diff options
| author | 魏曹先生 <1992414357@qq.com> | 2026-08-18 10:29:25 +0800 |
|---|---|---|
| committer | 魏曹先生 <1992414357@qq.com> | 2026-08-18 10:31:23 +0800 |
| commit | 721536eae99d468299c6d7f5ecd6b7dbeac4c34a (patch) | |
| tree | c9810fbd83630ea57d6e08c3a5d0e0c0d9672de5 | |
| parent | 8ea8e13f1a6b2a2942b78127e23d6783c5188ca5 (diff) | |
feat(ci-new): add markdown code block verification commands
Add `markdown-check` and `markdown-check-all` commands to verify Rust
code blocks in markdown files. Code blocks are compiled as test
projects, sharing temporary crates based on dependency configuration for
parallel execution.
| -rw-r--r-- | mingling_ci/help.txt | 2 | ||||
| -rw-r--r-- | mingling_ci/src/cmd.rs | 1 | ||||
| -rw-r--r-- | mingling_ci/src/cmd/cmd_markdown_check.rs | 128 | ||||
| -rw-r--r-- | mingling_ci/src/lib.rs | 2 | ||||
| -rw-r--r-- | mingling_ci/src/markdown.rs | 2 | ||||
| -rw-r--r-- | mingling_ci/src/markdown/project.rs | 347 | ||||
| -rw-r--r-- | mingling_ci/src/markdown/test.rs | 144 |
7 files changed, 626 insertions, 0 deletions
diff --git a/mingling_ci/help.txt b/mingling_ci/help.txt index 85d5c76..8dbc75b 100644 --- a/mingling_ci/help.txt +++ b/mingling_ci/help.txt @@ -9,6 +9,8 @@ FLAGS: COMMANDS: UTILS: + markdown-check <PATH> Verify rust code blocks in one markdown file + markdown-check-all Verify rust code blocks in all configured markdown files report-collect Collect and organize all inspection reports report-clean Clean up all reports diff --git a/mingling_ci/src/cmd.rs b/mingling_ci/src/cmd.rs index 30d65a1..b531b92 100644 --- a/mingling_ci/src/cmd.rs +++ b/mingling_ci/src/cmd.rs @@ -1,3 +1,4 @@ +pub(crate) mod cmd_markdown_check; pub(crate) mod cmd_report_clean; pub(crate) mod cmd_report_collect; pub(crate) mod cmd_show_features; diff --git a/mingling_ci/src/cmd/cmd_markdown_check.rs b/mingling_ci/src/cmd/cmd_markdown_check.rs new file mode 100644 index 0000000..c153832 --- /dev/null +++ b/mingling_ci/src/cmd/cmd_markdown_check.rs @@ -0,0 +1,128 @@ +use std::path::{Path, PathBuf}; + +use mingling::{ + Grouped, RenderResult, Routable, + macros::{buffer, command, renderer}, + res::ResExitCode, +}; + +use crate::Next; +use crate::markdown::project::parse_markdown; +use crate::markdown::test::try_test_markdown_project; +use crate::res::{CargoError, MessagePrinter}; + +const VERIFIED_DOCS: &str = ".config/verified-docs.toml"; + +#[command(node = "markdown-check")] +pub async fn markdown_check(args: Vec<String>) -> Next { + let Some(path_str) = args.first() else { + return ErrorMarkdownArgs("missing <path> argument".to_string()).to_chain(); + }; + let path = + std::env::current_dir().map_or_else(|_| PathBuf::from(path_str), |cwd| cwd.join(path_str)); + if !path.is_file() { + return ErrorMarkdownArgs(format!("{} is not a file", path.display())).to_chain(); + } + + let Ok(content) = std::fs::read_to_string(&path) else { + return ErrorMarkdownArgs(format!("failed to read {}", path.display())).to_chain(); + }; + let projects = parse_markdown(&content, &path.to_string_lossy()); + let fail_count = try_test_markdown_project(projects).await; + + ResultMarkdownCheck { fail_count }.to_chain() +} + +#[command(node = "markdown-check-all")] +pub async fn markdown_check_all() -> Next { + let Some(files) = verified_md_files() else { + return ErrorMarkdownConfig.to_chain(); + }; + + let mut projects = Vec::new(); + for (label, path) in files { + let Ok(content) = std::fs::read_to_string(&path) else { + continue; + }; + let source_file = format!("{label}/{}", path.file_name().unwrap().to_string_lossy()); + projects.extend(parse_markdown(&content, &source_file)); + } + let fail_count = try_test_markdown_project(projects).await; + + ResultMarkdownCheck { fail_count }.to_chain() +} + +/// Reads `verified-docs.toml` and collects all `.md` files: single files, +/// directories, or `**` globs (walked from the base directory). +fn verified_md_files() -> Option<Vec<(String, PathBuf)>> { + let content = std::fs::read_to_string(VERIFIED_DOCS).ok()?; + let table: toml::Table = content.parse().ok()?; + + let mut files: Vec<(String, PathBuf)> = Vec::new(); + for (label, value) in table.get("verified")?.as_table()? { + let value_str = value.as_str()?; + let candidate = PathBuf::from(value_str); + if candidate.is_dir() { + collect_md_files(&candidate, &mut files, label); + } else if candidate.is_file() { + files.push((label.clone(), candidate)); + } else if candidate.extension().is_none() { + // Glob like "docs/pages/**": walk the base directory. + let base = PathBuf::from(value_str.trim_end_matches("/**").trim_end_matches('*')); + if base.is_dir() { + collect_md_files(&base, &mut files, label); + } + } + } + + files.sort_by(|a, b| a.0.cmp(&b.0).then(a.1.cmp(&b.1))); + Some(files) +} + +/// Recursively collects all `.md` files under a directory. +fn collect_md_files(dir: &Path, files: &mut Vec<(String, PathBuf)>, label: &str) { + if let Ok(entries) = std::fs::read_dir(dir) { + for entry in entries.flatten() { + let path = entry.path(); + if path.is_dir() { + collect_md_files(&path, files, label); + } else if path.extension().is_some_and(|ext| ext == "md") { + files.push((label.to_string(), path)); + } + } + } +} + +/// Number of code blocks that failed to build. +#[derive(Grouped)] +pub struct ResultMarkdownCheck { + pub fail_count: usize, +} + +#[derive(Grouped, Default)] +pub struct ErrorMarkdownArgs(pub String); + +#[derive(Grouped, Default)] +pub struct ErrorMarkdownConfig; + +/// Silently sets a non-zero exit code when any block failed. +#[renderer(buffer)] +pub fn render_markdown_check(r: ResultMarkdownCheck, exit_code: &mut ResExitCode) { + if r.fail_count > 0 { + exit_code.exit_code = 1; + } +} + +#[renderer] +pub fn render_error_markdown_args(e: ErrorMarkdownArgs, error: &CargoError) -> RenderResult { + let render_result = RenderResult::new(); + error.println(vec![e.0]); + render_result +} + +#[renderer] +pub fn render_error_markdown_config(_: ErrorMarkdownConfig, error: &CargoError) -> RenderResult { + let render_result = RenderResult::new(); + error.println(vec![format!("failed to read {VERIFIED_DOCS}")]); + render_result +} diff --git a/mingling_ci/src/lib.rs b/mingling_ci/src/lib.rs index 8225bde..77170c6 100644 --- a/mingling_ci/src/lib.rs +++ b/mingling_ci/src/lib.rs @@ -14,6 +14,8 @@ pub mod res; /// Log exporter for CI reports pub mod reporter; +pub(crate) mod markdown; + #[help] pub fn render_fallback(_: EntryFallback) -> String { include_str!("../help.txt").to_string() diff --git a/mingling_ci/src/markdown.rs b/mingling_ci/src/markdown.rs new file mode 100644 index 0000000..b231a89 --- /dev/null +++ b/mingling_ci/src/markdown.rs @@ -0,0 +1,2 @@ +pub(crate) mod project; +pub(crate) mod test; diff --git a/mingling_ci/src/markdown/project.rs b/mingling_ci/src/markdown/project.rs new file mode 100644 index 0000000..d781b7e --- /dev/null +++ b/mingling_ci/src/markdown/project.rs @@ -0,0 +1,347 @@ +//! Model of a testable rust code block extracted from markdown: its dependency +//! configuration (features + deps) and the code itself. + +use std::fmt::Write as _; +use std::path::Path; + +/// A single testable `rust` code block, modeled as a test project. +pub(crate) struct MarkdownTestProject { + pub features: Vec<String>, + pub deps: Vec<(String, String)>, + pub code: String, + pub is_build_time: bool, + pub has_main: bool, + pub has_gen_program: bool, + pub source_file: String, + pub line: usize, +} + +impl MarkdownTestProject { + /// FNV-1a 64-bit hash over the dependency configuration (features + deps). + /// + /// Blocks with the same hash share one temporary crate and avoid redundant + /// recompilation. The input is sorted so the hash is stable. + #[must_use] + pub fn compute_hash(&self) -> String { + let mut features: Vec<&str> = self.features.iter().map(String::as_str).collect(); + features.sort_unstable(); + let mut dep_names: Vec<&str> = self.deps.iter().map(|(n, _)| n.as_str()).collect(); + dep_names.sort_unstable(); + let mut dep_versions: Vec<&str> = self.deps.iter().map(|(_, v)| v.as_str()).collect(); + dep_versions.sort_unstable(); + let mut deps: Vec<String> = self.deps.iter().map(|(n, v)| format!("{n}={v}")).collect(); + deps.sort(); + + let canonical = format!( + "{}\n{}\n{}\n{}", + features.join(","), + dep_names.join(","), + dep_versions.join(","), + deps.join(",") + ); + + // FNV-1a 64-bit — stable across runs (no random seed). + let mut hash: u64 = 0xcbf2_9ce4_8422_2325; + for &byte in canonical.as_bytes() { + hash ^= u64::from(byte); + hash = hash.wrapping_mul(0x0000_0100_0000_01b3); + } + format!("{hash:016x}") + } +} + +/// Parses all fenced `rust` blocks from markdown content. +/// +/// Blocks marked `// NOT VERIFIED` are skipped. +pub(crate) fn parse_markdown(content: &str, source_file: &str) -> Vec<MarkdownTestProject> { + let mut projects = Vec::new(); + let lines: Vec<&str> = content.lines().collect(); + let mut i = 0; + while i < lines.len() { + if lines[i].trim() == "```rust" { + if let Some(proj) = parse_block(&lines, i, source_file) { + projects.push(proj); + } + while i < lines.len() && lines[i].trim() != "```" { + i += 1; + } + } + i += 1; + } + projects +} + +/// Parses a single code block starting at a `rust` fence line. +fn parse_block(lines: &[&str], start: usize, source_file: &str) -> Option<MarkdownTestProject> { + let mut code_lines: Vec<String> = Vec::new(); + let mut features: Vec<String> = Vec::new(); + let mut not_verified = false; + let mut deps: Vec<(String, String)> = Vec::new(); + let mut has_main = false; + let mut has_gen_program = false; + let mut is_build_time = false; + + let mut idx = start + 1; + let mut in_header = true; + + while idx < lines.len() { + let raw_line = lines[idx]; + let trimmed = raw_line.trim(); + + if trimmed == "```" { + break; + } + + // `@@@` lines: hidden in the rendered docs (filtered by a docsify + // plugin) but must still compile. + if let Some(stripped) = trimmed.strip_prefix("@@@") { + in_header = false; + let code = stripped.trim_start(); + if code.contains("fn main") { + has_main = true; + } + if code.contains("gen_program!") { + has_gen_program = true; + } + code_lines.push(code.to_string()); + idx += 1; + continue; + } + + if in_header && trimmed == "// NOT VERIFIED" { + not_verified = true; + idx += 1; + continue; + } + if in_header && trimmed == "// BUILD TIME" { + is_build_time = true; + idx += 1; + continue; + } + if in_header && trimmed.starts_with("// ") { + if let Some(feat_str) = trimmed.strip_prefix("// Features:") { + let feat_str = feat_str.trim(); + if feat_str.starts_with('[') && feat_str.ends_with(']') { + let inner = &feat_str[1..feat_str.len() - 1]; + if !inner.is_empty() { + features = inner + .split(',') + .map(|s| s.trim().trim_matches('"').to_string()) + .filter(|s| !s.is_empty()) + .collect(); + } + } + idx += 1; + continue; + } + if trimmed == "// Dependencies:" { + idx += 1; + while idx < lines.len() { + let next = lines[idx].trim(); + if next == "```" { + break; + } + if let Some(dep_line) = next.strip_prefix("// ") { + if let Some((name, ver)) = dep_line.split_once(" = ") { + deps.push(( + name.trim().to_string(), + ver.trim().trim_matches('"').to_string(), + )); + } + idx += 1; + } else { + break; + } + } + continue; + } + } + + in_header = false; + if raw_line.contains("fn main") { + has_main = true; + } + if raw_line.contains("gen_program!") { + has_gen_program = true; + } + code_lines.push(raw_line.to_string()); + idx += 1; + } + + if code_lines.is_empty() || not_verified { + return None; + } + + Some(MarkdownTestProject { + features, + deps, + code: code_lines.join("\n"), + is_build_time, + has_main, + has_gen_program, + source_file: source_file.to_string(), + line: start + 1, + }) +} + +/// Builds the extra `[dependencies]` entries declared by a block's +/// `// Dependencies:` header comments. +/// +/// Markdown blocks declare companion crates like this: +/// +/// ```text +/// // Dependencies: +/// // serde = "1" +/// // clap = "4" +/// // tokio = { version = "1", features = ["full"] } +/// ``` +/// +/// Each `name = value` pair becomes one dependency of the generated test +/// crate (in addition to `mingling` itself), so doc blocks can freely use +/// external crates without repeating the whole manifest. +/// +/// # Special case: serde / clap +/// +/// Doc blocks pervasively derive serialization and argument parsing: +/// structural-renderer examples use `#[derive(Serialize)]`, the clap examples +/// use `#[derive(Parser)]` — and those derives live behind the `derive` +/// feature of `serde` / `clap`. Requiring every block to spell out +/// `// serde = { version = "1", features = ["derive"] }` would be +/// boilerplate repeated dozens of times, so the two crates automatically get +/// `features = ["derive"]` appended. +/// +/// Version values starting with `{` are inline tables (e.g. `tokio` with a +/// `features` list above) and are passed through verbatim — they already +/// carry their own features and must not be rewritten. +fn build_extra_deps(proj: &MarkdownTestProject) -> String { + let mut extra_deps = String::new(); + for (name, version) in &proj.deps { + if version.starts_with('{') { + // Inline table (path/features/…): the block already expressed its + // full dependency, so emit it unchanged. + let _ = writeln!(extra_deps, "{name} = {version}"); + } else if name == "serde" || name == "clap" { + // serde/clap derive: `#[derive(Serialize, Deserialize)]` and + // `#[derive(Parser)]` are used everywhere in the docs; auto-enable + // the `derive` feature to keep blocks terse. + let _ = writeln!( + extra_deps, + "{name} = {{ version = \"{version}\", features = [\"derive\"] }}" + ); + } else { + // Plain `name = "version"`. + let _ = writeln!(extra_deps, "{name} = \"{version}\""); + } + } + extra_deps +} + +/// Generates the `Cargo.toml` for a project. +/// +/// `manifest_path` is used to compute the relative path to the `mingling` crate. +pub(crate) fn generate_cargo_toml(proj: &MarkdownTestProject, manifest_path: &Path) -> String { + let features_str = if proj.features.is_empty() { + String::new() + } else { + let feats: Vec<String> = proj.features.iter().map(|f| format!("\"{f}\"")).collect(); + format!("features = [{}]", feats.join(", ")) + }; + + let extra_deps = build_extra_deps(proj); + + let mingling_path = find_mingling_relative_path(manifest_path); + let deps_section = if proj.features.is_empty() { + format!("[dependencies]\nmingling = {{ path = \"{mingling_path}\" }}\n{extra_deps}") + } else { + format!( + "[dependencies]\nmingling = {{ path = \"{mingling_path}\", {features_str} }}\n{extra_deps}" + ) + }; + + // Build-time projects mirror the features into [build-dependencies] so + // build.rs sees the same feature set. + let build_deps_section = if proj.is_build_time { + let feats: Vec<String> = proj.features.iter().map(|f| format!("\"{f}\"")).collect(); + let build_feats = if feats.is_empty() { + String::new() + } else { + format!("features = [{}]", feats.join(", ")) + }; + format!( + "\n[build-dependencies]\nmingling = {{ path = \"{mingling_path}\", {build_feats} }}\n" + ) + } else { + String::new() + }; + + format!( + r#"[package] + name = "test-doc" + version = "0.0.0" + edition = "2024" + +{deps_section}{build_deps_section} +[workspace] +"# + ) +} + +/// Computes the relative path from a manifest's parent directory to `mingling`. +/// +/// The process current directory is expected to be the project root. +fn find_mingling_relative_path(manifest_path: &Path) -> String { + let manifest_dir = manifest_path + .parent() + .expect("manifest path has no parent directory"); + let cwd = std::env::current_dir().expect("failed to get current directory"); + + let relative_to_root = manifest_dir.strip_prefix(&cwd).unwrap_or(manifest_dir); + let depth = relative_to_root.components().count(); + + let mut result = String::new(); + for _ in 0..depth { + result.push_str("../"); + } + result.push_str("mingling"); + result +} + +/// Generates `main.rs` for a project. +/// +/// Automatically prepends `use mingling::prelude::*;` and appends `fn main() {}` +/// and `gen_program!()` when the block does not provide them. +pub(crate) fn generate_main_rs(proj: &MarkdownTestProject) -> String { + let mut output = String::from("#![allow(dead_code)]\n#![allow(unused)]\n"); + + if !proj.code.contains("use mingling::prelude::*;") { + output.push_str("#[allow(unused_imports)]\nuse mingling::prelude::*;\n\n"); + } + output.push_str(&proj.code); + output.push('\n'); + + if !proj.has_main { + output.push_str("\nfn main() {}\n"); + } + if !proj.has_gen_program { + output.push_str("\nmingling::macros::gen_program!();\n"); + } + output +} + +/// Generates `build.rs` for a build-time project: the code wrapped in +/// `fn main() { }` unless the block already provides one. +pub(crate) fn generate_build_rs(proj: &MarkdownTestProject) -> String { + let mut output = String::from("#![allow(dead_code)]\n#![allow(unused)]\n"); + if proj.has_main { + output.push_str(&proj.code); + } else { + output.push_str("fn main() {\n"); + for line in proj.code.lines() { + output.push_str(" "); + output.push_str(line); + output.push('\n'); + } + output.push_str("}\n"); + } + output +} diff --git a/mingling_ci/src/markdown/test.rs b/mingling_ci/src/markdown/test.rs new file mode 100644 index 0000000..c010e8c --- /dev/null +++ b/mingling_ci/src/markdown/test.rs @@ -0,0 +1,144 @@ +//! Parallel execution of markdown test projects. + +use std::collections::BTreeMap; +use std::path::{Path, PathBuf}; + +use colored::Colorize; +use indicatif::{ProgressBar, ProgressStyle}; + +use super::project::{ + MarkdownTestProject, generate_build_rs, generate_cargo_toml, generate_main_rs, +}; + +/// Temporary root for the generated test crates. +const TEMP_BASE: &str = ".temp/doc-test"; + +/// Runs the given projects in parallel. +/// +/// Projects sharing a dependency hash share one temporary crate (written +/// serially within the group); groups run in parallel. Progress is shown on +/// stderr; failures print immediately. Returns the number of failed blocks. +pub(crate) async fn try_test_markdown_project(projs: Vec<MarkdownTestProject>) -> usize { + // Group by dependency hash for crate sharing. + let mut groups: BTreeMap<String, Vec<MarkdownTestProject>> = BTreeMap::new(); + for proj in projs { + groups.entry(proj.compute_hash()).or_default().push(proj); + } + + let total: usize = groups.values().map(Vec::len).sum(); + let pb = ProgressBar::new(total as u64); + pb.set_style( + ProgressStyle::default_bar() + .template(&format!( + "{} [{{bar:28}}] {{pos}}/{{len}}: {{msg}}", + " Testing".bold().bright_cyan() + )) + .unwrap() + .progress_chars("=> "), + ); + pb.set_message("blocks"); + + // One blocking task per group; blocks within a group are serial because + // they share the same crate directory. + let mut handles = Vec::new(); + for (hash, blocks) in groups { + let pb = pb.clone(); + handles.push(tokio::task::spawn_blocking(move || { + let crate_dir = PathBuf::from(TEMP_BASE).join(&hash); + let src_dir = crate_dir.join("src"); + let manifest_path = crate_dir.join("Cargo.toml"); + let cargo_toml = generate_cargo_toml(&blocks[0], &manifest_path); + + let mut failed = 0; + for proj in &blocks { + let label = format!("{}:{}", proj.source_file, proj.line); + pb.set_message(label.clone()); + + let main_rs = if proj.is_build_time { + generate_build_rs(proj) + } else { + generate_main_rs(proj) + }; + let (ok, err) = build_block( + &src_dir, + &manifest_path, + &cargo_toml, + &main_rs, + proj.is_build_time, + ); + pb.inc(1); + + if !ok { + failed += 1; + // Plain stderr: `pb.println` is swallowed on non-TTY (CI). + eprintln!(" {} {label}", "failed".bold().bright_red()); + eprintln!(" {label} FAILED:\n{err}"); + } + } + failed + })); + } + + let mut fail_count = 0; + for handle in handles { + if let Ok(failed) = handle.await { + fail_count += failed; + } + } + + pb.finish_and_clear(); + fail_count +} + +/// Writes the temporary crate files and runs `cargo check`. +/// +/// When `is_build_time` is true, the content goes to `build.rs` with a stub +/// `main.rs`; otherwise it goes to `src/main.rs`. +fn build_block( + src_dir: &Path, + manifest_path: &Path, + cargo_toml: &str, + content: &str, + is_build_time: bool, +) -> (bool, String) { + if let Err(e) = std::fs::create_dir_all(src_dir) { + return (false, format!("mkdir: {e}")); + } + if let Err(e) = std::fs::write(manifest_path, cargo_toml) { + return (false, format!("write Cargo.toml: {e}")); + } + + if is_build_time { + let crate_dir = manifest_path + .parent() + .expect("manifest path has a parent directory"); + if let Err(e) = std::fs::write(crate_dir.join("build.rs"), content) { + return (false, format!("write build.rs: {e}")); + } + if let Err(e) = std::fs::write(src_dir.join("main.rs"), "fn main() {}\n") { + return (false, format!("write main.rs: {e}")); + } + } else if let Err(e) = std::fs::write(src_dir.join("main.rs"), content) { + return (false, format!("write main.rs: {e}")); + } + + let output = std::process::Command::new("cargo") + .args(["check", "--color=always", "--manifest-path"]) + .arg(manifest_path) + .output(); + match output { + Ok(output) if output.status.success() => (true, String::new()), + Ok(output) => { + let mut log = String::from_utf8_lossy(&output.stdout).into_owned(); + log.push_str(&String::from_utf8_lossy(&output.stderr)); + let lines: Vec<&str> = log.lines().collect(); + let tail = &lines[lines.len().saturating_sub(20)..]; + let exit = output + .status + .code() + .map_or_else(|| "?".to_string(), |c| c.to_string()); + (false, format!("exit code {exit}\n{}", tail.join("\n"))) + } + Err(e) => (false, format!("failed to run cargo: {e}")), + } +} |
