diff options
| author | 魏曹先生 <1992414357@qq.com> | 2026-08-18 10:42:14 +0800 |
|---|---|---|
| committer | 魏曹先生 <1992414357@qq.com> | 2026-08-18 10:42:14 +0800 |
| commit | 8d06ff9eab3ac70b55f6d7437a6615b77fa38d80 (patch) | |
| tree | 3311aba119f3759a991e9d4e81cf60b19e78d36e /mingling_ci | |
| parent | b5941040586c3707620bbef6cf8ace4e565f7693 (diff) | |
feat(ci-new): add markdown structural comparison commands
Add `markdown-compare` and `markdown-compare-all` commands to verify
that translated docs maintain the same structure as the reference.
Compare heading levels, code fences, lists, quotes, and other
structural elements while allowing text differences.
Diffstat (limited to 'mingling_ci')
| -rw-r--r-- | mingling_ci/help.txt | 20 | ||||
| -rw-r--r-- | mingling_ci/src/cmd.rs | 1 | ||||
| -rw-r--r-- | mingling_ci/src/markdown.rs | 1 | ||||
| -rw-r--r-- | mingling_ci/src/markdown/compare.rs | 203 | ||||
| -rw-r--r-- | mingling_ci/src/task.rs | 2 | ||||
| -rw-r--r-- | mingling_ci/src/task/cmd_markdown_check.rs (renamed from mingling_ci/src/cmd/cmd_markdown_check.rs) | 2 | ||||
| -rw-r--r-- | mingling_ci/src/task/cmd_markdown_compare.rs | 221 |
7 files changed, 439 insertions, 11 deletions
diff --git a/mingling_ci/help.txt b/mingling_ci/help.txt index 8dbc75b..a810b41 100644 --- a/mingling_ci/help.txt +++ b/mingling_ci/help.txt @@ -9,15 +9,17 @@ FLAGS: COMMANDS: UTILS: - markdown-check <PATH> Verify rust code blocks in one markdown file - markdown-check-all Verify rust code blocks in all configured markdown files - report-collect Collect and organize all inspection reports - report-clean Clean up all reports + report-collect Collect and organize all inspection reports + report-clean Clean up all reports - show-features Print the docs.rs feature list of mingling - show-manifests Print all crate paths that need to be checked + show-features Print the docs.rs feature list of mingling + show-manifests Print all crate paths that need to be checked TASKS: - build-all Build all crates - clippy-all Run clippy with -D warnings on all crates - test-all Test all crates + markdown-check Verify rust code blocks in one markdown file + markdown-check-all <PATH> Verify rust code blocks in all configured markdown files + markdown-compare Compare the structure of two markdown files/dirs + markdown-compare-all <PATH> <PATH> Compare all translated docs against the reference + build-all Build all crates + clippy-all Run clippy with -D warnings on all crates + test-all Test all crates diff --git a/mingling_ci/src/cmd.rs b/mingling_ci/src/cmd.rs index b531b92..30d65a1 100644 --- a/mingling_ci/src/cmd.rs +++ b/mingling_ci/src/cmd.rs @@ -1,4 +1,3 @@ -pub(crate) mod cmd_markdown_check; pub(crate) mod cmd_report_clean; pub(crate) mod cmd_report_collect; pub(crate) mod cmd_show_features; diff --git a/mingling_ci/src/markdown.rs b/mingling_ci/src/markdown.rs index b231a89..75f2cbe 100644 --- a/mingling_ci/src/markdown.rs +++ b/mingling_ci/src/markdown.rs @@ -1,2 +1,3 @@ +pub(crate) mod compare; pub(crate) mod project; pub(crate) mod test; diff --git a/mingling_ci/src/markdown/compare.rs b/mingling_ci/src/markdown/compare.rs new file mode 100644 index 0000000..1bf3c57 --- /dev/null +++ b/mingling_ci/src/markdown/compare.rs @@ -0,0 +1,203 @@ +//! Structural comparison of markdown docs (reference vs translation). +//! +//! For each file pair the comparison uses a *structural signature*: one token +//! per line, classifying headings (both Markdown `#` and HTML `<hN>`), fenced +//! code blocks (including their language tag), `@@@` hidden-compilation lines, +//! blank lines, blockquotes, lists and plain text. Translated text is allowed +//! to differ; the structure is not. + +use std::path::{Path, PathBuf}; + +/// Collects all `.md` files under `dir`, returned relative to it. +pub(crate) fn collect_md_files(dir: &Path) -> Vec<PathBuf> { + let mut out = Vec::new(); + let mut stack = vec![dir.to_path_buf()]; + while let Some(current) = stack.pop() { + let Ok(entries) = std::fs::read_dir(¤t) else { + continue; + }; + for entry in entries.flatten() { + let path = entry.path(); + if path.is_dir() { + stack.push(path); + } else if path.extension().is_some_and(|e| e == "md") { + out.push(path.strip_prefix(dir).unwrap_or(&path).to_path_buf()); + } + } + } + out.sort(); + out +} + +/// Compares the structural signatures of two markdown files. +/// +/// Returns the human-readable diff lines (up to a small window) on the first +/// structural difference. +pub(crate) fn compare_signature(ref_path: &Path, lang_path: &Path) -> Result<(), Vec<String>> { + let ref_content = std::fs::read_to_string(ref_path).unwrap_or_default(); + let lang_content = std::fs::read_to_string(lang_path).unwrap_or_default(); + + let ref_sig = signature_of(&ref_content); + let lang_sig = signature_of(&lang_content); + + if ref_sig == lang_sig { + return Ok(()); + } + + let ref_lines: Vec<&str> = ref_content.lines().collect(); + let lang_lines: Vec<&str> = lang_content.lines().collect(); + + let mut diffs = Vec::new(); + let mut window = 0; + let max = ref_sig.len().max(lang_sig.len()); + for i in 0..max { + let ref_tok = ref_sig.get(i); + let lang_tok = lang_sig.get(i); + if ref_tok == lang_tok { + continue; + } + if window >= 5 { + diffs.push(format!("... ({}-line window truncated)", max - i)); + break; + } + window += 1; + let ref_line = ref_lines.get(i).copied().unwrap_or("<missing>"); + let lang_line = lang_lines.get(i).copied().unwrap_or("<missing>"); + diffs.push(format!("line {}", i + 1)); + diffs.push(format!( + "expect `{}` {}", + token_label(ref_tok.map_or("<eof>", String::as_str)), + display_line(ref_line) + )); + diffs.push(format!( + "found `{}` {}", + token_label(lang_tok.map_or("<eof>", String::as_str)), + display_line(lang_line) + )); + if ref_sig.len() != lang_sig.len() && window >= 5 { + diffs.push(format!( + "note: reference has {} lines, translation has {} lines", + ref_sig.len(), + lang_sig.len() + )); + break; + } + } + if diffs.is_empty() { + diffs.push("signatures differ in length (see line count note)".to_string()); + } + Err(diffs) +} + +/// Builds the structural signature of a markdown file. +fn signature_of(content: &str) -> Vec<String> { + let mut sig = Vec::new(); + let mut in_fence = false; + let mut fence_lang = String::new(); + + for raw_line in content.lines() { + let line = raw_line.trim(); + + if in_fence { + if line.starts_with("```") { + in_fence = false; + sig.push(format!("F:{fence_lang}")); + } else if line.starts_with("@@@") { + sig.push("A".to_string()); + } else if line.is_empty() { + sig.push("B".to_string()); + } else { + sig.push("P".to_string()); + } + continue; + } + + if line.starts_with("```") { + in_fence = true; + fence_lang = line.trim_start_matches("```").trim().to_string(); + sig.push(format!("F:{fence_lang}")); + } else if line.starts_with('#') { + let level = line.chars().take_while(|c| *c == '#').count(); + sig.push(format!("H{level}")); + } else if line.starts_with("<h") || line.starts_with("</h") { + // HTML headings (e.g. `<h1 align="center">` / `</h1>`) + let level = line + .trim_start_matches(['<', '/']) + .chars() + .next() + .and_then(|c| c.to_digit(10)) + .unwrap_or(1); + sig.push(format!("H{level}")); + } else if line.starts_with("@@@") { + sig.push("A".to_string()); + } else if line.is_empty() { + sig.push("B".to_string()); + } else if line.starts_with('>') { + sig.push("Q".to_string()); + } else if is_list_line(line) { + sig.push("L".to_string()); + } else { + sig.push("P".to_string()); + } + } + sig +} + +/// Human-readable label for a structural token. +fn token_label(token: &str) -> String { + match token { + "B" => "blank".to_string(), + "A" => "@@@".to_string(), + "Q" => "quote".to_string(), + "L" => "list".to_string(), + "P" => "text".to_string(), + t if t.starts_with('H') => format!("heading-{}", &t[1..]), + t if t.starts_with("F:") => { + let lang = &t[2..]; + if lang.is_empty() { + "fence".to_string() + } else { + format!("fence:{lang}") + } + } + _ => token.to_string(), + } +} + +/// Renders a source line for display: blank lines become `<blank>`. +fn display_line(line: &str) -> String { + if line.trim().is_empty() { + "<blank>".to_string() + } else { + truncate(line) + } +} + +fn truncate(line: &str) -> String { + const MAX: usize = 60; + if line.chars().count() <= MAX { + line.to_string() + } else { + let cut: String = line.chars().take(MAX).collect(); + format!("{cut}...") + } +} + +fn is_list_line(line: &str) -> bool { + let trimmed = line.trim_start(); + trimmed.starts_with("- ") + || trimmed.starts_with("* ") + || trimmed.starts_with("+ ") + || is_numbered_list(trimmed) +} + +/// A numbered list item: `1. text`, `1) text`, `10. text`, ... +fn is_numbered_list(line: &str) -> bool { + let digit_count = line.chars().take_while(char::is_ascii_digit).count(); + if digit_count == 0 { + return false; + } + let rest = &line[digit_count..]; + (rest.starts_with(". ") || rest.starts_with(") ")) + && rest.chars().nth(1).is_some_and(|c| c == ' ' || c == '\t') +} diff --git a/mingling_ci/src/task.rs b/mingling_ci/src/task.rs index b5161ad..92660bb 100644 --- a/mingling_ci/src/task.rs +++ b/mingling_ci/src/task.rs @@ -1,4 +1,6 @@ pub(crate) mod cmd_build; pub(crate) mod cmd_clippy; +pub(crate) mod cmd_markdown_check; +pub(crate) mod cmd_markdown_compare; pub(crate) mod cmd_test; pub(crate) mod run; diff --git a/mingling_ci/src/cmd/cmd_markdown_check.rs b/mingling_ci/src/task/cmd_markdown_check.rs index 6044846..2408636 100644 --- a/mingling_ci/src/cmd/cmd_markdown_check.rs +++ b/mingling_ci/src/task/cmd_markdown_check.rs @@ -74,7 +74,7 @@ pub async fn markdown_check_all() -> Next { } /// The file name without extension, e.g. `README.md` → `README`. -fn stem_of(path: &Path) -> String { +pub(crate) fn stem_of(path: &Path) -> String { path.file_stem() .unwrap_or_default() .to_string_lossy() diff --git a/mingling_ci/src/task/cmd_markdown_compare.rs b/mingling_ci/src/task/cmd_markdown_compare.rs new file mode 100644 index 0000000..b014f1e --- /dev/null +++ b/mingling_ci/src/task/cmd_markdown_compare.rs @@ -0,0 +1,221 @@ +use std::collections::BTreeSet; +use std::path::{Path, PathBuf}; + +use colored::Colorize; +use just_fmt::snake_case; +use mingling::{ + Grouped, Routable, + macros::{buffer, command, renderer}, + res::ResExitCode, +}; + +use crate::Next; +use crate::markdown::compare::{collect_md_files, compare_signature}; +use crate::reporter::{self, ReportResult}; +use crate::task::cmd_markdown_check::{ErrorMarkdownArgs, ErrorMarkdownConfig, stem_of}; + +const DOCS_DIR: &str = "./docs"; +const LANG_CONFIG: &str = ".config/docs-lang.txt"; + +/// One file-pair outcome of a structure comparison. +struct CompareOutcome { + item: String, + location: String, + ok: bool, + output: String, +} + +#[command(node = "markdown-compare")] +// `#[command]` rewrites an owned first param into the entry type, so the args +// must be passed by value even though the body only reads them. +#[allow(clippy::needless_pass_by_value)] +pub fn markdown_compare(args: Vec<String>) -> Next { + let [ref_arg, trans_arg] = args.as_slice() else { + return ErrorMarkdownArgs("missing <reference> and <translation> arguments".to_string()) + .to_chain(); + }; + let ref_path = cwd().join(ref_arg); + let trans_path = cwd().join(trans_arg); + + reporter::set_task("Markdown-Compare"); + let outcomes = if ref_path.is_dir() && trans_path.is_dir() { + compare_dirs(&ref_path, &trans_path, "doc") + } else if ref_path.is_file() && trans_path.is_file() { + compare_files(&ref_path, &trans_path, "doc") + } else { + return ErrorMarkdownArgs( + "both arguments must be files or both must be directories".to_string(), + ) + .to_chain(); + }; + let fail_count = export_outcomes(&outcomes); + reporter::flush(); + + ResultMarkdownCompare { fail_count }.to_chain() +} + +#[command(node = "markdown-compare-all")] +pub fn markdown_compare_all() -> Next { + let Some(langs) = lang_config() else { + return ErrorMarkdownConfig.to_chain(); + }; + let Some(reference) = langs.first() else { + return ErrorMarkdownConfig.to_chain(); + }; + let ref_dir = PathBuf::from(DOCS_DIR).join(reference); + if !ref_dir.is_dir() { + return ErrorMarkdownArgs(format!( + "reference docs directory `{}` does not exist", + ref_dir.display() + )) + .to_chain(); + } + + reporter::set_task("Markdown-Compare-All"); + let mut fail_count = 0; + for lang in &langs[1..] { + let lang_dir = PathBuf::from(DOCS_DIR).join(lang); + if !lang_dir.is_dir() { + eprintln!( + " {}: `{}` does not exist", + "ERROR".bright_red(), + lang_dir.display() + ); + fail_count += 1; + continue; + } + let outcomes = compare_dirs(&ref_dir, &lang_dir, &lang_key(lang)); + fail_count += export_outcomes(&outcomes); + } + reporter::flush(); + + ResultMarkdownCompare { fail_count }.to_chain() +} + +/// Compares one file pair (reference vs translation). +fn compare_files(ref_path: &Path, trans_path: &Path, prefix: &str) -> Vec<CompareOutcome> { + let item = format!("{prefix}-{}", snake_case!(&stem_of(ref_path))); + let location = trans_path.to_string_lossy().into_owned(); + match compare_signature(ref_path, trans_path) { + Ok(()) => vec![CompareOutcome { + item, + location, + ok: true, + output: String::new(), + }], + Err(diffs) => vec![CompareOutcome { + item, + location, + ok: false, + output: diffs.join("\n"), + }], + } +} + +/// Compares two directories: every `.md` file in the reference must exist in +/// the translation with the same structural signature; extra files are errors. +fn compare_dirs(ref_dir: &Path, trans_dir: &Path, prefix: &str) -> Vec<CompareOutcome> { + let ref_files = collect_md_files(ref_dir); + let ref_set: BTreeSet<PathBuf> = ref_files.iter().cloned().collect(); + let trans_set: BTreeSet<PathBuf> = collect_md_files(trans_dir).into_iter().collect(); + + let mut outcomes = Vec::new(); + for file in ref_files { + let item = format!("{prefix}-{}", snake_case!(&stem_of(&file))); + let trans_path = trans_dir.join(&file); + let location = trans_path.to_string_lossy().into_owned(); + if !trans_set.contains(&file) { + outcomes.push(CompareOutcome { + item, + location, + ok: false, + output: "missing in translation".to_string(), + }); + continue; + } + outcomes.push(match compare_signature(&ref_dir.join(&file), &trans_path) { + Ok(()) => CompareOutcome { + item, + location, + ok: true, + output: String::new(), + }, + Err(diffs) => CompareOutcome { + item, + location, + ok: false, + output: diffs.join("\n"), + }, + }); + } + + for file in trans_set.difference(&ref_set) { + let item = format!("{prefix}-{}", snake_case!(&stem_of(file))); + let trans_path = trans_dir.join(file); + outcomes.push(CompareOutcome { + item, + location: trans_path.to_string_lossy().into_owned(), + ok: false, + output: "extra file, not in reference".to_string(), + }); + } + outcomes +} + +/// Exports the outcomes via `reporter`; failures also print to stderr. +fn export_outcomes(outcomes: &[CompareOutcome]) -> usize { + let mut fail_count = 0; + for outcome in outcomes { + if outcome.ok { + reporter::export(&outcome.item, &outcome.location, ReportResult::Ok); + } else { + fail_count += 1; + eprintln!(" {} {}", "failed".bright_red(), outcome.item); + eprintln!(" {}\n{}", outcome.location, outcome.output); + reporter::export( + &outcome.item, + &outcome.location, + ReportResult::Error(outcome.output.clone()), + ); + } + } + fail_count +} + +/// Reads `.config/docs-lang.txt`: the first line is the reference directory +/// (relative to `./docs/`), the rest are translations that must mirror it. +fn lang_config() -> Option<Vec<String>> { + let content = std::fs::read_to_string(LANG_CONFIG).ok()?; + Some( + content + .lines() + .map(str::trim) + .filter(|l| !l.is_empty() && !l.starts_with('#')) + .map(|l| l.trim_start_matches("./").to_string()) + .collect(), + ) +} + +/// Turns a lang directory path into a report-item key, e.g. +/// `./_zh_CN/pages/` → `_zh_CN_pages`. +fn lang_key(lang: &str) -> String { + lang.trim_matches('/').replace('/', "_") +} + +fn cwd() -> PathBuf { + std::env::current_dir().unwrap_or_else(|_| PathBuf::from(".")) +} + +/// Number of files that failed the structure comparison. +#[derive(Grouped)] +pub struct ResultMarkdownCompare { + pub fail_count: usize, +} + +/// Silently sets a non-zero exit code when any comparison failed. +#[renderer(buffer)] +pub fn render_markdown_compare(r: ResultMarkdownCompare, exit_code: &mut ResExitCode) { + if r.fail_count > 0 { + exit_code.exit_code = 1; + } +} |
