aboutsummaryrefslogtreecommitdiff
path: root/mingling_ci/src/markdown
diff options
context:
space:
mode:
Diffstat (limited to 'mingling_ci/src/markdown')
-rw-r--r--mingling_ci/src/markdown/compare.rs203
-rw-r--r--mingling_ci/src/markdown/project.rs347
-rw-r--r--mingling_ci/src/markdown/test.rs152
3 files changed, 0 insertions, 702 deletions
diff --git a/mingling_ci/src/markdown/compare.rs b/mingling_ci/src/markdown/compare.rs
deleted file mode 100644
index 1bf3c57..0000000
--- a/mingling_ci/src/markdown/compare.rs
+++ /dev/null
@@ -1,203 +0,0 @@
-//! Structural comparison of markdown docs (reference vs translation).
-//!
-//! For each file pair the comparison uses a *structural signature*: one token
-//! per line, classifying headings (both Markdown `#` and HTML `<hN>`), fenced
-//! code blocks (including their language tag), `@@@` hidden-compilation lines,
-//! blank lines, blockquotes, lists and plain text. Translated text is allowed
-//! to differ; the structure is not.
-
-use std::path::{Path, PathBuf};
-
-/// Collects all `.md` files under `dir`, returned relative to it.
-pub(crate) fn collect_md_files(dir: &Path) -> Vec<PathBuf> {
- let mut out = Vec::new();
- let mut stack = vec![dir.to_path_buf()];
- while let Some(current) = stack.pop() {
- let Ok(entries) = std::fs::read_dir(&current) else {
- continue;
- };
- for entry in entries.flatten() {
- let path = entry.path();
- if path.is_dir() {
- stack.push(path);
- } else if path.extension().is_some_and(|e| e == "md") {
- out.push(path.strip_prefix(dir).unwrap_or(&path).to_path_buf());
- }
- }
- }
- out.sort();
- out
-}
-
-/// Compares the structural signatures of two markdown files.
-///
-/// Returns the human-readable diff lines (up to a small window) on the first
-/// structural difference.
-pub(crate) fn compare_signature(ref_path: &Path, lang_path: &Path) -> Result<(), Vec<String>> {
- let ref_content = std::fs::read_to_string(ref_path).unwrap_or_default();
- let lang_content = std::fs::read_to_string(lang_path).unwrap_or_default();
-
- let ref_sig = signature_of(&ref_content);
- let lang_sig = signature_of(&lang_content);
-
- if ref_sig == lang_sig {
- return Ok(());
- }
-
- let ref_lines: Vec<&str> = ref_content.lines().collect();
- let lang_lines: Vec<&str> = lang_content.lines().collect();
-
- let mut diffs = Vec::new();
- let mut window = 0;
- let max = ref_sig.len().max(lang_sig.len());
- for i in 0..max {
- let ref_tok = ref_sig.get(i);
- let lang_tok = lang_sig.get(i);
- if ref_tok == lang_tok {
- continue;
- }
- if window >= 5 {
- diffs.push(format!("... ({}-line window truncated)", max - i));
- break;
- }
- window += 1;
- let ref_line = ref_lines.get(i).copied().unwrap_or("<missing>");
- let lang_line = lang_lines.get(i).copied().unwrap_or("<missing>");
- diffs.push(format!("line {}", i + 1));
- diffs.push(format!(
- "expect `{}` {}",
- token_label(ref_tok.map_or("<eof>", String::as_str)),
- display_line(ref_line)
- ));
- diffs.push(format!(
- "found `{}` {}",
- token_label(lang_tok.map_or("<eof>", String::as_str)),
- display_line(lang_line)
- ));
- if ref_sig.len() != lang_sig.len() && window >= 5 {
- diffs.push(format!(
- "note: reference has {} lines, translation has {} lines",
- ref_sig.len(),
- lang_sig.len()
- ));
- break;
- }
- }
- if diffs.is_empty() {
- diffs.push("signatures differ in length (see line count note)".to_string());
- }
- Err(diffs)
-}
-
-/// Builds the structural signature of a markdown file.
-fn signature_of(content: &str) -> Vec<String> {
- let mut sig = Vec::new();
- let mut in_fence = false;
- let mut fence_lang = String::new();
-
- for raw_line in content.lines() {
- let line = raw_line.trim();
-
- if in_fence {
- if line.starts_with("```") {
- in_fence = false;
- sig.push(format!("F:{fence_lang}"));
- } else if line.starts_with("@@@") {
- sig.push("A".to_string());
- } else if line.is_empty() {
- sig.push("B".to_string());
- } else {
- sig.push("P".to_string());
- }
- continue;
- }
-
- if line.starts_with("```") {
- in_fence = true;
- fence_lang = line.trim_start_matches("```").trim().to_string();
- sig.push(format!("F:{fence_lang}"));
- } else if line.starts_with('#') {
- let level = line.chars().take_while(|c| *c == '#').count();
- sig.push(format!("H{level}"));
- } else if line.starts_with("<h") || line.starts_with("</h") {
- // HTML headings (e.g. `<h1 align="center">` / `</h1>`)
- let level = line
- .trim_start_matches(['<', '/'])
- .chars()
- .next()
- .and_then(|c| c.to_digit(10))
- .unwrap_or(1);
- sig.push(format!("H{level}"));
- } else if line.starts_with("@@@") {
- sig.push("A".to_string());
- } else if line.is_empty() {
- sig.push("B".to_string());
- } else if line.starts_with('>') {
- sig.push("Q".to_string());
- } else if is_list_line(line) {
- sig.push("L".to_string());
- } else {
- sig.push("P".to_string());
- }
- }
- sig
-}
-
-/// Human-readable label for a structural token.
-fn token_label(token: &str) -> String {
- match token {
- "B" => "blank".to_string(),
- "A" => "@@@".to_string(),
- "Q" => "quote".to_string(),
- "L" => "list".to_string(),
- "P" => "text".to_string(),
- t if t.starts_with('H') => format!("heading-{}", &t[1..]),
- t if t.starts_with("F:") => {
- let lang = &t[2..];
- if lang.is_empty() {
- "fence".to_string()
- } else {
- format!("fence:{lang}")
- }
- }
- _ => token.to_string(),
- }
-}
-
-/// Renders a source line for display: blank lines become `<blank>`.
-fn display_line(line: &str) -> String {
- if line.trim().is_empty() {
- "<blank>".to_string()
- } else {
- truncate(line)
- }
-}
-
-fn truncate(line: &str) -> String {
- const MAX: usize = 60;
- if line.chars().count() <= MAX {
- line.to_string()
- } else {
- let cut: String = line.chars().take(MAX).collect();
- format!("{cut}...")
- }
-}
-
-fn is_list_line(line: &str) -> bool {
- let trimmed = line.trim_start();
- trimmed.starts_with("- ")
- || trimmed.starts_with("* ")
- || trimmed.starts_with("+ ")
- || is_numbered_list(trimmed)
-}
-
-/// A numbered list item: `1. text`, `1) text`, `10. text`, ...
-fn is_numbered_list(line: &str) -> bool {
- let digit_count = line.chars().take_while(char::is_ascii_digit).count();
- if digit_count == 0 {
- return false;
- }
- let rest = &line[digit_count..];
- (rest.starts_with(". ") || rest.starts_with(") "))
- && rest.chars().nth(1).is_some_and(|c| c == ' ' || c == '\t')
-}
diff --git a/mingling_ci/src/markdown/project.rs b/mingling_ci/src/markdown/project.rs
deleted file mode 100644
index d781b7e..0000000
--- a/mingling_ci/src/markdown/project.rs
+++ /dev/null
@@ -1,347 +0,0 @@
-//! Model of a testable rust code block extracted from markdown: its dependency
-//! configuration (features + deps) and the code itself.
-
-use std::fmt::Write as _;
-use std::path::Path;
-
-/// A single testable `rust` code block, modeled as a test project.
-pub(crate) struct MarkdownTestProject {
- pub features: Vec<String>,
- pub deps: Vec<(String, String)>,
- pub code: String,
- pub is_build_time: bool,
- pub has_main: bool,
- pub has_gen_program: bool,
- pub source_file: String,
- pub line: usize,
-}
-
-impl MarkdownTestProject {
- /// FNV-1a 64-bit hash over the dependency configuration (features + deps).
- ///
- /// Blocks with the same hash share one temporary crate and avoid redundant
- /// recompilation. The input is sorted so the hash is stable.
- #[must_use]
- pub fn compute_hash(&self) -> String {
- let mut features: Vec<&str> = self.features.iter().map(String::as_str).collect();
- features.sort_unstable();
- let mut dep_names: Vec<&str> = self.deps.iter().map(|(n, _)| n.as_str()).collect();
- dep_names.sort_unstable();
- let mut dep_versions: Vec<&str> = self.deps.iter().map(|(_, v)| v.as_str()).collect();
- dep_versions.sort_unstable();
- let mut deps: Vec<String> = self.deps.iter().map(|(n, v)| format!("{n}={v}")).collect();
- deps.sort();
-
- let canonical = format!(
- "{}\n{}\n{}\n{}",
- features.join(","),
- dep_names.join(","),
- dep_versions.join(","),
- deps.join(",")
- );
-
- // FNV-1a 64-bit — stable across runs (no random seed).
- let mut hash: u64 = 0xcbf2_9ce4_8422_2325;
- for &byte in canonical.as_bytes() {
- hash ^= u64::from(byte);
- hash = hash.wrapping_mul(0x0000_0100_0000_01b3);
- }
- format!("{hash:016x}")
- }
-}
-
-/// Parses all fenced `rust` blocks from markdown content.
-///
-/// Blocks marked `// NOT VERIFIED` are skipped.
-pub(crate) fn parse_markdown(content: &str, source_file: &str) -> Vec<MarkdownTestProject> {
- let mut projects = Vec::new();
- let lines: Vec<&str> = content.lines().collect();
- let mut i = 0;
- while i < lines.len() {
- if lines[i].trim() == "```rust" {
- if let Some(proj) = parse_block(&lines, i, source_file) {
- projects.push(proj);
- }
- while i < lines.len() && lines[i].trim() != "```" {
- i += 1;
- }
- }
- i += 1;
- }
- projects
-}
-
-/// Parses a single code block starting at a `rust` fence line.
-fn parse_block(lines: &[&str], start: usize, source_file: &str) -> Option<MarkdownTestProject> {
- let mut code_lines: Vec<String> = Vec::new();
- let mut features: Vec<String> = Vec::new();
- let mut not_verified = false;
- let mut deps: Vec<(String, String)> = Vec::new();
- let mut has_main = false;
- let mut has_gen_program = false;
- let mut is_build_time = false;
-
- let mut idx = start + 1;
- let mut in_header = true;
-
- while idx < lines.len() {
- let raw_line = lines[idx];
- let trimmed = raw_line.trim();
-
- if trimmed == "```" {
- break;
- }
-
- // `@@@` lines: hidden in the rendered docs (filtered by a docsify
- // plugin) but must still compile.
- if let Some(stripped) = trimmed.strip_prefix("@@@") {
- in_header = false;
- let code = stripped.trim_start();
- if code.contains("fn main") {
- has_main = true;
- }
- if code.contains("gen_program!") {
- has_gen_program = true;
- }
- code_lines.push(code.to_string());
- idx += 1;
- continue;
- }
-
- if in_header && trimmed == "// NOT VERIFIED" {
- not_verified = true;
- idx += 1;
- continue;
- }
- if in_header && trimmed == "// BUILD TIME" {
- is_build_time = true;
- idx += 1;
- continue;
- }
- if in_header && trimmed.starts_with("// ") {
- if let Some(feat_str) = trimmed.strip_prefix("// Features:") {
- let feat_str = feat_str.trim();
- if feat_str.starts_with('[') && feat_str.ends_with(']') {
- let inner = &feat_str[1..feat_str.len() - 1];
- if !inner.is_empty() {
- features = inner
- .split(',')
- .map(|s| s.trim().trim_matches('"').to_string())
- .filter(|s| !s.is_empty())
- .collect();
- }
- }
- idx += 1;
- continue;
- }
- if trimmed == "// Dependencies:" {
- idx += 1;
- while idx < lines.len() {
- let next = lines[idx].trim();
- if next == "```" {
- break;
- }
- if let Some(dep_line) = next.strip_prefix("// ") {
- if let Some((name, ver)) = dep_line.split_once(" = ") {
- deps.push((
- name.trim().to_string(),
- ver.trim().trim_matches('"').to_string(),
- ));
- }
- idx += 1;
- } else {
- break;
- }
- }
- continue;
- }
- }
-
- in_header = false;
- if raw_line.contains("fn main") {
- has_main = true;
- }
- if raw_line.contains("gen_program!") {
- has_gen_program = true;
- }
- code_lines.push(raw_line.to_string());
- idx += 1;
- }
-
- if code_lines.is_empty() || not_verified {
- return None;
- }
-
- Some(MarkdownTestProject {
- features,
- deps,
- code: code_lines.join("\n"),
- is_build_time,
- has_main,
- has_gen_program,
- source_file: source_file.to_string(),
- line: start + 1,
- })
-}
-
-/// Builds the extra `[dependencies]` entries declared by a block's
-/// `// Dependencies:` header comments.
-///
-/// Markdown blocks declare companion crates like this:
-///
-/// ```text
-/// // Dependencies:
-/// // serde = "1"
-/// // clap = "4"
-/// // tokio = { version = "1", features = ["full"] }
-/// ```
-///
-/// Each `name = value` pair becomes one dependency of the generated test
-/// crate (in addition to `mingling` itself), so doc blocks can freely use
-/// external crates without repeating the whole manifest.
-///
-/// # Special case: serde / clap
-///
-/// Doc blocks pervasively derive serialization and argument parsing:
-/// structural-renderer examples use `#[derive(Serialize)]`, the clap examples
-/// use `#[derive(Parser)]` — and those derives live behind the `derive`
-/// feature of `serde` / `clap`. Requiring every block to spell out
-/// `// serde = { version = "1", features = ["derive"] }` would be
-/// boilerplate repeated dozens of times, so the two crates automatically get
-/// `features = ["derive"]` appended.
-///
-/// Version values starting with `{` are inline tables (e.g. `tokio` with a
-/// `features` list above) and are passed through verbatim — they already
-/// carry their own features and must not be rewritten.
-fn build_extra_deps(proj: &MarkdownTestProject) -> String {
- let mut extra_deps = String::new();
- for (name, version) in &proj.deps {
- if version.starts_with('{') {
- // Inline table (path/features/…): the block already expressed its
- // full dependency, so emit it unchanged.
- let _ = writeln!(extra_deps, "{name} = {version}");
- } else if name == "serde" || name == "clap" {
- // serde/clap derive: `#[derive(Serialize, Deserialize)]` and
- // `#[derive(Parser)]` are used everywhere in the docs; auto-enable
- // the `derive` feature to keep blocks terse.
- let _ = writeln!(
- extra_deps,
- "{name} = {{ version = \"{version}\", features = [\"derive\"] }}"
- );
- } else {
- // Plain `name = "version"`.
- let _ = writeln!(extra_deps, "{name} = \"{version}\"");
- }
- }
- extra_deps
-}
-
-/// Generates the `Cargo.toml` for a project.
-///
-/// `manifest_path` is used to compute the relative path to the `mingling` crate.
-pub(crate) fn generate_cargo_toml(proj: &MarkdownTestProject, manifest_path: &Path) -> String {
- let features_str = if proj.features.is_empty() {
- String::new()
- } else {
- let feats: Vec<String> = proj.features.iter().map(|f| format!("\"{f}\"")).collect();
- format!("features = [{}]", feats.join(", "))
- };
-
- let extra_deps = build_extra_deps(proj);
-
- let mingling_path = find_mingling_relative_path(manifest_path);
- let deps_section = if proj.features.is_empty() {
- format!("[dependencies]\nmingling = {{ path = \"{mingling_path}\" }}\n{extra_deps}")
- } else {
- format!(
- "[dependencies]\nmingling = {{ path = \"{mingling_path}\", {features_str} }}\n{extra_deps}"
- )
- };
-
- // Build-time projects mirror the features into [build-dependencies] so
- // build.rs sees the same feature set.
- let build_deps_section = if proj.is_build_time {
- let feats: Vec<String> = proj.features.iter().map(|f| format!("\"{f}\"")).collect();
- let build_feats = if feats.is_empty() {
- String::new()
- } else {
- format!("features = [{}]", feats.join(", "))
- };
- format!(
- "\n[build-dependencies]\nmingling = {{ path = \"{mingling_path}\", {build_feats} }}\n"
- )
- } else {
- String::new()
- };
-
- format!(
- r#"[package]
- name = "test-doc"
- version = "0.0.0"
- edition = "2024"
-
-{deps_section}{build_deps_section}
-[workspace]
-"#
- )
-}
-
-/// Computes the relative path from a manifest's parent directory to `mingling`.
-///
-/// The process current directory is expected to be the project root.
-fn find_mingling_relative_path(manifest_path: &Path) -> String {
- let manifest_dir = manifest_path
- .parent()
- .expect("manifest path has no parent directory");
- let cwd = std::env::current_dir().expect("failed to get current directory");
-
- let relative_to_root = manifest_dir.strip_prefix(&cwd).unwrap_or(manifest_dir);
- let depth = relative_to_root.components().count();
-
- let mut result = String::new();
- for _ in 0..depth {
- result.push_str("../");
- }
- result.push_str("mingling");
- result
-}
-
-/// Generates `main.rs` for a project.
-///
-/// Automatically prepends `use mingling::prelude::*;` and appends `fn main() {}`
-/// and `gen_program!()` when the block does not provide them.
-pub(crate) fn generate_main_rs(proj: &MarkdownTestProject) -> String {
- let mut output = String::from("#![allow(dead_code)]\n#![allow(unused)]\n");
-
- if !proj.code.contains("use mingling::prelude::*;") {
- output.push_str("#[allow(unused_imports)]\nuse mingling::prelude::*;\n\n");
- }
- output.push_str(&proj.code);
- output.push('\n');
-
- if !proj.has_main {
- output.push_str("\nfn main() {}\n");
- }
- if !proj.has_gen_program {
- output.push_str("\nmingling::macros::gen_program!();\n");
- }
- output
-}
-
-/// Generates `build.rs` for a build-time project: the code wrapped in
-/// `fn main() { }` unless the block already provides one.
-pub(crate) fn generate_build_rs(proj: &MarkdownTestProject) -> String {
- let mut output = String::from("#![allow(dead_code)]\n#![allow(unused)]\n");
- if proj.has_main {
- output.push_str(&proj.code);
- } else {
- output.push_str("fn main() {\n");
- for line in proj.code.lines() {
- output.push_str(" ");
- output.push_str(line);
- output.push('\n');
- }
- output.push_str("}\n");
- }
- output
-}
diff --git a/mingling_ci/src/markdown/test.rs b/mingling_ci/src/markdown/test.rs
deleted file mode 100644
index 8ecf18d..0000000
--- a/mingling_ci/src/markdown/test.rs
+++ /dev/null
@@ -1,152 +0,0 @@
-//! Parallel execution of markdown test projects.
-
-use std::collections::BTreeMap;
-use std::path::{Path, PathBuf};
-
-use colored::Colorize;
-
-use crate::progress::task_progress_bar;
-
-use super::project::{
- MarkdownTestProject, generate_build_rs, generate_cargo_toml, generate_main_rs,
-};
-
-/// Temporary root for the generated test crates.
-const TEMP_BASE: &str = ".temp/doc-test";
-
-/// Outcome of testing one code block.
-pub(crate) struct MarkdownBlockOutcome {
- pub source_file: String,
- pub line: usize,
- pub ok: bool,
- /// Failure detail; empty when `ok`.
- pub output: String,
-}
-
-/// Runs the given projects in parallel.
-///
-/// Projects sharing a dependency hash share one temporary crate (written
-/// serially within the group); groups run in parallel. Progress is shown on
-/// stderr; failures print there too. Returns one outcome per block.
-pub(crate) async fn try_test_markdown_project(
- projs: Vec<MarkdownTestProject>,
-) -> Vec<MarkdownBlockOutcome> {
- // Group by dependency hash for crate sharing.
- let mut groups: BTreeMap<String, Vec<MarkdownTestProject>> = BTreeMap::new();
- for proj in projs {
- groups.entry(proj.compute_hash()).or_default().push(proj);
- }
-
- let total: usize = groups.values().map(Vec::len).sum();
- let pb = task_progress_bar(total, "Testing");
- pb.set_message("blocks");
-
- // One blocking task per group; blocks within a group are serial because
- // they share the same crate directory.
- let mut handles = Vec::new();
- for (hash, blocks) in groups {
- let pb = pb.clone();
- handles.push(tokio::task::spawn_blocking(move || {
- let crate_dir = PathBuf::from(TEMP_BASE).join(&hash);
- let src_dir = crate_dir.join("src");
- let manifest_path = crate_dir.join("Cargo.toml");
- let cargo_toml = generate_cargo_toml(&blocks[0], &manifest_path);
-
- let mut group_outcomes = Vec::new();
- for proj in &blocks {
- let label = format!("{}:{}", proj.source_file, proj.line);
- pb.set_message(label.clone());
-
- let main_rs = if proj.is_build_time {
- generate_build_rs(proj)
- } else {
- generate_main_rs(proj)
- };
- let (ok, err) = build_block(
- &src_dir,
- &manifest_path,
- &cargo_toml,
- &main_rs,
- proj.is_build_time,
- );
- pb.inc(1);
-
- if !ok {
- // Plain stderr: `pb.println` is swallowed on non-TTY (CI).
- eprintln!(" {} {label}", "failed".bold().bright_red());
- eprintln!(" {label} FAILED:\n{err}");
- }
- group_outcomes.push(MarkdownBlockOutcome {
- source_file: proj.source_file.clone(),
- line: proj.line,
- ok,
- output: err,
- });
- }
- group_outcomes
- }));
- }
-
- let mut all_outcomes = Vec::new();
- for handle in handles {
- if let Ok(group_outcomes) = handle.await {
- all_outcomes.extend(group_outcomes);
- }
- }
-
- pb.finish_and_clear();
- all_outcomes
-}
-
-/// Writes the temporary crate files and runs `cargo check`.
-///
-/// When `is_build_time` is true, the content goes to `build.rs` with a stub
-/// `main.rs`; otherwise it goes to `src/main.rs`.
-fn build_block(
- src_dir: &Path,
- manifest_path: &Path,
- cargo_toml: &str,
- content: &str,
- is_build_time: bool,
-) -> (bool, String) {
- if let Err(e) = std::fs::create_dir_all(src_dir) {
- return (false, format!("mkdir: {e}"));
- }
- if let Err(e) = std::fs::write(manifest_path, cargo_toml) {
- return (false, format!("write Cargo.toml: {e}"));
- }
-
- if is_build_time {
- let crate_dir = manifest_path
- .parent()
- .expect("manifest path has a parent directory");
- if let Err(e) = std::fs::write(crate_dir.join("build.rs"), content) {
- return (false, format!("write build.rs: {e}"));
- }
- if let Err(e) = std::fs::write(src_dir.join("main.rs"), "fn main() {}\n") {
- return (false, format!("write main.rs: {e}"));
- }
- } else if let Err(e) = std::fs::write(src_dir.join("main.rs"), content) {
- return (false, format!("write main.rs: {e}"));
- }
-
- let output = std::process::Command::new("cargo")
- .args(["check", "--color=always", "--manifest-path"])
- .arg(manifest_path)
- .output();
- match output {
- Ok(output) if output.status.success() => (true, String::new()),
- Ok(output) => {
- let mut log = String::from_utf8_lossy(&output.stdout).into_owned();
- log.push_str(&String::from_utf8_lossy(&output.stderr));
- let lines: Vec<&str> = log.lines().collect();
- let tail = &lines[lines.len().saturating_sub(20)..];
- let exit = output
- .status
- .code()
- .map_or_else(|| "?".to_string(), |c| c.to_string());
- (false, format!("exit code {exit}\n{}", tail.join("\n")))
- }
- Err(e) => (false, format!("failed to run cargo: {e}")),
- }
-}
generated by cgit (git 2.34.1) at 2026-08-19 01:20:27 +0000