Refactor test-readme into generic docs code block verifier

5 files changed, 280 insertions, 119 deletions

diff --git a/dev_tools/src/bin/ci.rs b/dev_tools/src/bin/ci.rs
index 21763d1..ce8f058 100644
--- a/dev_tools/src/bin/ci.rs
+++ b/dev_tools/src/bin/ci.rs
@@ -73,7 +73,7 @@ fn ci() -> Result<(), i32> {
     clippy_all()?;
     test_all()?;
     test_examples()?;
-    test_readme()?;
+    test_docs_code_blocks()?;
     docs_refresh()?;
 
     run_cmd!("git add --renormalize .")?;
@@ -86,9 +86,9 @@ fn test_examples() -> Result<(), i32> {
     run_cmd!("cargo run --manifest-path dev_tools/Cargo.toml --bin test-examples")
 }
 
-fn test_readme() -> Result<(), i32> {
-    println_cargo_style!("Testing: readme code blocks");
-    run_cmd!("cargo run --manifest-path dev_tools/Cargo.toml --bin test-readme")
+fn test_docs_code_blocks() -> Result<(), i32> {
+    println_cargo_style!("Testing: documentation code blocks");
+    run_cmd!("cargo run --manifest-path dev_tools/Cargo.toml --bin test-all-markdown-code")
 }
 
 fn build_all() -> Result<(), i32> {
diff --git a/dev_tools/src/bin/test-all-markdown-code.rs b/dev_tools/src/bin/test-all-markdown-code.rs
new file mode 100644
index 0000000..c4e89be
--- /dev/null
+++ b/dev_tools/src/bin/test-all-markdown-code.rs
@@ -0,0 +1,182 @@
+use std::path::{Path, PathBuf};
+
+use colored::Colorize;
+use tools::verify::{
+    build_block, generate_cargo_toml, generate_main_rs, is_block_testable, parse_code_blocks,
+    write_summary_report,
+};
+use tools::{eprintln_cargo_style, println_cargo_style};
+
+/// Config from verified-docs.toml
+#[derive(serde::Deserialize)]
+struct Config {
+    verified: VerifiedPaths,
+}
+
+#[derive(serde::Deserialize)]
+struct VerifiedPaths {
+    readme: String,
+    #[serde(default)]
+    documents_en_us: Option<String>,
+    #[serde(default)]
+    documents_zh_cn: Option<String>,
+}
+
+fn main() {
+    #[cfg(windows)]
+    let _ = colored::control::set_virtual_terminal(true);
+
+    let config_path = PathBuf::from("verified-docs.toml");
+    if !config_path.exists() {
+        eprintln_cargo_style!("verified-docs.toml not found in current directory");
+        std::process::exit(1);
+    }
+
+    let config: Config = {
+        let content = std::fs::read_to_string(&config_path).unwrap_or_else(|_e| {
+            eprintln_cargo_style!("Failed to read verified-docs.toml");
+            std::process::exit(1);
+        });
+        toml::from_str(&content).unwrap_or_else(|_e| {
+            eprintln_cargo_style!("Failed to parse verified-docs.toml");
+            std::process::exit(1);
+        })
+    };
+
+    // Collect all markdown files from config
+    let mut files: Vec<(String, PathBuf)> = Vec::new();
+
+    // README
+    let readme_path = PathBuf::from(&config.verified.readme);
+    if readme_path.exists() {
+        files.push(("README".to_string(), readme_path));
+        println_cargo_style!("Source: found README.md");
+    }
+
+    // English docs
+    if let Some(pattern) = &config.verified.documents_en_us {
+        let base = pattern.trim_end_matches("/**").trim_end_matches('*');
+        let dir = PathBuf::from(base);
+        if dir.exists() && dir.is_dir() {
+            collect_md_files(&dir, &mut files, "en");
+            println_cargo_style!("Source: found docs/pages/");
+        }
+    }
+
+    // Chinese docs
+    if let Some(pattern) = &config.verified.documents_zh_cn {
+        let base = pattern.trim_end_matches("/**").trim_end_matches('*');
+        let dir = PathBuf::from(base);
+        if dir.exists() && dir.is_dir() {
+            collect_md_files(&dir, &mut files, "zh_CN");
+            println_cargo_style!("Source: found docs/_zh_CN/pages/");
+        }
+    }
+
+    if files.is_empty() {
+        eprintln_cargo_style!("No markdown files found to verify");
+        std::process::exit(1);
+    }
+
+    // Ensure temp directory exists
+    let temp_dir = PathBuf::from(".temp/docs-test");
+    let _ = std::fs::remove_dir_all(&temp_dir);
+
+    let mut all_blocks: Vec<(String, Vec<tools::verify::CodeBlock>)> = Vec::new();
+
+    for (label, path) in &files {
+        let content = std::fs::read_to_string(path).unwrap_or_else(|e| {
+            eprintln_cargo_style!("Failed to read {}: {}", path.display(), e);
+            String::new()
+        });
+        let source_file = format!("{label}/{}", path.file_name().unwrap().to_string_lossy());
+        let blocks = parse_code_blocks(&content, &source_file);
+        let testable: Vec<_> = blocks.into_iter().filter(is_block_testable).collect();
+        if !testable.is_empty() {
+            all_blocks.push((label.clone(), testable));
+        }
+    }
+
+    let total_testable: usize = all_blocks.iter().map(|(_, b)| b.len()).sum();
+
+    if total_testable == 0 {
+        println_cargo_style!("No testable code blocks found");
+        return;
+    }
+
+    println_cargo_style!(
+        "Test: found {total_testable} testable code blocks across {} files",
+        all_blocks.len()
+    );
+
+    // Build each block
+    let mut block_index = 0usize;
+    let mut passed = 0usize;
+    let mut failed = 0usize;
+    let mut results: Vec<(String, usize, bool, String)> = Vec::new();
+
+    for (_label, blocks) in &all_blocks {
+        for block in blocks {
+            block_index += 1;
+
+            let block_label = format!(
+                "Block {} ({}:{})",
+                block_index, block.source_file, block.line
+            );
+            print!("  Testing {block_label} ... ");
+
+            let package_name = format!("test-block-{block_index}");
+
+            let cargo_toml = generate_cargo_toml(block, &package_name);
+            let main_rs = generate_main_rs(block);
+            let src_dir = temp_dir.join("src");
+            let manifest_path = temp_dir.join("Cargo.toml");
+
+            let (ok, err) = build_block(&src_dir, &manifest_path, &cargo_toml, &main_rs);
+            if ok {
+                println!("{}", "passed".bold().bright_green());
+                passed += 1;
+                results.push((block.source_file.clone(), block.line, true, String::new()));
+            } else {
+                println!("{}", "failed".bold().bright_red());
+                failed += 1;
+                results.push((block.source_file.clone(), block.line, false, err.clone()));
+                eprintln_cargo_style!(format!("  {block_label} FAILED:\n{err}"));
+            }
+        }
+    }
+
+    let result_msg = format!("Result: {passed}/{total_testable} blocks passed");
+    println_cargo_style!(result_msg);
+
+    write_summary_report(
+        Path::new(".temp/DOCS-TEST-RESULT.md"),
+        "Documentation Code Block Test Report",
+        &results,
+        total_testable,
+        passed,
+        failed,
+    );
+
+    if failed > 0 {
+        let fail_msg = format!("{failed} block(s) failed to build");
+        eprintln_cargo_style!(fail_msg);
+        std::process::exit(1);
+    }
+
+    println_cargo_style!("Done: All verified code blocks build successfully!");
+}
+
+/// Recursively collect all `.md` files under a directory
+fn collect_md_files(dir: &Path, files: &mut Vec<(String, PathBuf)>, lang: &str) {
+    if let Ok(entries) = std::fs::read_dir(dir) {
+        for entry in entries.flatten() {
+            let path = entry.path();
+            if path.is_dir() {
+                collect_md_files(&path, files, lang);
+            } else if path.extension().is_some_and(|ext| ext == "md") {
+                files.push((lang.to_string(), path));
+            }
+        }
+    }
+}
diff --git a/dev_tools/src/lib.rs b/dev_tools/src/lib.rs
index 9eb1f75..ce897df 100644
--- a/dev_tools/src/lib.rs
+++ b/dev_tools/src/lib.rs
@@ -1,3 +1,5 @@
+pub mod verify;
+
 use colored::Colorize;
 
 #[macro_export]
diff --git a/dev_tools/src/bin/test-readme.rs b/dev_tools/src/verify.rs
index fd1962f..e94226c 100644
--- a/dev_tools/src/bin/test-readme.rs
+++ b/dev_tools/src/verify.rs
@@ -1,106 +1,39 @@
-use colored::Colorize;
-use std::path::{Path, PathBuf};
+use std::path::Path;
 
-use tools::{eprintln_cargo_style, println_cargo_style, run_cmd_and_capture_stderr};
+use crate::{println_cargo_style, run_cmd_and_capture_stderr};
 
-/// Represents a parsed code block from README.md
-struct CodeBlock {
-    /// The line number in README.md where this block starts
-    line: usize,
+/// Represents a parsed code block from a markdown file
+#[derive(Debug, Clone)]
+pub struct CodeBlock {
+    /// Source file path (for reporting)
+    pub source_file: String,
+    /// The line number in source file where this block starts
+    pub line: usize,
     /// The raw Rust source code
-    code: String,
+    pub code: String,
     /// Feature flags extracted from `// Features: [...]` comment
-    features: Vec<String>,
+    pub features: Vec<String>,
+    /// Whether the block had an explicit `// Features:` header
+    pub has_features_header: bool,
+    /// Whether the block has `// NOT VERIFIED` to opt out of testing
+    pub not_verified: bool,
     /// External dependencies extracted from `// Dependencies:` comments
-    external_deps: Vec<(String, String)>,
+    pub external_deps: Vec<(String, String)>,
     /// Whether this block has a `fn main` entry point
-    has_main: bool,
+    pub has_main: bool,
     /// Whether this block has `gen_program!()` call
-    has_gen_program: bool,
+    pub has_gen_program: bool,
 }
 
-fn main() {
-    #[cfg(windows)]
-    let _ = colored::control::set_virtual_terminal(true);
-
-    let readme_path = PathBuf::from("README.md");
-    if !readme_path.exists() {
-        eprintln_cargo_style!("README.md not found in current directory");
-        std::process::exit(1);
-    }
-
-    let content = std::fs::read_to_string(&readme_path).unwrap_or_else(|e| {
-        eprintln_cargo_style!("Failed to read README.md: {}", e);
-        std::process::exit(1);
-    });
-
-    let blocks = parse_code_blocks(&content);
-    if blocks.is_empty() {
-        eprintln_cargo_style!("No Rust code blocks found in README.md");
-        std::process::exit(1);
-    }
-
-    println_cargo_style!("Test: found {} Rust code blocks in README.md", blocks.len());
-
-    // Ensure temp directory exists
-    let temp_dir = PathBuf::from(".temp/readme-test");
-    let _ = std::fs::remove_dir_all(&temp_dir);
-    std::fs::create_dir_all(temp_dir.join("src")).unwrap_or_else(|e| {
-        eprintln_cargo_style!("Failed to create temp directory: {}", e);
-        std::process::exit(1);
-    });
-
-    let mut passed = 0usize;
-    let mut failed = 0usize;
-    let mut results: Vec<(usize, bool, String)> = Vec::new();
-
-    for (i, block) in blocks.iter().enumerate() {
-        let label = format!("Block {} (line {})", i + 1, block.line);
-        print!("  {label} ... ");
-
-        let (ok, err) = build_block(&temp_dir, block);
-        if ok {
-            println!("{}", "passed".bold().bright_green());
-            passed += 1;
-            results.push((block.line, true, String::new()));
-        } else {
-            println!("{}", "failed".bold().bright_red());
-            failed += 1;
-            results.push((block.line, false, err.clone()));
-            eprintln_cargo_style!("  {} FAILED:\n{}", label, err);
-        }
-    }
-
-    println_cargo_style!(
-        "Result: {passed}/{total} blocks passed",
-        total = blocks.len()
-    );
-
-    write_summary_report(
-        Path::new(".temp/README-TEST-RESULT.md"),
-        &results,
-        blocks.len(),
-        passed,
-        failed,
-    );
-
-    if failed > 0 {
-        eprintln_cargo_style!("{failed} block(s) failed to build");
-        std::process::exit(1);
-    }
-
-    println_cargo_style!("Done: All README code blocks build successfully!");
-}
-
-/// Parse all ```rust code blocks from README content
-fn parse_code_blocks(content: &str) -> Vec<CodeBlock> {
+/// Parse all ```rust code blocks from markdown content
+pub fn parse_code_blocks(content: &str, source_file: &str) -> Vec<CodeBlock> {
     let mut blocks = Vec::new();
     let lines: Vec<&str> = content.lines().collect();
     let mut i = 0;
 
     while i < lines.len() {
         if lines[i].trim() == "```rust" {
-            if let Some(block) = parse_single_block(&lines, i) {
+            if let Some(block) = parse_single_block(&lines, i, source_file) {
                 blocks.push(block);
             }
             i += 1;
@@ -115,11 +48,13 @@ fn parse_code_blocks(content: &str) -> Vec<CodeBlock> {
 }
 
 /// Parse a single code block starting at the ```rust line
-fn parse_single_block(lines: &[&str], start: usize) -> Option<CodeBlock> {
+fn parse_single_block(lines: &[&str], start: usize, source_file: &str) -> Option<CodeBlock> {
     let line_num = start + 1; // 1-based line number
 
     let mut code_lines: Vec<String> = Vec::new();
     let mut features: Vec<String> = Vec::new();
+    let mut has_features_header = false;
+    let mut not_verified = false;
     let mut external_deps: Vec<(String, String)> = Vec::new();
     let mut has_main = false;
     let mut has_gen_program = false;
@@ -136,8 +71,16 @@ fn parse_single_block(lines: &[&str], start: usize) -> Option<CodeBlock> {
         }
 
         // Parse header comments
+        // Check for NOT VERIFIED marker
+        if in_header && trimmed == "// NOT VERIFIED" {
+            not_verified = true;
+            idx += 1;
+            continue;
+        }
+
         if in_header && trimmed.starts_with("// ") {
             if trimmed.starts_with("// Features:") {
+                has_features_header = true;
                 let feat_str = trimmed.trim_start_matches("// Features:").trim();
                 if feat_str.starts_with('[') && feat_str.ends_with(']') {
                     let inner = &feat_str[1..feat_str.len() - 1];
@@ -195,9 +138,12 @@ fn parse_single_block(lines: &[&str], start: usize) -> Option<CodeBlock> {
     }
 
     Some(CodeBlock {
+        source_file: source_file.to_string(),
         line: line_num,
         code: code_lines.join("\n"),
         features,
+        has_features_header,
+        not_verified,
         external_deps,
         has_main,
         has_gen_program,
@@ -205,7 +151,7 @@ fn parse_single_block(lines: &[&str], start: usize) -> Option<CodeBlock> {
 }
 
 /// Generate a Cargo.toml for a block
-fn generate_cargo_toml(block: &CodeBlock) -> String {
+pub fn generate_cargo_toml(block: &CodeBlock, package_name: &str) -> String {
     let features_str = if !block.features.is_empty() {
         let feats: Vec<String> = block.features.iter().map(|f| format!("\"{f}\"")).collect();
         format!("features = [{}]", feats.join(", "))
@@ -231,16 +177,20 @@ fn generate_cargo_toml(block: &CodeBlock) -> String {
     }
 
     let deps_section = if features_str.is_empty() {
-        format!("[dependencies]\nmingling = {{ path = \"../../mingling\" }}\n{extra_deps}")
+        format!(
+            "[dependencies]\nmingling = {{ path = \"{}\" }}\n{extra_deps}",
+            find_mingling_relative_path()
+        )
     } else {
         format!(
-            "[dependencies]\nmingling = {{ path = \"../../mingling\", {features_str} }}\n{extra_deps}"
+            "[dependencies]\nmingling = {{ path = \"{}\", {features_str} }}\n{extra_deps}",
+            find_mingling_relative_path()
         )
     };
 
     format!(
         r#"[package]
-name = "readme-test-block"
+name = "{package_name}"
 version = "0.0.0"
 edition = "2024"
 
@@ -250,8 +200,14 @@ edition = "2024"
     )
 }
 
+/// Find the relative path from the temp test directory to mingling crate
+fn find_mingling_relative_path() -> &'static str {
+    // Tests run from project root, temp is under .temp/
+    "../../mingling"
+}
+
 /// Generate main.rs for a block
-fn generate_main_rs(block: &CodeBlock) -> String {
+pub fn generate_main_rs(block: &CodeBlock) -> String {
     let mut output = String::from("#![allow(dead_code)]\n\n");
 
     output.push_str(&block.code);
@@ -269,28 +225,28 @@ fn generate_main_rs(block: &CodeBlock) -> String {
 }
 
 /// Build a single code block as a Cargo project.
-/// Always writes to `{temp_dir}/Cargo.toml` and `{temp_dir}/src/main.rs`.
 /// Returns (success, error_message).
-fn build_block(temp_dir: &Path, block: &CodeBlock) -> (bool, String) {
-    let src_dir = temp_dir.join("src");
-    if let Err(e) = std::fs::create_dir_all(&src_dir) {
+pub fn build_block(
+    src_dir: &Path,
+    manifest_path: &Path,
+    cargo_toml: &str,
+    main_rs: &str,
+) -> (bool, String) {
+    if let Err(e) = std::fs::create_dir_all(src_dir) {
         return (false, format!("mkdir: {e}"));
     }
 
     // Write Cargo.toml
-    let cargo_toml = generate_cargo_toml(block);
-    if let Err(e) = std::fs::write(temp_dir.join("Cargo.toml"), &cargo_toml) {
+    if let Err(e) = std::fs::write(manifest_path, cargo_toml) {
         return (false, format!("write Cargo.toml: {e}"));
     }
 
     // Write main.rs
-    let main_rs = generate_main_rs(block);
-    if let Err(e) = std::fs::write(src_dir.join("main.rs"), &main_rs) {
+    if let Err(e) = std::fs::write(src_dir.join("main.rs"), main_rs) {
         return (false, format!("write main.rs: {e}"));
     }
 
-    // Build with release (single run via shared macro)
-    let manifest_path = temp_dir.join("Cargo.toml");
+    // Build with release
     match run_cmd_and_capture_stderr!(
         "cargo build --release --manifest-path {}",
         manifest_path.to_string_lossy()
@@ -305,36 +261,50 @@ fn build_block(temp_dir: &Path, block: &CodeBlock) -> (bool, String) {
     }
 }
 
-/// Write the .temp/README-TEST-RESULT.md summary report
-fn write_summary_report(
+/// Determine if a block should be treated as a test candidate.
+/// A block is NOT testable only if it has `// NOT VERIFIED` marker.
+pub fn is_block_testable(block: &CodeBlock) -> bool {
+    !block.not_verified
+}
+
+/// Write a summary report
+pub fn write_summary_report(
     path: &Path,
-    results: &[(usize, bool, String)],
+    title: &str,
+    results: &[(String, usize, bool, String)],
     total: usize,
     passed: usize,
     failed: usize,
 ) {
     let mut content = String::new();
-    content.push_str("# README Code Block Test Report\n\n");
+    content.push_str(&format!("# {title}\n\n"));
     content.push_str(&format!(
         "Tested **{total}** code blocks: **{passed}** passed, **{failed}** failed.\n\n"
     ));
     content.push_str("## Results\n\n");
-    content.push_str("| Block | Line | Status |\n");
-    content.push_str("|-------|------|--------|\n");
+    content.push_str("| Block | File | Line | Status |\n");
+    content.push_str("|-------|------|------|--------|\n");
 
-    for (i, (line, ok, _)) in results.iter().enumerate() {
+    for (i, (file, line, ok, _)) in results.iter().enumerate() {
         let status = if *ok { "PASS" } else { "FAIL" };
-        content.push_str(&format!("| {} | {} | {status} |\n", i + 1, line));
+        let short_file = file.rsplit('/').next().unwrap_or(file);
+        content.push_str(&format!(
+            "| {} | {} | {} | {status} |\n",
+            i + 1,
+            short_file,
+            line
+        ));
     }
 
-    let has_failures = results.iter().any(|(_, ok, _)| !ok);
+    let has_failures = results.iter().any(|(_, _, ok, _)| !ok);
     if has_failures {
         content.push_str("\n## Failed Blocks\n\n");
-        for (i, (line, ok, err)) in results.iter().enumerate() {
+        for (i, (file, line, ok, err)) in results.iter().enumerate() {
             if !ok {
                 content.push_str(&format!(
-                    "### Block {} (line {})\n\n```\n{err}\n```\n\n",
+                    "### Block {} (`{}`, line {})\n\n```\n{err}\n```\n\n",
                     i + 1,
+                    file,
                     line
                 ));
             }
diff --git a/verified-docs.toml b/verified-docs.toml
new file mode 100644
index 0000000..d66967f
--- /dev/null
+++ b/verified-docs.toml
@@ -0,0 +1,7 @@
+# Files marked in the following document,
+# all rust code blocks inside will be verified in CI to ensure they can compile
+
+[verified]
+readme = "./README.md"
+documents_en_us = "./docs/pages/**"
+documents_zh_cn = "./docs/_zh_CN/pages/**"