Parallelize markdown code block verification

4 files changed, 99 insertions, 34 deletions

diff --git a/dev_tools/Cargo.lock b/dev_tools/Cargo.lock
index 1180474..67acfc9 100644
--- a/dev_tools/Cargo.lock
+++ b/dev_tools/Cargo.lock
@@ -61,6 +61,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79"
 
 [[package]]
+name = "pin-project-lite"
+version = "0.2.17"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd"
+
+[[package]]
 name = "proc-macro2"
 version = "1.0.106"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -142,6 +148,27 @@ dependencies = [
 ]
 
 [[package]]
+name = "tokio"
+version = "1.52.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8fc7f01b389ac15039e4dc9531aa973a135d7a4135281b12d7c1bc79fd57fffe"
+dependencies = [
+ "pin-project-lite",
+ "tokio-macros",
+]
+
+[[package]]
+name = "tokio-macros"
+version = "2.7.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
 name = "toml"
 version = "0.8.23"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -191,6 +218,7 @@ dependencies = [
  "just_template",
  "serde",
  "serde_json",
+ "tokio",
  "toml",
 ]
 
diff --git a/dev_tools/Cargo.toml b/dev_tools/Cargo.toml
index a904bd5..7e9e332 100644
--- a/dev_tools/Cargo.toml
+++ b/dev_tools/Cargo.toml
@@ -17,3 +17,4 @@ colored = "3.1.1"
 toml = "0.8"
 serde = { version = "1", features = ["derive"] }
 serde_json = "1"
+tokio = { version = "1", features = ["rt-multi-thread", "macros"] }
diff --git a/dev_tools/src/bin/test-all-markdown-code.rs b/dev_tools/src/bin/test-all-markdown-code.rs
index e43053e..31aebdd 100644
--- a/dev_tools/src/bin/test-all-markdown-code.rs
+++ b/dev_tools/src/bin/test-all-markdown-code.rs
@@ -1,5 +1,6 @@
 use std::collections::HashMap;
 use std::env;
+use std::io::Write;
 use std::path::{Path, PathBuf};
 
 use colored::Colorize;
@@ -24,7 +25,8 @@ struct VerifiedPaths {
     documents_zh_cn: Option<String>,
 }
 
-fn main() {
+#[tokio::main]
+async fn main() {
     #[cfg(windows)]
     let _ = colored::control::set_virtual_terminal(true);
 
@@ -158,40 +160,76 @@ fn main() {
 
     let temp_base = PathBuf::from(".temp/doc-test");
 
-    // Build groups — same hash reuses the same Cargo.toml
+    // Sort groups by hash for deterministic output order
+    let mut group_vec: Vec<(String, Vec<(usize, tools::verify::CodeBlock)>)> =
+        groups.into_iter().collect();
+    group_vec.sort_by(|a, b| a.0.cmp(&b.0));
+
+    // Spawn a blocking task per group — groups run in parallel, blocks within a group are serial
+    let mut handles = Vec::new();
+    for (hash, blocks) in group_vec {
+        let temp_base = temp_base.clone();
+        let handle = tokio::task::spawn_blocking(move || {
+            let crate_dir = temp_base.join(&hash);
+            let src_dir = crate_dir.join("src");
+            let manifest_path = crate_dir.join("Cargo.toml");
+
+            // Buffer all output for this group so it prints contiguously
+            let mut output = String::new();
+
+            // Generate a single Cargo.toml for the whole group (all blocks share same deps)
+            let first_block = &blocks[0].1;
+            let cargo_toml = generate_cargo_toml(first_block, "test-doc", &manifest_path);
+
+            let mut group_results: Vec<(String, usize, bool, String)> = Vec::new();
+            for (block_idx, block) in &blocks {
+                let block_label =
+                    format!("Block {block_idx} ({}:{})", block.source_file, block.line);
+
+                let main_rs = generate_main_rs(block);
+                let (ok, err) = build_block(&src_dir, &manifest_path, &cargo_toml, &main_rs);
+                if ok {
+                    output.push_str(&format!(
+                        "  Testing {block_label} ... {}\n",
+                        "passed".bold().bright_green()
+                    ));
+                } else {
+                    output.push_str(&format!(
+                        "  Testing {block_label} ... {}\n",
+                        "failed".bold().bright_red()
+                    ));
+                    output.push_str(&format!("  {block_label} FAILED:\n{err}\n"));
+                }
+                group_results.push((block.source_file.clone(), block.line, ok, err));
+            }
+            (output, group_results)
+        });
+        handles.push(handle);
+    }
+
+    // Collect results from all groups
     let mut results: Vec<(String, usize, bool, String)> = Vec::new();
     let mut passed = 0usize;
     let mut failed = 0usize;
 
-    // Sort groups by hash for deterministic output order
-    let mut group_keys: Vec<&String> = groups.keys().collect();
-    group_keys.sort();
-
-    for hash in group_keys {
-        let blocks = &groups[hash];
-        let crate_dir = temp_base.join(hash);
-        let src_dir = crate_dir.join("src");
-        let manifest_path = crate_dir.join("Cargo.toml");
-
-        // Generate a single Cargo.toml for the whole group (all blocks share same deps)
-        let first_block = &blocks[0].1;
-        let cargo_toml = generate_cargo_toml(first_block, "test-doc", &manifest_path);
-
-        for (block_idx, block) in blocks {
-            let block_label = format!("Block {block_idx} ({}:{})", block.source_file, block.line);
-            print!("  Testing {block_label} ... ");
-
-            let main_rs = generate_main_rs(block);
-            let (ok, err) = build_block(&src_dir, &manifest_path, &cargo_toml, &main_rs);
-            if ok {
-                println!("{}", "passed".bold().bright_green());
-                passed += 1;
-                results.push((block.source_file.clone(), block.line, true, String::new()));
-            } else {
-                println!("{}", "failed".bold().bright_red());
-                failed += 1;
-                results.push((block.source_file.clone(), block.line, false, err.clone()));
-                eprintln_cargo_style!(format!("  {block_label} FAILED:\n{err}"));
+    for handle in handles {
+        match handle.await {
+            Ok((output, group_results)) => {
+                // Print entire group output at once, avoiding interleaving
+                print!("{output}");
+                let _ = std::io::stdout().flush();
+                for (file, line, ok, err) in group_results {
+                    if ok {
+                        passed += 1;
+                    } else {
+                        failed += 1;
+                    }
+                    results.push((file, line, ok, err));
+                }
+            }
+            Err(e) => {
+                eprintln_cargo_style!("Task panicked: {}", e);
+                std::process::exit(1);
             }
         }
     }
diff --git a/dev_tools/src/verify.rs b/dev_tools/src/verify.rs
index 319a38a..06db1b2 100644
--- a/dev_tools/src/verify.rs
+++ b/dev_tools/src/verify.rs
@@ -1,4 +1,3 @@
-use std::io::Write;
 use std::path::Path;
 
 use crate::println_cargo_style;
@@ -293,7 +292,7 @@ pub fn build_block(
         Err(e) => return (false, format!("spawn: {e}")),
     };
 
-    // Read stderr while it streams
+    // Read stderr (buffered, not forwarded — groups print their own output contiguously)
     use std::io::BufRead;
     let stderr_handle = child.stderr.take().unwrap();
     let reader = std::io::BufReader::new(stderr_handle);
@@ -301,7 +300,6 @@ pub fn build_block(
     for line in reader.lines() {
         match line {
             Ok(l) => {
-                let _ = writeln!(std::io::stderr(), "{l}");
                 captured.push_str(&l);
                 captured.push('\n');
             }