From addfbbf0b33a6251605990da73c2de5131766827 Mon Sep 17 00:00:00 2001 From: 魏曹先生 <1992414357@qq.com> Date: Wed, 24 Jun 2026 11:23:32 +0800 Subject: Run CI tasks in parallel with progress bars --- dev_tools/src/bin/ci.rs | 55 +++++++------ dev_tools/src/bin/test-all-markdown-code.rs | 53 ++++++------ dev_tools/src/bin/test-examples.rs | 65 +++++++++------ dev_tools/src/lib.rs | 122 ++++++++++++++++++++++++++++ 4 files changed, 217 insertions(+), 78 deletions(-) (limited to 'dev_tools/src') diff --git a/dev_tools/src/bin/ci.rs b/dev_tools/src/bin/ci.rs index d1071ab..1f536c9 100644 --- a/dev_tools/src/bin/ci.rs +++ b/dev_tools/src/bin/ci.rs @@ -2,7 +2,7 @@ use std::io::Write as _; use std::process::exit; use tools::{ - cargo_tomls, eprintln_cargo_style, println_cargo_style, run_cmd, wprintln_cargo_style, + cargo_tomls, crate_name_from, eprintln_cargo_style, println_cargo_style, run_cmd, run_parallel, }; fn get_ignore_dirs() -> Vec { @@ -128,73 +128,76 @@ fn ci(test_docs: bool, test_codes: bool, run_all: bool) -> Result<(), i32> { } fn test_examples() -> Result<(), i32> { - println_cargo_style!("Testing: examples"); - run_cmd!("cargo run --manifest-path dev_tools/Cargo.toml --bin test-examples") + run_cmd!("cargo run --manifest-path dev_tools/Cargo.toml --color always --bin test-examples") } fn test_docs_code_blocks() -> Result<(), i32> { - println_cargo_style!("Testing: documentation code blocks"); - run_cmd!("cargo run --manifest-path dev_tools/Cargo.toml --bin test-all-markdown-code") + run_cmd!( + "cargo run --manifest-path dev_tools/Cargo.toml --color always --bin test-all-markdown-code" + ) } fn build_all() -> Result<(), i32> { let ignore_dirs = get_ignore_dirs(); let cargo_tomls = cargo_tomls(); + let mut tasks = Vec::new(); for cargo_toml in cargo_tomls { let path = cargo_toml.parent().unwrap_or(std::path::Path::new("")); let path_str = path.to_string_lossy(); if ignore_dirs.iter().any(|d| path_str.contains(d.as_str())) { - wprintln_cargo_style!("Skipping: {} (ignored dir)", cargo_toml.to_string_lossy()); continue; } - println_cargo_style!("Build: {}", cargo_toml.to_string_lossy()); - run_cmd!( - "cargo build --manifest-path {}", + let label = format!("Build: {}", cargo_toml.to_string_lossy()); + let crate_name = crate_name_from(&cargo_toml); + let cmd = format!( + "cargo build --manifest-path {} --color always", cargo_toml.to_string_lossy() - )?; + ); + tasks.push((label, crate_name, cmd)); } - - Ok(()) + run_parallel("Building", tasks) } fn clippy_all() -> Result<(), i32> { let ignore_dirs = get_ignore_dirs(); let cargo_tomls = cargo_tomls(); + let mut tasks = Vec::new(); for cargo_toml in cargo_tomls { let path = cargo_toml.parent().unwrap_or(std::path::Path::new("")); let path_str = path.to_string_lossy(); if ignore_dirs.iter().any(|d| path_str.contains(d.as_str())) { - println_cargo_style!("Skipping: {} (ignored dir)", cargo_toml.to_string_lossy()); continue; } - println_cargo_style!("Clippy: {}", cargo_toml.to_string_lossy()); - run_cmd!( - "cargo clippy --manifest-path {} -- -D warnings", + let label = format!("Clippy: {}", cargo_toml.to_string_lossy()); + let crate_name = crate_name_from(&cargo_toml); + let cmd = format!( + "cargo clippy --manifest-path {} --color always -- -D warnings", cargo_toml.to_string_lossy() - )?; + ); + tasks.push((label, crate_name, cmd)); } - - Ok(()) + run_parallel("Clippy", tasks) } fn test_all() -> Result<(), i32> { let ignore_dirs = get_ignore_dirs(); let cargo_tomls = cargo_tomls(); + let mut tasks = Vec::new(); for cargo_toml in cargo_tomls { let path = cargo_toml.parent().unwrap_or(std::path::Path::new("")); let path_str = path.to_string_lossy(); if ignore_dirs.iter().any(|d| path_str.contains(d.as_str())) { - println_cargo_style!("Skipping: {} (ignored dir)", cargo_toml.to_string_lossy()); continue; } - println_cargo_style!("Testing: {}", cargo_toml.to_string_lossy()); - run_cmd!( - "cargo test --manifest-path {}", + let label = format!("Testing: {}", cargo_toml.to_string_lossy()); + let crate_name = crate_name_from(&cargo_toml); + let cmd = format!( + "cargo test --manifest-path {} --color always", cargo_toml.to_string_lossy() - )?; + ); + tasks.push((label, crate_name, cmd)); } - - Ok(()) + run_parallel("Testing", tasks) } fn docs_refresh() -> Result<(), i32> { diff --git a/dev_tools/src/bin/test-all-markdown-code.rs b/dev_tools/src/bin/test-all-markdown-code.rs index 31aebdd..a1acb22 100644 --- a/dev_tools/src/bin/test-all-markdown-code.rs +++ b/dev_tools/src/bin/test-all-markdown-code.rs @@ -1,9 +1,9 @@ use std::collections::HashMap; use std::env; -use std::io::Write; use std::path::{Path, PathBuf}; use colored::Colorize; +use indicatif::ProgressBar; use tools::verify::{ build_block, compute_block_hash, generate_cargo_toml, generate_main_rs, is_block_testable, parse_code_blocks, write_summary_report, @@ -70,7 +70,6 @@ async fn main() { let readme_path = PathBuf::from(&config.verified.readme); if readme_path.exists() { files.push(("README".to_string(), readme_path)); - println_cargo_style!("Source: found README.md"); } // English docs @@ -79,7 +78,6 @@ async fn main() { let dir = PathBuf::from(base); if dir.exists() && dir.is_dir() { collect_md_files(&dir, &mut files, "en"); - println_cargo_style!("Source: found docs/pages/"); } } @@ -89,13 +87,11 @@ async fn main() { let dir = PathBuf::from(base); if dir.exists() && dir.is_dir() { collect_md_files(&dir, &mut files, "zh_CN"); - println_cargo_style!("Source: found docs/_zh_CN/pages/"); } } // If a single file was specified, filter the list to only that file if let Some(ref target) = single_file { - // Canonicalize to compare paths reliably let target_canon = std::fs::canonicalize(target).unwrap_or_else(|_| target.clone()); files.retain(|(_, path)| { std::fs::canonicalize(path) @@ -109,7 +105,6 @@ async fn main() { ); std::process::exit(1); } - println_cargo_style!("Source: testing only file '{}'", target.display()); } if files.is_empty() { @@ -141,10 +136,18 @@ async fn main() { return; } - println_cargo_style!( - "Test: found {total_testable} testable code blocks across {} files", - files.len() + // Create a shared progress bar + let bar = ProgressBar::new(total_testable as u64); + bar.set_style( + indicatif::ProgressStyle::default_bar() + .template(&format!( + "{} [{{bar:28}}] {{pos}}/{{len}}: {{msg}}", + " Testing".bold().bright_cyan() + )) + .unwrap() + .progress_chars("=> "), ); + bar.set_message("blocks"); // Group blocks by dependency hash let mut groups: HashMap> = HashMap::new(); @@ -153,11 +156,6 @@ async fn main() { groups.entry(hash).or_default().push((idx, block)); } - println_cargo_style!( - "Cache: grouped into {} unique dependency configurations", - groups.len() - ); - let temp_base = PathBuf::from(".temp/doc-test"); // Sort groups by hash for deterministic output order @@ -169,14 +167,12 @@ async fn main() { let mut handles = Vec::new(); for (hash, blocks) in group_vec { let temp_base = temp_base.clone(); + let bar = bar.clone(); // clone shares the same underlying progress let handle = tokio::task::spawn_blocking(move || { let crate_dir = temp_base.join(&hash); let src_dir = crate_dir.join("src"); let manifest_path = crate_dir.join("Cargo.toml"); - // Buffer all output for this group so it prints contiguously - let mut output = String::new(); - // Generate a single Cargo.toml for the whole group (all blocks share same deps) let first_block = &blocks[0].1; let cargo_toml = generate_cargo_toml(first_block, "test-doc", &manifest_path); @@ -186,23 +182,23 @@ async fn main() { let block_label = format!("Block {block_idx} ({}:{})", block.source_file, block.line); + bar.set_message(block_label.clone()); + let main_rs = generate_main_rs(block); let (ok, err) = build_block(&src_dir, &manifest_path, &cargo_toml, &main_rs); if ok { - output.push_str(&format!( - " Testing {block_label} ... {}\n", - "passed".bold().bright_green() - )); + bar.inc(1); } else { - output.push_str(&format!( - " Testing {block_label} ... {}\n", + bar.inc(1); + bar.println(format!( + " {} {block_label}", "failed".bold().bright_red() )); - output.push_str(&format!(" {block_label} FAILED:\n{err}\n")); + bar.println(format!(" {block_label} FAILED:\n{err}")); } group_results.push((block.source_file.clone(), block.line, ok, err)); } - (output, group_results) + group_results }); handles.push(handle); } @@ -214,10 +210,7 @@ async fn main() { for handle in handles { match handle.await { - Ok((output, group_results)) => { - // Print entire group output at once, avoiding interleaving - print!("{output}"); - let _ = std::io::stdout().flush(); + Ok(group_results) => { for (file, line, ok, err) in group_results { if ok { passed += 1; @@ -234,6 +227,8 @@ async fn main() { } } + bar.finish_and_clear(); + let result_msg = format!("Result: {passed}/{total_testable} blocks passed"); println_cargo_style!(result_msg); diff --git a/dev_tools/src/bin/test-examples.rs b/dev_tools/src/bin/test-examples.rs index ddf5f7c..5153709 100644 --- a/dev_tools/src/bin/test-examples.rs +++ b/dev_tools/src/bin/test-examples.rs @@ -1,7 +1,9 @@ use std::collections::HashMap; +use colored::Colorize; +use indicatif::ProgressBar; use serde::Deserialize; -use tools::{eprintln_cargo_style, println_cargo_style, run_cmd}; +use tools::{eprintln_cargo_style, println_cargo_style}; #[derive(Deserialize)] struct TestConfig { @@ -26,7 +28,24 @@ fn main() { let _ = colored::control::set_virtual_terminal(true); let config = load_config(); - let (passed, total) = run_all_tests(&config); + + // Count total test cases upfront + let total: usize = config.test.values().map(|cases| cases.len()).sum(); + let bar = ProgressBar::new(total as u64); + bar.set_style( + indicatif::ProgressStyle::default_bar() + .template(&format!( + "{} [{{bar:28}}] {{pos}}/{{len}}: {{msg}}", + " Testing".bold().bright_cyan() + )) + .unwrap() + .progress_chars("=> "), + ); + bar.set_message("examples"); + + let passed = run_all_tests(&config, &bar); + + bar.finish_and_clear(); println_cargo_style!("Result: {}/{} tests passed", passed, total); @@ -49,38 +68,40 @@ fn load_config() -> TestConfig { }) } -/// Run all example test groups, return (passed, total) -fn run_all_tests(config: &TestConfig) -> (usize, usize) { - let mut total = 0; +/// Run all example test groups, return number passed +fn run_all_tests(config: &TestConfig, bar: &ProgressBar) -> usize { let mut passed = 0; for (example_name, test_cases) in &config.test { - println_cargo_style!("Test: {}", example_name); + bar.set_message(example_name.clone()); if !build_example(example_name) { - total += test_cases.len(); + bar.inc(test_cases.len() as u64); continue; } for test_case in test_cases { - total += 1; - if run_single_test(example_name, test_case) { + if run_single_test(example_name, test_case, bar) { passed += 1; } + bar.inc(1); } } - (passed, total) + passed } /// Build the example binary, return true on success fn build_example(example_name: &str) -> bool { let manifest = format!("examples/{example_name}/Cargo.toml"); - run_cmd!("cargo build --manifest-path {}", manifest).is_ok() + tools::run_cmd_capture(&format!( + "cargo build --manifest-path {manifest} --color always", + )) + .is_ok() } /// Run a single test case, return true on pass -fn run_single_test(example_name: &str, test_case: &TestCase) -> bool { +fn run_single_test(example_name: &str, test_case: &TestCase, bar: &ProgressBar) -> bool { let binary_path = format!(".temp/target/debug/{}", get_binary_name(example_name)); let args: Vec<&str> = test_case.command.split_whitespace().collect(); @@ -90,7 +111,7 @@ fn run_single_test(example_name: &str, test_case: &TestCase) -> bool { { Ok(o) => o, Err(e) => { - eprintln_cargo_style!("'{}' - failed to run: {}", test_case.command, e); + bar.println(format!("'{}' - failed to run: {}", test_case.command, e)); return false; } }; @@ -104,22 +125,20 @@ fn run_single_test(example_name: &str, test_case: &TestCase) -> bool { || actual_stdout.contains(&test_case.expect.result); if exit_ok && result_ok { - println_cargo_style!("Passed: '{}'", test_case.command); true } else { - eprintln_cargo_style!("'{}'", test_case.command); + bar.println(format!("failed: '{}'", test_case.command)); if !exit_ok { - eprintln_cargo_style!( - "Expected exit code: {}, actual: {}", - test_case.expect.exit_code, - actual_exit_code - ); + bar.println(format!( + " Expected exit code: {}, actual: {}", + test_case.expect.exit_code, actual_exit_code + )); } if !result_ok { - eprintln_cargo_style!("Expected output: {:?}", test_case.expect.result); - eprintln_cargo_style!("Actual stdout: {:?}", actual_stdout); + bar.println(format!(" Expected output: {:?}", test_case.expect.result)); + bar.println(format!(" Actual stdout: {:?}", actual_stdout)); if !actual_stderr.is_empty() { - eprintln_cargo_style!("Actual stderr: {:?}", actual_stderr); + bar.println(format!(" Actual stderr: {:?}", actual_stderr)); } } false diff --git a/dev_tools/src/lib.rs b/dev_tools/src/lib.rs index 13ed71f..2bd55a2 100644 --- a/dev_tools/src/lib.rs +++ b/dev_tools/src/lib.rs @@ -183,6 +183,128 @@ pub fn run_cmd_capture(cmd: impl Into) -> Result } } +/// Extract a crate-style name from a `Cargo.toml` path. +/// +/// Examples: +/// - `mingling_core/Cargo.toml` → `mingling_core` +/// - `.` → `(root)` +pub fn crate_name_from(path: &std::path::Path) -> String { + path.parent() + .and_then(|p| p.file_name()) + .and_then(|n| n.to_str()) + .unwrap_or("(root)") + .to_string() +} + +/// Run a list of `(label_for_errors, crate_name_for_bar, shell_command)` tuples +/// in parallel with a progress bar. +/// +/// - Success: silent, the bar tracks progress: +/// ` Building [============================] 32/32: mingling_core` +/// - Failure: `pb.println()` prints the error immediately above the bar. +pub fn run_parallel(phase: &str, tasks: Vec<(String, String, String)>) -> Result<(), i32> { + let n = tasks.len(); + if n == 0 { + return Ok(()); + } + + // Cargo-style prefix: right-aligned to 12 chars, bold bright cyan + let padding = " ".repeat(12 - phase.len()); + let styled_prefix = format!("{}{}", padding, phase.bold().bright_cyan()); + + let pb = indicatif::ProgressBar::new(n as u64); + pb.set_style( + indicatif::ProgressStyle::default_bar() + .template(&format!( + "{} [{{bar:28}}] {{pos}}/{{len}}: {{msg}}", + styled_prefix + )) + .unwrap() + .progress_chars("=> "), + ); + pb.set_position(0); + + // Pre-extract labels for error messages + let labels: Vec = tasks.iter().map(|(l, _, _)| l.clone()).collect(); + + let (tx, rx) = std::sync::mpsc::channel::<(usize, String, Result)>(); + + for (i, (_label, crate_name, cmd)) in tasks.into_iter().enumerate() { + let tx = tx.clone(); + std::thread::spawn(move || { + let result = run_cmd_capture(&cmd); + let _ = tx.send((i, crate_name, result)); + }); + } + drop(tx); + + let mut first_exit_code = 0; + + while let Ok((i, crate_name, result)) = rx.recv() { + pb.inc(1); + pb.set_message(crate_name); + + if let Err((code, output)) = result { + if first_exit_code == 0 { + first_exit_code = code; + } + pb.println(format!( + "{}: {} failed (exit code {})", + "error".bright_red().bold(), + labels[i], + code, + )); + if !output.is_empty() { + pb.println(output.trim_end().to_string()); + } + } + } + + pb.finish_and_clear(); + + if first_exit_code != 0 { + Err(first_exit_code) + } else { + Ok(()) + } +} + +/// Run a single shell command with a progress bar, capturing its output. +/// +/// - Success: bar clears silently. +/// - Failure: error is printed above the bar, then the bar clears. +pub fn run_cmd_with_progress(phase: &str, label: &str, cmd: String) -> Result<(), i32> { + let padding = " ".repeat(12 - phase.len()); + let styled_prefix = format!("{}{}", padding, phase.bold().bright_cyan()); + + let pb = indicatif::ProgressBar::new(1); + pb.set_style( + indicatif::ProgressStyle::default_bar() + .template(&format!( + "{} [{{bar:28}}] {{pos}}/{{len}}: {{msg}}", + styled_prefix + )) + .unwrap() + .progress_chars("=> "), + ); + pb.set_message(label.to_owned()); + + let result = run_cmd_capture(&cmd); + pb.inc(1); + pb.finish_and_clear(); + + match result { + Ok(_) => Ok(()), + Err((code, output)) => { + eprintln_cargo_style(format!("{} failed (exit code {})", label, code)); + if !output.is_empty() { + println!("{}", output.trim_end()); + } + Err(code) + } + } +} + #[must_use] pub fn cargo_tomls() -> Vec { let mut cargo_tomls = Vec::new(); -- cgit