diff options
Diffstat (limited to 'mingling_ci')
| -rw-r--r-- | mingling_ci/help.txt | 1 | ||||
| -rw-r--r-- | mingling_ci/src/cmd.rs | 1 | ||||
| -rw-r--r-- | mingling_ci/src/cmd/cmd_test_examples.rs | 78 | ||||
| -rw-r--r-- | mingling_ci/src/examples.rs | 186 | ||||
| -rw-r--r-- | mingling_ci/src/lib.rs | 1 |
5 files changed, 267 insertions, 0 deletions
diff --git a/mingling_ci/help.txt b/mingling_ci/help.txt index 7e55515..1391b37 100644 --- a/mingling_ci/help.txt +++ b/mingling_ci/help.txt @@ -22,3 +22,4 @@ COMMANDS: build-all Build all crates clippy-all Run clippy with -D warnings on all crates test-all Test all crates + test-examples Build examples and run their test.toml cases diff --git a/mingling_ci/src/cmd.rs b/mingling_ci/src/cmd.rs index 30d65a1..b9664fc 100644 --- a/mingling_ci/src/cmd.rs +++ b/mingling_ci/src/cmd.rs @@ -2,3 +2,4 @@ pub(crate) mod cmd_report_clean; pub(crate) mod cmd_report_collect; pub(crate) mod cmd_show_features; pub(crate) mod cmd_show_manifests; +pub(crate) mod cmd_test_examples; diff --git a/mingling_ci/src/cmd/cmd_test_examples.rs b/mingling_ci/src/cmd/cmd_test_examples.rs new file mode 100644 index 0000000..2b5526f --- /dev/null +++ b/mingling_ci/src/cmd/cmd_test_examples.rs @@ -0,0 +1,78 @@ +use colored::Colorize; +use indicatif::{ProgressBar, ProgressStyle}; +use mingling::{ + Grouped, Routable, + macros::{buffer, command, renderer}, + res::ResExitCode, +}; + +use crate::Next; +use crate::examples::{check_example, load_test_configs}; +use crate::reporter::{self, ReportResult}; + +#[command(node = "test-examples")] +pub async fn test_examples() -> Next { + reporter::set_task("Test-Examples"); + + let configs = load_test_configs(); + let total = configs.len(); + let pb = ProgressBar::new(total as u64); + pb.set_style( + ProgressStyle::default_bar() + .template(&format!( + "{} [{{bar:28}}] {{pos}}/{{len}}: {{msg}}", + " Testing".bold().bright_cyan() + )) + .unwrap() + .progress_chars("=> "), + ); + pb.set_message("examples"); + + // One blocking task per example: build + run its test cases. + let mut handles = Vec::new(); + for example in configs { + handles.push(tokio::task::spawn_blocking(move || check_example(example))); + } + + let mut fail_count = 0; + for handle in handles { + let Ok(outcome) = handle.await else { + continue; + }; + pb.set_message(outcome.name.clone()); + pb.inc(1); + + if outcome.ok { + reporter::export(&outcome.name, &outcome.location, ReportResult::Ok); + } else { + fail_count += 1; + // Plain stderr: `pb.println` is swallowed on non-TTY (CI). + eprintln!(" {} {}", "failed".bright_red(), outcome.name); + eprintln!(" {}", outcome.output); + reporter::export( + &outcome.name, + &outcome.location, + ReportResult::Error(outcome.output), + ); + } + } + + pb.finish_and_clear(); + reporter::flush(); + + ResultTestExamples { fail_count }.to_chain() +} + +/// Number of examples that failed to build or pass their tests. +#[derive(Grouped)] +pub struct ResultTestExamples { + pub fail_count: usize, +} + +/// Silently sets a non-zero exit code when any example failed. +#[renderer(buffer)] +pub fn render_test_examples(r: ResultTestExamples, exit_code: &mut ResExitCode) { + if r.fail_count > 0 { + exit_code.exit_code = 1; + } +} diff --git a/mingling_ci/src/examples.rs b/mingling_ci/src/examples.rs new file mode 100644 index 0000000..92d3475 --- /dev/null +++ b/mingling_ci/src/examples.rs @@ -0,0 +1,186 @@ +//! Example binary testing: build each example and run its `test.toml` cases. + +use std::process::Output; + +/// A single `[[runs]]` entry of an example's `test.toml`. +pub(crate) struct TestCase { + input: Vec<String>, + expect: Expect, +} + +struct Expect { + exit_code: i32, + result: String, +} + +/// One example and its test cases. +pub(crate) struct ExampleCase { + name: String, + cases: Vec<TestCase>, +} + +/// Outcome of checking one example. +pub(crate) struct ExampleOutcome { + pub name: String, + pub location: String, + pub ok: bool, + pub output: String, +} + +/// Loads `examples/<name>/test.toml` for every example that has one, in +/// alphabetical order of the example directory name. +pub(crate) fn load_test_configs() -> Vec<ExampleCase> { + let mut configs = Vec::new(); + if let Ok(entries) = std::fs::read_dir("examples") { + for entry in entries.flatten() { + let path = entry.path(); + if !path.is_dir() { + continue; + } + let test_toml = path.join("test.toml"); + if !test_toml.is_file() { + continue; + } + let name = path + .file_name() + .and_then(|n| n.to_str()) + .unwrap_or_default() + .to_string(); + let Ok(content) = std::fs::read_to_string(&test_toml) else { + continue; + }; + let Ok(table) = content.parse::<toml::Value>() else { + continue; + }; + let Some(cases) = parse_cases(&table) else { + continue; + }; + configs.push(ExampleCase { name, cases }); + } + } + configs.sort_by(|a, b| a.name.cmp(&b.name)); + configs +} + +fn parse_cases(table: &toml::Value) -> Option<Vec<TestCase>> { + let runs = table.get("runs")?.as_array()?; + let mut cases = Vec::new(); + for run in runs { + let input: Vec<String> = run + .get("input")? + .as_array()? + .iter() + .filter_map(|v| v.as_str().map(str::to_string)) + .collect(); + let expect = run.get("expect")?; + let exit_code = expect + .get("exit-code")? + .as_integer() + .and_then(|e| i32::try_from(e).ok()) + .unwrap_or(-1); + let result = expect + .get("result") + .and_then(|r| r.as_str()) + .unwrap_or_default() + .to_string(); + cases.push(TestCase { + input, + expect: Expect { exit_code, result }, + }); + } + Some(cases) +} + +/// Builds the example, then runs all of its test cases. +pub(crate) fn check_example(example: ExampleCase) -> ExampleOutcome { + let location = format!("./examples/{}", example.name); + + // Phase 1: build. + let manifest = format!("examples/{}/Cargo.toml", example.name); + let build = std::process::Command::new("cargo") + .args(["build", "--manifest-path", &manifest]) + .output(); + match build { + Ok(output) if !output.status.success() => ExampleOutcome { + name: example.name, + location, + ok: false, + output: build_error(&output), + }, + Err(e) => ExampleOutcome { + name: example.name, + location, + ok: false, + output: format!("failed to run cargo: {e}"), + }, + Ok(_) => { + // Phase 2: run the test cases against the built binary. + let mut failures = Vec::new(); + for case in &example.cases { + if let Err(detail) = run_case(&example.name, case) { + failures.push(detail); + } + } + ExampleOutcome { + name: example.name, + location, + ok: failures.is_empty(), + output: failures.join("\n\n"), + } + } + } +} + +/// Runs a single test case against the built binary. +fn run_case(name: &str, case: &TestCase) -> Result<(), String> { + let exe = if cfg!(target_os = "windows") { + ".exe" + } else { + "" + }; + let binary = format!(".temp/target/debug/{name}{exe}"); + + let output = std::process::Command::new(&binary) + .args(&case.input) + .output(); + let Ok(output) = output else { + return Err(format!("failed to run {binary}")); + }; + + let actual_exit_code = output.status.code().unwrap_or(-1); + let actual_stdout = String::from_utf8_lossy(&output.stdout).trim().to_string(); + let actual_stderr = String::from_utf8_lossy(&output.stderr).trim().to_string(); + + let exit_ok = actual_exit_code == case.expect.exit_code; + let result_ok = + actual_stdout == case.expect.result || actual_stdout.contains(&case.expect.result); + + if exit_ok && result_ok { + return Ok(()); + } + + let mut details = vec![format!("input: {}", case.input.join(" "))]; + if !exit_ok { + details.push(format!( + "expected exit code {}, actual {actual_exit_code}", + case.expect.exit_code + )); + } + if !result_ok { + details.push(format!("expected output {:?}", case.expect.result)); + details.push(format!("actual stdout {actual_stdout:?}")); + if !actual_stderr.is_empty() { + details.push(format!("actual stderr {actual_stderr:?}")); + } + } + Err(details.join("\n")) +} + +/// Tail of a failed build's combined output. +fn build_error(output: &Output) -> String { + let mut log = String::from_utf8_lossy(&output.stdout).into_owned(); + log.push_str(&String::from_utf8_lossy(&output.stderr)); + let lines: Vec<&str> = log.lines().collect(); + let tail = &lines[lines.len().saturating_sub(20)..]; + format!("build failed\n{}", tail.join("\n")) +} diff --git a/mingling_ci/src/lib.rs b/mingling_ci/src/lib.rs index 77170c6..3a3ffff 100644 --- a/mingling_ci/src/lib.rs +++ b/mingling_ci/src/lib.rs @@ -14,6 +14,7 @@ pub mod res; /// Log exporter for CI reports pub mod reporter; +pub(crate) mod examples; pub(crate) mod markdown; #[help] |