From e950257e3dc03c623c7bb2292cecad1e4f551ccd Mon Sep 17 00:00:00 2001 From: Vyncint Ng <115854244+vyncint@users.noreply.github.com> Date: Sat, 22 Aug 2026 10:13:48 +0700 Subject: [PATCH 1/2] fix(ptx-schedule): an example that declines to run is reported as a failure `RunKind::Skipped` became reachable in #1112, when `has_skip_marker` learned the second spelling examples use to decline. Nothing consumes it distinctly. Both places that read the baseline test for `Pass` and treat everything else alike: if !matches!(baseline.kind, RunKind::Pass) { ... println!("schedule-fuzz: BASELINE FAILURE: {:?}; ...") Ok(summary) if !matches!(summary.baseline.kind, RunKind::Pass) => { eprintln!("Error: baseline example did not pass; ..."); std::process::exit(1); } So `cargo oxide fuzz-schedule redux_f32` on anything below sm_100a prints `BASELINE FAILURE: Skipped`, then `Error: baseline example did not pass`, and exits 1. The example did not fail. It ran, declared it cannot run on this device, and exited 0 -- which is precisely what `scripts/smoketest.sh` reports as `PASS (skipped)`. #665 exists because reporting a decline as anything else made an arch-gated example indistinguishable from a broken one, and this is the same conflation in the campaign: sweeping a fleet, "below this example's arch floor" comes back red alongside a genuinely wedged device. Rather than repeat a `matches!` in two crates, the distinction becomes one typed definition: `RunKind::baseline_verdict()` returns `Usable`, `Declined` or `Broken`, and the campaign and the CLI both consult it. The match is exhaustive, so a new `RunKind` cannot join a class by accident -- adding one now fails with E0004 instead of silently reading as broken. Behaviour: `Declined` prints `BASELINE DECLINED` and, in the CLI, a `Note:` on stderr with exit **0**. Every other non-`Pass` baseline keeps `BASELINE FAILURE` and exit 1, now with the kind in the message so the reader knows whether it hung, crashed or wedged. `--fail-on-finding` is untouched and remains the switch for failing on real findings. If you would rather a decline carry a distinct nonzero code, that is a one-line change in the new arm. Two tests: the three classes pinned exhaustively over all eight `RunKind` variants, and that every *finding* kind is a `Broken` baseline -- a mismatch is a finding in a variant, but as the baseline it means there is no ground truth to compare against. The first fails with `Skipped` moved back into the broken class. Verified: 16 ptx-schedule tests and 177 cargo-oxide tests pass, fmt clean, `clippy --all-targets -- -D warnings` clean on both crates, and `cargo doc --document-private-items` clean. Signed-off-by: Vyncint Ng <115854244+vyncint@users.noreply.github.com> --- crates/cargo-oxide/src/main.rs | 18 +++++- crates/ptx-schedule/src/campaign.rs | 96 +++++++++++++++++++++++++++-- 2 files changed, 107 insertions(+), 7 deletions(-) diff --git a/crates/cargo-oxide/src/main.rs b/crates/cargo-oxide/src/main.rs index d7aaaee590..1c7b70efe8 100644 --- a/crates/cargo-oxide/src/main.rs +++ b/crates/cargo-oxide/src/main.rs @@ -838,11 +838,25 @@ fn main() { compare_output, }; match ptx_schedule::campaign::run_campaign(&options) { + // A decline is not a failure -- see `RunKind::baseline_verdict`. + // Exit 0 so sweeping a fleet does not read "below this + // example's arch floor" as a red run; `--fail-on-finding` + // remains the switch for failing on real findings. Ok(summary) - if !matches!(summary.baseline.kind, ptx_schedule::campaign::RunKind::Pass) => + if summary.baseline.kind.baseline_verdict() + == ptx_schedule::campaign::BaselineVerdict::Declined => { eprintln!( - "Error: baseline example did not pass; no schedule variants were run" + "Note: the example declined to run on this device; no schedule variants were run" + ); + } + Ok(summary) + if summary.baseline.kind.baseline_verdict() + != ptx_schedule::campaign::BaselineVerdict::Usable => + { + eprintln!( + "Error: baseline example did not pass ({:?}); no schedule variants were run", + summary.baseline.kind ); std::process::exit(1); } diff --git a/crates/ptx-schedule/src/campaign.rs b/crates/ptx-schedule/src/campaign.rs index 6344d8a22d..c673b1ab06 100644 --- a/crates/ptx-schedule/src/campaign.rs +++ b/crates/ptx-schedule/src/campaign.rs @@ -102,6 +102,42 @@ impl RunKind { fn is_finding(&self) -> bool { self.finding_label().is_some() } + + /// What this outcome means when it is the *baseline* run. + /// + /// The campaign and its callers have to agree on this, and the + /// distinction that matters is not pass/not-pass. An example that declares + /// it cannot run on this device has not failed: `scripts/smoketest.sh` + /// reports that decline as `PASS (skipped)`, and #665 exists because + /// reporting it as anything else made an arch-gated example + /// indistinguishable from a broken one. + /// + /// The match is exhaustive on purpose, so a new [`RunKind`] cannot join a + /// class by default. + pub fn baseline_verdict(&self) -> BaselineVerdict { + match self { + Self::Pass => BaselineVerdict::Usable, + Self::Skipped => BaselineVerdict::Declined, + Self::Hang + | Self::Crash + | Self::Mismatch + | Self::OutputChanged + | Self::GpuWedged + | Self::HarnessError => BaselineVerdict::Broken, + } + } +} + +/// Whether a baseline outcome lets a campaign proceed, and if not, why. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum BaselineVerdict { + /// The baseline passed; schedule variants are meaningful. + Usable, + /// The example declared it cannot run here. Not a failure. + Declined, + /// The baseline did not complete, so nothing could be concluded from a + /// variant that behaved the same way. + Broken, } #[derive(Clone, Debug, Deserialize, Serialize)] @@ -243,7 +279,7 @@ pub fn run_campaign(options: &CampaignOptions) -> Result Result println!( + "schedule-fuzz: BASELINE DECLINED: the example opted out on this \ + device; no schedule variants were run" + ), + _ => println!( + "schedule-fuzz: BASELINE FAILURE: {:?}; no schedule variants were run", + summary.baseline.kind + ), + } fs::write( output_dir.join("summary.json"), serde_json::to_vec_pretty(&summary)?, @@ -1025,6 +1067,50 @@ mod tests { assert!(parse_seed_range("8").is_err()); } + /// The baseline classes, pinned exhaustively. A decline must not be + /// lumped in with a failure: an arch-gated example on a device below its + /// floor is not a broken example, and #665 is the precedent for keeping + /// those apart. + #[test] + fn only_a_declined_baseline_sits_between_usable_and_broken() { + assert_eq!(RunKind::Pass.baseline_verdict(), BaselineVerdict::Usable); + assert_eq!( + RunKind::Skipped.baseline_verdict(), + BaselineVerdict::Declined + ); + for broken in [ + RunKind::Hang, + RunKind::Crash, + RunKind::Mismatch, + RunKind::OutputChanged, + RunKind::GpuWedged, + RunKind::HarnessError, + ] { + assert_eq!( + broken.baseline_verdict(), + BaselineVerdict::Broken, + "{broken:?}" + ); + } + } + + /// A finding is about a *variant*; as a baseline the same outcome means the + /// campaign has no ground truth to compare against, so it is broken, not a + /// finding to report. + #[test] + fn a_finding_as_the_baseline_is_a_broken_baseline() { + for kind in [ + RunKind::Hang, + RunKind::Crash, + RunKind::Mismatch, + RunKind::OutputChanged, + RunKind::GpuWedged, + ] { + assert!(kind.is_finding(), "{kind:?}"); + assert_eq!(kind.baseline_verdict(), BaselineVerdict::Broken, "{kind:?}"); + } + } + #[test] fn findings_use_schedule_sensitive_labels() { assert_eq!( From 327e8d649575e9166a0b3b4456082b03085b4721 Mon Sep 17 00:00:00 2001 From: nihalp Date: Sun, 23 Aug 2026 10:20:15 +0530 Subject: [PATCH 2/2] fix(ptx-schedule): pin baseline and CLI verdict policy Keep stronger process failures ahead of skips and make the CLI distinction between declined, broken, and finding-bearing campaigns explicit and testable. Signed-off-by: nihalp --- crates/cargo-oxide/src/main.rs | 100 ++++++++++++++++++++-------- crates/ptx-schedule/README.md | 3 + crates/ptx-schedule/src/campaign.rs | 37 ---------- 3 files changed, 74 insertions(+), 66 deletions(-) diff --git a/crates/cargo-oxide/src/main.rs b/crates/cargo-oxide/src/main.rs index 1c7b70efe8..8f1b2ef349 100644 --- a/crates/cargo-oxide/src/main.rs +++ b/crates/cargo-oxide/src/main.rs @@ -610,6 +610,31 @@ fn validate_materialization_cli(cli: &Cli) -> Result<(), String> { } } +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum FuzzScheduleDisposition { + Success, + Declined, + BrokenBaseline, + Findings(usize), +} + +fn fuzz_schedule_disposition( + baseline: ptx_schedule::campaign::BaselineVerdict, + finding_count: usize, + fail_on_finding: bool, +) -> FuzzScheduleDisposition { + use ptx_schedule::campaign::BaselineVerdict; + + match baseline { + BaselineVerdict::Declined => FuzzScheduleDisposition::Declined, + BaselineVerdict::Broken => FuzzScheduleDisposition::BrokenBaseline, + BaselineVerdict::Usable if fail_on_finding && finding_count > 0 => { + FuzzScheduleDisposition::Findings(finding_count) + } + BaselineVerdict::Usable => FuzzScheduleDisposition::Success, + } +} + fn main() { // Handle both invocation methods: // 1. Cargo subcommand: `cargo oxide run vecadd` → argv = ["cargo-oxide", "oxide", "run", "vecadd"] @@ -838,36 +863,29 @@ fn main() { compare_output, }; match ptx_schedule::campaign::run_campaign(&options) { - // A decline is not a failure -- see `RunKind::baseline_verdict`. - // Exit 0 so sweeping a fleet does not read "below this - // example's arch floor" as a red run; `--fail-on-finding` - // remains the switch for failing on real findings. - Ok(summary) - if summary.baseline.kind.baseline_verdict() - == ptx_schedule::campaign::BaselineVerdict::Declined => - { - eprintln!( + Ok(summary) => match fuzz_schedule_disposition( + summary.baseline.kind.baseline_verdict(), + summary.finding_count(), + fail_on_finding, + ) { + // A decline is not a finding. It stays successful even + // with --fail-on-finding and never runs schedule variants. + FuzzScheduleDisposition::Declined => eprintln!( "Note: the example declined to run on this device; no schedule variants were run" - ); - } - Ok(summary) - if summary.baseline.kind.baseline_verdict() - != ptx_schedule::campaign::BaselineVerdict::Usable => - { - eprintln!( - "Error: baseline example did not pass ({:?}); no schedule variants were run", - summary.baseline.kind - ); - std::process::exit(1); - } - Ok(summary) if fail_on_finding && summary.finding_count() > 0 => { - eprintln!( - "Error: {} schedule-sensitive finding(s) detected", - summary.finding_count() - ); - std::process::exit(1); - } - Ok(_) => {} + ), + FuzzScheduleDisposition::BrokenBaseline => { + eprintln!( + "Error: baseline example did not pass ({:?}); no schedule variants were run", + summary.baseline.kind + ); + std::process::exit(1); + } + FuzzScheduleDisposition::Findings(count) => { + eprintln!("Error: {count} schedule-sensitive finding(s) detected"); + std::process::exit(1); + } + FuzzScheduleDisposition::Success => {} + }, Err(error) => { eprintln!("Error: {error}"); std::process::exit(1); @@ -1114,6 +1132,30 @@ mod tests { args.iter().map(|arg| (*arg).to_string()).collect() } + #[test] + fn schedule_cli_exit_policy_keeps_declines_distinct_from_findings() { + use ptx_schedule::campaign::BaselineVerdict; + + for fail_on_finding in [false, true] { + assert_eq!( + fuzz_schedule_disposition(BaselineVerdict::Declined, 0, fail_on_finding), + FuzzScheduleDisposition::Declined + ); + assert_eq!( + fuzz_schedule_disposition(BaselineVerdict::Broken, 0, fail_on_finding), + FuzzScheduleDisposition::BrokenBaseline + ); + } + assert_eq!( + fuzz_schedule_disposition(BaselineVerdict::Usable, 2, false), + FuzzScheduleDisposition::Success + ); + assert_eq!( + fuzz_schedule_disposition(BaselineVerdict::Usable, 2, true), + FuzzScheduleDisposition::Findings(2) + ); + } + #[test] fn clean_parser_accepts_command_without_arguments() { let cli = diff --git a/crates/ptx-schedule/README.md b/crates/ptx-schedule/README.md index 8fb059298f..f5e70c6457 100644 --- a/crates/ptx-schedule/README.md +++ b/crates/ptx-schedule/README.md @@ -79,6 +79,9 @@ being mistaken for a reproducible schedule bug: | `GpuWedged` | the device stopped responding | | `HarnessError` | the campaign itself failed, not the kernel | +A skipped baseline runs no seeds and exits successfully, including with +`--fail-on-finding`. That flag fails only when a campaign finds a real variant. + ## Consumers | Crate | Uses it for | diff --git a/crates/ptx-schedule/src/campaign.rs b/crates/ptx-schedule/src/campaign.rs index c673b1ab06..bcefe18195 100644 --- a/crates/ptx-schedule/src/campaign.rs +++ b/crates/ptx-schedule/src/campaign.rs @@ -999,30 +999,6 @@ mod tests { } } - /// The two spellings `scripts/smoketest.sh` accepts, and the near-misses - /// that must not be mistaken for either. - #[test] - fn both_smoketest_skip_spellings_are_recognised() { - for declined in [ - "Skipping: cluster launch requires sm_90", - " SKIPPING: needs two devices", - "PASS (skipped): ldmatrix.m8n8.x4.b16 requires sm_75+; device is sm_70", - " Pass (Skipped): no peer access", - ] { - assert!(has_skip_marker(declined), "{declined}"); - } - - for ran in [ - "pass", - "pass: 1024 elements verified", - "success", - "no skipping: here", - "result was skipped by the host", - ] { - assert!(!has_skip_marker(ran), "{ran}"); - } - } - #[test] fn failures_take_priority_over_skip_markers() { assert!(matches!( @@ -1047,19 +1023,6 @@ mod tests { )); } - /// A declined run must not be reported as a passing baseline: the campaign - /// gates on `RunKind::Pass` and would otherwise sweep every seed against a - /// kernel that never launched. - #[test] - fn a_declined_run_is_skipped_not_passed() { - for declined in [ - "skipping: needs sm_90\n", - "pass (skipped): ldmatrix requires sm_75+\n", - ] { - assert!(has_skip_marker(declined), "{declined}"); - } - } - #[test] fn seed_ranges_are_half_open() { assert_eq!(parse_seed_range("3..8").unwrap(), (3, 8));