diff --git a/crates/cargo-oxide/src/main.rs b/crates/cargo-oxide/src/main.rs index d7aaaee590..8f1b2ef349 100644 --- a/crates/cargo-oxide/src/main.rs +++ b/crates/cargo-oxide/src/main.rs @@ -610,6 +610,31 @@ fn validate_materialization_cli(cli: &Cli) -> Result<(), String> { } } +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum FuzzScheduleDisposition { + Success, + Declined, + BrokenBaseline, + Findings(usize), +} + +fn fuzz_schedule_disposition( + baseline: ptx_schedule::campaign::BaselineVerdict, + finding_count: usize, + fail_on_finding: bool, +) -> FuzzScheduleDisposition { + use ptx_schedule::campaign::BaselineVerdict; + + match baseline { + BaselineVerdict::Declined => FuzzScheduleDisposition::Declined, + BaselineVerdict::Broken => FuzzScheduleDisposition::BrokenBaseline, + BaselineVerdict::Usable if fail_on_finding && finding_count > 0 => { + FuzzScheduleDisposition::Findings(finding_count) + } + BaselineVerdict::Usable => FuzzScheduleDisposition::Success, + } +} + fn main() { // Handle both invocation methods: // 1. Cargo subcommand: `cargo oxide run vecadd` → argv = ["cargo-oxide", "oxide", "run", "vecadd"] @@ -838,22 +863,29 @@ fn main() { compare_output, }; match ptx_schedule::campaign::run_campaign(&options) { - Ok(summary) - if !matches!(summary.baseline.kind, ptx_schedule::campaign::RunKind::Pass) => - { - eprintln!( - "Error: baseline example did not pass; no schedule variants were run" - ); - std::process::exit(1); - } - Ok(summary) if fail_on_finding && summary.finding_count() > 0 => { - eprintln!( - "Error: {} schedule-sensitive finding(s) detected", - summary.finding_count() - ); - std::process::exit(1); - } - Ok(_) => {} + Ok(summary) => match fuzz_schedule_disposition( + summary.baseline.kind.baseline_verdict(), + summary.finding_count(), + fail_on_finding, + ) { + // A decline is not a finding. It stays successful even + // with --fail-on-finding and never runs schedule variants. + FuzzScheduleDisposition::Declined => eprintln!( + "Note: the example declined to run on this device; no schedule variants were run" + ), + FuzzScheduleDisposition::BrokenBaseline => { + eprintln!( + "Error: baseline example did not pass ({:?}); no schedule variants were run", + summary.baseline.kind + ); + std::process::exit(1); + } + FuzzScheduleDisposition::Findings(count) => { + eprintln!("Error: {count} schedule-sensitive finding(s) detected"); + std::process::exit(1); + } + FuzzScheduleDisposition::Success => {} + }, Err(error) => { eprintln!("Error: {error}"); std::process::exit(1); @@ -1100,6 +1132,30 @@ mod tests { args.iter().map(|arg| (*arg).to_string()).collect() } + #[test] + fn schedule_cli_exit_policy_keeps_declines_distinct_from_findings() { + use ptx_schedule::campaign::BaselineVerdict; + + for fail_on_finding in [false, true] { + assert_eq!( + fuzz_schedule_disposition(BaselineVerdict::Declined, 0, fail_on_finding), + FuzzScheduleDisposition::Declined + ); + assert_eq!( + fuzz_schedule_disposition(BaselineVerdict::Broken, 0, fail_on_finding), + FuzzScheduleDisposition::BrokenBaseline + ); + } + assert_eq!( + fuzz_schedule_disposition(BaselineVerdict::Usable, 2, false), + FuzzScheduleDisposition::Success + ); + assert_eq!( + fuzz_schedule_disposition(BaselineVerdict::Usable, 2, true), + FuzzScheduleDisposition::Findings(2) + ); + } + #[test] fn clean_parser_accepts_command_without_arguments() { let cli = diff --git a/crates/ptx-schedule/README.md b/crates/ptx-schedule/README.md index 8fb059298f..f5e70c6457 100644 --- a/crates/ptx-schedule/README.md +++ b/crates/ptx-schedule/README.md @@ -79,6 +79,9 @@ being mistaken for a reproducible schedule bug: | `GpuWedged` | the device stopped responding | | `HarnessError` | the campaign itself failed, not the kernel | +A skipped baseline runs no seeds and exits successfully, including with +`--fail-on-finding`. That flag fails only when a campaign finds a real variant. + ## Consumers | Crate | Uses it for | diff --git a/crates/ptx-schedule/src/campaign.rs b/crates/ptx-schedule/src/campaign.rs index 6344d8a22d..bcefe18195 100644 --- a/crates/ptx-schedule/src/campaign.rs +++ b/crates/ptx-schedule/src/campaign.rs @@ -102,6 +102,42 @@ impl RunKind { fn is_finding(&self) -> bool { self.finding_label().is_some() } + + /// What this outcome means when it is the *baseline* run. + /// + /// The campaign and its callers have to agree on this, and the + /// distinction that matters is not pass/not-pass. An example that declares + /// it cannot run on this device has not failed: `scripts/smoketest.sh` + /// reports that decline as `PASS (skipped)`, and #665 exists because + /// reporting it as anything else made an arch-gated example + /// indistinguishable from a broken one. + /// + /// The match is exhaustive on purpose, so a new [`RunKind`] cannot join a + /// class by default. + pub fn baseline_verdict(&self) -> BaselineVerdict { + match self { + Self::Pass => BaselineVerdict::Usable, + Self::Skipped => BaselineVerdict::Declined, + Self::Hang + | Self::Crash + | Self::Mismatch + | Self::OutputChanged + | Self::GpuWedged + | Self::HarnessError => BaselineVerdict::Broken, + } + } +} + +/// Whether a baseline outcome lets a campaign proceed, and if not, why. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum BaselineVerdict { + /// The baseline passed; schedule variants are meaningful. + Usable, + /// The example declared it cannot run here. Not a failure. + Declined, + /// The baseline did not complete, so nothing could be concluded from a + /// variant that behaved the same way. + Broken, } #[derive(Clone, Debug, Deserialize, Serialize)] @@ -243,7 +279,7 @@ pub fn run_campaign(options: &CampaignOptions) -> Result Result println!( + "schedule-fuzz: BASELINE DECLINED: the example opted out on this \ + device; no schedule variants were run" + ), + _ => println!( + "schedule-fuzz: BASELINE FAILURE: {:?}; no schedule variants were run", + summary.baseline.kind + ), + } fs::write( output_dir.join("summary.json"), serde_json::to_vec_pretty(&summary)?, @@ -957,30 +999,6 @@ mod tests { } } - /// The two spellings `scripts/smoketest.sh` accepts, and the near-misses - /// that must not be mistaken for either. - #[test] - fn both_smoketest_skip_spellings_are_recognised() { - for declined in [ - "Skipping: cluster launch requires sm_90", - " SKIPPING: needs two devices", - "PASS (skipped): ldmatrix.m8n8.x4.b16 requires sm_75+; device is sm_70", - " Pass (Skipped): no peer access", - ] { - assert!(has_skip_marker(declined), "{declined}"); - } - - for ran in [ - "pass", - "pass: 1024 elements verified", - "success", - "no skipping: here", - "result was skipped by the host", - ] { - assert!(!has_skip_marker(ran), "{ran}"); - } - } - #[test] fn failures_take_priority_over_skip_markers() { assert!(matches!( @@ -1005,24 +1023,55 @@ mod tests { )); } - /// A declined run must not be reported as a passing baseline: the campaign - /// gates on `RunKind::Pass` and would otherwise sweep every seed against a - /// kernel that never launched. #[test] - fn a_declined_run_is_skipped_not_passed() { - for declined in [ - "skipping: needs sm_90\n", - "pass (skipped): ldmatrix requires sm_75+\n", + fn seed_ranges_are_half_open() { + assert_eq!(parse_seed_range("3..8").unwrap(), (3, 8)); + assert!(parse_seed_range("8..8").is_err()); + assert!(parse_seed_range("8").is_err()); + } + + /// The baseline classes, pinned exhaustively. A decline must not be + /// lumped in with a failure: an arch-gated example on a device below its + /// floor is not a broken example, and #665 is the precedent for keeping + /// those apart. + #[test] + fn only_a_declined_baseline_sits_between_usable_and_broken() { + assert_eq!(RunKind::Pass.baseline_verdict(), BaselineVerdict::Usable); + assert_eq!( + RunKind::Skipped.baseline_verdict(), + BaselineVerdict::Declined + ); + for broken in [ + RunKind::Hang, + RunKind::Crash, + RunKind::Mismatch, + RunKind::OutputChanged, + RunKind::GpuWedged, + RunKind::HarnessError, ] { - assert!(has_skip_marker(declined), "{declined}"); + assert_eq!( + broken.baseline_verdict(), + BaselineVerdict::Broken, + "{broken:?}" + ); } } + /// A finding is about a *variant*; as a baseline the same outcome means the + /// campaign has no ground truth to compare against, so it is broken, not a + /// finding to report. #[test] - fn seed_ranges_are_half_open() { - assert_eq!(parse_seed_range("3..8").unwrap(), (3, 8)); - assert!(parse_seed_range("8..8").is_err()); - assert!(parse_seed_range("8").is_err()); + fn a_finding_as_the_baseline_is_a_broken_baseline() { + for kind in [ + RunKind::Hang, + RunKind::Crash, + RunKind::Mismatch, + RunKind::OutputChanged, + RunKind::GpuWedged, + ] { + assert!(kind.is_finding(), "{kind:?}"); + assert_eq!(kind.baseline_verdict(), BaselineVerdict::Broken, "{kind:?}"); + } } #[test]