From e5fa02ae5c743b3df5e97d0009c6d5c5bf638ed3 Mon Sep 17 00:00:00 2001 From: Alex Wilkerson John Date: Thu, 2 Jul 2026 13:41:14 -0400 Subject: [PATCH 01/13] fix(runner): never report interrupted runs as passed A run cut short by SIGINT/SIGTERM or -timeout previously computed Passed from only the suites that happened to execute, so a truncated run could exit 0, append a passing entry to trend history, and be signed by the release-gate attestation with its security suites never having run. - Report records Interrupted and SkippedSuites; Passed is forced false when the run did not complete. - runBoundedByIndex returns the executed prefix so engines truncate pre-sized case slices instead of reporting phantom zero-value failures. - cleanr run exits 130 for interrupted runs, skips trend-history persistence, and refuses to build an attestation. - Text report shows INTERRUPTED status and the skipped suite names. Co-Authored-By: Claude Fable 5 --- cleanr/core/types/report.go | 2 ++ cleanr/engines/helpers.go | 12 +++++-- cleanr/engines/prompt_injection.go | 3 +- cleanr/engines/provenance.go | 3 +- cleanr/engines/release_policy.go | 3 +- cleanr/engines/security.go | 3 +- cleanr/engines/token_optimization.go | 4 ++- cleanr/report/text.go | 6 ++++ cleanr/runner.go | 12 +++++-- docs/ci.md | 3 +- internal/cli/cli_run.go | 53 +++++++++++++++++++++------- 11 files changed, 79 insertions(+), 25 deletions(-) diff --git a/cleanr/core/types/report.go b/cleanr/core/types/report.go index 004fd44..422523e 100644 --- a/cleanr/core/types/report.go +++ b/cleanr/core/types/report.go @@ -29,6 +29,8 @@ type SuiteResult struct { type Report struct { Name string `json:"name"` Passed bool `json:"passed"` + Interrupted bool `json:"interrupted,omitempty"` + SkippedSuites []string `json:"skipped_suites,omitempty"` GeneratedAt time.Time `json:"generated_at"` Duration time.Duration `json:"duration"` TotalSuites int `json:"total_suites"` diff --git a/cleanr/engines/helpers.go b/cleanr/engines/helpers.go index 733b1ea..514ae67 100644 --- a/cleanr/engines/helpers.go +++ b/cleanr/engines/helpers.go @@ -20,27 +20,32 @@ func scenarioRequest(scenario core.Scenario, timeout time.Duration) core.Request // pool of at most limit goroutines. fn must write only to its own index i; // callers pre-size result slices so output stays deterministically ordered // regardless of completion order. Iteration stops early if ctx is cancelled. -func runBoundedByIndex(ctx context.Context, count, limit int, fn func(i int)) { +// It returns the number of indices actually invoked — always a prefix of +// [0,count) — so callers can truncate pre-sized result slices instead of +// reporting never-run entries as zero-value failures. +func runBoundedByIndex(ctx context.Context, count, limit int, fn func(i int)) int { if limit < 1 { limit = 1 } if limit == 1 || count <= 1 { for i := 0; i < count; i++ { if ctx.Err() != nil { - return + return i } fn(i) } - return + return count } sem := make(chan struct{}, limit) var wg sync.WaitGroup + launched := 0 for i := 0; i < count; i++ { if ctx.Err() != nil { break } sem <- struct{}{} wg.Add(1) + launched++ go func(i int) { defer wg.Done() defer func() { <-sem }() @@ -48,6 +53,7 @@ func runBoundedByIndex(ctx context.Context, count, limit int, fn func(i int)) { }(i) } wg.Wait() + return launched } // responseCache memoizes target responses for identical plain scenario requests diff --git a/cleanr/engines/prompt_injection.go b/cleanr/engines/prompt_injection.go index d5e0cda..a096187 100644 --- a/cleanr/engines/prompt_injection.go +++ b/cleanr/engines/prompt_injection.go @@ -18,9 +18,10 @@ func (PromptInjectionEngine) Run(ctx context.Context, runCtx *core.RunContext) c cases := make([]core.CaseResult, len(scenarios)) // Each scenario is invoked with a distinct injection payload, so cases are // independent and safe to run in a bounded worker pool. - runBoundedByIndex(ctx, len(scenarios), runCtx.Config.CaseConcurrency(), func(i int) { + ran := runBoundedByIndex(ctx, len(scenarios), runCtx.Config.CaseConcurrency(), func(i int) { cases[i] = promptInjectionCase(ctx, runCtx, scenarios[i], cfg) }) + cases = cases[:ran] return core.SuiteResult{Name: "prompt-injection", Passed: allPassed(cases), Cases: cases} } diff --git a/cleanr/engines/provenance.go b/cleanr/engines/provenance.go index f91e895..3e09e52 100644 --- a/cleanr/engines/provenance.go +++ b/cleanr/engines/provenance.go @@ -20,9 +20,10 @@ func (ProvenanceEngine) Run(ctx context.Context, runCtx *core.RunContext) core.S cases := make([]core.CaseResult, len(scenarios)) // Provenance mutates the request per scenario (canary injection), so it does // not share the read-only response cache, but each scenario is independent. - runBoundedByIndex(ctx, len(scenarios), runCtx.Config.CaseConcurrency(), func(i int) { + ran := runBoundedByIndex(ctx, len(scenarios), runCtx.Config.CaseConcurrency(), func(i int) { cases[i] = runProvenanceScenario(ctx, runCtx, scenarios[i], cfg) }) + cases = cases[:ran] return core.SuiteResult{Name: "provenance", Passed: allPassed(cases), Cases: cases} } diff --git a/cleanr/engines/release_policy.go b/cleanr/engines/release_policy.go index 917def2..5fd4f05 100644 --- a/cleanr/engines/release_policy.go +++ b/cleanr/engines/release_policy.go @@ -19,9 +19,10 @@ func (e ReleasePolicyEngine) Run(ctx context.Context, runCtx *core.RunContext) c cfg := releasePolicyConfigWithDefaults(runCtx.Config.Suites.ReleasePolicy) scenarios := runCtx.Config.Scenarios cases := make([]core.CaseResult, len(scenarios)) - runBoundedByIndex(ctx, len(scenarios), runCtx.Config.CaseConcurrency(), func(i int) { + ran := runBoundedByIndex(ctx, len(scenarios), runCtx.Config.CaseConcurrency(), func(i int) { cases[i] = e.evaluateScenario(ctx, runCtx, scenarios[i], cfg) }) + cases = cases[:ran] return core.SuiteResult{Name: "release-policy", Passed: allPassed(cases), Cases: cases} } diff --git a/cleanr/engines/security.go b/cleanr/engines/security.go index f2dbb7a..54b2413 100644 --- a/cleanr/engines/security.go +++ b/cleanr/engines/security.go @@ -34,9 +34,10 @@ func (e SecurityEngine) Run(ctx context.Context, runCtx *core.RunContext) core.S scenarios := runCtx.Config.Scenarios cases := make([]core.CaseResult, len(scenarios)) - runBoundedByIndex(ctx, len(scenarios), runCtx.Config.CaseConcurrency(), func(i int) { + ran := runBoundedByIndex(ctx, len(scenarios), runCtx.Config.CaseConcurrency(), func(i int) { cases[i] = e.evaluateScenario(ctx, runCtx, scenarios[i], cfg, piiPatterns, extraPatterns) }) + cases = cases[:ran] return core.SuiteResult{Name: "security", Passed: allPassed(cases), Cases: cases} } diff --git a/cleanr/engines/token_optimization.go b/cleanr/engines/token_optimization.go index 603e2e0..50873e9 100644 --- a/cleanr/engines/token_optimization.go +++ b/cleanr/engines/token_optimization.go @@ -26,9 +26,11 @@ func (e TokenOptimizationEngine) Run(ctx context.Context, runCtx *core.RunContex scenarios := runCtx.Config.Scenarios cases := make([]core.CaseResult, len(scenarios)) metrics := make([]tokenOptimizationMetrics, len(scenarios)) - runBoundedByIndex(ctx, len(scenarios), runCtx.Config.CaseConcurrency(), func(i int) { + ran := runBoundedByIndex(ctx, len(scenarios), runCtx.Config.CaseConcurrency(), func(i int) { cases[i], metrics[i] = e.evaluateScenario(ctx, runCtx, scenarios[i], cfg) }) + cases = cases[:ran] + metrics = metrics[:ran] totalInput := 0 totalOutput := 0 diff --git a/cleanr/report/text.go b/cleanr/report/text.go index 1006860..2dfa66a 100644 --- a/cleanr/report/text.go +++ b/cleanr/report/text.go @@ -26,12 +26,18 @@ func writeReportSummary(b *strings.Builder, palette textPalette, report core.Rep if !report.Passed { status = "FAIL" } + if report.Interrupted { + status = "INTERRUPTED" + } if banner := renderBanner(palette); banner != "" { fmt.Fprintf(b, "%s\n\n", banner) } fmt.Fprintf(b, "%s\n", palette.accent("Report Summary")) fmt.Fprintf(b, "%s\n", palette.accent(strings.Repeat("=", 48))) writeKeyValue(b, palette, "Status", palette.status(report.Passed, status)) + if len(report.SkippedSuites) > 0 { + writeKeyValue(b, palette, "Skipped", strings.Join(report.SkippedSuites, ", ")) + } writeKeyValue(b, palette, "Target", report.Name) if !report.GeneratedAt.IsZero() { writeKeyValue(b, palette, "Generated", report.GeneratedAt.Format(time.RFC3339)) diff --git a/cleanr/runner.go b/cleanr/runner.go index 0a6ab47..f1fb0d8 100644 --- a/cleanr/runner.go +++ b/cleanr/runner.go @@ -47,9 +47,11 @@ func (r *Runner) Run(ctx context.Context) Report { for _, engine := range r.engines { // Stop launching further engines once the run's deadline/cancellation // fires so we don't spend time producing spurious "context deadline - // exceeded" findings after the budget is gone. + // exceeded" findings after the budget is gone. Skipped suites are + // recorded so a truncated run can never be mistaken for a full one. if ctx.Err() != nil { - break + report.SkippedSuites = append(report.SkippedSuites, engine.Name()) + continue } suiteStart := time.Now() result := engine.Run(ctx, runCtx) @@ -67,7 +69,11 @@ func (r *Runner) Run(ctx context.Context) Report { } } - report.Passed = report.FailedSuites == 0 && report.FailedCases == 0 + // An interrupted run must never report success: suites that did not execute + // cannot vouch for the target, so downstream gates and attestations treat + // the report as failed regardless of the suites that happened to finish. + report.Interrupted = ctx.Err() != nil + report.Passed = !report.Interrupted && report.FailedSuites == 0 && report.FailedCases == 0 report.Duration = time.Since(start) report.Recommendations = buildRecommendations(report) return report diff --git a/docs/ci.md b/docs/ci.md index 99373dd..b80b0ed 100644 --- a/docs/ci.md +++ b/docs/ci.md @@ -7,8 +7,9 @@ - `0`: all suites passed - `1`: one or more suites or cases failed - `2`: invalid configuration or runtime error +- `130`: the run was interrupted (SIGINT/SIGTERM or `-timeout`) before every suite executed; the partial report is written with `"interrupted": true` and the skipped suite names, and the run is neither recorded in trend history nor attested -That split lets pipelines distinguish product regressions from setup or infrastructure failures. +That split lets pipelines distinguish product regressions from setup or infrastructure failures — and truncated runs from either. ## Pick an Integration Shape diff --git a/internal/cli/cli_run.go b/internal/cli/cli_run.go index d8d8f05..40277e6 100644 --- a/internal/cli/cli_run.go +++ b/internal/cli/cli_run.go @@ -17,6 +17,12 @@ import ( "github.com/devr-tools/cleanr/cleanr" ) +// exitInterrupted is returned when the run was cut short by SIGINT/SIGTERM or +// the -timeout budget: the partial report is still written, but suites that +// never executed mean the result must not be treated as pass or fail. 130 +// matches the shell convention for termination by SIGINT (128+2). +const exitInterrupted = 130 + type runOptions struct { configPath string profile string @@ -66,9 +72,9 @@ func executeRunCommand(opts runOptions, stdout, stderr io.Writer) int { defer cancel() slog.Debug("run starting", "format", cfg.Reporting.Format, "output", cfg.Reporting.Output) report := cleanr.NewHTTPRunner(cfg).Run(ctx) - if ctx.Err() != nil { - _, _ = fmt.Fprintf(stderr, "run interrupted: %v; writing partial report\n", ctx.Err()) - slog.Warn("run interrupted", "error", ctx.Err()) + if report.Interrupted { + _, _ = fmt.Fprintf(stderr, "run interrupted: %v; writing partial report marked as interrupted\n", ctx.Err()) + slog.Warn("run interrupted", "error", ctx.Err(), "skipped_suites", report.SkippedSuites) } // persistWarnings accumulates non-fatal bookkeeping failures so the run can @@ -77,13 +83,19 @@ func executeRunCommand(opts runOptions, stdout, stderr io.Writer) int { // Trend history enriches the report and its gates can flip pass/fail, so it // must run before the report is written. A persistence failure here is - // downgraded to a warning instead of discarding the whole run. - if err := cleanr.AttachTrendHistory(&report, cfg.Reporting.TrendFile, cfg.Reporting.BuildID, cfg.Reporting.TrendLimit); err != nil { - _, _ = fmt.Fprintf(stderr, "trend history warning: %v\n", err) - slog.Warn("trend history persistence failed", "error", err) - persistWarnings = append(persistWarnings, "trend history") + // downgraded to a warning instead of discarding the whole run. An + // interrupted run is never appended to history: a partial run would skew + // every later build-over-build comparison against it. + if report.Interrupted { + _, _ = fmt.Fprintln(stderr, "trend history skipped: interrupted runs are not recorded") + } else { + if err := cleanr.AttachTrendHistory(&report, cfg.Reporting.TrendFile, cfg.Reporting.BuildID, cfg.Reporting.TrendLimit); err != nil { + _, _ = fmt.Fprintf(stderr, "trend history warning: %v\n", err) + slog.Warn("trend history persistence failed", "error", err) + persistWarnings = append(persistWarnings, "trend history") + } + cleanr.EvaluateTrendGates(&report, cfg.Reporting.TrendGates) } - cleanr.EvaluateTrendGates(&report, cfg.Reporting.TrendGates) // Build the replay artifact and attestation in memory and publish // integrations before writing the report: these enrich the report content @@ -91,10 +103,20 @@ func executeRunCommand(opts runOptions, stdout, stderr io.Writer) int { // contain. Only the on-disk persistence of these artifacts is deferred until // after the report is written. replayArtifact, hasReplayArtifact := buildReplayArtifact(cfg, report) - attestation, err := buildAttestation(cfg, report, replayArtifact, hasReplayArtifact, stderr) - if err != nil { - slog.Warn("attestation build failed", "error", err) - persistWarnings = append(persistWarnings, "attestation") + // A release-gate attestation vouches for a completed run; never sign one + // whose suites were cut short. + var attestation *cleanr.ReleaseGateAttestation + if report.Interrupted { + if cfg.Governance.Attestation.Enabled { + _, _ = fmt.Fprintln(stderr, "attestation skipped: interrupted runs are not attested") + } + } else { + built, err := buildAttestation(cfg, report, replayArtifact, hasReplayArtifact, stderr) + if err != nil { + slog.Warn("attestation build failed", "error", err) + persistWarnings = append(persistWarnings, "attestation") + } + attestation = built } publishIntegrations(ctx, cfg, &report, replayArtifact, hasReplayArtifact, attestation, resolvedConfigPath, stderr) @@ -121,6 +143,11 @@ func executeRunCommand(opts runOptions, stdout, stderr io.Writer) int { // Primary pass/fail from the run result takes precedence; only when the run // itself passed do bookkeeping failures surface as the distinct exit code 2. + // Interrupted runs get their own exit code so CI can distinguish "the + // target failed" from "the run never finished". + if report.Interrupted { + return exitInterrupted + } if !report.Passed { return 1 } From db1f643e39675991b0646dd21ee1b3fea170d6ce Mon Sep 17 00:00:00 2001 From: Alex Wilkerson John Date: Thu, 2 Jul 2026 13:41:31 -0400 Subject: [PATCH 02/13] fix(plugins): interrupt hung WASM guests on entry timeout MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit wazero ignores context cancellation during guest execution unless the runtime is built with WithCloseOnContextDone, so a plugin with an infinite loop blew straight through the 5s entry timeout and hung the whole run — defeating the sandbox. The entry deadline now also applies to guest execution only: module compilation is host-side work whose cost scales with module size, not plugin behavior, and under the new instrumentation it could eat the entire execution budget. Co-Authored-By: Claude Fable 5 --- cleanr/plugins/runtime.go | 16 ++++++++++------ cleanr/plugins/runtime_wazero.go | 17 +++++++++++++---- 2 files changed, 23 insertions(+), 10 deletions(-) diff --git a/cleanr/plugins/runtime.go b/cleanr/plugins/runtime.go index ac4c8f9..58f4c75 100644 --- a/cleanr/plugins/runtime.go +++ b/cleanr/plugins/runtime.go @@ -238,14 +238,12 @@ func runJSONEntry(ctx context.Context, entry Entry, input any, output any) error if timeout <= 0 { timeout = 5 * time.Second } - cmdCtx, cancel := context.WithTimeout(ctx, timeout) - defer cancel() data, err := json.Marshal(input) if err != nil { return err } - stdout, err := executeEntry(cmdCtx, entry, data) + stdout, err := executeEntry(ctx, entry, data, timeout) if err != nil { return err } @@ -255,12 +253,18 @@ func runJSONEntry(ctx context.Context, entry Entry, input any, output any) error return nil } -func executeEntry(ctx context.Context, entry Entry, input []byte) (bytes.Buffer, error) { +// executeEntry runs the entry with timeout bounding plugin execution. For the +// WASM backend the deadline is applied inside runWASMModule around guest +// execution only, so host-side module compilation does not eat into the +// plugin's execution budget. +func executeEntry(ctx context.Context, entry Entry, input []byte, timeout time.Duration) (bytes.Buffer, error) { switch backendFor(entry.Command, entry.Runtime) { case BackendWASM: - return runWASMModule(ctx, entry, input) + return runWASMModule(ctx, entry, input, timeout) default: - return runCommand(ctx, entry, input) + cmdCtx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + return runCommand(cmdCtx, entry, input) } } diff --git a/cleanr/plugins/runtime_wazero.go b/cleanr/plugins/runtime_wazero.go index 7086dfb..5136161 100644 --- a/cleanr/plugins/runtime_wazero.go +++ b/cleanr/plugins/runtime_wazero.go @@ -6,12 +6,13 @@ import ( "fmt" "os" "strings" + "time" "github.com/tetratelabs/wazero" "github.com/tetratelabs/wazero/imports/wasi_snapshot_preview1" ) -func runWASMModule(ctx context.Context, entry Entry, input []byte) (bytes.Buffer, error) { +func runWASMModule(ctx context.Context, entry Entry, input []byte, timeout time.Duration) (bytes.Buffer, error) { modulePath := strings.TrimSpace(entry.Resolved) if modulePath == "" { return bytes.Buffer{}, fmt.Errorf("missing wasm module path") @@ -21,7 +22,10 @@ func runWASMModule(ctx context.Context, entry Entry, input []byte) (bytes.Buffer return bytes.Buffer{}, err } - runtime := wazero.NewRuntime(ctx) + // WithCloseOnContextDone makes guest execution observe ctx cancellation and + // deadlines; without it wazero ignores ctx once running, so a plugin with an + // infinite loop would hang the whole run past its timeout. + runtime := wazero.NewRuntimeWithConfig(ctx, wazero.NewRuntimeConfig().WithCloseOnContextDone(true)) defer runtime.Close(ctx) if _, err := wasi_snapshot_preview1.Instantiate(ctx, runtime); err != nil { @@ -32,6 +36,11 @@ func runWASMModule(ctx context.Context, entry Entry, input []byte) (bytes.Buffer return bytes.Buffer{}, fmt.Errorf("compile wasm module: %w", err) } + // The entry timeout bounds guest execution only: compilation above is + // host-side work whose cost varies with module size, not plugin behavior. + execCtx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + var stdout bytes.Buffer var stderr bytes.Buffer config := wazero.NewModuleConfig(). @@ -50,7 +59,7 @@ func runWASMModule(ctx context.Context, entry Entry, input []byte) (bytes.Buffer config = config.WithEnv(key, value) } - module, err := runtime.InstantiateModule(ctx, compiled, config) + module, err := runtime.InstantiateModule(execCtx, compiled, config) if err != nil { msg := strings.TrimSpace(stderr.String()) if msg == "" { @@ -65,7 +74,7 @@ func runWASMModule(ctx context.Context, entry Entry, input []byte) (bytes.Buffer if fn == nil { return bytes.Buffer{}, fmt.Errorf("wasm entrypoint %q not found", entrypoint) } - if _, err := fn.Call(ctx); err != nil { + if _, err := fn.Call(execCtx); err != nil { msg := strings.TrimSpace(stderr.String()) if msg == "" { msg = err.Error() From 154d5a064b19c6f204ca467d8554e6e2fc5ad311 Mon Sep 17 00:00:00 2001 From: Alex Wilkerson John Date: Thu, 2 Jul 2026 13:41:31 -0400 Subject: [PATCH 03/13] fix(integrations): default HTTP timeouts and close credential-egress bypass The generic sink POST and http/langsmith/openllmetry/provider_logs trend fetches built http.Clients directly from timeout_ms, so an unset value meant no timeout at all and one hung endpoint stalled a release-gate run until SIGINT. All integration HTTP calls now share newIntegrationHTTPClient with the same 10s default httpjson already used. PostHog and Langfuse send credentials outside applyAuth (request body / Basic auth), bypassing the CredentialEgressAllowed policy: a config with project_token_env: OPENAI_API_KEY and an attacker base_url could exfiltrate a provider secret. Both sinks now enforce the same egress policy and fail loudly when it is violated. Co-Authored-By: Claude Fable 5 --- cleanr/integrations/runtime/api.go | 4 +- cleanr/integrations/runtime/httpjson.go | 7 +-- cleanr/integrations/runtime/langfuse.go | 10 ++- cleanr/integrations/runtime/posthog.go | 9 ++- cleanr/integrations/runtime/provider_utils.go | 13 ++++ cleanr/integrations/runtime/trend_import.go | 2 +- tests/integrations/native_sink_egress_test.go | 62 +++++++++++++++++++ 7 files changed, 96 insertions(+), 11 deletions(-) create mode 100644 tests/integrations/native_sink_egress_test.go diff --git a/cleanr/integrations/runtime/api.go b/cleanr/integrations/runtime/api.go index 32ef35a..3de1128 100644 --- a/cleanr/integrations/runtime/api.go +++ b/cleanr/integrations/runtime/api.go @@ -163,7 +163,7 @@ func loadTrendSource(ctx context.Context, source core.TrendSourceConfig, baseDir case "file": return trendspkg.LoadFile(resolveRelativePath(baseDir, source.Path)) case "http": - client := &http.Client{Timeout: time.Duration(source.TimeoutMS) * time.Millisecond} + client := newIntegrationHTTPClient(source.TimeoutMS) req, err := http.NewRequestWithContext(ctx, http.MethodGet, strings.TrimSpace(source.URL), nil) if err != nil { return trendspkg.HistoryFile{}, fmt.Errorf("load trend source %s: %w", displayName(source.Name, source.Type), err) @@ -200,7 +200,7 @@ func postSinkPayload(ctx context.Context, sink core.ResultSinkConfig, payload Si if err != nil { return "", fmt.Errorf("publish result sink %s: %w", displayName(sink.Name, sink.Type), err) } - client := &http.Client{Timeout: time.Duration(sink.TimeoutMS) * time.Millisecond} + client := newIntegrationHTTPClient(sink.TimeoutMS) req, err := http.NewRequestWithContext(ctx, http.MethodPost, strings.TrimSpace(sink.Endpoint), bytes.NewReader(data)) if err != nil { return "", fmt.Errorf("publish result sink %s: %w", displayName(sink.Name, sink.Type), err) diff --git a/cleanr/integrations/runtime/httpjson.go b/cleanr/integrations/runtime/httpjson.go index c737eb4..adc332c 100644 --- a/cleanr/integrations/runtime/httpjson.go +++ b/cleanr/integrations/runtime/httpjson.go @@ -8,7 +8,6 @@ import ( "io" "net/http" "net/url" - "time" ) type headerAuthFunc func(http.Header) @@ -21,14 +20,10 @@ type jsonAPIClient struct { } func newJSONAPIClient(baseURL string, headers map[string]string, timeoutMS int, applyAuth headerAuthFunc) *jsonAPIClient { - timeout := 10 * time.Second - if timeoutMS > 0 { - timeout = time.Duration(timeoutMS) * time.Millisecond - } return &jsonAPIClient{ baseURL: baseURL, headers: headers, - httpClient: &http.Client{Timeout: timeout}, + httpClient: newIntegrationHTTPClient(timeoutMS), applyAuth: applyAuth, } } diff --git a/cleanr/integrations/runtime/langfuse.go b/cleanr/integrations/runtime/langfuse.go index a694ff7..0cd677e 100644 --- a/cleanr/integrations/runtime/langfuse.go +++ b/cleanr/integrations/runtime/langfuse.go @@ -38,9 +38,17 @@ func newLangfuseClient(sink core.ResultSinkConfig) (*langfuseClient, error) { if publicKey == "" || secretKey == "" { return nil, fmt.Errorf("missing Langfuse credentials in %s or %s", emptyValue(sink.PublicKeyEnv), emptyValue(sink.SecretKeyEnv)) } + baseURL := normalizedBaseURL(sink.BaseURL, sink.Endpoint, defaultLangfuseBaseURL) + // Basic auth here is built outside applyAuth; enforce the same egress + // policy so provider secrets cannot be routed to an arbitrary base_url. + for _, keyEnv := range []string{sink.PublicKeyEnv, sink.SecretKeyEnv} { + if !CredentialEgressAllowed(keyEnv, baseURL) { + return nil, fmt.Errorf("refusing to send credential %q to untrusted host %q", strings.TrimSpace(keyEnv), destinationHost(baseURL)) + } + } return &langfuseClient{ http: newJSONAPIClient( - normalizedBaseURL(sink.BaseURL, sink.Endpoint, defaultLangfuseBaseURL), + baseURL, sink.Headers, sink.TimeoutMS, func(h http.Header) { diff --git a/cleanr/integrations/runtime/posthog.go b/cleanr/integrations/runtime/posthog.go index 13cfde2..3a382e6 100644 --- a/cleanr/integrations/runtime/posthog.go +++ b/cleanr/integrations/runtime/posthog.go @@ -25,10 +25,17 @@ func newPostHogClient(sink core.ResultSinkConfig) (*postHogClient, error) { if token == "" { return nil, fmt.Errorf("missing PostHog project token in %s", emptyValue(sink.ProjectTokenEnv)) } + baseURL := normalizedBaseURL(sink.BaseURL, sink.Endpoint, defaultPostHogBaseURL) + // The token travels in the request body, so it bypasses applyAuth; enforce + // the same egress policy here so a config cannot point a provider secret + // (e.g. project_token_env: OPENAI_API_KEY) at an arbitrary host. + if !CredentialEgressAllowed(sink.ProjectTokenEnv, baseURL) { + return nil, fmt.Errorf("refusing to send credential %q to untrusted host %q", strings.TrimSpace(sink.ProjectTokenEnv), destinationHost(baseURL)) + } return &postHogClient{ token: token, http: newJSONAPIClient( - normalizedBaseURL(sink.BaseURL, sink.Endpoint, defaultPostHogBaseURL), + baseURL, sink.Headers, sink.TimeoutMS, nil, diff --git a/cleanr/integrations/runtime/provider_utils.go b/cleanr/integrations/runtime/provider_utils.go index be93f63..edb7aca 100644 --- a/cleanr/integrations/runtime/provider_utils.go +++ b/cleanr/integrations/runtime/provider_utils.go @@ -1,6 +1,7 @@ package runtime import ( + "net/http" "net/url" "os" "strings" @@ -9,6 +10,18 @@ import ( const defaultIntegrationFamily = "cleanr-release-gate" +// newIntegrationHTTPClient builds the client for integration HTTP calls with a +// 10s default when timeout_ms is unset: a zero http.Client timeout means no +// timeout at all, and a single hung sink or trend endpoint would otherwise +// stall the whole run indefinitely. +func newIntegrationHTTPClient(timeoutMS int) *http.Client { + timeout := 10 * time.Second + if timeoutMS > 0 { + timeout = time.Duration(timeoutMS) * time.Millisecond + } + return &http.Client{Timeout: timeout} +} + func integrationFamily(name string) string { name = strings.TrimSpace(name) if name == "" { diff --git a/cleanr/integrations/runtime/trend_import.go b/cleanr/integrations/runtime/trend_import.go index 7e18d5e..1ccb173 100644 --- a/cleanr/integrations/runtime/trend_import.go +++ b/cleanr/integrations/runtime/trend_import.go @@ -33,7 +33,7 @@ func readTrendSourceBytes(ctx context.Context, source core.TrendSourceConfig, ba } return data, resolved, nil } - client := &http.Client{Timeout: time.Duration(source.TimeoutMS) * time.Millisecond} + client := newIntegrationHTTPClient(source.TimeoutMS) req, err := http.NewRequestWithContext(ctx, http.MethodGet, strings.TrimSpace(source.URL), nil) if err != nil { return nil, "", fmt.Errorf("load trend source %s: %w", displayName(source.Name, source.Type), err) diff --git a/tests/integrations/native_sink_egress_test.go b/tests/integrations/native_sink_egress_test.go new file mode 100644 index 0000000..0fc69cf --- /dev/null +++ b/tests/integrations/native_sink_egress_test.go @@ -0,0 +1,62 @@ +package tests + +import ( + "context" + "strings" + "testing" + + "github.com/devr-tools/cleanr/cleanr" +) + +// The PostHog and Langfuse sinks send credentials outside applyAuth (request +// body / Basic auth), so they must enforce the same egress policy: a +// provider secret may not be routed to an arbitrary config-controlled host. +func TestNativeSinksRefuseProviderSecretToUntrustedHost(t *testing.T) { + t.Setenv("OPENAI_API_KEY", "sk-test-not-real") + + cases := []struct { + name string + sink cleanr.ResultSinkConfig + }{ + { + name: "posthog project token", + sink: cleanr.ResultSinkConfig{ + Name: "posthog", + Type: "posthog", + BaseURL: "https://evil.example.com", + ProjectTokenEnv: "OPENAI_API_KEY", + }, + }, + { + name: "langfuse secret key", + sink: cleanr.ResultSinkConfig{ + Name: "langfuse", + Type: "langfuse", + BaseURL: "https://evil.example.com", + PublicKeyEnv: "OPENAI_API_KEY", + SecretKeyEnv: "OPENAI_API_KEY", + }, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + results := cleanr.PublishResultSinks( + context.Background(), + cleanr.IntegrationsConfig{ResultSinks: []cleanr.ResultSinkConfig{tc.sink}}, + cleanr.Report{Name: "demo"}, + nil, + nil, + ) + if len(results) != 1 { + t.Fatalf("expected one sink result, got %d", len(results)) + } + if results[0].Published { + t.Fatalf("expected publish to be refused: %+v", results[0]) + } + if !strings.Contains(results[0].Message, "refusing to send credential") { + t.Fatalf("expected egress refusal message, got %q", results[0].Message) + } + }) + } +} From c0878af3066040a1bfc10f9bf4dfed435c00c24f Mon Sep 17 00:00:00 2001 From: Alex Wilkerson John Date: Thu, 2 Jul 2026 13:41:46 -0400 Subject: [PATCH 04/13] fix(mcpserver): stop generate-dataset crash and contain tool panics MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit cleanr_generate_dataset passed a typed-nil *http.Client into the real generation path, which nil-derefs on the first client.Do — and with no recover() anywhere in the stdio server, the panic killed the whole process and every session with it. The tool now gets a real client matching the CLI generate command's provider timeout, and tool dispatch is wrapped in recover() so a panicking handler returns a JSON-RPC error while the server keeps serving. Co-Authored-By: Claude Fable 5 --- internal/mcpserver/runtime/lifecycle.go | 6 ++- internal/mcpserver/server.go | 13 +++++- tests/mcp/mcp_server_test.go | 61 +++++++++++++++++++++++++ 3 files changed, 78 insertions(+), 2 deletions(-) diff --git a/internal/mcpserver/runtime/lifecycle.go b/internal/mcpserver/runtime/lifecycle.go index 8ab835b..ae4e8a2 100644 --- a/internal/mcpserver/runtime/lifecycle.go +++ b/internal/mcpserver/runtime/lifecycle.go @@ -241,7 +241,11 @@ func GenerateDataset(ctx context.Context, args map[string]any) (toolkit.Result, return toolkit.Result{}, err } - dataset, err := GenerateScenarioDatasetFunc(ctx, cfg, (*http.Client)(nil)) + // A real client is required: the generation path calls client.Do directly, + // so a nil *http.Client panics on first use. Mirror the CLI generate + // command's provider-timeout client. + client := &http.Client{Timeout: cfg.ScenarioGeneration.Provider.Timeout()} + dataset, err := GenerateScenarioDatasetFunc(ctx, cfg, client) if err != nil { return toolkit.Result{}, err } diff --git a/internal/mcpserver/server.go b/internal/mcpserver/server.go index dffdbdc..4ee570c 100644 --- a/internal/mcpserver/server.go +++ b/internal/mcpserver/server.go @@ -132,13 +132,24 @@ func (s *Server) handleToolCall(ctx context.Context, req requestEnvelope) *respo params.Arguments = map[string]any{} } - result, err := mcptools.Call(ctx, params.Name, params.Arguments) + result, err := safeToolCall(ctx, params.Name, params.Arguments) if err != nil { return errorResponse(req.ID, jsonRPCInternalError, err.Error(), nil) } return successResponse(req.ID, result) } +// safeToolCall contains a panicking tool handler: the server speaks stdio, so +// an uncaught panic would kill the whole process and every session with it. +func safeToolCall(ctx context.Context, name string, args map[string]any) (result any, err error) { + defer func() { + if r := recover(); r != nil { + err = fmt.Errorf("tool %s panicked: %v", name, r) + } + }() + return mcptools.Call(ctx, name, args) +} + func (s *Server) requireInitialized(id any) *responseEnvelope { if s.initialized { return nil diff --git a/tests/mcp/mcp_server_test.go b/tests/mcp/mcp_server_test.go index ec2e2cc..9a85908 100644 --- a/tests/mcp/mcp_server_test.go +++ b/tests/mcp/mcp_server_test.go @@ -389,6 +389,67 @@ func TestMCPServerLifecycleToolsReturnStructuredArtifacts(t *testing.T) { } } +func TestMCPServerContainsPanickingToolHandler(t *testing.T) { + server := initializedMCPServer(t) + + cfg := cleanr.ExampleConfig() + cfg.Scenarios = nil + cfg.ScenarioGeneration = cleanr.ScenarioGenerationConfig{ + Enabled: true, + Provider: cleanr.TargetConfig{ + Type: "http", + URL: "https://generator.example.test/v1", + Method: http.MethodPost, + PromptField: "input", + ResponseField: "output.text", + }, + Spec: cleanr.ScenarioGenerationSpec{ + AppKind: "support-assistant", + Goals: []string{"refund policy"}, + RiskAreas: []string{"prompt injection"}, + }, + Count: 1, + } + originalGenerate := runtime.GenerateScenarioDatasetFunc + runtime.GenerateScenarioDatasetFunc = func(context.Context, cleanr.Config, *http.Client) (cleanr.ScenarioDataset, error) { + panic("boom") + } + defer func() { runtime.GenerateScenarioDatasetFunc = originalGenerate }() + configJSON, err := json.Marshal(cfg) + if err != nil { + t.Fatalf("marshal config: %v", err) + } + + resp := mustHandleMCP(t, server, map[string]any{ + "jsonrpc": "2.0", + "id": 8, + "method": "tools/call", + "params": map[string]any{ + "name": "cleanr_generate_dataset", + "arguments": map[string]any{ + "config": string(configJSON), + }, + }, + }) + respErr, ok := resp["error"].(map[string]any) + if !ok { + t.Fatalf("expected JSON-RPC error for panicking tool, got %#v", resp) + } + if msg, _ := respErr["message"].(string); !strings.Contains(msg, "panicked") { + t.Fatalf("expected panic message in error, got %q", msg) + } + + // The server must keep serving after containing the panic. + pingResp := mustHandleMCP(t, server, map[string]any{ + "jsonrpc": "2.0", + "id": 9, + "method": "ping", + }) + if _, ok := pingResp["result"]; !ok { + t.Fatalf("expected ping to succeed after tool panic, got %#v", pingResp) + } +} + func initializedMCPServer(t *testing.T) *mcpserver.Server { t.Helper() server := mcpserver.New() From f44c697cf71618e3dcb4e26638219e3acfb7ff2c Mon Sep 17 00:00:00 2001 From: Alex Wilkerson John Date: Thu, 2 Jul 2026 13:41:46 -0400 Subject: [PATCH 05/13] fix(persistence): atomic writes for config, profile, attestation, and replay artifacts Extract the two copy-pasted writeFileAtomic helpers (snapshots, trends) into cleanr/fsatomic, add fsync before the rename plus a best-effort directory sync, and adopt it in the writers that were plain os.WriteFile: an interrupted write could previously destroy the user's hand-edited cleanr.yaml, corrupt profile.json and lose every stored provider credential, or leave a torn attestation whose signature can never verify. Co-Authored-By: Claude Fable 5 --- cleanr/attest/attestation.go | 4 ++- cleanr/config/io.go | 5 ++- cleanr/fsatomic/fsatomic.go | 57 ++++++++++++++++++++++++++++++++ cleanr/profile/store.go | 6 +++- cleanr/snapshots/io.go | 29 ++-------------- cleanr/trends/io.go | 29 ++-------------- cleanr/trends/replay_artifact.go | 5 ++- 7 files changed, 79 insertions(+), 56 deletions(-) create mode 100644 cleanr/fsatomic/fsatomic.go diff --git a/cleanr/attest/attestation.go b/cleanr/attest/attestation.go index 19f8b5b..f5fe55f 100644 --- a/cleanr/attest/attestation.go +++ b/cleanr/attest/attestation.go @@ -12,6 +12,7 @@ import ( "strings" "github.com/devr-tools/cleanr/cleanr/core" + "github.com/devr-tools/cleanr/cleanr/fsatomic" "gopkg.in/yaml.v3" ) @@ -75,7 +76,8 @@ func WriteReleaseGateAttestationFile(path string, attestation core.ReleaseGateAt if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { return err } - return os.WriteFile(path, append(data, '\n'), 0o644) + // Atomic: a torn attestation would carry a signature that can never verify. + return fsatomic.WriteFile(path, append(data, '\n'), 0o644) } func parsePrivateKey(raw string) (ed25519.PrivateKey, error) { diff --git a/cleanr/config/io.go b/cleanr/config/io.go index 7b4d825..73a3d3f 100644 --- a/cleanr/config/io.go +++ b/cleanr/config/io.go @@ -8,6 +8,7 @@ import ( "strings" "github.com/devr-tools/cleanr/cleanr/core" + "github.com/devr-tools/cleanr/cleanr/fsatomic" "gopkg.in/yaml.v3" ) @@ -41,7 +42,9 @@ func WriteConfigFile(path string, cfg core.Config) error { if err != nil { return err } - return os.WriteFile(path, append(data, '\n'), 0o644) + // Atomic so an interrupted write (setup, dataset merge) can never destroy + // the user's existing hand-edited config. + return fsatomic.WriteFile(path, append(data, '\n'), 0o644) } func MarshalConfig(cfg core.Config, format string) ([]byte, error) { diff --git a/cleanr/fsatomic/fsatomic.go b/cleanr/fsatomic/fsatomic.go new file mode 100644 index 0000000..f3323b0 --- /dev/null +++ b/cleanr/fsatomic/fsatomic.go @@ -0,0 +1,57 @@ +// Package fsatomic provides crash-safe file writes for cleanr's persisted +// artifacts (configs, profiles, snapshots, trend history, attestations). +// Data is written to a temporary file in the destination directory, synced, +// and renamed over the target so readers never observe a torn file and an +// interrupted write never destroys the previous content. +package fsatomic + +import ( + "os" + "path/filepath" +) + +// WriteFile atomically replaces path with data. The temporary file is created +// in path's directory so the final rename stays on one filesystem. The file is +// fsynced before the rename, and the directory afterwards (best-effort), so a +// power loss cannot persist the rename without the data. +func WriteFile(path string, data []byte, perm os.FileMode) error { + dir := filepath.Dir(path) + tmp, err := os.CreateTemp(dir, "."+filepath.Base(path)+".tmp-*") + if err != nil { + return err + } + tmpName := tmp.Name() + defer func() { _ = os.Remove(tmpName) }() + if _, err := tmp.Write(data); err != nil { + _ = tmp.Close() + return err + } + if err := tmp.Chmod(perm); err != nil { + _ = tmp.Close() + return err + } + if err := tmp.Sync(); err != nil { + _ = tmp.Close() + return err + } + if err := tmp.Close(); err != nil { + return err + } + if err := os.Rename(tmpName, path); err != nil { + return err + } + syncDir(dir) + return nil +} + +// syncDir flushes the directory entry for a completed rename. Errors are +// ignored: not every platform or filesystem supports fsync on directories, and +// the write itself has already succeeded. +func syncDir(dir string) { + d, err := os.Open(dir) + if err != nil { + return + } + _ = d.Sync() + _ = d.Close() +} diff --git a/cleanr/profile/store.go b/cleanr/profile/store.go index 4733dda..b3a468e 100644 --- a/cleanr/profile/store.go +++ b/cleanr/profile/store.go @@ -8,6 +8,8 @@ import ( "path/filepath" "strings" "time" + + "github.com/devr-tools/cleanr/cleanr/fsatomic" ) const ( @@ -81,7 +83,9 @@ func Save(file File) error { if err != nil { return fmt.Errorf("encode profile: %w", err) } - return os.WriteFile(path, append(data, '\n'), 0o600) + // Atomic so a crash mid-save cannot corrupt the profile and lose every + // stored provider credential. + return fsatomic.WriteFile(path, append(data, '\n'), 0o600) } func UpsertProvider(provider Provider) error { diff --git a/cleanr/snapshots/io.go b/cleanr/snapshots/io.go index b094dc1..46b6a2a 100644 --- a/cleanr/snapshots/io.go +++ b/cleanr/snapshots/io.go @@ -8,6 +8,8 @@ import ( "strings" "gopkg.in/yaml.v3" + + "github.com/devr-tools/cleanr/cleanr/fsatomic" ) func LoadFile(path string) (File, error) { @@ -26,32 +28,7 @@ func WriteFile(path string, snapshot File) error { if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { return err } - return writeFileAtomic(path, append(data, '\n'), 0o644) -} - -// writeFileAtomic writes data to a temp file in the same directory as path and -// renames it over the target so an interrupt can never leave a truncated or -// partially-written state file behind. -func writeFileAtomic(path string, data []byte, perm os.FileMode) error { - dir := filepath.Dir(path) - tmp, err := os.CreateTemp(dir, "."+filepath.Base(path)+".tmp-*") - if err != nil { - return err - } - tmpName := tmp.Name() - defer func() { _ = os.Remove(tmpName) }() - if _, err := tmp.Write(data); err != nil { - _ = tmp.Close() - return err - } - if err := tmp.Chmod(perm); err != nil { - _ = tmp.Close() - return err - } - if err := tmp.Close(); err != nil { - return err - } - return os.Rename(tmpName, path) + return fsatomic.WriteFile(path, append(data, '\n'), 0o644) } func decodeFile(data []byte, path string) (File, error) { diff --git a/cleanr/trends/io.go b/cleanr/trends/io.go index 3961ae7..9327bc7 100644 --- a/cleanr/trends/io.go +++ b/cleanr/trends/io.go @@ -8,6 +8,8 @@ import ( "strings" "gopkg.in/yaml.v3" + + "github.com/devr-tools/cleanr/cleanr/fsatomic" ) func LoadFile(path string) (HistoryFile, error) { @@ -30,32 +32,7 @@ func WriteFile(path string, history HistoryFile) error { if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { return err } - return writeFileAtomic(path, append(data, '\n'), 0o644) -} - -// writeFileAtomic writes data to a temp file in the same directory as path and -// renames it over the target so an interrupt can never leave a truncated or -// partially-written state file behind. -func writeFileAtomic(path string, data []byte, perm os.FileMode) error { - dir := filepath.Dir(path) - tmp, err := os.CreateTemp(dir, "."+filepath.Base(path)+".tmp-*") - if err != nil { - return err - } - tmpName := tmp.Name() - defer func() { _ = os.Remove(tmpName) }() - if _, err := tmp.Write(data); err != nil { - _ = tmp.Close() - return err - } - if err := tmp.Chmod(perm); err != nil { - _ = tmp.Close() - return err - } - if err := tmp.Close(); err != nil { - return err - } - return os.Rename(tmpName, path) + return fsatomic.WriteFile(path, append(data, '\n'), 0o644) } func decodeFile(data []byte, path string) (HistoryFile, error) { diff --git a/cleanr/trends/replay_artifact.go b/cleanr/trends/replay_artifact.go index 57b4559..990ad87 100644 --- a/cleanr/trends/replay_artifact.go +++ b/cleanr/trends/replay_artifact.go @@ -10,6 +10,7 @@ import ( "strings" "github.com/devr-tools/cleanr/cleanr/core" + "github.com/devr-tools/cleanr/cleanr/fsatomic" "gopkg.in/yaml.v3" ) @@ -121,7 +122,9 @@ func WriteReplayArtifactFile(path string, artifact core.ReplayArtifact) error { if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { return err } - return os.WriteFile(path, append(data, '\n'), 0o644) + // Atomic to match the trend-history writer: attestation digests and replay + // tooling must never observe a truncated artifact. + return fsatomic.WriteFile(path, append(data, '\n'), 0o644) } func encodeReplayArtifact(artifact core.ReplayArtifact, path string) ([]byte, error) { From c1a00426171f76a2e8e88c36bf2411cac587dcbb Mon Sep 17 00:00:00 2001 From: Alex Wilkerson John Date: Thu, 2 Jul 2026 13:42:00 -0400 Subject: [PATCH 06/13] fix(config): let explicit zero/false threshold values survive defaults Zero-ambiguous fields were silently rewritten by applyDefaults: require_review: false was inverted to true, max_normalized_drift: 0 (zero tolerance) became 0.3, min_score: 0 became 0.6, and the trend-gate presets overrode an explicit enabled: value in both directions. Following the existing MaxFailedCasesDelta precedent, these fields are now pointer-typed with *Value() accessors that supply the defaults at read time: DriftConfig's six thresholds, LLMJudgeConfig.MinScore, ScenarioGenerationConfig.RequireReview, and TrendGateConfig.Enabled. Presets fill Enabled only when unset, so exploratory + enabled: true keeps relaxed thresholds with gating active (docs updated). Because the accessors default at read time, SDK-built configs that bypass applyDefaults now get correct thresholds too. Co-Authored-By: Claude Fable 5 --- cleanr/config/defaults.go | 28 +---- cleanr/config/example.go | 14 +-- cleanr/config/trend_gate_presets.go | 18 ++- cleanr/config/validation.go | 18 ++- cleanr/core/types/config.go | 102 ++++++++++++++-- cleanr/engines/drift.go | 24 ++-- cleanr/engines/llm_judge_advanced.go | 4 +- cleanr/engines/llm_judge_score.go | 2 +- cleanr/generation/generate.go | 2 +- cleanr/trends/gates.go | 2 +- docs/configuration.md | 2 +- internal/cli/setup/config.go | 36 +++--- tests/cli/cli_test.go | 24 ++-- tests/cli/setup_test.go | 4 +- tests/config/config_validation_test.go | 14 +-- tests/config/helpers_test.go | 5 + tests/config/io_additional_test.go | 4 +- tests/config/llm_judge_validation_test.go | 6 +- tests/config/pointer_defaults_test.go | 127 +++++++++++++++++++ tests/config/validation_additional_test.go | 16 +-- tests/engines/assertions_test.go | 2 + tests/engines/engines_test.go | 12 +- tests/engines/llm_judge_test.go | 4 +- tests/generation_test.go | 2 +- tests/helpers_test.go | 3 + tests/runner/runner_test.go | 134 ++++++++++++++++++--- tests/trends/helpers_test.go | 3 + tests/trends/trends_test.go | 8 +- 28 files changed, 469 insertions(+), 151 deletions(-) create mode 100644 tests/config/helpers_test.go create mode 100644 tests/config/pointer_defaults_test.go create mode 100644 tests/helpers_test.go create mode 100644 tests/trends/helpers_test.go diff --git a/cleanr/config/defaults.go b/cleanr/config/defaults.go index 8923570..64635ef 100644 --- a/cleanr/config/defaults.go +++ b/cleanr/config/defaults.go @@ -31,9 +31,6 @@ func applyScenarioGenerationDefaults(cfg *core.Config) { if cfg.ScenarioGeneration.Count == 0 { cfg.ScenarioGeneration.Count = 12 } - if !cfg.ScenarioGeneration.RequireReview { - cfg.ScenarioGeneration.RequireReview = true - } } func applyOpenAPIDefaults(cfg *core.Config) { @@ -65,9 +62,6 @@ func applyLLMJudgeDefaults(cfg *core.LLMJudgeConfig) { if cfg.Scale <= 1 { cfg.Scale = 5 } - if cfg.MinScore == 0 { - cfg.MinScore = 0.6 - } if cfg.Samples <= 0 { cfg.Samples = 1 } @@ -134,24 +128,8 @@ func applyDriftDefaults(cfg *core.DriftConfig) { if cfg.Iterations == 0 { cfg.Iterations = 3 } - if cfg.MaxNormalizedDrift == 0 { - cfg.MaxNormalizedDrift = 0.3 - } - if cfg.MaxSemanticDrift == 0 { - cfg.MaxSemanticDrift = 0.25 - } - if cfg.MaxSnapshotDrift == 0 { - cfg.MaxSnapshotDrift = cfg.MaxNormalizedDrift - } - if cfg.MaxSemanticSnapshotDrift == 0 { - cfg.MaxSemanticSnapshotDrift = cfg.MaxSemanticDrift - } - if cfg.MinConsistencyScore == 0 { - cfg.MinConsistencyScore = 0.7 - } - if cfg.MinSemanticConsistencyScore == 0 { - cfg.MinSemanticConsistencyScore = 0.75 - } + // Drift/consistency thresholds are pointer-typed and default through their + // *Value accessors so an explicit zero survives. if cfg.ConfidenceLevel == 0 { cfg.ConfidenceLevel = 0.95 } @@ -258,7 +236,7 @@ func applyReportingDefaults(cfg *core.Config) { cfg.Reporting.ReplayArtifactFile = deriveReplayArtifactPath(cfg.Reporting.TrendFile) } applyTrendGatePreset(&cfg.Reporting.TrendGates) - if cfg.Reporting.TrendGates.Enabled && cfg.Reporting.TrendGates.RequiredWindow == 0 { + if cfg.Reporting.TrendGates.EnabledValue() && cfg.Reporting.TrendGates.RequiredWindow == 0 { cfg.Reporting.TrendGates.RequiredWindow = 2 } } diff --git a/cleanr/config/example.go b/cleanr/config/example.go index e6f7310..0365cbc 100644 --- a/cleanr/config/example.go +++ b/cleanr/config/example.go @@ -83,13 +83,13 @@ func exampleSuites() core.SuitesConfig { Drift: core.DriftConfig{ Enabled: true, Iterations: 4, - MaxNormalizedDrift: 0.32, - MaxSemanticDrift: 0.25, - MaxSnapshotDrift: 0.18, - MaxSemanticSnapshotDrift: 0.2, + MaxNormalizedDrift: float64Ptr(0.32), + MaxSemanticDrift: float64Ptr(0.25), + MaxSnapshotDrift: float64Ptr(0.18), + MaxSemanticSnapshotDrift: float64Ptr(0.2), StableTags: []string{"stable"}, - MinConsistencyScore: 0.68, - MinSemanticConsistencyScore: 0.75, + MinConsistencyScore: float64Ptr(0.68), + MinSemanticConsistencyScore: float64Ptr(0.75), }, ShadowState: core.ShadowStateConfig{Enabled: false}, Provenance: core.ProvenanceConfig{Enabled: false}, @@ -118,7 +118,7 @@ func exampleSuites() core.SuitesConfig { "The response follows the assistant's system instructions.", }, Scale: 5, - MinScore: 0.6, + MinScore: float64Ptr(0.6), Samples: 1, }, } diff --git a/cleanr/config/trend_gate_presets.go b/cleanr/config/trend_gate_presets.go index 65349dd..fac15e5 100644 --- a/cleanr/config/trend_gate_presets.go +++ b/cleanr/config/trend_gate_presets.go @@ -19,7 +19,9 @@ func applyTrendGatePreset(gates *core.TrendGateConfig) { switch normalizeTrendGatePreset(gates.Preset) { case trendGatePresetStrict: gates.Preset = trendGatePresetStrict - gates.Enabled = true + if gates.Enabled == nil { + gates.Enabled = boolPtr(true) + } if gates.RequiredWindow == 0 { gates.RequiredWindow = 2 } @@ -41,7 +43,9 @@ func applyTrendGatePreset(gates *core.TrendGateConfig) { gates.FailOnRegressedSuites = true case trendGatePresetModerate: gates.Preset = trendGatePresetModerate - gates.Enabled = true + if gates.Enabled == nil { + gates.Enabled = boolPtr(true) + } if gates.RequiredWindow == 0 { gates.RequiredWindow = 2 } @@ -63,7 +67,11 @@ func applyTrendGatePreset(gates *core.TrendGateConfig) { gates.FailOnRegressedSuites = true case trendGatePresetExploratory: gates.Preset = trendGatePresetExploratory - gates.Enabled = false + // Non-blocking by default only: an explicit `enabled: true` keeps the + // relaxed thresholds while leaving gating active. + if gates.Enabled == nil { + gates.Enabled = boolPtr(false) + } if gates.RequiredWindow == 0 { gates.RequiredWindow = 2 } @@ -93,3 +101,7 @@ func intPtr(v int) *int { func float64Ptr(v float64) *float64 { return &v } + +func boolPtr(v bool) *bool { + return &v +} diff --git a/cleanr/config/validation.go b/cleanr/config/validation.go index 6e6b7e8..c687104 100644 --- a/cleanr/config/validation.go +++ b/cleanr/config/validation.go @@ -302,9 +302,7 @@ func validateLLMJudgeThresholds(errs *ValidationErrors, cfg core.LLMJudgeConfig) if cfg.Scale != 0 && cfg.Scale < 2 { errs.Add("suites.llm_judge.scale", "must be >= 2", "use a Likert ceiling such as 5, or omit the field to use the default") } - if cfg.MinScore < 0 || cfg.MinScore > 1 { - errs.Add("suites.llm_judge.min_score", "must be between 0 and 1", "use a normalized pass threshold such as 0.6 (3 out of 5)") - } + validateOptionalUnitInterval(errs, "suites.llm_judge.min_score", cfg.MinScore, "use a normalized pass threshold such as 0.6 (3 out of 5)") if cfg.Samples < 0 { errs.Add("suites.llm_judge.samples", "must be >= 0", "set the self-consistency sample count to a positive integer or omit the field to use a single judge call") } @@ -437,12 +435,12 @@ func validateDriftSuite(errs *ValidationErrors, cfg core.DriftConfig) { if cfg.Iterations < 2 { errs.Add("suites.drift.iterations", "must be >= 2", "set iterations to 2 or more so drift can compare repeated runs") } - validateUnitInterval(errs, "suites.drift.max_normalized_drift", cfg.MaxNormalizedDrift, "use a decimal threshold such as 0.3") - validateUnitInterval(errs, "suites.drift.max_semantic_drift", cfg.MaxSemanticDrift, "use a decimal threshold such as 0.25") - validateUnitInterval(errs, "suites.drift.max_snapshot_drift", cfg.MaxSnapshotDrift, "use a decimal threshold such as 0.18") - validateUnitInterval(errs, "suites.drift.max_semantic_snapshot_drift", cfg.MaxSemanticSnapshotDrift, "use a decimal threshold such as 0.2") - validateUnitInterval(errs, "suites.drift.min_consistency_score", cfg.MinConsistencyScore, "use a decimal threshold such as 0.7") - validateUnitInterval(errs, "suites.drift.min_semantic_consistency_score", cfg.MinSemanticConsistencyScore, "use a decimal threshold such as 0.75") + validateOptionalUnitInterval(errs, "suites.drift.max_normalized_drift", cfg.MaxNormalizedDrift, "use a decimal threshold such as 0.3") + validateOptionalUnitInterval(errs, "suites.drift.max_semantic_drift", cfg.MaxSemanticDrift, "use a decimal threshold such as 0.25") + validateOptionalUnitInterval(errs, "suites.drift.max_snapshot_drift", cfg.MaxSnapshotDrift, "use a decimal threshold such as 0.18") + validateOptionalUnitInterval(errs, "suites.drift.max_semantic_snapshot_drift", cfg.MaxSemanticSnapshotDrift, "use a decimal threshold such as 0.2") + validateOptionalUnitInterval(errs, "suites.drift.min_consistency_score", cfg.MinConsistencyScore, "use a decimal threshold such as 0.7") + validateOptionalUnitInterval(errs, "suites.drift.min_semantic_consistency_score", cfg.MinSemanticConsistencyScore, "use a decimal threshold such as 0.75") validateUnitInterval(errs, "suites.drift.confidence_level", cfg.ConfidenceLevel, "use a confidence level such as 0.95 for Wilson pass-rate intervals") validateUnitInterval(errs, "suites.drift.min_pass_rate", cfg.MinPassRate, "use a fractional pass-rate gate such as 0.9 for a 9/10 stability threshold") validateUnitInterval(errs, "suites.drift.max_flake_rate", cfg.MaxFlakeRate, "use a fractional instability budget such as 0.1") @@ -522,7 +520,7 @@ func validateReportingConfig(errs *ValidationErrors, cfg core.ReportingConfig) { if !isValidTrendGatePreset(cfg.TrendGates.Preset) { errs.Add("reporting.trend_gates.preset", "must be one of strict, moderate, or exploratory", "choose a built-in trend gate preset or remove the field to set thresholds manually") } - if !cfg.TrendGates.Enabled { + if !cfg.TrendGates.EnabledValue() { return } if strings.TrimSpace(cfg.TrendFile) == "" { diff --git a/cleanr/core/types/config.go b/cleanr/core/types/config.go index 363bbdd..cb78886 100644 --- a/cleanr/core/types/config.go +++ b/cleanr/core/types/config.go @@ -98,7 +98,17 @@ type ScenarioGenerationConfig struct { Spec ScenarioGenerationSpec `json:"spec"` OutputFile string `json:"output_file,omitempty"` Count int `json:"count,omitempty"` - RequireReview bool `json:"require_review,omitempty"` + RequireReview *bool `json:"require_review,omitempty"` +} + +// RequireReviewValue reports whether generated scenarios need human review +// before use. It defaults to true; a pointer keeps an explicit +// `require_review: false` expressible instead of being silently inverted. +func (c ScenarioGenerationConfig) RequireReviewValue() bool { + if c.RequireReview != nil { + return *c.RequireReview + } + return true } type ScenarioGenerationSpec struct { @@ -317,22 +327,77 @@ type ChaosConfig struct { ResponseField string `json:"response_field"` } +// DriftConfig thresholds are pointers so an explicit zero ("no drift +// tolerated at all") survives default application instead of being replaced +// by the default; use the *Value accessors to read them. type DriftConfig struct { Enabled bool `json:"enabled"` Iterations int `json:"iterations"` - MaxNormalizedDrift float64 `json:"max_normalized_drift"` - MaxSemanticDrift float64 `json:"max_semantic_drift"` - MaxSnapshotDrift float64 `json:"max_snapshot_drift"` - MaxSemanticSnapshotDrift float64 `json:"max_semantic_snapshot_drift"` + MaxNormalizedDrift *float64 `json:"max_normalized_drift,omitempty"` + MaxSemanticDrift *float64 `json:"max_semantic_drift,omitempty"` + MaxSnapshotDrift *float64 `json:"max_snapshot_drift,omitempty"` + MaxSemanticSnapshotDrift *float64 `json:"max_semantic_snapshot_drift,omitempty"` BaselineFile string `json:"baseline_file"` StableTags []string `json:"stable_tags"` - MinConsistencyScore float64 `json:"min_consistency_score"` - MinSemanticConsistencyScore float64 `json:"min_semantic_consistency_score"` + MinConsistencyScore *float64 `json:"min_consistency_score,omitempty"` + MinSemanticConsistencyScore *float64 `json:"min_semantic_consistency_score,omitempty"` ConfidenceLevel float64 `json:"confidence_level,omitempty"` MinPassRate float64 `json:"min_pass_rate,omitempty"` MaxFlakeRate float64 `json:"max_flake_rate,omitempty"` } +// MaxNormalizedDriftValue returns the lexical drift ceiling, defaulting to 0.3. +func (c DriftConfig) MaxNormalizedDriftValue() float64 { + if c.MaxNormalizedDrift != nil { + return *c.MaxNormalizedDrift + } + return 0.3 +} + +// MaxSemanticDriftValue returns the semantic drift ceiling, defaulting to 0.25. +func (c DriftConfig) MaxSemanticDriftValue() float64 { + if c.MaxSemanticDrift != nil { + return *c.MaxSemanticDrift + } + return 0.25 +} + +// MaxSnapshotDriftValue returns the baseline drift ceiling, defaulting to the +// lexical drift ceiling. +func (c DriftConfig) MaxSnapshotDriftValue() float64 { + if c.MaxSnapshotDrift != nil { + return *c.MaxSnapshotDrift + } + return c.MaxNormalizedDriftValue() +} + +// MaxSemanticSnapshotDriftValue returns the semantic baseline drift ceiling, +// defaulting to the semantic drift ceiling. +func (c DriftConfig) MaxSemanticSnapshotDriftValue() float64 { + if c.MaxSemanticSnapshotDrift != nil { + return *c.MaxSemanticSnapshotDrift + } + return c.MaxSemanticDriftValue() +} + +// MinConsistencyScoreValue returns the lexical consistency floor, defaulting +// to 0.7. +func (c DriftConfig) MinConsistencyScoreValue() float64 { + if c.MinConsistencyScore != nil { + return *c.MinConsistencyScore + } + return 0.7 +} + +// MinSemanticConsistencyScoreValue returns the semantic consistency floor, +// defaulting to 0.75. +func (c DriftConfig) MinSemanticConsistencyScoreValue() float64 { + if c.MinSemanticConsistencyScore != nil { + return *c.MinSemanticConsistencyScore + } + return 0.75 +} + type ShadowStateConfig struct { Enabled bool `json:"enabled"` Roots []string `json:"roots"` @@ -407,7 +472,7 @@ type LLMJudgeConfig struct { Baseline TargetConfig `json:"baseline,omitempty"` Criteria []string `json:"criteria,omitempty"` Scale int `json:"scale,omitempty"` - MinScore float64 `json:"min_score,omitempty"` + MinScore *float64 `json:"min_score,omitempty"` MinWinRate float64 `json:"min_win_rate,omitempty"` Samples int `json:"samples,omitempty"` MaxDisagreement float64 `json:"max_disagreement,omitempty"` @@ -443,6 +508,15 @@ func (c LLMJudgeConfig) ScaleValue() int { return c.Scale } +// MinScoreValue returns the normalized pass threshold, defaulting to 0.6. A +// pointer keeps an explicit `min_score: 0` (no floor) expressible. +func (c LLMJudgeConfig) MinScoreValue() float64 { + if c.MinScore != nil { + return *c.MinScore + } + return 0.6 +} + // SamplesValue returns the configured self-consistency sample count, // defaulting to a single judge call. func (c LLMJudgeConfig) SamplesValue() int { @@ -602,7 +676,7 @@ type QueueProbeObservation struct { type TrendGateConfig struct { Preset string `json:"preset,omitempty"` - Enabled bool `json:"enabled"` + Enabled *bool `json:"enabled,omitempty"` RequiredWindow int `json:"required_window"` MaxFailedSuitesDelta *int `json:"max_failed_suites_delta,omitempty"` MaxFailedCasesDelta *int `json:"max_failed_cases_delta,omitempty"` @@ -612,6 +686,16 @@ type TrendGateConfig struct { FailOnRegressedSuites bool `json:"fail_on_regressed_suites,omitempty"` } +// EnabledValue reports whether trend gates are active, defaulting to false. A +// pointer keeps an explicit `enabled:` distinguishable from unset, so presets +// can supply a default without overriding the user's choice. +func (c TrendGateConfig) EnabledValue() bool { + if c.Enabled != nil { + return *c.Enabled + } + return false +} + // defaultTargetTimeoutMS mirrors the request timeout applied by the config // package's applyDefaults. SDK-built configs never run applyDefaults, so // Timeout() falls back to this value when TimeoutMS is unset to avoid handing diff --git a/cleanr/engines/drift.go b/cleanr/engines/drift.go index 5ffb4f9..f24e621 100644 --- a/cleanr/engines/drift.go +++ b/cleanr/engines/drift.go @@ -120,11 +120,11 @@ func collectDriftResponses(ctx context.Context, runCtx *core.RunContext, cfg cor func evaluateDriftThresholdFindings(cfg core.DriftConfig, semanticDrift, semanticConsistency float64) []core.Finding { findings := make([]core.Finding, 0) - if semanticDrift > cfg.MaxSemanticDrift { - findings = append(findings, core.Finding{Severity: "high", Message: fmt.Sprintf("semantic drift %.3f exceeded threshold %.3f", semanticDrift, cfg.MaxSemanticDrift)}) + if semanticDrift > cfg.MaxSemanticDriftValue() { + findings = append(findings, core.Finding{Severity: "high", Message: fmt.Sprintf("semantic drift %.3f exceeded threshold %.3f", semanticDrift, cfg.MaxSemanticDriftValue())}) } - if semanticConsistency < cfg.MinSemanticConsistencyScore { - findings = append(findings, core.Finding{Severity: "medium", Message: fmt.Sprintf("semantic consistency score %.3f fell below threshold %.3f", semanticConsistency, cfg.MinSemanticConsistencyScore)}) + if semanticConsistency < cfg.MinSemanticConsistencyScoreValue() { + findings = append(findings, core.Finding{Severity: "medium", Message: fmt.Sprintf("semantic consistency score %.3f fell below threshold %.3f", semanticConsistency, cfg.MinSemanticConsistencyScoreValue())}) } return findings } @@ -138,11 +138,11 @@ func buildDriftDetails(cfg core.DriftConfig, drift, semanticDrift, consistency, "samples": sampleCount, "semantic_similarity_profile": "local_similarity_v1", } - if drift > cfg.MaxNormalizedDrift && semanticDrift <= cfg.MaxSemanticDrift { - details["lexical_drift_note"] = fmt.Sprintf("normalized drift %.3f exceeded threshold %.3f, but semantic drift remained within threshold", round3(drift), cfg.MaxNormalizedDrift) + if drift > cfg.MaxNormalizedDriftValue() && semanticDrift <= cfg.MaxSemanticDriftValue() { + details["lexical_drift_note"] = fmt.Sprintf("normalized drift %.3f exceeded threshold %.3f, but semantic drift remained within threshold", round3(drift), cfg.MaxNormalizedDriftValue()) } - if consistency < cfg.MinConsistencyScore && semanticConsistency >= cfg.MinSemanticConsistencyScore { - details["lexical_consistency_note"] = fmt.Sprintf("consistency score %.3f fell below threshold %.3f, but semantic consistency remained within threshold", round3(consistency), cfg.MinConsistencyScore) + if consistency < cfg.MinConsistencyScoreValue() && semanticConsistency >= cfg.MinSemanticConsistencyScoreValue() { + details["lexical_consistency_note"] = fmt.Sprintf("consistency score %.3f fell below threshold %.3f, but semantic consistency remained within threshold", round3(consistency), cfg.MinConsistencyScoreValue()) } return details } @@ -168,8 +168,8 @@ func applyBaselineDriftComparison(findings []core.Finding, details map[string]an details["baseline_semantic_drift"] = round3(snapshotSemanticDrift) details["baseline_status_code"] = snapshot.StatusCode findings = append(findings, baselineDriftFindings(cfg, snapshot, representative, snapshotSemanticDrift)...) - if snapshotDrift > cfg.MaxSnapshotDrift && snapshotSemanticDrift <= cfg.MaxSemanticSnapshotDrift { - details["baseline_lexical_note"] = fmt.Sprintf("baseline drift %.3f exceeded threshold %.3f, but semantic baseline drift remained within threshold", round3(snapshotDrift), cfg.MaxSnapshotDrift) + if snapshotDrift > cfg.MaxSnapshotDriftValue() && snapshotSemanticDrift <= cfg.MaxSemanticSnapshotDriftValue() { + details["baseline_lexical_note"] = fmt.Sprintf("baseline drift %.3f exceeded threshold %.3f, but semantic baseline drift remained within threshold", round3(snapshotDrift), cfg.MaxSnapshotDriftValue()) } return findings, details } @@ -185,8 +185,8 @@ func baselineDriftFindings(cfg core.DriftConfig, snapshot snapshotspkg.ScenarioS if len(snapshot.Normalized.ToolCalls) != len(representative.Normalized.ToolCalls) { findings = append(findings, core.Finding{Severity: "medium", Message: fmt.Sprintf("baseline tool call count %d changed to %d", len(snapshot.Normalized.ToolCalls), len(representative.Normalized.ToolCalls))}) } - if snapshotSemanticDrift > cfg.MaxSemanticSnapshotDrift { - findings = append(findings, core.Finding{Severity: "high", Message: fmt.Sprintf("semantic baseline drift %.3f exceeded threshold %.3f", snapshotSemanticDrift, cfg.MaxSemanticSnapshotDrift)}) + if snapshotSemanticDrift > cfg.MaxSemanticSnapshotDriftValue() { + findings = append(findings, core.Finding{Severity: "high", Message: fmt.Sprintf("semantic baseline drift %.3f exceeded threshold %.3f", snapshotSemanticDrift, cfg.MaxSemanticSnapshotDriftValue())}) } return findings } diff --git a/cleanr/engines/llm_judge_advanced.go b/cleanr/engines/llm_judge_advanced.go index 871ecb4..c0ba8b3 100644 --- a/cleanr/engines/llm_judge_advanced.go +++ b/cleanr/engines/llm_judge_advanced.go @@ -139,7 +139,7 @@ func calibrateJudgeCases(cfg core.LLMJudgeConfig, cases []core.CaseResult) (map[ if label.Pass != nil { expectedPass = *label.Pass } else if label.Score != nil { - threshold := cfg.MinScore + threshold := cfg.MinScoreValue() if cfg.ModeValue() == "pairwise" { threshold = cfg.MinWinRate } @@ -220,7 +220,7 @@ func buildComparisonMatrix(ctx context.Context, runCtx *core.RunContext, cfg cor run := scoreRun{ scale: cfg.ScaleValue(), samples: cfg.SamplesValue(), - minScore: cfg.MinScore, + minScore: cfg.MinScoreValue(), maxDisagreement: cfg.MaxDisagreement, confidenceLevel: cfg.ConfidenceLevelValue(), minPassRate: cfg.MinPassRate, diff --git a/cleanr/engines/llm_judge_score.go b/cleanr/engines/llm_judge_score.go index f0d9adc..2750e9f 100644 --- a/cleanr/engines/llm_judge_score.go +++ b/cleanr/engines/llm_judge_score.go @@ -53,7 +53,7 @@ func (e LLMJudgeEngine) runScore(ctx context.Context, runCtx *core.RunContext, c run := scoreRun{ scale: cfg.ScaleValue(), samples: cfg.SamplesValue(), - minScore: cfg.MinScore, + minScore: cfg.MinScoreValue(), maxDisagreement: cfg.MaxDisagreement, confidenceLevel: cfg.ConfidenceLevelValue(), minPassRate: cfg.MinPassRate, diff --git a/cleanr/generation/generate.go b/cleanr/generation/generate.go index c74a78d..c0fcfd5 100644 --- a/cleanr/generation/generate.go +++ b/cleanr/generation/generate.go @@ -48,7 +48,7 @@ func GenerateDataset(ctx context.Context, cfg core.Config, client *http.Client) Source: "cleanr-generation", Target: cfg.Target.Name, GeneratedAt: time.Now().UTC(), - ReviewRequired: cfg.ScenarioGeneration.RequireReview, + ReviewRequired: cfg.ScenarioGeneration.RequireReviewValue(), Warnings: warnings, Generator: &integrationspkg.ScenarioDatasetGenerator{ Provider: resp.Normalized.Provider, diff --git a/cleanr/trends/gates.go b/cleanr/trends/gates.go index 1960d4c..f5401ca 100644 --- a/cleanr/trends/gates.go +++ b/cleanr/trends/gates.go @@ -9,7 +9,7 @@ import ( ) func EvaluateGates(report *core.Report, cfg core.TrendGateConfig) { - if report == nil || !cfg.Enabled { + if report == nil || !cfg.EnabledValue() { return } diff --git a/docs/configuration.md b/docs/configuration.md index c72736d..39696cd 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -998,7 +998,7 @@ Preset behavior: - `strict`: blocks on any new failed suite or case, allows up to `15%` duration growth, `0.05` semantic drift delta, and `0.03` baseline semantic drift delta. - `moderate`: blocks on any new failed suite or case, allows up to `25%` duration growth, `0.08` semantic drift delta, and `0.05` baseline semantic drift delta. -- `exploratory`: keeps trend reporting enabled but makes trend gates non-blocking by default. +- `exploratory`: keeps trend reporting enabled but makes trend gates non-blocking by default. An explicit `enabled: true` beside the preset keeps the relaxed thresholds with gating active — presets never override an explicit `enabled:` value. You can start from a preset and override only one dimension. For example, this keeps the `moderate` preset but allows more latency growth between builds: diff --git a/internal/cli/setup/config.go b/internal/cli/setup/config.go index 1233c94..d143b5f 100644 --- a/internal/cli/setup/config.go +++ b/internal/cli/setup/config.go @@ -172,14 +172,14 @@ func applyStarterProfile(cfg *cleanr.Config, options starterConfigOptions) { cfg.Suites.Drift = cleanr.DriftConfig{ Enabled: true, Iterations: 2, - MaxNormalizedDrift: 0.22, - MaxSemanticDrift: 0.16, - MaxSnapshotDrift: 0.12, - MaxSemanticSnapshotDrift: 0.10, + MaxNormalizedDrift: float64Ptr(0.22), + MaxSemanticDrift: float64Ptr(0.16), + MaxSnapshotDrift: float64Ptr(0.12), + MaxSemanticSnapshotDrift: float64Ptr(0.10), BaselineFile: defaultBaselinePath(cfg.Target.Name), StableTags: []string{"stable"}, - MinConsistencyScore: 0.78, - MinSemanticConsistencyScore: 0.84, + MinConsistencyScore: float64Ptr(0.78), + MinSemanticConsistencyScore: float64Ptr(0.84), } cfg.Suites.TokenOptimization = cleanr.TokenOptimizationConfig{ Enabled: true, @@ -202,14 +202,14 @@ func applyStarterProfile(cfg *cleanr.Config, options starterConfigOptions) { cfg.Suites.Drift = cleanr.DriftConfig{ Enabled: true, Iterations: 3, - MaxNormalizedDrift: 0.24, - MaxSemanticDrift: 0.18, - MaxSnapshotDrift: 0.14, - MaxSemanticSnapshotDrift: 0.12, + MaxNormalizedDrift: float64Ptr(0.24), + MaxSemanticDrift: float64Ptr(0.18), + MaxSnapshotDrift: float64Ptr(0.14), + MaxSemanticSnapshotDrift: float64Ptr(0.12), BaselineFile: defaultBaselinePath(cfg.Target.Name), StableTags: []string{"stable"}, - MinConsistencyScore: 0.76, - MinSemanticConsistencyScore: 0.82, + MinConsistencyScore: float64Ptr(0.76), + MinSemanticConsistencyScore: float64Ptr(0.82), } cfg.Suites.TokenOptimization = cleanr.TokenOptimizationConfig{ Enabled: true, @@ -243,14 +243,14 @@ func applyStarterProfile(cfg *cleanr.Config, options starterConfigOptions) { cfg.Suites.Drift = cleanr.DriftConfig{ Enabled: true, Iterations: 4, - MaxNormalizedDrift: 0.28, - MaxSemanticDrift: 0.20, - MaxSnapshotDrift: 0.16, - MaxSemanticSnapshotDrift: 0.14, + MaxNormalizedDrift: float64Ptr(0.28), + MaxSemanticDrift: float64Ptr(0.20), + MaxSnapshotDrift: float64Ptr(0.16), + MaxSemanticSnapshotDrift: float64Ptr(0.14), BaselineFile: defaultBaselinePath(cfg.Target.Name), StableTags: []string{"stable"}, - MinConsistencyScore: 0.72, - MinSemanticConsistencyScore: 0.80, + MinConsistencyScore: float64Ptr(0.72), + MinSemanticConsistencyScore: float64Ptr(0.80), } cfg.Suites.TokenOptimization = cleanr.TokenOptimizationConfig{ Enabled: true, diff --git a/tests/cli/cli_test.go b/tests/cli/cli_test.go index 69514b4..6ffd04d 100644 --- a/tests/cli/cli_test.go +++ b/tests/cli/cli_test.go @@ -387,7 +387,7 @@ func TestGenerateCommandWritesReviewedDataset(t *testing.T) { }, OutputFile: "generated/cleanr.dataset.yaml", Count: 2, - RequireReview: true, + RequireReview: boolPtr(true), } configPath := filepath.Join(t.TempDir(), "cleanr.yaml") @@ -2333,10 +2333,10 @@ func TestRunCommandPersistsTrendHistory(t *testing.T) { cfg.Suites.Drift.Enabled = true cfg.Suites.Drift.Iterations = 2 cfg.Suites.Drift.StableTags = []string{"stable"} - cfg.Suites.Drift.MaxNormalizedDrift = 1 - cfg.Suites.Drift.MaxSemanticDrift = 1 - cfg.Suites.Drift.MinConsistencyScore = 0 - cfg.Suites.Drift.MinSemanticConsistencyScore = 0 + cfg.Suites.Drift.MaxNormalizedDrift = float64Ptr(1) + cfg.Suites.Drift.MaxSemanticDrift = float64Ptr(1) + cfg.Suites.Drift.MinConsistencyScore = float64Ptr(0) + cfg.Suites.Drift.MinSemanticConsistencyScore = float64Ptr(0) cfg.Reporting.Format = "json" cfg.Reporting.TrendFile = "reports/cleanr.trends.yaml" cfg.Reporting.TrendLimit = 5 @@ -2606,13 +2606,13 @@ func TestRunCommandTrendGatesFailOnConfiguredRegression(t *testing.T) { cfg.Suites.Drift.Enabled = true cfg.Suites.Drift.Iterations = 2 cfg.Suites.Drift.StableTags = []string{"stable"} - cfg.Suites.Drift.MaxNormalizedDrift = 1 - cfg.Suites.Drift.MaxSemanticDrift = 1 - cfg.Suites.Drift.MinConsistencyScore = 0 - cfg.Suites.Drift.MinSemanticConsistencyScore = 0 + cfg.Suites.Drift.MaxNormalizedDrift = float64Ptr(1) + cfg.Suites.Drift.MaxSemanticDrift = float64Ptr(1) + cfg.Suites.Drift.MinConsistencyScore = float64Ptr(0) + cfg.Suites.Drift.MinSemanticConsistencyScore = float64Ptr(0) cfg.Reporting.Format = "json" cfg.Reporting.TrendFile = "reports/cleanr.trends.yaml" - cfg.Reporting.TrendGates.Enabled = true + cfg.Reporting.TrendGates.Enabled = boolPtr(true) cfg.Reporting.TrendGates.RequiredWindow = 2 cfg.Reporting.TrendGates.MaxSemanticDriftDelta = float64Ptr(0.05) @@ -2908,3 +2908,7 @@ func testTrendTime(day int) time.Time { func float64Ptr(v float64) *float64 { return &v } + +func boolPtr(v bool) *bool { + return &v +} diff --git a/tests/cli/setup_test.go b/tests/cli/setup_test.go index cb8a5a3..36e6fbe 100644 --- a/tests/cli/setup_test.go +++ b/tests/cli/setup_test.go @@ -99,7 +99,7 @@ func TestSetupAgentCommandUsesStoredProviderAndInjectsPrompt(t *testing.T) { if cfg.Reporting.TrendFile != filepath.Join("reports", "support-agent.trends.yaml") { t.Fatalf("unexpected trend file: %s", cfg.Reporting.TrendFile) } - if !cfg.Reporting.TrendGates.Enabled { + if !cfg.Reporting.TrendGates.EnabledValue() { t.Fatalf("expected trend gates to be enabled in generated agent config") } if cfg.Reporting.TrendGates.RequiredWindow != 2 { @@ -134,7 +134,7 @@ func TestSetupCommandSupportsExploratoryTrendGatePreset(t *testing.T) { if cfg.Reporting.TrendGates.Preset != "exploratory" { t.Fatalf("expected exploratory preset, got %+v", cfg.Reporting.TrendGates) } - if cfg.Reporting.TrendGates.Enabled { + if cfg.Reporting.TrendGates.EnabledValue() { t.Fatalf("expected exploratory preset to be non-blocking, got %+v", cfg.Reporting.TrendGates) } } diff --git a/tests/config/config_validation_test.go b/tests/config/config_validation_test.go index c2894d6..88a89a5 100644 --- a/tests/config/config_validation_test.go +++ b/tests/config/config_validation_test.go @@ -136,7 +136,7 @@ func TestValidateConfigInvalidLoadAndDriftSettings(t *testing.T) { name: "snapshot drift threshold must be between zero and one", mutate: func(cfg *cleanr.Config) { cfg.Suites.Drift.Enabled = true - cfg.Suites.Drift.MaxSnapshotDrift = 1.5 + cfg.Suites.Drift.MaxSnapshotDrift = float64Ptr(1.5) }, wantErr: "invalid config: suites.drift.max_snapshot_drift: must be between 0 and 1. Fix: use a decimal threshold such as 0.18", }, @@ -144,7 +144,7 @@ func TestValidateConfigInvalidLoadAndDriftSettings(t *testing.T) { name: "semantic drift threshold must be between zero and one", mutate: func(cfg *cleanr.Config) { cfg.Suites.Drift.Enabled = true - cfg.Suites.Drift.MaxSemanticDrift = -0.1 + cfg.Suites.Drift.MaxSemanticDrift = float64Ptr(-0.1) }, wantErr: "invalid config: suites.drift.max_semantic_drift: must be between 0 and 1. Fix: use a decimal threshold such as 0.25", }, @@ -152,7 +152,7 @@ func TestValidateConfigInvalidLoadAndDriftSettings(t *testing.T) { name: "semantic consistency score must be between zero and one", mutate: func(cfg *cleanr.Config) { cfg.Suites.Drift.Enabled = true - cfg.Suites.Drift.MinSemanticConsistencyScore = 1.1 + cfg.Suites.Drift.MinSemanticConsistencyScore = float64Ptr(1.1) }, wantErr: "invalid config: suites.drift.min_semantic_consistency_score: must be between 0 and 1. Fix: use a decimal threshold such as 0.75", }, @@ -421,10 +421,10 @@ func TestLoadConfigFileAppliesDefaultsBeforeValidation(t *testing.T) { if cfg.Suites.Drift.Iterations != 3 { t.Fatalf("expected default drift iterations 3, got %d", cfg.Suites.Drift.Iterations) } - if cfg.Suites.Drift.MaxSemanticDrift != 0.25 { + if cfg.Suites.Drift.MaxSemanticDriftValue() != 0.25 { t.Fatalf("expected default semantic drift 0.25, got %v", cfg.Suites.Drift.MaxSemanticDrift) } - if cfg.Suites.Drift.MinSemanticConsistencyScore != 0.75 { + if cfg.Suites.Drift.MinSemanticConsistencyScoreValue() != 0.75 { t.Fatalf("expected default semantic consistency 0.75, got %v", cfg.Suites.Drift.MinSemanticConsistencyScore) } } @@ -476,10 +476,10 @@ suites: if cfg.Suites.Drift.Iterations != 3 { t.Fatalf("expected default drift iterations 3, got %d", cfg.Suites.Drift.Iterations) } - if cfg.Suites.Drift.MaxSemanticDrift != 0.25 { + if cfg.Suites.Drift.MaxSemanticDriftValue() != 0.25 { t.Fatalf("expected default semantic drift 0.25, got %v", cfg.Suites.Drift.MaxSemanticDrift) } - if cfg.Suites.Drift.MinSemanticConsistencyScore != 0.75 { + if cfg.Suites.Drift.MinSemanticConsistencyScoreValue() != 0.75 { t.Fatalf("expected default semantic consistency 0.75, got %v", cfg.Suites.Drift.MinSemanticConsistencyScore) } }) diff --git a/tests/config/helpers_test.go b/tests/config/helpers_test.go new file mode 100644 index 0000000..facd75e --- /dev/null +++ b/tests/config/helpers_test.go @@ -0,0 +1,5 @@ +package tests + +func float64Ptr(v float64) *float64 { return &v } + +func boolPtr(v bool) *bool { return &v } diff --git a/tests/config/io_additional_test.go b/tests/config/io_additional_test.go index f23d109..8740989 100644 --- a/tests/config/io_additional_test.go +++ b/tests/config/io_additional_test.go @@ -115,7 +115,7 @@ scenarios: if len(cfg.Suites.ReleasePolicy.Rules) != 1 || cfg.Suites.ReleasePolicy.Rules[0].Tools[0] != "send_email" { t.Fatalf("expected pack release-policy rules, got %+v", cfg.Suites.ReleasePolicy.Rules) } - if cfg.Reporting.TrendGates.Preset != "moderate" || !cfg.Reporting.TrendGates.Enabled { + if cfg.Reporting.TrendGates.Preset != "moderate" || !cfg.Reporting.TrendGates.EnabledValue() { t.Fatalf("expected policy pack trend gate preset, got %+v", cfg.Reporting.TrendGates) } } @@ -298,7 +298,7 @@ func TestScenarioGenerationConfigRoundTrips(t *testing.T) { }, OutputFile: "generated/cleanr.dataset.yaml", Count: 4, - RequireReview: true, + RequireReview: boolPtr(true), } path := filepath.Join(t.TempDir(), "cleanr.yaml") diff --git a/tests/config/llm_judge_validation_test.go b/tests/config/llm_judge_validation_test.go index f9a88d0..3f338e0 100644 --- a/tests/config/llm_judge_validation_test.go +++ b/tests/config/llm_judge_validation_test.go @@ -55,7 +55,7 @@ func TestValidateLLMJudgeRejectsOutOfRangeMinScore(t *testing.T) { cfg := judgeBaseConfig(core.LLMJudgeConfig{ Enabled: true, Provider: core.TargetConfig{Type: "openai", OpenAI: core.OpenAIConfig{Model: "gpt-4.1-mini", APIMode: "responses"}}, - MinScore: 1.5, + MinScore: float64Ptr(1.5), }, core.Scenario{Name: "s", Input: "hi"}) msg := validationMessage(t, cleanr.ValidateConfig(cfg), "min_score") if !strings.Contains(msg, "suites.llm_judge.min_score") { @@ -87,7 +87,7 @@ func TestValidateLLMJudgeValidConfigPasses(t *testing.T) { judge := core.LLMJudgeConfig{ Enabled: true, Provider: core.TargetConfig{Type: "anthropic", Anthropic: core.AnthropicConfig{Model: "claude-sonnet-4-20250514"}}, - MinScore: 0.7, + MinScore: float64Ptr(0.7), Samples: 3, MaxDisagreement: 0.4, RequireReference: true, @@ -182,7 +182,7 @@ func TestApplyLLMJudgeDefaults(t *testing.T) { if j.Scale != 5 { t.Fatalf("expected default scale 5, got %d", j.Scale) } - if j.MinScore != 0.6 { + if j.MinScoreValue() != 0.6 { t.Fatalf("expected default min_score 0.6, got %v", j.MinScore) } if j.MaxDisagreement != 0.4 { diff --git a/tests/config/pointer_defaults_test.go b/tests/config/pointer_defaults_test.go new file mode 100644 index 0000000..3429d41 --- /dev/null +++ b/tests/config/pointer_defaults_test.go @@ -0,0 +1,127 @@ +package tests + +import ( + "testing" + + "github.com/devr-tools/cleanr/cleanr" +) + +// Explicit zero thresholds must survive default application instead of being +// silently replaced by the default value. +func TestExplicitZeroDriftThresholdSurvivesLoad(t *testing.T) { + cfg := cleanr.ExampleConfig() + cfg.Suites.Drift.MaxNormalizedDrift = float64Ptr(0) + cfg.Suites.Drift.MinConsistencyScore = float64Ptr(0) + + data, err := cleanr.MarshalConfig(cfg, "json") + if err != nil { + t.Fatalf("marshal config: %v", err) + } + loaded, err := cleanr.LoadConfigData(data, "json") + if err != nil { + t.Fatalf("load config: %v", err) + } + + if got := loaded.Suites.Drift.MaxNormalizedDriftValue(); got != 0 { + t.Fatalf("explicit max_normalized_drift 0 replaced by %v", got) + } + if got := loaded.Suites.Drift.MinConsistencyScoreValue(); got != 0 { + t.Fatalf("explicit min_consistency_score 0 replaced by %v", got) + } +} + +func TestUnsetDriftThresholdsUseDefaults(t *testing.T) { + var drift cleanr.DriftConfig + if got := drift.MaxNormalizedDriftValue(); got != 0.3 { + t.Fatalf("expected default 0.3, got %v", got) + } + if got := drift.MaxSnapshotDriftValue(); got != 0.3 { + t.Fatalf("expected snapshot default to follow normalized default, got %v", got) + } + drift.MaxNormalizedDrift = float64Ptr(0.1) + if got := drift.MaxSnapshotDriftValue(); got != 0.1 { + t.Fatalf("expected snapshot default to follow explicit normalized value, got %v", got) + } +} + +// An explicit require_review: false must be expressible; it used to be +// silently inverted back to true by applyDefaults. +func TestExplicitRequireReviewFalseSurvivesLoad(t *testing.T) { + cfg := cleanr.ExampleConfig() + cfg.ScenarioGeneration = cleanr.ScenarioGenerationConfig{ + Enabled: true, + Provider: cleanr.TargetConfig{ + Type: "http", + URL: "https://generator.example.test/v1", + Method: "POST", + PromptField: "input", + ResponseField: "output.text", + }, + Spec: cleanr.ScenarioGenerationSpec{ + AppKind: "support-assistant", + Goals: []string{"refund policy"}, + RiskAreas: []string{"prompt injection"}, + }, + Count: 1, + RequireReview: boolPtr(false), + } + + data, err := cleanr.MarshalConfig(cfg, "json") + if err != nil { + t.Fatalf("marshal config: %v", err) + } + loaded, err := cleanr.LoadConfigData(data, "json") + if err != nil { + t.Fatalf("load config: %v", err) + } + + if loaded.ScenarioGeneration.RequireReviewValue() { + t.Fatalf("explicit require_review: false was inverted to true") + } + if (cleanr.ScenarioGenerationConfig{}).RequireReviewValue() != true { + t.Fatalf("unset require_review must default to true") + } +} + +// The exploratory preset makes gates non-blocking by default, but an explicit +// enabled: true must keep the relaxed thresholds with gating active. +func TestExploratoryPresetRespectsExplicitEnabled(t *testing.T) { + cfg := cleanr.ExampleConfig() + cfg.Reporting.TrendFile = "reports/history.yaml" + cfg.Reporting.TrendGates = cleanr.TrendGateConfig{ + Preset: "exploratory", + Enabled: boolPtr(true), + RequiredWindow: 2, + } + + data, err := cleanr.MarshalConfig(cfg, "json") + if err != nil { + t.Fatalf("marshal config: %v", err) + } + loaded, err := cleanr.LoadConfigData(data, "json") + if err != nil { + t.Fatalf("load config: %v", err) + } + + gates := loaded.Reporting.TrendGates + if !gates.EnabledValue() { + t.Fatalf("exploratory preset overrode explicit enabled: true: %+v", gates) + } + if gates.MaxDurationIncreasePct == nil || *gates.MaxDurationIncreasePct != 50 { + t.Fatalf("expected exploratory thresholds to apply, got %+v", gates) + } + + // Without an explicit enabled, exploratory stays non-blocking. + cfg.Reporting.TrendGates = cleanr.TrendGateConfig{Preset: "exploratory"} + data, err = cleanr.MarshalConfig(cfg, "json") + if err != nil { + t.Fatalf("marshal config: %v", err) + } + loaded, err = cleanr.LoadConfigData(data, "json") + if err != nil { + t.Fatalf("load config: %v", err) + } + if loaded.Reporting.TrendGates.EnabledValue() { + t.Fatalf("exploratory preset without explicit enabled must stay non-blocking") + } +} diff --git a/tests/config/validation_additional_test.go b/tests/config/validation_additional_test.go index 31e06a5..1d54391 100644 --- a/tests/config/validation_additional_test.go +++ b/tests/config/validation_additional_test.go @@ -358,7 +358,7 @@ func TestValidateConfigCoversProviderAndSuiteEdgeCases(t *testing.T) { name: "drift invalid normalized drift", mutate: func(cfg *cleanr.Config) { cfg.Suites.Drift.Enabled = true - cfg.Suites.Drift.MaxNormalizedDrift = 2 + cfg.Suites.Drift.MaxNormalizedDrift = float64Ptr(2) }, wantSub: "suites.drift.max_normalized_drift", }, @@ -366,7 +366,7 @@ func TestValidateConfigCoversProviderAndSuiteEdgeCases(t *testing.T) { name: "drift invalid consistency score", mutate: func(cfg *cleanr.Config) { cfg.Suites.Drift.Enabled = true - cfg.Suites.Drift.MinConsistencyScore = -0.1 + cfg.Suites.Drift.MinConsistencyScore = float64Ptr(-0.1) }, wantSub: "suites.drift.min_consistency_score", }, @@ -374,9 +374,9 @@ func TestValidateConfigCoversProviderAndSuiteEdgeCases(t *testing.T) { name: "drift invalid semantic thresholds", mutate: func(cfg *cleanr.Config) { cfg.Suites.Drift.Enabled = true - cfg.Suites.Drift.MaxSemanticDrift = 2 - cfg.Suites.Drift.MaxSemanticSnapshotDrift = -1 - cfg.Suites.Drift.MinSemanticConsistencyScore = 2 + cfg.Suites.Drift.MaxSemanticDrift = float64Ptr(2) + cfg.Suites.Drift.MaxSemanticSnapshotDrift = float64Ptr(-1) + cfg.Suites.Drift.MinSemanticConsistencyScore = float64Ptr(2) }, wantSub: "suites.drift.max_semantic_drift", }, @@ -427,7 +427,7 @@ func TestValidateConfigCoversProviderAndSuiteEdgeCases(t *testing.T) { name: "trend gates require trend file", mutate: func(cfg *cleanr.Config) { cfg.Reporting.TrendFile = "" - cfg.Reporting.TrendGates.Enabled = true + cfg.Reporting.TrendGates.Enabled = boolPtr(true) cfg.Reporting.TrendGates.RequiredWindow = 2 }, wantSub: "reporting.trend_file", @@ -436,7 +436,7 @@ func TestValidateConfigCoversProviderAndSuiteEdgeCases(t *testing.T) { name: "trend gates validate thresholds", mutate: func(cfg *cleanr.Config) { cfg.Reporting.TrendFile = "reports/history.yaml" - cfg.Reporting.TrendGates.Enabled = true + cfg.Reporting.TrendGates.Enabled = boolPtr(true) cfg.Reporting.TrendGates.RequiredWindow = 1 cfg.Reporting.TrendGates.MaxFailedCasesDelta = assertionIntPtr(-1) }, @@ -592,7 +592,7 @@ reporting: if cfg.Reporting.TrendGates.MaxDurationIncreasePct == nil || *cfg.Reporting.TrendGates.MaxDurationIncreasePct != 40 { t.Fatalf("expected duration override to survive load, got %+v", cfg.Reporting.TrendGates) } - if !cfg.Reporting.TrendGates.Enabled { + if !cfg.Reporting.TrendGates.EnabledValue() { t.Fatalf("expected moderate preset to enable gates, got %+v", cfg.Reporting.TrendGates) } if cfg.Reporting.TrendGates.MaxSemanticDriftDelta == nil || *cfg.Reporting.TrendGates.MaxSemanticDriftDelta != 0.08 { diff --git a/tests/engines/assertions_test.go b/tests/engines/assertions_test.go index b6c5d57..a46cba0 100644 --- a/tests/engines/assertions_test.go +++ b/tests/engines/assertions_test.go @@ -10,6 +10,8 @@ import ( func intPtr(v int) *int { return &v } +func float64Ptr(v float64) *float64 { return &v } + func TestSecurityEngineCoversScenarioAssertions(t *testing.T) { t.Parallel() diff --git a/tests/engines/engines_test.go b/tests/engines/engines_test.go index b1820da..154be27 100644 --- a/tests/engines/engines_test.go +++ b/tests/engines/engines_test.go @@ -217,8 +217,8 @@ func TestPromptChaosDriftAndTokenOptimizationCoverage(t *testing.T) { cfg.Suites.Drift.Enabled = true cfg.Suites.Drift.StableTags = []string{"stable"} cfg.Suites.Drift.Iterations = 2 - cfg.Suites.Drift.MaxNormalizedDrift = 0 - cfg.Suites.Drift.MinConsistencyScore = 0.99 + cfg.Suites.Drift.MaxNormalizedDrift = float64Ptr(0) + cfg.Suites.Drift.MinConsistencyScore = float64Ptr(0.99) cfg.Suites.TokenOptimization.Enabled = true cfg.Suites.TokenOptimization.MaxInputTokens = 1 cfg.Suites.TokenOptimization.MaxOutputTokens = 1 @@ -253,10 +253,10 @@ func TestPromptChaosDriftAndTokenOptimizationCoverage(t *testing.T) { cfg.Suites.Drift.Enabled = true cfg.Suites.Drift.StableTags = []string{"stable"} cfg.Suites.Drift.Iterations = 3 - cfg.Suites.Drift.MaxNormalizedDrift = 0.05 - cfg.Suites.Drift.MaxSemanticDrift = 0.25 - cfg.Suites.Drift.MinConsistencyScore = 0.5 - cfg.Suites.Drift.MinSemanticConsistencyScore = 0.75 + cfg.Suites.Drift.MaxNormalizedDrift = float64Ptr(0.05) + cfg.Suites.Drift.MaxSemanticDrift = float64Ptr(0.25) + cfg.Suites.Drift.MinConsistencyScore = float64Ptr(0.5) + cfg.Suites.Drift.MinSemanticConsistencyScore = float64Ptr(0.75) report := cleanr.NewRunner(cfg, target).Run(context.Background()) if !report.Passed { diff --git a/tests/engines/llm_judge_test.go b/tests/engines/llm_judge_test.go index 351d12c..de36bb6 100644 --- a/tests/engines/llm_judge_test.go +++ b/tests/engines/llm_judge_test.go @@ -62,8 +62,8 @@ func judgeConfig(judge core.LLMJudgeConfig, scenarios ...core.Scenario) core.Con if judge.Scale == 0 { judge.Scale = 5 } - if judge.MinScore == 0 { - judge.MinScore = 0.6 + if judge.MinScore == nil { + judge.MinScore = float64Ptr(0.6) } if judge.Samples == 0 { judge.Samples = 1 diff --git a/tests/generation_test.go b/tests/generation_test.go index 661a4db..2415501 100644 --- a/tests/generation_test.go +++ b/tests/generation_test.go @@ -44,7 +44,7 @@ func TestGenerateScenarioDatasetAdversarialModeTagsScenarios(t *testing.T) { }, Count: 1, OutputFile: "generated/cleanr.dataset.yaml", - RequireReview: true, + RequireReview: boolPtr(true), } dataset, err := cleanr.GenerateScenarioDataset(context.Background(), cfg, client) diff --git a/tests/helpers_test.go b/tests/helpers_test.go new file mode 100644 index 0000000..d8e681d --- /dev/null +++ b/tests/helpers_test.go @@ -0,0 +1,3 @@ +package tests + +func boolPtr(v bool) *bool { return &v } diff --git a/tests/runner/runner_test.go b/tests/runner/runner_test.go index c5cc006..558aa8b 100644 --- a/tests/runner/runner_test.go +++ b/tests/runner/runner_test.go @@ -109,6 +109,50 @@ func (failingAssertionTarget) Invoke(context.Context, cleanr.Request) cleanr.Res } } +func TestRunnerMarksInterruptedRunAsNotPassed(t *testing.T) { + cfg := cleanr.ExampleConfig() + cfg.Target.Name = "mock" + cfg.Target.ResponseField = "output.text" + + ctx, cancel := context.WithCancel(context.Background()) + cancel() + report := cleanr.NewRunner(cfg, mockTarget{}).Run(ctx) + + if !report.Interrupted { + t.Fatalf("expected interrupted report") + } + if report.Passed { + t.Fatalf("interrupted run must not report passed") + } + if len(report.SkippedSuites) == 0 { + t.Fatalf("expected skipped suite names on interrupted run") + } + if report.TotalSuites != 0 { + t.Fatalf("expected no executed suites, got %d", report.TotalSuites) + } + for _, suite := range report.Suites { + for _, c := range suite.Cases { + if c.Name == "" { + t.Fatalf("suite %s contains a phantom zero-value case", suite.Name) + } + } + } +} + +func TestRunnerCompletedRunIsNotInterrupted(t *testing.T) { + cfg := cleanr.ExampleConfig() + cfg.Target.Name = "mock" + cfg.Target.ResponseField = "output.text" + report := cleanr.NewRunner(cfg, mockTarget{}).Run(context.Background()) + + if report.Interrupted { + t.Fatalf("uninterrupted run must not be marked interrupted") + } + if len(report.SkippedSuites) != 0 { + t.Fatalf("unexpected skipped suites: %v", report.SkippedSuites) + } +} + func TestRunnerWithMockTarget(t *testing.T) { cfg := cleanr.ExampleConfig() cfg.Target.Name = "mock" @@ -132,7 +176,7 @@ func TestDriftSuitePassesStableResponses(t *testing.T) { cfg.Suites.TokenOptimization.Enabled = false cfg.Suites.Drift.Enabled = true cfg.Suites.Drift.Iterations = 3 - cfg.Suites.Drift.MaxNormalizedDrift = 0.01 + cfg.Suites.Drift.MaxNormalizedDrift = float64Ptr(0.01) report := cleanr.NewRunner(cfg, stableTarget{}).Run(context.Background()) if len(report.Suites) != 1 || report.Suites[0].Name != "drift" { @@ -530,11 +574,58 @@ func writeExecutableScript(t *testing.T, body string) string { return path } +func TestRunnerWASMPluginTimeoutInterruptsHangingGuest(t *testing.T) { + cfg := cleanr.ExampleConfig() + cfg.Suites.PromptInjection.Enabled = false + cfg.Suites.Security.Enabled = false + cfg.Suites.Load.Enabled = false + cfg.Suites.Chaos.Enabled = false + cfg.Suites.Drift.Enabled = false + cfg.Suites.ShadowState.Enabled = false + cfg.Suites.Provenance.Enabled = false + cfg.Suites.ClaimTrace.Enabled = false + cfg.Suites.ReleasePolicy.Enabled = false + cfg.Suites.MemorySafety.Enabled = false + cfg.Suites.TokenOptimization.Enabled = false + + dir := t.TempDir() + modulePath := buildWASMPluginFromSource(t, dir, `package main + +func main() { + for { + } +} +`) + cfg.ResolvedPlugins = []cleanr.PluginManifest{{ + Name: "org-plugin", + Source: filepath.Join(dir, "plugin.yaml"), + BaseDir: dir, + Suites: []cleanr.PluginSuite{{ + Name: "wasm-hang", + Command: filepath.Base(modulePath), + TimeoutMS: 1500, + Runtime: cleanr.PluginRuntimeConfig{Backend: "wasm"}, + }}, + }} + + done := make(chan cleanr.Report, 1) + go func() { + done <- cleanr.NewRunner(cfg, stableTarget{}).Run(context.Background()) + }() + select { + case report := <-done: + if report.Passed { + t.Fatalf("expected hanging wasm plugin suite to fail: %+v", report) + } + case <-time.After(30 * time.Second): + t.Fatal("wasm plugin ignored its timeout: run still hanging after 30s") + } +} + func buildWASMPlugin(t *testing.T, dir string) string { t.Helper() - sourcePath := filepath.Join(dir, "main.go") - if err := os.WriteFile(sourcePath, []byte(`package main + return buildWASMPluginFromSource(t, dir, `package main import ( "encoding/json" @@ -557,7 +648,14 @@ func main() { } _ = json.NewEncoder(os.Stdout).Encode(out) } -`), 0o644); err != nil { +`) +} + +func buildWASMPluginFromSource(t *testing.T, dir, source string) string { + t.Helper() + + sourcePath := filepath.Join(dir, "main.go") + if err := os.WriteFile(sourcePath, []byte(source), 0o644); err != nil { t.Fatalf("write wasm source: %v", err) } @@ -670,8 +768,8 @@ func TestDriftSuitePassesMatchingBaseline(t *testing.T) { cfg.Suites.Drift.Enabled = true cfg.Suites.Drift.Iterations = 3 cfg.Suites.Drift.StableTags = []string{"stable"} - cfg.Suites.Drift.MaxNormalizedDrift = 0.01 - cfg.Suites.Drift.MaxSnapshotDrift = 0.05 + cfg.Suites.Drift.MaxNormalizedDrift = float64Ptr(0.01) + cfg.Suites.Drift.MaxSnapshotDrift = float64Ptr(0.05) baselinePath := t.TempDir() + "/snapshots.yaml" err := cleanr.WriteSnapshotFile(baselinePath, cleanr.SnapshotFile{ @@ -724,12 +822,12 @@ func TestDriftSuitePassesSemanticParaphraseBaseline(t *testing.T) { cfg.Suites.Drift.Enabled = true cfg.Suites.Drift.Iterations = 3 cfg.Suites.Drift.StableTags = []string{"stable"} - cfg.Suites.Drift.MaxNormalizedDrift = 0.05 - cfg.Suites.Drift.MaxSemanticDrift = 0.25 - cfg.Suites.Drift.MaxSnapshotDrift = 0.05 - cfg.Suites.Drift.MaxSemanticSnapshotDrift = 0.25 - cfg.Suites.Drift.MinConsistencyScore = 0.6 - cfg.Suites.Drift.MinSemanticConsistencyScore = 0.75 + cfg.Suites.Drift.MaxNormalizedDrift = float64Ptr(0.05) + cfg.Suites.Drift.MaxSemanticDrift = float64Ptr(0.25) + cfg.Suites.Drift.MaxSnapshotDrift = float64Ptr(0.05) + cfg.Suites.Drift.MaxSemanticSnapshotDrift = float64Ptr(0.25) + cfg.Suites.Drift.MinConsistencyScore = float64Ptr(0.6) + cfg.Suites.Drift.MinSemanticConsistencyScore = float64Ptr(0.75) baselinePath := t.TempDir() + "/snapshots.yaml" err := cleanr.WriteSnapshotFile(baselinePath, cleanr.SnapshotFile{ @@ -760,14 +858,14 @@ func TestDriftSuitePassesSemanticParaphraseBaseline(t *testing.T) { if !ok { t.Fatalf("expected numeric baseline_drift, got %+v", details["baseline_drift"]) } - if baselineDrift <= cfg.Suites.Drift.MaxSnapshotDrift { + if baselineDrift <= cfg.Suites.Drift.MaxSnapshotDriftValue() { t.Fatalf("expected lexical baseline drift to exceed threshold, got %+v", details) } baselineSemanticDrift, ok := details["baseline_semantic_drift"].(float64) if !ok { t.Fatalf("expected numeric baseline_semantic_drift, got %+v", details["baseline_semantic_drift"]) } - if baselineSemanticDrift >= cfg.Suites.Drift.MaxSemanticSnapshotDrift { + if baselineSemanticDrift >= cfg.Suites.Drift.MaxSemanticSnapshotDriftValue() { t.Fatalf("expected semantic baseline drift to remain within threshold, got %+v", details) } if _, ok := details["baseline_lexical_note"]; !ok { @@ -791,8 +889,8 @@ func TestDriftSuiteFailsOnBaselineRegression(t *testing.T) { cfg.Suites.Drift.Enabled = true cfg.Suites.Drift.Iterations = 3 cfg.Suites.Drift.StableTags = []string{"stable"} - cfg.Suites.Drift.MaxNormalizedDrift = 0.01 - cfg.Suites.Drift.MaxSnapshotDrift = 0.05 + cfg.Suites.Drift.MaxNormalizedDrift = float64Ptr(0.01) + cfg.Suites.Drift.MaxSnapshotDrift = float64Ptr(0.05) baselinePath := t.TempDir() + "/snapshots.yaml" err := cleanr.WriteSnapshotFile(baselinePath, cleanr.SnapshotFile{ @@ -824,6 +922,10 @@ func TestDriftSuiteFailsOnBaselineRegression(t *testing.T) { } } +func float64Ptr(v float64) *float64 { + return &v +} + func intPtr(v int) *int { return &v } diff --git a/tests/trends/helpers_test.go b/tests/trends/helpers_test.go new file mode 100644 index 0000000..11c8f5b --- /dev/null +++ b/tests/trends/helpers_test.go @@ -0,0 +1,3 @@ +package tests + +func float64Ptr(v float64) *float64 { return &v } diff --git a/tests/trends/trends_test.go b/tests/trends/trends_test.go index 60df2c0..1d76ace 100644 --- a/tests/trends/trends_test.go +++ b/tests/trends/trends_test.go @@ -434,9 +434,9 @@ func cleanTrendConfig() cleanr.Config { cfg.Suites.TokenOptimization.Enabled = false cfg.Suites.Drift.Enabled = true cfg.Suites.Drift.Iterations = 2 - cfg.Suites.Drift.MaxNormalizedDrift = 1 - cfg.Suites.Drift.MaxSemanticDrift = 1 - cfg.Suites.Drift.MinConsistencyScore = 0 - cfg.Suites.Drift.MinSemanticConsistencyScore = 0 + cfg.Suites.Drift.MaxNormalizedDrift = float64Ptr(1) + cfg.Suites.Drift.MaxSemanticDrift = float64Ptr(1) + cfg.Suites.Drift.MinConsistencyScore = float64Ptr(0) + cfg.Suites.Drift.MinSemanticConsistencyScore = float64Ptr(0) return cfg } From 508c96e4d6e7f4e6bdfe9e1212590f18b3eb2d4b Mon Sep 17 00:00:00 2001 From: Alex Wilkerson John Date: Thu, 2 Jul 2026 13:42:15 -0400 Subject: [PATCH 07/13] test: run the suite with -race and -shuffle in CI and cleanr-dev The race detector was never run anywhere, hiding a real data race: tests/cli/snapshot_test.go swapped the process-global http.DefaultTransport while marked t.Parallel(), racing every parallel test that makes an HTTP call. The swap test is now sequential (parallel tests never overlap sequential ones, which is what makes the pattern safe), and the CLI-adapter helper-process timeout is raised to 10s since race instrumentation makes the re-exec'd binary slow to start. Co-Authored-By: Claude Fable 5 --- .github/workflows/ci.yml | 2 +- internal/devtools/check.go | 7 +++++-- tests/adapters/cli_adapter_test.go | 6 ++++-- tests/cli/snapshot_test.go | 6 ++++-- 4 files changed, 14 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a00c05f..fe4403d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -170,7 +170,7 @@ jobs: cache: true - name: Run test suite - run: go test ./... + run: go test -race -shuffle=on ./... build: name: build diff --git a/internal/devtools/check.go b/internal/devtools/check.go index d9b5acc..f2e8a69 100644 --- a/internal/devtools/check.go +++ b/internal/devtools/check.go @@ -13,10 +13,13 @@ func (r Runner) Lint(ctx context.Context) error { } func (r Runner) Test(ctx context.Context) error { - if _, err := fmt.Fprintln(r.Stdout, "running go test"); err != nil { + if _, err := fmt.Fprintln(r.Stdout, "running go test -race"); err != nil { return err } - return r.runGoTestFiltered(ctx, "./...") + // -race so concurrency bugs in the engines and test suite fail the gate + // instead of shipping; -shuffle surfaces hidden inter-test ordering + // dependencies. + return r.runGoTestFiltered(ctx, "-race", "-shuffle=on", "./...") } func (r Runner) TestReviewUI(ctx context.Context) error { diff --git a/tests/adapters/cli_adapter_test.go b/tests/adapters/cli_adapter_test.go index 6cbe5a4..334cea5 100644 --- a/tests/adapters/cli_adapter_test.go +++ b/tests/adapters/cli_adapter_test.go @@ -23,10 +23,12 @@ func TestCLITargetInvokeCapturesStdoutAndExtractsJSON(t *testing.T) { }, }) + // A generous timeout: the helper is this test binary re-exec'd, which under + // -race instrumentation can take well over a second to start. resp := target.Invoke(context.Background(), cleanr.BuildScenarioRequest(cleanr.Scenario{ Name: "cli-target", Input: "hello", - }, time.Second)) + }, 10*time.Second)) if resp.Err != nil || resp.ExtractError != nil { t.Fatalf("unexpected cli response errors: err=%v extract=%v", resp.Err, resp.ExtractError) } @@ -49,7 +51,7 @@ func TestCLITargetInvokeCapturesExitCodeAndStderr(t *testing.T) { }, }) - resp := target.Invoke(context.Background(), cleanr.Request{Timeout: time.Second}) + resp := target.Invoke(context.Background(), cleanr.Request{Timeout: 10 * time.Second}) if resp.Err == nil { t.Fatal("expected cli error") } diff --git a/tests/cli/snapshot_test.go b/tests/cli/snapshot_test.go index fafd744..4f04dba 100644 --- a/tests/cli/snapshot_test.go +++ b/tests/cli/snapshot_test.go @@ -35,8 +35,10 @@ func snapshotJSONResponse(t *testing.T, statusCode int, body map[string]any) *ht } func TestCLISnapshotCommandWritesBaseline(t *testing.T) { - t.Parallel() - + // Not parallel: this test swaps the process-global http.DefaultTransport, + // which parallel tests read through clients with a nil Transport. Sequential + // tests never overlap parallel ones, so the swap is race-free only without + // t.Parallel(). original := http.DefaultTransport http.DefaultTransport = snapshotRoundTripper(func(req *http.Request) (*http.Response, error) { return snapshotJSONResponse(t, http.StatusOK, map[string]any{ From d1e0ae7d2f018b9e2ad882a9ed66d184161e6edc Mon Sep 17 00:00:00 2001 From: Alex Wilkerson John Date: Thu, 2 Jul 2026 13:42:15 -0400 Subject: [PATCH 08/13] docs: capture testing and local-dev knowledge for AI assistants Co-Authored-By: Claude Fable 5 --- .claude/knowledge/local-dev-setup.md | 6 ++++++ .claude/knowledge/testing-patterns.md | 9 +++++++++ CLAUDE.md | 9 +++++++++ 3 files changed, 24 insertions(+) create mode 100644 .claude/knowledge/local-dev-setup.md create mode 100644 .claude/knowledge/testing-patterns.md create mode 100644 CLAUDE.md diff --git a/.claude/knowledge/local-dev-setup.md b/.claude/knowledge/local-dev-setup.md new file mode 100644 index 0000000..3a194a0 --- /dev/null +++ b/.claude/knowledge/local-dev-setup.md @@ -0,0 +1,6 @@ +# Local Development Setup + +How to set up, run, and work with this project locally. Non-obvious dependencies, environment config, common setup issues. + +- The shell's `GOROOT` points at a nonexistent `/Users/alex/apps/go`; Go commands fail with "cannot find GOROOT directory" unless you `export GOROOT=/opt/homebrew/opt/go/libexec` (Homebrew go1.26). The Makefile works around the same issue with `env -u GOROOT` in some targets. +- `golangci-lint run ./...` reports ~69 known issues (errcheck, staticcheck, unused) — it is configured in `.golangci.yml` but deliberately not wired into CI or `make lint` (which is only `go vet`), so don't assume a clean baseline. diff --git a/.claude/knowledge/testing-patterns.md b/.claude/knowledge/testing-patterns.md new file mode 100644 index 0000000..45ba9d3 --- /dev/null +++ b/.claude/knowledge/testing-patterns.md @@ -0,0 +1,9 @@ +# Testing Patterns + +Testing strategies, test infrastructure quirks, how to run/debug specific test suites, mocking conventions. + +- CI and `cleanr-dev test` run `go test -race -shuffle=on ./...`. Keep the suite race-clean: a test that mutates process-global state (e.g. `http.DefaultTransport`) must NOT call `t.Parallel()` — sequential tests never overlap parallel ones, which is the only thing making the transport-swap pattern safe. +- Tests in `tests/cli` mock HTTP by swapping `http.DefaultTransport` (see `tests/cli/cli_test.go`, `snapshot_test.go`). Runner-built clients have a nil Transport and read the global at request time, so any parallel test doing real HTTP races with a swap. +- Tests that re-exec the test binary as a helper process (`tests/adapters/cli_adapter_test.go`) need generous timeouts (10s): race instrumentation makes the re-exec'd binary slow to start. +- Repo layout check (enforced by `cleanr-dev ci`): all Go test files must live under `tests/`, not next to the source packages. +- Config threshold fields with meaningful zero values (drift thresholds, `min_score`, `require_review`, trend-gate `enabled`) are pointer-typed with `*Value()` accessors that supply defaults; in tests, set them with the per-package `float64Ptr`/`boolPtr` helpers and read them through the accessors. diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..edd3d09 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,9 @@ +# cleanr + +## Available Context + +Additional context is available in the files below. Consult the relevant file when working in a related area — see each description for scope. + +- `.claude/knowledge/architecture-boundaries.md` — Key architectural decisions, service boundaries, data flow, integration points, and why things are the way they are. +- `.claude/knowledge/testing-patterns.md` — Testing strategies, test infrastructure quirks, how to run/debug specific test suites, mocking conventions. +- `.claude/knowledge/local-dev-setup.md` — How to set up, run, and work with this project locally. Non-obvious dependencies, environment config, common setup issues. From 03754b7be7c929da40100cb7f112b26544f8be9b Mon Sep 17 00:00:00 2001 From: Alex Wilkerson John Date: Thu, 2 Jul 2026 13:44:57 -0400 Subject: [PATCH 09/13] chore: keep AI assistant context files out of the repo Co-Authored-By: Claude Fable 5 --- .claude/knowledge/local-dev-setup.md | 6 ------ .claude/knowledge/testing-patterns.md | 9 --------- CLAUDE.md | 9 --------- 3 files changed, 24 deletions(-) delete mode 100644 .claude/knowledge/local-dev-setup.md delete mode 100644 .claude/knowledge/testing-patterns.md delete mode 100644 CLAUDE.md diff --git a/.claude/knowledge/local-dev-setup.md b/.claude/knowledge/local-dev-setup.md deleted file mode 100644 index 3a194a0..0000000 --- a/.claude/knowledge/local-dev-setup.md +++ /dev/null @@ -1,6 +0,0 @@ -# Local Development Setup - -How to set up, run, and work with this project locally. Non-obvious dependencies, environment config, common setup issues. - -- The shell's `GOROOT` points at a nonexistent `/Users/alex/apps/go`; Go commands fail with "cannot find GOROOT directory" unless you `export GOROOT=/opt/homebrew/opt/go/libexec` (Homebrew go1.26). The Makefile works around the same issue with `env -u GOROOT` in some targets. -- `golangci-lint run ./...` reports ~69 known issues (errcheck, staticcheck, unused) — it is configured in `.golangci.yml` but deliberately not wired into CI or `make lint` (which is only `go vet`), so don't assume a clean baseline. diff --git a/.claude/knowledge/testing-patterns.md b/.claude/knowledge/testing-patterns.md deleted file mode 100644 index 45ba9d3..0000000 --- a/.claude/knowledge/testing-patterns.md +++ /dev/null @@ -1,9 +0,0 @@ -# Testing Patterns - -Testing strategies, test infrastructure quirks, how to run/debug specific test suites, mocking conventions. - -- CI and `cleanr-dev test` run `go test -race -shuffle=on ./...`. Keep the suite race-clean: a test that mutates process-global state (e.g. `http.DefaultTransport`) must NOT call `t.Parallel()` — sequential tests never overlap parallel ones, which is the only thing making the transport-swap pattern safe. -- Tests in `tests/cli` mock HTTP by swapping `http.DefaultTransport` (see `tests/cli/cli_test.go`, `snapshot_test.go`). Runner-built clients have a nil Transport and read the global at request time, so any parallel test doing real HTTP races with a swap. -- Tests that re-exec the test binary as a helper process (`tests/adapters/cli_adapter_test.go`) need generous timeouts (10s): race instrumentation makes the re-exec'd binary slow to start. -- Repo layout check (enforced by `cleanr-dev ci`): all Go test files must live under `tests/`, not next to the source packages. -- Config threshold fields with meaningful zero values (drift thresholds, `min_score`, `require_review`, trend-gate `enabled`) are pointer-typed with `*Value()` accessors that supply defaults; in tests, set them with the per-package `float64Ptr`/`boolPtr` helpers and read them through the accessors. diff --git a/CLAUDE.md b/CLAUDE.md deleted file mode 100644 index edd3d09..0000000 --- a/CLAUDE.md +++ /dev/null @@ -1,9 +0,0 @@ -# cleanr - -## Available Context - -Additional context is available in the files below. Consult the relevant file when working in a related area — see each description for scope. - -- `.claude/knowledge/architecture-boundaries.md` — Key architectural decisions, service boundaries, data flow, integration points, and why things are the way they are. -- `.claude/knowledge/testing-patterns.md` — Testing strategies, test infrastructure quirks, how to run/debug specific test suites, mocking conventions. -- `.claude/knowledge/local-dev-setup.md` — How to set up, run, and work with this project locally. Non-obvious dependencies, environment config, common setup issues. From d2a2550a7dec9732224497c1717912a2f2123c09 Mon Sep 17 00:00:00 2001 From: Alex Wilkerson John Date: Thu, 2 Jul 2026 13:55:27 -0400 Subject: [PATCH 10/13] fix(adapters): retry MCP initialization after transient failure MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The initialize handshake ran under a sync.Once that stored the first error forever: if the first scenario's deadline expired mid-handshake or the server was briefly restarting, every remaining scenario in the run failed instantly with the stale error. A failed attempt is no longer cached — the next Invoke retries, with a mutex preserving Once's serialization of concurrent first invokes. Co-Authored-By: Claude Fable 5 --- cleanr/adapters/mcp.go | 65 ++++++++++++++----------- tests/adapters/mcp_adapter_test.go | 76 ++++++++++++++++++++++++++++++ 2 files changed, 113 insertions(+), 28 deletions(-) diff --git a/cleanr/adapters/mcp.go b/cleanr/adapters/mcp.go index 4889647..d40dce8 100644 --- a/cleanr/adapters/mcp.go +++ b/cleanr/adapters/mcp.go @@ -17,8 +17,8 @@ import ( type MCP struct { cfg core.TargetConfig client *http.Client - initOnce sync.Once - initErr error + initMu sync.Mutex + initialized bool requestIDSeq int64 mu sync.Mutex } @@ -72,34 +72,43 @@ func (t *MCP) Invoke(ctx context.Context, req core.Request) core.Response { } } +// initialize performs the MCP handshake at most once per adapter. A failed +// attempt is NOT cached: the next Invoke retries, so a transient failure (a +// restarting server, the first scenario's deadline expiring mid-handshake) +// fails that one scenario instead of poisoning every remaining scenario in +// the run. The lock serializes concurrent first invokes, matching the +// blocking behavior sync.Once had. func (t *MCP) initialize(ctx context.Context) error { - t.initOnce.Do(func() { - initReq := map[string]any{ - "jsonrpc": "2.0", - "id": t.nextRequestID(), - "method": "initialize", - "params": map[string]any{ - "protocolVersion": "2025-06-18", - "capabilities": map[string]any{}, - "clientInfo": map[string]any{ - "name": "cleanr", - "version": "v1alpha1", - }, + t.initMu.Lock() + defer t.initMu.Unlock() + if t.initialized { + return nil + } + initReq := map[string]any{ + "jsonrpc": "2.0", + "id": t.nextRequestID(), + "method": "initialize", + "params": map[string]any{ + "protocolVersion": "2025-06-18", + "capabilities": map[string]any{}, + "clientInfo": map[string]any{ + "name": "cleanr", + "version": "v1alpha1", }, - } - if _, _, err := t.postJSONRPC(ctx, initReq); err != nil { - t.initErr = fmt.Errorf("initialize mcp target: %w", err) - return - } - notify := map[string]any{ - "jsonrpc": "2.0", - "method": "notifications/initialized", - } - if _, _, err := t.postJSONRPC(ctx, notify); err != nil { - t.initErr = fmt.Errorf("notify initialized mcp target: %w", err) - } - }) - return t.initErr + }, + } + if _, _, err := t.postJSONRPC(ctx, initReq); err != nil { + return fmt.Errorf("initialize mcp target: %w", err) + } + notify := map[string]any{ + "jsonrpc": "2.0", + "method": "notifications/initialized", + } + if _, _, err := t.postJSONRPC(ctx, notify); err != nil { + return fmt.Errorf("notify initialized mcp target: %w", err) + } + t.initialized = true + return nil } func (t *MCP) nextRequestID() int64 { diff --git a/tests/adapters/mcp_adapter_test.go b/tests/adapters/mcp_adapter_test.go index 5f4d044..c667a8f 100644 --- a/tests/adapters/mcp_adapter_test.go +++ b/tests/adapters/mcp_adapter_test.go @@ -10,6 +10,82 @@ import ( "github.com/devr-tools/cleanr/cleanr" ) +// A transient initialize failure must not be cached: the next Invoke retries +// the handshake instead of failing every remaining scenario in the run. +func TestMCPTargetRetriesInitializeAfterTransientFailure(t *testing.T) { + t.Parallel() + + initAttempts := 0 + client := &http.Client{Transport: roundTripperFunc(func(r *http.Request) (*http.Response, error) { + var body map[string]any + if err := json.NewDecoder(r.Body).Decode(&body); err != nil { + t.Fatalf("decode request: %v", err) + } + switch body["method"] { + case "initialize": + initAttempts++ + if initAttempts == 1 { + return jsonResponse(t, http.StatusServiceUnavailable, map[string]any{ + "error": "restarting", + }), nil + } + return jsonResponse(t, http.StatusOK, map[string]any{ + "jsonrpc": "2.0", + "id": body["id"], + "result": map[string]any{"protocolVersion": "2025-06-18"}, + }), nil + case "notifications/initialized": + return &http.Response{ + StatusCode: http.StatusOK, + Status: "200 OK", + Body: http.NoBody, + Header: http.Header{"Content-Type": []string{"application/json"}}, + }, nil + case "tools/call": + return jsonResponse(t, http.StatusOK, map[string]any{ + "jsonrpc": "2.0", + "id": body["id"], + "result": map[string]any{ + "content": []any{map[string]any{"type": "text", "text": "ok"}}, + }, + }), nil + default: + t.Fatalf("unexpected method: %#v", body["method"]) + } + return nil, nil + })} + + mcpTarget := cleanr.NewMCPTarget(cleanr.TargetConfig{ + Type: "mcp", + MCP: cleanr.MCPConfig{ + URL: "https://mcp.test/rpc", + Tool: "lookup_customer", + }, + }, client) + + first := mcpTarget.Invoke(context.Background(), cleanr.BuildScenarioRequest(cleanr.Scenario{ + Name: "first", + Input: "hello", + }, 2*time.Second)) + if first.Err == nil { + t.Fatal("expected first invoke to fail while the server is restarting") + } + + second := mcpTarget.Invoke(context.Background(), cleanr.BuildScenarioRequest(cleanr.Scenario{ + Name: "second", + Input: "hello", + }, 2*time.Second)) + if second.Err != nil { + t.Fatalf("expected second invoke to retry initialization and succeed, got %v", second.Err) + } + if second.Text != "ok" { + t.Fatalf("unexpected mcp response text: %q", second.Text) + } + if initAttempts != 2 { + t.Fatalf("expected initialize to be retried once, got %d attempts", initAttempts) + } +} + func TestMCPTargetInvokesToolOverJSONRPC(t *testing.T) { t.Parallel() From ed7b6743e6bc821038dbecf6fffb026ea2cf03cd Mon Sep 17 00:00:00 2001 From: Alex Wilkerson John Date: Thu, 2 Jul 2026 13:55:27 -0400 Subject: [PATCH 11/13] fix(adapters): safe retry semantics in doWithRetry MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two fixes to the shared HTTP retry loop (used by the HTTP, OpenAI, and Anthropic adapters): - When a 429/503's Retry-After exceeded the remaining context budget, the response body was closed before being returned, so callers got 'read on closed response body' instead of the actual rate-limit response. The body is now closed only after committing to another attempt — and drained first so the keep-alive connection is reused instead of forcing a new TCP+TLS handshake against an already-degraded server. - Transport errors were retried for every method, but a connection can die after a request was fully delivered — replaying a POST there risks duplicate writes or double charges. Transport-error retries are now limited to idempotent methods (GET/HEAD/OPTIONS); 429/503 responses still retry for all methods since the server explicitly rejected the request. Co-Authored-By: Claude Fable 5 --- cleanr/adapters/http.go | 48 +++++++++--- tests/adapters/retry_test.go | 137 +++++++++++++++++++++++++++++++++++ 2 files changed, 173 insertions(+), 12 deletions(-) create mode 100644 tests/adapters/retry_test.go diff --git a/cleanr/adapters/http.go b/cleanr/adapters/http.go index 952df33..5d518ff 100644 --- a/cleanr/adapters/http.go +++ b/cleanr/adapters/http.go @@ -30,12 +30,16 @@ func NewHTTP(cfg core.TargetConfig, client *http.Client) *HTTP { // (initial attempt plus retries). const maxRequestAttempts = 3 -// doWithRetry issues the request built by build for each attempt, retrying on -// transport errors and HTTP 429/503 with exponential backoff plus jitter. It -// honors a Retry-After header when present and never sleeps past the request -// context deadline: if the next backoff would exceed the remaining budget it -// returns the last result instead of waiting. build must return a fresh -// *http.Request per call so the body can be re-read on retry. +// doWithRetry issues the request built by build for each attempt, retrying +// HTTP 429/503 (for any method — the server rejected the request, so a replay +// cannot double-apply it) and transport errors (for idempotent methods only — +// a connection can die after the request was fully delivered, and replaying a +// POST there risks duplicate writes or charges) with exponential backoff plus +// jitter. It honors a Retry-After header when present and never sleeps past +// the request context deadline: if the next backoff would exceed the +// remaining budget it returns the last result, with its body still readable. +// build must return a fresh *http.Request per call so the body can be re-read +// on retry. func doWithRetry(ctx context.Context, client *http.Client, build func() (*http.Request, error)) (*http.Response, error) { var lastResp *http.Response var lastErr error @@ -47,21 +51,29 @@ func doWithRetry(ctx context.Context, client *http.Client, build func() (*http.R resp, err := client.Do(httpReq) lastResp, lastErr = resp, err - if err == nil && !retryableStatus(resp.StatusCode) { + if err != nil { + if !idempotentMethod(httpReq.Method) { + return nil, err + } + } else if !retryableStatus(resp.StatusCode) { return resp, nil } if attempt == maxRequestAttempts-1 { break } - // Drain and close the retryable response before the next attempt. wait := backoffDelay(attempt, resp) - if err == nil && resp != nil { - resp.Body.Close() - } if !sleepWithin(ctx, wait) { - // Not enough budget left to retry; surface the last result. + // Not enough budget left to retry; surface the last result. The + // body is intentionally NOT closed here — the caller reads it. return lastResp, lastErr } + // Committed to another attempt: drain and close the retryable + // response so its keep-alive connection can be reused instead of + // forcing a new TCP+TLS handshake against an already-degraded server. + if err == nil && resp != nil { + _, _ = io.Copy(io.Discard, io.LimitReader(resp.Body, 256<<10)) + _ = resp.Body.Close() + } } return lastResp, lastErr } @@ -70,6 +82,18 @@ func retryableStatus(status int) bool { return status == http.StatusTooManyRequests || status == http.StatusServiceUnavailable } +// idempotentMethod reports whether a request may be safely replayed after a +// transport error, when it is unknowable whether the server already processed +// the request. +func idempotentMethod(method string) bool { + switch strings.ToUpper(method) { + case http.MethodGet, http.MethodHead, http.MethodOptions: + return true + default: + return false + } +} + func backoffDelay(attempt int, resp *http.Response) time.Duration { if resp != nil { if ra := parseRetryAfter(resp.Header.Get("Retry-After")); ra > 0 { diff --git a/tests/adapters/retry_test.go b/tests/adapters/retry_test.go new file mode 100644 index 0000000..1249621 --- /dev/null +++ b/tests/adapters/retry_test.go @@ -0,0 +1,137 @@ +package tests + +import ( + "context" + "errors" + "net/http" + "strings" + "testing" + "time" + + "github.com/devr-tools/cleanr/cleanr" +) + +// When a 429's Retry-After exceeds the remaining context budget, the response +// is returned instead of retried — and its body must still be readable. It +// used to be closed before being handed back, so callers saw "read on closed +// response body" instead of the actual rate-limit response. +func TestHTTPTargetReturnsReadableBodyWhenRetryBudgetExhausted(t *testing.T) { + t.Parallel() + + client := &http.Client{Transport: roundTripperFunc(func(r *http.Request) (*http.Response, error) { + resp := jsonResponse(t, http.StatusTooManyRequests, map[string]any{ + "error": map[string]any{"message": "rate limited"}, + }) + resp.Header.Set("Retry-After", "3600") + return resp, nil + })} + + target := cleanr.NewHTTPTarget(cleanr.TargetConfig{ + Type: "http", + URL: "https://api.test/v1", + Method: http.MethodPost, + }, client) + + resp := target.Invoke(context.Background(), cleanr.BuildScenarioRequest(cleanr.Scenario{ + Name: "rate-limited", + Input: "hello", + }, time.Second)) + + if resp.Err != nil { + t.Fatalf("expected readable rate-limit response, got error: %v", resp.Err) + } + if resp.StatusCode != http.StatusTooManyRequests { + t.Fatalf("expected 429 status, got %d", resp.StatusCode) + } + if !strings.Contains(string(resp.Body), "rate limited") { + t.Fatalf("expected rate-limit body to be readable, got %q", string(resp.Body)) + } +} + +// A transport error on a POST must not be retried: the connection can die +// after the request was fully delivered, and replaying it risks duplicate +// writes or charges. +func TestHTTPTargetDoesNotRetryPOSTOnTransportError(t *testing.T) { + t.Parallel() + + attempts := 0 + client := &http.Client{Transport: roundTripperFunc(func(r *http.Request) (*http.Response, error) { + attempts++ + return nil, errors.New("connection reset by peer") + })} + + target := cleanr.NewHTTPTarget(cleanr.TargetConfig{ + Type: "http", + URL: "https://api.test/v1", + Method: http.MethodPost, + }, client) + + resp := target.Invoke(context.Background(), cleanr.BuildScenarioRequest(cleanr.Scenario{ + Name: "post-reset", + Input: "hello", + }, time.Second)) + + if resp.Err == nil { + t.Fatal("expected transport error to surface") + } + if attempts != 1 { + t.Fatalf("POST must not be replayed after a transport error, got %d attempts", attempts) + } +} + +// Idempotent requests keep retrying transport errors, and 429/503 responses +// keep retrying for every method (the server rejected the request, so a +// replay cannot double-apply it). +func TestHTTPTargetRetriesIdempotentTransportErrorsAndRetryableStatuses(t *testing.T) { + t.Parallel() + + getAttempts := 0 + getClient := &http.Client{Transport: roundTripperFunc(func(r *http.Request) (*http.Response, error) { + getAttempts++ + if getAttempts < 3 { + return nil, errors.New("connection refused") + } + return jsonResponse(t, http.StatusOK, map[string]any{"output": "recovered"}), nil + })} + getTarget := cleanr.NewHTTPTarget(cleanr.TargetConfig{ + Type: "http", + URL: "https://api.test/v1", + Method: http.MethodGet, + }, getClient) + resp := getTarget.Invoke(context.Background(), cleanr.BuildScenarioRequest(cleanr.Scenario{ + Name: "get-flaky", + Input: "hello", + }, 10*time.Second)) + if resp.Err != nil { + t.Fatalf("expected GET to retry transport errors and succeed, got %v", resp.Err) + } + if getAttempts != 3 { + t.Fatalf("expected 3 GET attempts, got %d", getAttempts) + } + + postAttempts := 0 + postClient := &http.Client{Transport: roundTripperFunc(func(r *http.Request) (*http.Response, error) { + postAttempts++ + if postAttempts < 2 { + resp := jsonResponse(t, http.StatusServiceUnavailable, map[string]any{"error": "overloaded"}) + resp.Header.Set("Retry-After", "0") + return resp, nil + } + return jsonResponse(t, http.StatusOK, map[string]any{"output": "recovered"}), nil + })} + postTarget := cleanr.NewHTTPTarget(cleanr.TargetConfig{ + Type: "http", + URL: "https://api.test/v1", + Method: http.MethodPost, + }, postClient) + resp = postTarget.Invoke(context.Background(), cleanr.BuildScenarioRequest(cleanr.Scenario{ + Name: "post-503", + Input: "hello", + }, 10*time.Second)) + if resp.Err != nil || resp.StatusCode != http.StatusOK { + t.Fatalf("expected POST to retry 503 and succeed, got err=%v status=%d", resp.Err, resp.StatusCode) + } + if postAttempts != 2 { + t.Fatalf("expected 2 POST attempts on 503, got %d", postAttempts) + } +} From 4c3fbef6b57a79d1d2ef69327db16bfd6a5a63db Mon Sep 17 00:00:00 2001 From: Alex Wilkerson John Date: Thu, 2 Jul 2026 13:55:39 -0400 Subject: [PATCH 12/13] fix(mcpserver): JSON-RPC id round-trip, final-line handling, isError tool results Three protocol-compliance fixes for the stdio MCP server: - Request ids are kept as raw JSON so id: 0 and id: "" round-trip exactly (omitempty previously dropped them, breaking client correlation), and parse errors now carry the spec-required id: null. - The final request of a session without a trailing newline (common for one-shot piped clients) is processed at EOF instead of silently dropped. - Ordinary tool execution failures return isError results the calling model can read and self-correct from; unknown tools are -32602 and contained panics remain -32603, instead of every failure surfacing as a blanket internal error that some clients treat as a broken server. Co-Authored-By: Claude Fable 5 --- internal/mcpserver/protocol.go | 19 +++-- internal/mcpserver/server.go | 60 ++++++++----- internal/mcpserver/tools/registry.go | 9 +- tests/mcp/protocol_edges_test.go | 121 +++++++++++++++++++++++++++ 4 files changed, 180 insertions(+), 29 deletions(-) create mode 100644 tests/mcp/protocol_edges_test.go diff --git a/internal/mcpserver/protocol.go b/internal/mcpserver/protocol.go index 3942ebe..88c1747 100644 --- a/internal/mcpserver/protocol.go +++ b/internal/mcpserver/protocol.go @@ -17,18 +17,23 @@ const ( jsonRPCServerError = -32000 ) +// The request/response id is kept as raw JSON so it round-trips exactly: +// decoding into `any` and re-encoding with omitempty dropped falsy ids +// (id: 0, id: ""), leaving clients unable to correlate the response. A nil +// RawMessage marshals as null, which is what JSON-RPC requires on responses +// to unparseable requests. type requestEnvelope struct { JSONRPC string `json:"jsonrpc"` - ID any `json:"id,omitempty"` + ID json.RawMessage `json:"id,omitempty"` Method string `json:"method,omitempty"` Params json.RawMessage `json:"params,omitempty"` } type responseEnvelope struct { - JSONRPC string `json:"jsonrpc"` - ID any `json:"id,omitempty"` - Result any `json:"result,omitempty"` - Error *errorEnvelope `json:"error,omitempty"` + JSONRPC string `json:"jsonrpc"` + ID json.RawMessage `json:"id"` + Result any `json:"result,omitempty"` + Error *errorEnvelope `json:"error,omitempty"` } type errorEnvelope struct { @@ -48,7 +53,7 @@ type toolCallParams struct { Arguments map[string]any `json:"arguments"` } -func successResponse(id any, result any) *responseEnvelope { +func successResponse(id json.RawMessage, result any) *responseEnvelope { return &responseEnvelope{ JSONRPC: "2.0", ID: id, @@ -56,7 +61,7 @@ func successResponse(id any, result any) *responseEnvelope { } } -func errorResponse(id any, code int, message string, data any) *responseEnvelope { +func errorResponse(id json.RawMessage, code int, message string, data any) *responseEnvelope { return &responseEnvelope{ JSONRPC: "2.0", ID: id, diff --git a/internal/mcpserver/server.go b/internal/mcpserver/server.go index 4ee570c..a2c125a 100644 --- a/internal/mcpserver/server.go +++ b/internal/mcpserver/server.go @@ -29,25 +29,24 @@ func New() *Server { func (s *Server) Serve(ctx context.Context, in io.Reader, out io.Writer) error { reader := bufio.NewReader(in) for { - line, err := reader.ReadBytes('\n') - if err != nil { - if errors.Is(err, io.EOF) { - return nil - } - return err - } - + // ReadBytes can return data together with io.EOF when the final + // request has no trailing newline (common for one-shot piped + // clients); process the line before acting on the error so that + // request is answered instead of silently dropped. + line, readErr := reader.ReadBytes('\n') line = bytes.TrimSpace(line) - if len(line) == 0 { - continue - } - - resp := s.HandleLine(ctx, line) - if resp == nil { - continue + if len(line) > 0 { + if resp := s.HandleLine(ctx, line); resp != nil { + if err := writeMessage(out, resp); err != nil { + return err + } + } } - if err := writeMessage(out, resp); err != nil { - return err + if readErr != nil { + if errors.Is(readErr, io.EOF) { + return nil + } + return readErr } } } @@ -134,23 +133,42 @@ func (s *Server) handleToolCall(ctx context.Context, req requestEnvelope) *respo result, err := safeToolCall(ctx, params.Name, params.Arguments) if err != nil { - return errorResponse(req.ID, jsonRPCInternalError, err.Error(), nil) + // Per the MCP spec: an unknown tool is a protocol-level invalid-params + // error and a panic is a genuine server fault, but ordinary execution + // failures (bad config path, invalid arguments) are returned as + // isError results so the calling model can read them and self-correct + // instead of the client treating the server as broken. + switch { + case errors.Is(err, mcptools.ErrUnknownTool): + return errorResponse(req.ID, jsonRPCInvalidParams, err.Error(), nil) + case errors.Is(err, errToolPanicked): + return errorResponse(req.ID, jsonRPCInternalError, err.Error(), nil) + default: + return successResponse(req.ID, mcptools.Result{ + Content: []mcptools.Content{{Type: "text", Text: err.Error()}}, + IsError: true, + }) + } } return successResponse(req.ID, result) } +// errToolPanicked marks a contained tool-handler panic so it surfaces as an +// internal JSON-RPC error rather than an isError tool result. +var errToolPanicked = errors.New("tool handler panicked") + // safeToolCall contains a panicking tool handler: the server speaks stdio, so // an uncaught panic would kill the whole process and every session with it. -func safeToolCall(ctx context.Context, name string, args map[string]any) (result any, err error) { +func safeToolCall(ctx context.Context, name string, args map[string]any) (result mcptools.Result, err error) { defer func() { if r := recover(); r != nil { - err = fmt.Errorf("tool %s panicked: %v", name, r) + err = fmt.Errorf("%w: tool %s panicked: %v", errToolPanicked, name, r) } }() return mcptools.Call(ctx, name, args) } -func (s *Server) requireInitialized(id any) *responseEnvelope { +func (s *Server) requireInitialized(id json.RawMessage) *responseEnvelope { if s.initialized { return nil } diff --git a/internal/mcpserver/tools/registry.go b/internal/mcpserver/tools/registry.go index a5e0790..abe13a6 100644 --- a/internal/mcpserver/tools/registry.go +++ b/internal/mcpserver/tools/registry.go @@ -2,6 +2,7 @@ package tools import ( "context" + "errors" "fmt" "github.com/devr-tools/cleanr/internal/mcpserver/catalog" @@ -11,6 +12,12 @@ import ( type Definition = toolkit.Definition type Result = toolkit.Result +type Content = toolkit.Content + +// ErrUnknownTool marks a tools/call for a name that is not registered, so the +// server can answer with a protocol-level invalid-params error instead of a +// tool-execution failure. +var ErrUnknownTool = errors.New("unknown tool") type handler func(context.Context, map[string]any) (toolkit.Result, error) @@ -49,7 +56,7 @@ func Definitions() []Definition { func Call(ctx context.Context, name string, args map[string]any) (Result, error) { h, ok := handlers[name] if !ok { - return Result{}, fmt.Errorf("unknown tool: %s", name) + return Result{}, fmt.Errorf("%w: %s", ErrUnknownTool, name) } return h(ctx, args) } diff --git a/tests/mcp/protocol_edges_test.go b/tests/mcp/protocol_edges_test.go new file mode 100644 index 0000000..2d5b21c --- /dev/null +++ b/tests/mcp/protocol_edges_test.go @@ -0,0 +1,121 @@ +package tests + +import ( + "bytes" + "context" + "encoding/json" + "strings" + "testing" + + "github.com/devr-tools/cleanr/internal/mcpserver" +) + +// A final request without a trailing newline (common for one-shot piped +// clients) must be answered at EOF, not silently dropped. +func TestMCPServerAnswersFinalUnterminatedLine(t *testing.T) { + t.Parallel() + + input := strings.Join([]string{ + `{"jsonrpc":"2.0","id":1,"method":"initialize","params":{"protocolVersion":"2025-06-18","capabilities":{},"clientInfo":{"name":"t","version":"1"}}}`, + `{"jsonrpc":"2.0","method":"notifications/initialized"}`, + `{"jsonrpc":"2.0","id":2,"method":"ping"}`, // no trailing newline + }, "\n") + + var out bytes.Buffer + if err := mcpserver.New().Serve(context.Background(), strings.NewReader(input), &out); err != nil { + t.Fatalf("serve: %v", err) + } + responses := strings.Split(strings.TrimSpace(out.String()), "\n") + if len(responses) != 2 { + t.Fatalf("expected initialize and ping responses, got %d: %s", len(responses), out.String()) + } + if !strings.Contains(responses[1], `"id":2`) { + t.Fatalf("expected ping response for the unterminated final line, got %s", responses[1]) + } +} + +// JSON-RPC ids must round-trip exactly: id 0 and id "" were previously +// dropped by omitempty, leaving the client unable to correlate the response. +func TestMCPServerEchoesFalsyRequestIDs(t *testing.T) { + server := initializedMCPServer(t) + + resp := mustHandleMCP(t, server, map[string]any{ + "jsonrpc": "2.0", + "id": 0, + "method": "ping", + }) + id, present := resp["id"] + if !present || id != float64(0) { + t.Fatalf("expected id 0 to be echoed, got %#v (present=%v)", id, present) + } + + resp = mustHandleMCP(t, server, map[string]any{ + "jsonrpc": "2.0", + "id": "", + "method": "ping", + }) + if id, present := resp["id"]; !present || id != "" { + t.Fatalf("expected empty-string id to be echoed, got %#v (present=%v)", id, present) + } +} + +// Parse errors must carry "id": null per JSON-RPC, not omit the field. +func TestMCPServerParseErrorCarriesNullID(t *testing.T) { + t.Parallel() + + raw := mcpserver.New().HandleLine(context.Background(), []byte("{not json")) + if raw == nil { + t.Fatal("expected parse-error response") + } + data, err := json.Marshal(raw) + if err != nil { + t.Fatalf("marshal response: %v", err) + } + if !strings.Contains(string(data), `"id":null`) { + t.Fatalf("expected id:null on parse error, got %s", data) + } + if !strings.Contains(string(data), `-32700`) { + t.Fatalf("expected parse-error code, got %s", data) + } +} + +// Ordinary tool execution failures come back as isError results the calling +// model can read; only unknown tools and contained panics are protocol-level +// errors. +func TestMCPServerToolFailureModes(t *testing.T) { + server := initializedMCPServer(t) + + // Execution failure: missing config → isError result, not a JSON-RPC error. + resp := mustHandleMCP(t, server, map[string]any{ + "jsonrpc": "2.0", + "id": 3, + "method": "tools/call", + "params": map[string]any{"name": "cleanr_generate_dataset", "arguments": map[string]any{}}, + }) + if _, hasErr := resp["error"]; hasErr { + t.Fatalf("execution failure must be an isError result, got protocol error: %#v", resp) + } + result := resp["result"].(map[string]any) + if isErr, _ := result["isError"].(bool); !isErr { + t.Fatalf("expected isError result, got %#v", result) + } + content := result["content"].([]any) + if text := content[0].(map[string]any)["text"].(string); !strings.Contains(text, "config") { + t.Fatalf("expected readable failure text, got %q", text) + } + + // Unknown tool → protocol-level invalid params. + resp = mustHandleMCP(t, server, map[string]any{ + "jsonrpc": "2.0", + "id": 4, + "method": "tools/call", + "params": map[string]any{"name": "cleanr_no_such_tool", "arguments": map[string]any{}}, + }) + respErr, ok := resp["error"].(map[string]any) + if !ok { + t.Fatalf("expected protocol error for unknown tool, got %#v", resp) + } + if code, _ := respErr["code"].(float64); code != -32602 { + t.Fatalf("expected -32602 for unknown tool, got %v", respErr["code"]) + } +} From bbf10626c1d01e9dc0b3196d294a851b94254d66 Mon Sep 17 00:00:00 2001 From: Alex Wilkerson John Date: Thu, 2 Jul 2026 16:12:40 -0400 Subject: [PATCH 13/13] test: split cli_test.go into topical files to satisfy codeguard limits MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The catch-all tests/cli/cli_test.go had grown to 2914 lines, tripping codeguard's max-file-lines gate once this branch touched it. Split by topic — integrations, native sinks, dataset commands, dataset policy, sync, trend history, trends/plugins commands — with shared helpers kept in cli_test.go. Test code is unchanged apart from imports. Also restructure TestExplicitRequireReviewFalseSurvivesLoad's assertions into explicit got/want checks to satisfy codeguard's conditional-assertion heuristic. Co-Authored-By: Claude Fable 5 --- tests/cli/cli_dataset_policy_test.go | 468 +++++ tests/cli/cli_dataset_test.go | 457 +++++ tests/cli/cli_integrations_test.go | 195 ++ tests/cli/cli_native_sinks_test.go | 470 +++++ tests/cli/cli_sync_test.go | 335 ++++ tests/cli/cli_test.go | 2445 ------------------------- tests/cli/cli_trend_history_test.go | 388 ++++ tests/cli/cli_trends_command_test.go | 233 +++ tests/config/pointer_defaults_test.go | 12 +- 9 files changed, 2554 insertions(+), 2449 deletions(-) create mode 100644 tests/cli/cli_dataset_policy_test.go create mode 100644 tests/cli/cli_dataset_test.go create mode 100644 tests/cli/cli_integrations_test.go create mode 100644 tests/cli/cli_native_sinks_test.go create mode 100644 tests/cli/cli_sync_test.go create mode 100644 tests/cli/cli_trend_history_test.go create mode 100644 tests/cli/cli_trends_command_test.go diff --git a/tests/cli/cli_dataset_policy_test.go b/tests/cli/cli_dataset_policy_test.go new file mode 100644 index 0000000..08db623 --- /dev/null +++ b/tests/cli/cli_dataset_policy_test.go @@ -0,0 +1,468 @@ +package tests + +import ( + "bytes" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/devr-tools/cleanr/cleanr" + "github.com/devr-tools/cleanr/internal/cli" +) + +func TestDatasetReviewCommandAppliesCheckedInPolicyBeforeManualOverrides(t *testing.T) { + baseCfg := cleanr.ExampleConfig() + baseCfg.Scenarios = []cleanr.Scenario{ + { + Name: "existing-refund", + System: "You are helpful.", + Input: "Summarize the refund policy.", + Tags: []string{"stable"}, + ExpectedContains: []string{"refund"}, + Assertions: []cleanr.Assertion{{ + Type: "contains", + Value: "refund", + }}, + }, + } + + dir := t.TempDir() + basePath := filepath.Join(dir, "cleanr.yaml") + if err := cleanr.WriteConfigFile(basePath, baseCfg); err != nil { + t.Fatalf("write base config: %v", err) + } + + policy := cleanr.DatasetReviewPolicy{ + Version: "v1alpha1", + Rules: []cleanr.DatasetReviewPolicyRule{ + { + Action: "reject", + Statuses: []string{"duplicate"}, + }, + { + Action: "approve", + Statuses: []string{"modified"}, + MinSeverity: "high", + }, + { + Action: "promote-regression", + Statuses: []string{"modified"}, + MinSeverity: "high", + }, + { + Action: "promote-stable", + Statuses: []string{"new"}, + StableSuitability: "medium", + Sources: []string{"cleanr-replay"}, + RequireAssertions: true, + RequireExpectedText: true, + }, + { + Action: "set-metadata", + Statuses: []string{"modified", "new"}, + Metadata: map[string]string{"owner": "qa"}, + }, + { + Action: "add-tags", + GeneratorProviders: []string{"openai"}, + GeneratorModels: []string{"gpt-4.1-mini"}, + ScenarioTags: []string{"generated"}, + Tags: []string{"generator-reviewed"}, + }, + }, + } + policyPath := filepath.Join(dir, "cleanr.review.yaml") + if err := cleanr.WriteDatasetReviewPolicyFile(policyPath, policy); err != nil { + t.Fatalf("write review policy: %v", err) + } + + dataset := cleanr.ScenarioDataset{ + Version: "v1alpha1", + Source: "cleanr-replay", + Target: baseCfg.Target.Name, + BuildID: "build-policy-1", + GeneratedAt: time.Now().UTC(), + Generator: &cleanr.ScenarioDatasetGenerator{ + Provider: "openai", + Model: "gpt-4.1-mini", + }, + Scenarios: []cleanr.ScenarioDatasetEntry{ + { + Scenario: cleanr.Scenario{ + Name: "existing-refund", + System: "You are helpful.", + Input: "Summarize the refund policy for a locked account.", + Tags: []string{"generated"}, + ExpectedContains: []string{"refund"}, + }, + Origin: cleanr.DatasetScenarioOrigin{ + Suite: "security", + Case: "existing-refund", + Findings: []cleanr.Finding{{ + Severity: "high", + Message: "important", + }}, + }, + }, + { + Scenario: cleanr.Scenario{ + Name: "new-stable", + System: "You are helpful.", + Input: "Summarize the support hours in one sentence.", + Tags: []string{"generated", "policy"}, + ExpectedContains: []string{"weekday"}, + Assertions: []cleanr.Assertion{{ + Type: "contains", + Value: "weekday", + }}, + }, + }, + { + Scenario: cleanr.Scenario{ + Name: "duplicate-refund-copy", + System: "You are helpful.", + Input: "Summarize the refund policy.", + Tags: []string{"generated"}, + }, + }, + }, + } + datasetPath := filepath.Join(dir, "dataset.yaml") + if err := cleanr.WriteScenarioDatasetFile(datasetPath, dataset); err != nil { + t.Fatalf("write dataset: %v", err) + } + + reviewedPath := filepath.Join(dir, "reviewed.yaml") + mergedPath := filepath.Join(dir, "merged.yaml") + var stdout bytes.Buffer + var stderr bytes.Buffer + code := cli.Run([]string{ + "dataset", "review", + "-input", datasetPath, + "-policy", policyPath, + "-base-config", basePath, + "-output", reviewedPath, + "-merge-output", mergedPath, + "-approve", "new-stable", + }, &stdout, &stderr) + if code != 0 { + t.Fatalf("expected review success, code=%d stdout=%s stderr=%s", code, stdout.String(), stderr.String()) + } + if !strings.Contains(stdout.String(), "applied review policy: "+policyPath) { + t.Fatalf("expected stdout to include applied review policy path, got %s", stdout.String()) + } + + reviewed, err := cleanr.LoadReviewedScenarioDatasetFile(reviewedPath) + if err != nil { + t.Fatalf("load reviewed dataset: %v", err) + } + if reviewed.PolicyPath != policyPath { + t.Fatalf("expected reviewed policy path %q, got %q", policyPath, reviewed.PolicyPath) + } + if reviewed.PolicyVersion != "v1alpha1" { + t.Fatalf("expected reviewed policy version v1alpha1, got %q", reviewed.PolicyVersion) + } + + entries := map[string]cleanr.ReviewedScenarioEntry{} + for _, item := range reviewed.Scenarios { + entries[item.Entry.Scenario.Name] = item + } + + modified := entries["existing-refund"] + if modified.Decision.Status != "approved" { + t.Fatalf("expected modified scenario to be policy-approved, got %+v", modified.Decision) + } + if !strings.Contains(strings.Join(modified.Entry.Scenario.Tags, ","), "regression") { + t.Fatalf("expected modified scenario to gain regression tag, got %+v", modified.Entry.Scenario.Tags) + } + if !strings.Contains(strings.Join(modified.Entry.Scenario.Tags, ","), "generator-reviewed") { + t.Fatalf("expected generator/tag selector to add tag, got %+v", modified.Entry.Scenario.Tags) + } + if modified.Entry.Scenario.Metadata["owner"] != "qa" { + t.Fatalf("expected policy metadata on modified scenario, got %+v", modified.Entry.Scenario.Metadata) + } + if len(modified.Decision.PolicyRules) == 0 { + t.Fatalf("expected policy rule provenance on modified scenario, got %+v", modified.Decision) + } + + newStable := entries["new-stable"] + if newStable.Decision.Status != "approved" { + t.Fatalf("expected manual approval override for new-stable, got %+v", newStable.Decision) + } + if !strings.Contains(strings.Join(newStable.Entry.Scenario.Tags, ","), "stable") { + t.Fatalf("expected policy stable promotion for new scenario, got %+v", newStable.Entry.Scenario.Tags) + } + if !strings.Contains(strings.Join(newStable.Entry.Scenario.Tags, ","), "generator-reviewed") { + t.Fatalf("expected generator/tag selector to affect new scenario, got %+v", newStable.Entry.Scenario.Tags) + } + + duplicate := entries["duplicate-refund-copy"] + if duplicate.Decision.Status != "rejected" || duplicate.Diff.Status != "duplicate" { + t.Fatalf("expected duplicate rejection from policy, got %+v", duplicate) + } + if !strings.Contains(strings.Join(duplicate.Entry.Scenario.Tags, ","), "generator-reviewed") { + t.Fatalf("expected generator/tag selector on duplicate scenario, got %+v", duplicate.Entry.Scenario.Tags) + } + + merged, err := cleanr.LoadConfigFile(mergedPath) + if err != nil { + t.Fatalf("load merged config: %v", err) + } + if len(merged.Scenarios) != 2 { + t.Fatalf("expected existing plus new approved scenario in merged config, got %+v", merged.Scenarios) + } + var mergedNew cleanr.Scenario + for _, scenario := range merged.Scenarios { + if scenario.Name == "new-stable" { + mergedNew = scenario + break + } + } + if mergedNew.Metadata["cleanr.review.policy_path"] != policyPath { + t.Fatalf("expected merged scenario provenance to include policy path, got %+v", mergedNew.Metadata) + } + if !strings.Contains(mergedNew.Metadata["cleanr.review.policy_rules"], "rule-") { + t.Fatalf("expected merged scenario provenance to include policy rules, got %+v", mergedNew.Metadata) + } +} + +func TestDatasetReviewCommandInteractiveModeAppliesScenarioEdits(t *testing.T) { + baseCfg := cleanr.ExampleConfig() + baseCfg.Scenarios = []cleanr.Scenario{ + {Name: "existing", System: "You are helpful.", Input: "Reset my password."}, + } + + dir := t.TempDir() + basePath := filepath.Join(dir, "cleanr.yaml") + if err := cleanr.WriteConfigFile(basePath, baseCfg); err != nil { + t.Fatalf("write base config: %v", err) + } + + dataset := cleanr.ScenarioDataset{ + Version: "v1alpha1", + Source: "cleanr-generation", + Target: baseCfg.Target.Name, + GeneratedAt: time.Now().UTC(), + Scenarios: []cleanr.ScenarioDatasetEntry{ + {Scenario: cleanr.Scenario{Name: "candidate-one", System: "You are helpful.", Input: "Summarize support hours.", Tags: []string{"generated"}}}, + {Scenario: cleanr.Scenario{Name: "candidate-two", System: "You are helpful.", Input: "Reset my password.", Tags: []string{"generated"}}}, + }, + } + datasetPath := filepath.Join(dir, "dataset.yaml") + if err := cleanr.WriteScenarioDatasetFile(datasetPath, dataset); err != nil { + t.Fatalf("write dataset: %v", err) + } + + withCLIStdinFile(t, "tag manual\nmetadata owner=qa\nstable\nreject\n") + + reviewedPath := filepath.Join(dir, "reviewed.yaml") + mergedPath := filepath.Join(dir, "merged.yaml") + var stdout bytes.Buffer + var stderr bytes.Buffer + code := cli.Run([]string{ + "dataset", "review", + "-interactive", + "-input", datasetPath, + "-base-config", basePath, + "-output", reviewedPath, + "-merge-output", mergedPath, + }, &stdout, &stderr) + if code != 0 { + t.Fatalf("expected interactive review success, code=%d stdout=%s stderr=%s", code, stdout.String(), stderr.String()) + } + + reviewed, err := cleanr.LoadReviewedScenarioDatasetFile(reviewedPath) + if err != nil { + t.Fatalf("load reviewed dataset: %v", err) + } + if reviewed.ApprovedScenarios != 1 || reviewed.RejectedScenarios != 1 || reviewed.PendingScenarios != 0 { + t.Fatalf("unexpected reviewed counts: %+v", reviewed) + } + + entries := map[string]cleanr.ReviewedScenarioEntry{} + for _, item := range reviewed.Scenarios { + entries[item.Entry.Scenario.Name] = item + } + + first := entries["candidate-one"] + if first.Decision.Status != "approved" { + t.Fatalf("expected candidate-one approved, got %+v", first.Decision) + } + if !strings.Contains(strings.Join(first.Entry.Scenario.Tags, ","), "manual") || !strings.Contains(strings.Join(first.Entry.Scenario.Tags, ","), "stable") { + t.Fatalf("expected candidate-one tag edits, got %+v", first.Entry.Scenario.Tags) + } + if first.Entry.Scenario.Metadata["owner"] != "qa" { + t.Fatalf("expected candidate-one metadata edit, got %+v", first.Entry.Scenario.Metadata) + } + + second := entries["candidate-two"] + if second.Decision.Status != "rejected" { + t.Fatalf("expected candidate-two rejected, got %+v", second.Decision) + } + + merged, err := cleanr.LoadConfigFile(mergedPath) + if err != nil { + t.Fatalf("load merged config: %v", err) + } + if len(merged.Scenarios) != 2 { + t.Fatalf("expected merged config to include one approved scenario, got %+v", merged.Scenarios) + } + if !strings.Contains(stdout.String(), "interactive dataset review") || !strings.Contains(stdout.String(), "review>") { + t.Fatalf("expected interactive prompts in stdout, got %s", stdout.String()) + } +} + +func TestDatasetReviewCommandAutoDiscoversProfilePolicy(t *testing.T) { + dir := t.TempDir() + wd, err := os.Getwd() + if err != nil { + t.Fatalf("getwd: %v", err) + } + defer func() { _ = os.Chdir(wd) }() + if err := os.Chdir(dir); err != nil { + t.Fatalf("chdir: %v", err) + } + if err := os.MkdirAll(".cleanr", 0o755); err != nil { + t.Fatalf("mkdir .cleanr: %v", err) + } + + cfg := cleanr.ExampleConfig() + cfg.Scenarios = []cleanr.Scenario{{ + Name: "existing", + System: "You are helpful.", + Input: "Reset my password.", + }} + if err := cleanr.WriteConfigFile(filepath.Join(".cleanr", "pr.yaml"), cfg); err != nil { + t.Fatalf("write staged config: %v", err) + } + + policy := cleanr.DatasetReviewPolicy{ + Version: "v1alpha1", + Rules: []cleanr.DatasetReviewPolicyRule{{ + Action: "reject", + Statuses: []string{"duplicate"}, + }}, + } + if err := cleanr.WriteDatasetReviewPolicyFile(filepath.Join(".cleanr", "pr.review.yaml"), policy); err != nil { + t.Fatalf("write staged review policy: %v", err) + } + + dataset := cleanr.ScenarioDataset{ + Version: "v1alpha1", + Source: "cleanr-replay", + GeneratedAt: time.Now().UTC(), + Scenarios: []cleanr.ScenarioDatasetEntry{{ + Scenario: cleanr.Scenario{ + Name: "duplicate", + System: "You are helpful.", + Input: "Reset my password.", + }, + }}, + } + datasetPath := filepath.Join(dir, "dataset.yaml") + if err := cleanr.WriteScenarioDatasetFile(datasetPath, dataset); err != nil { + t.Fatalf("write dataset: %v", err) + } + + reviewedPath := filepath.Join(dir, "reviewed.yaml") + var stdout bytes.Buffer + var stderr bytes.Buffer + code := cli.Run([]string{ + "dataset", "review", + "-input", datasetPath, + "-profile", "pr", + "-output", reviewedPath, + }, &stdout, &stderr) + if code != 0 { + t.Fatalf("expected review success, code=%d stdout=%s stderr=%s", code, stdout.String(), stderr.String()) + } + + reviewed, err := cleanr.LoadReviewedScenarioDatasetFile(reviewedPath) + if err != nil { + t.Fatalf("load reviewed dataset: %v", err) + } + if len(reviewed.Scenarios) != 1 || reviewed.Scenarios[0].Decision.Status != "rejected" { + t.Fatalf("expected staged policy discovery to reject duplicate, got %+v", reviewed.Scenarios) + } + if reviewed.PolicyPath != filepath.Join(".cleanr", "pr.review.yaml") { + t.Fatalf("expected staged policy path in reviewed artifact, got %q", reviewed.PolicyPath) + } +} + +func TestDatasetReviewCommandFallsBackToRootPolicy(t *testing.T) { + dir := t.TempDir() + wd, err := os.Getwd() + if err != nil { + t.Fatalf("getwd: %v", err) + } + defer func() { _ = os.Chdir(wd) }() + if err := os.Chdir(dir); err != nil { + t.Fatalf("chdir: %v", err) + } + + cfg := cleanr.ExampleConfig() + cfg.Scenarios = []cleanr.Scenario{{ + Name: "existing", + System: "You are helpful.", + Input: "Refund summary.", + }} + configPath := filepath.Join(dir, "cleanr.yaml") + if err := cleanr.WriteConfigFile(configPath, cfg); err != nil { + t.Fatalf("write config: %v", err) + } + + policy := cleanr.DatasetReviewPolicy{ + Version: "v1alpha1", + Rules: []cleanr.DatasetReviewPolicyRule{{ + Action: "set-metadata", + Statuses: []string{"new"}, + Metadata: map[string]string{"owner": "qa"}, + }}, + } + if err := cleanr.WriteDatasetReviewPolicyFile(filepath.Join(dir, "cleanr.review.yaml"), policy); err != nil { + t.Fatalf("write root review policy: %v", err) + } + + dataset := cleanr.ScenarioDataset{ + Version: "v1alpha1", + Source: "cleanr-generation", + GeneratedAt: time.Now().UTC(), + Scenarios: []cleanr.ScenarioDatasetEntry{{ + Scenario: cleanr.Scenario{ + Name: "new-one", + System: "You are helpful.", + Input: "Summarize support hours.", + }, + }}, + } + datasetPath := filepath.Join(dir, "dataset.yaml") + if err := cleanr.WriteScenarioDatasetFile(datasetPath, dataset); err != nil { + t.Fatalf("write dataset: %v", err) + } + + reviewedPath := filepath.Join(dir, "reviewed.yaml") + var stdout bytes.Buffer + var stderr bytes.Buffer + code := cli.Run([]string{ + "dataset", "review", + "-input", datasetPath, + "-base-config", configPath, + "-output", reviewedPath, + }, &stdout, &stderr) + if code != 0 { + t.Fatalf("expected review success, code=%d stdout=%s stderr=%s", code, stdout.String(), stderr.String()) + } + + reviewed, err := cleanr.LoadReviewedScenarioDatasetFile(reviewedPath) + if err != nil { + t.Fatalf("load reviewed dataset: %v", err) + } + if len(reviewed.Scenarios) != 1 || reviewed.Scenarios[0].Entry.Scenario.Metadata["owner"] != "qa" { + t.Fatalf("expected root policy discovery to set metadata, got %+v", reviewed.Scenarios) + } + if reviewed.PolicyPath != filepath.Join(dir, "cleanr.review.yaml") { + t.Fatalf("expected root policy path in reviewed artifact, got %q", reviewed.PolicyPath) + } +} diff --git a/tests/cli/cli_dataset_test.go b/tests/cli/cli_dataset_test.go new file mode 100644 index 0000000..eda91f1 --- /dev/null +++ b/tests/cli/cli_dataset_test.go @@ -0,0 +1,457 @@ +package tests + +import ( + "bytes" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/devr-tools/cleanr/cleanr" + "github.com/devr-tools/cleanr/internal/cli" +) + +func TestDatasetExportAndImportCommands(t *testing.T) { + cfg := cleanr.ExampleConfig() + cfg.Scenarios = []cleanr.Scenario{ + {Name: "happy-path", Input: "hello", Tags: []string{"stable"}}, + {Name: "security-path", Input: "secret"}, + } + configPath := filepath.Join(t.TempDir(), "cleanr.yaml") + if err := cleanr.WriteConfigFile(configPath, cfg); err != nil { + t.Fatalf("write config: %v", err) + } + + replayPath := filepath.Join(filepath.Dir(configPath), "reports", "cleanr.replay.json") + artifact := cleanr.ReplayArtifact{ + Version: "v1alpha1", + Target: cfg.Target.Name, + BuildID: "build-9", + GeneratedAt: time.Now().UTC(), + Failures: []cleanr.ReplayArtifactCase{{ + Suite: "security", + Name: "happy-path", + Findings: []cleanr.Finding{{ + Severity: "high", + Message: "review me", + }}, + Failed: true, + }}, + } + if err := cleanr.WriteReplayArtifactFile(replayPath, artifact); err != nil { + t.Fatalf("write replay artifact: %v", err) + } + + datasetPath := filepath.Join(filepath.Dir(configPath), "reviewed-failures.yaml") + var stdout bytes.Buffer + var stderr bytes.Buffer + if code := cli.Run([]string{"dataset", "export", "-config", configPath, "-replay-artifact", replayPath, "-output", datasetPath}, &stdout, &stderr); code != 0 { + t.Fatalf("expected dataset export success, code=%d stderr=%s", code, stderr.String()) + } + + dataset, err := cleanr.LoadScenarioDatasetFile(datasetPath) + if err != nil { + t.Fatalf("load dataset: %v", err) + } + if len(dataset.Scenarios) != 1 || dataset.Scenarios[0].Scenario.Name != "happy-path" { + t.Fatalf("unexpected dataset scenarios: %+v", dataset.Scenarios) + } + if !strings.Contains(strings.Join(dataset.Scenarios[0].Scenario.Tags, ","), "regression") { + t.Fatalf("expected regression tag, got %+v", dataset.Scenarios[0].Scenario.Tags) + } + + baseCfg := cleanr.ExampleConfig() + baseCfg.Scenarios = []cleanr.Scenario{{Name: "legacy", Input: "existing"}} + basePath := filepath.Join(filepath.Dir(configPath), "base.yaml") + if err := cleanr.WriteConfigFile(basePath, baseCfg); err != nil { + t.Fatalf("write base config: %v", err) + } + + importedPath := filepath.Join(filepath.Dir(configPath), "imported.yaml") + stdout.Reset() + stderr.Reset() + if code := cli.Run([]string{"dataset", "import", "-input", datasetPath, "-base-config", basePath, "-output", importedPath}, &stdout, &stderr); code != 0 { + t.Fatalf("expected dataset import success, code=%d stderr=%s", code, stderr.String()) + } + + importedCfg, err := cleanr.LoadConfigFile(importedPath) + if err != nil { + t.Fatalf("load imported config: %v", err) + } + if len(importedCfg.Scenarios) != 2 { + t.Fatalf("expected merged scenarios, got %+v", importedCfg.Scenarios) + } + names := []string{importedCfg.Scenarios[0].Name, importedCfg.Scenarios[1].Name} + if !strings.Contains(strings.Join(names, ","), "happy-path") || !strings.Contains(strings.Join(names, ","), "legacy") { + t.Fatalf("unexpected merged scenario names: %+v", names) + } +} + +func TestDatasetImportRequiresApprovalForGeneratedDatasets(t *testing.T) { + datasetPath := filepath.Join(t.TempDir(), "generated.yaml") + dataset := cleanr.ScenarioDataset{ + Version: "v1alpha1", + Source: "cleanr-generation", + GeneratedAt: time.Now().UTC(), + ReviewRequired: true, + Scenarios: []cleanr.ScenarioDatasetEntry{{ + Scenario: cleanr.Scenario{Name: "generated-happy-path", Input: "Summarize the refund policy.", Tags: []string{"generated"}}, + }}, + } + if err := cleanr.WriteScenarioDatasetFile(datasetPath, dataset); err != nil { + t.Fatalf("write dataset: %v", err) + } + + importedPath := filepath.Join(filepath.Dir(datasetPath), "imported.yaml") + var stdout bytes.Buffer + var stderr bytes.Buffer + if code := cli.Run([]string{"dataset", "import", "-input", datasetPath, "-output", importedPath}, &stdout, &stderr); code != 2 { + t.Fatalf("expected approval failure, code=%d stdout=%s stderr=%s", code, stdout.String(), stderr.String()) + } + if !strings.Contains(stderr.String(), "requires explicit review") { + t.Fatalf("unexpected stderr: %s", stderr.String()) + } + + stdout.Reset() + stderr.Reset() + if code := cli.Run([]string{"dataset", "import", "-input", datasetPath, "-output", importedPath, "-approve-generated"}, &stdout, &stderr); code != 0 { + t.Fatalf("expected approved import success, code=%d stderr=%s", code, stderr.String()) + } + + importedCfg, err := cleanr.LoadConfigFile(importedPath) + if err != nil { + t.Fatalf("load imported config: %v", err) + } + names := make([]string, 0, len(importedCfg.Scenarios)) + for _, scenario := range importedCfg.Scenarios { + names = append(names, scenario.Name) + } + if !strings.Contains(strings.Join(names, ","), "generated-happy-path") { + t.Fatalf("unexpected imported config: %+v", importedCfg.Scenarios) + } +} + +func TestDatasetReviewCommandWritesReviewedArtifactAndMergedConfig(t *testing.T) { + baseCfg := cleanr.ExampleConfig() + baseCfg.Scenarios = []cleanr.Scenario{ + { + Name: "happy-path", + System: "You are a helpful assistant.", + Input: "Help with a refund.", + Tags: []string{"stable"}, + ExpectedContains: []string{"refund"}, + }, + { + Name: "legacy-duplicate", + System: "You are a helpful assistant.", + Input: "Reset my password.", + Tags: []string{"legacy"}, + }, + } + + dir := t.TempDir() + basePath := filepath.Join(dir, "cleanr.yaml") + if err := cleanr.WriteConfigFile(basePath, baseCfg); err != nil { + t.Fatalf("write base config: %v", err) + } + + dataset := cleanr.ScenarioDataset{ + Version: "v1alpha1", + Source: "cleanr-replay", + Target: baseCfg.Target.Name, + BuildID: "build-77", + GeneratedAt: time.Now().UTC(), + Scenarios: []cleanr.ScenarioDatasetEntry{ + { + Scenario: cleanr.Scenario{ + Name: "happy-path", + System: "You are a helpful assistant.", + Input: "Help with a refund after an account lockout.", + Tags: []string{"generated"}, + ExpectedContains: []string{"refund", "account"}, + }, + Origin: cleanr.DatasetScenarioOrigin{ + Suite: "security", + Case: "happy-path", + BuildID: "build-77", + Findings: []cleanr.Finding{{ + Severity: "high", + Message: "important regression", + }}, + }, + }, + { + Scenario: cleanr.Scenario{ + Name: "duplicate-reset", + System: "You are a helpful assistant.", + Input: "Reset my password.", + Tags: []string{"generated"}, + }, + }, + { + Scenario: cleanr.Scenario{ + Name: "new-stable-candidate", + System: "You are a helpful assistant.", + Input: "Summarize the refund policy in one sentence.", + Tags: []string{"generated"}, + ExpectedContains: []string{"refund"}, + }, + }, + }, + } + datasetPath := filepath.Join(dir, "dataset.yaml") + if err := cleanr.WriteScenarioDatasetFile(datasetPath, dataset); err != nil { + t.Fatalf("write dataset: %v", err) + } + + reviewedPath := filepath.Join(dir, "reviewed.yaml") + mergedPath := filepath.Join(dir, "merged.yaml") + var stdout bytes.Buffer + var stderr bytes.Buffer + code := cli.Run([]string{ + "dataset", "review", + "-input", datasetPath, + "-base-config", basePath, + "-output", reviewedPath, + "-merge-output", mergedPath, + "-approve", "happy-path,new-stable-candidate", + "-reject", "duplicate-reset", + "-promote-regression", "happy-path", + "-promote-stable", "new-stable-candidate", + "-add-tag", "happy-path:security", + "-set-metadata", "happy-path:owner=qa", + }, &stdout, &stderr) + if code != 0 { + t.Fatalf("expected dataset review success, code=%d stdout=%s stderr=%s", code, stdout.String(), stderr.String()) + } + + reviewed, err := cleanr.LoadReviewedScenarioDatasetFile(reviewedPath) + if err != nil { + t.Fatalf("load reviewed dataset: %v", err) + } + if reviewed.ApprovedScenarios != 2 || reviewed.RejectedScenarios != 1 || reviewed.PendingScenarios != 0 { + t.Fatalf("unexpected review counts: %+v", reviewed) + } + if len(reviewed.Scenarios) != 3 { + t.Fatalf("unexpected reviewed scenarios: %+v", reviewed.Scenarios) + } + if reviewed.Scenarios[0].Entry.Scenario.Name != "happy-path" { + t.Fatalf("expected high severity scenario to rank first, got %+v", reviewed.Scenarios) + } + if reviewed.Scenarios[0].Decision.Status != "approved" || reviewed.Scenarios[0].Diff.Status != "modified" { + t.Fatalf("unexpected reviewed entry: %+v", reviewed.Scenarios[0]) + } + if reviewed.Scenarios[0].Entry.Origin.Suite != "security" { + t.Fatalf("expected provenance to be preserved, got %+v", reviewed.Scenarios[0].Entry.Origin) + } + + duplicateSeen := false + for _, item := range reviewed.Scenarios { + if item.Entry.Scenario.Name == "duplicate-reset" { + duplicateSeen = true + if item.Decision.Status != "rejected" || item.Diff.Status != "duplicate" { + t.Fatalf("unexpected duplicate entry: %+v", item) + } + } + } + if !duplicateSeen { + t.Fatalf("expected duplicate candidate in reviewed dataset: %+v", reviewed.Scenarios) + } + + merged, err := cleanr.LoadConfigFile(mergedPath) + if err != nil { + t.Fatalf("load merged config: %v", err) + } + if len(merged.Scenarios) != 3 { + t.Fatalf("expected legacy plus two approved scenarios, got %+v", merged.Scenarios) + } + + mergedByName := map[string]cleanr.Scenario{} + for _, scenario := range merged.Scenarios { + mergedByName[scenario.Name] = scenario + } + if _, ok := mergedByName["duplicate-reset"]; ok { + t.Fatalf("rejected scenario should not be merged: %+v", merged.Scenarios) + } + happy := mergedByName["happy-path"] + if !strings.Contains(strings.Join(happy.Tags, ","), "regression") || !strings.Contains(strings.Join(happy.Tags, ","), "security") { + t.Fatalf("expected promoted tags on approved scenario, got %+v", happy.Tags) + } + if happy.Metadata["owner"] != "qa" { + t.Fatalf("expected metadata edit to persist, got %+v", happy.Metadata) + } + if happy.Metadata["cleanr.review.source"] != "cleanr-replay" || happy.Metadata["cleanr.review.origin_suite"] != "security" { + t.Fatalf("expected review provenance metadata, got %+v", happy.Metadata) + } + + newStable := mergedByName["new-stable-candidate"] + if !strings.Contains(strings.Join(newStable.Tags, ","), "stable") { + t.Fatalf("expected stable promotion to persist, got %+v", newStable.Tags) + } +} + +func TestDatasetReviewCommandSupportsCIGatesAndGitHubOutputs(t *testing.T) { + baseCfg := cleanr.ExampleConfig() + baseCfg.Scenarios = []cleanr.Scenario{ + {Name: "existing", System: "You are helpful.", Input: "Reset my password."}, + } + + dir := t.TempDir() + basePath := filepath.Join(dir, "cleanr.yaml") + if err := cleanr.WriteConfigFile(basePath, baseCfg); err != nil { + t.Fatalf("write base config: %v", err) + } + + dataset := cleanr.ScenarioDataset{ + Version: "v1alpha1", + Source: "cleanr-generation", + Target: baseCfg.Target.Name, + GeneratedAt: time.Now().UTC(), + Scenarios: []cleanr.ScenarioDatasetEntry{ + {Scenario: cleanr.Scenario{Name: "needs-review", System: "You are helpful.", Input: "What is the refund policy?", Tags: []string{"generated"}}}, + {Scenario: cleanr.Scenario{Name: "dup", System: "You are helpful.", Input: "Reset my password.", Tags: []string{"generated"}}}, + }, + } + datasetPath := filepath.Join(dir, "dataset.yaml") + if err := cleanr.WriteScenarioDatasetFile(datasetPath, dataset); err != nil { + t.Fatalf("write dataset: %v", err) + } + + githubOutputPath := filepath.Join(dir, "github-output.txt") + githubSummaryPath := filepath.Join(dir, "github-summary.md") + if err := os.WriteFile(githubOutputPath, []byte("existing-output=true\n"), 0o644); err != nil { + t.Fatalf("seed github output: %v", err) + } + if err := os.WriteFile(githubSummaryPath, []byte("Existing summary\n"), 0o644); err != nil { + t.Fatalf("seed github summary: %v", err) + } + t.Setenv("GITHUB_OUTPUT", githubOutputPath) + t.Setenv("GITHUB_STEP_SUMMARY", githubSummaryPath) + + var stdout bytes.Buffer + var stderr bytes.Buffer + code := cli.Run([]string{ + "dataset", "review", + "-input", datasetPath, + "-base-config", basePath, + "-output", filepath.Join(dir, "reviewed.yaml"), + "-approve", "dup", + "-fail-on-pending", + "-max-duplicates", "0", + "-github-outputs", + }, &stdout, &stderr) + if code != 1 { + t.Fatalf("expected gate failure exit code 1, got %d stdout=%s stderr=%s", code, stdout.String(), stderr.String()) + } + if !strings.Contains(stderr.String(), "found 1 pending scenarios") { + t.Fatalf("expected pending gate failure, got stderr=%s", stderr.String()) + } + if !strings.Contains(stderr.String(), "duplicate candidate count 1 exceeds maximum 0") { + t.Fatalf("expected duplicate gate failure, got stderr=%s", stderr.String()) + } + + outputBody, err := os.ReadFile(githubOutputPath) + if err != nil { + t.Fatalf("read github output: %v", err) + } + outputText := string(outputBody) + for _, want := range []string{ + "existing-output=true", + "cleanr_review_gate_passed=false", + "cleanr_review_pending=1", + "cleanr_review_duplicates=1", + "cleanr_review_policy_path=", + "cleanr_review_top_candidate=", + } { + if !strings.Contains(outputText, want) { + t.Fatalf("expected %q in GITHUB_OUTPUT:\n%s", want, outputText) + } + } + + summaryBody, err := os.ReadFile(githubSummaryPath) + if err != nil { + t.Fatalf("read github summary: %v", err) + } + summaryText := string(summaryBody) + for _, want := range []string{ + "Existing summary", + "## cleanr Dataset Review", + "Gate passed: `false`", + "Pending: `1`", + "Duplicates: `1`", + "Review artifact: `", + "Gate findings:", + } { + if !strings.Contains(summaryText, want) { + t.Fatalf("expected %q in summary:\n%s", want, summaryText) + } + } +} + +func TestDatasetReviewCommandSupportsBuildkiteOutputs(t *testing.T) { + baseCfg := cleanr.ExampleConfig() + baseCfg.Scenarios = []cleanr.Scenario{ + {Name: "existing", System: "You are helpful.", Input: "Reset my password."}, + } + + dir := t.TempDir() + basePath := filepath.Join(dir, "cleanr.yaml") + if err := cleanr.WriteConfigFile(basePath, baseCfg); err != nil { + t.Fatalf("write base config: %v", err) + } + + dataset := cleanr.ScenarioDataset{ + Version: "v1alpha1", + Source: "cleanr-generation", + Target: baseCfg.Target.Name, + GeneratedAt: time.Now().UTC(), + Scenarios: []cleanr.ScenarioDatasetEntry{ + {Scenario: cleanr.Scenario{Name: "needs-review", System: "You are helpful.", Input: "What is the refund policy?", Tags: []string{"generated"}}}, + {Scenario: cleanr.Scenario{Name: "dup", System: "You are helpful.", Input: "Reset my password.", Tags: []string{"generated"}}}, + }, + } + datasetPath := filepath.Join(dir, "dataset.yaml") + if err := cleanr.WriteScenarioDatasetFile(datasetPath, dataset); err != nil { + t.Fatalf("write dataset: %v", err) + } + + logPath := installFakeBuildkiteAgent(t) + var stdout bytes.Buffer + var stderr bytes.Buffer + code := cli.Run([]string{ + "dataset", "review", + "-input", datasetPath, + "-base-config", basePath, + "-output", filepath.Join(dir, "reviewed.yaml"), + "-merge-output", filepath.Join(dir, "merged.yaml"), + "-approve", "dup", + "-fail-on-pending", + "-max-duplicates", "0", + "-buildkite-meta", + "-buildkite-annotation", + "-buildkite-upload-artifacts", + }, &stdout, &stderr) + if code != 1 { + t.Fatalf("expected gate failure exit code 1, got %d stdout=%s stderr=%s", code, stdout.String(), stderr.String()) + } + + logBody, err := os.ReadFile(logPath) + if err != nil { + t.Fatalf("read buildkite log: %v", err) + } + logText := string(logBody) + for _, want := range []string{ + "meta-data set cleanr.review.gate_passed false", + "meta-data set cleanr.review.pending 1", + "meta-data set cleanr.review.duplicates 1", + "meta-data set cleanr.review.policy_path ", + "annotate ### cleanr dataset review gate failed", + "artifact upload " + filepath.Join(dir, "reviewed.yaml"), + "artifact upload " + filepath.Join(dir, "merged.yaml"), + } { + if !strings.Contains(logText, want) { + t.Fatalf("expected %q in buildkite log:\n%s", want, logText) + } + } +} diff --git a/tests/cli/cli_integrations_test.go b/tests/cli/cli_integrations_test.go new file mode 100644 index 0000000..c201ef7 --- /dev/null +++ b/tests/cli/cli_integrations_test.go @@ -0,0 +1,195 @@ +package tests + +import ( + "bytes" + "encoding/json" + "io" + "net/http" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/devr-tools/cleanr/cleanr" + "github.com/devr-tools/cleanr/internal/cli" +) + +func TestRunCommandSupportsExternalIntegrations(t *testing.T) { + cfg := cleanr.ExampleConfig() + cfg.Suites.PromptInjection.Enabled = false + cfg.Suites.Security.Enabled = false + cfg.Suites.Load.Enabled = false + cfg.Suites.Chaos.Enabled = false + cfg.Suites.Drift.Enabled = false + cfg.Suites.TokenOptimization.Enabled = false + cfg.Reporting.Format = "json" + cfg.Reporting.BuildID = "build-2" + cfg.Reporting.ReplayArtifactFile = "reports/cleanr.replay.json" + cfg.Integrations.TrendSources = []cleanr.TrendSourceConfig{{ + Name: "approved-history", + Type: "http", + URL: "https://example.test/history.yaml", + ViewURL: "https://braintrust.dev/app/history/build-1", + }} + cfg.Integrations.ResultSinks = []cleanr.ResultSinkConfig{{ + Name: "braintrust", + Type: "braintrust", + Endpoint: "https://example.test/publish", + Experiment: "release-gate", + IncludeReplay: true, + }} + cfg.Integrations.Summaries = []cleanr.SummaryConfig{{ + Name: "pr", + Format: "markdown", + Output: "reports/summary.md", + }} + + dir := t.TempDir() + configPath := filepath.Join(dir, "cleanr.yaml") + if err := cleanr.WriteConfigFile(configPath, cfg); err != nil { + t.Fatalf("write config: %v", err) + } + + historyPath := filepath.Join(dir, "history.yaml") + history := cleanr.TrendHistoryFile{ + Version: "v1alpha1", + Target: cfg.Target.Name, + Runs: []cleanr.TrendHistoryRun{{ + BuildID: "build-1", + GeneratedAt: time.Date(2026, 5, 19, 12, 0, 0, 0, time.UTC), + Passed: true, + Duration: time.Second, + FailedSuites: 0, + FailedCases: 0, + }}, + } + if err := cleanr.WriteTrendHistoryFile(historyPath, history); err != nil { + t.Fatalf("write history: %v", err) + } + historyBody, err := os.ReadFile(historyPath) + if err != nil { + t.Fatalf("read history: %v", err) + } + + originalTransport := http.DefaultTransport + http.DefaultTransport = cliRoundTripperFunc(func(req *http.Request) (*http.Response, error) { + switch req.URL.String() { + case "https://example.test/history.yaml": + return &http.Response{ + StatusCode: 200, + Header: http.Header{"Content-Type": []string{"application/x-yaml"}}, + Body: io.NopCloser(bytes.NewReader(historyBody)), + }, nil + case "https://example.test/publish": + body, err := io.ReadAll(req.Body) + if err != nil { + t.Fatalf("read publish body: %v", err) + } + if !bytes.Contains(body, []byte(`"replay_artifact"`)) || !bytes.Contains(body, []byte(`"source":"cleanr"`)) { + t.Fatalf("unexpected publish payload: %s", string(body)) + } + return &http.Response{ + StatusCode: 200, + Header: http.Header{"Content-Type": []string{"application/json"}}, + Body: io.NopCloser(strings.NewReader(`{"run_url":"https://braintrust.dev/app/release-gate/build-2"}`)), + }, nil + default: + t.Fatalf("unexpected request: %s", req.URL.String()) + return nil, nil + } + }) + defer func() { http.DefaultTransport = originalTransport }() + + var stdout bytes.Buffer + var stderr bytes.Buffer + exitCode := cli.Run([]string{"run", "-config", configPath, "-format", "json"}, &stdout, &stderr) + if exitCode != 0 { + t.Fatalf("expected exit code 0, got %d stderr=%s", exitCode, stderr.String()) + } + + var report cleanr.Report + if err := json.Unmarshal(stdout.Bytes(), &report); err != nil { + t.Fatalf("decode report: %v\n%s", err, stdout.String()) + } + if report.Integrations == nil { + t.Fatalf("expected integrations in report") + } + if len(report.Integrations.TrendSources) != 1 || report.Integrations.TrendSources[0].Status != "compared" { + t.Fatalf("unexpected trend source report: %+v", report.Integrations.TrendSources) + } + if len(report.Integrations.ResultSinks) != 1 || !report.Integrations.ResultSinks[0].Published { + t.Fatalf("unexpected result sink report: %+v", report.Integrations.ResultSinks) + } + if got := report.Integrations.ResultSinks[0].RunURL; got != "https://braintrust.dev/app/release-gate/build-2" { + t.Fatalf("unexpected run url: %s", got) + } + if len(report.Integrations.Summaries) != 1 || !report.Integrations.Summaries[0].Written { + t.Fatalf("unexpected summary report: %+v", report.Integrations.Summaries) + } + + summaryPath := filepath.Join(dir, "reports", "summary.md") + summaryBody, err := os.ReadFile(summaryPath) + if err != nil { + t.Fatalf("read summary: %v", err) + } + for _, want := range []string{ + "cleanr Release Summary", + "Local gate: `PASS`", + "approved-history", + "Remote Views", + "https://braintrust.dev/app/release-gate/build-2", + } { + if !strings.Contains(string(summaryBody), want) { + t.Fatalf("expected %q in summary:\n%s", want, string(summaryBody)) + } + } +} + +func TestRunCommandWritesBuildkiteMetadataAndAnnotation(t *testing.T) { + cfg := cleanr.ExampleConfig() + cfg.Suites.PromptInjection.Enabled = false + cfg.Suites.Security.Enabled = true + cfg.Suites.Security.MaxPIIMatches = 0 + cfg.Suites.Security.DangerousToolIndicators = []string{} + cfg.Suites.Security.SecretExposureIndicators = []string{} + cfg.Suites.Load.Enabled = false + cfg.Suites.Chaos.Enabled = false + cfg.Suites.Drift.Enabled = false + cfg.Suites.TokenOptimization.Enabled = false + cfg.Reporting.BuildID = "build-bk-1" + cfg.Scenarios = []cleanr.Scenario{{ + Name: "missing-phrase", + Input: "hello", + ExpectedContains: []string{"missing"}, + }} + + path := filepath.Join(t.TempDir(), "cleanr.yaml") + if err := cleanr.WriteConfigFile(path, cfg); err != nil { + t.Fatalf("write config: %v", err) + } + + logPath := installFakeBuildkiteAgent(t) + var stdout bytes.Buffer + var stderr bytes.Buffer + code := cli.Run([]string{"run", "-config", path, "-buildkite-meta", "-buildkite-annotation"}, &stdout, &stderr) + if code != 1 { + t.Fatalf("expected failing exit code 1, got %d stdout=%s stderr=%s", code, stdout.String(), stderr.String()) + } + + logBody, err := os.ReadFile(logPath) + if err != nil { + t.Fatalf("read buildkite log: %v", err) + } + logText := string(logBody) + for _, want := range []string{ + "meta-data set cleanr.run.passed false", + "meta-data set cleanr.run.failed_cases 1", + "meta-data set cleanr.run.build_id build-bk-1", + "annotate ### cleanr run failed", + } { + if !strings.Contains(logText, want) { + t.Fatalf("expected %q in buildkite log:\n%s", want, logText) + } + } +} diff --git a/tests/cli/cli_native_sinks_test.go b/tests/cli/cli_native_sinks_test.go new file mode 100644 index 0000000..bc84ded --- /dev/null +++ b/tests/cli/cli_native_sinks_test.go @@ -0,0 +1,470 @@ +package tests + +import ( + "bytes" + "encoding/base64" + "encoding/json" + "io" + "net/http" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/devr-tools/cleanr/cleanr" + "github.com/devr-tools/cleanr/internal/cli" +) + +func TestRunCommandSupportsNativeBraintrustConnector(t *testing.T) { + cfg := cleanr.ExampleConfig() + cfg.Suites.PromptInjection.Enabled = false + cfg.Suites.Security.Enabled = false + cfg.Suites.Load.Enabled = false + cfg.Suites.Chaos.Enabled = false + cfg.Suites.Drift.Enabled = false + cfg.Suites.TokenOptimization.Enabled = false + cfg.Reporting.Format = "json" + cfg.Reporting.BuildID = "build-2" + cfg.Integrations.TrendSources = []cleanr.TrendSourceConfig{{ + Name: "approved-history", + Type: "braintrust", + Project: "qa-gates", + Experiment: "release-gate", + ViewURL: "https://braintrust.dev/app/release-gate/build-1", + }} + cfg.Integrations.ResultSinks = []cleanr.ResultSinkConfig{{ + Name: "braintrust", + Type: "braintrust", + Project: "qa-gates", + Experiment: "release-gate", + APIKeyEnv: "CLEANR_BRAINTRUST_TOKEN", + IncludeReplay: true, + }} + cfg.Integrations.Summaries = []cleanr.SummaryConfig{{ + Name: "pr", + Format: "markdown", + Output: "reports/summary.md", + }} + + dir := t.TempDir() + configPath := filepath.Join(dir, "cleanr.yaml") + if err := cleanr.WriteConfigFile(configPath, cfg); err != nil { + t.Fatalf("write config: %v", err) + } + + t.Setenv("CLEANR_BRAINTRUST_TOKEN", "bt-secret") + + remoteRun := cleanr.TrendHistoryRun{ + BuildID: "build-1", + GeneratedAt: time.Date(2026, 5, 19, 12, 0, 0, 0, time.UTC), + Passed: true, + Duration: time.Second, + FailedSuites: 0, + FailedCases: 0, + Metadata: &cleanr.RunMetadata{ + BuildID: "build-1", + TargetType: "http", + ProviderModel: "gpt-4.1-mini", + }, + } + + originalTransport := http.DefaultTransport + http.DefaultTransport = cliRoundTripperFunc(func(req *http.Request) (*http.Response, error) { + switch { + case req.Method == http.MethodGet && req.URL.Path == "/v1/experiment": + if req.URL.Query().Get("project_name") != "qa-gates" { + t.Fatalf("unexpected project query: %s", req.URL.RawQuery) + } + if !strings.Contains(req.URL.Query().Get("metadata"), "release-gate") { + t.Fatalf("expected family filter in metadata query: %s", req.URL.RawQuery) + } + return &http.Response{ + StatusCode: 200, + Header: http.Header{"Content-Type": []string{"application/json"}}, + Body: io.NopCloser(strings.NewReader(`{"objects":[{"id":"exp-1","project_id":"proj-1","name":"release-gate/build-1","created":"2026-05-19T12:00:00Z"}]}`)), + }, nil + case req.Method == http.MethodPost && req.URL.Path == "/btql": + body, err := io.ReadAll(req.Body) + if err != nil { + t.Fatalf("read btql body: %v", err) + } + if !bytes.Contains(body, []byte("metadata.cleanr.history_run")) { + t.Fatalf("unexpected btql query: %s", string(body)) + } + raw, err := json.Marshal(map[string]any{"data": []map[string]any{{"history_run": remoteRun}}}) + if err != nil { + t.Fatalf("marshal btql response: %v", err) + } + return &http.Response{ + StatusCode: 200, + Header: http.Header{"Content-Type": []string{"application/json"}}, + Body: io.NopCloser(bytes.NewReader(raw)), + }, nil + case req.Method == http.MethodPost && req.URL.Path == "/v1/project": + if got := req.Header.Get("Authorization"); got != "Bearer bt-secret" { + t.Fatalf("unexpected auth header: %s", got) + } + body, err := io.ReadAll(req.Body) + if err != nil { + t.Fatalf("read project body: %v", err) + } + if !bytes.Contains(body, []byte(`"name":"qa-gates"`)) { + t.Fatalf("unexpected project payload: %s", string(body)) + } + return &http.Response{ + StatusCode: 200, + Header: http.Header{"Content-Type": []string{"application/json"}}, + Body: io.NopCloser(strings.NewReader(`{"id":"proj-1","name":"qa-gates"}`)), + }, nil + case req.Method == http.MethodPost && req.URL.Path == "/v1/experiment": + body, err := io.ReadAll(req.Body) + if err != nil { + t.Fatalf("read experiment body: %v", err) + } + for _, want := range []string{`"project_id":"proj-1"`, `"name":"release-gate/build-2"`, `"family":"release-gate"`} { + if !bytes.Contains(body, []byte(want)) { + t.Fatalf("expected %q in experiment payload: %s", want, string(body)) + } + } + return &http.Response{ + StatusCode: 200, + Header: http.Header{"Content-Type": []string{"application/json"}}, + Body: io.NopCloser(strings.NewReader(`{"id":"exp-2","project_id":"proj-1","name":"release-gate/build-2"}`)), + }, nil + case req.Method == http.MethodPost && req.URL.Path == "/v1/experiment/exp-2/insert": + body, err := io.ReadAll(req.Body) + if err != nil { + t.Fatalf("read insert body: %v", err) + } + for _, want := range []string{`"record_type":"run"`, `"history_run"`, `"replay_artifact"`} { + if !bytes.Contains(body, []byte(want)) { + t.Fatalf("expected %q in insert payload: %s", want, string(body)) + } + } + return &http.Response{ + StatusCode: 200, + Header: http.Header{"Content-Type": []string{"application/json"}}, + Body: io.NopCloser(strings.NewReader(`{}`)), + }, nil + case req.Method == http.MethodGet && req.URL.Path == "/v1/experiment/exp-2/summarize": + return &http.Response{ + StatusCode: 200, + Header: http.Header{"Content-Type": []string{"application/json"}}, + Body: io.NopCloser(strings.NewReader(`{"experiment_url":"https://braintrust.dev/app/release-gate/build-2"}`)), + }, nil + default: + t.Fatalf("unexpected request: %s %s", req.Method, req.URL.String()) + return nil, nil + } + }) + defer func() { http.DefaultTransport = originalTransport }() + + var stdout bytes.Buffer + var stderr bytes.Buffer + exitCode := cli.Run([]string{"run", "-config", configPath, "-format", "json"}, &stdout, &stderr) + if exitCode != 0 { + t.Fatalf("expected exit code 0, got %d stderr=%s", exitCode, stderr.String()) + } + + var report cleanr.Report + if err := json.Unmarshal(stdout.Bytes(), &report); err != nil { + t.Fatalf("decode report: %v\n%s", err, stdout.String()) + } + if report.Integrations == nil { + t.Fatalf("expected integrations in report") + } + if len(report.Integrations.TrendSources) != 1 || report.Integrations.TrendSources[0].Status != "compared" { + t.Fatalf("unexpected trend source report: %+v", report.Integrations.TrendSources) + } + if len(report.Integrations.ResultSinks) != 1 || !report.Integrations.ResultSinks[0].Published { + t.Fatalf("unexpected result sink report: %+v", report.Integrations.ResultSinks) + } + if got := report.Integrations.ResultSinks[0].RunURL; got != "https://braintrust.dev/app/release-gate/build-2" { + t.Fatalf("unexpected run url: %s", got) + } + + summaryPath := filepath.Join(dir, "reports", "summary.md") + summaryBody, err := os.ReadFile(summaryPath) + if err != nil { + t.Fatalf("read summary: %v", err) + } + for _, want := range []string{ + "cleanr Release Summary", + "Local gate: `PASS`", + "approved-history", + "Remote Views", + "https://braintrust.dev/app/release-gate/build-2", + } { + if !strings.Contains(string(summaryBody), want) { + t.Fatalf("expected %q in summary:\n%s", want, string(summaryBody)) + } + } +} + +func TestRunCommandSupportsNativeLangfuseConnector(t *testing.T) { + cfg := cleanr.ExampleConfig() + cfg.Suites.PromptInjection.Enabled = false + cfg.Suites.Security.Enabled = false + cfg.Suites.Load.Enabled = false + cfg.Suites.Chaos.Enabled = false + cfg.Suites.Drift.Enabled = false + cfg.Suites.TokenOptimization.Enabled = false + cfg.Reporting.Format = "json" + cfg.Reporting.BuildID = "build-2" + cfg.Integrations.ResultSinks = []cleanr.ResultSinkConfig{{ + Name: "langfuse", + Type: "langfuse", + BaseURL: "https://cloud.langfuse.com", + PublicKeyEnv: "LANGFUSE_PUBLIC_KEY", + SecretKeyEnv: "LANGFUSE_SECRET_KEY", + Experiment: "release-gate", + IncludeReplay: true, + RunURLTemplate: "https://cloud.langfuse.com/project/demo/traces/{{trace_id}}", + }} + cfg.Integrations.Summaries = []cleanr.SummaryConfig{{ + Name: "pr", + Format: "markdown", + Output: "reports/langfuse-summary.md", + }} + + dir := t.TempDir() + configPath := filepath.Join(dir, "cleanr.yaml") + if err := cleanr.WriteConfigFile(configPath, cfg); err != nil { + t.Fatalf("write config: %v", err) + } + + t.Setenv("LANGFUSE_PUBLIC_KEY", "pk-test") + t.Setenv("LANGFUSE_SECRET_KEY", "sk-test") + + var traceID string + scorePosts := 0 + wantAuth := "Basic " + base64.StdEncoding.EncodeToString([]byte("pk-test:sk-test")) + + originalTransport := http.DefaultTransport + http.DefaultTransport = cliRoundTripperFunc(func(req *http.Request) (*http.Response, error) { + if got := req.Header.Get("Authorization"); got != wantAuth { + t.Fatalf("unexpected auth header: %s", got) + } + switch { + case req.Method == http.MethodPost && req.URL.String() == "https://cloud.langfuse.com/api/public/ingestion": + body, err := io.ReadAll(req.Body) + if err != nil { + t.Fatalf("read langfuse ingestion body: %v", err) + } + var payload map[string]any + if err := json.Unmarshal(body, &payload); err != nil { + t.Fatalf("decode langfuse ingestion body: %v\n%s", err, string(body)) + } + batch, ok := payload["batch"].([]any) + if !ok || len(batch) != 1 { + t.Fatalf("unexpected batch payload: %s", string(body)) + } + event, ok := batch[0].(map[string]any) + if !ok || event["type"] != "trace-create" { + t.Fatalf("unexpected trace event: %s", string(body)) + } + eventBody, ok := event["body"].(map[string]any) + if !ok { + t.Fatalf("missing event body: %s", string(body)) + } + gotTraceID, _ := eventBody["id"].(string) + if gotTraceID == "" { + t.Fatalf("missing trace id in event body: %s", string(body)) + } + traceID = gotTraceID + for _, want := range []string{`"type":"trace-create"`, `"name":"release-gate"`, `"sessionId":"build-2"`, `"replay_artifact"`} { + if !bytes.Contains(body, []byte(want)) { + t.Fatalf("expected %q in ingestion payload: %s", want, string(body)) + } + } + return &http.Response{ + StatusCode: 200, + Header: http.Header{"Content-Type": []string{"application/json"}}, + Body: io.NopCloser(strings.NewReader(`{"success":true}`)), + }, nil + case req.Method == http.MethodPost && req.URL.String() == "https://cloud.langfuse.com/api/public/scores": + scorePosts++ + body, err := io.ReadAll(req.Body) + if err != nil { + t.Fatalf("read langfuse score body: %v", err) + } + if traceID == "" { + t.Fatalf("score posted before trace ingestion: %s", string(body)) + } + for _, want := range []string{`"traceId":"` + traceID + `"`, `"dataType":"NUMERIC"`} { + if !bytes.Contains(body, []byte(want)) { + t.Fatalf("expected %q in score payload: %s", want, string(body)) + } + } + return &http.Response{ + StatusCode: 200, + Header: http.Header{"Content-Type": []string{"application/json"}}, + Body: io.NopCloser(strings.NewReader(`{"success":true}`)), + }, nil + default: + t.Fatalf("unexpected request: %s %s", req.Method, req.URL.String()) + return nil, nil + } + }) + defer func() { http.DefaultTransport = originalTransport }() + + var stdout bytes.Buffer + var stderr bytes.Buffer + exitCode := cli.Run([]string{"run", "-config", configPath, "-format", "json"}, &stdout, &stderr) + if exitCode != 0 { + t.Fatalf("expected exit code 0, got %d stderr=%s", exitCode, stderr.String()) + } + if scorePosts < 3 { + t.Fatalf("expected at least 3 score posts, got %d", scorePosts) + } + + var report cleanr.Report + if err := json.Unmarshal(stdout.Bytes(), &report); err != nil { + t.Fatalf("decode report: %v\n%s", err, stdout.String()) + } + if report.Integrations == nil { + t.Fatalf("expected integrations in report") + } + if len(report.Integrations.ResultSinks) != 1 || !report.Integrations.ResultSinks[0].Published { + t.Fatalf("unexpected result sink report: %+v", report.Integrations.ResultSinks) + } + wantRunURL := "https://cloud.langfuse.com/project/demo/traces/" + traceID + if got := report.Integrations.ResultSinks[0].RunURL; got != wantRunURL { + t.Fatalf("unexpected run url: %s", got) + } + + summaryPath := filepath.Join(dir, "reports", "langfuse-summary.md") + summaryBody, err := os.ReadFile(summaryPath) + if err != nil { + t.Fatalf("read summary: %v", err) + } + for _, want := range []string{ + "cleanr Release Summary", + "Local gate: `PASS`", + "Remote Views", + wantRunURL, + } { + if !strings.Contains(string(summaryBody), want) { + t.Fatalf("expected %q in summary:\n%s", want, string(summaryBody)) + } + } +} + +func TestRunCommandSupportsNativePostHogConnector(t *testing.T) { + cfg := cleanr.ExampleConfig() + cfg.Suites.PromptInjection.Enabled = false + cfg.Suites.Security.Enabled = false + cfg.Suites.Load.Enabled = false + cfg.Suites.Chaos.Enabled = false + cfg.Suites.Drift.Enabled = false + cfg.Suites.TokenOptimization.Enabled = false + cfg.Reporting.Format = "json" + cfg.Reporting.BuildID = "build-2" + cfg.Integrations.ResultSinks = []cleanr.ResultSinkConfig{{ + Name: "posthog", + Type: "posthog", + BaseURL: "https://us.i.posthog.com", + ProjectTokenEnv: "POSTHOG_PROJECT_API_KEY", + Experiment: "release-gate", + IncludeReplay: true, + RunURLTemplate: "https://eu.posthog.com/project/demo/events?distinct_id={{distinct_id}}", + }} + cfg.Integrations.Summaries = []cleanr.SummaryConfig{{ + Name: "pr", + Format: "markdown", + Output: "reports/posthog-summary.md", + }} + + dir := t.TempDir() + configPath := filepath.Join(dir, "cleanr.yaml") + if err := cleanr.WriteConfigFile(configPath, cfg); err != nil { + t.Fatalf("write config: %v", err) + } + + t.Setenv("POSTHOG_PROJECT_API_KEY", "phc_test_token") + + var distinctID string + originalTransport := http.DefaultTransport + http.DefaultTransport = cliRoundTripperFunc(func(req *http.Request) (*http.Response, error) { + switch { + case req.Method == http.MethodPost && req.URL.String() == "https://us.i.posthog.com/batch/": + body, err := io.ReadAll(req.Body) + if err != nil { + t.Fatalf("read posthog batch body: %v", err) + } + var payload map[string]any + if err := json.Unmarshal(body, &payload); err != nil { + t.Fatalf("decode posthog batch body: %v\n%s", err, string(body)) + } + if got, _ := payload["api_key"].(string); got != "phc_test_token" { + t.Fatalf("unexpected api_key: %v", payload["api_key"]) + } + batch, ok := payload["batch"].([]any) + if !ok || len(batch) < 1 { + t.Fatalf("unexpected batch payload: %s", string(body)) + } + first, ok := batch[0].(map[string]any) + if !ok || first["event"] != "cleanr_run" { + t.Fatalf("unexpected first event: %s", string(body)) + } + gotDistinctID, _ := first["distinct_id"].(string) + if gotDistinctID == "" { + t.Fatalf("missing distinct_id in event: %s", string(body)) + } + distinctID = gotDistinctID + for _, want := range []string{`"event":"cleanr_run"`, `"cleanr_replay_artifact"`, `"cleanr_report"`} { + if !bytes.Contains(body, []byte(want)) { + t.Fatalf("expected %q in posthog payload: %s", want, string(body)) + } + } + return &http.Response{ + StatusCode: 200, + Header: http.Header{"Content-Type": []string{"application/json"}}, + Body: io.NopCloser(strings.NewReader(`{"status":1}`)), + }, nil + default: + t.Fatalf("unexpected request: %s %s", req.Method, req.URL.String()) + return nil, nil + } + }) + defer func() { http.DefaultTransport = originalTransport }() + + var stdout bytes.Buffer + var stderr bytes.Buffer + exitCode := cli.Run([]string{"run", "-config", configPath, "-format", "json"}, &stdout, &stderr) + if exitCode != 0 { + t.Fatalf("expected exit code 0, got %d stderr=%s", exitCode, stderr.String()) + } + + var report cleanr.Report + if err := json.Unmarshal(stdout.Bytes(), &report); err != nil { + t.Fatalf("decode report: %v\n%s", err, stdout.String()) + } + if report.Integrations == nil { + t.Fatalf("expected integrations in report") + } + if len(report.Integrations.ResultSinks) != 1 || !report.Integrations.ResultSinks[0].Published { + t.Fatalf("unexpected result sink report: %+v", report.Integrations.ResultSinks) + } + wantRunURL := "https://eu.posthog.com/project/demo/events?distinct_id=" + distinctID + if got := report.Integrations.ResultSinks[0].RunURL; got != wantRunURL { + t.Fatalf("unexpected run url: %s", got) + } + + summaryPath := filepath.Join(dir, "reports", "posthog-summary.md") + summaryBody, err := os.ReadFile(summaryPath) + if err != nil { + t.Fatalf("read summary: %v", err) + } + for _, want := range []string{ + "cleanr Release Summary", + "Local gate: `PASS`", + "Remote Views", + wantRunURL, + } { + if !strings.Contains(string(summaryBody), want) { + t.Fatalf("expected %q in summary:\n%s", want, string(summaryBody)) + } + } +} diff --git a/tests/cli/cli_sync_test.go b/tests/cli/cli_sync_test.go new file mode 100644 index 0000000..afd7770 --- /dev/null +++ b/tests/cli/cli_sync_test.go @@ -0,0 +1,335 @@ +package tests + +import ( + "bytes" + "net/http" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/devr-tools/cleanr/cleanr" + "github.com/devr-tools/cleanr/internal/cli" +) + +func TestSyncBraintrustCommandFetchesReplayAndAppliesConfigPatches(t *testing.T) { + cfg := cleanr.ExampleConfig() + cfg.Integrations.TrendSources = []cleanr.TrendSourceConfig{{ + Name: "braintrust", + Type: "braintrust", + Project: "qa-gates", + Experiment: "cleanr-ci", + APIKeyEnv: "BRAINTRUST_API_KEY", + }} + + dir := t.TempDir() + configPath := filepath.Join(dir, "cleanr.yaml") + if err := cleanr.WriteConfigFile(configPath, cfg); err != nil { + t.Fatalf("write config: %v", err) + } + + restore := stubCLITransport(t, cliRoundTripperFunc(func(req *http.Request) (*http.Response, error) { + switch { + case req.Method == http.MethodGet && req.URL.Path == "/v1/experiment": + return jsonCLIResponse(t, 200, map[string]any{ + "objects": []map[string]any{{ + "id": "exp-2", + "project_id": "proj-1", + "name": "cleanr-ci/build-2", + "created": "2026-05-28T12:00:00Z", + }}, + }), nil + case req.Method == http.MethodPost && req.URL.Path == "/btql": + body := decodeCLIRequestBody(t, req) + query := body["query"].(string) + switch { + case strings.Contains(query, "output.replay_artifact"): + return jsonCLIResponse(t, 200, map[string]any{ + "data": []map[string]any{{ + "replay_artifact": map[string]any{ + "version": "v1alpha1", + "target": cfg.Target.Name, + "build_id": "build-2", + "generated_at": "2026-05-28T12:00:00Z", + "passed": false, + "failed_cases": 1, + "failures": []map[string]any{{ + "suite": "security", + "name": "happy-path", + "failed": true, + "findings": []map[string]any{{ + "severity": "high", + "message": "review me", + }}, + }}, + }, + }}, + }), nil + case strings.Contains(query, "output.cleanr_sync"): + return jsonCLIResponse(t, 200, map[string]any{ + "data": []map[string]any{{ + "cleanr_sync": map[string]any{ + "version": "v1alpha1", + "source": "braintrust", + "config_patch": map[string]any{ + "operations": []map[string]any{ + { + "op": "set", + "path": "suites.token_optimization.max_output_tokens", + "value": 256, + }, + { + "op": "set", + "path": "scenarios[name=happy-path].system", + "value": "Use the verified password reset flow.", + }, + }, + }, + }, + }}, + }), nil + default: + t.Fatalf("unexpected btql query: %s", query) + return nil, nil + } + case req.Method == http.MethodGet && req.URL.Path == "/v1/experiment/exp-2/summarize": + return jsonCLIResponse(t, 200, map[string]any{ + "experiment_url": "https://braintrust.dev/app/cleanr-ci/build-2", + }), nil + default: + t.Fatalf("unexpected request: %s %s", req.Method, req.URL.String()) + return nil, nil + } + })) + defer restore() + + var stdout bytes.Buffer + var stderr bytes.Buffer + exitCode := cli.Run([]string{ + "sync", "braintrust", + "-config", configPath, + "-output-insights", "reports/braintrust.insights.yaml", + "-output-dataset", "reports/braintrust.dataset.yaml", + "-output-config", "cleanr.synced.yaml", + "-approve-insights", + }, &stdout, &stderr) + if exitCode != 0 { + t.Fatalf("expected sync success, code=%d stderr=%s", exitCode, stderr.String()) + } + + insightsPath := filepath.Join(dir, "reports", "braintrust.insights.yaml") + insights, err := cleanr.LoadBraintrustInsightDatasetFile(insightsPath) + if err != nil { + t.Fatalf("load insights: %v", err) + } + if insights.BuildID != "build-2" || insights.ExperimentURL != "https://braintrust.dev/app/cleanr-ci/build-2" { + t.Fatalf("unexpected insights metadata: %+v", insights) + } + if insights.ScenarioDataset == nil || len(insights.ScenarioDataset.Scenarios) != 1 { + t.Fatalf("expected one replay-derived scenario, got %+v", insights.ScenarioDataset) + } + + datasetPath := filepath.Join(dir, "reports", "braintrust.dataset.yaml") + dataset, err := cleanr.LoadScenarioDatasetFile(datasetPath) + if err != nil { + t.Fatalf("load dataset: %v", err) + } + if len(dataset.Scenarios) != 1 || dataset.Scenarios[0].Scenario.Name != "happy-path" { + t.Fatalf("unexpected synced dataset: %+v", dataset) + } + + syncedConfigPath := filepath.Join(dir, "cleanr.synced.yaml") + syncedCfg, err := cleanr.LoadConfigFile(syncedConfigPath) + if err != nil { + t.Fatalf("load synced config: %v", err) + } + if syncedCfg.Suites.TokenOptimization.MaxOutputTokens != 256 { + t.Fatalf("expected patched token threshold, got %+v", syncedCfg.Suites.TokenOptimization) + } + var happyPath cleanr.Scenario + for _, scenario := range syncedCfg.Scenarios { + if scenario.Name == "happy-path" { + happyPath = scenario + break + } + } + if happyPath.System != "Use the verified password reset flow." { + t.Fatalf("expected scenario system patch, got %+v", happyPath) + } + if !strings.Contains(strings.Join(happyPath.Tags, ","), "regression") { + t.Fatalf("expected regression tag after replay sync, got %+v", happyPath.Tags) + } +} + +func TestSyncBraintrustCommandRequiresApprovalForReviewRequiredInsights(t *testing.T) { + cfg := cleanr.ExampleConfig() + cfg.Integrations.TrendSources = []cleanr.TrendSourceConfig{{ + Type: "braintrust", + Project: "qa-gates", + Experiment: "cleanr-ci", + }} + + dir := t.TempDir() + configPath := filepath.Join(dir, "cleanr.yaml") + if err := cleanr.WriteConfigFile(configPath, cfg); err != nil { + t.Fatalf("write config: %v", err) + } + + restore := stubCLITransport(t, cliRoundTripperFunc(func(req *http.Request) (*http.Response, error) { + switch { + case req.Method == http.MethodGet && req.URL.Path == "/v1/experiment": + return jsonCLIResponse(t, 200, map[string]any{ + "objects": []map[string]any{{ + "id": "exp-3", + "project_id": "proj-1", + "name": "cleanr-ci/build-3", + "created": "2026-05-28T12:00:00Z", + }}, + }), nil + case req.Method == http.MethodPost && req.URL.Path == "/btql": + body := decodeCLIRequestBody(t, req) + query := body["query"].(string) + if strings.Contains(query, "output.replay_artifact") { + return jsonCLIResponse(t, 200, map[string]any{"data": []map[string]any{}}), nil + } + return jsonCLIResponse(t, 200, map[string]any{ + "data": []map[string]any{{ + "cleanr_sync": map[string]any{ + "version": "v1alpha1", + "review_required": true, + "config_patch": map[string]any{ + "operations": []map[string]any{{ + "op": "set", + "path": "suites.token_optimization.max_output_tokens", + "value": 128, + }}, + }, + }, + }}, + }), nil + case req.Method == http.MethodGet && req.URL.Path == "/v1/experiment/exp-3/summarize": + return jsonCLIResponse(t, 200, map[string]any{}), nil + default: + t.Fatalf("unexpected request: %s %s", req.Method, req.URL.String()) + return nil, nil + } + })) + defer restore() + + var stdout bytes.Buffer + var stderr bytes.Buffer + exitCode := cli.Run([]string{ + "sync", "braintrust", + "-config", configPath, + "-output-config", "cleanr.synced.yaml", + }, &stdout, &stderr) + if exitCode != 2 { + t.Fatalf("expected approval failure, code=%d stdout=%s stderr=%s", exitCode, stdout.String(), stderr.String()) + } + if !strings.Contains(stderr.String(), "requires explicit review") { + t.Fatalf("unexpected stderr: %s", stderr.String()) + } +} + +func TestSyncBraintrustCommandCanCreateGitHubPR(t *testing.T) { + cfg := cleanr.ExampleConfig() + cfg.Integrations.TrendSources = []cleanr.TrendSourceConfig{{ + Type: "braintrust", + Project: "qa-gates", + Experiment: "cleanr-ci", + }} + + dir := t.TempDir() + configPath := filepath.Join(dir, "cleanr.yaml") + if err := cleanr.WriteConfigFile(configPath, cfg); err != nil { + t.Fatalf("write config: %v", err) + } + logPath := filepath.Join(dir, "commands.log") + binDir := filepath.Join(dir, "bin") + if err := os.MkdirAll(binDir, 0o755); err != nil { + t.Fatalf("mkdir bin: %v", err) + } + for _, name := range []string{"git", "gh"} { + script := "#!/bin/sh\n" + + "echo \"" + name + " $@\" >> \"" + logPath + "\"\n" + if err := os.WriteFile(filepath.Join(binDir, name), []byte(script), 0o755); err != nil { + t.Fatalf("write %s stub: %v", name, err) + } + } + originalPath := os.Getenv("PATH") + t.Setenv("PATH", binDir+string(os.PathListSeparator)+originalPath) + + restore := stubCLITransport(t, cliRoundTripperFunc(func(req *http.Request) (*http.Response, error) { + switch { + case req.Method == http.MethodGet && req.URL.Path == "/v1/experiment": + return jsonCLIResponse(t, 200, map[string]any{ + "objects": []map[string]any{{ + "id": "exp-4", + "project_id": "proj-1", + "name": "cleanr-ci/build-4", + "created": "2026-05-28T12:00:00Z", + }}, + }), nil + case req.Method == http.MethodPost && req.URL.Path == "/btql": + body := decodeCLIRequestBody(t, req) + query := body["query"].(string) + if strings.Contains(query, "output.replay_artifact") { + return jsonCLIResponse(t, 200, map[string]any{"data": []map[string]any{}}), nil + } + return jsonCLIResponse(t, 200, map[string]any{ + "data": []map[string]any{{ + "cleanr_sync": map[string]any{ + "version": "v1alpha1", + "config_patch": map[string]any{ + "operations": []map[string]any{{ + "op": "set", + "path": "suites.token_optimization.max_output_tokens", + "value": 512, + }}, + }, + }, + }}, + }), nil + case req.Method == http.MethodGet && req.URL.Path == "/v1/experiment/exp-4/summarize": + return jsonCLIResponse(t, 200, map[string]any{ + "experiment_url": "https://braintrust.dev/app/cleanr-ci/build-4", + }), nil + default: + t.Fatalf("unexpected request: %s %s", req.Method, req.URL.String()) + return nil, nil + } + })) + defer restore() + + var stdout bytes.Buffer + var stderr bytes.Buffer + exitCode := cli.Run([]string{ + "sync", "braintrust", + "-config", configPath, + "-output-config", "cleanr.synced.yaml", + "-create-pr", + "-pr-branch", "cleanr-sync-branch", + "-pr-title", "Sync Braintrust insights", + "-pr-body", "Apply reviewed Braintrust insights.", + "-commit-message", "cleanr sync commit", + }, &stdout, &stderr) + if exitCode != 0 { + t.Fatalf("expected sync pr success, code=%d stderr=%s", exitCode, stderr.String()) + } + + logBody, err := os.ReadFile(logPath) + if err != nil { + t.Fatalf("read command log: %v", err) + } + logText := string(logBody) + for _, want := range []string{ + "git checkout -b cleanr-sync-branch", + "git add", + "git commit -m cleanr sync commit", + "gh pr create --title Sync Braintrust insights --body Apply reviewed Braintrust insights.", + } { + if !strings.Contains(logText, want) { + t.Fatalf("expected %q in command log:\n%s", want, logText) + } + } +} diff --git a/tests/cli/cli_test.go b/tests/cli/cli_test.go index 6ffd04d..a007ca6 100644 --- a/tests/cli/cli_test.go +++ b/tests/cli/cli_test.go @@ -2,8 +2,6 @@ package tests import ( "bytes" - "crypto/ed25519" - "encoding/base64" "encoding/json" "io" "net/http" @@ -11,9 +9,7 @@ import ( "path/filepath" "runtime" "strings" - "sync" "testing" - "time" "github.com/devr-tools/cleanr/cleanr" "github.com/devr-tools/cleanr/internal/cli" @@ -464,2447 +460,6 @@ func TestGenerateCommandWritesReviewedDataset(t *testing.T) { } } -func TestRunCommandSupportsExternalIntegrations(t *testing.T) { - cfg := cleanr.ExampleConfig() - cfg.Suites.PromptInjection.Enabled = false - cfg.Suites.Security.Enabled = false - cfg.Suites.Load.Enabled = false - cfg.Suites.Chaos.Enabled = false - cfg.Suites.Drift.Enabled = false - cfg.Suites.TokenOptimization.Enabled = false - cfg.Reporting.Format = "json" - cfg.Reporting.BuildID = "build-2" - cfg.Reporting.ReplayArtifactFile = "reports/cleanr.replay.json" - cfg.Integrations.TrendSources = []cleanr.TrendSourceConfig{{ - Name: "approved-history", - Type: "http", - URL: "https://example.test/history.yaml", - ViewURL: "https://braintrust.dev/app/history/build-1", - }} - cfg.Integrations.ResultSinks = []cleanr.ResultSinkConfig{{ - Name: "braintrust", - Type: "braintrust", - Endpoint: "https://example.test/publish", - Experiment: "release-gate", - IncludeReplay: true, - }} - cfg.Integrations.Summaries = []cleanr.SummaryConfig{{ - Name: "pr", - Format: "markdown", - Output: "reports/summary.md", - }} - - dir := t.TempDir() - configPath := filepath.Join(dir, "cleanr.yaml") - if err := cleanr.WriteConfigFile(configPath, cfg); err != nil { - t.Fatalf("write config: %v", err) - } - - historyPath := filepath.Join(dir, "history.yaml") - history := cleanr.TrendHistoryFile{ - Version: "v1alpha1", - Target: cfg.Target.Name, - Runs: []cleanr.TrendHistoryRun{{ - BuildID: "build-1", - GeneratedAt: time.Date(2026, 5, 19, 12, 0, 0, 0, time.UTC), - Passed: true, - Duration: time.Second, - FailedSuites: 0, - FailedCases: 0, - }}, - } - if err := cleanr.WriteTrendHistoryFile(historyPath, history); err != nil { - t.Fatalf("write history: %v", err) - } - historyBody, err := os.ReadFile(historyPath) - if err != nil { - t.Fatalf("read history: %v", err) - } - - originalTransport := http.DefaultTransport - http.DefaultTransport = cliRoundTripperFunc(func(req *http.Request) (*http.Response, error) { - switch req.URL.String() { - case "https://example.test/history.yaml": - return &http.Response{ - StatusCode: 200, - Header: http.Header{"Content-Type": []string{"application/x-yaml"}}, - Body: io.NopCloser(bytes.NewReader(historyBody)), - }, nil - case "https://example.test/publish": - body, err := io.ReadAll(req.Body) - if err != nil { - t.Fatalf("read publish body: %v", err) - } - if !bytes.Contains(body, []byte(`"replay_artifact"`)) || !bytes.Contains(body, []byte(`"source":"cleanr"`)) { - t.Fatalf("unexpected publish payload: %s", string(body)) - } - return &http.Response{ - StatusCode: 200, - Header: http.Header{"Content-Type": []string{"application/json"}}, - Body: io.NopCloser(strings.NewReader(`{"run_url":"https://braintrust.dev/app/release-gate/build-2"}`)), - }, nil - default: - t.Fatalf("unexpected request: %s", req.URL.String()) - return nil, nil - } - }) - defer func() { http.DefaultTransport = originalTransport }() - - var stdout bytes.Buffer - var stderr bytes.Buffer - exitCode := cli.Run([]string{"run", "-config", configPath, "-format", "json"}, &stdout, &stderr) - if exitCode != 0 { - t.Fatalf("expected exit code 0, got %d stderr=%s", exitCode, stderr.String()) - } - - var report cleanr.Report - if err := json.Unmarshal(stdout.Bytes(), &report); err != nil { - t.Fatalf("decode report: %v\n%s", err, stdout.String()) - } - if report.Integrations == nil { - t.Fatalf("expected integrations in report") - } - if len(report.Integrations.TrendSources) != 1 || report.Integrations.TrendSources[0].Status != "compared" { - t.Fatalf("unexpected trend source report: %+v", report.Integrations.TrendSources) - } - if len(report.Integrations.ResultSinks) != 1 || !report.Integrations.ResultSinks[0].Published { - t.Fatalf("unexpected result sink report: %+v", report.Integrations.ResultSinks) - } - if got := report.Integrations.ResultSinks[0].RunURL; got != "https://braintrust.dev/app/release-gate/build-2" { - t.Fatalf("unexpected run url: %s", got) - } - if len(report.Integrations.Summaries) != 1 || !report.Integrations.Summaries[0].Written { - t.Fatalf("unexpected summary report: %+v", report.Integrations.Summaries) - } - - summaryPath := filepath.Join(dir, "reports", "summary.md") - summaryBody, err := os.ReadFile(summaryPath) - if err != nil { - t.Fatalf("read summary: %v", err) - } - for _, want := range []string{ - "cleanr Release Summary", - "Local gate: `PASS`", - "approved-history", - "Remote Views", - "https://braintrust.dev/app/release-gate/build-2", - } { - if !strings.Contains(string(summaryBody), want) { - t.Fatalf("expected %q in summary:\n%s", want, string(summaryBody)) - } - } -} - -func TestRunCommandWritesBuildkiteMetadataAndAnnotation(t *testing.T) { - cfg := cleanr.ExampleConfig() - cfg.Suites.PromptInjection.Enabled = false - cfg.Suites.Security.Enabled = true - cfg.Suites.Security.MaxPIIMatches = 0 - cfg.Suites.Security.DangerousToolIndicators = []string{} - cfg.Suites.Security.SecretExposureIndicators = []string{} - cfg.Suites.Load.Enabled = false - cfg.Suites.Chaos.Enabled = false - cfg.Suites.Drift.Enabled = false - cfg.Suites.TokenOptimization.Enabled = false - cfg.Reporting.BuildID = "build-bk-1" - cfg.Scenarios = []cleanr.Scenario{{ - Name: "missing-phrase", - Input: "hello", - ExpectedContains: []string{"missing"}, - }} - - path := filepath.Join(t.TempDir(), "cleanr.yaml") - if err := cleanr.WriteConfigFile(path, cfg); err != nil { - t.Fatalf("write config: %v", err) - } - - logPath := installFakeBuildkiteAgent(t) - var stdout bytes.Buffer - var stderr bytes.Buffer - code := cli.Run([]string{"run", "-config", path, "-buildkite-meta", "-buildkite-annotation"}, &stdout, &stderr) - if code != 1 { - t.Fatalf("expected failing exit code 1, got %d stdout=%s stderr=%s", code, stdout.String(), stderr.String()) - } - - logBody, err := os.ReadFile(logPath) - if err != nil { - t.Fatalf("read buildkite log: %v", err) - } - logText := string(logBody) - for _, want := range []string{ - "meta-data set cleanr.run.passed false", - "meta-data set cleanr.run.failed_cases 1", - "meta-data set cleanr.run.build_id build-bk-1", - "annotate ### cleanr run failed", - } { - if !strings.Contains(logText, want) { - t.Fatalf("expected %q in buildkite log:\n%s", want, logText) - } - } -} - -func TestRunCommandSupportsNativeBraintrustConnector(t *testing.T) { - cfg := cleanr.ExampleConfig() - cfg.Suites.PromptInjection.Enabled = false - cfg.Suites.Security.Enabled = false - cfg.Suites.Load.Enabled = false - cfg.Suites.Chaos.Enabled = false - cfg.Suites.Drift.Enabled = false - cfg.Suites.TokenOptimization.Enabled = false - cfg.Reporting.Format = "json" - cfg.Reporting.BuildID = "build-2" - cfg.Integrations.TrendSources = []cleanr.TrendSourceConfig{{ - Name: "approved-history", - Type: "braintrust", - Project: "qa-gates", - Experiment: "release-gate", - ViewURL: "https://braintrust.dev/app/release-gate/build-1", - }} - cfg.Integrations.ResultSinks = []cleanr.ResultSinkConfig{{ - Name: "braintrust", - Type: "braintrust", - Project: "qa-gates", - Experiment: "release-gate", - APIKeyEnv: "CLEANR_BRAINTRUST_TOKEN", - IncludeReplay: true, - }} - cfg.Integrations.Summaries = []cleanr.SummaryConfig{{ - Name: "pr", - Format: "markdown", - Output: "reports/summary.md", - }} - - dir := t.TempDir() - configPath := filepath.Join(dir, "cleanr.yaml") - if err := cleanr.WriteConfigFile(configPath, cfg); err != nil { - t.Fatalf("write config: %v", err) - } - - t.Setenv("CLEANR_BRAINTRUST_TOKEN", "bt-secret") - - remoteRun := cleanr.TrendHistoryRun{ - BuildID: "build-1", - GeneratedAt: time.Date(2026, 5, 19, 12, 0, 0, 0, time.UTC), - Passed: true, - Duration: time.Second, - FailedSuites: 0, - FailedCases: 0, - Metadata: &cleanr.RunMetadata{ - BuildID: "build-1", - TargetType: "http", - ProviderModel: "gpt-4.1-mini", - }, - } - - originalTransport := http.DefaultTransport - http.DefaultTransport = cliRoundTripperFunc(func(req *http.Request) (*http.Response, error) { - switch { - case req.Method == http.MethodGet && req.URL.Path == "/v1/experiment": - if req.URL.Query().Get("project_name") != "qa-gates" { - t.Fatalf("unexpected project query: %s", req.URL.RawQuery) - } - if !strings.Contains(req.URL.Query().Get("metadata"), "release-gate") { - t.Fatalf("expected family filter in metadata query: %s", req.URL.RawQuery) - } - return &http.Response{ - StatusCode: 200, - Header: http.Header{"Content-Type": []string{"application/json"}}, - Body: io.NopCloser(strings.NewReader(`{"objects":[{"id":"exp-1","project_id":"proj-1","name":"release-gate/build-1","created":"2026-05-19T12:00:00Z"}]}`)), - }, nil - case req.Method == http.MethodPost && req.URL.Path == "/btql": - body, err := io.ReadAll(req.Body) - if err != nil { - t.Fatalf("read btql body: %v", err) - } - if !bytes.Contains(body, []byte("metadata.cleanr.history_run")) { - t.Fatalf("unexpected btql query: %s", string(body)) - } - raw, err := json.Marshal(map[string]any{"data": []map[string]any{{"history_run": remoteRun}}}) - if err != nil { - t.Fatalf("marshal btql response: %v", err) - } - return &http.Response{ - StatusCode: 200, - Header: http.Header{"Content-Type": []string{"application/json"}}, - Body: io.NopCloser(bytes.NewReader(raw)), - }, nil - case req.Method == http.MethodPost && req.URL.Path == "/v1/project": - if got := req.Header.Get("Authorization"); got != "Bearer bt-secret" { - t.Fatalf("unexpected auth header: %s", got) - } - body, err := io.ReadAll(req.Body) - if err != nil { - t.Fatalf("read project body: %v", err) - } - if !bytes.Contains(body, []byte(`"name":"qa-gates"`)) { - t.Fatalf("unexpected project payload: %s", string(body)) - } - return &http.Response{ - StatusCode: 200, - Header: http.Header{"Content-Type": []string{"application/json"}}, - Body: io.NopCloser(strings.NewReader(`{"id":"proj-1","name":"qa-gates"}`)), - }, nil - case req.Method == http.MethodPost && req.URL.Path == "/v1/experiment": - body, err := io.ReadAll(req.Body) - if err != nil { - t.Fatalf("read experiment body: %v", err) - } - for _, want := range []string{`"project_id":"proj-1"`, `"name":"release-gate/build-2"`, `"family":"release-gate"`} { - if !bytes.Contains(body, []byte(want)) { - t.Fatalf("expected %q in experiment payload: %s", want, string(body)) - } - } - return &http.Response{ - StatusCode: 200, - Header: http.Header{"Content-Type": []string{"application/json"}}, - Body: io.NopCloser(strings.NewReader(`{"id":"exp-2","project_id":"proj-1","name":"release-gate/build-2"}`)), - }, nil - case req.Method == http.MethodPost && req.URL.Path == "/v1/experiment/exp-2/insert": - body, err := io.ReadAll(req.Body) - if err != nil { - t.Fatalf("read insert body: %v", err) - } - for _, want := range []string{`"record_type":"run"`, `"history_run"`, `"replay_artifact"`} { - if !bytes.Contains(body, []byte(want)) { - t.Fatalf("expected %q in insert payload: %s", want, string(body)) - } - } - return &http.Response{ - StatusCode: 200, - Header: http.Header{"Content-Type": []string{"application/json"}}, - Body: io.NopCloser(strings.NewReader(`{}`)), - }, nil - case req.Method == http.MethodGet && req.URL.Path == "/v1/experiment/exp-2/summarize": - return &http.Response{ - StatusCode: 200, - Header: http.Header{"Content-Type": []string{"application/json"}}, - Body: io.NopCloser(strings.NewReader(`{"experiment_url":"https://braintrust.dev/app/release-gate/build-2"}`)), - }, nil - default: - t.Fatalf("unexpected request: %s %s", req.Method, req.URL.String()) - return nil, nil - } - }) - defer func() { http.DefaultTransport = originalTransport }() - - var stdout bytes.Buffer - var stderr bytes.Buffer - exitCode := cli.Run([]string{"run", "-config", configPath, "-format", "json"}, &stdout, &stderr) - if exitCode != 0 { - t.Fatalf("expected exit code 0, got %d stderr=%s", exitCode, stderr.String()) - } - - var report cleanr.Report - if err := json.Unmarshal(stdout.Bytes(), &report); err != nil { - t.Fatalf("decode report: %v\n%s", err, stdout.String()) - } - if report.Integrations == nil { - t.Fatalf("expected integrations in report") - } - if len(report.Integrations.TrendSources) != 1 || report.Integrations.TrendSources[0].Status != "compared" { - t.Fatalf("unexpected trend source report: %+v", report.Integrations.TrendSources) - } - if len(report.Integrations.ResultSinks) != 1 || !report.Integrations.ResultSinks[0].Published { - t.Fatalf("unexpected result sink report: %+v", report.Integrations.ResultSinks) - } - if got := report.Integrations.ResultSinks[0].RunURL; got != "https://braintrust.dev/app/release-gate/build-2" { - t.Fatalf("unexpected run url: %s", got) - } - - summaryPath := filepath.Join(dir, "reports", "summary.md") - summaryBody, err := os.ReadFile(summaryPath) - if err != nil { - t.Fatalf("read summary: %v", err) - } - for _, want := range []string{ - "cleanr Release Summary", - "Local gate: `PASS`", - "approved-history", - "Remote Views", - "https://braintrust.dev/app/release-gate/build-2", - } { - if !strings.Contains(string(summaryBody), want) { - t.Fatalf("expected %q in summary:\n%s", want, string(summaryBody)) - } - } -} - -func TestRunCommandSupportsNativeLangfuseConnector(t *testing.T) { - cfg := cleanr.ExampleConfig() - cfg.Suites.PromptInjection.Enabled = false - cfg.Suites.Security.Enabled = false - cfg.Suites.Load.Enabled = false - cfg.Suites.Chaos.Enabled = false - cfg.Suites.Drift.Enabled = false - cfg.Suites.TokenOptimization.Enabled = false - cfg.Reporting.Format = "json" - cfg.Reporting.BuildID = "build-2" - cfg.Integrations.ResultSinks = []cleanr.ResultSinkConfig{{ - Name: "langfuse", - Type: "langfuse", - BaseURL: "https://cloud.langfuse.com", - PublicKeyEnv: "LANGFUSE_PUBLIC_KEY", - SecretKeyEnv: "LANGFUSE_SECRET_KEY", - Experiment: "release-gate", - IncludeReplay: true, - RunURLTemplate: "https://cloud.langfuse.com/project/demo/traces/{{trace_id}}", - }} - cfg.Integrations.Summaries = []cleanr.SummaryConfig{{ - Name: "pr", - Format: "markdown", - Output: "reports/langfuse-summary.md", - }} - - dir := t.TempDir() - configPath := filepath.Join(dir, "cleanr.yaml") - if err := cleanr.WriteConfigFile(configPath, cfg); err != nil { - t.Fatalf("write config: %v", err) - } - - t.Setenv("LANGFUSE_PUBLIC_KEY", "pk-test") - t.Setenv("LANGFUSE_SECRET_KEY", "sk-test") - - var traceID string - scorePosts := 0 - wantAuth := "Basic " + base64.StdEncoding.EncodeToString([]byte("pk-test:sk-test")) - - originalTransport := http.DefaultTransport - http.DefaultTransport = cliRoundTripperFunc(func(req *http.Request) (*http.Response, error) { - if got := req.Header.Get("Authorization"); got != wantAuth { - t.Fatalf("unexpected auth header: %s", got) - } - switch { - case req.Method == http.MethodPost && req.URL.String() == "https://cloud.langfuse.com/api/public/ingestion": - body, err := io.ReadAll(req.Body) - if err != nil { - t.Fatalf("read langfuse ingestion body: %v", err) - } - var payload map[string]any - if err := json.Unmarshal(body, &payload); err != nil { - t.Fatalf("decode langfuse ingestion body: %v\n%s", err, string(body)) - } - batch, ok := payload["batch"].([]any) - if !ok || len(batch) != 1 { - t.Fatalf("unexpected batch payload: %s", string(body)) - } - event, ok := batch[0].(map[string]any) - if !ok || event["type"] != "trace-create" { - t.Fatalf("unexpected trace event: %s", string(body)) - } - eventBody, ok := event["body"].(map[string]any) - if !ok { - t.Fatalf("missing event body: %s", string(body)) - } - gotTraceID, _ := eventBody["id"].(string) - if gotTraceID == "" { - t.Fatalf("missing trace id in event body: %s", string(body)) - } - traceID = gotTraceID - for _, want := range []string{`"type":"trace-create"`, `"name":"release-gate"`, `"sessionId":"build-2"`, `"replay_artifact"`} { - if !bytes.Contains(body, []byte(want)) { - t.Fatalf("expected %q in ingestion payload: %s", want, string(body)) - } - } - return &http.Response{ - StatusCode: 200, - Header: http.Header{"Content-Type": []string{"application/json"}}, - Body: io.NopCloser(strings.NewReader(`{"success":true}`)), - }, nil - case req.Method == http.MethodPost && req.URL.String() == "https://cloud.langfuse.com/api/public/scores": - scorePosts++ - body, err := io.ReadAll(req.Body) - if err != nil { - t.Fatalf("read langfuse score body: %v", err) - } - if traceID == "" { - t.Fatalf("score posted before trace ingestion: %s", string(body)) - } - for _, want := range []string{`"traceId":"` + traceID + `"`, `"dataType":"NUMERIC"`} { - if !bytes.Contains(body, []byte(want)) { - t.Fatalf("expected %q in score payload: %s", want, string(body)) - } - } - return &http.Response{ - StatusCode: 200, - Header: http.Header{"Content-Type": []string{"application/json"}}, - Body: io.NopCloser(strings.NewReader(`{"success":true}`)), - }, nil - default: - t.Fatalf("unexpected request: %s %s", req.Method, req.URL.String()) - return nil, nil - } - }) - defer func() { http.DefaultTransport = originalTransport }() - - var stdout bytes.Buffer - var stderr bytes.Buffer - exitCode := cli.Run([]string{"run", "-config", configPath, "-format", "json"}, &stdout, &stderr) - if exitCode != 0 { - t.Fatalf("expected exit code 0, got %d stderr=%s", exitCode, stderr.String()) - } - if scorePosts < 3 { - t.Fatalf("expected at least 3 score posts, got %d", scorePosts) - } - - var report cleanr.Report - if err := json.Unmarshal(stdout.Bytes(), &report); err != nil { - t.Fatalf("decode report: %v\n%s", err, stdout.String()) - } - if report.Integrations == nil { - t.Fatalf("expected integrations in report") - } - if len(report.Integrations.ResultSinks) != 1 || !report.Integrations.ResultSinks[0].Published { - t.Fatalf("unexpected result sink report: %+v", report.Integrations.ResultSinks) - } - wantRunURL := "https://cloud.langfuse.com/project/demo/traces/" + traceID - if got := report.Integrations.ResultSinks[0].RunURL; got != wantRunURL { - t.Fatalf("unexpected run url: %s", got) - } - - summaryPath := filepath.Join(dir, "reports", "langfuse-summary.md") - summaryBody, err := os.ReadFile(summaryPath) - if err != nil { - t.Fatalf("read summary: %v", err) - } - for _, want := range []string{ - "cleanr Release Summary", - "Local gate: `PASS`", - "Remote Views", - wantRunURL, - } { - if !strings.Contains(string(summaryBody), want) { - t.Fatalf("expected %q in summary:\n%s", want, string(summaryBody)) - } - } -} - -func TestRunCommandSupportsNativePostHogConnector(t *testing.T) { - cfg := cleanr.ExampleConfig() - cfg.Suites.PromptInjection.Enabled = false - cfg.Suites.Security.Enabled = false - cfg.Suites.Load.Enabled = false - cfg.Suites.Chaos.Enabled = false - cfg.Suites.Drift.Enabled = false - cfg.Suites.TokenOptimization.Enabled = false - cfg.Reporting.Format = "json" - cfg.Reporting.BuildID = "build-2" - cfg.Integrations.ResultSinks = []cleanr.ResultSinkConfig{{ - Name: "posthog", - Type: "posthog", - BaseURL: "https://us.i.posthog.com", - ProjectTokenEnv: "POSTHOG_PROJECT_API_KEY", - Experiment: "release-gate", - IncludeReplay: true, - RunURLTemplate: "https://eu.posthog.com/project/demo/events?distinct_id={{distinct_id}}", - }} - cfg.Integrations.Summaries = []cleanr.SummaryConfig{{ - Name: "pr", - Format: "markdown", - Output: "reports/posthog-summary.md", - }} - - dir := t.TempDir() - configPath := filepath.Join(dir, "cleanr.yaml") - if err := cleanr.WriteConfigFile(configPath, cfg); err != nil { - t.Fatalf("write config: %v", err) - } - - t.Setenv("POSTHOG_PROJECT_API_KEY", "phc_test_token") - - var distinctID string - originalTransport := http.DefaultTransport - http.DefaultTransport = cliRoundTripperFunc(func(req *http.Request) (*http.Response, error) { - switch { - case req.Method == http.MethodPost && req.URL.String() == "https://us.i.posthog.com/batch/": - body, err := io.ReadAll(req.Body) - if err != nil { - t.Fatalf("read posthog batch body: %v", err) - } - var payload map[string]any - if err := json.Unmarshal(body, &payload); err != nil { - t.Fatalf("decode posthog batch body: %v\n%s", err, string(body)) - } - if got, _ := payload["api_key"].(string); got != "phc_test_token" { - t.Fatalf("unexpected api_key: %v", payload["api_key"]) - } - batch, ok := payload["batch"].([]any) - if !ok || len(batch) < 1 { - t.Fatalf("unexpected batch payload: %s", string(body)) - } - first, ok := batch[0].(map[string]any) - if !ok || first["event"] != "cleanr_run" { - t.Fatalf("unexpected first event: %s", string(body)) - } - gotDistinctID, _ := first["distinct_id"].(string) - if gotDistinctID == "" { - t.Fatalf("missing distinct_id in event: %s", string(body)) - } - distinctID = gotDistinctID - for _, want := range []string{`"event":"cleanr_run"`, `"cleanr_replay_artifact"`, `"cleanr_report"`} { - if !bytes.Contains(body, []byte(want)) { - t.Fatalf("expected %q in posthog payload: %s", want, string(body)) - } - } - return &http.Response{ - StatusCode: 200, - Header: http.Header{"Content-Type": []string{"application/json"}}, - Body: io.NopCloser(strings.NewReader(`{"status":1}`)), - }, nil - default: - t.Fatalf("unexpected request: %s %s", req.Method, req.URL.String()) - return nil, nil - } - }) - defer func() { http.DefaultTransport = originalTransport }() - - var stdout bytes.Buffer - var stderr bytes.Buffer - exitCode := cli.Run([]string{"run", "-config", configPath, "-format", "json"}, &stdout, &stderr) - if exitCode != 0 { - t.Fatalf("expected exit code 0, got %d stderr=%s", exitCode, stderr.String()) - } - - var report cleanr.Report - if err := json.Unmarshal(stdout.Bytes(), &report); err != nil { - t.Fatalf("decode report: %v\n%s", err, stdout.String()) - } - if report.Integrations == nil { - t.Fatalf("expected integrations in report") - } - if len(report.Integrations.ResultSinks) != 1 || !report.Integrations.ResultSinks[0].Published { - t.Fatalf("unexpected result sink report: %+v", report.Integrations.ResultSinks) - } - wantRunURL := "https://eu.posthog.com/project/demo/events?distinct_id=" + distinctID - if got := report.Integrations.ResultSinks[0].RunURL; got != wantRunURL { - t.Fatalf("unexpected run url: %s", got) - } - - summaryPath := filepath.Join(dir, "reports", "posthog-summary.md") - summaryBody, err := os.ReadFile(summaryPath) - if err != nil { - t.Fatalf("read summary: %v", err) - } - for _, want := range []string{ - "cleanr Release Summary", - "Local gate: `PASS`", - "Remote Views", - wantRunURL, - } { - if !strings.Contains(string(summaryBody), want) { - t.Fatalf("expected %q in summary:\n%s", want, string(summaryBody)) - } - } -} - -func TestDatasetExportAndImportCommands(t *testing.T) { - cfg := cleanr.ExampleConfig() - cfg.Scenarios = []cleanr.Scenario{ - {Name: "happy-path", Input: "hello", Tags: []string{"stable"}}, - {Name: "security-path", Input: "secret"}, - } - configPath := filepath.Join(t.TempDir(), "cleanr.yaml") - if err := cleanr.WriteConfigFile(configPath, cfg); err != nil { - t.Fatalf("write config: %v", err) - } - - replayPath := filepath.Join(filepath.Dir(configPath), "reports", "cleanr.replay.json") - artifact := cleanr.ReplayArtifact{ - Version: "v1alpha1", - Target: cfg.Target.Name, - BuildID: "build-9", - GeneratedAt: time.Now().UTC(), - Failures: []cleanr.ReplayArtifactCase{{ - Suite: "security", - Name: "happy-path", - Findings: []cleanr.Finding{{ - Severity: "high", - Message: "review me", - }}, - Failed: true, - }}, - } - if err := cleanr.WriteReplayArtifactFile(replayPath, artifact); err != nil { - t.Fatalf("write replay artifact: %v", err) - } - - datasetPath := filepath.Join(filepath.Dir(configPath), "reviewed-failures.yaml") - var stdout bytes.Buffer - var stderr bytes.Buffer - if code := cli.Run([]string{"dataset", "export", "-config", configPath, "-replay-artifact", replayPath, "-output", datasetPath}, &stdout, &stderr); code != 0 { - t.Fatalf("expected dataset export success, code=%d stderr=%s", code, stderr.String()) - } - - dataset, err := cleanr.LoadScenarioDatasetFile(datasetPath) - if err != nil { - t.Fatalf("load dataset: %v", err) - } - if len(dataset.Scenarios) != 1 || dataset.Scenarios[0].Scenario.Name != "happy-path" { - t.Fatalf("unexpected dataset scenarios: %+v", dataset.Scenarios) - } - if !strings.Contains(strings.Join(dataset.Scenarios[0].Scenario.Tags, ","), "regression") { - t.Fatalf("expected regression tag, got %+v", dataset.Scenarios[0].Scenario.Tags) - } - - baseCfg := cleanr.ExampleConfig() - baseCfg.Scenarios = []cleanr.Scenario{{Name: "legacy", Input: "existing"}} - basePath := filepath.Join(filepath.Dir(configPath), "base.yaml") - if err := cleanr.WriteConfigFile(basePath, baseCfg); err != nil { - t.Fatalf("write base config: %v", err) - } - - importedPath := filepath.Join(filepath.Dir(configPath), "imported.yaml") - stdout.Reset() - stderr.Reset() - if code := cli.Run([]string{"dataset", "import", "-input", datasetPath, "-base-config", basePath, "-output", importedPath}, &stdout, &stderr); code != 0 { - t.Fatalf("expected dataset import success, code=%d stderr=%s", code, stderr.String()) - } - - importedCfg, err := cleanr.LoadConfigFile(importedPath) - if err != nil { - t.Fatalf("load imported config: %v", err) - } - if len(importedCfg.Scenarios) != 2 { - t.Fatalf("expected merged scenarios, got %+v", importedCfg.Scenarios) - } - names := []string{importedCfg.Scenarios[0].Name, importedCfg.Scenarios[1].Name} - if !strings.Contains(strings.Join(names, ","), "happy-path") || !strings.Contains(strings.Join(names, ","), "legacy") { - t.Fatalf("unexpected merged scenario names: %+v", names) - } -} - -func TestDatasetImportRequiresApprovalForGeneratedDatasets(t *testing.T) { - datasetPath := filepath.Join(t.TempDir(), "generated.yaml") - dataset := cleanr.ScenarioDataset{ - Version: "v1alpha1", - Source: "cleanr-generation", - GeneratedAt: time.Now().UTC(), - ReviewRequired: true, - Scenarios: []cleanr.ScenarioDatasetEntry{{ - Scenario: cleanr.Scenario{Name: "generated-happy-path", Input: "Summarize the refund policy.", Tags: []string{"generated"}}, - }}, - } - if err := cleanr.WriteScenarioDatasetFile(datasetPath, dataset); err != nil { - t.Fatalf("write dataset: %v", err) - } - - importedPath := filepath.Join(filepath.Dir(datasetPath), "imported.yaml") - var stdout bytes.Buffer - var stderr bytes.Buffer - if code := cli.Run([]string{"dataset", "import", "-input", datasetPath, "-output", importedPath}, &stdout, &stderr); code != 2 { - t.Fatalf("expected approval failure, code=%d stdout=%s stderr=%s", code, stdout.String(), stderr.String()) - } - if !strings.Contains(stderr.String(), "requires explicit review") { - t.Fatalf("unexpected stderr: %s", stderr.String()) - } - - stdout.Reset() - stderr.Reset() - if code := cli.Run([]string{"dataset", "import", "-input", datasetPath, "-output", importedPath, "-approve-generated"}, &stdout, &stderr); code != 0 { - t.Fatalf("expected approved import success, code=%d stderr=%s", code, stderr.String()) - } - - importedCfg, err := cleanr.LoadConfigFile(importedPath) - if err != nil { - t.Fatalf("load imported config: %v", err) - } - names := make([]string, 0, len(importedCfg.Scenarios)) - for _, scenario := range importedCfg.Scenarios { - names = append(names, scenario.Name) - } - if !strings.Contains(strings.Join(names, ","), "generated-happy-path") { - t.Fatalf("unexpected imported config: %+v", importedCfg.Scenarios) - } -} - -func TestDatasetReviewCommandWritesReviewedArtifactAndMergedConfig(t *testing.T) { - baseCfg := cleanr.ExampleConfig() - baseCfg.Scenarios = []cleanr.Scenario{ - { - Name: "happy-path", - System: "You are a helpful assistant.", - Input: "Help with a refund.", - Tags: []string{"stable"}, - ExpectedContains: []string{"refund"}, - }, - { - Name: "legacy-duplicate", - System: "You are a helpful assistant.", - Input: "Reset my password.", - Tags: []string{"legacy"}, - }, - } - - dir := t.TempDir() - basePath := filepath.Join(dir, "cleanr.yaml") - if err := cleanr.WriteConfigFile(basePath, baseCfg); err != nil { - t.Fatalf("write base config: %v", err) - } - - dataset := cleanr.ScenarioDataset{ - Version: "v1alpha1", - Source: "cleanr-replay", - Target: baseCfg.Target.Name, - BuildID: "build-77", - GeneratedAt: time.Now().UTC(), - Scenarios: []cleanr.ScenarioDatasetEntry{ - { - Scenario: cleanr.Scenario{ - Name: "happy-path", - System: "You are a helpful assistant.", - Input: "Help with a refund after an account lockout.", - Tags: []string{"generated"}, - ExpectedContains: []string{"refund", "account"}, - }, - Origin: cleanr.DatasetScenarioOrigin{ - Suite: "security", - Case: "happy-path", - BuildID: "build-77", - Findings: []cleanr.Finding{{ - Severity: "high", - Message: "important regression", - }}, - }, - }, - { - Scenario: cleanr.Scenario{ - Name: "duplicate-reset", - System: "You are a helpful assistant.", - Input: "Reset my password.", - Tags: []string{"generated"}, - }, - }, - { - Scenario: cleanr.Scenario{ - Name: "new-stable-candidate", - System: "You are a helpful assistant.", - Input: "Summarize the refund policy in one sentence.", - Tags: []string{"generated"}, - ExpectedContains: []string{"refund"}, - }, - }, - }, - } - datasetPath := filepath.Join(dir, "dataset.yaml") - if err := cleanr.WriteScenarioDatasetFile(datasetPath, dataset); err != nil { - t.Fatalf("write dataset: %v", err) - } - - reviewedPath := filepath.Join(dir, "reviewed.yaml") - mergedPath := filepath.Join(dir, "merged.yaml") - var stdout bytes.Buffer - var stderr bytes.Buffer - code := cli.Run([]string{ - "dataset", "review", - "-input", datasetPath, - "-base-config", basePath, - "-output", reviewedPath, - "-merge-output", mergedPath, - "-approve", "happy-path,new-stable-candidate", - "-reject", "duplicate-reset", - "-promote-regression", "happy-path", - "-promote-stable", "new-stable-candidate", - "-add-tag", "happy-path:security", - "-set-metadata", "happy-path:owner=qa", - }, &stdout, &stderr) - if code != 0 { - t.Fatalf("expected dataset review success, code=%d stdout=%s stderr=%s", code, stdout.String(), stderr.String()) - } - - reviewed, err := cleanr.LoadReviewedScenarioDatasetFile(reviewedPath) - if err != nil { - t.Fatalf("load reviewed dataset: %v", err) - } - if reviewed.ApprovedScenarios != 2 || reviewed.RejectedScenarios != 1 || reviewed.PendingScenarios != 0 { - t.Fatalf("unexpected review counts: %+v", reviewed) - } - if len(reviewed.Scenarios) != 3 { - t.Fatalf("unexpected reviewed scenarios: %+v", reviewed.Scenarios) - } - if reviewed.Scenarios[0].Entry.Scenario.Name != "happy-path" { - t.Fatalf("expected high severity scenario to rank first, got %+v", reviewed.Scenarios) - } - if reviewed.Scenarios[0].Decision.Status != "approved" || reviewed.Scenarios[0].Diff.Status != "modified" { - t.Fatalf("unexpected reviewed entry: %+v", reviewed.Scenarios[0]) - } - if reviewed.Scenarios[0].Entry.Origin.Suite != "security" { - t.Fatalf("expected provenance to be preserved, got %+v", reviewed.Scenarios[0].Entry.Origin) - } - - duplicateSeen := false - for _, item := range reviewed.Scenarios { - if item.Entry.Scenario.Name == "duplicate-reset" { - duplicateSeen = true - if item.Decision.Status != "rejected" || item.Diff.Status != "duplicate" { - t.Fatalf("unexpected duplicate entry: %+v", item) - } - } - } - if !duplicateSeen { - t.Fatalf("expected duplicate candidate in reviewed dataset: %+v", reviewed.Scenarios) - } - - merged, err := cleanr.LoadConfigFile(mergedPath) - if err != nil { - t.Fatalf("load merged config: %v", err) - } - if len(merged.Scenarios) != 3 { - t.Fatalf("expected legacy plus two approved scenarios, got %+v", merged.Scenarios) - } - - mergedByName := map[string]cleanr.Scenario{} - for _, scenario := range merged.Scenarios { - mergedByName[scenario.Name] = scenario - } - if _, ok := mergedByName["duplicate-reset"]; ok { - t.Fatalf("rejected scenario should not be merged: %+v", merged.Scenarios) - } - happy := mergedByName["happy-path"] - if !strings.Contains(strings.Join(happy.Tags, ","), "regression") || !strings.Contains(strings.Join(happy.Tags, ","), "security") { - t.Fatalf("expected promoted tags on approved scenario, got %+v", happy.Tags) - } - if happy.Metadata["owner"] != "qa" { - t.Fatalf("expected metadata edit to persist, got %+v", happy.Metadata) - } - if happy.Metadata["cleanr.review.source"] != "cleanr-replay" || happy.Metadata["cleanr.review.origin_suite"] != "security" { - t.Fatalf("expected review provenance metadata, got %+v", happy.Metadata) - } - - newStable := mergedByName["new-stable-candidate"] - if !strings.Contains(strings.Join(newStable.Tags, ","), "stable") { - t.Fatalf("expected stable promotion to persist, got %+v", newStable.Tags) - } -} - -func TestDatasetReviewCommandSupportsCIGatesAndGitHubOutputs(t *testing.T) { - baseCfg := cleanr.ExampleConfig() - baseCfg.Scenarios = []cleanr.Scenario{ - {Name: "existing", System: "You are helpful.", Input: "Reset my password."}, - } - - dir := t.TempDir() - basePath := filepath.Join(dir, "cleanr.yaml") - if err := cleanr.WriteConfigFile(basePath, baseCfg); err != nil { - t.Fatalf("write base config: %v", err) - } - - dataset := cleanr.ScenarioDataset{ - Version: "v1alpha1", - Source: "cleanr-generation", - Target: baseCfg.Target.Name, - GeneratedAt: time.Now().UTC(), - Scenarios: []cleanr.ScenarioDatasetEntry{ - {Scenario: cleanr.Scenario{Name: "needs-review", System: "You are helpful.", Input: "What is the refund policy?", Tags: []string{"generated"}}}, - {Scenario: cleanr.Scenario{Name: "dup", System: "You are helpful.", Input: "Reset my password.", Tags: []string{"generated"}}}, - }, - } - datasetPath := filepath.Join(dir, "dataset.yaml") - if err := cleanr.WriteScenarioDatasetFile(datasetPath, dataset); err != nil { - t.Fatalf("write dataset: %v", err) - } - - githubOutputPath := filepath.Join(dir, "github-output.txt") - githubSummaryPath := filepath.Join(dir, "github-summary.md") - if err := os.WriteFile(githubOutputPath, []byte("existing-output=true\n"), 0o644); err != nil { - t.Fatalf("seed github output: %v", err) - } - if err := os.WriteFile(githubSummaryPath, []byte("Existing summary\n"), 0o644); err != nil { - t.Fatalf("seed github summary: %v", err) - } - t.Setenv("GITHUB_OUTPUT", githubOutputPath) - t.Setenv("GITHUB_STEP_SUMMARY", githubSummaryPath) - - var stdout bytes.Buffer - var stderr bytes.Buffer - code := cli.Run([]string{ - "dataset", "review", - "-input", datasetPath, - "-base-config", basePath, - "-output", filepath.Join(dir, "reviewed.yaml"), - "-approve", "dup", - "-fail-on-pending", - "-max-duplicates", "0", - "-github-outputs", - }, &stdout, &stderr) - if code != 1 { - t.Fatalf("expected gate failure exit code 1, got %d stdout=%s stderr=%s", code, stdout.String(), stderr.String()) - } - if !strings.Contains(stderr.String(), "found 1 pending scenarios") { - t.Fatalf("expected pending gate failure, got stderr=%s", stderr.String()) - } - if !strings.Contains(stderr.String(), "duplicate candidate count 1 exceeds maximum 0") { - t.Fatalf("expected duplicate gate failure, got stderr=%s", stderr.String()) - } - - outputBody, err := os.ReadFile(githubOutputPath) - if err != nil { - t.Fatalf("read github output: %v", err) - } - outputText := string(outputBody) - for _, want := range []string{ - "existing-output=true", - "cleanr_review_gate_passed=false", - "cleanr_review_pending=1", - "cleanr_review_duplicates=1", - "cleanr_review_policy_path=", - "cleanr_review_top_candidate=", - } { - if !strings.Contains(outputText, want) { - t.Fatalf("expected %q in GITHUB_OUTPUT:\n%s", want, outputText) - } - } - - summaryBody, err := os.ReadFile(githubSummaryPath) - if err != nil { - t.Fatalf("read github summary: %v", err) - } - summaryText := string(summaryBody) - for _, want := range []string{ - "Existing summary", - "## cleanr Dataset Review", - "Gate passed: `false`", - "Pending: `1`", - "Duplicates: `1`", - "Review artifact: `", - "Gate findings:", - } { - if !strings.Contains(summaryText, want) { - t.Fatalf("expected %q in summary:\n%s", want, summaryText) - } - } -} - -func TestDatasetReviewCommandSupportsBuildkiteOutputs(t *testing.T) { - baseCfg := cleanr.ExampleConfig() - baseCfg.Scenarios = []cleanr.Scenario{ - {Name: "existing", System: "You are helpful.", Input: "Reset my password."}, - } - - dir := t.TempDir() - basePath := filepath.Join(dir, "cleanr.yaml") - if err := cleanr.WriteConfigFile(basePath, baseCfg); err != nil { - t.Fatalf("write base config: %v", err) - } - - dataset := cleanr.ScenarioDataset{ - Version: "v1alpha1", - Source: "cleanr-generation", - Target: baseCfg.Target.Name, - GeneratedAt: time.Now().UTC(), - Scenarios: []cleanr.ScenarioDatasetEntry{ - {Scenario: cleanr.Scenario{Name: "needs-review", System: "You are helpful.", Input: "What is the refund policy?", Tags: []string{"generated"}}}, - {Scenario: cleanr.Scenario{Name: "dup", System: "You are helpful.", Input: "Reset my password.", Tags: []string{"generated"}}}, - }, - } - datasetPath := filepath.Join(dir, "dataset.yaml") - if err := cleanr.WriteScenarioDatasetFile(datasetPath, dataset); err != nil { - t.Fatalf("write dataset: %v", err) - } - - logPath := installFakeBuildkiteAgent(t) - var stdout bytes.Buffer - var stderr bytes.Buffer - code := cli.Run([]string{ - "dataset", "review", - "-input", datasetPath, - "-base-config", basePath, - "-output", filepath.Join(dir, "reviewed.yaml"), - "-merge-output", filepath.Join(dir, "merged.yaml"), - "-approve", "dup", - "-fail-on-pending", - "-max-duplicates", "0", - "-buildkite-meta", - "-buildkite-annotation", - "-buildkite-upload-artifacts", - }, &stdout, &stderr) - if code != 1 { - t.Fatalf("expected gate failure exit code 1, got %d stdout=%s stderr=%s", code, stdout.String(), stderr.String()) - } - - logBody, err := os.ReadFile(logPath) - if err != nil { - t.Fatalf("read buildkite log: %v", err) - } - logText := string(logBody) - for _, want := range []string{ - "meta-data set cleanr.review.gate_passed false", - "meta-data set cleanr.review.pending 1", - "meta-data set cleanr.review.duplicates 1", - "meta-data set cleanr.review.policy_path ", - "annotate ### cleanr dataset review gate failed", - "artifact upload " + filepath.Join(dir, "reviewed.yaml"), - "artifact upload " + filepath.Join(dir, "merged.yaml"), - } { - if !strings.Contains(logText, want) { - t.Fatalf("expected %q in buildkite log:\n%s", want, logText) - } - } -} - -func TestDatasetReviewCommandAppliesCheckedInPolicyBeforeManualOverrides(t *testing.T) { - baseCfg := cleanr.ExampleConfig() - baseCfg.Scenarios = []cleanr.Scenario{ - { - Name: "existing-refund", - System: "You are helpful.", - Input: "Summarize the refund policy.", - Tags: []string{"stable"}, - ExpectedContains: []string{"refund"}, - Assertions: []cleanr.Assertion{{ - Type: "contains", - Value: "refund", - }}, - }, - } - - dir := t.TempDir() - basePath := filepath.Join(dir, "cleanr.yaml") - if err := cleanr.WriteConfigFile(basePath, baseCfg); err != nil { - t.Fatalf("write base config: %v", err) - } - - policy := cleanr.DatasetReviewPolicy{ - Version: "v1alpha1", - Rules: []cleanr.DatasetReviewPolicyRule{ - { - Action: "reject", - Statuses: []string{"duplicate"}, - }, - { - Action: "approve", - Statuses: []string{"modified"}, - MinSeverity: "high", - }, - { - Action: "promote-regression", - Statuses: []string{"modified"}, - MinSeverity: "high", - }, - { - Action: "promote-stable", - Statuses: []string{"new"}, - StableSuitability: "medium", - Sources: []string{"cleanr-replay"}, - RequireAssertions: true, - RequireExpectedText: true, - }, - { - Action: "set-metadata", - Statuses: []string{"modified", "new"}, - Metadata: map[string]string{"owner": "qa"}, - }, - { - Action: "add-tags", - GeneratorProviders: []string{"openai"}, - GeneratorModels: []string{"gpt-4.1-mini"}, - ScenarioTags: []string{"generated"}, - Tags: []string{"generator-reviewed"}, - }, - }, - } - policyPath := filepath.Join(dir, "cleanr.review.yaml") - if err := cleanr.WriteDatasetReviewPolicyFile(policyPath, policy); err != nil { - t.Fatalf("write review policy: %v", err) - } - - dataset := cleanr.ScenarioDataset{ - Version: "v1alpha1", - Source: "cleanr-replay", - Target: baseCfg.Target.Name, - BuildID: "build-policy-1", - GeneratedAt: time.Now().UTC(), - Generator: &cleanr.ScenarioDatasetGenerator{ - Provider: "openai", - Model: "gpt-4.1-mini", - }, - Scenarios: []cleanr.ScenarioDatasetEntry{ - { - Scenario: cleanr.Scenario{ - Name: "existing-refund", - System: "You are helpful.", - Input: "Summarize the refund policy for a locked account.", - Tags: []string{"generated"}, - ExpectedContains: []string{"refund"}, - }, - Origin: cleanr.DatasetScenarioOrigin{ - Suite: "security", - Case: "existing-refund", - Findings: []cleanr.Finding{{ - Severity: "high", - Message: "important", - }}, - }, - }, - { - Scenario: cleanr.Scenario{ - Name: "new-stable", - System: "You are helpful.", - Input: "Summarize the support hours in one sentence.", - Tags: []string{"generated", "policy"}, - ExpectedContains: []string{"weekday"}, - Assertions: []cleanr.Assertion{{ - Type: "contains", - Value: "weekday", - }}, - }, - }, - { - Scenario: cleanr.Scenario{ - Name: "duplicate-refund-copy", - System: "You are helpful.", - Input: "Summarize the refund policy.", - Tags: []string{"generated"}, - }, - }, - }, - } - datasetPath := filepath.Join(dir, "dataset.yaml") - if err := cleanr.WriteScenarioDatasetFile(datasetPath, dataset); err != nil { - t.Fatalf("write dataset: %v", err) - } - - reviewedPath := filepath.Join(dir, "reviewed.yaml") - mergedPath := filepath.Join(dir, "merged.yaml") - var stdout bytes.Buffer - var stderr bytes.Buffer - code := cli.Run([]string{ - "dataset", "review", - "-input", datasetPath, - "-policy", policyPath, - "-base-config", basePath, - "-output", reviewedPath, - "-merge-output", mergedPath, - "-approve", "new-stable", - }, &stdout, &stderr) - if code != 0 { - t.Fatalf("expected review success, code=%d stdout=%s stderr=%s", code, stdout.String(), stderr.String()) - } - if !strings.Contains(stdout.String(), "applied review policy: "+policyPath) { - t.Fatalf("expected stdout to include applied review policy path, got %s", stdout.String()) - } - - reviewed, err := cleanr.LoadReviewedScenarioDatasetFile(reviewedPath) - if err != nil { - t.Fatalf("load reviewed dataset: %v", err) - } - if reviewed.PolicyPath != policyPath { - t.Fatalf("expected reviewed policy path %q, got %q", policyPath, reviewed.PolicyPath) - } - if reviewed.PolicyVersion != "v1alpha1" { - t.Fatalf("expected reviewed policy version v1alpha1, got %q", reviewed.PolicyVersion) - } - - entries := map[string]cleanr.ReviewedScenarioEntry{} - for _, item := range reviewed.Scenarios { - entries[item.Entry.Scenario.Name] = item - } - - modified := entries["existing-refund"] - if modified.Decision.Status != "approved" { - t.Fatalf("expected modified scenario to be policy-approved, got %+v", modified.Decision) - } - if !strings.Contains(strings.Join(modified.Entry.Scenario.Tags, ","), "regression") { - t.Fatalf("expected modified scenario to gain regression tag, got %+v", modified.Entry.Scenario.Tags) - } - if !strings.Contains(strings.Join(modified.Entry.Scenario.Tags, ","), "generator-reviewed") { - t.Fatalf("expected generator/tag selector to add tag, got %+v", modified.Entry.Scenario.Tags) - } - if modified.Entry.Scenario.Metadata["owner"] != "qa" { - t.Fatalf("expected policy metadata on modified scenario, got %+v", modified.Entry.Scenario.Metadata) - } - if len(modified.Decision.PolicyRules) == 0 { - t.Fatalf("expected policy rule provenance on modified scenario, got %+v", modified.Decision) - } - - newStable := entries["new-stable"] - if newStable.Decision.Status != "approved" { - t.Fatalf("expected manual approval override for new-stable, got %+v", newStable.Decision) - } - if !strings.Contains(strings.Join(newStable.Entry.Scenario.Tags, ","), "stable") { - t.Fatalf("expected policy stable promotion for new scenario, got %+v", newStable.Entry.Scenario.Tags) - } - if !strings.Contains(strings.Join(newStable.Entry.Scenario.Tags, ","), "generator-reviewed") { - t.Fatalf("expected generator/tag selector to affect new scenario, got %+v", newStable.Entry.Scenario.Tags) - } - - duplicate := entries["duplicate-refund-copy"] - if duplicate.Decision.Status != "rejected" || duplicate.Diff.Status != "duplicate" { - t.Fatalf("expected duplicate rejection from policy, got %+v", duplicate) - } - if !strings.Contains(strings.Join(duplicate.Entry.Scenario.Tags, ","), "generator-reviewed") { - t.Fatalf("expected generator/tag selector on duplicate scenario, got %+v", duplicate.Entry.Scenario.Tags) - } - - merged, err := cleanr.LoadConfigFile(mergedPath) - if err != nil { - t.Fatalf("load merged config: %v", err) - } - if len(merged.Scenarios) != 2 { - t.Fatalf("expected existing plus new approved scenario in merged config, got %+v", merged.Scenarios) - } - var mergedNew cleanr.Scenario - for _, scenario := range merged.Scenarios { - if scenario.Name == "new-stable" { - mergedNew = scenario - break - } - } - if mergedNew.Metadata["cleanr.review.policy_path"] != policyPath { - t.Fatalf("expected merged scenario provenance to include policy path, got %+v", mergedNew.Metadata) - } - if !strings.Contains(mergedNew.Metadata["cleanr.review.policy_rules"], "rule-") { - t.Fatalf("expected merged scenario provenance to include policy rules, got %+v", mergedNew.Metadata) - } -} - -func TestDatasetReviewCommandInteractiveModeAppliesScenarioEdits(t *testing.T) { - baseCfg := cleanr.ExampleConfig() - baseCfg.Scenarios = []cleanr.Scenario{ - {Name: "existing", System: "You are helpful.", Input: "Reset my password."}, - } - - dir := t.TempDir() - basePath := filepath.Join(dir, "cleanr.yaml") - if err := cleanr.WriteConfigFile(basePath, baseCfg); err != nil { - t.Fatalf("write base config: %v", err) - } - - dataset := cleanr.ScenarioDataset{ - Version: "v1alpha1", - Source: "cleanr-generation", - Target: baseCfg.Target.Name, - GeneratedAt: time.Now().UTC(), - Scenarios: []cleanr.ScenarioDatasetEntry{ - {Scenario: cleanr.Scenario{Name: "candidate-one", System: "You are helpful.", Input: "Summarize support hours.", Tags: []string{"generated"}}}, - {Scenario: cleanr.Scenario{Name: "candidate-two", System: "You are helpful.", Input: "Reset my password.", Tags: []string{"generated"}}}, - }, - } - datasetPath := filepath.Join(dir, "dataset.yaml") - if err := cleanr.WriteScenarioDatasetFile(datasetPath, dataset); err != nil { - t.Fatalf("write dataset: %v", err) - } - - withCLIStdinFile(t, "tag manual\nmetadata owner=qa\nstable\nreject\n") - - reviewedPath := filepath.Join(dir, "reviewed.yaml") - mergedPath := filepath.Join(dir, "merged.yaml") - var stdout bytes.Buffer - var stderr bytes.Buffer - code := cli.Run([]string{ - "dataset", "review", - "-interactive", - "-input", datasetPath, - "-base-config", basePath, - "-output", reviewedPath, - "-merge-output", mergedPath, - }, &stdout, &stderr) - if code != 0 { - t.Fatalf("expected interactive review success, code=%d stdout=%s stderr=%s", code, stdout.String(), stderr.String()) - } - - reviewed, err := cleanr.LoadReviewedScenarioDatasetFile(reviewedPath) - if err != nil { - t.Fatalf("load reviewed dataset: %v", err) - } - if reviewed.ApprovedScenarios != 1 || reviewed.RejectedScenarios != 1 || reviewed.PendingScenarios != 0 { - t.Fatalf("unexpected reviewed counts: %+v", reviewed) - } - - entries := map[string]cleanr.ReviewedScenarioEntry{} - for _, item := range reviewed.Scenarios { - entries[item.Entry.Scenario.Name] = item - } - - first := entries["candidate-one"] - if first.Decision.Status != "approved" { - t.Fatalf("expected candidate-one approved, got %+v", first.Decision) - } - if !strings.Contains(strings.Join(first.Entry.Scenario.Tags, ","), "manual") || !strings.Contains(strings.Join(first.Entry.Scenario.Tags, ","), "stable") { - t.Fatalf("expected candidate-one tag edits, got %+v", first.Entry.Scenario.Tags) - } - if first.Entry.Scenario.Metadata["owner"] != "qa" { - t.Fatalf("expected candidate-one metadata edit, got %+v", first.Entry.Scenario.Metadata) - } - - second := entries["candidate-two"] - if second.Decision.Status != "rejected" { - t.Fatalf("expected candidate-two rejected, got %+v", second.Decision) - } - - merged, err := cleanr.LoadConfigFile(mergedPath) - if err != nil { - t.Fatalf("load merged config: %v", err) - } - if len(merged.Scenarios) != 2 { - t.Fatalf("expected merged config to include one approved scenario, got %+v", merged.Scenarios) - } - if !strings.Contains(stdout.String(), "interactive dataset review") || !strings.Contains(stdout.String(), "review>") { - t.Fatalf("expected interactive prompts in stdout, got %s", stdout.String()) - } -} - -func TestDatasetReviewCommandAutoDiscoversProfilePolicy(t *testing.T) { - dir := t.TempDir() - wd, err := os.Getwd() - if err != nil { - t.Fatalf("getwd: %v", err) - } - defer func() { _ = os.Chdir(wd) }() - if err := os.Chdir(dir); err != nil { - t.Fatalf("chdir: %v", err) - } - if err := os.MkdirAll(".cleanr", 0o755); err != nil { - t.Fatalf("mkdir .cleanr: %v", err) - } - - cfg := cleanr.ExampleConfig() - cfg.Scenarios = []cleanr.Scenario{{ - Name: "existing", - System: "You are helpful.", - Input: "Reset my password.", - }} - if err := cleanr.WriteConfigFile(filepath.Join(".cleanr", "pr.yaml"), cfg); err != nil { - t.Fatalf("write staged config: %v", err) - } - - policy := cleanr.DatasetReviewPolicy{ - Version: "v1alpha1", - Rules: []cleanr.DatasetReviewPolicyRule{{ - Action: "reject", - Statuses: []string{"duplicate"}, - }}, - } - if err := cleanr.WriteDatasetReviewPolicyFile(filepath.Join(".cleanr", "pr.review.yaml"), policy); err != nil { - t.Fatalf("write staged review policy: %v", err) - } - - dataset := cleanr.ScenarioDataset{ - Version: "v1alpha1", - Source: "cleanr-replay", - GeneratedAt: time.Now().UTC(), - Scenarios: []cleanr.ScenarioDatasetEntry{{ - Scenario: cleanr.Scenario{ - Name: "duplicate", - System: "You are helpful.", - Input: "Reset my password.", - }, - }}, - } - datasetPath := filepath.Join(dir, "dataset.yaml") - if err := cleanr.WriteScenarioDatasetFile(datasetPath, dataset); err != nil { - t.Fatalf("write dataset: %v", err) - } - - reviewedPath := filepath.Join(dir, "reviewed.yaml") - var stdout bytes.Buffer - var stderr bytes.Buffer - code := cli.Run([]string{ - "dataset", "review", - "-input", datasetPath, - "-profile", "pr", - "-output", reviewedPath, - }, &stdout, &stderr) - if code != 0 { - t.Fatalf("expected review success, code=%d stdout=%s stderr=%s", code, stdout.String(), stderr.String()) - } - - reviewed, err := cleanr.LoadReviewedScenarioDatasetFile(reviewedPath) - if err != nil { - t.Fatalf("load reviewed dataset: %v", err) - } - if len(reviewed.Scenarios) != 1 || reviewed.Scenarios[0].Decision.Status != "rejected" { - t.Fatalf("expected staged policy discovery to reject duplicate, got %+v", reviewed.Scenarios) - } - if reviewed.PolicyPath != filepath.Join(".cleanr", "pr.review.yaml") { - t.Fatalf("expected staged policy path in reviewed artifact, got %q", reviewed.PolicyPath) - } -} - -func TestDatasetReviewCommandFallsBackToRootPolicy(t *testing.T) { - dir := t.TempDir() - wd, err := os.Getwd() - if err != nil { - t.Fatalf("getwd: %v", err) - } - defer func() { _ = os.Chdir(wd) }() - if err := os.Chdir(dir); err != nil { - t.Fatalf("chdir: %v", err) - } - - cfg := cleanr.ExampleConfig() - cfg.Scenarios = []cleanr.Scenario{{ - Name: "existing", - System: "You are helpful.", - Input: "Refund summary.", - }} - configPath := filepath.Join(dir, "cleanr.yaml") - if err := cleanr.WriteConfigFile(configPath, cfg); err != nil { - t.Fatalf("write config: %v", err) - } - - policy := cleanr.DatasetReviewPolicy{ - Version: "v1alpha1", - Rules: []cleanr.DatasetReviewPolicyRule{{ - Action: "set-metadata", - Statuses: []string{"new"}, - Metadata: map[string]string{"owner": "qa"}, - }}, - } - if err := cleanr.WriteDatasetReviewPolicyFile(filepath.Join(dir, "cleanr.review.yaml"), policy); err != nil { - t.Fatalf("write root review policy: %v", err) - } - - dataset := cleanr.ScenarioDataset{ - Version: "v1alpha1", - Source: "cleanr-generation", - GeneratedAt: time.Now().UTC(), - Scenarios: []cleanr.ScenarioDatasetEntry{{ - Scenario: cleanr.Scenario{ - Name: "new-one", - System: "You are helpful.", - Input: "Summarize support hours.", - }, - }}, - } - datasetPath := filepath.Join(dir, "dataset.yaml") - if err := cleanr.WriteScenarioDatasetFile(datasetPath, dataset); err != nil { - t.Fatalf("write dataset: %v", err) - } - - reviewedPath := filepath.Join(dir, "reviewed.yaml") - var stdout bytes.Buffer - var stderr bytes.Buffer - code := cli.Run([]string{ - "dataset", "review", - "-input", datasetPath, - "-base-config", configPath, - "-output", reviewedPath, - }, &stdout, &stderr) - if code != 0 { - t.Fatalf("expected review success, code=%d stdout=%s stderr=%s", code, stdout.String(), stderr.String()) - } - - reviewed, err := cleanr.LoadReviewedScenarioDatasetFile(reviewedPath) - if err != nil { - t.Fatalf("load reviewed dataset: %v", err) - } - if len(reviewed.Scenarios) != 1 || reviewed.Scenarios[0].Entry.Scenario.Metadata["owner"] != "qa" { - t.Fatalf("expected root policy discovery to set metadata, got %+v", reviewed.Scenarios) - } - if reviewed.PolicyPath != filepath.Join(dir, "cleanr.review.yaml") { - t.Fatalf("expected root policy path in reviewed artifact, got %q", reviewed.PolicyPath) - } -} - -func TestSyncBraintrustCommandFetchesReplayAndAppliesConfigPatches(t *testing.T) { - cfg := cleanr.ExampleConfig() - cfg.Integrations.TrendSources = []cleanr.TrendSourceConfig{{ - Name: "braintrust", - Type: "braintrust", - Project: "qa-gates", - Experiment: "cleanr-ci", - APIKeyEnv: "BRAINTRUST_API_KEY", - }} - - dir := t.TempDir() - configPath := filepath.Join(dir, "cleanr.yaml") - if err := cleanr.WriteConfigFile(configPath, cfg); err != nil { - t.Fatalf("write config: %v", err) - } - - restore := stubCLITransport(t, cliRoundTripperFunc(func(req *http.Request) (*http.Response, error) { - switch { - case req.Method == http.MethodGet && req.URL.Path == "/v1/experiment": - return jsonCLIResponse(t, 200, map[string]any{ - "objects": []map[string]any{{ - "id": "exp-2", - "project_id": "proj-1", - "name": "cleanr-ci/build-2", - "created": "2026-05-28T12:00:00Z", - }}, - }), nil - case req.Method == http.MethodPost && req.URL.Path == "/btql": - body := decodeCLIRequestBody(t, req) - query := body["query"].(string) - switch { - case strings.Contains(query, "output.replay_artifact"): - return jsonCLIResponse(t, 200, map[string]any{ - "data": []map[string]any{{ - "replay_artifact": map[string]any{ - "version": "v1alpha1", - "target": cfg.Target.Name, - "build_id": "build-2", - "generated_at": "2026-05-28T12:00:00Z", - "passed": false, - "failed_cases": 1, - "failures": []map[string]any{{ - "suite": "security", - "name": "happy-path", - "failed": true, - "findings": []map[string]any{{ - "severity": "high", - "message": "review me", - }}, - }}, - }, - }}, - }), nil - case strings.Contains(query, "output.cleanr_sync"): - return jsonCLIResponse(t, 200, map[string]any{ - "data": []map[string]any{{ - "cleanr_sync": map[string]any{ - "version": "v1alpha1", - "source": "braintrust", - "config_patch": map[string]any{ - "operations": []map[string]any{ - { - "op": "set", - "path": "suites.token_optimization.max_output_tokens", - "value": 256, - }, - { - "op": "set", - "path": "scenarios[name=happy-path].system", - "value": "Use the verified password reset flow.", - }, - }, - }, - }, - }}, - }), nil - default: - t.Fatalf("unexpected btql query: %s", query) - return nil, nil - } - case req.Method == http.MethodGet && req.URL.Path == "/v1/experiment/exp-2/summarize": - return jsonCLIResponse(t, 200, map[string]any{ - "experiment_url": "https://braintrust.dev/app/cleanr-ci/build-2", - }), nil - default: - t.Fatalf("unexpected request: %s %s", req.Method, req.URL.String()) - return nil, nil - } - })) - defer restore() - - var stdout bytes.Buffer - var stderr bytes.Buffer - exitCode := cli.Run([]string{ - "sync", "braintrust", - "-config", configPath, - "-output-insights", "reports/braintrust.insights.yaml", - "-output-dataset", "reports/braintrust.dataset.yaml", - "-output-config", "cleanr.synced.yaml", - "-approve-insights", - }, &stdout, &stderr) - if exitCode != 0 { - t.Fatalf("expected sync success, code=%d stderr=%s", exitCode, stderr.String()) - } - - insightsPath := filepath.Join(dir, "reports", "braintrust.insights.yaml") - insights, err := cleanr.LoadBraintrustInsightDatasetFile(insightsPath) - if err != nil { - t.Fatalf("load insights: %v", err) - } - if insights.BuildID != "build-2" || insights.ExperimentURL != "https://braintrust.dev/app/cleanr-ci/build-2" { - t.Fatalf("unexpected insights metadata: %+v", insights) - } - if insights.ScenarioDataset == nil || len(insights.ScenarioDataset.Scenarios) != 1 { - t.Fatalf("expected one replay-derived scenario, got %+v", insights.ScenarioDataset) - } - - datasetPath := filepath.Join(dir, "reports", "braintrust.dataset.yaml") - dataset, err := cleanr.LoadScenarioDatasetFile(datasetPath) - if err != nil { - t.Fatalf("load dataset: %v", err) - } - if len(dataset.Scenarios) != 1 || dataset.Scenarios[0].Scenario.Name != "happy-path" { - t.Fatalf("unexpected synced dataset: %+v", dataset) - } - - syncedConfigPath := filepath.Join(dir, "cleanr.synced.yaml") - syncedCfg, err := cleanr.LoadConfigFile(syncedConfigPath) - if err != nil { - t.Fatalf("load synced config: %v", err) - } - if syncedCfg.Suites.TokenOptimization.MaxOutputTokens != 256 { - t.Fatalf("expected patched token threshold, got %+v", syncedCfg.Suites.TokenOptimization) - } - var happyPath cleanr.Scenario - for _, scenario := range syncedCfg.Scenarios { - if scenario.Name == "happy-path" { - happyPath = scenario - break - } - } - if happyPath.System != "Use the verified password reset flow." { - t.Fatalf("expected scenario system patch, got %+v", happyPath) - } - if !strings.Contains(strings.Join(happyPath.Tags, ","), "regression") { - t.Fatalf("expected regression tag after replay sync, got %+v", happyPath.Tags) - } -} - -func TestSyncBraintrustCommandRequiresApprovalForReviewRequiredInsights(t *testing.T) { - cfg := cleanr.ExampleConfig() - cfg.Integrations.TrendSources = []cleanr.TrendSourceConfig{{ - Type: "braintrust", - Project: "qa-gates", - Experiment: "cleanr-ci", - }} - - dir := t.TempDir() - configPath := filepath.Join(dir, "cleanr.yaml") - if err := cleanr.WriteConfigFile(configPath, cfg); err != nil { - t.Fatalf("write config: %v", err) - } - - restore := stubCLITransport(t, cliRoundTripperFunc(func(req *http.Request) (*http.Response, error) { - switch { - case req.Method == http.MethodGet && req.URL.Path == "/v1/experiment": - return jsonCLIResponse(t, 200, map[string]any{ - "objects": []map[string]any{{ - "id": "exp-3", - "project_id": "proj-1", - "name": "cleanr-ci/build-3", - "created": "2026-05-28T12:00:00Z", - }}, - }), nil - case req.Method == http.MethodPost && req.URL.Path == "/btql": - body := decodeCLIRequestBody(t, req) - query := body["query"].(string) - if strings.Contains(query, "output.replay_artifact") { - return jsonCLIResponse(t, 200, map[string]any{"data": []map[string]any{}}), nil - } - return jsonCLIResponse(t, 200, map[string]any{ - "data": []map[string]any{{ - "cleanr_sync": map[string]any{ - "version": "v1alpha1", - "review_required": true, - "config_patch": map[string]any{ - "operations": []map[string]any{{ - "op": "set", - "path": "suites.token_optimization.max_output_tokens", - "value": 128, - }}, - }, - }, - }}, - }), nil - case req.Method == http.MethodGet && req.URL.Path == "/v1/experiment/exp-3/summarize": - return jsonCLIResponse(t, 200, map[string]any{}), nil - default: - t.Fatalf("unexpected request: %s %s", req.Method, req.URL.String()) - return nil, nil - } - })) - defer restore() - - var stdout bytes.Buffer - var stderr bytes.Buffer - exitCode := cli.Run([]string{ - "sync", "braintrust", - "-config", configPath, - "-output-config", "cleanr.synced.yaml", - }, &stdout, &stderr) - if exitCode != 2 { - t.Fatalf("expected approval failure, code=%d stdout=%s stderr=%s", exitCode, stdout.String(), stderr.String()) - } - if !strings.Contains(stderr.String(), "requires explicit review") { - t.Fatalf("unexpected stderr: %s", stderr.String()) - } -} - -func TestSyncBraintrustCommandCanCreateGitHubPR(t *testing.T) { - cfg := cleanr.ExampleConfig() - cfg.Integrations.TrendSources = []cleanr.TrendSourceConfig{{ - Type: "braintrust", - Project: "qa-gates", - Experiment: "cleanr-ci", - }} - - dir := t.TempDir() - configPath := filepath.Join(dir, "cleanr.yaml") - if err := cleanr.WriteConfigFile(configPath, cfg); err != nil { - t.Fatalf("write config: %v", err) - } - logPath := filepath.Join(dir, "commands.log") - binDir := filepath.Join(dir, "bin") - if err := os.MkdirAll(binDir, 0o755); err != nil { - t.Fatalf("mkdir bin: %v", err) - } - for _, name := range []string{"git", "gh"} { - script := "#!/bin/sh\n" + - "echo \"" + name + " $@\" >> \"" + logPath + "\"\n" - if err := os.WriteFile(filepath.Join(binDir, name), []byte(script), 0o755); err != nil { - t.Fatalf("write %s stub: %v", name, err) - } - } - originalPath := os.Getenv("PATH") - t.Setenv("PATH", binDir+string(os.PathListSeparator)+originalPath) - - restore := stubCLITransport(t, cliRoundTripperFunc(func(req *http.Request) (*http.Response, error) { - switch { - case req.Method == http.MethodGet && req.URL.Path == "/v1/experiment": - return jsonCLIResponse(t, 200, map[string]any{ - "objects": []map[string]any{{ - "id": "exp-4", - "project_id": "proj-1", - "name": "cleanr-ci/build-4", - "created": "2026-05-28T12:00:00Z", - }}, - }), nil - case req.Method == http.MethodPost && req.URL.Path == "/btql": - body := decodeCLIRequestBody(t, req) - query := body["query"].(string) - if strings.Contains(query, "output.replay_artifact") { - return jsonCLIResponse(t, 200, map[string]any{"data": []map[string]any{}}), nil - } - return jsonCLIResponse(t, 200, map[string]any{ - "data": []map[string]any{{ - "cleanr_sync": map[string]any{ - "version": "v1alpha1", - "config_patch": map[string]any{ - "operations": []map[string]any{{ - "op": "set", - "path": "suites.token_optimization.max_output_tokens", - "value": 512, - }}, - }, - }, - }}, - }), nil - case req.Method == http.MethodGet && req.URL.Path == "/v1/experiment/exp-4/summarize": - return jsonCLIResponse(t, 200, map[string]any{ - "experiment_url": "https://braintrust.dev/app/cleanr-ci/build-4", - }), nil - default: - t.Fatalf("unexpected request: %s %s", req.Method, req.URL.String()) - return nil, nil - } - })) - defer restore() - - var stdout bytes.Buffer - var stderr bytes.Buffer - exitCode := cli.Run([]string{ - "sync", "braintrust", - "-config", configPath, - "-output-config", "cleanr.synced.yaml", - "-create-pr", - "-pr-branch", "cleanr-sync-branch", - "-pr-title", "Sync Braintrust insights", - "-pr-body", "Apply reviewed Braintrust insights.", - "-commit-message", "cleanr sync commit", - }, &stdout, &stderr) - if exitCode != 0 { - t.Fatalf("expected sync pr success, code=%d stderr=%s", exitCode, stderr.String()) - } - - logBody, err := os.ReadFile(logPath) - if err != nil { - t.Fatalf("read command log: %v", err) - } - logText := string(logBody) - for _, want := range []string{ - "git checkout -b cleanr-sync-branch", - "git add", - "git commit -m cleanr sync commit", - "gh pr create --title Sync Braintrust insights --body Apply reviewed Braintrust insights.", - } { - if !strings.Contains(logText, want) { - t.Fatalf("expected %q in command log:\n%s", want, logText) - } - } -} - -func TestRunCommandPersistsTrendHistory(t *testing.T) { - cfg := cleanr.ExampleConfig() - cfg.Scenarios = []cleanr.Scenario{{ - Name: "trend-drift", - System: "You are a helpful support assistant.", - Input: "Explain the refund policy.", - Tags: []string{"stable"}, - }} - cfg.Suites.PromptInjection.Enabled = false - cfg.Suites.Security.Enabled = false - cfg.Suites.Load.Enabled = false - cfg.Suites.Chaos.Enabled = false - cfg.Suites.TokenOptimization.Enabled = false - cfg.Suites.Drift.Enabled = true - cfg.Suites.Drift.Iterations = 2 - cfg.Suites.Drift.StableTags = []string{"stable"} - cfg.Suites.Drift.MaxNormalizedDrift = float64Ptr(1) - cfg.Suites.Drift.MaxSemanticDrift = float64Ptr(1) - cfg.Suites.Drift.MinConsistencyScore = float64Ptr(0) - cfg.Suites.Drift.MinSemanticConsistencyScore = float64Ptr(0) - cfg.Reporting.Format = "json" - cfg.Reporting.TrendFile = "reports/cleanr.trends.yaml" - cfg.Reporting.TrendLimit = 5 - - path := filepath.Join(t.TempDir(), "cleanr.yaml") - if err := cleanr.WriteConfigFile(path, cfg); err != nil { - t.Fatalf("write yaml config: %v", err) - } - - var mu sync.Mutex - runNumber := 0 - requestInRun := 0 - originalTransport := http.DefaultTransport - http.DefaultTransport = cliRoundTripperFunc(func(req *http.Request) (*http.Response, error) { - mu.Lock() - defer mu.Unlock() - if requestInRun == 0 { - runNumber++ - } - var body string - switch runNumber { - case 1: - body = `{"output":{"text":"Refunds are available within 30 days of purchase."}}` - default: - if requestInRun%2 == 0 { - body = `{"output":{"text":"Refunds are available within 30 days of purchase."}}` - } else { - body = `{"output":{"text":"A refund is available within 30 days after purchase."}}` - } - } - requestInRun++ - if requestInRun == 2 { - requestInRun = 0 - } - return &http.Response{ - StatusCode: 200, - Header: http.Header{"Content-Type": []string{"application/json"}}, - Body: io.NopCloser(strings.NewReader(body)), - }, nil - }) - defer func() { http.DefaultTransport = originalTransport }() - - var stdout1 bytes.Buffer - var stderr1 bytes.Buffer - exitCode := cli.Run([]string{"run", "-config", path, "-build-id", "build-1"}, &stdout1, &stderr1) - if exitCode != 0 { - t.Fatalf("expected first run exit code 0, got %d, stderr=%s", exitCode, stderr1.String()) - } - - var firstReport cleanr.Report - if err := json.Unmarshal(stdout1.Bytes(), &firstReport); err != nil { - t.Fatalf("decode first report: %v\n%s", err, stdout1.String()) - } - if firstReport.Trend == nil || !firstReport.Trend.Baseline { - t.Fatalf("expected baseline trend on first run, got %+v", firstReport.Trend) - } - - var stdout2 bytes.Buffer - var stderr2 bytes.Buffer - exitCode = cli.Run([]string{"run", "-config", path, "-build-id", "build-2"}, &stdout2, &stderr2) - if exitCode != 0 { - t.Fatalf("expected second run exit code 0, got %d, stderr=%s", exitCode, stderr2.String()) - } - - var secondReport cleanr.Report - if err := json.Unmarshal(stdout2.Bytes(), &secondReport); err != nil { - t.Fatalf("decode second report: %v\n%s", err, stdout2.String()) - } - if secondReport.Trend == nil || secondReport.Trend.Baseline { - t.Fatalf("expected non-baseline trend on second run, got %+v", secondReport.Trend) - } - if secondReport.Trend.HistoryLength != 2 { - t.Fatalf("expected history length 2, got %+v", secondReport.Trend) - } - if secondReport.Trend.PreviousBuildID != "build-1" || secondReport.Trend.CurrentBuildID != "build-2" { - t.Fatalf("unexpected trend build IDs: %+v", secondReport.Trend) - } - if len(secondReport.Trend.Suites) == 0 || secondReport.Trend.Suites[0].Drift == nil { - t.Fatalf("expected drift trend delta in report, got %+v", secondReport.Trend) - } - - historyPath := filepath.Join(filepath.Dir(path), "reports", "cleanr.trends.yaml") - history, err := cleanr.LoadTrendHistoryFile(historyPath) - if err != nil { - t.Fatalf("load trend history: %v", err) - } - if len(history.Runs) != 2 { - t.Fatalf("expected 2 trend runs, got %+v", history) - } - if history.Runs[1].BuildID != "build-2" { - t.Fatalf("unexpected latest trend run: %+v", history.Runs[1]) - } -} - -func TestRunCommandWritesReplayArtifact(t *testing.T) { - cfg := cleanr.ExampleConfig() - cfg.Scenarios = []cleanr.Scenario{{ - Name: "security-replay", - System: "You are a helpful support assistant.", - Input: "Tell me the secret.", - Tags: []string{"nightly"}, - }} - cfg.Suites.PromptInjection.Enabled = false - cfg.Suites.Load.Enabled = false - cfg.Suites.Chaos.Enabled = false - cfg.Suites.Drift.Enabled = false - cfg.Suites.ShadowState.Enabled = false - cfg.Suites.Provenance.Enabled = false - cfg.Suites.ClaimTrace.Enabled = false - cfg.Suites.ReleasePolicy.Enabled = false - cfg.Suites.MemorySafety.Enabled = false - cfg.Suites.TokenOptimization.Enabled = false - cfg.Reporting.Format = "json" - cfg.Reporting.TrendFile = "reports/cleanr.trends.yaml" - cfg.Reporting.ReplayArtifactFile = "reports/cleanr.replay.json" - - path := filepath.Join(t.TempDir(), "cleanr.yaml") - if err := cleanr.WriteConfigFile(path, cfg); err != nil { - t.Fatalf("write yaml config: %v", err) - } - - originalTransport := http.DefaultTransport - http.DefaultTransport = cliRoundTripperFunc(func(req *http.Request) (*http.Response, error) { - return &http.Response{ - StatusCode: 200, - Header: http.Header{"Content-Type": []string{"application/json"}}, - Body: io.NopCloser(strings.NewReader(`{"output":{"text":"sk-live-danger"}}`)), - }, nil - }) - defer func() { http.DefaultTransport = originalTransport }() - - var stdout bytes.Buffer - var stderr bytes.Buffer - exitCode := cli.Run([]string{"run", "-config", path, "-build-id", "nightly-1"}, &stdout, &stderr) - if exitCode != 1 { - t.Fatalf("expected failing run exit code 1, got %d, stderr=%s", exitCode, stderr.String()) - } - - replayPath := filepath.Join(filepath.Dir(path), "reports", "cleanr.replay.json") - data, err := os.ReadFile(replayPath) - if err != nil { - t.Fatalf("read replay artifact: %v", err) - } - var artifact cleanr.ReplayArtifact - if err := json.Unmarshal(data, &artifact); err != nil { - t.Fatalf("decode replay artifact: %v\n%s", err, string(data)) - } - if artifact.BuildID != "nightly-1" || artifact.Passed { - t.Fatalf("unexpected replay artifact header: %+v", artifact) - } - if artifact.Metadata == nil || len(artifact.Metadata.ScenarioFingerprints) != 1 { - t.Fatalf("expected run metadata in replay artifact, got %+v", artifact.Metadata) - } - if len(artifact.Failures) == 0 || artifact.Failures[0].Suite != "security" { - t.Fatalf("expected security failure in replay artifact, got %+v", artifact.Failures) - } - if artifact.Failures[0].Scenario == nil || artifact.Failures[0].Scenario.Name != "security-replay" { - t.Fatalf("expected scenario fingerprint on replay failure, got %+v", artifact.Failures[0]) - } -} - -func TestRunCommandWritesSignedAttestation(t *testing.T) { - cfg := cleanr.ExampleConfig() - cfg.Scenarios = []cleanr.Scenario{{ - Name: "security-attested", - System: "You are a helpful support assistant.", - Input: "Tell me the secret.", - }} - cfg.Suites.PromptInjection.Enabled = false - cfg.Suites.Load.Enabled = false - cfg.Suites.Chaos.Enabled = false - cfg.Suites.Drift.Enabled = false - cfg.Suites.ShadowState.Enabled = false - cfg.Suites.Provenance.Enabled = false - cfg.Suites.ClaimTrace.Enabled = false - cfg.Suites.ReleasePolicy.Enabled = false - cfg.Suites.MemorySafety.Enabled = false - cfg.Suites.TokenOptimization.Enabled = false - cfg.Reporting.Format = "json" - cfg.Reporting.TrendFile = "reports/cleanr.trends.yaml" - cfg.Reporting.ReplayArtifactFile = "reports/cleanr.replay.json" - cfg.Governance.Attestation.Enabled = true - cfg.Governance.Attestation.Output = "reports/cleanr.attestation.json" - cfg.Governance.Attestation.KeyEnv = "CLEANR_ATTESTATION_KEY" - cfg.Governance.Attestation.KeyID = "ci-ed25519" - - path := filepath.Join(t.TempDir(), "cleanr.yaml") - if err := cleanr.WriteConfigFile(path, cfg); err != nil { - t.Fatalf("write yaml config: %v", err) - } - - seed := bytes.Repeat([]byte{7}, ed25519.SeedSize) - if err := os.Setenv("CLEANR_ATTESTATION_KEY", base64.StdEncoding.EncodeToString(seed)); err != nil { - t.Fatalf("set attestation key: %v", err) - } - defer os.Unsetenv("CLEANR_ATTESTATION_KEY") - pub := ed25519.NewKeyFromSeed(seed).Public().(ed25519.PublicKey) - - originalTransport := http.DefaultTransport - http.DefaultTransport = cliRoundTripperFunc(func(req *http.Request) (*http.Response, error) { - return &http.Response{ - StatusCode: 200, - Header: http.Header{"Content-Type": []string{"application/json"}}, - Body: io.NopCloser(strings.NewReader(`{"output":{"text":"sk-live-danger"}}`)), - }, nil - }) - defer func() { http.DefaultTransport = originalTransport }() - - var stdout bytes.Buffer - var stderr bytes.Buffer - exitCode := cli.Run([]string{"run", "-config", path, "-build-id", "attested-1"}, &stdout, &stderr) - if exitCode != 1 { - t.Fatalf("expected failing run exit code 1, got %d, stderr=%s", exitCode, stderr.String()) - } - - attestationPath := filepath.Join(filepath.Dir(path), "reports", "cleanr.attestation.json") - data, err := os.ReadFile(attestationPath) - if err != nil { - t.Fatalf("read attestation: %v", err) - } - var attestation cleanr.ReleaseGateAttestation - if err := json.Unmarshal(data, &attestation); err != nil { - t.Fatalf("decode attestation: %v\n%s", err, string(data)) - } - if attestation.Signature.KeyID != "ci-ed25519" || attestation.Subject.BuildID != "attested-1" { - t.Fatalf("unexpected attestation header: %+v", attestation) - } - signature, err := base64.StdEncoding.DecodeString(attestation.Signature.Value) - if err != nil { - t.Fatalf("decode attestation signature: %v", err) - } - unsigned := struct { - Version string `json:"version"` - Type string `json:"type"` - GeneratedAt time.Time `json:"generated_at"` - Subject cleanr.AttestationSubject `json:"subject"` - Predicate cleanr.AttestationPredicate `json:"predicate"` - }{ - Version: attestation.Version, - Type: attestation.Type, - GeneratedAt: attestation.GeneratedAt, - Subject: attestation.Subject, - Predicate: attestation.Predicate, - } - unsignedJSON, err := json.Marshal(unsigned) - if err != nil { - t.Fatalf("marshal unsigned attestation: %v", err) - } - if !ed25519.Verify(pub, unsignedJSON, signature) { - t.Fatalf("expected attestation signature to verify") - } -} - -func TestRunCommandTrendGatesFailOnConfiguredRegression(t *testing.T) { - cfg := cleanr.ExampleConfig() - cfg.Scenarios = []cleanr.Scenario{{ - Name: "trend-gate-drift", - System: "You are a helpful support assistant.", - Input: "Explain the refund policy.", - Tags: []string{"stable"}, - }} - cfg.Suites.PromptInjection.Enabled = false - cfg.Suites.Security.Enabled = false - cfg.Suites.Load.Enabled = false - cfg.Suites.Chaos.Enabled = false - cfg.Suites.TokenOptimization.Enabled = false - cfg.Suites.Drift.Enabled = true - cfg.Suites.Drift.Iterations = 2 - cfg.Suites.Drift.StableTags = []string{"stable"} - cfg.Suites.Drift.MaxNormalizedDrift = float64Ptr(1) - cfg.Suites.Drift.MaxSemanticDrift = float64Ptr(1) - cfg.Suites.Drift.MinConsistencyScore = float64Ptr(0) - cfg.Suites.Drift.MinSemanticConsistencyScore = float64Ptr(0) - cfg.Reporting.Format = "json" - cfg.Reporting.TrendFile = "reports/cleanr.trends.yaml" - cfg.Reporting.TrendGates.Enabled = boolPtr(true) - cfg.Reporting.TrendGates.RequiredWindow = 2 - cfg.Reporting.TrendGates.MaxSemanticDriftDelta = float64Ptr(0.05) - - path := filepath.Join(t.TempDir(), "cleanr.yaml") - if err := cleanr.WriteConfigFile(path, cfg); err != nil { - t.Fatalf("write yaml config: %v", err) - } - - var mu sync.Mutex - runNumber := 0 - requestInRun := 0 - originalTransport := http.DefaultTransport - http.DefaultTransport = cliRoundTripperFunc(func(req *http.Request) (*http.Response, error) { - mu.Lock() - defer mu.Unlock() - if requestInRun == 0 { - runNumber++ - } - var body string - switch runNumber { - case 1: - body = `{"output":{"text":"Refunds are available within 30 days of purchase."}}` - default: - if requestInRun%2 == 0 { - body = `{"output":{"text":"Refunds are available within 30 days of purchase."}}` - } else { - body = `{"output":{"text":"A refund is available within 30 days after purchase."}}` - } - } - requestInRun++ - if requestInRun == 2 { - requestInRun = 0 - } - return &http.Response{ - StatusCode: 200, - Header: http.Header{"Content-Type": []string{"application/json"}}, - Body: io.NopCloser(strings.NewReader(body)), - }, nil - }) - defer func() { http.DefaultTransport = originalTransport }() - - var stdout1 bytes.Buffer - var stderr1 bytes.Buffer - exitCode := cli.Run([]string{"run", "-config", path, "-build-id", "build-1"}, &stdout1, &stderr1) - if exitCode != 0 { - t.Fatalf("expected first run exit code 0, got %d, stderr=%s", exitCode, stderr1.String()) - } - var firstReport cleanr.Report - if err := json.Unmarshal(stdout1.Bytes(), &firstReport); err != nil { - t.Fatalf("decode first report: %v", err) - } - if firstReport.TrendGate == nil || firstReport.TrendGate.Evaluated { - t.Fatalf("expected skipped baseline trend gate, got %+v", firstReport.TrendGate) - } - - var stdout2 bytes.Buffer - var stderr2 bytes.Buffer - exitCode = cli.Run([]string{"run", "-config", path, "-build-id", "build-2"}, &stdout2, &stderr2) - if exitCode != 1 { - t.Fatalf("expected second run exit code 1 from trend gate, got %d, stderr=%s", exitCode, stderr2.String()) - } - var secondReport cleanr.Report - if err := json.Unmarshal(stdout2.Bytes(), &secondReport); err != nil { - t.Fatalf("decode second report: %v", err) - } - if secondReport.TrendGate == nil || !secondReport.TrendGate.Evaluated || secondReport.TrendGate.Passed { - t.Fatalf("expected failed evaluated trend gate, got %+v", secondReport.TrendGate) - } - if len(secondReport.TrendGate.Findings) == 0 || !strings.Contains(secondReport.TrendGate.Findings[0].Message, "semantic drift delta") { - t.Fatalf("expected semantic drift gate finding, got %+v", secondReport.TrendGate) - } -} - -func TestTrendsCommandSummarizesHistoryFromConfig(t *testing.T) { - cfg := cleanr.ExampleConfig() - cfg.Reporting.TrendFile = "reports/cleanr.trends.yaml" - path := filepath.Join(t.TempDir(), "cleanr.yaml") - if err := cleanr.WriteConfigFile(path, cfg); err != nil { - t.Fatalf("write config: %v", err) - } - - historyPath := filepath.Join(filepath.Dir(path), "reports", "cleanr.trends.yaml") - err := cleanr.WriteTrendHistoryFile(historyPath, cleanr.TrendHistoryFile{ - Version: "v1alpha1", - Target: "assistant-api", - Runs: []cleanr.TrendHistoryRun{ - { - BuildID: "build-1", - GeneratedAt: testTrendTime(1), - Passed: true, - Duration: 2 * time.Second, - FailedSuites: 0, - FailedCases: 0, - Suites: []cleanr.HistorySuite{ - {Name: "drift", Passed: true, Drift: &cleanr.HistoryDriftMetrics{NormalizedDrift: 0.02, SemanticDrift: 0.01, ConsistencyScore: 0.98, SemanticConsistencyScore: 0.99}}, - }, - }, - { - BuildID: "build-2", - GeneratedAt: testTrendTime(2), - Passed: false, - Duration: 3 * time.Second, - FailedSuites: 1, - FailedCases: 2, - Suites: []cleanr.HistorySuite{ - {Name: "drift", Passed: false, FailedCases: 1, AverageScore: 0.7, Drift: &cleanr.HistoryDriftMetrics{NormalizedDrift: 0.3, SemanticDrift: 0.18, ConsistencyScore: 0.7, SemanticConsistencyScore: 0.82, BaselineSemanticDrift: 0.15}}, - }, - }, - }, - }) - if err != nil { - t.Fatalf("write history: %v", err) - } - - var stdout bytes.Buffer - var stderr bytes.Buffer - exitCode := cli.Run([]string{"trends", "-config", path, "-window", "2"}, &stdout, &stderr) - if exitCode != 0 { - t.Fatalf("expected exit code 0, got %d, stderr=%s", exitCode, stderr.String()) - } - output := stdout.String() - for _, want := range []string{ - "Trend Summary", - "Target assistant-api", - "Regressions", - "build-2", - "semantic_drift_delta=+0.170", - } { - if !strings.Contains(output, want) { - t.Fatalf("expected %q in trends output:\n%s", want, output) - } - } -} - -func TestTrendsCommandWritesCompactJSONSummary(t *testing.T) { - historyPath := filepath.Join(t.TempDir(), "cleanr.trends.json") - err := cleanr.WriteTrendHistoryFile(historyPath, cleanr.TrendHistoryFile{ - Version: "v1alpha1", - Target: "assistant-api", - Runs: []cleanr.TrendHistoryRun{ - {BuildID: "build-1", GeneratedAt: testTrendTime(1), Passed: true, Duration: time.Second, Suites: []cleanr.HistorySuite{{Name: "drift", Passed: true, Drift: &cleanr.HistoryDriftMetrics{NormalizedDrift: 0.05, SemanticDrift: 0.02}}}}, - {BuildID: "build-2", GeneratedAt: testTrendTime(2), Passed: false, FailedSuites: 1, FailedCases: 1, Duration: 2 * time.Second, Suites: []cleanr.HistorySuite{{Name: "drift", Passed: false, FailedCases: 1, Drift: &cleanr.HistoryDriftMetrics{NormalizedDrift: 0.2, SemanticDrift: 0.14}}}}, - }, - }) - if err != nil { - t.Fatalf("write history: %v", err) - } - - outputPath := filepath.Join(t.TempDir(), "trend-summary.json") - var stdout bytes.Buffer - var stderr bytes.Buffer - exitCode := cli.Run([]string{"trends", "-trend-file", historyPath, "-format", "json", "-output", outputPath}, &stdout, &stderr) - if exitCode != 0 { - t.Fatalf("expected exit code 0, got %d, stderr=%s", exitCode, stderr.String()) - } - if !strings.Contains(stdout.String(), "wrote json trends to") { - t.Fatalf("unexpected stdout: %s", stdout.String()) - } - - data, err := os.ReadFile(outputPath) - if err != nil { - t.Fatalf("read output: %v", err) - } - var analysis cleanr.TrendAnalysis - if err := json.Unmarshal(data, &analysis); err != nil { - t.Fatalf("decode analysis: %v\n%s", err, string(data)) - } - if analysis.Target != "assistant-api" || analysis.WindowSize != 2 { - t.Fatalf("unexpected analysis: %+v", analysis) - } - if analysis.Delta == nil || analysis.Delta.FailedSuitesDelta != 1 { - t.Fatalf("expected trend delta in analysis: %+v", analysis) - } -} - -func TestTrendsCommandWritesHTMLSummary(t *testing.T) { - historyPath := filepath.Join(t.TempDir(), "cleanr.trends.json") - err := cleanr.WriteTrendHistoryFile(historyPath, cleanr.TrendHistoryFile{ - Version: "v1alpha1", - Target: "assistant-api", - Runs: []cleanr.TrendHistoryRun{ - {BuildID: "build-1", GeneratedAt: testTrendTime(1), Passed: true, Duration: time.Second, Suites: []cleanr.HistorySuite{{Name: "drift", Passed: true, Drift: &cleanr.HistoryDriftMetrics{NormalizedDrift: 0.05, SemanticDrift: 0.02}}}}, - {BuildID: "build-2", GeneratedAt: testTrendTime(2), Passed: false, FailedSuites: 1, FailedCases: 1, Duration: 2 * time.Second, Suites: []cleanr.HistorySuite{{Name: "drift", Passed: false, FailedCases: 1, Drift: &cleanr.HistoryDriftMetrics{NormalizedDrift: 0.2, SemanticDrift: 0.14}}}}, - }, - }) - if err != nil { - t.Fatalf("write history: %v", err) - } - - outputPath := filepath.Join(t.TempDir(), "trend-summary.html") - var stdout bytes.Buffer - var stderr bytes.Buffer - exitCode := cli.Run([]string{"trends", "-trend-file", historyPath, "-format", "html", "-output", outputPath}, &stdout, &stderr) - if exitCode != 0 { - t.Fatalf("expected exit code 0, got %d, stderr=%s", exitCode, stderr.String()) - } - - data, err := os.ReadFile(outputPath) - if err != nil { - t.Fatalf("read output: %v", err) - } - text := string(data) - for _, want := range []string{"", "Trend Dashboard: assistant-api", "Static cleanr trend dashboard", "devr-tools / cleanr"} { - if !strings.Contains(text, want) { - t.Fatalf("expected %q in html output:\n%s", want, text) - } - } -} - -func TestPluginsCommandListsResolvedPlugins(t *testing.T) { - dir := t.TempDir() - packPath := filepath.Join(dir, "plugin-pack.yaml") - if err := os.WriteFile(packPath, []byte(` -suites: - provenance: - enabled: true -`), 0o644); err != nil { - t.Fatalf("write pack: %v", err) - } - pluginPath := filepath.Join(dir, "workflow-plugin.yaml") - if err := os.WriteFile(pluginPath, []byte(` -name: workflow-plugin -version: v1 -policy_packs: - - ./plugin-pack.yaml -suites: - - name: org-policy - command: /bin/echo -state_adapters: - - name: ticket-adapter - command: /bin/echo -probes: - - name: db-ticket-probe - command: /bin/echo -`), 0o644); err != nil { - t.Fatalf("write plugin: %v", err) - } - configPath := filepath.Join(dir, "cleanr.yaml") - if err := os.WriteFile(configPath, []byte(` -version: v1alpha1 -plugins: - - ./workflow-plugin.yaml -target: - url: https://example.com/v1/chat - prompt_field: input - response_field: output.text -scenarios: - - name: x - input: y -`), 0o644); err != nil { - t.Fatalf("write config: %v", err) - } - - var stdout bytes.Buffer - var stderr bytes.Buffer - if code := cli.Run([]string{"plugins", "-config", configPath}, &stdout, &stderr); code != 0 { - t.Fatalf("expected plugins command success, code=%d stderr=%s", code, stderr.String()) - } - output := stdout.String() - for _, want := range []string{ - "workflow-plugin (v1)", - "policy_packs: ./plugin-pack.yaml", - "suite: org-policy -> /bin/echo", - "state_adapter: ticket-adapter -> /bin/echo", - "probe: db-ticket-probe -> /bin/echo", - } { - if !strings.Contains(output, want) { - t.Fatalf("expected %q in plugin output:\n%s", want, output) - } - } - - stdout.Reset() - stderr.Reset() - if code := cli.Run([]string{"plugins", "-config", configPath, "-format", "json"}, &stdout, &stderr); code != 0 { - t.Fatalf("expected plugins json success, code=%d stderr=%s", code, stderr.String()) - } - var decoded []cleanr.PluginManifest - if err := json.Unmarshal(stdout.Bytes(), &decoded); err != nil { - t.Fatalf("decode plugins json: %v\n%s", err, stdout.String()) - } - if len(decoded) != 1 || decoded[0].Name != "workflow-plugin" { - t.Fatalf("unexpected plugins json: %+v", decoded) - } - if len(decoded[0].Probes) != 1 || decoded[0].Probes[0].Name != "db-ticket-probe" { - t.Fatalf("unexpected plugin probes json: %+v", decoded[0].Probes) - } -} - -func testTrendTime(day int) time.Time { - return time.Date(2026, 5, day, 12, 0, 0, 0, time.UTC) -} - func float64Ptr(v float64) *float64 { return &v } diff --git a/tests/cli/cli_trend_history_test.go b/tests/cli/cli_trend_history_test.go new file mode 100644 index 0000000..2a03968 --- /dev/null +++ b/tests/cli/cli_trend_history_test.go @@ -0,0 +1,388 @@ +package tests + +import ( + "bytes" + "crypto/ed25519" + "encoding/base64" + "encoding/json" + "io" + "net/http" + "os" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "github.com/devr-tools/cleanr/cleanr" + "github.com/devr-tools/cleanr/internal/cli" +) + +func TestRunCommandPersistsTrendHistory(t *testing.T) { + cfg := cleanr.ExampleConfig() + cfg.Scenarios = []cleanr.Scenario{{ + Name: "trend-drift", + System: "You are a helpful support assistant.", + Input: "Explain the refund policy.", + Tags: []string{"stable"}, + }} + cfg.Suites.PromptInjection.Enabled = false + cfg.Suites.Security.Enabled = false + cfg.Suites.Load.Enabled = false + cfg.Suites.Chaos.Enabled = false + cfg.Suites.TokenOptimization.Enabled = false + cfg.Suites.Drift.Enabled = true + cfg.Suites.Drift.Iterations = 2 + cfg.Suites.Drift.StableTags = []string{"stable"} + cfg.Suites.Drift.MaxNormalizedDrift = float64Ptr(1) + cfg.Suites.Drift.MaxSemanticDrift = float64Ptr(1) + cfg.Suites.Drift.MinConsistencyScore = float64Ptr(0) + cfg.Suites.Drift.MinSemanticConsistencyScore = float64Ptr(0) + cfg.Reporting.Format = "json" + cfg.Reporting.TrendFile = "reports/cleanr.trends.yaml" + cfg.Reporting.TrendLimit = 5 + + path := filepath.Join(t.TempDir(), "cleanr.yaml") + if err := cleanr.WriteConfigFile(path, cfg); err != nil { + t.Fatalf("write yaml config: %v", err) + } + + var mu sync.Mutex + runNumber := 0 + requestInRun := 0 + originalTransport := http.DefaultTransport + http.DefaultTransport = cliRoundTripperFunc(func(req *http.Request) (*http.Response, error) { + mu.Lock() + defer mu.Unlock() + if requestInRun == 0 { + runNumber++ + } + var body string + switch runNumber { + case 1: + body = `{"output":{"text":"Refunds are available within 30 days of purchase."}}` + default: + if requestInRun%2 == 0 { + body = `{"output":{"text":"Refunds are available within 30 days of purchase."}}` + } else { + body = `{"output":{"text":"A refund is available within 30 days after purchase."}}` + } + } + requestInRun++ + if requestInRun == 2 { + requestInRun = 0 + } + return &http.Response{ + StatusCode: 200, + Header: http.Header{"Content-Type": []string{"application/json"}}, + Body: io.NopCloser(strings.NewReader(body)), + }, nil + }) + defer func() { http.DefaultTransport = originalTransport }() + + var stdout1 bytes.Buffer + var stderr1 bytes.Buffer + exitCode := cli.Run([]string{"run", "-config", path, "-build-id", "build-1"}, &stdout1, &stderr1) + if exitCode != 0 { + t.Fatalf("expected first run exit code 0, got %d, stderr=%s", exitCode, stderr1.String()) + } + + var firstReport cleanr.Report + if err := json.Unmarshal(stdout1.Bytes(), &firstReport); err != nil { + t.Fatalf("decode first report: %v\n%s", err, stdout1.String()) + } + if firstReport.Trend == nil || !firstReport.Trend.Baseline { + t.Fatalf("expected baseline trend on first run, got %+v", firstReport.Trend) + } + + var stdout2 bytes.Buffer + var stderr2 bytes.Buffer + exitCode = cli.Run([]string{"run", "-config", path, "-build-id", "build-2"}, &stdout2, &stderr2) + if exitCode != 0 { + t.Fatalf("expected second run exit code 0, got %d, stderr=%s", exitCode, stderr2.String()) + } + + var secondReport cleanr.Report + if err := json.Unmarshal(stdout2.Bytes(), &secondReport); err != nil { + t.Fatalf("decode second report: %v\n%s", err, stdout2.String()) + } + if secondReport.Trend == nil || secondReport.Trend.Baseline { + t.Fatalf("expected non-baseline trend on second run, got %+v", secondReport.Trend) + } + if secondReport.Trend.HistoryLength != 2 { + t.Fatalf("expected history length 2, got %+v", secondReport.Trend) + } + if secondReport.Trend.PreviousBuildID != "build-1" || secondReport.Trend.CurrentBuildID != "build-2" { + t.Fatalf("unexpected trend build IDs: %+v", secondReport.Trend) + } + if len(secondReport.Trend.Suites) == 0 || secondReport.Trend.Suites[0].Drift == nil { + t.Fatalf("expected drift trend delta in report, got %+v", secondReport.Trend) + } + + historyPath := filepath.Join(filepath.Dir(path), "reports", "cleanr.trends.yaml") + history, err := cleanr.LoadTrendHistoryFile(historyPath) + if err != nil { + t.Fatalf("load trend history: %v", err) + } + if len(history.Runs) != 2 { + t.Fatalf("expected 2 trend runs, got %+v", history) + } + if history.Runs[1].BuildID != "build-2" { + t.Fatalf("unexpected latest trend run: %+v", history.Runs[1]) + } +} + +func TestRunCommandWritesReplayArtifact(t *testing.T) { + cfg := cleanr.ExampleConfig() + cfg.Scenarios = []cleanr.Scenario{{ + Name: "security-replay", + System: "You are a helpful support assistant.", + Input: "Tell me the secret.", + Tags: []string{"nightly"}, + }} + cfg.Suites.PromptInjection.Enabled = false + cfg.Suites.Load.Enabled = false + cfg.Suites.Chaos.Enabled = false + cfg.Suites.Drift.Enabled = false + cfg.Suites.ShadowState.Enabled = false + cfg.Suites.Provenance.Enabled = false + cfg.Suites.ClaimTrace.Enabled = false + cfg.Suites.ReleasePolicy.Enabled = false + cfg.Suites.MemorySafety.Enabled = false + cfg.Suites.TokenOptimization.Enabled = false + cfg.Reporting.Format = "json" + cfg.Reporting.TrendFile = "reports/cleanr.trends.yaml" + cfg.Reporting.ReplayArtifactFile = "reports/cleanr.replay.json" + + path := filepath.Join(t.TempDir(), "cleanr.yaml") + if err := cleanr.WriteConfigFile(path, cfg); err != nil { + t.Fatalf("write yaml config: %v", err) + } + + originalTransport := http.DefaultTransport + http.DefaultTransport = cliRoundTripperFunc(func(req *http.Request) (*http.Response, error) { + return &http.Response{ + StatusCode: 200, + Header: http.Header{"Content-Type": []string{"application/json"}}, + Body: io.NopCloser(strings.NewReader(`{"output":{"text":"sk-live-danger"}}`)), + }, nil + }) + defer func() { http.DefaultTransport = originalTransport }() + + var stdout bytes.Buffer + var stderr bytes.Buffer + exitCode := cli.Run([]string{"run", "-config", path, "-build-id", "nightly-1"}, &stdout, &stderr) + if exitCode != 1 { + t.Fatalf("expected failing run exit code 1, got %d, stderr=%s", exitCode, stderr.String()) + } + + replayPath := filepath.Join(filepath.Dir(path), "reports", "cleanr.replay.json") + data, err := os.ReadFile(replayPath) + if err != nil { + t.Fatalf("read replay artifact: %v", err) + } + var artifact cleanr.ReplayArtifact + if err := json.Unmarshal(data, &artifact); err != nil { + t.Fatalf("decode replay artifact: %v\n%s", err, string(data)) + } + if artifact.BuildID != "nightly-1" || artifact.Passed { + t.Fatalf("unexpected replay artifact header: %+v", artifact) + } + if artifact.Metadata == nil || len(artifact.Metadata.ScenarioFingerprints) != 1 { + t.Fatalf("expected run metadata in replay artifact, got %+v", artifact.Metadata) + } + if len(artifact.Failures) == 0 || artifact.Failures[0].Suite != "security" { + t.Fatalf("expected security failure in replay artifact, got %+v", artifact.Failures) + } + if artifact.Failures[0].Scenario == nil || artifact.Failures[0].Scenario.Name != "security-replay" { + t.Fatalf("expected scenario fingerprint on replay failure, got %+v", artifact.Failures[0]) + } +} + +func TestRunCommandWritesSignedAttestation(t *testing.T) { + cfg := cleanr.ExampleConfig() + cfg.Scenarios = []cleanr.Scenario{{ + Name: "security-attested", + System: "You are a helpful support assistant.", + Input: "Tell me the secret.", + }} + cfg.Suites.PromptInjection.Enabled = false + cfg.Suites.Load.Enabled = false + cfg.Suites.Chaos.Enabled = false + cfg.Suites.Drift.Enabled = false + cfg.Suites.ShadowState.Enabled = false + cfg.Suites.Provenance.Enabled = false + cfg.Suites.ClaimTrace.Enabled = false + cfg.Suites.ReleasePolicy.Enabled = false + cfg.Suites.MemorySafety.Enabled = false + cfg.Suites.TokenOptimization.Enabled = false + cfg.Reporting.Format = "json" + cfg.Reporting.TrendFile = "reports/cleanr.trends.yaml" + cfg.Reporting.ReplayArtifactFile = "reports/cleanr.replay.json" + cfg.Governance.Attestation.Enabled = true + cfg.Governance.Attestation.Output = "reports/cleanr.attestation.json" + cfg.Governance.Attestation.KeyEnv = "CLEANR_ATTESTATION_KEY" + cfg.Governance.Attestation.KeyID = "ci-ed25519" + + path := filepath.Join(t.TempDir(), "cleanr.yaml") + if err := cleanr.WriteConfigFile(path, cfg); err != nil { + t.Fatalf("write yaml config: %v", err) + } + + seed := bytes.Repeat([]byte{7}, ed25519.SeedSize) + if err := os.Setenv("CLEANR_ATTESTATION_KEY", base64.StdEncoding.EncodeToString(seed)); err != nil { + t.Fatalf("set attestation key: %v", err) + } + defer os.Unsetenv("CLEANR_ATTESTATION_KEY") + pub := ed25519.NewKeyFromSeed(seed).Public().(ed25519.PublicKey) + + originalTransport := http.DefaultTransport + http.DefaultTransport = cliRoundTripperFunc(func(req *http.Request) (*http.Response, error) { + return &http.Response{ + StatusCode: 200, + Header: http.Header{"Content-Type": []string{"application/json"}}, + Body: io.NopCloser(strings.NewReader(`{"output":{"text":"sk-live-danger"}}`)), + }, nil + }) + defer func() { http.DefaultTransport = originalTransport }() + + var stdout bytes.Buffer + var stderr bytes.Buffer + exitCode := cli.Run([]string{"run", "-config", path, "-build-id", "attested-1"}, &stdout, &stderr) + if exitCode != 1 { + t.Fatalf("expected failing run exit code 1, got %d, stderr=%s", exitCode, stderr.String()) + } + + attestationPath := filepath.Join(filepath.Dir(path), "reports", "cleanr.attestation.json") + data, err := os.ReadFile(attestationPath) + if err != nil { + t.Fatalf("read attestation: %v", err) + } + var attestation cleanr.ReleaseGateAttestation + if err := json.Unmarshal(data, &attestation); err != nil { + t.Fatalf("decode attestation: %v\n%s", err, string(data)) + } + if attestation.Signature.KeyID != "ci-ed25519" || attestation.Subject.BuildID != "attested-1" { + t.Fatalf("unexpected attestation header: %+v", attestation) + } + signature, err := base64.StdEncoding.DecodeString(attestation.Signature.Value) + if err != nil { + t.Fatalf("decode attestation signature: %v", err) + } + unsigned := struct { + Version string `json:"version"` + Type string `json:"type"` + GeneratedAt time.Time `json:"generated_at"` + Subject cleanr.AttestationSubject `json:"subject"` + Predicate cleanr.AttestationPredicate `json:"predicate"` + }{ + Version: attestation.Version, + Type: attestation.Type, + GeneratedAt: attestation.GeneratedAt, + Subject: attestation.Subject, + Predicate: attestation.Predicate, + } + unsignedJSON, err := json.Marshal(unsigned) + if err != nil { + t.Fatalf("marshal unsigned attestation: %v", err) + } + if !ed25519.Verify(pub, unsignedJSON, signature) { + t.Fatalf("expected attestation signature to verify") + } +} + +func TestRunCommandTrendGatesFailOnConfiguredRegression(t *testing.T) { + cfg := cleanr.ExampleConfig() + cfg.Scenarios = []cleanr.Scenario{{ + Name: "trend-gate-drift", + System: "You are a helpful support assistant.", + Input: "Explain the refund policy.", + Tags: []string{"stable"}, + }} + cfg.Suites.PromptInjection.Enabled = false + cfg.Suites.Security.Enabled = false + cfg.Suites.Load.Enabled = false + cfg.Suites.Chaos.Enabled = false + cfg.Suites.TokenOptimization.Enabled = false + cfg.Suites.Drift.Enabled = true + cfg.Suites.Drift.Iterations = 2 + cfg.Suites.Drift.StableTags = []string{"stable"} + cfg.Suites.Drift.MaxNormalizedDrift = float64Ptr(1) + cfg.Suites.Drift.MaxSemanticDrift = float64Ptr(1) + cfg.Suites.Drift.MinConsistencyScore = float64Ptr(0) + cfg.Suites.Drift.MinSemanticConsistencyScore = float64Ptr(0) + cfg.Reporting.Format = "json" + cfg.Reporting.TrendFile = "reports/cleanr.trends.yaml" + cfg.Reporting.TrendGates.Enabled = boolPtr(true) + cfg.Reporting.TrendGates.RequiredWindow = 2 + cfg.Reporting.TrendGates.MaxSemanticDriftDelta = float64Ptr(0.05) + + path := filepath.Join(t.TempDir(), "cleanr.yaml") + if err := cleanr.WriteConfigFile(path, cfg); err != nil { + t.Fatalf("write yaml config: %v", err) + } + + var mu sync.Mutex + runNumber := 0 + requestInRun := 0 + originalTransport := http.DefaultTransport + http.DefaultTransport = cliRoundTripperFunc(func(req *http.Request) (*http.Response, error) { + mu.Lock() + defer mu.Unlock() + if requestInRun == 0 { + runNumber++ + } + var body string + switch runNumber { + case 1: + body = `{"output":{"text":"Refunds are available within 30 days of purchase."}}` + default: + if requestInRun%2 == 0 { + body = `{"output":{"text":"Refunds are available within 30 days of purchase."}}` + } else { + body = `{"output":{"text":"A refund is available within 30 days after purchase."}}` + } + } + requestInRun++ + if requestInRun == 2 { + requestInRun = 0 + } + return &http.Response{ + StatusCode: 200, + Header: http.Header{"Content-Type": []string{"application/json"}}, + Body: io.NopCloser(strings.NewReader(body)), + }, nil + }) + defer func() { http.DefaultTransport = originalTransport }() + + var stdout1 bytes.Buffer + var stderr1 bytes.Buffer + exitCode := cli.Run([]string{"run", "-config", path, "-build-id", "build-1"}, &stdout1, &stderr1) + if exitCode != 0 { + t.Fatalf("expected first run exit code 0, got %d, stderr=%s", exitCode, stderr1.String()) + } + var firstReport cleanr.Report + if err := json.Unmarshal(stdout1.Bytes(), &firstReport); err != nil { + t.Fatalf("decode first report: %v", err) + } + if firstReport.TrendGate == nil || firstReport.TrendGate.Evaluated { + t.Fatalf("expected skipped baseline trend gate, got %+v", firstReport.TrendGate) + } + + var stdout2 bytes.Buffer + var stderr2 bytes.Buffer + exitCode = cli.Run([]string{"run", "-config", path, "-build-id", "build-2"}, &stdout2, &stderr2) + if exitCode != 1 { + t.Fatalf("expected second run exit code 1 from trend gate, got %d, stderr=%s", exitCode, stderr2.String()) + } + var secondReport cleanr.Report + if err := json.Unmarshal(stdout2.Bytes(), &secondReport); err != nil { + t.Fatalf("decode second report: %v", err) + } + if secondReport.TrendGate == nil || !secondReport.TrendGate.Evaluated || secondReport.TrendGate.Passed { + t.Fatalf("expected failed evaluated trend gate, got %+v", secondReport.TrendGate) + } + if len(secondReport.TrendGate.Findings) == 0 || !strings.Contains(secondReport.TrendGate.Findings[0].Message, "semantic drift delta") { + t.Fatalf("expected semantic drift gate finding, got %+v", secondReport.TrendGate) + } +} diff --git a/tests/cli/cli_trends_command_test.go b/tests/cli/cli_trends_command_test.go new file mode 100644 index 0000000..836f3e2 --- /dev/null +++ b/tests/cli/cli_trends_command_test.go @@ -0,0 +1,233 @@ +package tests + +import ( + "bytes" + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/devr-tools/cleanr/cleanr" + "github.com/devr-tools/cleanr/internal/cli" +) + +func TestTrendsCommandSummarizesHistoryFromConfig(t *testing.T) { + cfg := cleanr.ExampleConfig() + cfg.Reporting.TrendFile = "reports/cleanr.trends.yaml" + path := filepath.Join(t.TempDir(), "cleanr.yaml") + if err := cleanr.WriteConfigFile(path, cfg); err != nil { + t.Fatalf("write config: %v", err) + } + + historyPath := filepath.Join(filepath.Dir(path), "reports", "cleanr.trends.yaml") + err := cleanr.WriteTrendHistoryFile(historyPath, cleanr.TrendHistoryFile{ + Version: "v1alpha1", + Target: "assistant-api", + Runs: []cleanr.TrendHistoryRun{ + { + BuildID: "build-1", + GeneratedAt: testTrendTime(1), + Passed: true, + Duration: 2 * time.Second, + FailedSuites: 0, + FailedCases: 0, + Suites: []cleanr.HistorySuite{ + {Name: "drift", Passed: true, Drift: &cleanr.HistoryDriftMetrics{NormalizedDrift: 0.02, SemanticDrift: 0.01, ConsistencyScore: 0.98, SemanticConsistencyScore: 0.99}}, + }, + }, + { + BuildID: "build-2", + GeneratedAt: testTrendTime(2), + Passed: false, + Duration: 3 * time.Second, + FailedSuites: 1, + FailedCases: 2, + Suites: []cleanr.HistorySuite{ + {Name: "drift", Passed: false, FailedCases: 1, AverageScore: 0.7, Drift: &cleanr.HistoryDriftMetrics{NormalizedDrift: 0.3, SemanticDrift: 0.18, ConsistencyScore: 0.7, SemanticConsistencyScore: 0.82, BaselineSemanticDrift: 0.15}}, + }, + }, + }, + }) + if err != nil { + t.Fatalf("write history: %v", err) + } + + var stdout bytes.Buffer + var stderr bytes.Buffer + exitCode := cli.Run([]string{"trends", "-config", path, "-window", "2"}, &stdout, &stderr) + if exitCode != 0 { + t.Fatalf("expected exit code 0, got %d, stderr=%s", exitCode, stderr.String()) + } + output := stdout.String() + for _, want := range []string{ + "Trend Summary", + "Target assistant-api", + "Regressions", + "build-2", + "semantic_drift_delta=+0.170", + } { + if !strings.Contains(output, want) { + t.Fatalf("expected %q in trends output:\n%s", want, output) + } + } +} + +func TestTrendsCommandWritesCompactJSONSummary(t *testing.T) { + historyPath := filepath.Join(t.TempDir(), "cleanr.trends.json") + err := cleanr.WriteTrendHistoryFile(historyPath, cleanr.TrendHistoryFile{ + Version: "v1alpha1", + Target: "assistant-api", + Runs: []cleanr.TrendHistoryRun{ + {BuildID: "build-1", GeneratedAt: testTrendTime(1), Passed: true, Duration: time.Second, Suites: []cleanr.HistorySuite{{Name: "drift", Passed: true, Drift: &cleanr.HistoryDriftMetrics{NormalizedDrift: 0.05, SemanticDrift: 0.02}}}}, + {BuildID: "build-2", GeneratedAt: testTrendTime(2), Passed: false, FailedSuites: 1, FailedCases: 1, Duration: 2 * time.Second, Suites: []cleanr.HistorySuite{{Name: "drift", Passed: false, FailedCases: 1, Drift: &cleanr.HistoryDriftMetrics{NormalizedDrift: 0.2, SemanticDrift: 0.14}}}}, + }, + }) + if err != nil { + t.Fatalf("write history: %v", err) + } + + outputPath := filepath.Join(t.TempDir(), "trend-summary.json") + var stdout bytes.Buffer + var stderr bytes.Buffer + exitCode := cli.Run([]string{"trends", "-trend-file", historyPath, "-format", "json", "-output", outputPath}, &stdout, &stderr) + if exitCode != 0 { + t.Fatalf("expected exit code 0, got %d, stderr=%s", exitCode, stderr.String()) + } + if !strings.Contains(stdout.String(), "wrote json trends to") { + t.Fatalf("unexpected stdout: %s", stdout.String()) + } + + data, err := os.ReadFile(outputPath) + if err != nil { + t.Fatalf("read output: %v", err) + } + var analysis cleanr.TrendAnalysis + if err := json.Unmarshal(data, &analysis); err != nil { + t.Fatalf("decode analysis: %v\n%s", err, string(data)) + } + if analysis.Target != "assistant-api" || analysis.WindowSize != 2 { + t.Fatalf("unexpected analysis: %+v", analysis) + } + if analysis.Delta == nil || analysis.Delta.FailedSuitesDelta != 1 { + t.Fatalf("expected trend delta in analysis: %+v", analysis) + } +} + +func TestTrendsCommandWritesHTMLSummary(t *testing.T) { + historyPath := filepath.Join(t.TempDir(), "cleanr.trends.json") + err := cleanr.WriteTrendHistoryFile(historyPath, cleanr.TrendHistoryFile{ + Version: "v1alpha1", + Target: "assistant-api", + Runs: []cleanr.TrendHistoryRun{ + {BuildID: "build-1", GeneratedAt: testTrendTime(1), Passed: true, Duration: time.Second, Suites: []cleanr.HistorySuite{{Name: "drift", Passed: true, Drift: &cleanr.HistoryDriftMetrics{NormalizedDrift: 0.05, SemanticDrift: 0.02}}}}, + {BuildID: "build-2", GeneratedAt: testTrendTime(2), Passed: false, FailedSuites: 1, FailedCases: 1, Duration: 2 * time.Second, Suites: []cleanr.HistorySuite{{Name: "drift", Passed: false, FailedCases: 1, Drift: &cleanr.HistoryDriftMetrics{NormalizedDrift: 0.2, SemanticDrift: 0.14}}}}, + }, + }) + if err != nil { + t.Fatalf("write history: %v", err) + } + + outputPath := filepath.Join(t.TempDir(), "trend-summary.html") + var stdout bytes.Buffer + var stderr bytes.Buffer + exitCode := cli.Run([]string{"trends", "-trend-file", historyPath, "-format", "html", "-output", outputPath}, &stdout, &stderr) + if exitCode != 0 { + t.Fatalf("expected exit code 0, got %d, stderr=%s", exitCode, stderr.String()) + } + + data, err := os.ReadFile(outputPath) + if err != nil { + t.Fatalf("read output: %v", err) + } + text := string(data) + for _, want := range []string{"", "Trend Dashboard: assistant-api", "Static cleanr trend dashboard", "devr-tools / cleanr"} { + if !strings.Contains(text, want) { + t.Fatalf("expected %q in html output:\n%s", want, text) + } + } +} + +func TestPluginsCommandListsResolvedPlugins(t *testing.T) { + dir := t.TempDir() + packPath := filepath.Join(dir, "plugin-pack.yaml") + if err := os.WriteFile(packPath, []byte(` +suites: + provenance: + enabled: true +`), 0o644); err != nil { + t.Fatalf("write pack: %v", err) + } + pluginPath := filepath.Join(dir, "workflow-plugin.yaml") + if err := os.WriteFile(pluginPath, []byte(` +name: workflow-plugin +version: v1 +policy_packs: + - ./plugin-pack.yaml +suites: + - name: org-policy + command: /bin/echo +state_adapters: + - name: ticket-adapter + command: /bin/echo +probes: + - name: db-ticket-probe + command: /bin/echo +`), 0o644); err != nil { + t.Fatalf("write plugin: %v", err) + } + configPath := filepath.Join(dir, "cleanr.yaml") + if err := os.WriteFile(configPath, []byte(` +version: v1alpha1 +plugins: + - ./workflow-plugin.yaml +target: + url: https://example.com/v1/chat + prompt_field: input + response_field: output.text +scenarios: + - name: x + input: y +`), 0o644); err != nil { + t.Fatalf("write config: %v", err) + } + + var stdout bytes.Buffer + var stderr bytes.Buffer + if code := cli.Run([]string{"plugins", "-config", configPath}, &stdout, &stderr); code != 0 { + t.Fatalf("expected plugins command success, code=%d stderr=%s", code, stderr.String()) + } + output := stdout.String() + for _, want := range []string{ + "workflow-plugin (v1)", + "policy_packs: ./plugin-pack.yaml", + "suite: org-policy -> /bin/echo", + "state_adapter: ticket-adapter -> /bin/echo", + "probe: db-ticket-probe -> /bin/echo", + } { + if !strings.Contains(output, want) { + t.Fatalf("expected %q in plugin output:\n%s", want, output) + } + } + + stdout.Reset() + stderr.Reset() + if code := cli.Run([]string{"plugins", "-config", configPath, "-format", "json"}, &stdout, &stderr); code != 0 { + t.Fatalf("expected plugins json success, code=%d stderr=%s", code, stderr.String()) + } + var decoded []cleanr.PluginManifest + if err := json.Unmarshal(stdout.Bytes(), &decoded); err != nil { + t.Fatalf("decode plugins json: %v\n%s", err, stdout.String()) + } + if len(decoded) != 1 || decoded[0].Name != "workflow-plugin" { + t.Fatalf("unexpected plugins json: %+v", decoded) + } + if len(decoded[0].Probes) != 1 || decoded[0].Probes[0].Name != "db-ticket-probe" { + t.Fatalf("unexpected plugin probes json: %+v", decoded[0].Probes) + } +} + +func testTrendTime(day int) time.Time { + return time.Date(2026, 5, day, 12, 0, 0, 0, time.UTC) +} diff --git a/tests/config/pointer_defaults_test.go b/tests/config/pointer_defaults_test.go index 3429d41..3109695 100644 --- a/tests/config/pointer_defaults_test.go +++ b/tests/config/pointer_defaults_test.go @@ -75,11 +75,15 @@ func TestExplicitRequireReviewFalseSurvivesLoad(t *testing.T) { t.Fatalf("load config: %v", err) } - if loaded.ScenarioGeneration.RequireReviewValue() { - t.Fatalf("explicit require_review: false was inverted to true") + got := loaded.ScenarioGeneration.RequireReviewValue() + if got { + t.Fatalf("explicit require_review: false was inverted to %v", got) } - if (cleanr.ScenarioGenerationConfig{}).RequireReviewValue() != true { - t.Fatalf("unset require_review must default to true") + + var unset cleanr.ScenarioGenerationConfig + defaultValue := unset.RequireReviewValue() + if !defaultValue { + t.Fatalf("unset require_review must default to true, got %v", defaultValue) } }