|
| 1 | +import { |
| 2 | + agentIntentSchema, |
| 3 | + agentPageContextSchema, |
| 4 | + isRevisableBlock, |
| 5 | + safeParseStoredViewBlock, |
| 6 | + safeParseTriggerUri, |
| 7 | + suggestedPromptSchema, |
| 8 | + viewBlockSchema, |
| 9 | + watchIdentity, |
| 10 | + watchSpecSchema, |
| 11 | + SUGGESTED_PROMPT_CAP, |
| 12 | + type Evidence, |
| 13 | +} from "@internal/dashboard-agent-contracts"; |
| 14 | +import { readdirSync, readFileSync, statSync } from "node:fs"; |
| 15 | +import { join } from "node:path"; |
| 16 | +import { describe, expect, it } from "vitest"; |
| 17 | +import * as fixtures from "./fixtures"; |
| 18 | +import { DEMO_ID_PREFIX, DEMO_MARKER } from "./ids"; |
| 19 | + |
| 20 | +const DEMO_DIR = __dirname; |
| 21 | + |
| 22 | +function walk(dir: string): string[] { |
| 23 | + return readdirSync(dir).flatMap((entry) => { |
| 24 | + const path = join(dir, entry); |
| 25 | + return statSync(path).isDirectory() ? walk(path) : [path]; |
| 26 | + }); |
| 27 | +} |
| 28 | + |
| 29 | +const sourceFiles = walk(DEMO_DIR).filter( |
| 30 | + (path) => /\.(ts|tsx)$/.test(path) && !path.endsWith(".test.ts") |
| 31 | +); |
| 32 | + |
| 33 | +function importSpecifiers(source: string): string[] { |
| 34 | + return [...source.matchAll(/(?:import|export)[\s\S]*?from\s+["']([^"']+)["']/g)].map( |
| 35 | + (match) => match[1]! |
| 36 | + ); |
| 37 | +} |
| 38 | + |
| 39 | +describe("demo ids", () => { |
| 40 | + it("namespaces investigation, hypothesis, watch and prompt ids", () => { |
| 41 | + for (const investigation of Object.values(fixtures.demoInvestigations)) { |
| 42 | + expect(investigation.investigationId.startsWith(DEMO_ID_PREFIX)).toBe(true); |
| 43 | + for (const hypothesis of investigation.hypotheses) { |
| 44 | + expect(hypothesis.id.startsWith(DEMO_ID_PREFIX)).toBe(true); |
| 45 | + } |
| 46 | + } |
| 47 | + for (const watch of fixtures.demoWatches.row) { |
| 48 | + expect(watch.id.startsWith(DEMO_ID_PREFIX)).toBe(true); |
| 49 | + } |
| 50 | + for (const prompts of Object.values(fixtures.demoPromptSets)) { |
| 51 | + for (const prompt of prompts) { |
| 52 | + expect(prompt.id.startsWith(DEMO_ID_PREFIX)).toBe(true); |
| 53 | + } |
| 54 | + } |
| 55 | + }); |
| 56 | + |
| 57 | + it("marks every resource id, so nothing can pass for a real one", () => { |
| 58 | + for (const value of Object.values(fixtures.demoViewBlocks)) { |
| 59 | + if (value.type === "diagnosis") { |
| 60 | + expect(value.runId).toContain(DEMO_MARKER); |
| 61 | + } |
| 62 | + } |
| 63 | + for (const id of Object.values({ |
| 64 | + failedRunId: fixtures.demoInvestigationConcluded.runId, |
| 65 | + slowRunId: fixtures.demoInvestigationInconclusive.runId, |
| 66 | + })) { |
| 67 | + expect(id).toContain(DEMO_MARKER); |
| 68 | + } |
| 69 | + }); |
| 70 | +}); |
| 71 | + |
| 72 | +describe("view block fixtures", () => { |
| 73 | + const blocks = Object.values(fixtures.demoViewBlocks); |
| 74 | + |
| 75 | + it("parses every block through the lenient stored-block schema", () => { |
| 76 | + for (const block of blocks) { |
| 77 | + const result = safeParseStoredViewBlock(block); |
| 78 | + expect(result.success, JSON.stringify(result.error?.issues)).toBe(true); |
| 79 | + } |
| 80 | + }); |
| 81 | + |
| 82 | + it("parses the enveloped blocks through the strict schema too", () => { |
| 83 | + for (const block of [ |
| 84 | + fixtures.demoDiagnosisBlockFirstPass, |
| 85 | + fixtures.demoDiagnosisBlockRevised, |
| 86 | + fixtures.demoChartBlock, |
| 87 | + ]) { |
| 88 | + expect(viewBlockSchema.safeParse(block).success).toBe(true); |
| 89 | + expect(isRevisableBlock(block)).toBe(true); |
| 90 | + } |
| 91 | + }); |
| 92 | + |
| 93 | + it("keeps one legacy, envelope-less block that is not revisable", () => { |
| 94 | + const legacy = fixtures.demoLegacyDiagnosisBlock; |
| 95 | + expect(viewBlockSchema.safeParse(legacy).success).toBe(false); |
| 96 | + expect(safeParseStoredViewBlock(legacy).success).toBe(true); |
| 97 | + expect(isRevisableBlock(legacy)).toBe(false); |
| 98 | + }); |
| 99 | + |
| 100 | + it("revises a block by id rather than emitting a second one", () => { |
| 101 | + expect(fixtures.demoDiagnosisBlockRevised.id).toBe(fixtures.demoDiagnosisBlockFirstPass.id); |
| 102 | + expect(fixtures.demoDiagnosisBlockRevised.revision).toBeGreaterThan( |
| 103 | + fixtures.demoDiagnosisBlockFirstPass.revision |
| 104 | + ); |
| 105 | + }); |
| 106 | +}); |
| 107 | + |
| 108 | +describe("investigation fixtures", () => { |
| 109 | + const investigations = Object.values(fixtures.demoInvestigations); |
| 110 | + |
| 111 | + const allEvidence = (): Evidence[] => |
| 112 | + investigations.flatMap((investigation) => [ |
| 113 | + ...investigation.evidence, |
| 114 | + ...investigation.hypotheses.flatMap((hypothesis) => hypothesis.evidence), |
| 115 | + ]); |
| 116 | + |
| 117 | + it("cites only valid trigger:// URIs, with the kind matching the URI", () => { |
| 118 | + for (const evidence of allEvidence()) { |
| 119 | + const parsed = safeParseTriggerUri(evidence.uri); |
| 120 | + expect(parsed.success, `${evidence.uri}: ${!parsed.success ? parsed.error : ""}`).toBe(true); |
| 121 | + if (parsed.success) expect(parsed.data.kind).toBe(evidence.kind); |
| 122 | + expect(evidence.uri).toContain(DEMO_MARKER); |
| 123 | + } |
| 124 | + }); |
| 125 | + |
| 126 | + it("only offers a fix when it concluded, and only 'check next' when it didn't", () => { |
| 127 | + for (const investigation of investigations) { |
| 128 | + if (investigation.outcome === "concluded") { |
| 129 | + expect(investigation.remediation).toBeTruthy(); |
| 130 | + expect(investigation.checkNext).toBeUndefined(); |
| 131 | + } else { |
| 132 | + expect(investigation.remediation).toBeUndefined(); |
| 133 | + } |
| 134 | + if (investigation.outcome === "inconclusive") { |
| 135 | + expect(investigation.checkNext?.length).toBeGreaterThan(0); |
| 136 | + } |
| 137 | + } |
| 138 | + }); |
| 139 | + |
| 140 | + it("gives the concluded card at least two settled hypotheses", () => { |
| 141 | + const settled = fixtures.demoInvestigationConcluded.hypotheses.filter( |
| 142 | + (hypothesis) => hypothesis.verdict !== "testing" |
| 143 | + ); |
| 144 | + expect(settled.length).toBeGreaterThanOrEqual(2); |
| 145 | + expect(settled.some((h) => h.verdict === "validated")).toBe(true); |
| 146 | + expect(settled.some((h) => h.verdict === "invalidated")).toBe(true); |
| 147 | + expect( |
| 148 | + fixtures.demoInvestigationConcluded.hypotheses.every( |
| 149 | + (h) => h.verdict === "testing" || h.finding |
| 150 | + ) |
| 151 | + ).toBe(true); |
| 152 | + }); |
| 153 | + |
| 154 | + it("keeps a streaming revision with a hypothesis still testing", () => { |
| 155 | + expect(fixtures.demoInvestigationStreamingRev1.investigationId).toBe( |
| 156 | + fixtures.demoInvestigationStreamingRev0.investigationId |
| 157 | + ); |
| 158 | + expect(fixtures.demoInvestigationStreamingRev1.revision).toBeGreaterThan( |
| 159 | + fixtures.demoInvestigationStreamingRev0.revision |
| 160 | + ); |
| 161 | + expect( |
| 162 | + fixtures.demoInvestigationStreamingRev1.hypotheses.some((h) => h.verdict === "testing") |
| 163 | + ).toBe(true); |
| 164 | + }); |
| 165 | + |
| 166 | + it("hedges the dirty-commit variant with the agreed wording", () => { |
| 167 | + expect(fixtures.demoInvestigationDirtyCommit.caveat?.kind).toBe("dirty_commit"); |
| 168 | + expect(fixtures.demoInvestigationDirtyCommit.caveat?.message).toContain( |
| 169 | + "nearest repository snapshot" |
| 170 | + ); |
| 171 | + }); |
| 172 | + |
| 173 | + it("cites file:line@sha in the show-code turn", () => { |
| 174 | + expect(fixtures.demoShowCodeMarkdown).toMatch(/\.ts:\d+(-\d+)?@[0-9a-z]{7}/); |
| 175 | + expect(fixtures.demoShowCodeMarkdown).toContain("```diff"); |
| 176 | + }); |
| 177 | +}); |
| 178 | + |
| 179 | +describe("watch fixtures", () => { |
| 180 | + it("validates every spec against the contracts schema", () => { |
| 181 | + for (const watch of fixtures.demoWatches.row) { |
| 182 | + const result = watchSpecSchema.safeParse(watch.spec); |
| 183 | + expect(result.success, JSON.stringify(result.error?.issues)).toBe(true); |
| 184 | + } |
| 185 | + }); |
| 186 | + |
| 187 | + it("derives the chip identity from the spec", () => { |
| 188 | + for (const watch of fixtures.demoWatches.row) { |
| 189 | + expect(watch.identity).toBe(watchIdentity(watch.spec)); |
| 190 | + } |
| 191 | + }); |
| 192 | + |
| 193 | + it("covers every watch status and offers cancel only while active", () => { |
| 194 | + const statuses = new Set(fixtures.demoWatches.row.map((watch) => watch.status)); |
| 195 | + expect(statuses).toEqual(new Set(["active", "fired", "expired", "cancelled"])); |
| 196 | + for (const watch of fixtures.demoWatches.row) { |
| 197 | + expect(watch.cancellable).toBe(watch.status === "active"); |
| 198 | + } |
| 199 | + }); |
| 200 | + |
| 201 | + it("has an expiry narration that admits it could not verify", () => { |
| 202 | + expect(fixtures.demoWatchNarration.expiryUnverified).toContain("couldn't verify"); |
| 203 | + }); |
| 204 | +}); |
| 205 | + |
| 206 | +describe("intent fixtures", () => { |
| 207 | + it("validates every intent and marks propose_fix non-executable", () => { |
| 208 | + for (const demoIntent of Object.values(fixtures.demoIntents)) { |
| 209 | + const result = agentIntentSchema.safeParse(demoIntent.intent); |
| 210 | + expect(result.success, JSON.stringify(result.error?.issues)).toBe(true); |
| 211 | + expect(demoIntent.executable).toBe(demoIntent.intent.kind !== "propose_fix"); |
| 212 | + } |
| 213 | + }); |
| 214 | +}); |
| 215 | + |
| 216 | +describe("page context and prompt fixtures", () => { |
| 217 | + it("validates every page context", () => { |
| 218 | + for (const context of Object.values(fixtures.demoPageContexts)) { |
| 219 | + const result = agentPageContextSchema.safeParse(context); |
| 220 | + expect(result.success, JSON.stringify(result.error?.issues)).toBe(true); |
| 221 | + } |
| 222 | + }); |
| 223 | + |
| 224 | + it("covers all four signal kinds", () => { |
| 225 | + const kinds = new Set(fixtures.demoSignalsByPriority.map((signal) => signal.kind)); |
| 226 | + expect(kinds).toEqual( |
| 227 | + new Set(["fresh_failure", "waiting_run", "slow_run", "concurrency_saturation"]) |
| 228 | + ); |
| 229 | + }); |
| 230 | + |
| 231 | + it("validates every chip, stays under the cap, and promotes at most one", () => { |
| 232 | + for (const prompts of Object.values(fixtures.demoPromptSets)) { |
| 233 | + expect(prompts.length).toBeLessThanOrEqual(SUGGESTED_PROMPT_CAP); |
| 234 | + expect(prompts.filter((prompt) => prompt.source === "promoted").length).toBeLessThanOrEqual( |
| 235 | + 1 |
| 236 | + ); |
| 237 | + for (const prompt of prompts) { |
| 238 | + expect(suggestedPromptSchema.safeParse(prompt).success).toBe(true); |
| 239 | + } |
| 240 | + } |
| 241 | + }); |
| 242 | + |
| 243 | + it("drops dismissed chips from the resolved row", () => { |
| 244 | + for (const id of fixtures.demoDismissedPromptIds) { |
| 245 | + expect(fixtures.demoPromptsAfterDismissal.some((prompt) => prompt.id === id)).toBe(false); |
| 246 | + } |
| 247 | + }); |
| 248 | +}); |
| 249 | + |
| 250 | +describe("report fixtures", () => { |
| 251 | + it("covers a healthy and a degraded verdict", () => { |
| 252 | + expect(fixtures.demoHealthyReport.summary.severity).toBe("ok"); |
| 253 | + expect(fixtures.demoDegradedReport.summary.severity).toBe("crit"); |
| 254 | + }); |
| 255 | + |
| 256 | + it("references only metrics the report carries, and only links it declares", () => { |
| 257 | + for (const vm of Object.values(fixtures.demoReports)) { |
| 258 | + const metricIds = new Set(vm.metrics.map((metric) => metric.id)); |
| 259 | + for (const finding of vm.findings) { |
| 260 | + for (const id of finding.metricIds) expect(metricIds.has(id), id).toBe(true); |
| 261 | + } |
| 262 | + const linkKeys = new Set(vm.links.map((link) => link.key)); |
| 263 | + for (const entry of vm.footer) { |
| 264 | + if (entry.link) expect(linkKeys.has(entry.link), entry.link).toBe(true); |
| 265 | + } |
| 266 | + expect(vm.footer.length).toBeLessThanOrEqual(3); |
| 267 | + } |
| 268 | + }); |
| 269 | +}); |
| 270 | + |
| 271 | +describe("chart fixtures", () => { |
| 272 | + it("has a row for every configured column", () => { |
| 273 | + const columns = fixtures.demoChart.columns.map((column) => column.name); |
| 274 | + for (const row of fixtures.demoChart.rows) { |
| 275 | + expect(Object.keys(row).sort()).toEqual([...columns].sort()); |
| 276 | + } |
| 277 | + expect(columns).toContain(fixtures.demoChart.config.xAxisColumn); |
| 278 | + for (const y of fixtures.demoChart.config.yAxisColumns) expect(columns).toContain(y); |
| 279 | + }); |
| 280 | +}); |
| 281 | + |
| 282 | +describe("isolation", () => { |
| 283 | + it("imports no server module and no route", () => { |
| 284 | + for (const path of sourceFiles) { |
| 285 | + const specifiers = importSpecifiers(readFileSync(path, "utf8")); |
| 286 | + for (const specifier of specifiers) { |
| 287 | + expect(specifier.includes(".server"), `${path} -> ${specifier}`).toBe(false); |
| 288 | + expect(/routes?\//.test(specifier), `${path} -> ${specifier}`).toBe(false); |
| 289 | + expect(specifier.includes("~/db"), `${path} -> ${specifier}`).toBe(false); |
| 290 | + } |
| 291 | + } |
| 292 | + }); |
| 293 | + |
| 294 | + it("makes no network calls", () => { |
| 295 | + for (const path of sourceFiles) { |
| 296 | + const source = readFileSync(path, "utf8"); |
| 297 | + expect(/\bfetch\s*\(/.test(source), path).toBe(false); |
| 298 | + expect(/\buseFetcher\b/.test(source), path).toBe(false); |
| 299 | + } |
| 300 | + }); |
| 301 | + |
| 302 | + it("has no server file of its own", () => { |
| 303 | + expect(sourceFiles.filter((path) => path.endsWith(".server.ts"))).toEqual([]); |
| 304 | + }); |
| 305 | +}); |
0 commit comments