From 4cd6fe1bbba8eef5237f190662777d71f52d3375 Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Sat, 18 Jul 2026 11:38:43 -0600 Subject: [PATCH 1/5] refactor(knowledge): modularize KB and RAG internals --- src/kb-improvement.ts | 2722 +---------------------- src/kb-improvement/activation.ts | 298 +++ src/kb-improvement/contracts.ts | 520 +++++ src/kb-improvement/evaluation.ts | 442 ++++ src/kb-improvement/run.ts | 222 ++ src/kb-improvement/state.ts | 266 +++ src/kb-improvement/transition.ts | 623 ++++++ src/kb-improvement/workspace.ts | 477 ++++ src/rag-eval.ts | 950 +------- src/rag-eval/answer.ts | 279 +++ src/rag-eval/calibration.ts | 69 + src/rag-eval/contracts.ts | 181 ++ src/rag-eval/knowledge-base.ts | 79 + src/rag-eval/providers.ts | 91 + src/rag-eval/scoring.ts | 269 +++ tests/kb-improvement.test.ts | 1867 ---------------- tests/kb-improvement/activation.test.ts | 599 +++++ tests/kb-improvement/candidate.test.ts | 410 ++++ tests/kb-improvement/integrity.test.ts | 343 +++ tests/kb-improvement/promotion.test.ts | 396 ++++ tests/support/kb-improvement.ts | 201 ++ 21 files changed, 5850 insertions(+), 5454 deletions(-) create mode 100644 src/kb-improvement/activation.ts create mode 100644 src/kb-improvement/contracts.ts create mode 100644 src/kb-improvement/evaluation.ts create mode 100644 src/kb-improvement/run.ts create mode 100644 src/kb-improvement/state.ts create mode 100644 src/kb-improvement/transition.ts create mode 100644 src/kb-improvement/workspace.ts create mode 100644 src/rag-eval/answer.ts create mode 100644 src/rag-eval/calibration.ts create mode 100644 src/rag-eval/contracts.ts create mode 100644 src/rag-eval/knowledge-base.ts create mode 100644 src/rag-eval/providers.ts create mode 100644 src/rag-eval/scoring.ts delete mode 100644 tests/kb-improvement.test.ts create mode 100644 tests/kb-improvement/activation.test.ts create mode 100644 tests/kb-improvement/candidate.test.ts create mode 100644 tests/kb-improvement/integrity.test.ts create mode 100644 tests/kb-improvement/promotion.test.ts create mode 100644 tests/support/kb-improvement.ts diff --git a/src/kb-improvement.ts b/src/kb-improvement.ts index 158f653..9765193 100644 --- a/src/kb-improvement.ts +++ b/src/kb-improvement.ts @@ -1,2671 +1,51 @@ -import { createHash } from 'node:crypto' -import { cp, lstat, mkdir, mkdtemp, rm, stat } from 'node:fs/promises' -import { tmpdir } from 'node:os' -import { dirname, isAbsolute, join, relative, resolve, sep } from 'node:path' -import { - canonicalJson, - contentHash, - type RunRecord, - validateRunRecord, -} from '@tangle-network/agent-eval' -import { - type AgentImprovementActivation, - type AgentImprovementActivationResult, - agentImprovementActivationResultSchema, - agentImprovementActivationSchema, - canonicalCandidateDigest, - omitTopLevelDigest, - type Sha256Digest, - sha256DigestSchema, -} from '@tangle-network/agent-interface' -import { z } from 'zod' -import { - isMissingFile, - listRegularFilesWithinRoot, - readRegularFileWithinRoot, - renameDurable, - withSafeDirectory, - writeJsonDurableWithinRoot, -} from './durable-fs' -import type { - BuildEvalKnowledgeBundleOptions, - EvalKnowledgeBundleBuildResult, - KnowledgeReadinessSpec, -} from './eval-readiness' -import { - applyKnowledgeFileTransaction, - assertKnowledgeMutationPath, - finishKnowledgeFileTransaction, - type KnowledgeFileMutation, - type KnowledgeFileTransaction, - type KnowledgeFileTransactionPlanEntry, - knowledgeFileTransactionPlanHash, - prepareKnowledgeFileTransaction, - rollbackKnowledgeFileTransaction, -} from './file-transaction' -import { sha256, slugify, stableId } from './ids' -import { buildKnowledgeIndex, writeKnowledgeIndex } from './indexer' -import { acquireDurableFileLock, withKnowledgeMutation, withKnowledgeRead } from './mutation-lock' -import { - type KnowledgeBaseQualityOptions, - type KnowledgeBaseQualityReport, - scoreKnowledgeBaseIndex, -} from './rag-eval' -import { - type RagKnowledgeImprovementPhase, - type RagKnowledgeResearchOptions, - type RagKnowledgeUpdateInput, - type RagKnowledgeUpdateResult, - type RunRagKnowledgeImprovementLoopOptions, - type RunRagKnowledgeImprovementLoopResult, - runRagKnowledgeImprovementLoop, -} from './rag-improvement-loop' -import { readinessFor } from './readiness-helpers' -import type { RunKnowledgeResearchLoopOptions } from './research-loop' -import type { RunRetrievalImprovementLoopOptions } from './retrieval-eval' -import { layoutFor } from './store' -import type { KnowledgeIndex } from './types' -import { - type ValidateKnowledgeOptions, - type ValidateKnowledgeResult, - validateKnowledgeIndex, -} from './validate' - -export type KnowledgeImprovementStatus = - | 'running' - | 'candidate-ready' - | 'promoted' - | 'rejected' - | 'blocked' - -interface KnowledgeImprovementMetricProvenanceBase { - evaluator: string - version: string -} - -export type KnowledgeImprovementMetricProvenance = - | (KnowledgeImprovementMetricProvenanceBase & { - method: 'deterministic' - }) - | (KnowledgeImprovementMetricProvenanceBase & { - method: 'sampled' | 'composite' - corpusHash: string - runRecords: RunRecord[] - }) - | (KnowledgeImprovementMetricProvenanceBase & { - method: 'model' - model: string - corpusHash: string - runRecords: RunRecord[] - }) - -export interface KnowledgeImprovementMetric { - score: number - passed: boolean - dimensions?: Record - notes?: string - provenance: KnowledgeImprovementMetricProvenance -} - -export interface KnowledgeImprovementEvaluationInput { - runId: string - iteration: number - root: string - baselineRoot: string - candidateRoot: string - baselineIndex: KnowledgeIndex - candidateIndex: KnowledgeIndex - baseHash: string - candidateHash: string - validation: ValidateKnowledgeResult - readiness?: EvalKnowledgeBundleBuildResult - kbQuality: KnowledgeBaseQualityReport - lifecycle?: RunRagKnowledgeImprovementLoopResult - signal?: AbortSignal -} - -export type KnowledgeImprovementEvaluator = ( - input: KnowledgeImprovementEvaluationInput, -) => Promise | KnowledgeImprovementMetric - -export interface KnowledgeImprovementCandidateRecord { - iteration: number - candidateId: string - baseHash: string - candidateHash?: string - evidenceHash?: string - promotionPlanHash?: string - status: KnowledgeImprovementStatus - createdAt: string - updatedAt: string -} - -export interface KnowledgeImprovementRunState { - runId: string - root: string - goal: string - status: KnowledgeImprovementStatus - baseHash: string - createdAt: string - updatedAt: string - ownerId?: string - candidates: KnowledgeImprovementCandidateRecord[] - promotedCandidateId?: string - blockedReason?: string -} - -export interface KnowledgeImprovementResult { - runId: string - state: KnowledgeImprovementRunState - candidate?: KnowledgeImprovementCandidateRecord - evaluation?: KnowledgeImprovementMetric - lifecycle?: RunRagKnowledgeImprovementLoopResult - promoted: boolean - blocked: boolean -} - -export type KnowledgeImprovementTarget = 'candidate' | 'baseline' - -export interface KnowledgeImprovementMutationReceipt { - target: KnowledgeImprovementTarget - beforeHash: string - afterHash: string - changed: boolean - transactionId: string | null - recovered: boolean -} - -export interface KnowledgeImprovementMutationResult extends KnowledgeImprovementResult { - candidate: KnowledgeImprovementCandidateRecord - mutation: KnowledgeImprovementMutationReceipt - activationResult?: AgentImprovementActivationResult -} - -export interface KnowledgeImprovementActivationPersistence { - activation: AgentImprovementActivation - attemptedAt: string - identity: string - /** May run again after interruption; keep this deterministic and free of external side effects. */ - createResult( - mutation: KnowledgeImprovementMutationReceipt, - ): Promise | AgentImprovementActivationResult -} - -const digestSchema = z.string().regex(/^[a-f0-9]{64}$/) -const runIdSchema = z.string().min(1).max(2_048) -const safePathSegmentSchema = z - .string() - .min(1) - .max(128) - .regex(/^[A-Za-z0-9][A-Za-z0-9._-]*$/) - -const knowledgeImprovementMutationReceiptSchema = z - .object({ - target: z.enum(['candidate', 'baseline']), - beforeHash: digestSchema, - afterHash: digestSchema, - changed: z.boolean(), - transactionId: z.string().uuid().nullable(), - recovered: z.boolean(), - }) - .strict() - -const knowledgeImprovementActivationRecordSchema = z - .object({ - kind: z.literal('knowledge-improvement-activation-result'), - candidateId: safePathSegmentSchema, - mutation: knowledgeImprovementMutationReceiptSchema, - result: agentImprovementActivationResultSchema, - }) - .strict() -type KnowledgeImprovementActivationRecord = z.infer< - typeof knowledgeImprovementActivationRecordSchema -> -const improvementStatusSchema = z.enum([ - 'running', - 'candidate-ready', - 'promoted', - 'rejected', - 'blocked', -]) -const runRecordSchema = z.custom((value) => { - try { - validateRunRecord(value) - return true - } catch { - return false - } -}, 'invalid agent-eval RunRecord') -const deterministicMetricProvenanceSchema = z - .object({ - evaluator: z.string().min(1), - version: z.string().min(1), - method: z.literal('deterministic'), - }) - .strict() -const measuredMetricProvenanceSchema = z - .object({ - evaluator: z.string().min(1), - version: z.string().min(1), - method: z.enum(['sampled', 'composite']), - corpusHash: digestSchema, - runRecords: z.array(runRecordSchema).min(1), - }) - .strict() -const modelMetricProvenanceSchema = z - .object({ - evaluator: z.string().min(1), - version: z.string().min(1), - method: z.literal('model'), - model: z.string().min(1), - corpusHash: digestSchema, - runRecords: z.array(runRecordSchema).min(1), - }) - .strict() -const improvementMetricSchema = z - .object({ - score: z.number().finite().min(0).max(1), - passed: z.boolean(), - dimensions: z.record(z.string(), z.number().finite()).optional(), - notes: z.string().optional(), - provenance: z.discriminatedUnion('method', [ - deterministicMetricProvenanceSchema, - measuredMetricProvenanceSchema, - modelMetricProvenanceSchema, - ]), - }) - .strict() - .superRefine((metric, context) => { - if (metric.provenance.method === 'deterministic') return - const actualCorpusHash = contentHash(metric.provenance.runRecords) - if (actualCorpusHash !== metric.provenance.corpusHash) { - context.addIssue({ - code: 'custom', - path: ['provenance', 'corpusHash'], - message: 'metric corpus hash must bind the complete RunRecord array', - }) - } - }) -const candidateRecordSchema = z - .object({ - iteration: z.number().int().positive(), - candidateId: safePathSegmentSchema, - baseHash: digestSchema, - candidateHash: digestSchema.optional(), - evidenceHash: digestSchema.optional(), - promotionPlanHash: digestSchema.optional(), - status: improvementStatusSchema, - createdAt: z.iso.datetime(), - updatedAt: z.iso.datetime(), - }) - .strict() - -export const KnowledgeImprovementRunStateSchema = z - .object({ - runId: runIdSchema, - root: z.string().min(1), - goal: z.string().min(1), - status: improvementStatusSchema, - baseHash: digestSchema, - createdAt: z.iso.datetime(), - updatedAt: z.iso.datetime(), - ownerId: z.string().min(1).optional(), - candidates: z.array(candidateRecordSchema), - promotedCandidateId: safePathSegmentSchema.optional(), - blockedReason: z.string().min(1).optional(), - }) - .strict() - .superRefine((state, context) => { - const candidateIds = new Set() - for (const [index, candidate] of state.candidates.entries()) { - if (candidateIds.has(candidate.candidateId)) { - context.addIssue({ - code: 'custom', - path: ['candidates', index, 'candidateId'], - message: 'candidate ids must be unique within an improvement run', - }) - } - candidateIds.add(candidate.candidateId) - if (candidate.baseHash !== state.baseHash) { - context.addIssue({ - code: 'custom', - path: ['candidates', index, 'baseHash'], - message: 'candidate base hash must match its improvement run', - }) - } - if (candidate.status === 'candidate-ready') { - if (!candidate.candidateHash || !candidate.evidenceHash || !candidate.promotionPlanHash) { - context.addIssue({ - code: 'custom', - path: ['candidates', index], - message: 'ready candidates require content, evidence, and promotion-plan identities', - }) - } - } - if ( - candidate.status === 'promoted' && - (!candidate.candidateHash || !candidate.evidenceHash || !candidate.promotionPlanHash) - ) { - context.addIssue({ - code: 'custom', - path: ['candidates', index], - message: 'promoted candidates require content, evidence, and promotion-plan identities', - }) - } - if ( - candidate.status === 'promoted' && - Boolean(candidate.evidenceHash) !== Boolean(candidate.promotionPlanHash) - ) { - context.addIssue({ - code: 'custom', - path: ['candidates', index], - message: 'promoted candidate evidence and promotion-plan identities must appear together', - }) - } - if (candidate.status === 'promoted' && state.status !== 'promoted') { - context.addIssue({ - code: 'custom', - path: ['candidates', index, 'status'], - message: 'only a promoted run may contain a promoted candidate', - }) - } - } - if (state.status === 'promoted') { - const promoted = state.candidates.filter( - (candidate) => candidate.candidateId === state.promotedCandidateId, - ) - if (promoted.length !== 1 || promoted[0]?.status !== 'promoted') { - context.addIssue({ - code: 'custom', - path: ['promotedCandidateId'], - message: 'promoted state must identify exactly one promoted candidate', - }) - } - } else if (state.promotedCandidateId !== undefined) { - context.addIssue({ - code: 'custom', - path: ['promotedCandidateId'], - message: 'only a promoted run may identify a promoted candidate', - }) - } - if ( - state.status === 'candidate-ready' && - state.candidates.filter((candidate) => candidate.status === 'candidate-ready').length !== 1 - ) { - context.addIssue({ - code: 'custom', - path: ['status'], - message: 'candidate-ready state must contain exactly one ready candidate', - }) - } - if (state.status === 'blocked' && state.blockedReason === undefined) { - context.addIssue({ - code: 'custom', - path: ['blockedReason'], - message: 'blocked state must include a reason', - }) - } - }) - -export const KnowledgeImprovementEvidenceSchema = z - .object({ - kind: z.literal('knowledge-improvement-evidence'), - runId: runIdSchema, - candidateId: safePathSegmentSchema, - iteration: z.number().int().positive(), - goalHash: digestSchema, - baseHash: digestSchema, - candidateHash: digestSchema, - promotionPlanHash: digestSchema, - validation: z.unknown(), - readiness: z.unknown().nullable(), - kbQuality: z.unknown(), - evaluation: improvementMetricSchema, - lifecycle: z.unknown().nullable(), - }) - .strict() - -export type KnowledgeImprovementEvidence = z.infer - -/** Portable identity of one measured candidate. Paths and mutable run state are deliberately excluded. */ -export const KnowledgeImprovementCandidateRefSchema = z - .object({ - kind: z.literal('knowledge-improvement-candidate'), - runId: runIdSchema, - candidateId: safePathSegmentSchema, - goalHash: digestSchema, - baseHash: digestSchema, - candidateHash: digestSchema, - evidenceHash: digestSchema, - promotionPlanHash: digestSchema, - }) - .strict() - -export type KnowledgeImprovementCandidateRef = z.infer< - typeof KnowledgeImprovementCandidateRefSchema -> - -export interface PromoteKnowledgeCandidateOptions { - root: string - candidate: KnowledgeImprovementCandidateRef - activation?: KnowledgeImprovementActivationPersistence - ownerId?: string - leaseTtlMs?: number - now?: () => Date - onState?: (state: KnowledgeImprovementRunState) => Promise | void -} - -export type RestoreKnowledgeCandidateBaselineOptions = PromoteKnowledgeCandidateOptions - -export interface LoadKnowledgeImprovementActivationResultOptions { - root: string - candidate: KnowledgeImprovementCandidateRef - activation: AgentImprovementActivation - identity: string -} - -export interface UseKnowledgeImprovementCandidateOptions { - root: string - candidate: KnowledgeImprovementCandidateRef -} - -export interface ResolvedKnowledgeImprovementComparisonSnapshot { - root: string - hash: string -} - -export interface ResolvedKnowledgeImprovementComparison { - reference: KnowledgeImprovementCandidateRef - evaluation: KnowledgeImprovementMetric - baseline: ResolvedKnowledgeImprovementComparisonSnapshot - candidate: ResolvedKnowledgeImprovementComparisonSnapshot -} - -export interface ResolvedKnowledgeImprovementCandidate { - root: string - candidate: KnowledgeImprovementCandidateRef - evaluation: KnowledgeImprovementMetric -} - -export interface KnowledgeImprovementRetrievalOptions - extends Omit { - runDir?: RunRetrievalImprovementLoopOptions['runDir'] -} - -export interface KnowledgeImprovementUpdateInput extends RagKnowledgeUpdateInput { - runId: string - iteration: number - candidateId: string - root: string - baselineRoot: string - candidateRoot: string - baseHash: string -} - -export type KnowledgeImprovementUpdate = ( - input: KnowledgeImprovementUpdateInput, -) => Promise | RagKnowledgeUpdateResult - -export interface KnowledgeImprovementOptions { - root: string - goal: string - runId?: string - ownerId?: string - leaseTtlMs?: number - resume?: boolean - maxCandidates?: number - candidateResearchIterations?: number - strict?: ValidateKnowledgeOptions['strict'] - readinessSpecs?: KnowledgeReadinessSpec[] - readinessTaskId?: string - readiness?: Omit - kbQuality?: KnowledgeBaseQualityOptions - step?: RunKnowledgeResearchLoopOptions['step'] - knowledgeResearch?: Omit - retrieval?: KnowledgeImprovementRetrievalOptions - diagnose?: NonNullable - acquireKnowledge?: NonNullable - updateKnowledge?: KnowledgeImprovementUpdate - evaluateAnswers?: NonNullable - decidePromotion?: NonNullable - enabledPhases?: readonly RagKnowledgeImprovementPhase[] - requiredPhases?: readonly RagKnowledgeImprovementPhase[] - evaluate?: KnowledgeImprovementEvaluator - signal?: AbortSignal - now?: () => Date - onState?: (state: KnowledgeImprovementRunState) => Promise | void -} - -interface LeaseHandle { - ownerId: string - assertOwned(): void - release(): Promise -} - -const DEFAULT_LEASE_TTL_MS = 15 * 60 * 1000 -const UPDATE_PHASES: readonly RagKnowledgeImprovementPhase[] = [ - 'knowledge-acquisition', - 'knowledge-update', -] -const EVALUATION_PHASES: readonly RagKnowledgeImprovementPhase[] = [ - 'retrieval-tuning', - 'gap-diagnosis', - 'answer-quality', - 'promotion', -] - -export function knowledgeImprovementRunId(root: string, goal: string): string { - return stableId('kimpr', `${root}:${goal}`) -} - -export function knowledgeImprovementRunDir(root: string, runId: string): string { - const parsedRunId = runIdSchema.parse(runId) - const safeRunId = safePathSegmentSchema.safeParse(parsedRunId) - const runSegment = safeRunId.success - ? safeRunId.data - : `${slugify(parsedRunId).slice(0, 72)}-${sha256(parsedRunId).slice(0, 16)}` - const improvementsDir = join(layoutFor(root).cacheDir, 'improvements') - const runDir = join(improvementsDir, runSegment) - const resolvedImprovementsDir = resolve(improvementsDir) - const resolvedRunDir = resolve(runDir) - if (!resolvedRunDir.startsWith(`${resolvedImprovementsDir}${sep}`)) { - throw new Error('knowledge improvement run directory escaped its root') - } - return runDir -} - -async function withKnowledgeImprovementRun( - root: string, - runId: string, - create: boolean, - use: (runDir: string) => Promise | T, -): Promise { - const runDir = knowledgeImprovementRunDir(root, runId) - const relativePath = descendantPath(root, runDir) - if (!relativePath) throw new Error('knowledge improvement run directory escaped its root') - return withSafeDirectory(root, relativePath, create, async (openedRunDir) => { - const result = await use(openedRunDir) - const openedIdentity = await stat(openedRunDir) - const currentIdentity = await withSafeDirectory(root, relativePath, false, (currentRunDir) => - stat(currentRunDir), - ) - if (openedIdentity.dev !== currentIdentity.dev || openedIdentity.ino !== currentIdentity.ino) { - throw new Error('knowledge improvement run directory changed during use') - } - return result - }) -} - -export async function loadKnowledgeImprovementState( - root: string, - runId: string, -): Promise { - const expectedRunId = runIdSchema.parse(runId) - try { - return await withKnowledgeImprovementRun(root, expectedRunId, false, (runDir) => - loadKnowledgeImprovementStateFromRun(root, expectedRunId, runDir), - ) - } catch (error) { - if (isMissingFile(error)) return null - throw error - } -} - -/** Load the durable result for one exact activation without changing knowledge or run state. */ -export async function loadKnowledgeImprovementActivationResult( - options: LoadKnowledgeImprovementActivationResultOptions, -): Promise { - assertExactCandidatePlatform() - const candidate = Object.freeze(KnowledgeImprovementCandidateRefSchema.parse(options.candidate)) - const activation = verifyCanonicalKnowledgeActivation(options.activation) - const target = targetForKnowledgeActivation(activation) - assertKnowledgeActivationAuthority(activation, candidate, target, options.identity) - return withKnowledgeImprovementRun(options.root, candidate.runId, false, async (runDir) => { - const record = await loadKnowledgeActivationRecord( - runDir, - candidate, - activation, - target, - options.identity, - ) - return record?.result ?? null - }) -} - -async function loadKnowledgeImprovementStateFromRun( - root: string, - runId: string, - runDir: string, -): Promise { - const stateFile = await readRegularFileWithinRoot(runDir, 'state.json') - const raw = JSON.parse(stateFile.bytes.toString('utf8')) as unknown - const state = KnowledgeImprovementRunStateSchema.parse(raw) as KnowledgeImprovementRunState - if (state.runId !== runId) { - throw new Error('knowledge improvement state does not match the requested run') - } - if (resolve(state.root) !== resolve(root)) { - throw new Error('knowledge improvement state does not match the requested root') - } - for (const candidate of state.candidates) { - if (candidate.status === 'running') await assertCandidateWorkspace(runDir, candidate) - } - return state -} - -/** Freeze the exact knowledge bytes and measured evidence a later approval may promote. */ -export function knowledgeImprovementCandidateRef( - result: Pick, -): KnowledgeImprovementCandidateRef { - if (!result.candidate) throw new Error('knowledge improvement result has no candidate') - return candidateRefFor(result.runId, result.state, result.candidate) -} - -/** Use both frozen sides of one measured comparison in isolated, integrity-checked copies. */ -export async function withKnowledgeImprovementComparison( - options: UseKnowledgeImprovementCandidateOptions, - use: (comparison: ResolvedKnowledgeImprovementComparison) => Promise | T, -): Promise { - assertExactCandidatePlatform() - const reference = Object.freeze(KnowledgeImprovementCandidateRefSchema.parse(options.candidate)) - return withKnowledgeImprovementRun(options.root, reference.runId, false, async (runDir) => { - const state = await loadKnowledgeImprovementStateFromRun(options.root, reference.runId, runDir) - return withMeasuredCandidateSnapshot(options.root, runDir, state, reference, (resolved) => - withBaselineSnapshot(runDir, reference.baseHash, (baselineRoot) => - withIsolatedKnowledgeCopy(baselineRoot, reference.baseHash, 'baseline', (baseline) => - withIsolatedKnowledgeCopy( - resolved.root, - reference.candidateHash, - 'candidate', - (candidate) => - use( - Object.freeze({ - reference, - evaluation: immutableJsonValue(structuredClone(resolved.evidence.evaluation)), - baseline: Object.freeze({ root: baseline, hash: reference.baseHash }), - candidate: Object.freeze({ root: candidate, hash: reference.candidateHash }), - }), - ), - ), - ), - ), - ) - }) -} - -/** Use the frozen candidate side of one measured comparison. */ -export async function withKnowledgeImprovementCandidate( - options: UseKnowledgeImprovementCandidateOptions, - use: (candidate: ResolvedKnowledgeImprovementCandidate) => Promise | T, -): Promise { - assertExactCandidatePlatform() - const candidateRef = Object.freeze( - KnowledgeImprovementCandidateRefSchema.parse(options.candidate), - ) - return withKnowledgeImprovementRun(options.root, candidateRef.runId, false, async (runDir) => { - const state = await loadKnowledgeImprovementStateFromRun( - options.root, - candidateRef.runId, - runDir, - ) - return withMeasuredCandidateSnapshot(options.root, runDir, state, candidateRef, (resolved) => - withIsolatedKnowledgeCopy(resolved.root, candidateRef.candidateHash, 'candidate', (root) => - use( - Object.freeze({ - root, - candidate: candidateRef, - evaluation: immutableJsonValue(structuredClone(resolved.evidence.evaluation)), - }), - ), - ), - ) - }) -} - -/** Promote one previously measured candidate without rerunning research or evaluation. */ -export async function promoteKnowledgeCandidate( - options: PromoteKnowledgeCandidateOptions, -): Promise { - return transitionKnowledgeCandidate(options, 'candidate') -} - -/** Restore the frozen baseline paired with one previously measured candidate. */ -export async function restoreKnowledgeCandidateBaseline( - options: RestoreKnowledgeCandidateBaselineOptions, -): Promise { - return transitionKnowledgeCandidate(options, 'baseline') -} - -async function transitionKnowledgeCandidate( - options: PromoteKnowledgeCandidateOptions, - target: KnowledgeImprovementTarget, -): Promise { - assertExactCandidatePlatform() - const candidateRef = Object.freeze( - KnowledgeImprovementCandidateRefSchema.parse(options.candidate), - ) - const now = options.now ?? (() => new Date()) - const activation = options.activation - ? resolveKnowledgeActivationPersistence(options.activation, candidateRef, target) - : undefined - return withKnowledgeImprovementRun(options.root, candidateRef.runId, false, async (runDir) => { - const lease = await acquireRunLease(runDir, { - ownerId: options.ownerId ?? `pid-${process.pid}`, - ttlMs: options.leaseTtlMs ?? DEFAULT_LEASE_TTL_MS, - }) - try { - lease.assertOwned() - const state = await loadKnowledgeImprovementStateFromRun( - options.root, - candidateRef.runId, - runDir, - ) - return await applyKnowledgeCandidateTarget( - { - root: options.root, - runDir, - state, - candidateRef, - leaseTtlMs: options.leaseTtlMs ?? DEFAULT_LEASE_TTL_MS, - assertRunOwned: lease.assertOwned, - now, - onState: options.onState, - activation, - }, - target, - ) - } finally { - await lease.release() - } - }) -} - -export async function improveKnowledgeBase( - options: KnowledgeImprovementOptions, -): Promise { - assertExactCandidatePlatform() - assertKnowledgeImprovementOptions(options) - const now = options.now ?? (() => new Date()) - const runId = runIdSchema.parse( - options.runId ?? knowledgeImprovementRunId(options.root, options.goal), - ) - return withKnowledgeImprovementRun(options.root, runId, true, (runDir) => - improveKnowledgeBaseInRun(options, runId, runDir, now), - ) -} - -async function improveKnowledgeBaseInRun( - options: KnowledgeImprovementOptions, - runId: string, - runDir: string, - now: () => Date, -): Promise { - const lease = await acquireRunLease(runDir, { - ownerId: options.ownerId ?? `pid-${process.pid}`, - ttlMs: options.leaseTtlMs ?? DEFAULT_LEASE_TTL_MS, - }) - - try { - lease.assertOwned() - let state = - options.resume === false - ? null - : await loadKnowledgeImprovementStateFromRun(options.root, runId, runDir).catch((error) => { - if (isMissingFile(error)) return null - throw error - }) - if (!state) { - const baseHash = await hashKnowledgeBase(options.root) - await createBaselineSnapshot(runDir, options.root, baseHash) - state = { - runId, - root: options.root, - goal: options.goal, - status: 'running', - baseHash, - createdAt: now().toISOString(), - updatedAt: now().toISOString(), - ownerId: lease.ownerId, - candidates: [], - } - await saveState(runDir, state, options.onState) - await appendLedger(runDir, { type: 'run.created', runId, baseHash }) - } - if (state.goal !== options.goal) { - throw new Error('knowledge improvement state does not match the requested goal') - } - const promotedCandidateId = state.promotedCandidateId - const promotedCandidate = - state.status === 'promoted' - ? state.candidates.find((candidate) => candidate.candidateId === promotedCandidateId) - : undefined - if (state.status === 'promoted' && !promotedCandidate) { - throw new Error('promoted knowledge state has no promoted candidate') - } - if (state.status === 'promoted') { - const promoted = promotedCandidate! - const promotedState = state - return withKnowledgeMutation(options.root, async () => { - const currentHash = await hashKnowledgeBase(options.root) - if (currentHash !== promoted.candidateHash) { - throw new Error( - `promoted knowledge base changed: expected ${promoted.candidateHash}, got ${currentHash}`, - ) - } - const evidence = await assertCandidateEvidence( - runDir, - candidateRefFor(runId, promotedState, promoted), - ) - return { - runId, - state: promotedState, - candidate: promoted, - evaluation: evidence.evaluation, - promoted: true, - blocked: false, - } - }) - } - await withKnowledgeMutation(options.root, () => undefined) - await ensureBaselineSnapshot(runDir, options.root, state.baseHash) - - if (state.status === 'blocked') { - return { runId, state, promoted: false, blocked: true } - } - - const maxCandidates = Math.max(1, options.maxCandidates ?? 1) - let candidate = findActiveCandidate(state) - let lastRejectedCandidate: KnowledgeImprovementCandidateRecord | undefined - let lastRejectedEvaluation: KnowledgeImprovementMetric | undefined - let lifecycle: RunRagKnowledgeImprovementLoopResult | undefined - - while (candidate || state.candidates.length < maxCandidates) { - if (!candidate) { - const currentHash = await hashKnowledgeBase(options.root) - if (currentHash !== state.baseHash) { - state = await blockRun( - runDir, - state, - `base changed before candidate creation: expected ${state.baseHash}, got ${currentHash}`, - options.onState, - now, - ) - return { runId, state, promoted: false, blocked: true } - } - const activeState = state - candidate = await withBaselineSnapshot(runDir, activeState.baseHash, (baselineRoot) => - createCandidateWorkspace(runDir, activeState, baselineRoot, now), - ) - state.candidates.push(candidate) - state.status = 'running' - state.updatedAt = now().toISOString() - await saveState(runDir, state, options.onState) - await appendLedger(runDir, { - type: 'candidate.created', - runId, - candidateId: candidate.candidateId, - iteration: candidate.iteration, - }) - } - - const measured = await measureCandidate(runId, runDir, state, candidate, options, now) - candidate = measured.candidate - const evaluation = measured.evaluation - lifecycle = measured.lifecycle - - if (evaluation.passed) { - candidate.status = 'candidate-ready' - candidate.updatedAt = now().toISOString() - state.status = 'candidate-ready' - state.updatedAt = now().toISOString() - await saveState(runDir, state, options.onState) - await appendLedger(runDir, { - type: 'candidate.ready', - runId, - candidateId: candidate.candidateId, - }) - break - } - - candidate.status = 'rejected' - state.status = 'running' - state.updatedAt = now().toISOString() - await saveState(runDir, state, options.onState) - await appendLedger(runDir, { - type: 'candidate.rejected', - runId, - candidateId: candidate.candidateId, - }) - lastRejectedCandidate = candidate - lastRejectedEvaluation = evaluation - candidate = undefined - } - - if (!candidate) { - state.status = 'rejected' - state.updatedAt = now().toISOString() - await saveState(runDir, state, options.onState) - return { - runId, - state, - candidate: lastRejectedCandidate, - ...(lastRejectedEvaluation ? { evaluation: lastRejectedEvaluation } : {}), - lifecycle, - promoted: false, - blocked: false, - } - } - - const evidence = await assertCandidateEvidence(runDir, candidateRefFor(runId, state, candidate)) - return { - runId, - state, - candidate, - evaluation: evidence.evaluation, - lifecycle, - promoted: false, - blocked: false, - } - } finally { - await lease.release() - } -} - -interface KnowledgeCandidateTransitionInput { - root: string - runDir: string - state: KnowledgeImprovementRunState - candidateRef: KnowledgeImprovementCandidateRef - activation?: ResolvedKnowledgeImprovementActivationPersistence - leaseTtlMs: number - assertRunOwned(): void - now: () => Date - onState?: KnowledgeImprovementOptions['onState'] - lifecycle?: RunRagKnowledgeImprovementLoopResult -} - -interface ResolvedKnowledgeImprovementActivationPersistence - extends KnowledgeImprovementActivationPersistence { - activation: AgentImprovementActivation -} - -async function applyKnowledgeCandidateTarget( - input: KnowledgeCandidateTransitionInput, - target: KnowledgeImprovementTarget, -): Promise { - const { candidateRef, runDir, state } = input - assertStateIdentity(input.root, candidateRef, state) - const candidate = state.candidates.find((entry) => entry.candidateId === candidateRef.candidateId) - if ( - !candidate || - canonicalJson(candidateIdentityFor(candidateRef.runId, state, candidate)) !== - canonicalJson(candidateRef) - ) { - throw new Error('knowledge candidate approval does not match the measured candidate') - } - - const action = target === 'candidate' ? 'promotion' : 'restore' - const desiredHash = target === 'candidate' ? candidateRef.candidateHash : candidateRef.baseHash - const purpose = knowledgeCandidateTransitionPurpose(candidateRef, target) - const recoveryOwner = input.activation - ? `knowledge-improvement-activation:${input.activation.activation.digest}` - : 'knowledge-improvement-candidate-transition' - return withKnowledgeMutation( - input.root, - async (mutationLock) => { - const assertOwned = () => { - mutationLock.assertOwned() - input.assertRunOwned() - } - assertOwned() - const transactionRoot = mutationLock.transactionRoot - const recovery = - mutationLock.recovery?.purpose === purpose ? mutationLock.recovery : undefined - const existingActivation = input.activation - ? await loadKnowledgeActivationRecord( - runDir, - candidateRef, - input.activation.activation, - target, - input.activation.identity, - ) - : null - if (existingActivation) { - if (recovery?.direction === 'rollback') { - throw new Error('stored knowledge activation result conflicts with a pending rollback') - } - if (recovery) { - const recoveredHash = await hashKnowledgeBase(input.root) - if (recoveredHash !== existingActivation.mutation.afterHash) { - throw new Error('stored knowledge activation result does not match the recovered files') - } - await finishKnowledgeFileTransaction({ - root: input.root, - transactionRoot, - transaction: recovery.transaction, - assertOwned, - }) - } - if (existingActivation.result.outcome.status === 'conflict') { - const reason = candidateTransitionConflictReason( - target, - candidateRef, - existingActivation.mutation.afterHash, - ) - const blocked = await blockCandidateTransition( - input, - candidate, - target, - reason, - existingActivation.mutation.afterHash, - ) - return { ...blocked, activationResult: existingActivation.result } - } - return candidateTransitionResult( - input, - candidate, - target === 'candidate' && state.status === 'promoted', - false, - existingActivation.mutation, - existingActivation.result, - ) - } - if (recovery?.direction === 'rollback') { - await finishKnowledgeFileTransaction({ - root: input.root, - transactionRoot, - transaction: recovery.transaction, - assertOwned, - }) - } - const recovered = recovery?.direction === 'apply' ? recovery : undefined - let pending: KnowledgeFileTransaction | null = - input.activation && recovered ? recovered.transaction : null - const currentHash = await hashKnowledgeBase(input.root) - let transactionId = recovered?.transactionId ?? null - if (recovered && currentHash !== desiredHash) { - throw new Error(`recovered knowledge ${action} does not match the approved target`) - } - if (state.status === 'promoted' && state.promotedCandidateId !== candidate.candidateId) { - throw new Error( - `knowledge run already promoted '${state.promotedCandidateId ?? 'unknown'}'`, - ) - } - if ( - target === 'candidate' && - state.status === 'promoted' && - currentHash !== candidateRef.candidateHash - ) { - throw new Error( - `promoted knowledge base changed: expected ${candidateRef.candidateHash}, got ${currentHash}`, - ) - } - if (state.status !== 'promoted' && state.status !== 'candidate-ready') { - throw new Error( - `knowledge candidate is not ready for ${action}: run=${state.status}, candidate=${candidate.status}`, - ) - } - if (currentHash !== state.baseHash && currentHash !== candidateRef.candidateHash) { - const reason = - target === 'candidate' - ? `base changed before promotion: expected ${state.baseHash}, got ${currentHash}` - : `knowledge changed before restore: expected ${candidateRef.candidateHash}, got ${currentHash}` - const mutation = Object.freeze({ - target, - beforeHash: currentHash, - afterHash: currentHash, - changed: false, - transactionId: null, - recovered: false, - }) satisfies KnowledgeImprovementMutationReceipt - const activationResult = input.activation - ? await persistKnowledgeActivationResult( - runDir, - candidateRef, - input.activation, - target, - mutation, - ) - : undefined - const blocked = await blockCandidateTransition( - input, - candidate, - target, - reason, - currentHash, - ) - return activationResult ? { ...blocked, activationResult } : blocked - } - - if (currentHash !== desiredHash && !pending) { - pending = await withMeasuredCandidateSnapshot( - input.root, - runDir, - state, - candidateRef, - (resolved) => - withBaselineSnapshot(runDir, state.baseHash, async (baselineRoot) => { - const sourceRoot = target === 'candidate' ? baselineRoot : resolved.root - const targetRoot = target === 'candidate' ? resolved.root : baselineRoot - const plan = await knowledgeFilePlanEntries(sourceRoot, targetRoot) - assertCandidateTransitionPlan(plan, candidateRef, target) - return prepareKnowledgeFileTransaction({ - root: input.root, - transactionRoot, - purpose, - recoveryOwner, - mutations: await knowledgePlanMutations(targetRoot, plan), - includeUnchanged: true, - now: input.now, - }) - }), - ) - if (!pending) { - throw new Error(`knowledge ${action} plan unexpectedly contained no file changes`) - } - transactionId = pending.transactionId - try { - assertCandidateTransitionTransaction(pending, candidateRef, target) - } catch (error) { - try { - await rollbackKnowledgeFileTransaction({ - root: input.root, - transactionRoot, - transaction: pending, - beforeCommit: assertOwned, - }) - await finishKnowledgeFileTransaction({ - root: input.root, - transactionRoot, - transaction: pending, - assertOwned, - }) - } catch (cleanupError) { - throw new AggregateError( - [error, cleanupError], - `invalid knowledge ${action} transaction could not be removed`, - ) - } - throw error - } - } - try { - if (pending) { - await applyKnowledgeFileTransaction({ - root: input.root, - transactionRoot, - transaction: pending, - beforeCommit: assertOwned, - }) - } - assertOwned() - if ((await hashKnowledgeBase(input.root)) !== desiredHash) { - throw new Error(`knowledge ${action} content does not match the approved target`) - } - await writeKnowledgeIndex(input.root) - } catch (error) { - if (!pending) throw error - if (recovered && input.activation) throw error - try { - assertOwned() - } catch (ownershipError) { - throw new AggregateError( - [error, ownershipError], - `knowledge ${action} lost its lock and left the transaction pending`, - ) - } - try { - await rollbackKnowledgeFileTransaction({ - root: input.root, - transactionRoot, - transaction: pending, - beforeCommit: assertOwned, - }) - await finishKnowledgeFileTransaction({ - root: input.root, - transactionRoot, - transaction: pending, - assertOwned, - }) - await writeKnowledgeIndex(input.root) - } catch (rollbackError) { - throw new AggregateError( - [error, rollbackError], - `knowledge ${action} failed and could not restore the previous files`, - ) - } - throw error - } - candidate.status = target === 'candidate' ? 'promoted' : 'candidate-ready' - candidate.updatedAt = input.now().toISOString() - state.status = candidate.status - if (target === 'candidate') state.promotedCandidateId = candidate.candidateId - else delete state.promotedCandidateId - delete state.blockedReason - state.updatedAt = input.now().toISOString() - await saveState(runDir, state, input.onState) - await ensureCandidateTransitionEvent(runDir, candidateRef, target) - let finalHash = await hashKnowledgeBase(input.root) - if (finalHash !== desiredHash) { - throw new Error(`knowledge ${action} changed before its result was returned`) - } - const mutation = Object.freeze({ - target, - beforeHash: transactionId ? sourceKnowledgeHash(candidateRef, target) : currentHash, - afterHash: finalHash, - changed: transactionId !== null, - transactionId, - recovered: recovered !== undefined, - }) satisfies KnowledgeImprovementMutationReceipt - const activationResult = input.activation - ? await persistKnowledgeActivationResult( - runDir, - candidateRef, - input.activation, - target, - mutation, - ) - : undefined - if ((await hashKnowledgeBase(input.root)) !== finalHash) { - throw new Error(`knowledge ${action} changed while its result was persisted`) - } - if (pending) { - await finishKnowledgeFileTransaction({ - root: input.root, - transactionRoot, - transaction: pending, - assertOwned, - }) - } - finalHash = await hashKnowledgeBase(input.root) - if (finalHash !== desiredHash) { - throw new Error(`knowledge ${action} changed before its result was returned`) - } - return candidateTransitionResult( - input, - candidate, - target === 'candidate', - false, - mutation, - activationResult, - ) - }, - { - staleMs: input.leaseTtlMs, - resumeTransaction: { - purpose, - recoveryOwner, - validate: (transaction) => - assertCandidateTransitionTransaction(transaction, candidateRef, target), - deferFinish: input.activation !== undefined, - }, - }, - ) -} - -function candidateTransitionConflictReason( - target: KnowledgeImprovementTarget, - candidate: KnowledgeImprovementCandidateRef, - currentHash: string, -): string { - return target === 'candidate' - ? `base changed before promotion: expected ${candidate.baseHash}, got ${currentHash}` - : `knowledge changed before restore: expected ${candidate.candidateHash}, got ${currentHash}` -} - -async function blockCandidateTransition( - input: KnowledgeCandidateTransitionInput, - candidate: KnowledgeImprovementCandidateRecord, - target: KnowledgeImprovementTarget, - reason: string, - currentHash: string, -): Promise { - if (input.state.status === 'promoted') { - candidate.status = 'blocked' - candidate.updatedAt = input.now().toISOString() - delete input.state.promotedCandidateId - } - await blockRun(input.runDir, input.state, reason, input.onState, input.now) - await appendLedger(input.runDir, { - type: target === 'candidate' ? 'promotion.blocked' : 'restore.blocked', - runId: input.candidateRef.runId, - candidateId: candidate.candidateId, - reason, - }) - return candidateTransitionResult(input, candidate, false, true, { - target, - beforeHash: currentHash, - afterHash: currentHash, - changed: false, - transactionId: null, - recovered: false, - }) -} - -function candidateTransitionResult( - input: KnowledgeCandidateTransitionInput, - candidate: KnowledgeImprovementCandidateRecord, - promoted: boolean, - blocked: boolean, - mutation: KnowledgeImprovementMutationReceipt, - activationResult?: AgentImprovementActivationResult, -): KnowledgeImprovementMutationResult { - return { - runId: input.candidateRef.runId, - state: input.state, - candidate, - ...(input.lifecycle ? { lifecycle: input.lifecycle } : {}), - mutation, - ...(activationResult ? { activationResult } : {}), - promoted, - blocked, - } -} - -function sourceKnowledgeHash( - candidate: KnowledgeImprovementCandidateRef, - target: KnowledgeImprovementTarget, -): string { - return target === 'candidate' ? candidate.baseHash : candidate.candidateHash -} - -function targetForKnowledgeActivation( - activation: AgentImprovementActivation, -): KnowledgeImprovementTarget { - return activation.intent === 'activate-candidate' ? 'candidate' : 'baseline' -} - -function resolveKnowledgeActivationPersistence( - input: KnowledgeImprovementActivationPersistence, - candidate: KnowledgeImprovementCandidateRef, - target: KnowledgeImprovementTarget, -): ResolvedKnowledgeImprovementActivationPersistence { - const activation = verifyCanonicalKnowledgeActivation(input.activation) - assertKnowledgeActivationAuthority(activation, candidate, target, input.identity) - const attemptedAt = z.iso.datetime().parse(input.attemptedAt) - if ( - Date.parse(attemptedAt) < Date.parse(activation.authorizedAt) || - Date.parse(attemptedAt) >= Date.parse(activation.expiresAt) - ) { - throw new Error('knowledge activation attempt is outside its authorization window') - } - return Object.freeze({ - activation, - attemptedAt, - identity: input.identity, - createResult: input.createResult, - }) -} - -function assertKnowledgeActivationAuthority( - activation: AgentImprovementActivation, - candidate: KnowledgeImprovementCandidateRef, - target: KnowledgeImprovementTarget, - identity: string, -): void { - if (!identity.trim()) throw new Error('knowledge activation identity is required') - if (targetForKnowledgeActivation(activation) !== target) { - throw new Error('knowledge activation intent does not match the requested transition') - } - if (activation.targets.length !== 1) { - throw new Error('knowledge activation requires exactly one target') - } - const authorizedTarget = activation.targets[0] - if (authorizedTarget.surface !== 'knowledge' || authorizedTarget.identity !== identity) { - throw new Error('knowledge activation target does not match this knowledge base') - } - if ( - authorizedTarget.expectedBaseDigest !== - prefixedKnowledgeDigest(sourceKnowledgeHash(candidate, target)) - ) { - throw new Error('knowledge activation does not authorize the measured source state') - } -} - -function verifyCanonicalKnowledgeActivation( - value: AgentImprovementActivation, -): AgentImprovementActivation { - const activation = agentImprovementActivationSchema.parse(value) - if (canonicalCandidateDigest(omitTopLevelDigest(activation)) !== activation.digest) { - throw new Error('knowledge activation digest does not match its canonical content') - } - return immutableJsonValue(structuredClone(activation)) -} - -function verifyCanonicalKnowledgeActivationResult( - value: AgentImprovementActivationResult, -): AgentImprovementActivationResult { - const result = agentImprovementActivationResultSchema.parse(value) - if (canonicalCandidateDigest(omitTopLevelDigest(result)) !== result.digest) { - throw new Error('knowledge activation result digest does not match its canonical content') - } - return immutableJsonValue(structuredClone(result)) -} - -async function persistKnowledgeActivationResult( - runDir: string, - candidate: KnowledgeImprovementCandidateRef, - persistence: ResolvedKnowledgeImprovementActivationPersistence, - target: KnowledgeImprovementTarget, - mutation: KnowledgeImprovementMutationReceipt, -): Promise { - const result = assertKnowledgeActivationResult( - persistence.activation, - candidate, - target, - persistence.identity, - mutation, - await persistence.createResult(Object.freeze({ ...mutation })), - persistence.attemptedAt, - ) - const record = knowledgeImprovementActivationRecordSchema.parse({ - kind: 'knowledge-improvement-activation-result', - candidateId: candidate.candidateId, - mutation, - result, - }) - const existing = await loadKnowledgeActivationRecord( - runDir, - candidate, - persistence.activation, - target, - persistence.identity, - ) - if (existing) { - if (canonicalJson(existing) !== canonicalJson(record)) { - throw new Error('knowledge activation result identity conflicts with durable content') - } - return existing.result - } - await writeJsonDurableWithinRoot( - runDir, - knowledgeActivationResultPath(persistence.activation.digest), - record, - ) - const stored = await loadKnowledgeActivationRecord( - runDir, - candidate, - persistence.activation, - target, - persistence.identity, - ) - if (!stored || canonicalJson(stored) !== canonicalJson(record)) { - throw new Error('knowledge activation result was not durably persisted') - } - return stored.result -} - -async function loadKnowledgeActivationRecord( - runDir: string, - candidate: KnowledgeImprovementCandidateRef, - activation: AgentImprovementActivation, - target: KnowledgeImprovementTarget, - identity: string, -): Promise { - let raw: unknown - try { - const file = await readRegularFileWithinRoot( - runDir, - knowledgeActivationResultPath(activation.digest), - ) - raw = JSON.parse(file.bytes.toString('utf8')) as unknown - } catch (error) { - if (isMissingFile(error)) return null - throw error - } - const record = knowledgeImprovementActivationRecordSchema.parse(raw) - if (record.candidateId !== candidate.candidateId) { - throw new Error('knowledge activation result belongs to another candidate') - } - const result = assertKnowledgeActivationResult( - activation, - candidate, - target, - identity, - record.mutation, - record.result, - ) - return immutableJsonValue({ ...record, result }) -} - -function assertKnowledgeActivationResult( - activation: AgentImprovementActivation, - candidate: KnowledgeImprovementCandidateRef, - target: KnowledgeImprovementTarget, - identity: string, - mutationInput: KnowledgeImprovementMutationReceipt, - resultInput: AgentImprovementActivationResult, - attemptedAt?: string, -): AgentImprovementActivationResult { - assertKnowledgeActivationAuthority(activation, candidate, target, identity) - const mutation = knowledgeImprovementMutationReceiptSchema.parse(mutationInput) - const result = verifyCanonicalKnowledgeActivationResult(resultInput) - if ( - result.idempotencyKey !== activation.digest || - (attemptedAt !== undefined && result.attemptedAt !== attemptedAt) || - Date.parse(result.attemptedAt) < Date.parse(activation.authorizedAt) || - Date.parse(result.attemptedAt) >= Date.parse(activation.expiresAt) || - mutation.target !== target || - mutation.changed !== (mutation.beforeHash !== mutation.afterHash) || - (!mutation.changed && (mutation.transactionId !== null || mutation.recovered)) - ) { - throw new Error('knowledge activation result does not bind its authorized mutation') - } - - const sourceDigest = prefixedKnowledgeDigest(sourceKnowledgeHash(candidate, target)) - const desiredDigest = prefixedKnowledgeDigest( - target === 'candidate' ? candidate.candidateHash : candidate.baseHash, - ) - const beforeDigest = prefixedKnowledgeDigest(mutation.beforeHash) - const afterDigest = prefixedKnowledgeDigest(mutation.afterHash) - const outcome = result.outcome - if (mutation.changed) { - if ( - mutation.transactionId === null || - beforeDigest !== sourceDigest || - afterDigest !== desiredDigest || - outcome.status !== 'applied' || - outcome.transactionId !== mutation.transactionId || - outcome.targets.length !== 1 || - outcome.targets[0]?.surface !== 'knowledge' || - outcome.targets[0]?.identity !== identity || - outcome.targets[0]?.beforeDigest !== beforeDigest || - outcome.targets[0]?.afterDigest !== afterDigest - ) { - throw new Error('knowledge activation result does not prove the applied transaction') - } - return result - } - - const expectedStatus = afterDigest === desiredDigest ? 'already-applied' : 'conflict' - if ( - afterDigest === sourceDigest || - outcome.status !== expectedStatus || - outcome.targets.length !== 1 || - outcome.targets[0]?.surface !== 'knowledge' || - outcome.targets[0]?.identity !== identity || - outcome.targets[0]?.currentDigest !== afterDigest - ) { - throw new Error('knowledge activation result does not prove the observed target state') - } - return result -} - -function knowledgeActivationResultPath(digest: Sha256Digest): string { - const parsed = sha256DigestSchema.parse(digest) - return `activation-results/${parsed.slice('sha256:'.length)}.json` -} - -function prefixedKnowledgeDigest(hash: string): Sha256Digest { - return sha256DigestSchema.parse(`sha256:${digestSchema.parse(hash)}`) -} - -function immutableJsonValue(value: T): T { - if (value === null || typeof value !== 'object') return value - for (const child of Object.values(value)) immutableJsonValue(child) - return Object.freeze(value) -} - -function candidateRefFor( - runId: string, - state: KnowledgeImprovementRunState, - candidate: KnowledgeImprovementCandidateRecord, -): KnowledgeImprovementCandidateRef { - if (candidate.status !== 'candidate-ready' && candidate.status !== 'promoted') { - throw new Error(`knowledge candidate '${candidate.candidateId}' is not ready`) - } - return candidateIdentityFor(runId, state, candidate) -} - -function candidateIdentityFor( - runId: string, - state: KnowledgeImprovementRunState, - candidate: KnowledgeImprovementCandidateRecord, -): KnowledgeImprovementCandidateRef { - if (!candidate.candidateHash) { - throw new Error(`knowledge candidate '${candidate.candidateId}' has no content hash`) - } - if (!candidate.evidenceHash) { - throw new Error(`knowledge candidate '${candidate.candidateId}' has no evidence hash`) - } - if (!candidate.promotionPlanHash) { - throw new Error(`knowledge candidate '${candidate.candidateId}' has no promotion plan hash`) - } - return Object.freeze({ - kind: 'knowledge-improvement-candidate', - runId, - candidateId: candidate.candidateId, - goalHash: sha256(state.goal), - baseHash: candidate.baseHash, - candidateHash: candidate.candidateHash, - evidenceHash: candidate.evidenceHash, - promotionPlanHash: candidate.promotionPlanHash, - }) -} - -async function withMeasuredCandidateSnapshot( - liveRoot: string, - runDir: string, - state: KnowledgeImprovementRunState, - candidateRef: KnowledgeImprovementCandidateRef, - use: (snapshot: { - root: string - candidate: KnowledgeImprovementCandidateRecord - evidence: KnowledgeImprovementEvidence - }) => Promise | T, -): Promise { - assertStateIdentity(liveRoot, candidateRef, state) - const candidate = state.candidates.find((entry) => entry.candidateId === candidateRef.candidateId) - if (!candidate) { - throw new Error(`knowledge candidate '${candidateRef.candidateId}' does not exist`) - } - const expectedRef = candidateRefFor(candidateRef.runId, state, candidate) - if (canonicalJson(expectedRef) !== canonicalJson(candidateRef)) { - throw new Error('knowledge candidate approval does not match the measured candidate') - } - const evidence = await assertCandidateEvidence(runDir, candidateRef) - const relativePath = join( - 'candidates', - candidate.candidateId, - 'snapshots', - candidateRef.candidateHash, - ) - return withSafeDirectory(runDir, relativePath, false, async (root) => { - if ((await hashKnowledgeBase(root)) !== candidateRef.candidateHash) { - throw new Error('knowledge candidate snapshot changed after approval') - } - const result = await use({ root, candidate, evidence }) - if ((await hashKnowledgeBase(root)) !== candidateRef.candidateHash) { - throw new Error('knowledge candidate snapshot changed during use') - } - return result - }) -} - -async function withIsolatedKnowledgeCopy( - sourceRoot: string, - expectedHash: string, - target: KnowledgeImprovementTarget, - use: (root: string) => Promise | T, -): Promise { - const isolationRoot = await mkdtemp(join(tmpdir(), 'agent-knowledge-snapshot-')) - const snapshotRoot = join(isolationRoot, 'snapshot') - try { - await copyKnowledgeWorkspace(sourceRoot, snapshotRoot) - if ((await hashKnowledgeBase(snapshotRoot)) !== expectedHash) { - throw new Error(`isolated knowledge ${target} does not match its measured content`) - } - const result = await use(snapshotRoot) - if ((await hashKnowledgeBase(snapshotRoot)) !== expectedHash) { - throw new Error(`knowledge ${target} snapshot changed during use`) - } - return result - } finally { - await rm(isolationRoot, { recursive: true, force: true }) - } -} - -async function assertCandidateEvidence( - runDir: string, - candidate: KnowledgeImprovementCandidateRef, -): Promise { - const evidence = KnowledgeImprovementEvidenceSchema.parse( - JSON.parse( - ( - await readRegularFileWithinRoot( - runDir, - candidateEvidenceRelativePath(candidate.candidateId), - ) - ).bytes.toString('utf8'), - ), - ) - const actualHash = contentHash(evidence) - if (actualHash !== candidate.evidenceHash) { - throw new Error( - `knowledge candidate evidence changed after approval: expected ${candidate.evidenceHash}, got ${actualHash}`, - ) - } - if ( - evidence.runId !== candidate.runId || - evidence.candidateId !== candidate.candidateId || - evidence.goalHash !== candidate.goalHash || - evidence.baseHash !== candidate.baseHash || - evidence.candidateHash !== candidate.candidateHash || - evidence.promotionPlanHash !== candidate.promotionPlanHash || - evidence.evaluation.passed !== true - ) { - throw new Error('knowledge candidate evidence does not match the approved candidate') - } - return evidence -} - -function assertStateIdentity( - root: string, - candidateRef: KnowledgeImprovementCandidateRef, - state: KnowledgeImprovementRunState, -): void { - if (state.runId !== candidateRef.runId) { - throw new Error('knowledge candidate run identity does not match persisted state') - } - if (resolve(state.root) !== resolve(root)) { - throw new Error('knowledge candidate root does not match persisted state') - } - if (sha256(state.goal) !== candidateRef.goalHash) { - throw new Error('knowledge candidate goal does not match persisted state') - } - if (state.baseHash !== candidateRef.baseHash) { - throw new Error('knowledge candidate base does not match persisted state') - } -} - -async function assertCandidateWorkspace( - runDir: string, - candidate: Pick, -): Promise { - await withCandidateWorkspace(runDir, candidate, () => undefined) -} - -async function withCandidateWorkspace( - runDir: string, - candidate: Pick, - use: (candidateRoot: string) => Promise | T, -): Promise { - return withSafeDirectory( - runDir, - join('candidates', safePathSegmentSchema.parse(candidate.candidateId), 'workspace'), - false, - use, - ) -} - -function assertKnowledgeImprovementOptions(options: KnowledgeImprovementOptions): void { - if (options.step && options.knowledgeResearch?.step) { - throw new Error('improveKnowledgeBase accepts either step or knowledgeResearch.step, not both') - } - const updateDrivers = [ - Boolean(options.step ?? options.knowledgeResearch), - Boolean(options.updateKnowledge), - ].filter(Boolean).length - if (updateDrivers > 1) { - throw new Error( - 'improveKnowledgeBase accepts only one knowledge-update driver: knowledgeResearch or updateKnowledge', - ) - } -} - -async function measureCandidate( - runId: string, - runDir: string, - state: KnowledgeImprovementRunState, - candidate: KnowledgeImprovementCandidateRecord, - options: KnowledgeImprovementOptions, - now: () => Date, -): Promise<{ - candidate: KnowledgeImprovementCandidateRecord - evaluation: KnowledgeImprovementMetric - lifecycle?: RunRagKnowledgeImprovementLoopResult -}> { - return withCandidateWorkspace(runDir, candidate, async (candidateRoot) => { - const currentCandidateHash = await hashKnowledgeBase(candidateRoot) - if ( - candidate.status === 'candidate-ready' && - candidate.candidateHash === currentCandidateHash && - candidate.evidenceHash !== undefined && - candidate.promotionPlanHash !== undefined - ) { - const evidence = await assertCandidateEvidence( - runDir, - candidateRefFor(runId, state, candidate), - ) - return { candidate, evaluation: evidence.evaluation } - } - - clearCandidateMeasurement(candidate) - const lifecycles: RunRagKnowledgeImprovementLoopResult[] = [] - if (candidate.status === 'running') { - const updateLifecycle = await runCandidateUpdateLifecycle( - runId, - candidate, - candidateRoot, - options, - now, - ) - if (updateLifecycle) lifecycles.push(updateLifecycle) - } - return withFrozenCandidateWorkspace(runDir, candidate, candidateRoot, async (snapshot) => { - const evaluationLifecycle = await runCandidateEvaluationLifecycle( - runDir, - candidate, - snapshot.root, - options, - now, - ) - if (evaluationLifecycle) lifecycles.push(evaluationLifecycle) - const lifecycle = mergeLifecycleResults(options.goal, lifecycles) - const measured = await evaluateCandidate( - runDir, - state, - candidate, - snapshot, - lifecycle, - options, - now, - ) - return { ...measured, ...(lifecycle ? { lifecycle } : {}) } - }) - }) -} - -async function runCandidateUpdateLifecycle( - runId: string, - candidate: KnowledgeImprovementCandidateRecord, - candidateRoot: string, - options: KnowledgeImprovementOptions, - now: () => Date, -): Promise { - if (!shouldRunUpdateStage(options)) return undefined - const lifecycle = await runRagKnowledgeImprovementLoop({ - goal: options.goal, - acquireKnowledge: options.acquireKnowledge, - knowledgeResearch: candidateKnowledgeResearchOptions(candidateRoot, options), - updateKnowledge: candidateUpdateHook(runId, candidate, candidateRoot, options), - enabledPhases: selectedStagePhases(options, UPDATE_PHASES), - requiredPhases: selectedStageRequiredPhases(options, UPDATE_PHASES), - signal: options.signal, - now, - }) - return lifecycle -} - -async function runCandidateEvaluationLifecycle( - runDir: string, - candidate: KnowledgeImprovementCandidateRecord, - candidateRoot: string, - options: KnowledgeImprovementOptions, - now: () => Date, -): Promise { - if (!shouldRunEvaluationStage(options)) return undefined - const candidateIndex = await buildKnowledgeIndex(candidateRoot) - const lifecycle = await runRagKnowledgeImprovementLoop({ - goal: options.goal, - retrieval: options.retrieval - ? { - ...options.retrieval, - index: candidateIndex, - runDir: options.retrieval.runDir ?? join(runDir, 'retrieval', candidate.candidateId), - } - : undefined, - diagnose: options.diagnose, - evaluateAnswers: options.evaluateAnswers, - promote: options.decidePromotion, - enabledPhases: selectedStagePhases(options, EVALUATION_PHASES), - requiredPhases: selectedStageRequiredPhases(options, EVALUATION_PHASES), - signal: options.signal, - now, - }) - return lifecycle -} - -function candidateKnowledgeResearchOptions( - candidateRoot: string, - options: KnowledgeImprovementOptions, -): RagKnowledgeResearchOptions | undefined { - if (!options.step && !options.knowledgeResearch) return undefined - const { step: researchStep, ...rest } = options.knowledgeResearch ?? {} - const step = options.step ?? researchStep - return { - ...rest, - root: candidateRoot, - step, - maxIterations: - rest.maxIterations ?? (step ? (options.candidateResearchIterations ?? 3) : undefined), - strict: rest.strict ?? options.strict, - readinessSpecs: rest.readinessSpecs ?? options.readinessSpecs, - readinessTaskId: rest.readinessTaskId ?? options.readinessTaskId, - readiness: rest.readiness ?? options.readiness, - } -} - -function candidateUpdateHook( - runId: string, - candidate: KnowledgeImprovementCandidateRecord, - candidateRoot: string, - options: KnowledgeImprovementOptions, -): RunRagKnowledgeImprovementLoopOptions['updateKnowledge'] { - if (!options.updateKnowledge) return undefined - return (input) => - options.updateKnowledge!({ - ...input, - runId, - iteration: candidate.iteration, - candidateId: candidate.candidateId, - root: candidateRoot, - baselineRoot: options.root, - candidateRoot, - baseHash: candidate.baseHash, - }) -} - -function shouldRunUpdateStage(options: KnowledgeImprovementOptions): boolean { - const phases = selectedStagePhases(options, UPDATE_PHASES) - if (phases.length === 0) return false - return Boolean( - options.acquireKnowledge || - options.step || - options.knowledgeResearch || - options.updateKnowledge || - selectedStageRequiredPhases(options, UPDATE_PHASES).length > 0, - ) -} - -function shouldRunEvaluationStage(options: KnowledgeImprovementOptions): boolean { - const phases = selectedStagePhases(options, EVALUATION_PHASES) - if (phases.length === 0) return false - return Boolean( - options.retrieval || - options.diagnose || - options.evaluateAnswers || - options.decidePromotion || - selectedStageRequiredPhases(options, EVALUATION_PHASES).length > 0, - ) -} - -function selectedStagePhases( - options: Pick, - stagePhases: readonly RagKnowledgeImprovementPhase[], -): RagKnowledgeImprovementPhase[] { - const requested = options.enabledPhases ?? stagePhases - return requested.filter((phase) => stagePhases.includes(phase)) -} - -function selectedStageRequiredPhases( - options: Pick, - stagePhases: readonly RagKnowledgeImprovementPhase[], -): RagKnowledgeImprovementPhase[] { - return (options.requiredPhases ?? []).filter((phase) => stagePhases.includes(phase)) -} - -function mergeLifecycleResults( - goal: string, - lifecycles: readonly RunRagKnowledgeImprovementLoopResult[], -): RunRagKnowledgeImprovementLoopResult | undefined { - if (lifecycles.length === 0) return undefined - return { - goal, - phases: lifecycles.flatMap((lifecycle) => lifecycle.phases), - retrieval: lastDefined(lifecycles.map((lifecycle) => lifecycle.retrieval)), - findings: lifecycles.flatMap((lifecycle) => lifecycle.findings), - acquisition: lastDefined(lifecycles.map((lifecycle) => lifecycle.acquisition)), - knowledgeUpdate: lastDefined(lifecycles.map((lifecycle) => lifecycle.knowledgeUpdate)), - answerQuality: lastDefined(lifecycles.map((lifecycle) => lifecycle.answerQuality)), - promotion: lastDefined(lifecycles.map((lifecycle) => lifecycle.promotion)), - } -} - -function lastDefined(values: readonly (T | undefined)[]): T | undefined { - for (let index = values.length - 1; index >= 0; index -= 1) { - if (values[index] !== undefined) return values[index] - } - return undefined -} - -async function evaluateCandidate( - runDir: string, - state: KnowledgeImprovementRunState, - candidate: KnowledgeImprovementCandidateRecord, - snapshot: { root: string; hash: string }, - lifecycle: RunRagKnowledgeImprovementLoopResult | undefined, - options: KnowledgeImprovementOptions, - now: () => Date, -): Promise<{ - candidate: KnowledgeImprovementCandidateRecord - evaluation: KnowledgeImprovementMetric -}> { - return withBaselineSnapshot(runDir, state.baseHash, async (baselineRoot) => { - const [baselineIndex, candidateIndex] = await Promise.all([ - buildKnowledgeIndex(baselineRoot), - buildKnowledgeIndex(snapshot.root), - ]) - const validation = validateKnowledgeIndex(candidateIndex, { strict: options.strict }) - const readiness = readinessFor(options, candidateIndex) - const kbQuality = scoreKnowledgeBaseIndex(candidateIndex, { - strict: options.strict, - ...options.kbQuality, - }) - const candidateHash = snapshot.hash - const metric = - options.evaluate?.({ - runId: state.runId, - iteration: candidate.iteration, - root: options.root, - baselineRoot, - candidateRoot: snapshot.root, - baselineIndex, - candidateIndex, - baseHash: state.baseHash, - candidateHash, - validation, - readiness, - kbQuality, - lifecycle, - signal: options.signal, - }) ?? - defaultKnowledgeImprovementMetric( - validation, - readiness, - options.readinessSpecs, - kbQuality, - lifecycle, - ) - const evaluation = applyLifecycleFailures(normalizeMetric(await metric), lifecycle) - const measuredHash = await hashKnowledgeBase(snapshot.root) - if (measuredHash !== candidateHash) { - throw new Error( - `knowledge candidate changed during evaluation: expected ${candidateHash}, got ${measuredHash}`, - ) - } - candidate.candidateHash = candidateHash - candidate.promotionPlanHash = knowledgeFileTransactionPlanHash( - await knowledgeFilePlanEntries(baselineRoot, snapshot.root), - ) - const evidence = KnowledgeImprovementEvidenceSchema.parse( - JSON.parse( - JSON.stringify({ - kind: 'knowledge-improvement-evidence', - runId: state.runId, - candidateId: candidate.candidateId, - iteration: candidate.iteration, - goalHash: sha256(state.goal), - baseHash: candidate.baseHash, - candidateHash, - promotionPlanHash: candidate.promotionPlanHash, - validation, - readiness: readiness ?? null, - kbQuality, - evaluation, - lifecycle: lifecycle ?? null, - }), - ), - ) - candidate.evidenceHash = contentHash(evidence) - candidate.updatedAt = now().toISOString() - await writeJsonDurableWithinRoot( - runDir, - candidateEvidenceRelativePath(candidate.candidateId), - evidence, - ) - await appendLedger(runDir, { - type: 'candidate.evaluated', - runId: state.runId, - candidateId: candidate.candidateId, - score: evaluation.score, - passed: evaluation.passed, - }) - return { candidate, evaluation } - }) -} - -function defaultKnowledgeImprovementMetric( - validation: ValidateKnowledgeResult, - readiness: EvalKnowledgeBundleBuildResult | undefined, - readinessSpecs: readonly KnowledgeReadinessSpec[] | undefined, - kbQuality: KnowledgeBaseQualityReport, - lifecycle: RunRagKnowledgeImprovementLoopResult | undefined, -): KnowledgeImprovementMetric { - const blockingMissing = readiness?.report.blockingMissingRequirements.length ?? 0 - const blockingTotal = readinessSpecs?.filter((spec) => spec.importance === 'blocking').length ?? 0 - const blockingReadiness = - blockingTotal === 0 ? 1 : Math.max(0, blockingTotal - blockingMissing) / blockingTotal - const answerQuality = lifecycle?.answerQuality - ? average(Object.values(lifecycle.answerQuality.metrics).filter(Number.isFinite)) - : 1 - const promotionDecision = lifecycle?.promotion ? (lifecycle.promotion.promoted ? 1 : 0) : 1 - const dimensions = { - validation: validation.ok ? 1 : 0, - kb_quality: kbQuality.ok ? 1 : 0, - blocking_readiness: blockingReadiness, - answer_quality: answerQuality, - promotion_decision: promotionDecision, - } - const failedReasons = [ - validation.ok ? undefined : 'candidate validation failed', - kbQuality.ok ? undefined : 'candidate KB quality check failed', - blockingMissing === 0 - ? undefined - : `${blockingMissing}/${blockingTotal} blocking knowledge requirements still missing`, - ].filter((reason): reason is string => Boolean(reason)) - return { - score: average(Object.values(dimensions)), - passed: failedReasons.length === 0, - dimensions, - notes: - failedReasons.length === 0 ? 'candidate passed configured checks' : failedReasons.join('; '), - provenance: { - evaluator: '@tangle-network/agent-knowledge/default-knowledge-improvement-metric', - version: '1', - method: 'deterministic', - }, - } -} - -function applyLifecycleFailures( - metric: KnowledgeImprovementMetric, - lifecycle: RunRagKnowledgeImprovementLoopResult | undefined, -): KnowledgeImprovementMetric { - const reasons = [ - metric.notes, - lifecycle?.answerQuality && !lifecycle.answerQuality.passed - ? 'answer quality failed' - : undefined, - lifecycle?.promotion && !lifecycle.promotion.promoted - ? `promotion decision held: ${lifecycle.promotion.reason}` - : undefined, - ].filter((reason): reason is string => Boolean(reason)) - const forcedFailure = - Boolean(lifecycle?.answerQuality && !lifecycle.answerQuality.passed) || - Boolean(lifecycle?.promotion && !lifecycle.promotion.promoted) - return { - ...metric, - passed: metric.passed && !forcedFailure, - notes: reasons.length > 0 ? reasons.join('; ') : metric.notes, - } -} - -function normalizeMetric(metric: KnowledgeImprovementMetric): KnowledgeImprovementMetric { - return improvementMetricSchema.parse(metric) -} - -async function createCandidateWorkspace( - runDir: string, - state: KnowledgeImprovementRunState, - root: string, - now: () => Date, -): Promise { - const iteration = state.candidates.length + 1 - const candidateId = stableId('kcand', `${state.runId}:${iteration}:${now().toISOString()}`) - const candidateRoot = candidateWorkspacePath(runDir, candidateId) - await copyKnowledgeWorkspace(root, candidateRoot) - const createdAt = now().toISOString() - return { - iteration, - candidateId, - baseHash: state.baseHash, - status: 'running', - createdAt, - updatedAt: createdAt, - } -} - -function candidateWorkspacePath(runDir: string, candidateId: string): string { - return join(runDir, 'candidates', safePathSegmentSchema.parse(candidateId), 'workspace') -} - -function baselineSnapshotPath(runDir: string): string { - return join(runDir, 'baseline') -} - -async function createBaselineSnapshot( - runDir: string, - root: string, - expectedHash: string, -): Promise { - const target = baselineSnapshotPath(runDir) - try { - await assertBaselineSnapshot(runDir, expectedHash) - return - } catch (error) { - if (!isMissingFile(error)) throw error - } - const preparation = await mkdtemp(join(runDir, 'baseline-prepare-')) - let activated = false - try { - await copyKnowledgeWorkspace(root, preparation) - const actualHash = await hashKnowledgeBase(preparation) - if (actualHash !== expectedHash) { - throw new Error( - `knowledge base changed while baseline was frozen: expected ${expectedHash}, got ${actualHash}`, - ) - } - await renameDurable(preparation, target) - activated = true - } finally { - if (!activated) await rm(preparation, { recursive: true, force: true }) - } -} - -async function ensureBaselineSnapshot( - runDir: string, - root: string, - expectedHash: string, -): Promise { - try { - await assertBaselineSnapshot(runDir, expectedHash) - } catch (error) { - if (!isMissingFile(error)) throw error - const liveHash = await hashKnowledgeBase(root) - if (liveHash !== expectedHash) { - throw new Error( - 'knowledge improvement baseline snapshot is missing and cannot be reconstructed', - ) - } - await createBaselineSnapshot(runDir, root, expectedHash) - } -} - -async function assertBaselineSnapshot(runDir: string, expectedHash: string): Promise { - await withBaselineSnapshot(runDir, expectedHash, () => undefined) -} - -async function withBaselineSnapshot( - runDir: string, - expectedHash: string, - use: (baselineRoot: string) => Promise | T, -): Promise { - return withSafeDirectory(runDir, 'baseline', false, async (baselineRoot) => { - const actualHash = await hashKnowledgeBase(baselineRoot) - if (actualHash !== expectedHash) { - throw new Error( - `knowledge improvement baseline changed: expected ${expectedHash}, got ${actualHash}`, - ) - } - return use(baselineRoot) - }) -} - -async function withFrozenCandidateWorkspace( - runDir: string, - candidate: KnowledgeImprovementCandidateRecord, - candidateRoot: string, - use: (snapshot: { root: string; hash: string }) => Promise | T, -): Promise { - const snapshotsPath = join( - 'candidates', - safePathSegmentSchema.parse(candidate.candidateId), - 'snapshots', - ) - return withSafeDirectory(runDir, snapshotsPath, true, async (snapshotsDir) => { - const preparation = await mkdtemp(join(snapshotsDir, 'prepare-')) - let activated = false - try { - await copyKnowledgeWorkspace(candidateRoot, preparation) - const hash = await hashKnowledgeBase(preparation) - try { - const result = await withSafeDirectory(snapshotsDir, hash, false, async (existing) => { - if ((await hashKnowledgeBase(existing)) !== hash) { - throw new Error('knowledge candidate snapshot does not match its content identity') - } - return use({ root: existing, hash }) - }) - await rm(preparation, { recursive: true, force: true }) - activated = true - return result - } catch (error) { - if (!isMissingFile(error)) throw error - } - await renameDurable(preparation, join(snapshotsDir, hash)) - activated = true - return withSafeDirectory(snapshotsDir, hash, false, (root) => use({ root, hash })) - } finally { - if (!activated) await rm(preparation, { recursive: true, force: true }) - } - }) -} - -function clearCandidateMeasurement(candidate: KnowledgeImprovementCandidateRecord): void { - delete candidate.candidateHash - delete candidate.evidenceHash - delete candidate.promotionPlanHash -} - -function findActiveCandidate( - state: KnowledgeImprovementRunState, -): KnowledgeImprovementCandidateRecord | undefined { - return [...state.candidates] - .reverse() - .find((candidate) => candidate.status === 'candidate-ready' || candidate.status === 'running') -} - -async function copyKnowledgeWorkspace(sourceRoot: string, targetRoot: string): Promise { - await rm(targetRoot, { recursive: true, force: true }) - await mkdir(join(targetRoot, 'knowledge'), { recursive: true }) - await mkdir(join(targetRoot, 'raw', 'sources'), { recursive: true }) - await copyIfExists(join(sourceRoot, 'knowledge'), join(targetRoot, 'knowledge')) - await copyIfExists(join(sourceRoot, 'raw'), join(targetRoot, 'raw')) - await copyIfExists( - join(layoutFor(sourceRoot).cacheDir, 'sources.json'), - join(layoutFor(targetRoot).cacheDir, 'sources.json'), - ) - await writeKnowledgeIndex(targetRoot) -} - -function knowledgeCandidateTransitionPurpose( - candidate: KnowledgeImprovementCandidateRef, - target: KnowledgeImprovementTarget, -): string { - const action = target === 'candidate' ? 'promotion' : 'restore' - return `knowledge-${action}:${contentHash(candidate)}` -} - -async function knowledgeFilePlanEntries( - sourceRoot: string, - targetRoot: string, -): Promise { - const [before, after] = await Promise.all([ - knowledgeHashEntries(sourceRoot), - knowledgeHashEntries(targetRoot), - ]) - const beforeByPath = new Map(before.map((entry) => [entry.path, entry])) - const afterByPath = new Map(after.map((entry) => [entry.path, entry])) - const paths = [ - ...new Set([...before.map((entry) => entry.path), ...after.map((entry) => entry.path)]), - ].sort((left, right) => left.localeCompare(right)) - return paths.map((path) => { - assertKnowledgeMutationPath(path) - const beforeEntry = beforeByPath.get(path) - const afterEntry = afterByPath.get(path) - return { - path, - beforeHash: beforeEntry?.transactionHash ?? null, - afterHash: afterEntry?.transactionHash ?? null, - ...(beforeEntry ? { beforeMode: beforeEntry.mode } : {}), - ...(afterEntry ? { afterMode: afterEntry.mode } : {}), - } - }) -} - -async function knowledgePlanMutations( - targetRoot: string, - plan: readonly KnowledgeFileTransactionPlanEntry[], -): Promise { - return Promise.all( - plan.map(async (entry) => { - if (entry.afterHash === null) return { path: entry.path, content: null } - const file = await readRegularFileWithinRoot(targetRoot, entry.path) - const actualHash = createHash('sha256').update(file.bytes).digest('hex') - if (actualHash !== entry.afterHash || file.mode !== entry.afterMode) { - throw new Error(`knowledge target file changed before activation: ${entry.path}`) - } - return { path: entry.path, content: file.bytes, mode: file.mode } - }), - ) -} - -function assertCandidateTransitionPlan( - plan: readonly KnowledgeFileTransactionPlanEntry[], - candidate: KnowledgeImprovementCandidateRef, - target: KnowledgeImprovementTarget, -): void { - const approvedDirection = target === 'candidate' ? plan : reverseKnowledgeFilePlan(plan) - const actualPlanHash = knowledgeFileTransactionPlanHash(approvedDirection) - if (actualPlanHash !== candidate.promotionPlanHash) { - throw new Error( - `knowledge candidate plan changed after approval: expected ${candidate.promotionPlanHash}, got ${actualPlanHash}`, - ) - } -} - -function assertCandidateTransitionTransaction( - transaction: KnowledgeFileTransaction, - candidate: KnowledgeImprovementCandidateRef, - target: KnowledgeImprovementTarget, -): void { - assertCandidateTransitionPlan(transaction.entries, candidate, target) -} - -function reverseKnowledgeFilePlan( - plan: readonly KnowledgeFileTransactionPlanEntry[], -): KnowledgeFileTransactionPlanEntry[] { - return plan.map((entry) => ({ - path: entry.path, - beforeHash: entry.afterHash, - afterHash: entry.beforeHash, - ...(entry.afterMode === undefined ? {} : { beforeMode: entry.afterMode }), - ...(entry.beforeMode === undefined ? {} : { afterMode: entry.beforeMode }), - })) -} - -async function ensureCandidateTransitionEvent( - runDir: string, - candidateRef: KnowledgeImprovementCandidateRef, - target: KnowledgeImprovementTarget, -): Promise { - if (await hasCandidateTransitionEvent(runDir, candidateRef, target)) return - await appendLedger(runDir, { - type: target === 'candidate' ? 'candidate.promoted' : 'candidate.restored', - runId: candidateRef.runId, - candidateId: candidateRef.candidateId, - candidateHash: candidateRef.candidateHash, - evidenceHash: candidateRef.evidenceHash, - promotionPlanHash: candidateRef.promotionPlanHash, - }) -} - -async function hasCandidateTransitionEvent( - runDir: string, - candidateRef: KnowledgeImprovementCandidateRef, - target: KnowledgeImprovementTarget, -): Promise { - const eventType = target === 'candidate' ? 'candidate.promoted' : 'candidate.restored' - let matched = false - for (const row of await loadKnowledgeImprovementEventsFromRun(runDir)) { - if (row.type !== eventType || row.candidateId !== candidateRef.candidateId) continue - if ( - row.runId !== candidateRef.runId || - row.candidateHash !== candidateRef.candidateHash || - row.evidenceHash !== candidateRef.evidenceHash || - row.promotionPlanHash !== candidateRef.promotionPlanHash - ) { - throw new Error('persisted knowledge activation event conflicts with the approved candidate') - } - matched = true - } - return matched -} - -export interface KnowledgeImprovementEvent extends Record { - at: string - type: string -} - -export async function loadKnowledgeImprovementEvents( - root: string, - runId: string, -): Promise { - const parsedRunId = runIdSchema.parse(runId) - try { - return await withKnowledgeImprovementRun(root, parsedRunId, false, (runDir) => - loadKnowledgeImprovementEventsFromRun(runDir), - ) - } catch (error) { - if (isMissingFile(error)) return [] - throw error - } -} - -async function loadKnowledgeImprovementEventsFromRun( - runDir: string, -): Promise { - const events: KnowledgeImprovementEvent[] = [] - try { - for (const file of await listRegularFilesWithinRoot(runDir, 'events')) { - const name = file.path.slice('events/'.length) - if (name.includes('/') || !name.endsWith('.json')) { - throw new Error(`knowledge event store contains an unsupported entry: ${name}`) - } - events.push(parseKnowledgeImprovementEvent(JSON.parse(file.bytes.toString('utf8')))) - } - } catch (error) { - if (!isMissingFile(error)) throw error - } - - const unique = new Map() - for (const event of events) { - const { at: _at, ...semantic } = event - unique.set(contentHash(semantic), event) - } - return [...unique.values()].sort((left, right) => left.at.localeCompare(right.at)) -} - -function parseKnowledgeImprovementEvent(value: unknown): KnowledgeImprovementEvent { - if (!value || typeof value !== 'object' || Array.isArray(value)) { - throw new Error('knowledge improvement event is not an object') - } - const event = value as Record - if (typeof event.at !== 'string' || typeof event.type !== 'string') { - throw new Error('knowledge improvement event is missing at or type') - } - return event as KnowledgeImprovementEvent -} - -async function copyIfExists(source: string, target: string): Promise { - let sourceStat: Awaited> - try { - sourceStat = await lstat(source) - } catch (error) { - if (isMissingFile(error)) return - throw error - } - if (!sourceStat.isDirectory() && !sourceStat.isFile()) { - throw new Error(`knowledge surface contains an unsupported filesystem entry: ${source}`) - } - await mkdir(dirname(target), { recursive: true }) - await cp(source, target, { recursive: sourceStat.isDirectory(), dereference: false }) -} - -export async function hashKnowledgeBase(root: string): Promise { - return withKnowledgeRead(root, () => hashKnowledgeBaseUnlocked(root)) -} - -async function hashKnowledgeBaseUnlocked(root: string): Promise { - const entries = await knowledgeHashEntries(root) - return sha256(JSON.stringify(entries.map(({ path, hash, mode }) => ({ path, hash, mode })))) -} - -interface KnowledgeFileIdentity { - path: string - hash: string - transactionHash: string - mode: number -} - -async function knowledgeHashEntries(root: string): Promise { - const entries: KnowledgeFileIdentity[] = [] - for (const rel of ['knowledge', 'raw']) { - try { - for (const file of await listRegularFilesWithinRoot(root, rel)) { - entries.push(knowledgeFileIdentity(file.path, file.bytes, file.mode)) - } - } catch (error) { - if (!isMissingFile(error)) throw error - } - } - const sourceRegistry = relative(root, layoutFor(root).sourceRegistryPath).replace(/\\/g, '/') - try { - const file = await readRegularFileWithinRoot(root, sourceRegistry) - entries.push(knowledgeFileIdentity(sourceRegistry, file.bytes, file.mode)) - } catch (error) { - if (!isMissingFile(error)) throw error - } - entries.sort((a, b) => a.path.localeCompare(b.path)) - return entries -} - -function knowledgeFileIdentity(path: string, bytes: Buffer, mode: number): KnowledgeFileIdentity { - return { - path, - hash: sha256(bytes.toString('base64')), - transactionHash: createHash('sha256').update(bytes).digest('hex'), - mode, - } -} - -async function acquireRunLease( - runDir: string, - options: { ownerId: string; ttlMs: number }, -): Promise { - const path = join(runDir, 'run.lock.durable') - const acquired = await acquireDurableFileLock(runDir, { - lockfilePath: path, - staleMs: options.ttlMs, - }) - return { - ownerId: options.ownerId, - assertOwned: acquired.assertOwned, - release: acquired.release, - } -} - -async function blockRun( - runDir: string, - state: KnowledgeImprovementRunState, - reason: string, - onState: KnowledgeImprovementOptions['onState'], - now: () => Date, -): Promise { - state.status = 'blocked' - state.blockedReason = reason - state.updatedAt = now().toISOString() - await saveState(runDir, state, onState) - return state -} - -async function saveState( - runDir: string, - state: KnowledgeImprovementRunState, - onState?: KnowledgeImprovementOptions['onState'], -): Promise { - await writeJsonDurableWithinRoot( - runDir, - 'state.json', - KnowledgeImprovementRunStateSchema.parse(state), - ) - await onState?.(state) -} - -async function appendLedger(runDir: string, value: Record): Promise { - const type = value.type - if (typeof type !== 'string' || type.length === 0) { - throw new Error('knowledge improvement event requires a type') - } - const relativePath = join('events', `${contentHash(value)}.json`).replace(/\\/g, '/') - try { - const file = await readRegularFileWithinRoot(runDir, relativePath) - const existing = parseKnowledgeImprovementEvent(JSON.parse(file.bytes.toString('utf8'))) - const { at: _at, ...semantic } = existing - if (canonicalJson(semantic) !== canonicalJson(value)) { - throw new Error('knowledge improvement event identity conflicts with durable content') - } - return - } catch (error) { - if (!isMissingFile(error)) throw error - } - await writeJsonDurableWithinRoot(runDir, relativePath, { - at: new Date().toISOString(), - ...value, - }) -} - -function candidateEvidenceRelativePath(candidateId: string): string { - return join('candidates', safePathSegmentSchema.parse(candidateId), 'evidence.json').replace( - /\\/g, - '/', - ) -} - -function descendantPath(root: string, path: string): string | undefined { - const value = relative(resolve(root), resolve(path)).replace(/\\/g, '/') - if (value === '' || value === '..' || value.startsWith('../') || isAbsolute(value)) - return undefined - return value -} - -function assertExactCandidatePlatform(): void { - if (process.platform !== 'linux') { - throw new Error('exact knowledge candidate workflows require Linux directory descriptors') - } -} - -function average(values: readonly number[]): number { - const finite = values.filter(Number.isFinite) - if (finite.length === 0) return 0 - return finite.reduce((sum, value) => sum + value, 0) / finite.length -} +export { loadKnowledgeImprovementActivationResult } from './kb-improvement/activation' +export type { + KnowledgeImprovementActivationPersistence, + KnowledgeImprovementCandidateRecord, + KnowledgeImprovementCandidateRef, + KnowledgeImprovementEvaluationInput, + KnowledgeImprovementEvaluator, + KnowledgeImprovementEvidence, + KnowledgeImprovementMetric, + KnowledgeImprovementMetricProvenance, + KnowledgeImprovementMutationReceipt, + KnowledgeImprovementMutationResult, + KnowledgeImprovementOptions, + KnowledgeImprovementResult, + KnowledgeImprovementRetrievalOptions, + KnowledgeImprovementRunState, + KnowledgeImprovementStatus, + KnowledgeImprovementTarget, + KnowledgeImprovementUpdate, + KnowledgeImprovementUpdateInput, + LoadKnowledgeImprovementActivationResultOptions, + PromoteKnowledgeCandidateOptions, + ResolvedKnowledgeImprovementCandidate, + ResolvedKnowledgeImprovementComparison, + ResolvedKnowledgeImprovementComparisonSnapshot, + RestoreKnowledgeCandidateBaselineOptions, + UseKnowledgeImprovementCandidateOptions, +} from './kb-improvement/contracts' +export { + KnowledgeImprovementCandidateRefSchema, + KnowledgeImprovementEvidenceSchema, + KnowledgeImprovementRunStateSchema, +} from './kb-improvement/contracts' +export { improveKnowledgeBase } from './kb-improvement/run' +export type { KnowledgeImprovementEvent } from './kb-improvement/state' +export { + knowledgeImprovementRunDir, + knowledgeImprovementRunId, + loadKnowledgeImprovementEvents, + loadKnowledgeImprovementState, +} from './kb-improvement/state' +export { + promoteKnowledgeCandidate, + restoreKnowledgeCandidateBaseline, +} from './kb-improvement/transition' +export { + hashKnowledgeBase, + knowledgeImprovementCandidateRef, + withKnowledgeImprovementCandidate, + withKnowledgeImprovementComparison, +} from './kb-improvement/workspace' diff --git a/src/kb-improvement/activation.ts b/src/kb-improvement/activation.ts new file mode 100644 index 0000000..38f4ac4 --- /dev/null +++ b/src/kb-improvement/activation.ts @@ -0,0 +1,298 @@ +import { canonicalJson } from '@tangle-network/agent-eval' +import { + type AgentImprovementActivation, + type AgentImprovementActivationResult, + agentImprovementActivationResultSchema, + agentImprovementActivationSchema, + canonicalCandidateDigest, + omitTopLevelDigest, + type Sha256Digest, + sha256DigestSchema, +} from '@tangle-network/agent-interface' +import { z } from 'zod' +import { isMissingFile, readRegularFileWithinRoot, writeJsonDurableWithinRoot } from '../durable-fs' +import type { + KnowledgeImprovementActivationPersistence, + KnowledgeImprovementActivationRecord, + KnowledgeImprovementCandidateRef, + KnowledgeImprovementMutationReceipt, + KnowledgeImprovementTarget, + LoadKnowledgeImprovementActivationResultOptions, +} from './contracts' +import { + digestSchema, + KnowledgeImprovementCandidateRefSchema, + knowledgeImprovementActivationRecordSchema, + knowledgeImprovementMutationReceiptSchema, +} from './contracts' +import { assertExactCandidatePlatform, withKnowledgeImprovementRun } from './state' + +/** Load the durable result for one exact activation without changing knowledge or run state. */ +export async function loadKnowledgeImprovementActivationResult( + options: LoadKnowledgeImprovementActivationResultOptions, +): Promise { + assertExactCandidatePlatform() + const candidate = Object.freeze(KnowledgeImprovementCandidateRefSchema.parse(options.candidate)) + const activation = verifyCanonicalKnowledgeActivation(options.activation) + const target = targetForKnowledgeActivation(activation) + assertKnowledgeActivationAuthority(activation, candidate, target, options.identity) + return withKnowledgeImprovementRun(options.root, candidate.runId, false, async (runDir) => { + const record = await loadKnowledgeActivationRecord( + runDir, + candidate, + activation, + target, + options.identity, + ) + return record?.result ?? null + }) +} + +export interface ResolvedKnowledgeImprovementActivationPersistence + extends KnowledgeImprovementActivationPersistence { + activation: AgentImprovementActivation +} + +export function sourceKnowledgeHash( + candidate: KnowledgeImprovementCandidateRef, + target: KnowledgeImprovementTarget, +): string { + return target === 'candidate' ? candidate.baseHash : candidate.candidateHash +} + +function targetForKnowledgeActivation( + activation: AgentImprovementActivation, +): KnowledgeImprovementTarget { + return activation.intent === 'activate-candidate' ? 'candidate' : 'baseline' +} + +export function resolveKnowledgeActivationPersistence( + input: KnowledgeImprovementActivationPersistence, + candidate: KnowledgeImprovementCandidateRef, + target: KnowledgeImprovementTarget, +): ResolvedKnowledgeImprovementActivationPersistence { + const activation = verifyCanonicalKnowledgeActivation(input.activation) + assertKnowledgeActivationAuthority(activation, candidate, target, input.identity) + const attemptedAt = z.iso.datetime().parse(input.attemptedAt) + if ( + Date.parse(attemptedAt) < Date.parse(activation.authorizedAt) || + Date.parse(attemptedAt) >= Date.parse(activation.expiresAt) + ) { + throw new Error('knowledge activation attempt is outside its authorization window') + } + return Object.freeze({ + activation, + attemptedAt, + identity: input.identity, + createResult: input.createResult, + }) +} + +function assertKnowledgeActivationAuthority( + activation: AgentImprovementActivation, + candidate: KnowledgeImprovementCandidateRef, + target: KnowledgeImprovementTarget, + identity: string, +): void { + if (!identity.trim()) throw new Error('knowledge activation identity is required') + if (targetForKnowledgeActivation(activation) !== target) { + throw new Error('knowledge activation intent does not match the requested transition') + } + if (activation.targets.length !== 1) { + throw new Error('knowledge activation requires exactly one target') + } + const authorizedTarget = activation.targets[0] + if (authorizedTarget.surface !== 'knowledge' || authorizedTarget.identity !== identity) { + throw new Error('knowledge activation target does not match this knowledge base') + } + if ( + authorizedTarget.expectedBaseDigest !== + prefixedKnowledgeDigest(sourceKnowledgeHash(candidate, target)) + ) { + throw new Error('knowledge activation does not authorize the measured source state') + } +} + +function verifyCanonicalKnowledgeActivation( + value: AgentImprovementActivation, +): AgentImprovementActivation { + const activation = agentImprovementActivationSchema.parse(value) + if (canonicalCandidateDigest(omitTopLevelDigest(activation)) !== activation.digest) { + throw new Error('knowledge activation digest does not match its canonical content') + } + return immutableJsonValue(structuredClone(activation)) +} + +function verifyCanonicalKnowledgeActivationResult( + value: AgentImprovementActivationResult, +): AgentImprovementActivationResult { + const result = agentImprovementActivationResultSchema.parse(value) + if (canonicalCandidateDigest(omitTopLevelDigest(result)) !== result.digest) { + throw new Error('knowledge activation result digest does not match its canonical content') + } + return immutableJsonValue(structuredClone(result)) +} + +export async function persistKnowledgeActivationResult( + runDir: string, + candidate: KnowledgeImprovementCandidateRef, + persistence: ResolvedKnowledgeImprovementActivationPersistence, + target: KnowledgeImprovementTarget, + mutation: KnowledgeImprovementMutationReceipt, +): Promise { + const result = assertKnowledgeActivationResult( + persistence.activation, + candidate, + target, + persistence.identity, + mutation, + await persistence.createResult(Object.freeze({ ...mutation })), + persistence.attemptedAt, + ) + const record = knowledgeImprovementActivationRecordSchema.parse({ + kind: 'knowledge-improvement-activation-result', + candidateId: candidate.candidateId, + mutation, + result, + }) + const existing = await loadKnowledgeActivationRecord( + runDir, + candidate, + persistence.activation, + target, + persistence.identity, + ) + if (existing) { + if (canonicalJson(existing) !== canonicalJson(record)) { + throw new Error('knowledge activation result identity conflicts with durable content') + } + return existing.result + } + await writeJsonDurableWithinRoot( + runDir, + knowledgeActivationResultPath(persistence.activation.digest), + record, + ) + const stored = await loadKnowledgeActivationRecord( + runDir, + candidate, + persistence.activation, + target, + persistence.identity, + ) + if (!stored || canonicalJson(stored) !== canonicalJson(record)) { + throw new Error('knowledge activation result was not durably persisted') + } + return stored.result +} + +export async function loadKnowledgeActivationRecord( + runDir: string, + candidate: KnowledgeImprovementCandidateRef, + activation: AgentImprovementActivation, + target: KnowledgeImprovementTarget, + identity: string, +): Promise { + let raw: unknown + try { + const file = await readRegularFileWithinRoot( + runDir, + knowledgeActivationResultPath(activation.digest), + ) + raw = JSON.parse(file.bytes.toString('utf8')) as unknown + } catch (error) { + if (isMissingFile(error)) return null + throw error + } + const record = knowledgeImprovementActivationRecordSchema.parse(raw) + if (record.candidateId !== candidate.candidateId) { + throw new Error('knowledge activation result belongs to another candidate') + } + const result = assertKnowledgeActivationResult( + activation, + candidate, + target, + identity, + record.mutation, + record.result, + ) + return immutableJsonValue({ ...record, result }) +} + +function assertKnowledgeActivationResult( + activation: AgentImprovementActivation, + candidate: KnowledgeImprovementCandidateRef, + target: KnowledgeImprovementTarget, + identity: string, + mutationInput: KnowledgeImprovementMutationReceipt, + resultInput: AgentImprovementActivationResult, + attemptedAt?: string, +): AgentImprovementActivationResult { + assertKnowledgeActivationAuthority(activation, candidate, target, identity) + const mutation = knowledgeImprovementMutationReceiptSchema.parse(mutationInput) + const result = verifyCanonicalKnowledgeActivationResult(resultInput) + if ( + result.idempotencyKey !== activation.digest || + (attemptedAt !== undefined && result.attemptedAt !== attemptedAt) || + Date.parse(result.attemptedAt) < Date.parse(activation.authorizedAt) || + Date.parse(result.attemptedAt) >= Date.parse(activation.expiresAt) || + mutation.target !== target || + mutation.changed !== (mutation.beforeHash !== mutation.afterHash) || + (!mutation.changed && (mutation.transactionId !== null || mutation.recovered)) + ) { + throw new Error('knowledge activation result does not bind its authorized mutation') + } + + const sourceDigest = prefixedKnowledgeDigest(sourceKnowledgeHash(candidate, target)) + const desiredDigest = prefixedKnowledgeDigest( + target === 'candidate' ? candidate.candidateHash : candidate.baseHash, + ) + const beforeDigest = prefixedKnowledgeDigest(mutation.beforeHash) + const afterDigest = prefixedKnowledgeDigest(mutation.afterHash) + const outcome = result.outcome + if (mutation.changed) { + if ( + mutation.transactionId === null || + beforeDigest !== sourceDigest || + afterDigest !== desiredDigest || + outcome.status !== 'applied' || + outcome.transactionId !== mutation.transactionId || + outcome.targets.length !== 1 || + outcome.targets[0]?.surface !== 'knowledge' || + outcome.targets[0]?.identity !== identity || + outcome.targets[0]?.beforeDigest !== beforeDigest || + outcome.targets[0]?.afterDigest !== afterDigest + ) { + throw new Error('knowledge activation result does not prove the applied transaction') + } + return result + } + + const expectedStatus = afterDigest === desiredDigest ? 'already-applied' : 'conflict' + if ( + afterDigest === sourceDigest || + outcome.status !== expectedStatus || + outcome.targets.length !== 1 || + outcome.targets[0]?.surface !== 'knowledge' || + outcome.targets[0]?.identity !== identity || + outcome.targets[0]?.currentDigest !== afterDigest + ) { + throw new Error('knowledge activation result does not prove the observed target state') + } + return result +} + +function knowledgeActivationResultPath(digest: Sha256Digest): string { + const parsed = sha256DigestSchema.parse(digest) + return `activation-results/${parsed.slice('sha256:'.length)}.json` +} + +function prefixedKnowledgeDigest(hash: string): Sha256Digest { + return sha256DigestSchema.parse(`sha256:${digestSchema.parse(hash)}`) +} + +export function immutableJsonValue(value: T): T { + if (value === null || typeof value !== 'object') return value + for (const child of Object.values(value)) immutableJsonValue(child) + return Object.freeze(value) +} diff --git a/src/kb-improvement/contracts.ts b/src/kb-improvement/contracts.ts new file mode 100644 index 0000000..4e8df08 --- /dev/null +++ b/src/kb-improvement/contracts.ts @@ -0,0 +1,520 @@ +import { contentHash, type RunRecord, validateRunRecord } from '@tangle-network/agent-eval' +import { + type AgentImprovementActivation, + type AgentImprovementActivationResult, + agentImprovementActivationResultSchema, +} from '@tangle-network/agent-interface' +import { z } from 'zod' +import type { + BuildEvalKnowledgeBundleOptions, + EvalKnowledgeBundleBuildResult, + KnowledgeReadinessSpec, +} from '../eval-readiness' +import type { KnowledgeBaseQualityOptions, KnowledgeBaseQualityReport } from '../rag-eval' +import type { + RagKnowledgeImprovementPhase, + RagKnowledgeResearchOptions, + RagKnowledgeUpdateInput, + RagKnowledgeUpdateResult, + RunRagKnowledgeImprovementLoopOptions, + RunRagKnowledgeImprovementLoopResult, +} from '../rag-improvement-loop' +import type { RunKnowledgeResearchLoopOptions } from '../research-loop' +import type { RunRetrievalImprovementLoopOptions } from '../retrieval-eval' +import type { KnowledgeIndex } from '../types' +import type { ValidateKnowledgeOptions, ValidateKnowledgeResult } from '../validate' + +export type KnowledgeImprovementStatus = + | 'running' + | 'candidate-ready' + | 'promoted' + | 'rejected' + | 'blocked' + +interface KnowledgeImprovementMetricProvenanceBase { + evaluator: string + version: string +} + +export type KnowledgeImprovementMetricProvenance = + | (KnowledgeImprovementMetricProvenanceBase & { + method: 'deterministic' + }) + | (KnowledgeImprovementMetricProvenanceBase & { + method: 'sampled' | 'composite' + corpusHash: string + runRecords: RunRecord[] + }) + | (KnowledgeImprovementMetricProvenanceBase & { + method: 'model' + model: string + corpusHash: string + runRecords: RunRecord[] + }) + +export interface KnowledgeImprovementMetric { + score: number + passed: boolean + dimensions?: Record + notes?: string + provenance: KnowledgeImprovementMetricProvenance +} + +export interface KnowledgeImprovementEvaluationInput { + runId: string + iteration: number + root: string + baselineRoot: string + candidateRoot: string + baselineIndex: KnowledgeIndex + candidateIndex: KnowledgeIndex + baseHash: string + candidateHash: string + validation: ValidateKnowledgeResult + readiness?: EvalKnowledgeBundleBuildResult + kbQuality: KnowledgeBaseQualityReport + lifecycle?: RunRagKnowledgeImprovementLoopResult + signal?: AbortSignal +} + +export type KnowledgeImprovementEvaluator = ( + input: KnowledgeImprovementEvaluationInput, +) => Promise | KnowledgeImprovementMetric + +export interface KnowledgeImprovementCandidateRecord { + iteration: number + candidateId: string + baseHash: string + candidateHash?: string + evidenceHash?: string + promotionPlanHash?: string + status: KnowledgeImprovementStatus + createdAt: string + updatedAt: string +} + +export interface KnowledgeImprovementRunState { + runId: string + root: string + goal: string + status: KnowledgeImprovementStatus + baseHash: string + createdAt: string + updatedAt: string + ownerId?: string + candidates: KnowledgeImprovementCandidateRecord[] + promotedCandidateId?: string + blockedReason?: string +} + +export interface KnowledgeImprovementResult { + runId: string + state: KnowledgeImprovementRunState + candidate?: KnowledgeImprovementCandidateRecord + evaluation?: KnowledgeImprovementMetric + lifecycle?: RunRagKnowledgeImprovementLoopResult + promoted: boolean + blocked: boolean +} + +export type KnowledgeImprovementTarget = 'candidate' | 'baseline' + +export interface KnowledgeImprovementMutationReceipt { + target: KnowledgeImprovementTarget + beforeHash: string + afterHash: string + changed: boolean + transactionId: string | null + recovered: boolean +} + +export interface KnowledgeImprovementMutationResult extends KnowledgeImprovementResult { + candidate: KnowledgeImprovementCandidateRecord + mutation: KnowledgeImprovementMutationReceipt + activationResult?: AgentImprovementActivationResult +} + +export interface KnowledgeImprovementActivationPersistence { + activation: AgentImprovementActivation + attemptedAt: string + identity: string + /** May run again after interruption; keep this deterministic and free of external side effects. */ + createResult( + mutation: KnowledgeImprovementMutationReceipt, + ): Promise | AgentImprovementActivationResult +} + +export const digestSchema = z.string().regex(/^[a-f0-9]{64}$/) + +export const runIdSchema = z.string().min(1).max(2_048) + +export const safePathSegmentSchema = z + .string() + .min(1) + .max(128) + .regex(/^[A-Za-z0-9][A-Za-z0-9._-]*$/) + +export const knowledgeImprovementMutationReceiptSchema = z + .object({ + target: z.enum(['candidate', 'baseline']), + beforeHash: digestSchema, + afterHash: digestSchema, + changed: z.boolean(), + transactionId: z.string().uuid().nullable(), + recovered: z.boolean(), + }) + .strict() + +export const knowledgeImprovementActivationRecordSchema = z + .object({ + kind: z.literal('knowledge-improvement-activation-result'), + candidateId: safePathSegmentSchema, + mutation: knowledgeImprovementMutationReceiptSchema, + result: agentImprovementActivationResultSchema, + }) + .strict() + +export type KnowledgeImprovementActivationRecord = z.infer< + typeof knowledgeImprovementActivationRecordSchema +> + +const improvementStatusSchema = z.enum([ + 'running', + 'candidate-ready', + 'promoted', + 'rejected', + 'blocked', +]) + +const runRecordSchema = z.custom((value) => { + try { + validateRunRecord(value) + return true + } catch { + return false + } +}, 'invalid agent-eval RunRecord') + +const deterministicMetricProvenanceSchema = z + .object({ + evaluator: z.string().min(1), + version: z.string().min(1), + method: z.literal('deterministic'), + }) + .strict() + +const measuredMetricProvenanceSchema = z + .object({ + evaluator: z.string().min(1), + version: z.string().min(1), + method: z.enum(['sampled', 'composite']), + corpusHash: digestSchema, + runRecords: z.array(runRecordSchema).min(1), + }) + .strict() + +const modelMetricProvenanceSchema = z + .object({ + evaluator: z.string().min(1), + version: z.string().min(1), + method: z.literal('model'), + model: z.string().min(1), + corpusHash: digestSchema, + runRecords: z.array(runRecordSchema).min(1), + }) + .strict() + +export const improvementMetricSchema = z + .object({ + score: z.number().finite().min(0).max(1), + passed: z.boolean(), + dimensions: z.record(z.string(), z.number().finite()).optional(), + notes: z.string().optional(), + provenance: z.discriminatedUnion('method', [ + deterministicMetricProvenanceSchema, + measuredMetricProvenanceSchema, + modelMetricProvenanceSchema, + ]), + }) + .strict() + .superRefine((metric, context) => { + if (metric.provenance.method === 'deterministic') return + const actualCorpusHash = contentHash(metric.provenance.runRecords) + if (actualCorpusHash !== metric.provenance.corpusHash) { + context.addIssue({ + code: 'custom', + path: ['provenance', 'corpusHash'], + message: 'metric corpus hash must bind the complete RunRecord array', + }) + } + }) + +const candidateRecordSchema = z + .object({ + iteration: z.number().int().positive(), + candidateId: safePathSegmentSchema, + baseHash: digestSchema, + candidateHash: digestSchema.optional(), + evidenceHash: digestSchema.optional(), + promotionPlanHash: digestSchema.optional(), + status: improvementStatusSchema, + createdAt: z.iso.datetime(), + updatedAt: z.iso.datetime(), + }) + .strict() + +export const KnowledgeImprovementRunStateSchema = z + .object({ + runId: runIdSchema, + root: z.string().min(1), + goal: z.string().min(1), + status: improvementStatusSchema, + baseHash: digestSchema, + createdAt: z.iso.datetime(), + updatedAt: z.iso.datetime(), + ownerId: z.string().min(1).optional(), + candidates: z.array(candidateRecordSchema), + promotedCandidateId: safePathSegmentSchema.optional(), + blockedReason: z.string().min(1).optional(), + }) + .strict() + .superRefine((state, context) => { + const candidateIds = new Set() + for (const [index, candidate] of state.candidates.entries()) { + if (candidateIds.has(candidate.candidateId)) { + context.addIssue({ + code: 'custom', + path: ['candidates', index, 'candidateId'], + message: 'candidate ids must be unique within an improvement run', + }) + } + candidateIds.add(candidate.candidateId) + if (candidate.baseHash !== state.baseHash) { + context.addIssue({ + code: 'custom', + path: ['candidates', index, 'baseHash'], + message: 'candidate base hash must match its improvement run', + }) + } + if (candidate.status === 'candidate-ready') { + if (!candidate.candidateHash || !candidate.evidenceHash || !candidate.promotionPlanHash) { + context.addIssue({ + code: 'custom', + path: ['candidates', index], + message: 'ready candidates require content, evidence, and promotion-plan identities', + }) + } + } + if ( + candidate.status === 'promoted' && + (!candidate.candidateHash || !candidate.evidenceHash || !candidate.promotionPlanHash) + ) { + context.addIssue({ + code: 'custom', + path: ['candidates', index], + message: 'promoted candidates require content, evidence, and promotion-plan identities', + }) + } + if ( + candidate.status === 'promoted' && + Boolean(candidate.evidenceHash) !== Boolean(candidate.promotionPlanHash) + ) { + context.addIssue({ + code: 'custom', + path: ['candidates', index], + message: 'promoted candidate evidence and promotion-plan identities must appear together', + }) + } + if (candidate.status === 'promoted' && state.status !== 'promoted') { + context.addIssue({ + code: 'custom', + path: ['candidates', index, 'status'], + message: 'only a promoted run may contain a promoted candidate', + }) + } + } + if (state.status === 'promoted') { + const promoted = state.candidates.filter( + (candidate) => candidate.candidateId === state.promotedCandidateId, + ) + if (promoted.length !== 1 || promoted[0]?.status !== 'promoted') { + context.addIssue({ + code: 'custom', + path: ['promotedCandidateId'], + message: 'promoted state must identify exactly one promoted candidate', + }) + } + } else if (state.promotedCandidateId !== undefined) { + context.addIssue({ + code: 'custom', + path: ['promotedCandidateId'], + message: 'only a promoted run may identify a promoted candidate', + }) + } + if ( + state.status === 'candidate-ready' && + state.candidates.filter((candidate) => candidate.status === 'candidate-ready').length !== 1 + ) { + context.addIssue({ + code: 'custom', + path: ['status'], + message: 'candidate-ready state must contain exactly one ready candidate', + }) + } + if (state.status === 'blocked' && state.blockedReason === undefined) { + context.addIssue({ + code: 'custom', + path: ['blockedReason'], + message: 'blocked state must include a reason', + }) + } + }) + +export const KnowledgeImprovementEvidenceSchema = z + .object({ + kind: z.literal('knowledge-improvement-evidence'), + runId: runIdSchema, + candidateId: safePathSegmentSchema, + iteration: z.number().int().positive(), + goalHash: digestSchema, + baseHash: digestSchema, + candidateHash: digestSchema, + promotionPlanHash: digestSchema, + validation: z.unknown(), + readiness: z.unknown().nullable(), + kbQuality: z.unknown(), + evaluation: improvementMetricSchema, + lifecycle: z.unknown().nullable(), + }) + .strict() + +export type KnowledgeImprovementEvidence = z.infer + +/** Portable identity of one measured candidate. Paths and mutable run state are deliberately excluded. */ +export const KnowledgeImprovementCandidateRefSchema = z + .object({ + kind: z.literal('knowledge-improvement-candidate'), + runId: runIdSchema, + candidateId: safePathSegmentSchema, + goalHash: digestSchema, + baseHash: digestSchema, + candidateHash: digestSchema, + evidenceHash: digestSchema, + promotionPlanHash: digestSchema, + }) + .strict() + +export type KnowledgeImprovementCandidateRef = z.infer< + typeof KnowledgeImprovementCandidateRefSchema +> + +export interface PromoteKnowledgeCandidateOptions { + root: string + candidate: KnowledgeImprovementCandidateRef + activation?: KnowledgeImprovementActivationPersistence + ownerId?: string + leaseTtlMs?: number + now?: () => Date + onState?: (state: KnowledgeImprovementRunState) => Promise | void +} + +export type RestoreKnowledgeCandidateBaselineOptions = PromoteKnowledgeCandidateOptions + +export interface LoadKnowledgeImprovementActivationResultOptions { + root: string + candidate: KnowledgeImprovementCandidateRef + activation: AgentImprovementActivation + identity: string +} + +export interface UseKnowledgeImprovementCandidateOptions { + root: string + candidate: KnowledgeImprovementCandidateRef +} + +export interface ResolvedKnowledgeImprovementComparisonSnapshot { + root: string + hash: string +} + +export interface ResolvedKnowledgeImprovementComparison { + reference: KnowledgeImprovementCandidateRef + evaluation: KnowledgeImprovementMetric + baseline: ResolvedKnowledgeImprovementComparisonSnapshot + candidate: ResolvedKnowledgeImprovementComparisonSnapshot +} + +export interface ResolvedKnowledgeImprovementCandidate { + root: string + candidate: KnowledgeImprovementCandidateRef + evaluation: KnowledgeImprovementMetric +} + +export interface KnowledgeImprovementRetrievalOptions + extends Omit { + runDir?: RunRetrievalImprovementLoopOptions['runDir'] +} + +export interface KnowledgeImprovementUpdateInput extends RagKnowledgeUpdateInput { + runId: string + iteration: number + candidateId: string + root: string + baselineRoot: string + candidateRoot: string + baseHash: string +} + +export type KnowledgeImprovementUpdate = ( + input: KnowledgeImprovementUpdateInput, +) => Promise | RagKnowledgeUpdateResult + +export interface KnowledgeImprovementOptions { + root: string + goal: string + runId?: string + ownerId?: string + leaseTtlMs?: number + resume?: boolean + maxCandidates?: number + candidateResearchIterations?: number + strict?: ValidateKnowledgeOptions['strict'] + readinessSpecs?: KnowledgeReadinessSpec[] + readinessTaskId?: string + readiness?: Omit + kbQuality?: KnowledgeBaseQualityOptions + step?: RunKnowledgeResearchLoopOptions['step'] + knowledgeResearch?: Omit + retrieval?: KnowledgeImprovementRetrievalOptions + diagnose?: NonNullable + acquireKnowledge?: NonNullable + updateKnowledge?: KnowledgeImprovementUpdate + evaluateAnswers?: NonNullable + decidePromotion?: NonNullable + enabledPhases?: readonly RagKnowledgeImprovementPhase[] + requiredPhases?: readonly RagKnowledgeImprovementPhase[] + evaluate?: KnowledgeImprovementEvaluator + signal?: AbortSignal + now?: () => Date + onState?: (state: KnowledgeImprovementRunState) => Promise | void +} + +export interface LeaseHandle { + ownerId: string + assertOwned(): void + release(): Promise +} + +export const DEFAULT_LEASE_TTL_MS = 15 * 60 * 1000 + +export const UPDATE_PHASES: readonly RagKnowledgeImprovementPhase[] = [ + 'knowledge-acquisition', + 'knowledge-update', +] + +export const EVALUATION_PHASES: readonly RagKnowledgeImprovementPhase[] = [ + 'retrieval-tuning', + 'gap-diagnosis', + 'answer-quality', + 'promotion', +] diff --git a/src/kb-improvement/evaluation.ts b/src/kb-improvement/evaluation.ts new file mode 100644 index 0000000..f4f816b --- /dev/null +++ b/src/kb-improvement/evaluation.ts @@ -0,0 +1,442 @@ +import { join } from 'node:path' +import { contentHash } from '@tangle-network/agent-eval' +import { writeJsonDurableWithinRoot } from '../durable-fs' +import type { EvalKnowledgeBundleBuildResult, KnowledgeReadinessSpec } from '../eval-readiness' +import { knowledgeFileTransactionPlanHash } from '../file-transaction' +import { sha256 } from '../ids' +import { buildKnowledgeIndex } from '../indexer' +import { type KnowledgeBaseQualityReport, scoreKnowledgeBaseIndex } from '../rag-eval' +import { + type RagKnowledgeImprovementPhase, + type RagKnowledgeResearchOptions, + type RunRagKnowledgeImprovementLoopOptions, + type RunRagKnowledgeImprovementLoopResult, + runRagKnowledgeImprovementLoop, +} from '../rag-improvement-loop' +import { readinessFor } from '../readiness-helpers' +import { type ValidateKnowledgeResult, validateKnowledgeIndex } from '../validate' +import type { + KnowledgeImprovementCandidateRecord, + KnowledgeImprovementMetric, + KnowledgeImprovementOptions, + KnowledgeImprovementRunState, +} from './contracts' +import { + EVALUATION_PHASES, + improvementMetricSchema, + KnowledgeImprovementEvidenceSchema, + UPDATE_PHASES, +} from './contracts' +import { appendLedger, candidateEvidenceRelativePath, withCandidateWorkspace } from './state' +import { knowledgeFilePlanEntries } from './transition' +import { + assertCandidateEvidence, + candidateRefFor, + clearCandidateMeasurement, + hashKnowledgeBase, + withBaselineSnapshot, + withFrozenCandidateWorkspace, +} from './workspace' + +export function assertKnowledgeImprovementOptions(options: KnowledgeImprovementOptions): void { + if (options.step && options.knowledgeResearch?.step) { + throw new Error('improveKnowledgeBase accepts either step or knowledgeResearch.step, not both') + } + const updateDrivers = [ + Boolean(options.step ?? options.knowledgeResearch), + Boolean(options.updateKnowledge), + ].filter(Boolean).length + if (updateDrivers > 1) { + throw new Error( + 'improveKnowledgeBase accepts only one knowledge-update driver: knowledgeResearch or updateKnowledge', + ) + } +} + +export async function measureCandidate( + runId: string, + runDir: string, + state: KnowledgeImprovementRunState, + candidate: KnowledgeImprovementCandidateRecord, + options: KnowledgeImprovementOptions, + now: () => Date, +): Promise<{ + candidate: KnowledgeImprovementCandidateRecord + evaluation: KnowledgeImprovementMetric + lifecycle?: RunRagKnowledgeImprovementLoopResult +}> { + return withCandidateWorkspace(runDir, candidate, async (candidateRoot) => { + const currentCandidateHash = await hashKnowledgeBase(candidateRoot) + if ( + candidate.status === 'candidate-ready' && + candidate.candidateHash === currentCandidateHash && + candidate.evidenceHash !== undefined && + candidate.promotionPlanHash !== undefined + ) { + const evidence = await assertCandidateEvidence( + runDir, + candidateRefFor(runId, state, candidate), + ) + return { candidate, evaluation: evidence.evaluation } + } + + clearCandidateMeasurement(candidate) + const lifecycles: RunRagKnowledgeImprovementLoopResult[] = [] + if (candidate.status === 'running') { + const updateLifecycle = await runCandidateUpdateLifecycle( + runId, + candidate, + candidateRoot, + options, + now, + ) + if (updateLifecycle) lifecycles.push(updateLifecycle) + } + return withFrozenCandidateWorkspace(runDir, candidate, candidateRoot, async (snapshot) => { + const evaluationLifecycle = await runCandidateEvaluationLifecycle( + runDir, + candidate, + snapshot.root, + options, + now, + ) + if (evaluationLifecycle) lifecycles.push(evaluationLifecycle) + const lifecycle = mergeLifecycleResults(options.goal, lifecycles) + const measured = await evaluateCandidate( + runDir, + state, + candidate, + snapshot, + lifecycle, + options, + now, + ) + return { ...measured, ...(lifecycle ? { lifecycle } : {}) } + }) + }) +} + +async function runCandidateUpdateLifecycle( + runId: string, + candidate: KnowledgeImprovementCandidateRecord, + candidateRoot: string, + options: KnowledgeImprovementOptions, + now: () => Date, +): Promise { + if (!shouldRunUpdateStage(options)) return undefined + const lifecycle = await runRagKnowledgeImprovementLoop({ + goal: options.goal, + acquireKnowledge: options.acquireKnowledge, + knowledgeResearch: candidateKnowledgeResearchOptions(candidateRoot, options), + updateKnowledge: candidateUpdateHook(runId, candidate, candidateRoot, options), + enabledPhases: selectedStagePhases(options, UPDATE_PHASES), + requiredPhases: selectedStageRequiredPhases(options, UPDATE_PHASES), + signal: options.signal, + now, + }) + return lifecycle +} + +async function runCandidateEvaluationLifecycle( + runDir: string, + candidate: KnowledgeImprovementCandidateRecord, + candidateRoot: string, + options: KnowledgeImprovementOptions, + now: () => Date, +): Promise { + if (!shouldRunEvaluationStage(options)) return undefined + const candidateIndex = await buildKnowledgeIndex(candidateRoot) + const lifecycle = await runRagKnowledgeImprovementLoop({ + goal: options.goal, + retrieval: options.retrieval + ? { + ...options.retrieval, + index: candidateIndex, + runDir: options.retrieval.runDir ?? join(runDir, 'retrieval', candidate.candidateId), + } + : undefined, + diagnose: options.diagnose, + evaluateAnswers: options.evaluateAnswers, + promote: options.decidePromotion, + enabledPhases: selectedStagePhases(options, EVALUATION_PHASES), + requiredPhases: selectedStageRequiredPhases(options, EVALUATION_PHASES), + signal: options.signal, + now, + }) + return lifecycle +} + +function candidateKnowledgeResearchOptions( + candidateRoot: string, + options: KnowledgeImprovementOptions, +): RagKnowledgeResearchOptions | undefined { + if (!options.step && !options.knowledgeResearch) return undefined + const { step: researchStep, ...rest } = options.knowledgeResearch ?? {} + const step = options.step ?? researchStep + return { + ...rest, + root: candidateRoot, + step, + maxIterations: + rest.maxIterations ?? (step ? (options.candidateResearchIterations ?? 3) : undefined), + strict: rest.strict ?? options.strict, + readinessSpecs: rest.readinessSpecs ?? options.readinessSpecs, + readinessTaskId: rest.readinessTaskId ?? options.readinessTaskId, + readiness: rest.readiness ?? options.readiness, + } +} + +function candidateUpdateHook( + runId: string, + candidate: KnowledgeImprovementCandidateRecord, + candidateRoot: string, + options: KnowledgeImprovementOptions, +): RunRagKnowledgeImprovementLoopOptions['updateKnowledge'] { + if (!options.updateKnowledge) return undefined + return (input) => + options.updateKnowledge!({ + ...input, + runId, + iteration: candidate.iteration, + candidateId: candidate.candidateId, + root: candidateRoot, + baselineRoot: options.root, + candidateRoot, + baseHash: candidate.baseHash, + }) +} + +function shouldRunUpdateStage(options: KnowledgeImprovementOptions): boolean { + const phases = selectedStagePhases(options, UPDATE_PHASES) + if (phases.length === 0) return false + return Boolean( + options.acquireKnowledge || + options.step || + options.knowledgeResearch || + options.updateKnowledge || + selectedStageRequiredPhases(options, UPDATE_PHASES).length > 0, + ) +} + +function shouldRunEvaluationStage(options: KnowledgeImprovementOptions): boolean { + const phases = selectedStagePhases(options, EVALUATION_PHASES) + if (phases.length === 0) return false + return Boolean( + options.retrieval || + options.diagnose || + options.evaluateAnswers || + options.decidePromotion || + selectedStageRequiredPhases(options, EVALUATION_PHASES).length > 0, + ) +} + +function selectedStagePhases( + options: Pick, + stagePhases: readonly RagKnowledgeImprovementPhase[], +): RagKnowledgeImprovementPhase[] { + const requested = options.enabledPhases ?? stagePhases + return requested.filter((phase) => stagePhases.includes(phase)) +} + +function selectedStageRequiredPhases( + options: Pick, + stagePhases: readonly RagKnowledgeImprovementPhase[], +): RagKnowledgeImprovementPhase[] { + return (options.requiredPhases ?? []).filter((phase) => stagePhases.includes(phase)) +} + +function mergeLifecycleResults( + goal: string, + lifecycles: readonly RunRagKnowledgeImprovementLoopResult[], +): RunRagKnowledgeImprovementLoopResult | undefined { + if (lifecycles.length === 0) return undefined + return { + goal, + phases: lifecycles.flatMap((lifecycle) => lifecycle.phases), + retrieval: lastDefined(lifecycles.map((lifecycle) => lifecycle.retrieval)), + findings: lifecycles.flatMap((lifecycle) => lifecycle.findings), + acquisition: lastDefined(lifecycles.map((lifecycle) => lifecycle.acquisition)), + knowledgeUpdate: lastDefined(lifecycles.map((lifecycle) => lifecycle.knowledgeUpdate)), + answerQuality: lastDefined(lifecycles.map((lifecycle) => lifecycle.answerQuality)), + promotion: lastDefined(lifecycles.map((lifecycle) => lifecycle.promotion)), + } +} + +function lastDefined(values: readonly (T | undefined)[]): T | undefined { + for (let index = values.length - 1; index >= 0; index -= 1) { + if (values[index] !== undefined) return values[index] + } + return undefined +} + +async function evaluateCandidate( + runDir: string, + state: KnowledgeImprovementRunState, + candidate: KnowledgeImprovementCandidateRecord, + snapshot: { root: string; hash: string }, + lifecycle: RunRagKnowledgeImprovementLoopResult | undefined, + options: KnowledgeImprovementOptions, + now: () => Date, +): Promise<{ + candidate: KnowledgeImprovementCandidateRecord + evaluation: KnowledgeImprovementMetric +}> { + return withBaselineSnapshot(runDir, state.baseHash, async (baselineRoot) => { + const [baselineIndex, candidateIndex] = await Promise.all([ + buildKnowledgeIndex(baselineRoot), + buildKnowledgeIndex(snapshot.root), + ]) + const validation = validateKnowledgeIndex(candidateIndex, { strict: options.strict }) + const readiness = readinessFor(options, candidateIndex) + const kbQuality = scoreKnowledgeBaseIndex(candidateIndex, { + strict: options.strict, + ...options.kbQuality, + }) + const candidateHash = snapshot.hash + const metric = + options.evaluate?.({ + runId: state.runId, + iteration: candidate.iteration, + root: options.root, + baselineRoot, + candidateRoot: snapshot.root, + baselineIndex, + candidateIndex, + baseHash: state.baseHash, + candidateHash, + validation, + readiness, + kbQuality, + lifecycle, + signal: options.signal, + }) ?? + defaultKnowledgeImprovementMetric( + validation, + readiness, + options.readinessSpecs, + kbQuality, + lifecycle, + ) + const evaluation = applyLifecycleFailures(normalizeMetric(await metric), lifecycle) + const measuredHash = await hashKnowledgeBase(snapshot.root) + if (measuredHash !== candidateHash) { + throw new Error( + `knowledge candidate changed during evaluation: expected ${candidateHash}, got ${measuredHash}`, + ) + } + candidate.candidateHash = candidateHash + candidate.promotionPlanHash = knowledgeFileTransactionPlanHash( + await knowledgeFilePlanEntries(baselineRoot, snapshot.root), + ) + const evidence = KnowledgeImprovementEvidenceSchema.parse( + JSON.parse( + JSON.stringify({ + kind: 'knowledge-improvement-evidence', + runId: state.runId, + candidateId: candidate.candidateId, + iteration: candidate.iteration, + goalHash: sha256(state.goal), + baseHash: candidate.baseHash, + candidateHash, + promotionPlanHash: candidate.promotionPlanHash, + validation, + readiness: readiness ?? null, + kbQuality, + evaluation, + lifecycle: lifecycle ?? null, + }), + ), + ) + candidate.evidenceHash = contentHash(evidence) + candidate.updatedAt = now().toISOString() + await writeJsonDurableWithinRoot( + runDir, + candidateEvidenceRelativePath(candidate.candidateId), + evidence, + ) + await appendLedger(runDir, { + type: 'candidate.evaluated', + runId: state.runId, + candidateId: candidate.candidateId, + score: evaluation.score, + passed: evaluation.passed, + }) + return { candidate, evaluation } + }) +} + +function defaultKnowledgeImprovementMetric( + validation: ValidateKnowledgeResult, + readiness: EvalKnowledgeBundleBuildResult | undefined, + readinessSpecs: readonly KnowledgeReadinessSpec[] | undefined, + kbQuality: KnowledgeBaseQualityReport, + lifecycle: RunRagKnowledgeImprovementLoopResult | undefined, +): KnowledgeImprovementMetric { + const blockingMissing = readiness?.report.blockingMissingRequirements.length ?? 0 + const blockingTotal = readinessSpecs?.filter((spec) => spec.importance === 'blocking').length ?? 0 + const blockingReadiness = + blockingTotal === 0 ? 1 : Math.max(0, blockingTotal - blockingMissing) / blockingTotal + const answerQuality = lifecycle?.answerQuality + ? average(Object.values(lifecycle.answerQuality.metrics).filter(Number.isFinite)) + : 1 + const promotionDecision = lifecycle?.promotion ? (lifecycle.promotion.promoted ? 1 : 0) : 1 + const dimensions = { + validation: validation.ok ? 1 : 0, + kb_quality: kbQuality.ok ? 1 : 0, + blocking_readiness: blockingReadiness, + answer_quality: answerQuality, + promotion_decision: promotionDecision, + } + const failedReasons = [ + validation.ok ? undefined : 'candidate validation failed', + kbQuality.ok ? undefined : 'candidate KB quality check failed', + blockingMissing === 0 + ? undefined + : `${blockingMissing}/${blockingTotal} blocking knowledge requirements still missing`, + ].filter((reason): reason is string => Boolean(reason)) + return { + score: average(Object.values(dimensions)), + passed: failedReasons.length === 0, + dimensions, + notes: + failedReasons.length === 0 ? 'candidate passed configured checks' : failedReasons.join('; '), + provenance: { + evaluator: '@tangle-network/agent-knowledge/default-knowledge-improvement-metric', + version: '1', + method: 'deterministic', + }, + } +} + +function applyLifecycleFailures( + metric: KnowledgeImprovementMetric, + lifecycle: RunRagKnowledgeImprovementLoopResult | undefined, +): KnowledgeImprovementMetric { + const reasons = [ + metric.notes, + lifecycle?.answerQuality && !lifecycle.answerQuality.passed + ? 'answer quality failed' + : undefined, + lifecycle?.promotion && !lifecycle.promotion.promoted + ? `promotion decision held: ${lifecycle.promotion.reason}` + : undefined, + ].filter((reason): reason is string => Boolean(reason)) + const forcedFailure = + Boolean(lifecycle?.answerQuality && !lifecycle.answerQuality.passed) || + Boolean(lifecycle?.promotion && !lifecycle.promotion.promoted) + return { + ...metric, + passed: metric.passed && !forcedFailure, + notes: reasons.length > 0 ? reasons.join('; ') : metric.notes, + } +} + +function normalizeMetric(metric: KnowledgeImprovementMetric): KnowledgeImprovementMetric { + return improvementMetricSchema.parse(metric) +} + +function average(values: readonly number[]): number { + const finite = values.filter(Number.isFinite) + if (finite.length === 0) return 0 + return finite.reduce((sum, value) => sum + value, 0) / finite.length +} diff --git a/src/kb-improvement/run.ts b/src/kb-improvement/run.ts new file mode 100644 index 0000000..898a980 --- /dev/null +++ b/src/kb-improvement/run.ts @@ -0,0 +1,222 @@ +import { isMissingFile } from '../durable-fs' +import { withKnowledgeMutation } from '../mutation-lock' +import type { RunRagKnowledgeImprovementLoopResult } from '../rag-improvement-loop' +import type { + KnowledgeImprovementCandidateRecord, + KnowledgeImprovementMetric, + KnowledgeImprovementOptions, + KnowledgeImprovementResult, +} from './contracts' +import { DEFAULT_LEASE_TTL_MS, runIdSchema } from './contracts' +import { assertKnowledgeImprovementOptions, measureCandidate } from './evaluation' +import { + acquireRunLease, + appendLedger, + assertExactCandidatePlatform, + blockRun, + findActiveCandidate, + knowledgeImprovementRunId, + loadKnowledgeImprovementStateFromRun, + saveState, + withKnowledgeImprovementRun, +} from './state' +import { + assertCandidateEvidence, + candidateRefFor, + createBaselineSnapshot, + createCandidateWorkspace, + ensureBaselineSnapshot, + hashKnowledgeBase, + withBaselineSnapshot, +} from './workspace' + +export async function improveKnowledgeBase( + options: KnowledgeImprovementOptions, +): Promise { + assertExactCandidatePlatform() + assertKnowledgeImprovementOptions(options) + const now = options.now ?? (() => new Date()) + const runId = runIdSchema.parse( + options.runId ?? knowledgeImprovementRunId(options.root, options.goal), + ) + return withKnowledgeImprovementRun(options.root, runId, true, (runDir) => + improveKnowledgeBaseInRun(options, runId, runDir, now), + ) +} + +async function improveKnowledgeBaseInRun( + options: KnowledgeImprovementOptions, + runId: string, + runDir: string, + now: () => Date, +): Promise { + const lease = await acquireRunLease(runDir, { + ownerId: options.ownerId ?? `pid-${process.pid}`, + ttlMs: options.leaseTtlMs ?? DEFAULT_LEASE_TTL_MS, + }) + + try { + lease.assertOwned() + let state = + options.resume === false + ? null + : await loadKnowledgeImprovementStateFromRun(options.root, runId, runDir).catch((error) => { + if (isMissingFile(error)) return null + throw error + }) + if (!state) { + const baseHash = await hashKnowledgeBase(options.root) + await createBaselineSnapshot(runDir, options.root, baseHash) + state = { + runId, + root: options.root, + goal: options.goal, + status: 'running', + baseHash, + createdAt: now().toISOString(), + updatedAt: now().toISOString(), + ownerId: lease.ownerId, + candidates: [], + } + await saveState(runDir, state, options.onState) + await appendLedger(runDir, { type: 'run.created', runId, baseHash }) + } + if (state.goal !== options.goal) { + throw new Error('knowledge improvement state does not match the requested goal') + } + const promotedCandidateId = state.promotedCandidateId + const promotedCandidate = + state.status === 'promoted' + ? state.candidates.find((candidate) => candidate.candidateId === promotedCandidateId) + : undefined + if (state.status === 'promoted' && !promotedCandidate) { + throw new Error('promoted knowledge state has no promoted candidate') + } + if (state.status === 'promoted') { + const promoted = promotedCandidate! + const promotedState = state + return withKnowledgeMutation(options.root, async () => { + const currentHash = await hashKnowledgeBase(options.root) + if (currentHash !== promoted.candidateHash) { + throw new Error( + `promoted knowledge base changed: expected ${promoted.candidateHash}, got ${currentHash}`, + ) + } + const evidence = await assertCandidateEvidence( + runDir, + candidateRefFor(runId, promotedState, promoted), + ) + return { + runId, + state: promotedState, + candidate: promoted, + evaluation: evidence.evaluation, + promoted: true, + blocked: false, + } + }) + } + await withKnowledgeMutation(options.root, () => undefined) + await ensureBaselineSnapshot(runDir, options.root, state.baseHash) + + if (state.status === 'blocked') { + return { runId, state, promoted: false, blocked: true } + } + + const maxCandidates = Math.max(1, options.maxCandidates ?? 1) + let candidate = findActiveCandidate(state) + let lastRejectedCandidate: KnowledgeImprovementCandidateRecord | undefined + let lastRejectedEvaluation: KnowledgeImprovementMetric | undefined + let lifecycle: RunRagKnowledgeImprovementLoopResult | undefined + + while (candidate || state.candidates.length < maxCandidates) { + if (!candidate) { + const currentHash = await hashKnowledgeBase(options.root) + if (currentHash !== state.baseHash) { + state = await blockRun( + runDir, + state, + `base changed before candidate creation: expected ${state.baseHash}, got ${currentHash}`, + options.onState, + now, + ) + return { runId, state, promoted: false, blocked: true } + } + const activeState = state + candidate = await withBaselineSnapshot(runDir, activeState.baseHash, (baselineRoot) => + createCandidateWorkspace(runDir, activeState, baselineRoot, now), + ) + state.candidates.push(candidate) + state.status = 'running' + state.updatedAt = now().toISOString() + await saveState(runDir, state, options.onState) + await appendLedger(runDir, { + type: 'candidate.created', + runId, + candidateId: candidate.candidateId, + iteration: candidate.iteration, + }) + } + + const measured = await measureCandidate(runId, runDir, state, candidate, options, now) + candidate = measured.candidate + const evaluation = measured.evaluation + lifecycle = measured.lifecycle + + if (evaluation.passed) { + candidate.status = 'candidate-ready' + candidate.updatedAt = now().toISOString() + state.status = 'candidate-ready' + state.updatedAt = now().toISOString() + await saveState(runDir, state, options.onState) + await appendLedger(runDir, { + type: 'candidate.ready', + runId, + candidateId: candidate.candidateId, + }) + break + } + + candidate.status = 'rejected' + state.status = 'running' + state.updatedAt = now().toISOString() + await saveState(runDir, state, options.onState) + await appendLedger(runDir, { + type: 'candidate.rejected', + runId, + candidateId: candidate.candidateId, + }) + lastRejectedCandidate = candidate + lastRejectedEvaluation = evaluation + candidate = undefined + } + + if (!candidate) { + state.status = 'rejected' + state.updatedAt = now().toISOString() + await saveState(runDir, state, options.onState) + return { + runId, + state, + candidate: lastRejectedCandidate, + ...(lastRejectedEvaluation ? { evaluation: lastRejectedEvaluation } : {}), + lifecycle, + promoted: false, + blocked: false, + } + } + + const evidence = await assertCandidateEvidence(runDir, candidateRefFor(runId, state, candidate)) + return { + runId, + state, + candidate, + evaluation: evidence.evaluation, + lifecycle, + promoted: false, + blocked: false, + } + } finally { + await lease.release() + } +} diff --git a/src/kb-improvement/state.ts b/src/kb-improvement/state.ts new file mode 100644 index 0000000..ac340f6 --- /dev/null +++ b/src/kb-improvement/state.ts @@ -0,0 +1,266 @@ +import { stat } from 'node:fs/promises' +import { isAbsolute, join, relative, resolve, sep } from 'node:path' +import { canonicalJson, contentHash } from '@tangle-network/agent-eval' +import { + isMissingFile, + listRegularFilesWithinRoot, + readRegularFileWithinRoot, + withSafeDirectory, + writeJsonDurableWithinRoot, +} from '../durable-fs' +import { sha256, slugify, stableId } from '../ids' +import { acquireDurableFileLock } from '../mutation-lock' +import { layoutFor } from '../store' +import type { + KnowledgeImprovementCandidateRecord, + KnowledgeImprovementOptions, + KnowledgeImprovementRunState, + LeaseHandle, +} from './contracts' +import { KnowledgeImprovementRunStateSchema, runIdSchema, safePathSegmentSchema } from './contracts' + +export function knowledgeImprovementRunId(root: string, goal: string): string { + return stableId('kimpr', `${root}:${goal}`) +} + +export function knowledgeImprovementRunDir(root: string, runId: string): string { + const parsedRunId = runIdSchema.parse(runId) + const safeRunId = safePathSegmentSchema.safeParse(parsedRunId) + const runSegment = safeRunId.success + ? safeRunId.data + : `${slugify(parsedRunId).slice(0, 72)}-${sha256(parsedRunId).slice(0, 16)}` + const improvementsDir = join(layoutFor(root).cacheDir, 'improvements') + const runDir = join(improvementsDir, runSegment) + const resolvedImprovementsDir = resolve(improvementsDir) + const resolvedRunDir = resolve(runDir) + if (!resolvedRunDir.startsWith(`${resolvedImprovementsDir}${sep}`)) { + throw new Error('knowledge improvement run directory escaped its root') + } + return runDir +} + +export async function withKnowledgeImprovementRun( + root: string, + runId: string, + create: boolean, + use: (runDir: string) => Promise | T, +): Promise { + const runDir = knowledgeImprovementRunDir(root, runId) + const relativePath = descendantPath(root, runDir) + if (!relativePath) throw new Error('knowledge improvement run directory escaped its root') + return withSafeDirectory(root, relativePath, create, async (openedRunDir) => { + const result = await use(openedRunDir) + const openedIdentity = await stat(openedRunDir) + const currentIdentity = await withSafeDirectory(root, relativePath, false, (currentRunDir) => + stat(currentRunDir), + ) + if (openedIdentity.dev !== currentIdentity.dev || openedIdentity.ino !== currentIdentity.ino) { + throw new Error('knowledge improvement run directory changed during use') + } + return result + }) +} + +export async function loadKnowledgeImprovementState( + root: string, + runId: string, +): Promise { + const expectedRunId = runIdSchema.parse(runId) + try { + return await withKnowledgeImprovementRun(root, expectedRunId, false, (runDir) => + loadKnowledgeImprovementStateFromRun(root, expectedRunId, runDir), + ) + } catch (error) { + if (isMissingFile(error)) return null + throw error + } +} + +export async function loadKnowledgeImprovementStateFromRun( + root: string, + runId: string, + runDir: string, +): Promise { + const stateFile = await readRegularFileWithinRoot(runDir, 'state.json') + const raw = JSON.parse(stateFile.bytes.toString('utf8')) as unknown + const state = KnowledgeImprovementRunStateSchema.parse(raw) as KnowledgeImprovementRunState + if (state.runId !== runId) { + throw new Error('knowledge improvement state does not match the requested run') + } + if (resolve(state.root) !== resolve(root)) { + throw new Error('knowledge improvement state does not match the requested root') + } + for (const candidate of state.candidates) { + if (candidate.status === 'running') await assertCandidateWorkspace(runDir, candidate) + } + return state +} + +async function assertCandidateWorkspace( + runDir: string, + candidate: Pick, +): Promise { + await withCandidateWorkspace(runDir, candidate, () => undefined) +} + +export async function withCandidateWorkspace( + runDir: string, + candidate: Pick, + use: (candidateRoot: string) => Promise | T, +): Promise { + return withSafeDirectory( + runDir, + join('candidates', safePathSegmentSchema.parse(candidate.candidateId), 'workspace'), + false, + use, + ) +} + +export function findActiveCandidate( + state: KnowledgeImprovementRunState, +): KnowledgeImprovementCandidateRecord | undefined { + return [...state.candidates] + .reverse() + .find((candidate) => candidate.status === 'candidate-ready' || candidate.status === 'running') +} + +export interface KnowledgeImprovementEvent extends Record { + at: string + type: string +} + +export async function loadKnowledgeImprovementEvents( + root: string, + runId: string, +): Promise { + const parsedRunId = runIdSchema.parse(runId) + try { + return await withKnowledgeImprovementRun(root, parsedRunId, false, (runDir) => + loadKnowledgeImprovementEventsFromRun(runDir), + ) + } catch (error) { + if (isMissingFile(error)) return [] + throw error + } +} + +export async function loadKnowledgeImprovementEventsFromRun( + runDir: string, +): Promise { + const events: KnowledgeImprovementEvent[] = [] + try { + for (const file of await listRegularFilesWithinRoot(runDir, 'events')) { + const name = file.path.slice('events/'.length) + if (name.includes('/') || !name.endsWith('.json')) { + throw new Error(`knowledge event store contains an unsupported entry: ${name}`) + } + events.push(parseKnowledgeImprovementEvent(JSON.parse(file.bytes.toString('utf8')))) + } + } catch (error) { + if (!isMissingFile(error)) throw error + } + + const unique = new Map() + for (const event of events) { + const { at: _at, ...semantic } = event + unique.set(contentHash(semantic), event) + } + return [...unique.values()].sort((left, right) => left.at.localeCompare(right.at)) +} + +function parseKnowledgeImprovementEvent(value: unknown): KnowledgeImprovementEvent { + if (!value || typeof value !== 'object' || Array.isArray(value)) { + throw new Error('knowledge improvement event is not an object') + } + const event = value as Record + if (typeof event.at !== 'string' || typeof event.type !== 'string') { + throw new Error('knowledge improvement event is missing at or type') + } + return event as KnowledgeImprovementEvent +} + +export async function acquireRunLease( + runDir: string, + options: { ownerId: string; ttlMs: number }, +): Promise { + const path = join(runDir, 'run.lock.durable') + const acquired = await acquireDurableFileLock(runDir, { + lockfilePath: path, + staleMs: options.ttlMs, + }) + return { + ownerId: options.ownerId, + assertOwned: acquired.assertOwned, + release: acquired.release, + } +} + +export async function blockRun( + runDir: string, + state: KnowledgeImprovementRunState, + reason: string, + onState: KnowledgeImprovementOptions['onState'], + now: () => Date, +): Promise { + state.status = 'blocked' + state.blockedReason = reason + state.updatedAt = now().toISOString() + await saveState(runDir, state, onState) + return state +} + +export async function saveState( + runDir: string, + state: KnowledgeImprovementRunState, + onState?: KnowledgeImprovementOptions['onState'], +): Promise { + await writeJsonDurableWithinRoot( + runDir, + 'state.json', + KnowledgeImprovementRunStateSchema.parse(state), + ) + await onState?.(state) +} + +export async function appendLedger(runDir: string, value: Record): Promise { + const type = value.type + if (typeof type !== 'string' || type.length === 0) { + throw new Error('knowledge improvement event requires a type') + } + const relativePath = join('events', `${contentHash(value)}.json`).replace(/\\/g, '/') + try { + const file = await readRegularFileWithinRoot(runDir, relativePath) + const existing = parseKnowledgeImprovementEvent(JSON.parse(file.bytes.toString('utf8'))) + const { at: _at, ...semantic } = existing + if (canonicalJson(semantic) !== canonicalJson(value)) { + throw new Error('knowledge improvement event identity conflicts with durable content') + } + return + } catch (error) { + if (!isMissingFile(error)) throw error + } + await writeJsonDurableWithinRoot(runDir, relativePath, { + at: new Date().toISOString(), + ...value, + }) +} + +export function candidateEvidenceRelativePath(candidateId: string): string { + return join('candidates', safePathSegmentSchema.parse(candidateId), 'evidence.json').replace( + /\\/g, + '/', + ) +} + +function descendantPath(root: string, path: string): string | undefined { + const value = relative(resolve(root), resolve(path)).replace(/\\/g, '/') + if (value === '' || value === '..' || value.startsWith('../') || isAbsolute(value)) + return undefined + return value +} + +export function assertExactCandidatePlatform(): void { + if (process.platform !== 'linux') { + throw new Error('exact knowledge candidate workflows require Linux directory descriptors') + } +} diff --git a/src/kb-improvement/transition.ts b/src/kb-improvement/transition.ts new file mode 100644 index 0000000..73c1511 --- /dev/null +++ b/src/kb-improvement/transition.ts @@ -0,0 +1,623 @@ +import { createHash } from 'node:crypto' +import { canonicalJson, contentHash } from '@tangle-network/agent-eval' +import type { AgentImprovementActivationResult } from '@tangle-network/agent-interface' +import { readRegularFileWithinRoot } from '../durable-fs' +import { + applyKnowledgeFileTransaction, + assertKnowledgeMutationPath, + finishKnowledgeFileTransaction, + type KnowledgeFileMutation, + type KnowledgeFileTransaction, + type KnowledgeFileTransactionPlanEntry, + knowledgeFileTransactionPlanHash, + prepareKnowledgeFileTransaction, + rollbackKnowledgeFileTransaction, +} from '../file-transaction' +import { writeKnowledgeIndex } from '../indexer' +import { withKnowledgeMutation } from '../mutation-lock' +import type { RunRagKnowledgeImprovementLoopResult } from '../rag-improvement-loop' +import type { ResolvedKnowledgeImprovementActivationPersistence } from './activation' +import { + loadKnowledgeActivationRecord, + persistKnowledgeActivationResult, + resolveKnowledgeActivationPersistence, + sourceKnowledgeHash, +} from './activation' +import type { + KnowledgeImprovementCandidateRecord, + KnowledgeImprovementCandidateRef, + KnowledgeImprovementMutationReceipt, + KnowledgeImprovementMutationResult, + KnowledgeImprovementOptions, + KnowledgeImprovementRunState, + KnowledgeImprovementTarget, + PromoteKnowledgeCandidateOptions, + RestoreKnowledgeCandidateBaselineOptions, +} from './contracts' +import { DEFAULT_LEASE_TTL_MS, KnowledgeImprovementCandidateRefSchema } from './contracts' +import { + acquireRunLease, + appendLedger, + assertExactCandidatePlatform, + blockRun, + loadKnowledgeImprovementEventsFromRun, + loadKnowledgeImprovementStateFromRun, + saveState, + withKnowledgeImprovementRun, +} from './state' +import { + assertStateIdentity, + candidateIdentityFor, + hashKnowledgeBase, + knowledgeHashEntries, + withBaselineSnapshot, + withMeasuredCandidateSnapshot, +} from './workspace' + +/** Promote one previously measured candidate without rerunning research or evaluation. */ +export async function promoteKnowledgeCandidate( + options: PromoteKnowledgeCandidateOptions, +): Promise { + return transitionKnowledgeCandidate(options, 'candidate') +} + +/** Restore the frozen baseline paired with one previously measured candidate. */ +export async function restoreKnowledgeCandidateBaseline( + options: RestoreKnowledgeCandidateBaselineOptions, +): Promise { + return transitionKnowledgeCandidate(options, 'baseline') +} + +async function transitionKnowledgeCandidate( + options: PromoteKnowledgeCandidateOptions, + target: KnowledgeImprovementTarget, +): Promise { + assertExactCandidatePlatform() + const candidateRef = Object.freeze( + KnowledgeImprovementCandidateRefSchema.parse(options.candidate), + ) + const now = options.now ?? (() => new Date()) + const activation = options.activation + ? resolveKnowledgeActivationPersistence(options.activation, candidateRef, target) + : undefined + return withKnowledgeImprovementRun(options.root, candidateRef.runId, false, async (runDir) => { + const lease = await acquireRunLease(runDir, { + ownerId: options.ownerId ?? `pid-${process.pid}`, + ttlMs: options.leaseTtlMs ?? DEFAULT_LEASE_TTL_MS, + }) + try { + lease.assertOwned() + const state = await loadKnowledgeImprovementStateFromRun( + options.root, + candidateRef.runId, + runDir, + ) + return await applyKnowledgeCandidateTarget( + { + root: options.root, + runDir, + state, + candidateRef, + leaseTtlMs: options.leaseTtlMs ?? DEFAULT_LEASE_TTL_MS, + assertRunOwned: lease.assertOwned, + now, + onState: options.onState, + activation, + }, + target, + ) + } finally { + await lease.release() + } + }) +} + +interface KnowledgeCandidateTransitionInput { + root: string + runDir: string + state: KnowledgeImprovementRunState + candidateRef: KnowledgeImprovementCandidateRef + activation?: ResolvedKnowledgeImprovementActivationPersistence + leaseTtlMs: number + assertRunOwned(): void + now: () => Date + onState?: KnowledgeImprovementOptions['onState'] + lifecycle?: RunRagKnowledgeImprovementLoopResult +} + +async function applyKnowledgeCandidateTarget( + input: KnowledgeCandidateTransitionInput, + target: KnowledgeImprovementTarget, +): Promise { + const { candidateRef, runDir, state } = input + assertStateIdentity(input.root, candidateRef, state) + const candidate = state.candidates.find((entry) => entry.candidateId === candidateRef.candidateId) + if ( + !candidate || + canonicalJson(candidateIdentityFor(candidateRef.runId, state, candidate)) !== + canonicalJson(candidateRef) + ) { + throw new Error('knowledge candidate approval does not match the measured candidate') + } + + const action = target === 'candidate' ? 'promotion' : 'restore' + const desiredHash = target === 'candidate' ? candidateRef.candidateHash : candidateRef.baseHash + const purpose = knowledgeCandidateTransitionPurpose(candidateRef, target) + const recoveryOwner = input.activation + ? `knowledge-improvement-activation:${input.activation.activation.digest}` + : 'knowledge-improvement-candidate-transition' + return withKnowledgeMutation( + input.root, + async (mutationLock) => { + const assertOwned = () => { + mutationLock.assertOwned() + input.assertRunOwned() + } + assertOwned() + const transactionRoot = mutationLock.transactionRoot + const recovery = + mutationLock.recovery?.purpose === purpose ? mutationLock.recovery : undefined + const existingActivation = input.activation + ? await loadKnowledgeActivationRecord( + runDir, + candidateRef, + input.activation.activation, + target, + input.activation.identity, + ) + : null + if (existingActivation) { + if (recovery?.direction === 'rollback') { + throw new Error('stored knowledge activation result conflicts with a pending rollback') + } + if (recovery) { + const recoveredHash = await hashKnowledgeBase(input.root) + if (recoveredHash !== existingActivation.mutation.afterHash) { + throw new Error('stored knowledge activation result does not match the recovered files') + } + await finishKnowledgeFileTransaction({ + root: input.root, + transactionRoot, + transaction: recovery.transaction, + assertOwned, + }) + } + if (existingActivation.result.outcome.status === 'conflict') { + const reason = candidateTransitionConflictReason( + target, + candidateRef, + existingActivation.mutation.afterHash, + ) + const blocked = await blockCandidateTransition( + input, + candidate, + target, + reason, + existingActivation.mutation.afterHash, + ) + return { ...blocked, activationResult: existingActivation.result } + } + return candidateTransitionResult( + input, + candidate, + target === 'candidate' && state.status === 'promoted', + false, + existingActivation.mutation, + existingActivation.result, + ) + } + if (recovery?.direction === 'rollback') { + await finishKnowledgeFileTransaction({ + root: input.root, + transactionRoot, + transaction: recovery.transaction, + assertOwned, + }) + } + const recovered = recovery?.direction === 'apply' ? recovery : undefined + let pending: KnowledgeFileTransaction | null = + input.activation && recovered ? recovered.transaction : null + const currentHash = await hashKnowledgeBase(input.root) + let transactionId = recovered?.transactionId ?? null + if (recovered && currentHash !== desiredHash) { + throw new Error(`recovered knowledge ${action} does not match the approved target`) + } + if (state.status === 'promoted' && state.promotedCandidateId !== candidate.candidateId) { + throw new Error( + `knowledge run already promoted '${state.promotedCandidateId ?? 'unknown'}'`, + ) + } + if ( + target === 'candidate' && + state.status === 'promoted' && + currentHash !== candidateRef.candidateHash + ) { + throw new Error( + `promoted knowledge base changed: expected ${candidateRef.candidateHash}, got ${currentHash}`, + ) + } + if (state.status !== 'promoted' && state.status !== 'candidate-ready') { + throw new Error( + `knowledge candidate is not ready for ${action}: run=${state.status}, candidate=${candidate.status}`, + ) + } + if (currentHash !== state.baseHash && currentHash !== candidateRef.candidateHash) { + const reason = + target === 'candidate' + ? `base changed before promotion: expected ${state.baseHash}, got ${currentHash}` + : `knowledge changed before restore: expected ${candidateRef.candidateHash}, got ${currentHash}` + const mutation = Object.freeze({ + target, + beforeHash: currentHash, + afterHash: currentHash, + changed: false, + transactionId: null, + recovered: false, + }) satisfies KnowledgeImprovementMutationReceipt + const activationResult = input.activation + ? await persistKnowledgeActivationResult( + runDir, + candidateRef, + input.activation, + target, + mutation, + ) + : undefined + const blocked = await blockCandidateTransition( + input, + candidate, + target, + reason, + currentHash, + ) + return activationResult ? { ...blocked, activationResult } : blocked + } + + if (currentHash !== desiredHash && !pending) { + pending = await withMeasuredCandidateSnapshot( + input.root, + runDir, + state, + candidateRef, + (resolved) => + withBaselineSnapshot(runDir, state.baseHash, async (baselineRoot) => { + const sourceRoot = target === 'candidate' ? baselineRoot : resolved.root + const targetRoot = target === 'candidate' ? resolved.root : baselineRoot + const plan = await knowledgeFilePlanEntries(sourceRoot, targetRoot) + assertCandidateTransitionPlan(plan, candidateRef, target) + return prepareKnowledgeFileTransaction({ + root: input.root, + transactionRoot, + purpose, + recoveryOwner, + mutations: await knowledgePlanMutations(targetRoot, plan), + includeUnchanged: true, + now: input.now, + }) + }), + ) + if (!pending) { + throw new Error(`knowledge ${action} plan unexpectedly contained no file changes`) + } + transactionId = pending.transactionId + try { + assertCandidateTransitionTransaction(pending, candidateRef, target) + } catch (error) { + try { + await rollbackKnowledgeFileTransaction({ + root: input.root, + transactionRoot, + transaction: pending, + beforeCommit: assertOwned, + }) + await finishKnowledgeFileTransaction({ + root: input.root, + transactionRoot, + transaction: pending, + assertOwned, + }) + } catch (cleanupError) { + throw new AggregateError( + [error, cleanupError], + `invalid knowledge ${action} transaction could not be removed`, + ) + } + throw error + } + } + try { + if (pending) { + await applyKnowledgeFileTransaction({ + root: input.root, + transactionRoot, + transaction: pending, + beforeCommit: assertOwned, + }) + } + assertOwned() + if ((await hashKnowledgeBase(input.root)) !== desiredHash) { + throw new Error(`knowledge ${action} content does not match the approved target`) + } + await writeKnowledgeIndex(input.root) + } catch (error) { + if (!pending) throw error + if (recovered && input.activation) throw error + try { + assertOwned() + } catch (ownershipError) { + throw new AggregateError( + [error, ownershipError], + `knowledge ${action} lost its lock and left the transaction pending`, + ) + } + try { + await rollbackKnowledgeFileTransaction({ + root: input.root, + transactionRoot, + transaction: pending, + beforeCommit: assertOwned, + }) + await finishKnowledgeFileTransaction({ + root: input.root, + transactionRoot, + transaction: pending, + assertOwned, + }) + await writeKnowledgeIndex(input.root) + } catch (rollbackError) { + throw new AggregateError( + [error, rollbackError], + `knowledge ${action} failed and could not restore the previous files`, + ) + } + throw error + } + candidate.status = target === 'candidate' ? 'promoted' : 'candidate-ready' + candidate.updatedAt = input.now().toISOString() + state.status = candidate.status + if (target === 'candidate') state.promotedCandidateId = candidate.candidateId + else delete state.promotedCandidateId + delete state.blockedReason + state.updatedAt = input.now().toISOString() + await saveState(runDir, state, input.onState) + await ensureCandidateTransitionEvent(runDir, candidateRef, target) + let finalHash = await hashKnowledgeBase(input.root) + if (finalHash !== desiredHash) { + throw new Error(`knowledge ${action} changed before its result was returned`) + } + const mutation = Object.freeze({ + target, + beforeHash: transactionId ? sourceKnowledgeHash(candidateRef, target) : currentHash, + afterHash: finalHash, + changed: transactionId !== null, + transactionId, + recovered: recovered !== undefined, + }) satisfies KnowledgeImprovementMutationReceipt + const activationResult = input.activation + ? await persistKnowledgeActivationResult( + runDir, + candidateRef, + input.activation, + target, + mutation, + ) + : undefined + if ((await hashKnowledgeBase(input.root)) !== finalHash) { + throw new Error(`knowledge ${action} changed while its result was persisted`) + } + if (pending) { + await finishKnowledgeFileTransaction({ + root: input.root, + transactionRoot, + transaction: pending, + assertOwned, + }) + } + finalHash = await hashKnowledgeBase(input.root) + if (finalHash !== desiredHash) { + throw new Error(`knowledge ${action} changed before its result was returned`) + } + return candidateTransitionResult( + input, + candidate, + target === 'candidate', + false, + mutation, + activationResult, + ) + }, + { + staleMs: input.leaseTtlMs, + resumeTransaction: { + purpose, + recoveryOwner, + validate: (transaction) => + assertCandidateTransitionTransaction(transaction, candidateRef, target), + deferFinish: input.activation !== undefined, + }, + }, + ) +} + +function candidateTransitionConflictReason( + target: KnowledgeImprovementTarget, + candidate: KnowledgeImprovementCandidateRef, + currentHash: string, +): string { + return target === 'candidate' + ? `base changed before promotion: expected ${candidate.baseHash}, got ${currentHash}` + : `knowledge changed before restore: expected ${candidate.candidateHash}, got ${currentHash}` +} + +async function blockCandidateTransition( + input: KnowledgeCandidateTransitionInput, + candidate: KnowledgeImprovementCandidateRecord, + target: KnowledgeImprovementTarget, + reason: string, + currentHash: string, +): Promise { + if (input.state.status === 'promoted') { + candidate.status = 'blocked' + candidate.updatedAt = input.now().toISOString() + delete input.state.promotedCandidateId + } + await blockRun(input.runDir, input.state, reason, input.onState, input.now) + await appendLedger(input.runDir, { + type: target === 'candidate' ? 'promotion.blocked' : 'restore.blocked', + runId: input.candidateRef.runId, + candidateId: candidate.candidateId, + reason, + }) + return candidateTransitionResult(input, candidate, false, true, { + target, + beforeHash: currentHash, + afterHash: currentHash, + changed: false, + transactionId: null, + recovered: false, + }) +} + +function candidateTransitionResult( + input: KnowledgeCandidateTransitionInput, + candidate: KnowledgeImprovementCandidateRecord, + promoted: boolean, + blocked: boolean, + mutation: KnowledgeImprovementMutationReceipt, + activationResult?: AgentImprovementActivationResult, +): KnowledgeImprovementMutationResult { + return { + runId: input.candidateRef.runId, + state: input.state, + candidate, + ...(input.lifecycle ? { lifecycle: input.lifecycle } : {}), + mutation, + ...(activationResult ? { activationResult } : {}), + promoted, + blocked, + } +} + +function knowledgeCandidateTransitionPurpose( + candidate: KnowledgeImprovementCandidateRef, + target: KnowledgeImprovementTarget, +): string { + const action = target === 'candidate' ? 'promotion' : 'restore' + return `knowledge-${action}:${contentHash(candidate)}` +} + +export async function knowledgeFilePlanEntries( + sourceRoot: string, + targetRoot: string, +): Promise { + const [before, after] = await Promise.all([ + knowledgeHashEntries(sourceRoot), + knowledgeHashEntries(targetRoot), + ]) + const beforeByPath = new Map(before.map((entry) => [entry.path, entry])) + const afterByPath = new Map(after.map((entry) => [entry.path, entry])) + const paths = [ + ...new Set([...before.map((entry) => entry.path), ...after.map((entry) => entry.path)]), + ].sort((left, right) => left.localeCompare(right)) + return paths.map((path) => { + assertKnowledgeMutationPath(path) + const beforeEntry = beforeByPath.get(path) + const afterEntry = afterByPath.get(path) + return { + path, + beforeHash: beforeEntry?.transactionHash ?? null, + afterHash: afterEntry?.transactionHash ?? null, + ...(beforeEntry ? { beforeMode: beforeEntry.mode } : {}), + ...(afterEntry ? { afterMode: afterEntry.mode } : {}), + } + }) +} + +async function knowledgePlanMutations( + targetRoot: string, + plan: readonly KnowledgeFileTransactionPlanEntry[], +): Promise { + return Promise.all( + plan.map(async (entry) => { + if (entry.afterHash === null) return { path: entry.path, content: null } + const file = await readRegularFileWithinRoot(targetRoot, entry.path) + const actualHash = createHash('sha256').update(file.bytes).digest('hex') + if (actualHash !== entry.afterHash || file.mode !== entry.afterMode) { + throw new Error(`knowledge target file changed before activation: ${entry.path}`) + } + return { path: entry.path, content: file.bytes, mode: file.mode } + }), + ) +} + +function assertCandidateTransitionPlan( + plan: readonly KnowledgeFileTransactionPlanEntry[], + candidate: KnowledgeImprovementCandidateRef, + target: KnowledgeImprovementTarget, +): void { + const approvedDirection = target === 'candidate' ? plan : reverseKnowledgeFilePlan(plan) + const actualPlanHash = knowledgeFileTransactionPlanHash(approvedDirection) + if (actualPlanHash !== candidate.promotionPlanHash) { + throw new Error( + `knowledge candidate plan changed after approval: expected ${candidate.promotionPlanHash}, got ${actualPlanHash}`, + ) + } +} + +function assertCandidateTransitionTransaction( + transaction: KnowledgeFileTransaction, + candidate: KnowledgeImprovementCandidateRef, + target: KnowledgeImprovementTarget, +): void { + assertCandidateTransitionPlan(transaction.entries, candidate, target) +} + +function reverseKnowledgeFilePlan( + plan: readonly KnowledgeFileTransactionPlanEntry[], +): KnowledgeFileTransactionPlanEntry[] { + return plan.map((entry) => ({ + path: entry.path, + beforeHash: entry.afterHash, + afterHash: entry.beforeHash, + ...(entry.afterMode === undefined ? {} : { beforeMode: entry.afterMode }), + ...(entry.beforeMode === undefined ? {} : { afterMode: entry.beforeMode }), + })) +} + +async function ensureCandidateTransitionEvent( + runDir: string, + candidateRef: KnowledgeImprovementCandidateRef, + target: KnowledgeImprovementTarget, +): Promise { + if (await hasCandidateTransitionEvent(runDir, candidateRef, target)) return + await appendLedger(runDir, { + type: target === 'candidate' ? 'candidate.promoted' : 'candidate.restored', + runId: candidateRef.runId, + candidateId: candidateRef.candidateId, + candidateHash: candidateRef.candidateHash, + evidenceHash: candidateRef.evidenceHash, + promotionPlanHash: candidateRef.promotionPlanHash, + }) +} + +async function hasCandidateTransitionEvent( + runDir: string, + candidateRef: KnowledgeImprovementCandidateRef, + target: KnowledgeImprovementTarget, +): Promise { + const eventType = target === 'candidate' ? 'candidate.promoted' : 'candidate.restored' + let matched = false + for (const row of await loadKnowledgeImprovementEventsFromRun(runDir)) { + if (row.type !== eventType || row.candidateId !== candidateRef.candidateId) continue + if ( + row.runId !== candidateRef.runId || + row.candidateHash !== candidateRef.candidateHash || + row.evidenceHash !== candidateRef.evidenceHash || + row.promotionPlanHash !== candidateRef.promotionPlanHash + ) { + throw new Error('persisted knowledge activation event conflicts with the approved candidate') + } + matched = true + } + return matched +} diff --git a/src/kb-improvement/workspace.ts b/src/kb-improvement/workspace.ts new file mode 100644 index 0000000..d830d66 --- /dev/null +++ b/src/kb-improvement/workspace.ts @@ -0,0 +1,477 @@ +import { createHash } from 'node:crypto' +import { cp, lstat, mkdir, mkdtemp, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { dirname, join, relative, resolve } from 'node:path' +import { canonicalJson, contentHash } from '@tangle-network/agent-eval' +import { + isMissingFile, + listRegularFilesWithinRoot, + readRegularFileWithinRoot, + renameDurable, + withSafeDirectory, +} from '../durable-fs' +import { sha256, stableId } from '../ids' +import { writeKnowledgeIndex } from '../indexer' +import { withKnowledgeRead } from '../mutation-lock' +import { layoutFor } from '../store' +import { immutableJsonValue } from './activation' +import type { + KnowledgeImprovementCandidateRecord, + KnowledgeImprovementCandidateRef, + KnowledgeImprovementEvidence, + KnowledgeImprovementResult, + KnowledgeImprovementRunState, + KnowledgeImprovementTarget, + ResolvedKnowledgeImprovementCandidate, + ResolvedKnowledgeImprovementComparison, + UseKnowledgeImprovementCandidateOptions, +} from './contracts' +import { + KnowledgeImprovementCandidateRefSchema, + KnowledgeImprovementEvidenceSchema, + safePathSegmentSchema, +} from './contracts' +import { + assertExactCandidatePlatform, + candidateEvidenceRelativePath, + loadKnowledgeImprovementStateFromRun, + withKnowledgeImprovementRun, +} from './state' + +/** Freeze the exact knowledge bytes and measured evidence a later approval may promote. */ +export function knowledgeImprovementCandidateRef( + result: Pick, +): KnowledgeImprovementCandidateRef { + if (!result.candidate) throw new Error('knowledge improvement result has no candidate') + return candidateRefFor(result.runId, result.state, result.candidate) +} + +/** Use both frozen sides of one measured comparison in isolated, integrity-checked copies. */ +export async function withKnowledgeImprovementComparison( + options: UseKnowledgeImprovementCandidateOptions, + use: (comparison: ResolvedKnowledgeImprovementComparison) => Promise | T, +): Promise { + assertExactCandidatePlatform() + const reference = Object.freeze(KnowledgeImprovementCandidateRefSchema.parse(options.candidate)) + return withKnowledgeImprovementRun(options.root, reference.runId, false, async (runDir) => { + const state = await loadKnowledgeImprovementStateFromRun(options.root, reference.runId, runDir) + return withMeasuredCandidateSnapshot(options.root, runDir, state, reference, (resolved) => + withBaselineSnapshot(runDir, reference.baseHash, (baselineRoot) => + withIsolatedKnowledgeCopy(baselineRoot, reference.baseHash, 'baseline', (baseline) => + withIsolatedKnowledgeCopy( + resolved.root, + reference.candidateHash, + 'candidate', + (candidate) => + use( + Object.freeze({ + reference, + evaluation: immutableJsonValue(structuredClone(resolved.evidence.evaluation)), + baseline: Object.freeze({ root: baseline, hash: reference.baseHash }), + candidate: Object.freeze({ root: candidate, hash: reference.candidateHash }), + }), + ), + ), + ), + ), + ) + }) +} + +/** Use the frozen candidate side of one measured comparison. */ +export async function withKnowledgeImprovementCandidate( + options: UseKnowledgeImprovementCandidateOptions, + use: (candidate: ResolvedKnowledgeImprovementCandidate) => Promise | T, +): Promise { + assertExactCandidatePlatform() + const candidateRef = Object.freeze( + KnowledgeImprovementCandidateRefSchema.parse(options.candidate), + ) + return withKnowledgeImprovementRun(options.root, candidateRef.runId, false, async (runDir) => { + const state = await loadKnowledgeImprovementStateFromRun( + options.root, + candidateRef.runId, + runDir, + ) + return withMeasuredCandidateSnapshot(options.root, runDir, state, candidateRef, (resolved) => + withIsolatedKnowledgeCopy(resolved.root, candidateRef.candidateHash, 'candidate', (root) => + use( + Object.freeze({ + root, + candidate: candidateRef, + evaluation: immutableJsonValue(structuredClone(resolved.evidence.evaluation)), + }), + ), + ), + ) + }) +} + +export function candidateRefFor( + runId: string, + state: KnowledgeImprovementRunState, + candidate: KnowledgeImprovementCandidateRecord, +): KnowledgeImprovementCandidateRef { + if (candidate.status !== 'candidate-ready' && candidate.status !== 'promoted') { + throw new Error(`knowledge candidate '${candidate.candidateId}' is not ready`) + } + return candidateIdentityFor(runId, state, candidate) +} + +export function candidateIdentityFor( + runId: string, + state: KnowledgeImprovementRunState, + candidate: KnowledgeImprovementCandidateRecord, +): KnowledgeImprovementCandidateRef { + if (!candidate.candidateHash) { + throw new Error(`knowledge candidate '${candidate.candidateId}' has no content hash`) + } + if (!candidate.evidenceHash) { + throw new Error(`knowledge candidate '${candidate.candidateId}' has no evidence hash`) + } + if (!candidate.promotionPlanHash) { + throw new Error(`knowledge candidate '${candidate.candidateId}' has no promotion plan hash`) + } + return Object.freeze({ + kind: 'knowledge-improvement-candidate', + runId, + candidateId: candidate.candidateId, + goalHash: sha256(state.goal), + baseHash: candidate.baseHash, + candidateHash: candidate.candidateHash, + evidenceHash: candidate.evidenceHash, + promotionPlanHash: candidate.promotionPlanHash, + }) +} + +export async function withMeasuredCandidateSnapshot( + liveRoot: string, + runDir: string, + state: KnowledgeImprovementRunState, + candidateRef: KnowledgeImprovementCandidateRef, + use: (snapshot: { + root: string + candidate: KnowledgeImprovementCandidateRecord + evidence: KnowledgeImprovementEvidence + }) => Promise | T, +): Promise { + assertStateIdentity(liveRoot, candidateRef, state) + const candidate = state.candidates.find((entry) => entry.candidateId === candidateRef.candidateId) + if (!candidate) { + throw new Error(`knowledge candidate '${candidateRef.candidateId}' does not exist`) + } + const expectedRef = candidateRefFor(candidateRef.runId, state, candidate) + if (canonicalJson(expectedRef) !== canonicalJson(candidateRef)) { + throw new Error('knowledge candidate approval does not match the measured candidate') + } + const evidence = await assertCandidateEvidence(runDir, candidateRef) + const relativePath = join( + 'candidates', + candidate.candidateId, + 'snapshots', + candidateRef.candidateHash, + ) + return withSafeDirectory(runDir, relativePath, false, async (root) => { + if ((await hashKnowledgeBase(root)) !== candidateRef.candidateHash) { + throw new Error('knowledge candidate snapshot changed after approval') + } + const result = await use({ root, candidate, evidence }) + if ((await hashKnowledgeBase(root)) !== candidateRef.candidateHash) { + throw new Error('knowledge candidate snapshot changed during use') + } + return result + }) +} + +async function withIsolatedKnowledgeCopy( + sourceRoot: string, + expectedHash: string, + target: KnowledgeImprovementTarget, + use: (root: string) => Promise | T, +): Promise { + const isolationRoot = await mkdtemp(join(tmpdir(), 'agent-knowledge-snapshot-')) + const snapshotRoot = join(isolationRoot, 'snapshot') + try { + await copyKnowledgeWorkspace(sourceRoot, snapshotRoot) + if ((await hashKnowledgeBase(snapshotRoot)) !== expectedHash) { + throw new Error(`isolated knowledge ${target} does not match its measured content`) + } + const result = await use(snapshotRoot) + if ((await hashKnowledgeBase(snapshotRoot)) !== expectedHash) { + throw new Error(`knowledge ${target} snapshot changed during use`) + } + return result + } finally { + await rm(isolationRoot, { recursive: true, force: true }) + } +} + +export async function assertCandidateEvidence( + runDir: string, + candidate: KnowledgeImprovementCandidateRef, +): Promise { + const evidence = KnowledgeImprovementEvidenceSchema.parse( + JSON.parse( + ( + await readRegularFileWithinRoot( + runDir, + candidateEvidenceRelativePath(candidate.candidateId), + ) + ).bytes.toString('utf8'), + ), + ) + const actualHash = contentHash(evidence) + if (actualHash !== candidate.evidenceHash) { + throw new Error( + `knowledge candidate evidence changed after approval: expected ${candidate.evidenceHash}, got ${actualHash}`, + ) + } + if ( + evidence.runId !== candidate.runId || + evidence.candidateId !== candidate.candidateId || + evidence.goalHash !== candidate.goalHash || + evidence.baseHash !== candidate.baseHash || + evidence.candidateHash !== candidate.candidateHash || + evidence.promotionPlanHash !== candidate.promotionPlanHash || + evidence.evaluation.passed !== true + ) { + throw new Error('knowledge candidate evidence does not match the approved candidate') + } + return evidence +} + +export function assertStateIdentity( + root: string, + candidateRef: KnowledgeImprovementCandidateRef, + state: KnowledgeImprovementRunState, +): void { + if (state.runId !== candidateRef.runId) { + throw new Error('knowledge candidate run identity does not match persisted state') + } + if (resolve(state.root) !== resolve(root)) { + throw new Error('knowledge candidate root does not match persisted state') + } + if (sha256(state.goal) !== candidateRef.goalHash) { + throw new Error('knowledge candidate goal does not match persisted state') + } + if (state.baseHash !== candidateRef.baseHash) { + throw new Error('knowledge candidate base does not match persisted state') + } +} + +export async function createCandidateWorkspace( + runDir: string, + state: KnowledgeImprovementRunState, + root: string, + now: () => Date, +): Promise { + const iteration = state.candidates.length + 1 + const candidateId = stableId('kcand', `${state.runId}:${iteration}:${now().toISOString()}`) + const candidateRoot = candidateWorkspacePath(runDir, candidateId) + await copyKnowledgeWorkspace(root, candidateRoot) + const createdAt = now().toISOString() + return { + iteration, + candidateId, + baseHash: state.baseHash, + status: 'running', + createdAt, + updatedAt: createdAt, + } +} + +function candidateWorkspacePath(runDir: string, candidateId: string): string { + return join(runDir, 'candidates', safePathSegmentSchema.parse(candidateId), 'workspace') +} + +function baselineSnapshotPath(runDir: string): string { + return join(runDir, 'baseline') +} + +export async function createBaselineSnapshot( + runDir: string, + root: string, + expectedHash: string, +): Promise { + const target = baselineSnapshotPath(runDir) + try { + await assertBaselineSnapshot(runDir, expectedHash) + return + } catch (error) { + if (!isMissingFile(error)) throw error + } + const preparation = await mkdtemp(join(runDir, 'baseline-prepare-')) + let activated = false + try { + await copyKnowledgeWorkspace(root, preparation) + const actualHash = await hashKnowledgeBase(preparation) + if (actualHash !== expectedHash) { + throw new Error( + `knowledge base changed while baseline was frozen: expected ${expectedHash}, got ${actualHash}`, + ) + } + await renameDurable(preparation, target) + activated = true + } finally { + if (!activated) await rm(preparation, { recursive: true, force: true }) + } +} + +export async function ensureBaselineSnapshot( + runDir: string, + root: string, + expectedHash: string, +): Promise { + try { + await assertBaselineSnapshot(runDir, expectedHash) + } catch (error) { + if (!isMissingFile(error)) throw error + const liveHash = await hashKnowledgeBase(root) + if (liveHash !== expectedHash) { + throw new Error( + 'knowledge improvement baseline snapshot is missing and cannot be reconstructed', + ) + } + await createBaselineSnapshot(runDir, root, expectedHash) + } +} + +async function assertBaselineSnapshot(runDir: string, expectedHash: string): Promise { + await withBaselineSnapshot(runDir, expectedHash, () => undefined) +} + +export async function withBaselineSnapshot( + runDir: string, + expectedHash: string, + use: (baselineRoot: string) => Promise | T, +): Promise { + return withSafeDirectory(runDir, 'baseline', false, async (baselineRoot) => { + const actualHash = await hashKnowledgeBase(baselineRoot) + if (actualHash !== expectedHash) { + throw new Error( + `knowledge improvement baseline changed: expected ${expectedHash}, got ${actualHash}`, + ) + } + return use(baselineRoot) + }) +} + +export async function withFrozenCandidateWorkspace( + runDir: string, + candidate: KnowledgeImprovementCandidateRecord, + candidateRoot: string, + use: (snapshot: { root: string; hash: string }) => Promise | T, +): Promise { + const snapshotsPath = join( + 'candidates', + safePathSegmentSchema.parse(candidate.candidateId), + 'snapshots', + ) + return withSafeDirectory(runDir, snapshotsPath, true, async (snapshotsDir) => { + const preparation = await mkdtemp(join(snapshotsDir, 'prepare-')) + let activated = false + try { + await copyKnowledgeWorkspace(candidateRoot, preparation) + const hash = await hashKnowledgeBase(preparation) + try { + const result = await withSafeDirectory(snapshotsDir, hash, false, async (existing) => { + if ((await hashKnowledgeBase(existing)) !== hash) { + throw new Error('knowledge candidate snapshot does not match its content identity') + } + return use({ root: existing, hash }) + }) + await rm(preparation, { recursive: true, force: true }) + activated = true + return result + } catch (error) { + if (!isMissingFile(error)) throw error + } + await renameDurable(preparation, join(snapshotsDir, hash)) + activated = true + return withSafeDirectory(snapshotsDir, hash, false, (root) => use({ root, hash })) + } finally { + if (!activated) await rm(preparation, { recursive: true, force: true }) + } + }) +} + +export function clearCandidateMeasurement(candidate: KnowledgeImprovementCandidateRecord): void { + delete candidate.candidateHash + delete candidate.evidenceHash + delete candidate.promotionPlanHash +} + +async function copyKnowledgeWorkspace(sourceRoot: string, targetRoot: string): Promise { + await rm(targetRoot, { recursive: true, force: true }) + await mkdir(join(targetRoot, 'knowledge'), { recursive: true }) + await mkdir(join(targetRoot, 'raw', 'sources'), { recursive: true }) + await copyIfExists(join(sourceRoot, 'knowledge'), join(targetRoot, 'knowledge')) + await copyIfExists(join(sourceRoot, 'raw'), join(targetRoot, 'raw')) + await copyIfExists( + join(layoutFor(sourceRoot).cacheDir, 'sources.json'), + join(layoutFor(targetRoot).cacheDir, 'sources.json'), + ) + await writeKnowledgeIndex(targetRoot) +} + +async function copyIfExists(source: string, target: string): Promise { + let sourceStat: Awaited> + try { + sourceStat = await lstat(source) + } catch (error) { + if (isMissingFile(error)) return + throw error + } + if (!sourceStat.isDirectory() && !sourceStat.isFile()) { + throw new Error(`knowledge surface contains an unsupported filesystem entry: ${source}`) + } + await mkdir(dirname(target), { recursive: true }) + await cp(source, target, { recursive: sourceStat.isDirectory(), dereference: false }) +} + +export async function hashKnowledgeBase(root: string): Promise { + return withKnowledgeRead(root, () => hashKnowledgeBaseUnlocked(root)) +} + +async function hashKnowledgeBaseUnlocked(root: string): Promise { + const entries = await knowledgeHashEntries(root) + return sha256(JSON.stringify(entries.map(({ path, hash, mode }) => ({ path, hash, mode })))) +} + +export interface KnowledgeFileIdentity { + path: string + hash: string + transactionHash: string + mode: number +} + +export async function knowledgeHashEntries(root: string): Promise { + const entries: KnowledgeFileIdentity[] = [] + for (const rel of ['knowledge', 'raw']) { + try { + for (const file of await listRegularFilesWithinRoot(root, rel)) { + entries.push(knowledgeFileIdentity(file.path, file.bytes, file.mode)) + } + } catch (error) { + if (!isMissingFile(error)) throw error + } + } + const sourceRegistry = relative(root, layoutFor(root).sourceRegistryPath).replace(/\\/g, '/') + try { + const file = await readRegularFileWithinRoot(root, sourceRegistry) + entries.push(knowledgeFileIdentity(sourceRegistry, file.bytes, file.mode)) + } catch (error) { + if (!isMissingFile(error)) throw error + } + entries.sort((a, b) => a.path.localeCompare(b.path)) + return entries +} + +function knowledgeFileIdentity(path: string, bytes: Buffer, mode: number): KnowledgeFileIdentity { + return { + path, + hash: sha256(bytes.toString('base64')), + transactionHash: createHash('sha256').update(bytes).digest('hex'), + mode, + } +} diff --git a/src/rag-eval.ts b/src/rag-eval.ts index c0968de..1d49a38 100644 --- a/src/rag-eval.ts +++ b/src/rag-eval.ts @@ -1,916 +1,34 @@ -import type { JsonValue, JudgeConfig, Scenario } from '@tangle-network/agent-eval/campaign' -import { groundClaimInText } from './claim-grounding' -import { lintKnowledgeIndex } from './lint' -import type { RagAnswerQualityResult, RagGapFinding } from './rag-improvement-loop' -import type { KnowledgeIndex } from './types' -import { validateKnowledgeIndex } from './validate' - -export type RagEvalProvider = - | 'agent-knowledge' - | 'ragas' - | 'deepeval' - | 'trulens' - | 'ragchecker' - | 'custom' - -export type RagEvalMetricKey = - | 'context_precision' - | 'context_recall' - | 'context_relevance' - | 'context_sufficiency' - | 'faithfulness' - | 'groundedness' - | 'answer_relevance' - | 'answer_correctness' - | 'citation_support' - | 'abstention' - | 'unsupported_answer_rate' - -export type RagEvalSlice = - | 'known-answer' - | 'paraphrase' - | 'distractor' - | 'freshness' - | 'multi-source' - | 'unanswerable' - | 'long-tail' - | 'custom' - -export interface RagEvalContext { - id: string - text: string - rank?: number - pageId?: string - sourceId?: string - anchorId?: string - stale?: boolean - metadata?: Record -} - -export interface RagEvalCitation { - id: string - claimId?: string - contextId?: string - pageId?: string - sourceId?: string - anchorId?: string - quote?: string - metadata?: Record -} - -export interface RagEvalClaim { - id: string - text: string - citationIds?: readonly string[] - metadata?: Record -} - -export interface RagRequiredContext { - id?: string - text?: string - pageId?: string - sourceId?: string - anchorId?: string -} - -export interface RagAnswerEvalScenario extends Scenario { - kind: 'rag-answer-eval' - query: string - referenceAnswer?: string - expectedClaims?: readonly string[] - forbiddenClaims?: readonly string[] - requiredContext?: readonly RagRequiredContext[] - unanswerable?: boolean - requireCitations?: boolean - slices?: readonly RagEvalSlice[] - thresholds?: Partial> -} - -export interface ExternalRagEvalScore { - provider: RagEvalProvider | string - scores: Record - reasons?: Record - metadata?: Record -} - -export interface RagAnswerEvalArtifact { - query: string - answer: string - contexts: readonly RagEvalContext[] - claims?: readonly RagEvalClaim[] - citations?: readonly RagEvalCitation[] - abstained?: boolean - durationMs?: number - costUsd?: number - externalScores?: readonly ExternalRagEvalScore[] - metadata?: Record -} - -export interface RagAnswerMetricSummary { - metrics: Record - composite: number - passed: boolean - findings: readonly RagGapFinding[] - claimCount: number - supportedClaimCount: number - citedClaimCount: number - supportedCitationCount: number - matchedRequiredContextCount: number - requiredContextCount: number - providerScores: Record> -} - -export interface RagAnswerQualityJudgeOptions { - name?: string - thresholds?: Partial> - weights?: Partial> - externalScorePolicy?: 'prefer-external' | 'deterministic-first' - minClaimSupport?: number -} - -export interface RagAnswerEvalCase { - scenario: RagAnswerEvalScenario - artifact: RagAnswerEvalArtifact -} - -export interface RagAnswerQualityHookOptions { - scenarios: readonly RagAnswerEvalScenario[] - run: (scenario: RagAnswerEvalScenario) => MaybePromise - externalEvaluator?: ( - item: RagAnswerEvalCase, - ) => MaybePromise - thresholds?: Partial> - weights?: Partial> -} - -export interface RagCalibrationOptions { - scenario: RagAnswerEvalScenario - strong: RagAnswerEvalArtifact - weak: RagAnswerEvalArtifact - judge?: JudgeConfig - minStrongScore?: number - maxWeakScore?: number - signal?: AbortSignal -} - -export interface RagCalibrationResult { - passed: boolean - strongScore: number - weakScore: number - gap: number -} - -export interface KnowledgeBaseQualityOptions { - now?: Date - strict?: boolean - minCitationRate?: number - maxStaleSourceRate?: number -} - -export interface KnowledgeBaseQualityReport { - ok: boolean - metrics: { - page_count: number - source_count: number - citation_rate: number - source_backed_page_rate: number - stale_source_rate: number - duplicate_source_hash_rate: number - lint_error_count: number - lint_warning_count: number - } - findings: readonly RagGapFinding[] -} - -type MaybePromise = T | Promise - -const defaultThresholds: Partial> = { - context_precision: 0.5, - context_recall: 0.8, - context_relevance: 0.5, - context_sufficiency: 0.8, - faithfulness: 0.9, - answer_relevance: 0.7, - citation_support: 0.9, - abstention: 1, - unsupported_answer_rate: 0, -} - -const defaultWeights: Partial> = { - context_relevance: 1, - context_sufficiency: 1, - faithfulness: 2, - answer_relevance: 1, - citation_support: 1, - abstention: 1, -} - -const metricAliases: Record = { - answer_correctness: 'answer_correctness', - answer_relevance: 'answer_relevance', - answer_relevancy: 'answer_relevance', - answerrelevancy: 'answer_relevance', - context_precision: 'context_precision', - context_recall: 'context_recall', - context_relevance: 'context_relevance', - context_relevancy: 'context_relevance', - context_sufficiency: 'context_sufficiency', - contextual_precision: 'context_precision', - contextual_recall: 'context_recall', - contextual_relevancy: 'context_relevance', - faithfulness: 'faithfulness', - groundedness: 'groundedness', - claim_recall: 'context_recall', - claim_precision: 'faithfulness', - citation_support: 'citation_support', - abstention: 'abstention', - unsupported_answer_rate: 'unsupported_answer_rate', -} - -export function ragAnswerQualityJudge( - options: RagAnswerQualityJudgeOptions = {}, -): JudgeConfig { - return { - name: options.name ?? 'rag-answer-quality', - dimensions: [ - { key: 'context_precision', description: 'share of retrieved context that is useful' }, - { key: 'context_recall', description: 'share of required evidence present in context' }, - { key: 'context_relevance', description: 'context relevance to the query and answer target' }, - { key: 'context_sufficiency', description: 'whether context is enough to answer' }, - { key: 'faithfulness', description: 'share of answer claims supported by retrieved context' }, - { key: 'answer_relevance', description: 'answer addresses the user query' }, - { key: 'answer_correctness', description: 'answer contains expected claims' }, - { key: 'citation_support', description: 'citations support the claims they cite' }, - { key: 'abstention', description: 'answer abstains exactly when the case is unanswerable' }, - ], - appliesTo: (scenario) => scenario.kind === 'rag-answer-eval', - async score({ artifact, scenario }) { - const summary = scoreRagAnswerArtifact(artifact, scenario, options) - return { - dimensions: summary.metrics, - composite: summary.composite, - notes: summary.findings.map((finding) => finding.message).join('; '), - } - }, - } -} - -export function scoreRagAnswerArtifact( - artifact: RagAnswerEvalArtifact, - scenario: RagAnswerEvalScenario, - options: RagAnswerQualityJudgeOptions = {}, -): RagAnswerMetricSummary { - const claims = normalizeClaims(artifact) - const abstained = artifact.abstained ?? looksLikeAbstention(artifact.answer) - const requiredTargets = requiredContextTargets(scenario) - const matchedRequiredContextCount = requiredTargets.filter((target) => - artifact.contexts.some((context) => contextMatchesTarget(context, target)), - ).length - const requiredContextCount = requiredTargets.length - const contextRecall = - requiredContextCount === 0 - ? neutralScore(Boolean(scenario.unanswerable) || artifact.contexts.length > 0) - : matchedRequiredContextCount / requiredContextCount - const relevantContextCount = artifact.contexts.filter((context) => - contextIsRelevant(context, scenario), - ).length - const contextPrecision = - artifact.contexts.length === 0 - ? scenario.unanswerable - ? 1 - : 0 - : relevantContextCount / artifact.contexts.length - const contextRelevance = - artifact.contexts.length === 0 - ? scenario.unanswerable - ? 1 - : 0 - : average(artifact.contexts.map((context) => contextRelevanceScore(context, scenario))) - const contextSufficiency = requiredContextCount === 0 ? contextRelevance : contextRecall - const support = claims.map((claim) => claimSupport(claim.text, artifact.contexts, options)) - const supportedClaimCount = support.filter(Boolean).length - const faithfulness = - claims.length === 0 ? (abstained ? 1 : 0) : supportedClaimCount / Math.max(1, claims.length) - const citation = scoreCitations(claims, artifact, scenario, options) - const answerRelevance = scoreAnswerRelevance(artifact, scenario, abstained) - const answerCorrectness = scoreAnswerCorrectness(artifact, scenario, abstained) - const abstention = scenario.unanswerable ? (abstained ? 1 : 0) : abstained ? 0 : 1 - const unsupportedAnswerRate = - claims.length === 0 ? (abstained ? 0 : 1) : 1 - supportedClaimCount / Math.max(1, claims.length) - - const deterministicMetrics: Record = { - context_precision: clamp01(contextPrecision), - context_recall: clamp01(contextRecall), - context_relevance: clamp01(contextRelevance), - context_sufficiency: clamp01(contextSufficiency), - faithfulness: clamp01(faithfulness), - groundedness: clamp01(faithfulness), - answer_relevance: clamp01(answerRelevance), - answer_correctness: clamp01(answerCorrectness), - citation_support: clamp01(citation.support), - abstention: clamp01(abstention), - unsupported_answer_rate: clamp01(unsupportedAnswerRate), - } - const providerScores = normalizeExternalRagScores(artifact.externalScores ?? []) - const metrics = mergeMetrics(deterministicMetrics, providerScores, options.externalScorePolicy) - const thresholds = { ...defaultThresholds, ...scenario.thresholds, ...options.thresholds } - const findings = diagnoseRagAnswerFailure(metrics, scenario, thresholds) - const composite = weightedComposite(metrics, options.weights ?? defaultWeights) - - return { - metrics, - composite, - passed: findings.every( - (finding) => finding.severity !== 'error' && finding.severity !== 'critical', - ), - findings, - claimCount: claims.length, - supportedClaimCount, - citedClaimCount: citation.citedClaimCount, - supportedCitationCount: citation.supportedCitationCount, - matchedRequiredContextCount, - requiredContextCount, - providerScores, - } -} - -export function diagnoseRagAnswerFailure( - metrics: Record, - scenario: RagAnswerEvalScenario, - thresholds: Partial> = defaultThresholds, -): RagGapFinding[] { - const findings: RagGapFinding[] = [] - if (below(metrics.context_recall, thresholds.context_recall)) { - findings.push({ - id: `${scenario.id}:context-recall`, - kind: scenario.slices?.includes('multi-source') - ? 'missing-multihop-evidence' - : 'retrieval-miss', - severity: 'error', - scenarioId: scenario.id, - message: 'Required evidence was not present in retrieved context.', - evidence: { context_recall: metrics.context_recall }, - }) - } - if (below(metrics.context_precision, thresholds.context_precision)) { - findings.push({ - id: `${scenario.id}:context-precision`, - kind: 'retrieval-noise', - severity: 'warning', - scenarioId: scenario.id, - message: 'Retrieved context contains too much irrelevant material.', - evidence: { context_precision: metrics.context_precision }, - }) - } - if (below(metrics.faithfulness, thresholds.faithfulness)) { - findings.push({ - id: `${scenario.id}:faithfulness`, - kind: 'generator-unsupported-claim', - severity: 'error', - scenarioId: scenario.id, - message: 'The answer contains claims not supported by retrieved context.', - evidence: { faithfulness: metrics.faithfulness }, - }) - } - if (below(metrics.citation_support, thresholds.citation_support)) { - findings.push({ - id: `${scenario.id}:citation-support`, - kind: 'citation-mismatch', - severity: 'error', - scenarioId: scenario.id, - message: 'Citations do not support the claims they are attached to.', - evidence: { citation_support: metrics.citation_support }, - }) - } - if (below(metrics.abstention, thresholds.abstention)) { - findings.push({ - id: `${scenario.id}:abstention`, - kind: 'incorrect-abstention', - severity: 'error', - scenarioId: scenario.id, - message: scenario.unanswerable - ? 'The system answered a case marked unanswerable.' - : 'The system abstained from an answerable case.', - evidence: { abstention: metrics.abstention }, - }) - } - return findings -} - -export function createRagAnswerQualityHook( - options: RagAnswerQualityHookOptions, -): () => Promise { - return async () => { - const summaries: RagAnswerMetricSummary[] = [] - const findings: RagGapFinding[] = [] - for (const scenario of options.scenarios) { - const initialArtifact = await options.run(scenario) - const external = await options.externalEvaluator?.({ scenario, artifact: initialArtifact }) - const artifact = external - ? { - ...initialArtifact, - externalScores: [ - ...(initialArtifact.externalScores ?? []), - ...(Array.isArray(external) ? external : [external]), - ], - } - : initialArtifact - const summary = scoreRagAnswerArtifact(artifact, scenario, { - thresholds: options.thresholds, - weights: options.weights, - }) - summaries.push(summary) - findings.push(...summary.findings) - } - const metrics = aggregateRagAnswerMetrics(summaries) - return { - passed: findings.length === 0, - metrics, - findings, - metadata: { scenarioCount: options.scenarios.length }, - } - } -} - -export async function calibrateRagAnswerJudge( - options: RagCalibrationOptions, -): Promise { - const judge = options.judge ?? ragAnswerQualityJudge() - const strong = await judge.score({ - artifact: options.strong, - scenario: options.scenario, - signal: options.signal ?? new AbortController().signal, - }) - const weak = await judge.score({ - artifact: options.weak, - scenario: options.scenario, - signal: options.signal ?? new AbortController().signal, - }) - const strongScore = strong.composite - const weakScore = weak.composite - const minStrongScore = options.minStrongScore ?? 0.7 - const maxWeakScore = options.maxWeakScore ?? 0.3 - return { - passed: strongScore >= minStrongScore && weakScore <= maxWeakScore, - strongScore, - weakScore, - gap: strongScore - weakScore, - } -} - -export function normalizeExternalRagScores( - scores: readonly ExternalRagEvalScore[], -): Record> { - const normalized: Record> = {} - for (const item of scores) { - const provider = item.provider || 'custom' - const providerScores = (normalized[provider] ?? {}) as Record - for (const [key, value] of Object.entries(item.scores)) { - const canonical = metricAliases[normalizeMetricName(key)] - if (!canonical) continue - if (Number.isFinite(value)) providerScores[canonical] = clamp01(value) - } - normalized[provider] = providerScores - } - return normalized -} - -export function toRagasEvaluationRows(cases: readonly RagAnswerEvalCase[]) { - return cases.map(({ scenario, artifact }) => ({ - user_input: scenario.query, - response: artifact.answer, - retrieved_contexts: artifact.contexts.map((context) => context.text), - reference: scenario.referenceAnswer, - reference_contexts: requiredContextTexts(scenario), - })) -} - -export function toDeepEvalTestCases(cases: readonly RagAnswerEvalCase[]) { - return cases.map(({ scenario, artifact }) => ({ - input: scenario.query, - actual_output: artifact.answer, - expected_output: scenario.referenceAnswer, - retrieval_context: artifact.contexts.map((context) => context.text), - context: requiredContextTexts(scenario), - })) -} - -export function toTruLensRecords(cases: readonly RagAnswerEvalCase[]) { - return cases.map(({ scenario, artifact }) => ({ - input: scenario.query, - output: artifact.answer, - context: artifact.contexts.map((context) => context.text).join('\n\n'), - })) -} - -export function toRagCheckerRecords(cases: readonly RagAnswerEvalCase[]) { - return cases.map(({ scenario, artifact }) => ({ - query_id: scenario.id, - query: scenario.query, - gt_answer: scenario.referenceAnswer, - response: artifact.answer, - retrieved_context: artifact.contexts.map((context) => ({ - doc_id: context.id, - text: context.text, - })), - claims: normalizeClaims(artifact).map((claim) => claim.text), - })) -} - -export function scoreKnowledgeBaseIndex( - index: KnowledgeIndex, - options: KnowledgeBaseQualityOptions = {}, -): KnowledgeBaseQualityReport { - const validation = validateKnowledgeIndex(index, { strict: options.strict }) - const lintFindings = lintKnowledgeIndex(index) - const now = options.now ?? new Date() - const staleSourceCount = index.sources.filter((source) => { - return source.validUntil ? Date.parse(source.validUntil) < now.getTime() : false - }).length - const sourceHashCounts = new Map() - for (const source of index.sources) { - sourceHashCounts.set(source.contentHash, (sourceHashCounts.get(source.contentHash) ?? 0) + 1) - } - const duplicateSourceHashCount = [...sourceHashCounts.values()].filter( - (count) => count > 1, - ).length - const pagesWithInlineCitations = index.pages.filter( - (page) => sourceRefs(page.text).length > 0, - ).length - const pagesWithSources = index.pages.filter((page) => page.sourceIds.length > 0).length - const metrics = { - page_count: index.pages.length, - source_count: index.sources.length, - citation_rate: index.pages.length === 0 ? 1 : pagesWithInlineCitations / index.pages.length, - source_backed_page_rate: index.pages.length === 0 ? 1 : pagesWithSources / index.pages.length, - stale_source_rate: index.sources.length === 0 ? 0 : staleSourceCount / index.sources.length, - duplicate_source_hash_rate: - index.sources.length === 0 ? 0 : duplicateSourceHashCount / index.sources.length, - lint_error_count: validation.findings.filter((finding) => finding.severity === 'error').length, - lint_warning_count: lintFindings.filter((finding) => finding.severity === 'warning').length, - } - const findings: RagGapFinding[] = [] - if (metrics.lint_error_count > 0) { - findings.push({ - id: 'kb:lint-errors', - kind: 'unknown', - severity: 'critical', - message: `${metrics.lint_error_count} validation error(s) in the knowledge index.`, - evidence: { lint_error_count: metrics.lint_error_count }, - }) - } - if (metrics.citation_rate < (options.minCitationRate ?? 0)) { - findings.push({ - id: 'kb:citation-rate', - kind: 'missing-source', - severity: 'error', - message: 'Inline citation coverage is below the configured minimum.', - evidence: { citation_rate: metrics.citation_rate }, - }) - } - if (metrics.stale_source_rate > (options.maxStaleSourceRate ?? 1)) { - findings.push({ - id: 'kb:stale-source-rate', - kind: 'stale-source', - severity: 'error', - message: 'Stale source rate exceeds the configured maximum.', - evidence: { stale_source_rate: metrics.stale_source_rate }, - }) - } - return { ok: findings.length === 0, metrics, findings } -} - -function requiredContextTargets(scenario: RagAnswerEvalScenario): RagRequiredContext[] { - if (scenario.requiredContext?.length) return [...scenario.requiredContext] - return (scenario.expectedClaims ?? []).map((text) => ({ text })) -} - -function requiredContextTexts(scenario: RagAnswerEvalScenario): string[] { - return requiredContextTargets(scenario).flatMap((target) => (target.text ? [target.text] : [])) -} - -function contextMatchesTarget(context: RagEvalContext, target: RagRequiredContext): boolean { - if (target.id && context.id === target.id) return true - if (target.pageId && context.pageId === target.pageId) return true - if (target.sourceId && context.sourceId === target.sourceId) return true - if (target.anchorId && context.anchorId === target.anchorId) return true - if (target.text && textSupportScore(target.text, context.text) >= 0.7) return true - return false -} - -function contextIsRelevant(context: RagEvalContext, scenario: RagAnswerEvalScenario): boolean { - return contextRelevanceScore(context, scenario) >= 0.3 -} - -function contextRelevanceScore(context: RagEvalContext, scenario: RagAnswerEvalScenario): number { - const targets = [ - scenario.query, - scenario.referenceAnswer, - ...(scenario.expectedClaims ?? []), - ...requiredContextTexts(scenario), - ].filter((value): value is string => Boolean(value?.trim())) - if (targets.length === 0) return 1 - return Math.max(...targets.map((target) => textSupportScore(target, context.text))) -} - -function claimSupport( - claim: string, - contexts: readonly RagEvalContext[], - options: Pick, -): boolean { - if (contexts.length === 0) return false - const combined = contexts.map((context) => context.text).join('\n\n') - const deterministic = groundClaimInText(claim, combined, { - minOverlap: options.minClaimSupport ?? 0.7, - }) - return ( - deterministic.grounded || textSupportScore(claim, combined) >= (options.minClaimSupport ?? 0.7) - ) -} - -function scoreCitations( - claims: readonly RagEvalClaim[], - artifact: RagAnswerEvalArtifact, - scenario: RagAnswerEvalScenario, - options: Pick, -): { support: number; citedClaimCount: number; supportedCitationCount: number } { - if (claims.length === 0) - return { support: scenario.unanswerable ? 1 : 0, citedClaimCount: 0, supportedCitationCount: 0 } - const citations = artifact.citations ?? [] - const citedClaims = claims.filter((claim) => citationsForClaim(claim, citations).length > 0) - if (citedClaims.length === 0) { - return { - support: scenario.requireCitations ? 0 : 1, - citedClaimCount: 0, - supportedCitationCount: 0, - } - } - let supportedCitationCount = 0 - for (const claim of citedClaims) { - const citedContexts = contextsForClaim(claim, citations, artifact.contexts) - if (claimSupport(claim.text, citedContexts, options)) supportedCitationCount += 1 - } - return { - support: supportedCitationCount / citedClaims.length, - citedClaimCount: citedClaims.length, - supportedCitationCount, - } -} - -function citationsForClaim( - claim: RagEvalClaim, - citations: readonly RagEvalCitation[], -): RagEvalCitation[] { - const claimCitationIds = new Set(claim.citationIds ?? []) - return citations.filter((citation) => { - return citation.claimId === claim.id || (citation.id && claimCitationIds.has(citation.id)) - }) -} - -function contextsForClaim( - claim: RagEvalClaim, - citations: readonly RagEvalCitation[], - contexts: readonly RagEvalContext[], -): readonly RagEvalContext[] { - const matchedCitations = citationsForClaim(claim, citations) - const matchedContexts = contexts.filter((context) => - matchedCitations.some((citation) => { - if (citation.contextId && citation.contextId === context.id) return true - if (citation.pageId && citation.pageId === context.pageId) return true - if (citation.sourceId && citation.sourceId === context.sourceId) return true - if (citation.anchorId && citation.anchorId === context.anchorId) return true - return false - }), - ) - return matchedContexts.length > 0 ? matchedContexts : contexts -} - -function scoreAnswerRelevance( - artifact: RagAnswerEvalArtifact, - scenario: RagAnswerEvalScenario, - abstained: boolean, -): number { - if (scenario.unanswerable) return abstained ? 1 : 0 - if (abstained) return 0 - const targets = [ - scenario.referenceAnswer, - ...(scenario.expectedClaims ?? []), - scenario.query, - ].filter((value): value is string => Boolean(value?.trim())) - return Math.max(...targets.map((target) => textSupportScore(target, artifact.answer))) -} - -function scoreAnswerCorrectness( - artifact: RagAnswerEvalArtifact, - scenario: RagAnswerEvalScenario, - abstained: boolean, -): number { - if (scenario.unanswerable) return abstained ? 1 : 0 - const expected = scenario.expectedClaims ?? [] - const forbidden = scenario.forbiddenClaims ?? [] - const expectedScore = - expected.length === 0 - ? scoreAnswerRelevance(artifact, scenario, abstained) - : average(expected.map((claim) => textSupportScore(claim, artifact.answer))) - const forbiddenPenalty = - forbidden.length === 0 - ? 0 - : forbidden.filter((claim) => textSupportScore(claim, artifact.answer) >= 0.7).length / - forbidden.length - return clamp01(expectedScore * (1 - forbiddenPenalty)) -} - -function normalizeClaims(artifact: RagAnswerEvalArtifact): RagEvalClaim[] { - if (artifact.claims?.length) return [...artifact.claims] - return splitSentences(artifact.answer).map((text, index) => ({ id: `claim-${index + 1}`, text })) -} - -function splitSentences(text: string): string[] { - return text - .split(/(?<=[.!?])\s+/) - .map((sentence) => sentence.trim()) - .filter((sentence) => sentence.length > 0 && !looksLikeAbstention(sentence)) -} - -function looksLikeAbstention(answer: string): boolean { - const normalized = answer.toLowerCase() - return [ - "i don't know", - 'i do not know', - 'not enough information', - 'cannot answer', - "can't answer", - 'insufficient information', - 'no reliable answer', - ].some((phrase) => normalized.includes(phrase)) -} - -function mergeMetrics( - deterministic: Record, - providerScores: Record>, - policy: RagAnswerQualityJudgeOptions['externalScorePolicy'] = 'prefer-external', -): Record { - if (policy === 'deterministic-first') return deterministic - const merged = { ...deterministic } - for (const scores of Object.values(providerScores)) { - for (const [key, value] of Object.entries(scores) as Array<[RagEvalMetricKey, number]>) { - merged[key] = value - if (key === 'groundedness') merged.faithfulness = value - if (key === 'faithfulness') merged.groundedness = value - } - } - return merged -} - -function aggregateRagAnswerMetrics( - summaries: readonly RagAnswerMetricSummary[], -): Record { - const keys = new Set() - for (const summary of summaries) { - for (const key of Object.keys(summary.metrics) as RagEvalMetricKey[]) keys.add(key) - } - const out: Record = {} - for (const key of [...keys].sort()) { - out[key] = average(summaries.map((summary) => summary.metrics[key])) - } - out.composite = average(summaries.map((summary) => summary.composite)) - return out -} - -function weightedComposite( - metrics: Record, - weights: Partial>, -): number { - let weighted = 0 - let total = 0 - for (const [key, weight] of Object.entries(weights) as Array<[RagEvalMetricKey, number]>) { - if (!Number.isFinite(weight) || weight <= 0) continue - const metric = metrics[key] - if (!Number.isFinite(metric)) continue - const value = key === 'unsupported_answer_rate' ? 1 - metric : metric - weighted += value * weight - total += weight - } - return total === 0 ? 0 : weighted / total -} - -function normalizeMetricName(metric: string): string { - return metric - .trim() - .toLowerCase() - .replace(/[@/.-]+/g, '_') - .replace(/\s+/g, '_') -} - -function textSupportScore(needle: string, haystack: string): number { - const needleWords = contentWords(needle) - if (needleWords.length === 0) return 0 - const haystackWords = new Set(contentWords(haystack)) - const present = needleWords.filter((word) => haystackWords.has(word)) - return present.length / needleWords.length -} - -function contentWords(text: string): string[] { - const normalized = text - .toLowerCase() - .replace(/[^\p{L}\p{N}\s]+/gu, ' ') - .split(/\s+/) - .map((word) => stem(word.trim())) - .filter((word) => (word.length >= 3 || /^\d+$/.test(word)) && !stopwords.has(word)) - return [...new Set(normalized)] -} - -function stem(word: string): string { - if (word.endsWith('ies') && word.length > 4) return `${word.slice(0, -3)}y` - if (word.endsWith('ing') && word.length > 5) return word.slice(0, -3) - if (word.endsWith('ed') && word.length > 4) return word.slice(0, -2) - if (word.endsWith('s') && word.length > 3) return word.slice(0, -1) - return word -} - -function sourceRefs(text: string): string[] { - const refs: string[] = [] - const regex = /\[\^([A-Za-z0-9_-]+)(?:#([A-Za-z0-9_.:-]+))?\]/g - let match = regex.exec(text) - while (match !== null) { - refs.push(match[0]) - match = regex.exec(text) - } - return refs -} - -function below(metric: number, threshold: number | undefined): boolean { - return threshold !== undefined && metric < threshold -} - -function neutralScore(condition: boolean): number { - return condition ? 1 : 0 -} - -function average(values: readonly number[]): number { - const finite = values.filter(Number.isFinite) - return finite.length === 0 ? 0 : finite.reduce((sum, value) => sum + value, 0) / finite.length -} - -function clamp01(value: number): number { - if (!Number.isFinite(value)) return 0 - return Math.max(0, Math.min(1, value)) -} - -const stopwords = new Set([ - 'the', - 'a', - 'an', - 'and', - 'or', - 'but', - 'of', - 'to', - 'in', - 'on', - 'for', - 'with', - 'as', - 'by', - 'at', - 'from', - 'that', - 'this', - 'these', - 'those', - 'it', - 'its', - 'is', - 'are', - 'was', - 'were', - 'be', - 'been', - 'being', - 'has', - 'have', - 'had', - 'can', - 'will', - 'would', - 'should', - 'may', - 'might', - 'not', - 'no', - 'than', - 'then', - 'over', - 'under', - 'about', - 'into', - 'their', - 'they', - 'them', - 'what', - 'when', - 'where', - 'who', - 'why', - 'how', -]) +export { calibrateRagAnswerJudge, createRagAnswerQualityHook } from './rag-eval/calibration' +export type { + ExternalRagEvalScore, + KnowledgeBaseQualityOptions, + KnowledgeBaseQualityReport, + RagAnswerEvalArtifact, + RagAnswerEvalCase, + RagAnswerEvalScenario, + RagAnswerMetricSummary, + RagAnswerQualityHookOptions, + RagAnswerQualityJudgeOptions, + RagCalibrationOptions, + RagCalibrationResult, + RagEvalCitation, + RagEvalClaim, + RagEvalContext, + RagEvalMetricKey, + RagEvalProvider, + RagEvalSlice, + RagRequiredContext, +} from './rag-eval/contracts' +export { scoreKnowledgeBaseIndex } from './rag-eval/knowledge-base' +export { + normalizeExternalRagScores, + toDeepEvalTestCases, + toRagasEvaluationRows, + toRagCheckerRecords, + toTruLensRecords, +} from './rag-eval/providers' +export { + diagnoseRagAnswerFailure, + ragAnswerQualityJudge, + scoreRagAnswerArtifact, +} from './rag-eval/scoring' diff --git a/src/rag-eval/answer.ts b/src/rag-eval/answer.ts new file mode 100644 index 0000000..ed1ac05 --- /dev/null +++ b/src/rag-eval/answer.ts @@ -0,0 +1,279 @@ +import { groundClaimInText } from '../claim-grounding' +import type { + RagAnswerEvalArtifact, + RagAnswerEvalScenario, + RagAnswerQualityJudgeOptions, + RagEvalCitation, + RagEvalClaim, + RagEvalContext, + RagRequiredContext, +} from './contracts' + +export function requiredContextTargets(scenario: RagAnswerEvalScenario): RagRequiredContext[] { + if (scenario.requiredContext?.length) return [...scenario.requiredContext] + return (scenario.expectedClaims ?? []).map((text) => ({ text })) +} + +export function requiredContextTexts(scenario: RagAnswerEvalScenario): string[] { + return requiredContextTargets(scenario).flatMap((target) => (target.text ? [target.text] : [])) +} + +export function contextMatchesTarget(context: RagEvalContext, target: RagRequiredContext): boolean { + if (target.id && context.id === target.id) return true + if (target.pageId && context.pageId === target.pageId) return true + if (target.sourceId && context.sourceId === target.sourceId) return true + if (target.anchorId && context.anchorId === target.anchorId) return true + if (target.text && textSupportScore(target.text, context.text) >= 0.7) return true + return false +} + +export function contextIsRelevant( + context: RagEvalContext, + scenario: RagAnswerEvalScenario, +): boolean { + return contextRelevanceScore(context, scenario) >= 0.3 +} + +export function contextRelevanceScore( + context: RagEvalContext, + scenario: RagAnswerEvalScenario, +): number { + const targets = [ + scenario.query, + scenario.referenceAnswer, + ...(scenario.expectedClaims ?? []), + ...requiredContextTexts(scenario), + ].filter((value): value is string => Boolean(value?.trim())) + if (targets.length === 0) return 1 + return Math.max(...targets.map((target) => textSupportScore(target, context.text))) +} + +export function claimSupport( + claim: string, + contexts: readonly RagEvalContext[], + options: Pick, +): boolean { + if (contexts.length === 0) return false + const combined = contexts.map((context) => context.text).join('\n\n') + const deterministic = groundClaimInText(claim, combined, { + minOverlap: options.minClaimSupport ?? 0.7, + }) + return ( + deterministic.grounded || textSupportScore(claim, combined) >= (options.minClaimSupport ?? 0.7) + ) +} + +export function scoreCitations( + claims: readonly RagEvalClaim[], + artifact: RagAnswerEvalArtifact, + scenario: RagAnswerEvalScenario, + options: Pick, +): { support: number; citedClaimCount: number; supportedCitationCount: number } { + if (claims.length === 0) + return { support: scenario.unanswerable ? 1 : 0, citedClaimCount: 0, supportedCitationCount: 0 } + const citations = artifact.citations ?? [] + const citedClaims = claims.filter((claim) => citationsForClaim(claim, citations).length > 0) + if (citedClaims.length === 0) { + return { + support: scenario.requireCitations ? 0 : 1, + citedClaimCount: 0, + supportedCitationCount: 0, + } + } + let supportedCitationCount = 0 + for (const claim of citedClaims) { + const citedContexts = contextsForClaim(claim, citations, artifact.contexts) + if (claimSupport(claim.text, citedContexts, options)) supportedCitationCount += 1 + } + return { + support: supportedCitationCount / citedClaims.length, + citedClaimCount: citedClaims.length, + supportedCitationCount, + } +} + +function citationsForClaim( + claim: RagEvalClaim, + citations: readonly RagEvalCitation[], +): RagEvalCitation[] { + const claimCitationIds = new Set(claim.citationIds ?? []) + return citations.filter((citation) => { + return citation.claimId === claim.id || (citation.id && claimCitationIds.has(citation.id)) + }) +} + +function contextsForClaim( + claim: RagEvalClaim, + citations: readonly RagEvalCitation[], + contexts: readonly RagEvalContext[], +): readonly RagEvalContext[] { + const matchedCitations = citationsForClaim(claim, citations) + const matchedContexts = contexts.filter((context) => + matchedCitations.some((citation) => { + if (citation.contextId && citation.contextId === context.id) return true + if (citation.pageId && citation.pageId === context.pageId) return true + if (citation.sourceId && citation.sourceId === context.sourceId) return true + if (citation.anchorId && citation.anchorId === context.anchorId) return true + return false + }), + ) + return matchedContexts.length > 0 ? matchedContexts : contexts +} + +export function scoreAnswerRelevance( + artifact: RagAnswerEvalArtifact, + scenario: RagAnswerEvalScenario, + abstained: boolean, +): number { + if (scenario.unanswerable) return abstained ? 1 : 0 + if (abstained) return 0 + const targets = [ + scenario.referenceAnswer, + ...(scenario.expectedClaims ?? []), + scenario.query, + ].filter((value): value is string => Boolean(value?.trim())) + return Math.max(...targets.map((target) => textSupportScore(target, artifact.answer))) +} + +export function scoreAnswerCorrectness( + artifact: RagAnswerEvalArtifact, + scenario: RagAnswerEvalScenario, + abstained: boolean, +): number { + if (scenario.unanswerable) return abstained ? 1 : 0 + const expected = scenario.expectedClaims ?? [] + const forbidden = scenario.forbiddenClaims ?? [] + const expectedScore = + expected.length === 0 + ? scoreAnswerRelevance(artifact, scenario, abstained) + : average(expected.map((claim) => textSupportScore(claim, artifact.answer))) + const forbiddenPenalty = + forbidden.length === 0 + ? 0 + : forbidden.filter((claim) => textSupportScore(claim, artifact.answer) >= 0.7).length / + forbidden.length + return clamp01(expectedScore * (1 - forbiddenPenalty)) +} + +export function normalizeClaims(artifact: RagAnswerEvalArtifact): RagEvalClaim[] { + if (artifact.claims?.length) return [...artifact.claims] + return splitSentences(artifact.answer).map((text, index) => ({ id: `claim-${index + 1}`, text })) +} + +function splitSentences(text: string): string[] { + return text + .split(/(?<=[.!?])\s+/) + .map((sentence) => sentence.trim()) + .filter((sentence) => sentence.length > 0 && !looksLikeAbstention(sentence)) +} + +export function looksLikeAbstention(answer: string): boolean { + const normalized = answer.toLowerCase() + return [ + "i don't know", + 'i do not know', + 'not enough information', + 'cannot answer', + "can't answer", + 'insufficient information', + 'no reliable answer', + ].some((phrase) => normalized.includes(phrase)) +} + +function textSupportScore(needle: string, haystack: string): number { + const needleWords = contentWords(needle) + if (needleWords.length === 0) return 0 + const haystackWords = new Set(contentWords(haystack)) + const present = needleWords.filter((word) => haystackWords.has(word)) + return present.length / needleWords.length +} + +function contentWords(text: string): string[] { + const normalized = text + .toLowerCase() + .replace(/[^\p{L}\p{N}\s]+/gu, ' ') + .split(/\s+/) + .map((word) => stem(word.trim())) + .filter((word) => (word.length >= 3 || /^\d+$/.test(word)) && !stopwords.has(word)) + return [...new Set(normalized)] +} + +function stem(word: string): string { + if (word.endsWith('ies') && word.length > 4) return `${word.slice(0, -3)}y` + if (word.endsWith('ing') && word.length > 5) return word.slice(0, -3) + if (word.endsWith('ed') && word.length > 4) return word.slice(0, -2) + if (word.endsWith('s') && word.length > 3) return word.slice(0, -1) + return word +} + +export function neutralScore(condition: boolean): number { + return condition ? 1 : 0 +} + +export function average(values: readonly number[]): number { + const finite = values.filter(Number.isFinite) + return finite.length === 0 ? 0 : finite.reduce((sum, value) => sum + value, 0) / finite.length +} + +export function clamp01(value: number): number { + if (!Number.isFinite(value)) return 0 + return Math.max(0, Math.min(1, value)) +} + +const stopwords = new Set([ + 'the', + 'a', + 'an', + 'and', + 'or', + 'but', + 'of', + 'to', + 'in', + 'on', + 'for', + 'with', + 'as', + 'by', + 'at', + 'from', + 'that', + 'this', + 'these', + 'those', + 'it', + 'its', + 'is', + 'are', + 'was', + 'were', + 'be', + 'been', + 'being', + 'has', + 'have', + 'had', + 'can', + 'will', + 'would', + 'should', + 'may', + 'might', + 'not', + 'no', + 'than', + 'then', + 'over', + 'under', + 'about', + 'into', + 'their', + 'they', + 'them', + 'what', + 'when', + 'where', + 'who', + 'why', + 'how', +]) diff --git a/src/rag-eval/calibration.ts b/src/rag-eval/calibration.ts new file mode 100644 index 0000000..9d58d01 --- /dev/null +++ b/src/rag-eval/calibration.ts @@ -0,0 +1,69 @@ +import type { RagAnswerQualityResult, RagGapFinding } from '../rag-improvement-loop' +import type { + RagAnswerMetricSummary, + RagAnswerQualityHookOptions, + RagCalibrationOptions, + RagCalibrationResult, +} from './contracts' +import { aggregateRagAnswerMetrics, ragAnswerQualityJudge, scoreRagAnswerArtifact } from './scoring' + +export function createRagAnswerQualityHook( + options: RagAnswerQualityHookOptions, +): () => Promise { + return async () => { + const summaries: RagAnswerMetricSummary[] = [] + const findings: RagGapFinding[] = [] + for (const scenario of options.scenarios) { + const initialArtifact = await options.run(scenario) + const external = await options.externalEvaluator?.({ scenario, artifact: initialArtifact }) + const artifact = external + ? { + ...initialArtifact, + externalScores: [ + ...(initialArtifact.externalScores ?? []), + ...(Array.isArray(external) ? external : [external]), + ], + } + : initialArtifact + const summary = scoreRagAnswerArtifact(artifact, scenario, { + thresholds: options.thresholds, + weights: options.weights, + }) + summaries.push(summary) + findings.push(...summary.findings) + } + const metrics = aggregateRagAnswerMetrics(summaries) + return { + passed: findings.length === 0, + metrics, + findings, + metadata: { scenarioCount: options.scenarios.length }, + } + } +} + +export async function calibrateRagAnswerJudge( + options: RagCalibrationOptions, +): Promise { + const judge = options.judge ?? ragAnswerQualityJudge() + const strong = await judge.score({ + artifact: options.strong, + scenario: options.scenario, + signal: options.signal ?? new AbortController().signal, + }) + const weak = await judge.score({ + artifact: options.weak, + scenario: options.scenario, + signal: options.signal ?? new AbortController().signal, + }) + const strongScore = strong.composite + const weakScore = weak.composite + const minStrongScore = options.minStrongScore ?? 0.7 + const maxWeakScore = options.maxWeakScore ?? 0.3 + return { + passed: strongScore >= minStrongScore && weakScore <= maxWeakScore, + strongScore, + weakScore, + gap: strongScore - weakScore, + } +} diff --git a/src/rag-eval/contracts.ts b/src/rag-eval/contracts.ts new file mode 100644 index 0000000..612d8af --- /dev/null +++ b/src/rag-eval/contracts.ts @@ -0,0 +1,181 @@ +import type { JsonValue, JudgeConfig, Scenario } from '@tangle-network/agent-eval/campaign' +import type { RagGapFinding } from '../rag-improvement-loop' + +export type RagEvalProvider = + | 'agent-knowledge' + | 'ragas' + | 'deepeval' + | 'trulens' + | 'ragchecker' + | 'custom' + +export type RagEvalMetricKey = + | 'context_precision' + | 'context_recall' + | 'context_relevance' + | 'context_sufficiency' + | 'faithfulness' + | 'groundedness' + | 'answer_relevance' + | 'answer_correctness' + | 'citation_support' + | 'abstention' + | 'unsupported_answer_rate' + +export type RagEvalSlice = + | 'known-answer' + | 'paraphrase' + | 'distractor' + | 'freshness' + | 'multi-source' + | 'unanswerable' + | 'long-tail' + | 'custom' + +export interface RagEvalContext { + id: string + text: string + rank?: number + pageId?: string + sourceId?: string + anchorId?: string + stale?: boolean + metadata?: Record +} + +export interface RagEvalCitation { + id: string + claimId?: string + contextId?: string + pageId?: string + sourceId?: string + anchorId?: string + quote?: string + metadata?: Record +} + +export interface RagEvalClaim { + id: string + text: string + citationIds?: readonly string[] + metadata?: Record +} + +export interface RagRequiredContext { + id?: string + text?: string + pageId?: string + sourceId?: string + anchorId?: string +} + +export interface RagAnswerEvalScenario extends Scenario { + kind: 'rag-answer-eval' + query: string + referenceAnswer?: string + expectedClaims?: readonly string[] + forbiddenClaims?: readonly string[] + requiredContext?: readonly RagRequiredContext[] + unanswerable?: boolean + requireCitations?: boolean + slices?: readonly RagEvalSlice[] + thresholds?: Partial> +} + +export interface ExternalRagEvalScore { + provider: RagEvalProvider | string + scores: Record + reasons?: Record + metadata?: Record +} + +export interface RagAnswerEvalArtifact { + query: string + answer: string + contexts: readonly RagEvalContext[] + claims?: readonly RagEvalClaim[] + citations?: readonly RagEvalCitation[] + abstained?: boolean + durationMs?: number + costUsd?: number + externalScores?: readonly ExternalRagEvalScore[] + metadata?: Record +} + +export interface RagAnswerMetricSummary { + metrics: Record + composite: number + passed: boolean + findings: readonly RagGapFinding[] + claimCount: number + supportedClaimCount: number + citedClaimCount: number + supportedCitationCount: number + matchedRequiredContextCount: number + requiredContextCount: number + providerScores: Record> +} + +export interface RagAnswerQualityJudgeOptions { + name?: string + thresholds?: Partial> + weights?: Partial> + externalScorePolicy?: 'prefer-external' | 'deterministic-first' + minClaimSupport?: number +} + +export interface RagAnswerEvalCase { + scenario: RagAnswerEvalScenario + artifact: RagAnswerEvalArtifact +} + +export interface RagAnswerQualityHookOptions { + scenarios: readonly RagAnswerEvalScenario[] + run: (scenario: RagAnswerEvalScenario) => MaybePromise + externalEvaluator?: ( + item: RagAnswerEvalCase, + ) => MaybePromise + thresholds?: Partial> + weights?: Partial> +} + +export interface RagCalibrationOptions { + scenario: RagAnswerEvalScenario + strong: RagAnswerEvalArtifact + weak: RagAnswerEvalArtifact + judge?: JudgeConfig + minStrongScore?: number + maxWeakScore?: number + signal?: AbortSignal +} + +export interface RagCalibrationResult { + passed: boolean + strongScore: number + weakScore: number + gap: number +} + +export interface KnowledgeBaseQualityOptions { + now?: Date + strict?: boolean + minCitationRate?: number + maxStaleSourceRate?: number +} + +export interface KnowledgeBaseQualityReport { + ok: boolean + metrics: { + page_count: number + source_count: number + citation_rate: number + source_backed_page_rate: number + stale_source_rate: number + duplicate_source_hash_rate: number + lint_error_count: number + lint_warning_count: number + } + findings: readonly RagGapFinding[] +} + +type MaybePromise = T | Promise diff --git a/src/rag-eval/knowledge-base.ts b/src/rag-eval/knowledge-base.ts new file mode 100644 index 0000000..503a0bb --- /dev/null +++ b/src/rag-eval/knowledge-base.ts @@ -0,0 +1,79 @@ +import { lintKnowledgeIndex } from '../lint' +import type { RagGapFinding } from '../rag-improvement-loop' +import type { KnowledgeIndex } from '../types' +import { validateKnowledgeIndex } from '../validate' +import type { KnowledgeBaseQualityOptions, KnowledgeBaseQualityReport } from './contracts' + +export function scoreKnowledgeBaseIndex( + index: KnowledgeIndex, + options: KnowledgeBaseQualityOptions = {}, +): KnowledgeBaseQualityReport { + const validation = validateKnowledgeIndex(index, { strict: options.strict }) + const lintFindings = lintKnowledgeIndex(index) + const now = options.now ?? new Date() + const staleSourceCount = index.sources.filter((source) => { + return source.validUntil ? Date.parse(source.validUntil) < now.getTime() : false + }).length + const sourceHashCounts = new Map() + for (const source of index.sources) { + sourceHashCounts.set(source.contentHash, (sourceHashCounts.get(source.contentHash) ?? 0) + 1) + } + const duplicateSourceHashCount = [...sourceHashCounts.values()].filter( + (count) => count > 1, + ).length + const pagesWithInlineCitations = index.pages.filter( + (page) => sourceRefs(page.text).length > 0, + ).length + const pagesWithSources = index.pages.filter((page) => page.sourceIds.length > 0).length + const metrics = { + page_count: index.pages.length, + source_count: index.sources.length, + citation_rate: index.pages.length === 0 ? 1 : pagesWithInlineCitations / index.pages.length, + source_backed_page_rate: index.pages.length === 0 ? 1 : pagesWithSources / index.pages.length, + stale_source_rate: index.sources.length === 0 ? 0 : staleSourceCount / index.sources.length, + duplicate_source_hash_rate: + index.sources.length === 0 ? 0 : duplicateSourceHashCount / index.sources.length, + lint_error_count: validation.findings.filter((finding) => finding.severity === 'error').length, + lint_warning_count: lintFindings.filter((finding) => finding.severity === 'warning').length, + } + const findings: RagGapFinding[] = [] + if (metrics.lint_error_count > 0) { + findings.push({ + id: 'kb:lint-errors', + kind: 'unknown', + severity: 'critical', + message: `${metrics.lint_error_count} validation error(s) in the knowledge index.`, + evidence: { lint_error_count: metrics.lint_error_count }, + }) + } + if (metrics.citation_rate < (options.minCitationRate ?? 0)) { + findings.push({ + id: 'kb:citation-rate', + kind: 'missing-source', + severity: 'error', + message: 'Inline citation coverage is below the configured minimum.', + evidence: { citation_rate: metrics.citation_rate }, + }) + } + if (metrics.stale_source_rate > (options.maxStaleSourceRate ?? 1)) { + findings.push({ + id: 'kb:stale-source-rate', + kind: 'stale-source', + severity: 'error', + message: 'Stale source rate exceeds the configured maximum.', + evidence: { stale_source_rate: metrics.stale_source_rate }, + }) + } + return { ok: findings.length === 0, metrics, findings } +} + +function sourceRefs(text: string): string[] { + const refs: string[] = [] + const regex = /\[\^([A-Za-z0-9_-]+)(?:#([A-Za-z0-9_.:-]+))?\]/g + let match = regex.exec(text) + while (match !== null) { + refs.push(match[0]) + match = regex.exec(text) + } + return refs +} diff --git a/src/rag-eval/providers.ts b/src/rag-eval/providers.ts new file mode 100644 index 0000000..22ea3a7 --- /dev/null +++ b/src/rag-eval/providers.ts @@ -0,0 +1,91 @@ +import { clamp01, normalizeClaims, requiredContextTexts } from './answer' +import type { ExternalRagEvalScore, RagAnswerEvalCase, RagEvalMetricKey } from './contracts' + +const metricAliases: Record = { + answer_correctness: 'answer_correctness', + answer_relevance: 'answer_relevance', + answer_relevancy: 'answer_relevance', + answerrelevancy: 'answer_relevance', + context_precision: 'context_precision', + context_recall: 'context_recall', + context_relevance: 'context_relevance', + context_relevancy: 'context_relevance', + context_sufficiency: 'context_sufficiency', + contextual_precision: 'context_precision', + contextual_recall: 'context_recall', + contextual_relevancy: 'context_relevance', + faithfulness: 'faithfulness', + groundedness: 'groundedness', + claim_recall: 'context_recall', + claim_precision: 'faithfulness', + citation_support: 'citation_support', + abstention: 'abstention', + unsupported_answer_rate: 'unsupported_answer_rate', +} + +export function normalizeExternalRagScores( + scores: readonly ExternalRagEvalScore[], +): Record> { + const normalized: Record> = {} + for (const item of scores) { + const provider = item.provider || 'custom' + const providerScores = (normalized[provider] ?? {}) as Record + for (const [key, value] of Object.entries(item.scores)) { + const canonical = metricAliases[normalizeMetricName(key)] + if (!canonical) continue + if (Number.isFinite(value)) providerScores[canonical] = clamp01(value) + } + normalized[provider] = providerScores + } + return normalized +} + +export function toRagasEvaluationRows(cases: readonly RagAnswerEvalCase[]) { + return cases.map(({ scenario, artifact }) => ({ + user_input: scenario.query, + response: artifact.answer, + retrieved_contexts: artifact.contexts.map((context) => context.text), + reference: scenario.referenceAnswer, + reference_contexts: requiredContextTexts(scenario), + })) +} + +export function toDeepEvalTestCases(cases: readonly RagAnswerEvalCase[]) { + return cases.map(({ scenario, artifact }) => ({ + input: scenario.query, + actual_output: artifact.answer, + expected_output: scenario.referenceAnswer, + retrieval_context: artifact.contexts.map((context) => context.text), + context: requiredContextTexts(scenario), + })) +} + +export function toTruLensRecords(cases: readonly RagAnswerEvalCase[]) { + return cases.map(({ scenario, artifact }) => ({ + input: scenario.query, + output: artifact.answer, + context: artifact.contexts.map((context) => context.text).join('\n\n'), + })) +} + +export function toRagCheckerRecords(cases: readonly RagAnswerEvalCase[]) { + return cases.map(({ scenario, artifact }) => ({ + query_id: scenario.id, + query: scenario.query, + gt_answer: scenario.referenceAnswer, + response: artifact.answer, + retrieved_context: artifact.contexts.map((context) => ({ + doc_id: context.id, + text: context.text, + })), + claims: normalizeClaims(artifact).map((claim) => claim.text), + })) +} + +function normalizeMetricName(metric: string): string { + return metric + .trim() + .toLowerCase() + .replace(/[@/.-]+/g, '_') + .replace(/\s+/g, '_') +} diff --git a/src/rag-eval/scoring.ts b/src/rag-eval/scoring.ts new file mode 100644 index 0000000..38577bd --- /dev/null +++ b/src/rag-eval/scoring.ts @@ -0,0 +1,269 @@ +import type { JudgeConfig } from '@tangle-network/agent-eval/campaign' +import type { RagGapFinding } from '../rag-improvement-loop' +import { + average, + claimSupport, + clamp01, + contextIsRelevant, + contextMatchesTarget, + contextRelevanceScore, + looksLikeAbstention, + neutralScore, + normalizeClaims, + requiredContextTargets, + scoreAnswerCorrectness, + scoreAnswerRelevance, + scoreCitations, +} from './answer' +import type { + RagAnswerEvalArtifact, + RagAnswerEvalScenario, + RagAnswerMetricSummary, + RagAnswerQualityJudgeOptions, + RagEvalMetricKey, +} from './contracts' +import { normalizeExternalRagScores } from './providers' + +const defaultThresholds: Partial> = { + context_precision: 0.5, + context_recall: 0.8, + context_relevance: 0.5, + context_sufficiency: 0.8, + faithfulness: 0.9, + answer_relevance: 0.7, + citation_support: 0.9, + abstention: 1, + unsupported_answer_rate: 0, +} + +const defaultWeights: Partial> = { + context_relevance: 1, + context_sufficiency: 1, + faithfulness: 2, + answer_relevance: 1, + citation_support: 1, + abstention: 1, +} + +export function ragAnswerQualityJudge( + options: RagAnswerQualityJudgeOptions = {}, +): JudgeConfig { + return { + name: options.name ?? 'rag-answer-quality', + dimensions: [ + { key: 'context_precision', description: 'share of retrieved context that is useful' }, + { key: 'context_recall', description: 'share of required evidence present in context' }, + { key: 'context_relevance', description: 'context relevance to the query and answer target' }, + { key: 'context_sufficiency', description: 'whether context is enough to answer' }, + { key: 'faithfulness', description: 'share of answer claims supported by retrieved context' }, + { key: 'answer_relevance', description: 'answer addresses the user query' }, + { key: 'answer_correctness', description: 'answer contains expected claims' }, + { key: 'citation_support', description: 'citations support the claims they cite' }, + { key: 'abstention', description: 'answer abstains exactly when the case is unanswerable' }, + ], + appliesTo: (scenario) => scenario.kind === 'rag-answer-eval', + async score({ artifact, scenario }) { + const summary = scoreRagAnswerArtifact(artifact, scenario, options) + return { + dimensions: summary.metrics, + composite: summary.composite, + notes: summary.findings.map((finding) => finding.message).join('; '), + } + }, + } +} + +export function scoreRagAnswerArtifact( + artifact: RagAnswerEvalArtifact, + scenario: RagAnswerEvalScenario, + options: RagAnswerQualityJudgeOptions = {}, +): RagAnswerMetricSummary { + const claims = normalizeClaims(artifact) + const abstained = artifact.abstained ?? looksLikeAbstention(artifact.answer) + const requiredTargets = requiredContextTargets(scenario) + const matchedRequiredContextCount = requiredTargets.filter((target) => + artifact.contexts.some((context) => contextMatchesTarget(context, target)), + ).length + const requiredContextCount = requiredTargets.length + const contextRecall = + requiredContextCount === 0 + ? neutralScore(Boolean(scenario.unanswerable) || artifact.contexts.length > 0) + : matchedRequiredContextCount / requiredContextCount + const relevantContextCount = artifact.contexts.filter((context) => + contextIsRelevant(context, scenario), + ).length + const contextPrecision = + artifact.contexts.length === 0 + ? scenario.unanswerable + ? 1 + : 0 + : relevantContextCount / artifact.contexts.length + const contextRelevance = + artifact.contexts.length === 0 + ? scenario.unanswerable + ? 1 + : 0 + : average(artifact.contexts.map((context) => contextRelevanceScore(context, scenario))) + const contextSufficiency = requiredContextCount === 0 ? contextRelevance : contextRecall + const support = claims.map((claim) => claimSupport(claim.text, artifact.contexts, options)) + const supportedClaimCount = support.filter(Boolean).length + const faithfulness = + claims.length === 0 ? (abstained ? 1 : 0) : supportedClaimCount / Math.max(1, claims.length) + const citation = scoreCitations(claims, artifact, scenario, options) + const answerRelevance = scoreAnswerRelevance(artifact, scenario, abstained) + const answerCorrectness = scoreAnswerCorrectness(artifact, scenario, abstained) + const abstention = scenario.unanswerable ? (abstained ? 1 : 0) : abstained ? 0 : 1 + const unsupportedAnswerRate = + claims.length === 0 ? (abstained ? 0 : 1) : 1 - supportedClaimCount / Math.max(1, claims.length) + + const deterministicMetrics: Record = { + context_precision: clamp01(contextPrecision), + context_recall: clamp01(contextRecall), + context_relevance: clamp01(contextRelevance), + context_sufficiency: clamp01(contextSufficiency), + faithfulness: clamp01(faithfulness), + groundedness: clamp01(faithfulness), + answer_relevance: clamp01(answerRelevance), + answer_correctness: clamp01(answerCorrectness), + citation_support: clamp01(citation.support), + abstention: clamp01(abstention), + unsupported_answer_rate: clamp01(unsupportedAnswerRate), + } + const providerScores = normalizeExternalRagScores(artifact.externalScores ?? []) + const metrics = mergeMetrics(deterministicMetrics, providerScores, options.externalScorePolicy) + const thresholds = { ...defaultThresholds, ...scenario.thresholds, ...options.thresholds } + const findings = diagnoseRagAnswerFailure(metrics, scenario, thresholds) + const composite = weightedComposite(metrics, options.weights ?? defaultWeights) + + return { + metrics, + composite, + passed: findings.every( + (finding) => finding.severity !== 'error' && finding.severity !== 'critical', + ), + findings, + claimCount: claims.length, + supportedClaimCount, + citedClaimCount: citation.citedClaimCount, + supportedCitationCount: citation.supportedCitationCount, + matchedRequiredContextCount, + requiredContextCount, + providerScores, + } +} + +export function diagnoseRagAnswerFailure( + metrics: Record, + scenario: RagAnswerEvalScenario, + thresholds: Partial> = defaultThresholds, +): RagGapFinding[] { + const findings: RagGapFinding[] = [] + if (below(metrics.context_recall, thresholds.context_recall)) { + findings.push({ + id: `${scenario.id}:context-recall`, + kind: scenario.slices?.includes('multi-source') + ? 'missing-multihop-evidence' + : 'retrieval-miss', + severity: 'error', + scenarioId: scenario.id, + message: 'Required evidence was not present in retrieved context.', + evidence: { context_recall: metrics.context_recall }, + }) + } + if (below(metrics.context_precision, thresholds.context_precision)) { + findings.push({ + id: `${scenario.id}:context-precision`, + kind: 'retrieval-noise', + severity: 'warning', + scenarioId: scenario.id, + message: 'Retrieved context contains too much irrelevant material.', + evidence: { context_precision: metrics.context_precision }, + }) + } + if (below(metrics.faithfulness, thresholds.faithfulness)) { + findings.push({ + id: `${scenario.id}:faithfulness`, + kind: 'generator-unsupported-claim', + severity: 'error', + scenarioId: scenario.id, + message: 'The answer contains claims not supported by retrieved context.', + evidence: { faithfulness: metrics.faithfulness }, + }) + } + if (below(metrics.citation_support, thresholds.citation_support)) { + findings.push({ + id: `${scenario.id}:citation-support`, + kind: 'citation-mismatch', + severity: 'error', + scenarioId: scenario.id, + message: 'Citations do not support the claims they are attached to.', + evidence: { citation_support: metrics.citation_support }, + }) + } + if (below(metrics.abstention, thresholds.abstention)) { + findings.push({ + id: `${scenario.id}:abstention`, + kind: 'incorrect-abstention', + severity: 'error', + scenarioId: scenario.id, + message: scenario.unanswerable + ? 'The system answered a case marked unanswerable.' + : 'The system abstained from an answerable case.', + evidence: { abstention: metrics.abstention }, + }) + } + return findings +} + +function mergeMetrics( + deterministic: Record, + providerScores: Record>, + policy: RagAnswerQualityJudgeOptions['externalScorePolicy'] = 'prefer-external', +): Record { + if (policy === 'deterministic-first') return deterministic + const merged = { ...deterministic } + for (const scores of Object.values(providerScores)) { + for (const [key, value] of Object.entries(scores) as Array<[RagEvalMetricKey, number]>) { + merged[key] = value + if (key === 'groundedness') merged.faithfulness = value + if (key === 'faithfulness') merged.groundedness = value + } + } + return merged +} + +export function aggregateRagAnswerMetrics( + summaries: readonly RagAnswerMetricSummary[], +): Record { + const keys = new Set() + for (const summary of summaries) { + for (const key of Object.keys(summary.metrics) as RagEvalMetricKey[]) keys.add(key) + } + const out: Record = {} + for (const key of [...keys].sort()) { + out[key] = average(summaries.map((summary) => summary.metrics[key])) + } + out.composite = average(summaries.map((summary) => summary.composite)) + return out +} + +function weightedComposite( + metrics: Record, + weights: Partial>, +): number { + let weighted = 0 + let total = 0 + for (const [key, weight] of Object.entries(weights) as Array<[RagEvalMetricKey, number]>) { + if (!Number.isFinite(weight) || weight <= 0) continue + const metric = metrics[key] + if (!Number.isFinite(metric)) continue + const value = key === 'unsupported_answer_rate' ? 1 - metric : metric + weighted += value * weight + total += weight + } + return total === 0 ? 0 : weighted / total +} + +function below(metric: number, threshold: number | undefined): boolean { + return threshold !== undefined && metric < threshold +} diff --git a/tests/kb-improvement.test.ts b/tests/kb-improvement.test.ts deleted file mode 100644 index bfcb723..0000000 --- a/tests/kb-improvement.test.ts +++ /dev/null @@ -1,1867 +0,0 @@ -import { createHash } from 'node:crypto' -import { - chmod, - mkdir, - mkdtemp, - readdir, - readFile, - rename, - rm, - stat, - symlink, - writeFile, -} from 'node:fs/promises' -import { tmpdir } from 'node:os' -import { dirname, join, relative } from 'node:path' -import { - type AgentImprovementActivation, - type AgentImprovementActivationResult, - canonicalCandidateDigest, - type Sha256Digest, -} from '@tangle-network/agent-interface' -import { describe, expect, it } from 'vitest' -import { - applyKnowledgeWriteBlocks, - buildEvalKnowledgeBundle, - buildKnowledgeIndex, - defineReadinessSpec, - evaluateKnowledgeBaseReadiness, - hashKnowledgeBase, - improveKnowledgeBase, - initKnowledgeBase, - inspectPendingKnowledgeMutation, - type KnowledgeImprovementCandidateRef, - type KnowledgeImprovementMutationReceipt, - knowledgeImprovementCandidateRef, - knowledgeImprovementRunDir, - loadKnowledgeImprovementActivationResult, - loadKnowledgeImprovementEvents, - loadKnowledgeImprovementState, - promoteKnowledgeCandidate, - recoverPendingKnowledgeMutation, - restoreKnowledgeCandidateBaseline, - sha256, - stableId, - withKnowledgeImprovementCandidate, - withKnowledgeImprovementComparison, -} from '../src/index' -import { withKnowledgeMutation } from '../src/mutation-lock' - -async function withKb(fn: (root: string) => Promise): Promise { - const root = await mkdtemp(join(tmpdir(), 'agent-knowledge-improve-')) - try { - await initKnowledgeBase(root) - await fn(root) - } finally { - await rm(root, { recursive: true, force: true }) - } -} - -async function withEmptyRoot(fn: (root: string) => Promise): Promise { - const root = await mkdtemp(join(tmpdir(), 'agent-knowledge-improve-empty-')) - try { - await fn(root) - } finally { - await rm(root, { recursive: true, force: true }) - } -} - -function mutableCandidateRoot( - root: string, - result: { runId: string; candidate?: { candidateId: string } }, -): string { - if (!result.candidate) throw new Error('knowledge improvement result has no candidate') - return join( - knowledgeImprovementRunDir(root, result.runId), - 'candidates', - result.candidate.candidateId, - 'workspace', - ) -} - -const refundSpec = defineReadinessSpec({ - id: 'refund-policy', - description: 'Refund policy support knowledge', - query: 'refund policy customer billing refund', - requiredFor: ['support-agent'], - importance: 'blocking', - minHits: 1, - minSources: 1, -}) - -function refundSource() { - const text = 'The billing support refund policy allows refunds within 30 days with receipt proof.' - const uri = 'research://refund-policy' - return { - uri, - text, - title: 'Refund Policy Source', - id: stableId('src', `${sha256(text)}:${uri}`), - } -} - -function refundProposal(sourceId: string, extra = ''): string { - return [ - '---FILE: knowledge/support/refund-policy.md---', - '---', - 'id: refund-policy', - 'title: Refund Policy', - 'sources:', - ` - ${sourceId}`, - '---', - '# Refund Policy', - 'Billing support can grant a customer refund within 30 days when receipt proof is present.', - extra, - '---END FILE---', - ].join('\n') -} - -function passingMetric() { - return { - score: 1, - passed: true, - provenance: { - evaluator: 'agent-knowledge-test', - version: '1', - method: 'deterministic' as const, - }, - } -} - -function candidateDigest(seed: string): Sha256Digest { - return canonicalCandidateDigest({ seed }) -} - -function canonicalDocument>( - material: T, -): T & { digest: Sha256Digest } { - return { ...material, digest: canonicalCandidateDigest(material) } -} - -function knowledgeActivation( - candidate: KnowledgeImprovementCandidateRef, - intent: AgentImprovementActivation['intent'], - identity = 'knowledge:test', -): AgentImprovementActivation { - const expectedBaseHash = - intent === 'activate-candidate' ? candidate.baseHash : candidate.candidateHash - return canonicalDocument({ - kind: 'agent-improvement-activation' as const, - proposalDigest: candidateDigest('proposal'), - reviewDigest: candidateDigest('review'), - experimentDigest: candidateDigest('experiment'), - candidateBundleDigest: candidateDigest('candidate-bundle'), - intent, - targets: [ - { - surface: 'knowledge' as const, - identity, - expectedBaseDigest: `sha256:${expectedBaseHash}` as Sha256Digest, - }, - ] as AgentImprovementActivation['targets'], - fundingOwner: 'tenant:test', - authorizedBy: 'reviewer:test', - authorizedAt: '2026-07-17T00:00:00.000Z', - expiresAt: '2026-07-18T00:00:00.000Z', - }) -} - -function knowledgeActivationResult( - activation: AgentImprovementActivation, - candidate: KnowledgeImprovementCandidateRef, - mutation: KnowledgeImprovementMutationReceipt, - attemptedAt: string, - identity = 'knowledge:test', -): AgentImprovementActivationResult { - const desiredHash = - activation.intent === 'activate-candidate' ? candidate.candidateHash : candidate.baseHash - const outcome: AgentImprovementActivationResult['outcome'] = mutation.changed - ? { - status: 'applied', - transactionId: mutation.transactionId!, - targets: [ - { - surface: 'knowledge', - identity, - beforeDigest: `sha256:${mutation.beforeHash}`, - afterDigest: `sha256:${mutation.afterHash}`, - }, - ], - } - : mutation.afterHash === desiredHash - ? { - status: 'already-applied', - targets: [ - { - surface: 'knowledge', - identity, - currentDigest: `sha256:${mutation.afterHash}`, - }, - ], - } - : { - status: 'conflict', - targets: [ - { - surface: 'knowledge', - identity, - currentDigest: `sha256:${mutation.afterHash}`, - }, - ], - } - return canonicalDocument({ - kind: 'agent-improvement-activation-result' as const, - idempotencyKey: activation.digest, - attemptedAt, - completedAt: attemptedAt, - outcome, - }) -} - -async function improveAndPromote(options: Parameters[0]) { - const staged = await improveKnowledgeBase(options) - const promoted = await promoteKnowledgeCandidate({ - root: options.root, - candidate: knowledgeImprovementCandidateRef(staged), - }) - return { ...promoted, evaluation: staged.evaluation, lifecycle: staged.lifecycle } -} - -describe('improveKnowledgeBase', () => { - it('leaves the live knowledge base unchanged unless promotion is explicit', async () => { - await withEmptyRoot(async (root) => { - const before = await hashKnowledgeBase(root) - await expect(stat(join(root, '.agent-knowledge'))).rejects.toMatchObject({ code: 'ENOENT' }) - const result = await improveKnowledgeBase({ - root, - goal: 'Create a measured candidate without applying it', - runId: 'propose-only-default', - updateKnowledge: async ({ candidateRoot }) => { - await writeFile(join(candidateRoot, 'knowledge', 'candidate.md'), '# Candidate\n') - return { applied: true, summary: 'created candidate' } - }, - evaluate: passingMetric, - }) - - expect(result.promoted).toBe(false) - expect(result.state.status).toBe('candidate-ready') - expect(result.candidate?.candidateHash).not.toBe(before) - await expect(hashKnowledgeBase(root)).resolves.toBe(before) - await expect(readFile(join(root, 'knowledge', 'candidate.md'), 'utf8')).rejects.toMatchObject( - { - code: 'ENOENT', - }, - ) - await expect(stat(join(root, 'knowledge'))).rejects.toMatchObject({ code: 'ENOENT' }) - await expect(stat(join(root, 'raw'))).rejects.toMatchObject({ code: 'ENOENT' }) - await expect( - readFile(join(root, '.agent-knowledge', 'sources.json'), 'utf8'), - ).rejects.toMatchObject({ code: 'ENOENT' }) - }) - }) - - it('does not reapply a promoted candidate when the improvement run is reopened', async () => { - await withEmptyRoot(async (root) => { - const options = { - root, - goal: 'Keep candidate generation separate from activation', - runId: 'reopen-promoted-run', - updateKnowledge: async ({ candidateRoot }: { candidateRoot: string }) => { - await writeFile(join(candidateRoot, 'knowledge', 'candidate.md'), '# Candidate\n') - return { applied: true, summary: 'created candidate' } - }, - evaluate: passingMetric, - } - const staged = await improveKnowledgeBase(options) - const candidate = knowledgeImprovementCandidateRef(staged) - await promoteKnowledgeCandidate({ root, candidate }) - await rm(join(root, 'knowledge', 'candidate.md')) - const liveBeforeReopen = await hashKnowledgeBase(root) - - await expect(improveKnowledgeBase(options)).rejects.toThrow(/promoted knowledge base changed/) - expect(await hashKnowledgeBase(root)).toBe(liveBeforeReopen) - await expect(readFile(join(root, 'knowledge', 'candidate.md'), 'utf8')).rejects.toMatchObject( - { code: 'ENOENT' }, - ) - }) - }) - - it('rejects evaluator results without provenance', async () => { - await withKb(async (root) => { - await expect( - improveKnowledgeBase({ - root, - goal: 'Reject an anonymous evaluation', - runId: 'anonymous-evaluator', - updateKnowledge: async ({ candidateRoot }) => { - await writeFile(join(candidateRoot, 'knowledge', 'candidate.md'), '# Candidate\n') - return { applied: true, summary: 'created candidate' } - }, - evaluate: (() => ({ score: 1, passed: true })) as never, - }), - ).rejects.toThrow(/provenance/) - }) - }) - - it('does not create improvement state through a symbolic-link metadata directory', async () => { - await withEmptyRoot(async (root) => { - const outside = await mkdtemp(join(tmpdir(), 'agent-knowledge-improvement-outside-')) - try { - await symlink(outside, join(root, '.agent-knowledge')) - await expect( - improveKnowledgeBase({ - root, - goal: 'Reject redirected improvement state', - runId: 'redirected-state', - }), - ).rejects.toThrow(/unsafe directory/) - await expect(stat(join(outside, 'improvements'))).rejects.toMatchObject({ code: 'ENOENT' }) - } finally { - await rm(outside, { recursive: true, force: true }) - } - }) - }) - - it.skipIf(process.platform !== 'linux')( - 'rejects a run directory swapped while candidate work is active', - async () => { - await withKb(async (root) => { - const outside = await mkdtemp(join(tmpdir(), 'agent-knowledge-run-outside-')) - const runId = 'run-directory-swap' - const runDir = knowledgeImprovementRunDir(root, runId) - try { - await expect( - improveKnowledgeBase({ - root, - goal: 'Reject redirected run state', - runId, - async updateKnowledge({ candidateRoot }) { - await writeFile(join(candidateRoot, 'knowledge', 'candidate.md'), '# Candidate\n') - await rename(runDir, `${runDir}-original`) - await symlink(outside, runDir) - return { applied: true, summary: 'candidate written before redirect' } - }, - evaluate: passingMetric, - }), - ).rejects.toThrow(/unsafe directory|changed during use/) - await expect(readFile(join(outside, 'state.json'), 'utf8')).rejects.toMatchObject({ - code: 'ENOENT', - }) - } finally { - await rm(outside, { recursive: true, force: true }) - } - }) - }, - ) - - it('evaluates root-level KB readiness without running an improvement loop', async () => { - await withKb(async (root) => { - const empty = await evaluateKnowledgeBaseReadiness({ - root, - goal: 'Build billing support refund-policy knowledge', - readinessSpecs: [refundSpec], - strict: true, - }) - - expect(empty.ready).toBe(false) - expect(empty.dimensions.blocking_readiness).toBe(0) - expect(empty.summary).toContain('blocking readiness requirement') - - const source = refundSource() - const improved = await improveAndPromote({ - root, - goal: 'Build billing support refund-policy knowledge', - runId: 'readiness-evaluator', - readinessSpecs: [refundSpec], - strict: true, - step: () => ({ - done: true, - sourceTexts: [{ uri: source.uri, text: source.text, title: source.title }], - proposalText: refundProposal(source.id), - }), - }) - expect(improved.promoted).toBe(true) - - const ready = await evaluateKnowledgeBaseReadiness({ - root, - goal: 'Build billing support refund-policy knowledge', - readinessSpecs: [refundSpec], - strict: true, - }) - - expect(ready.ready).toBe(true) - expect(ready.summary).toBe('knowledge base passed readiness checks') - expect(ready.dimensions).toMatchObject({ - validation: 1, - kb_quality: 1, - blocking_readiness: 1, - }) - }) - }) - - it('promotes a candidate KB after readiness separates weak from strong', async () => { - await withKb(async (root) => { - const emptyIndex = await buildKnowledgeIndex(root) - const emptyReadiness = buildEvalKnowledgeBundle({ - taskId: 'support-agent', - index: emptyIndex, - specs: [refundSpec], - }) - expect(emptyReadiness.report.blockingMissingRequirements).toHaveLength(1) - - const source = refundSource() - const result = await improveAndPromote({ - root, - goal: 'Build billing support refund-policy knowledge', - runId: 'refund-improvement', - readinessSpecs: [refundSpec], - strict: true, - step: () => ({ - done: true, - sourceTexts: [{ uri: source.uri, text: source.text, title: source.title }], - proposalText: refundProposal(source.id), - }), - }) - - expect(result.promoted).toBe(true) - expect(result.state.status).toBe('promoted') - expect(result.evaluation?.score).toBe(1) - expect( - result.lifecycle?.phases.some( - (phase) => phase.phase === 'knowledge-update' && phase.status === 'completed', - ), - ).toBe(true) - - const promotedIndex = await buildKnowledgeIndex(root) - expect(promotedIndex.pages.map((page) => page.id)).toContain('refund-policy') - const promotedReadiness = buildEvalKnowledgeBundle({ - taskId: 'support-agent', - index: promotedIndex, - specs: [refundSpec], - }) - expect(promotedReadiness.report.blockingMissingRequirements).toHaveLength(0) - }) - }) - - it('resumes a candidate workspace after an external edit and promotes the edited artifact', async () => { - await withKb(async (root) => { - const source = refundSource() - const first = await improveKnowledgeBase({ - root, - goal: 'Build billing support refund-policy knowledge', - runId: 'resume-edit', - readinessSpecs: [refundSpec], - strict: true, - step: () => ({ - done: true, - sourceTexts: [{ uri: source.uri, text: source.text, title: source.title }], - proposalText: refundProposal(source.id), - }), - }) - - expect(first.promoted).toBe(false) - expect(first.state.status).toBe('candidate-ready') - const candidateRoot = mutableCandidateRoot(root, first) - - const editedPage = join(candidateRoot!, 'knowledge', 'support', 'refund-escalation.md') - await mkdir(dirname(editedPage), { recursive: true }) - await writeFile( - editedPage, - [ - '---', - 'id: refund-escalation', - 'title: Refund Escalation', - 'sources:', - ` - ${source.id}`, - '---', - '# Refund Escalation', - 'Escalate refund requests that lack receipt proof.', - ].join('\n'), - ) - - const resumed = await improveAndPromote({ - root, - goal: 'Build billing support refund-policy knowledge', - runId: 'resume-edit', - readinessSpecs: [refundSpec], - strict: true, - }) - - expect(resumed.promoted).toBe(true) - const promoted = await readFile( - join(root, 'knowledge', 'support', 'refund-escalation.md'), - 'utf8', - ) - expect(promoted).toContain('Escalate refund requests') - }) - }) - - it('promotes one exact approved candidate without rerunning candidate work', async () => { - await withKb(async (root) => { - const source = refundSource() - let updateCalls = 0 - const staged = await improveKnowledgeBase({ - root, - goal: 'Build billing support refund-policy knowledge', - runId: 'approved-candidate', - readinessSpecs: [refundSpec], - strict: true, - step: () => { - updateCalls += 1 - return { - done: true, - sourceTexts: [{ uri: source.uri, text: source.text, title: source.title }], - proposalText: refundProposal(source.id), - } - }, - }) - const candidate = knowledgeImprovementCandidateRef(staged) - - const promoted = await promoteKnowledgeCandidate({ root, candidate }) - expect(promoted.promoted).toBe(true) - expect(promoted.state.promotedCandidateId).toBe(candidate.candidateId) - expect(updateCalls).toBe(1) - await expect( - readFile(join(root, 'knowledge', 'support', 'refund-policy.md'), 'utf8'), - ).resolves.toContain('refund within 30 days') - - const repeated = await promoteKnowledgeCandidate({ root, candidate }) - expect(repeated.promoted).toBe(true) - expect(repeated.state.promotedCandidateId).toBe(candidate.candidateId) - expect(repeated.mutation).toEqual({ - target: 'candidate', - beforeHash: candidate.candidateHash, - afterHash: candidate.candidateHash, - changed: false, - transactionId: null, - recovered: false, - }) - expect(updateCalls).toBe(1) - }) - }) - - it('restores the exact frozen baseline and can apply the same candidate again', async () => { - await withKb(async (root) => { - const originalPage = join(root, 'knowledge', 'original.md') - const originalSource = join(root, 'raw', 'sources', 'original.txt') - await writeFile(originalPage, '# Original\n') - await writeFile(originalSource, 'original evidence\n') - const baseHash = await hashKnowledgeBase(root) - const staged = await improveKnowledgeBase({ - root, - goal: 'Replace the original knowledge with a measured candidate', - runId: 'restore-baseline', - updateKnowledge: async ({ candidateRoot }) => { - await writeFile(join(candidateRoot, 'knowledge', 'original.md'), '# Changed\n') - await writeFile(join(candidateRoot, 'knowledge', 'candidate.md'), '# Candidate\n') - await rm(join(candidateRoot, 'raw', 'sources', 'original.txt')) - return { applied: true, summary: 'changed, added, and deleted measured files' } - }, - evaluate: passingMetric, - }) - const candidate = knowledgeImprovementCandidateRef(staged) - - const promoted = await promoteKnowledgeCandidate({ root, candidate }) - expect(promoted).toMatchObject({ - promoted: true, - blocked: false, - }) - expect(promoted.mutation).toEqual({ - target: 'candidate', - beforeHash: baseHash, - afterHash: candidate.candidateHash, - changed: true, - transactionId: expect.any(String), - recovered: false, - }) - expect(await hashKnowledgeBase(root)).toBe(candidate.candidateHash) - - const restored = await restoreKnowledgeCandidateBaseline({ root, candidate }) - expect(restored).toMatchObject({ - promoted: false, - blocked: false, - state: { status: 'candidate-ready' }, - candidate: { status: 'candidate-ready' }, - }) - expect(restored.mutation).toEqual({ - target: 'baseline', - beforeHash: candidate.candidateHash, - afterHash: baseHash, - changed: true, - transactionId: expect.any(String), - recovered: false, - }) - expect(restored.state.promotedCandidateId).toBeUndefined() - expect(await hashKnowledgeBase(root)).toBe(baseHash) - await expect(readFile(originalPage, 'utf8')).resolves.toBe('# Original\n') - await expect(readFile(originalSource, 'utf8')).resolves.toBe('original evidence\n') - await expect(readFile(join(root, 'knowledge', 'candidate.md'), 'utf8')).rejects.toMatchObject( - { - code: 'ENOENT', - }, - ) - - await expect(promoteKnowledgeCandidate({ root, candidate })).resolves.toMatchObject({ - promoted: true, - blocked: false, - }) - expect(await hashKnowledgeBase(root)).toBe(candidate.candidateHash) - const events = await loadKnowledgeImprovementEvents(root, candidate.runId) - expect(events.filter((event) => event.type === 'candidate.promoted')).toHaveLength(1) - expect(events.filter((event) => event.type === 'candidate.restored')).toHaveLength(1) - }) - }) - - it('finishes an interrupted restore without repeating the mutation', async () => { - await withKb(async (root) => { - const staged = await improveKnowledgeBase({ - root, - goal: 'Create knowledge that can be restored after interruption', - runId: 'interrupted-restore', - updateKnowledge: async ({ candidateRoot }) => { - await writeFile(join(candidateRoot, 'knowledge', 'candidate.md'), '# Candidate\n') - return { applied: true, summary: 'created measured knowledge' } - }, - evaluate: passingMetric, - }) - const candidate = knowledgeImprovementCandidateRef(staged) - await promoteKnowledgeCandidate({ root, candidate }) - - await expect( - restoreKnowledgeCandidateBaseline({ - root, - candidate, - onState() { - throw new Error('operator stopped after restore state persistence') - }, - }), - ).rejects.toThrow(/operator stopped/) - - const recovered = await restoreKnowledgeCandidateBaseline({ root, candidate }) - expect(recovered).toMatchObject({ - promoted: false, - blocked: false, - state: { status: 'candidate-ready' }, - mutation: { - target: 'baseline', - beforeHash: candidate.candidateHash, - afterHash: candidate.baseHash, - changed: true, - transactionId: expect.any(String), - recovered: true, - }, - }) - expect(await hashKnowledgeBase(root)).toBe(candidate.baseHash) - const events = await loadKnowledgeImprovementEvents(root, candidate.runId) - expect(events.filter((event) => event.type === 'candidate.restored')).toHaveLength(1) - const transactions = await readdir(join(root, '.agent-knowledge', 'file-transactions')) - expect(transactions.filter((entry) => entry.startsWith('active-'))).toEqual([]) - - await expect(restoreKnowledgeCandidateBaseline({ root, candidate })).resolves.toMatchObject({ - promoted: false, - blocked: false, - }) - const repeatedEvents = await loadKnowledgeImprovementEvents(root, candidate.runId) - expect(repeatedEvents.filter((event) => event.type === 'candidate.restored')).toHaveLength(1) - }) - }) - - it('blocks restore without overwriting knowledge changed after promotion', async () => { - await withKb(async (root) => { - const page = join(root, 'knowledge', 'candidate.md') - const staged = await improveKnowledgeBase({ - root, - goal: 'Preserve concurrent edits during restore', - runId: 'restore-conflict', - updateKnowledge: async ({ candidateRoot }) => { - await writeFile(join(candidateRoot, 'knowledge', 'candidate.md'), '# Candidate\n') - return { applied: true, summary: 'created measured knowledge' } - }, - evaluate: passingMetric, - }) - const candidate = knowledgeImprovementCandidateRef(staged) - await promoteKnowledgeCandidate({ root, candidate }) - await writeFile(page, '# Concurrent change\n') - - const restored = await restoreKnowledgeCandidateBaseline({ root, candidate }) - - expect(restored).toMatchObject({ - promoted: false, - blocked: true, - state: { status: 'blocked' }, - candidate: { status: 'blocked' }, - }) - expect(restored.state.promotedCandidateId).toBeUndefined() - await expect(readFile(page, 'utf8')).resolves.toBe('# Concurrent change\n') - }) - }) - - it('finishes an interrupted approved promotion without rerunning candidate work', async () => { - await withKb(async (root) => { - const source = refundSource() - let updateCalls = 0 - const staged = await improveKnowledgeBase({ - root, - goal: 'Build billing support refund-policy knowledge', - runId: 'interrupted-promotion', - readinessSpecs: [refundSpec], - strict: true, - step: () => { - updateCalls += 1 - return { - done: true, - sourceTexts: [{ uri: source.uri, text: source.text, title: source.title }], - proposalText: refundProposal(source.id), - } - }, - }) - const candidate = knowledgeImprovementCandidateRef(staged) - - await expect( - promoteKnowledgeCandidate({ - root, - candidate, - onState() { - throw new Error('operator stopped after state persistence') - }, - }), - ).rejects.toThrow(/operator stopped/) - - const runDir = knowledgeImprovementRunDir(root, candidate.runId) - await rm(join(runDir, 'candidates', candidate.candidateId, 'snapshots'), { - recursive: true, - force: true, - }) - await rm(join(runDir, 'candidates', candidate.candidateId, 'evidence.json'), { - force: true, - }) - await expect( - applyKnowledgeWriteBlocks( - root, - ['---FILE: knowledge/unapproved.md---', '# Unapproved', '---END FILE---'].join('\n'), - ), - ).rejects.toThrow(/requires its owner to resume/) - const recovered = await promoteKnowledgeCandidate({ root, candidate }) - - expect(recovered.promoted).toBe(true) - expect(recovered.mutation).toEqual({ - target: 'candidate', - beforeHash: candidate.baseHash, - afterHash: candidate.candidateHash, - changed: true, - transactionId: expect.any(String), - recovered: true, - }) - expect(updateCalls).toBe(1) - const events = await loadKnowledgeImprovementEvents(root, candidate.runId) - expect(events.filter((event) => event.type === 'candidate.promoted')).toHaveLength(1) - const transactionEntries = await readdir(join(root, '.agent-knowledge', 'file-transactions')) - expect(transactionEntries.filter((entry) => entry.startsWith('active-'))).toEqual([]) - }) - }) - - it('persists an approved result before closing its file transaction and never reapplies it', async () => { - await withKb(async (root) => { - const staged = await improveKnowledgeBase({ - root, - goal: 'Keep approval results recoverable with their knowledge mutation', - runId: 'protected-activation-recovery', - updateKnowledge: async ({ candidateRoot }) => { - await writeFile(join(candidateRoot, 'knowledge', 'candidate.md'), '# Candidate\n') - return { applied: true, summary: 'created measured knowledge' } - }, - evaluate: passingMetric, - }) - const candidate = knowledgeImprovementCandidateRef(staged) - const activation = knowledgeActivation(candidate, 'activate-candidate') - const attemptedAt = '2026-07-17T12:00:00.000Z' - const transactionRoot = join(root, '.agent-knowledge', 'file-transactions') - let interruptedMutation: KnowledgeImprovementMutationReceipt | undefined - - await expect( - promoteKnowledgeCandidate({ - root, - candidate, - activation: { - activation, - attemptedAt, - identity: 'knowledge:test', - createResult(mutation) { - interruptedMutation = mutation - throw new Error('activation result store unavailable') - }, - }, - }), - ).rejects.toThrow(/activation result store unavailable/) - expect(interruptedMutation).toMatchObject({ - target: 'candidate', - beforeHash: candidate.baseHash, - afterHash: candidate.candidateHash, - changed: true, - transactionId: expect.any(String), - recovered: false, - }) - await expect(hashKnowledgeBase(root)).rejects.toThrow(/requires its owner to resume/) - await expect(readFile(join(root, 'knowledge', 'candidate.md'), 'utf8')).resolves.toBe( - '# Candidate\n', - ) - expect( - (await readdir(transactionRoot)).filter((entry) => entry.startsWith('active-')), - ).toHaveLength(1) - const pending = await inspectPendingKnowledgeMutation(root) - expect(pending).toMatchObject({ - transactionId: interruptedMutation?.transactionId, - recoveryOwner: `knowledge-improvement-activation:${activation.digest}`, - }) - await expect( - recoverPendingKnowledgeMutation(root, { - transactionId: pending!.transactionId, - action: 'apply', - }), - ).rejects.toThrow(/must be resumed by 'knowledge-improvement-activation:sha256:/) - await expect(promoteKnowledgeCandidate({ root, candidate })).rejects.toThrow( - /must be resumed by 'knowledge-improvement-activation:sha256:/, - ) - await expect( - loadKnowledgeImprovementActivationResult({ - root, - candidate, - activation, - identity: 'knowledge:test', - }), - ).resolves.toBeNull() - await expect( - improveKnowledgeBase({ - root, - goal: 'Keep approval results recoverable with their knowledge mutation', - runId: 'protected-activation-recovery', - }), - ).rejects.toThrow(/requires its owner to resume/) - - const recovered = await promoteKnowledgeCandidate({ - root, - candidate, - activation: { - activation, - attemptedAt, - identity: 'knowledge:test', - createResult: (mutation) => - knowledgeActivationResult(activation, candidate, mutation, attemptedAt), - }, - }) - expect(recovered.mutation).toEqual({ - ...interruptedMutation, - recovered: true, - }) - expect(recovered.activationResult?.outcome.status).toBe('applied') - expect( - (await readdir(transactionRoot)).filter((entry) => entry.startsWith('active-')), - ).toEqual([]) - await expect( - loadKnowledgeImprovementActivationResult({ - root, - candidate, - activation, - identity: 'knowledge:test', - }), - ).resolves.toEqual(recovered.activationResult) - - await restoreKnowledgeCandidateBaseline({ root, candidate }) - await expect(hashKnowledgeBase(root)).resolves.toBe(candidate.baseHash) - const retried = await promoteKnowledgeCandidate({ - root, - candidate, - activation: { - activation, - attemptedAt, - identity: 'knowledge:test', - createResult() { - throw new Error('durable activation must not be recomputed') - }, - }, - }) - expect(retried.activationResult).toEqual(recovered.activationResult) - await expect(hashKnowledgeBase(root)).resolves.toBe(candidate.baseHash) - }) - }) - - it('repairs blocked run state and its event from a durable conflict result', async () => { - await withKb(async (root) => { - const staged = await improveKnowledgeBase({ - root, - goal: 'Keep conflict results and run state consistent', - runId: 'activation-conflict-recovery', - updateKnowledge: async ({ candidateRoot }) => { - await writeFile(join(candidateRoot, 'knowledge', 'candidate.md'), '# Candidate\n') - return { applied: true, summary: 'created measured knowledge' } - }, - evaluate: passingMetric, - }) - const candidate = knowledgeImprovementCandidateRef(staged) - await promoteKnowledgeCandidate({ root, candidate }) - await writeFile(join(root, 'knowledge', 'candidate.md'), '# Concurrent change\n') - const activation = knowledgeActivation(candidate, 'restore-baseline') - const attemptedAt = '2026-07-17T12:00:00.000Z' - - await expect( - restoreKnowledgeCandidateBaseline({ - root, - candidate, - activation: { - activation, - attemptedAt, - identity: 'knowledge:test', - createResult: (mutation) => - knowledgeActivationResult(activation, candidate, mutation, attemptedAt), - }, - onState() { - throw new Error('operator stopped before conflict event persistence') - }, - }), - ).rejects.toThrow(/operator stopped before conflict event persistence/) - const stored = await loadKnowledgeImprovementActivationResult({ - root, - candidate, - activation, - identity: 'knowledge:test', - }) - expect(stored?.outcome.status).toBe('conflict') - expect( - (await loadKnowledgeImprovementEvents(root, candidate.runId)).filter( - (event) => event.type === 'restore.blocked', - ), - ).toHaveLength(0) - - const recovered = await restoreKnowledgeCandidateBaseline({ - root, - candidate, - activation: { - activation, - attemptedAt, - identity: 'knowledge:test', - createResult() { - throw new Error('durable conflict result must not be recomputed') - }, - }, - }) - expect(recovered).toMatchObject({ - promoted: false, - blocked: true, - state: { status: 'blocked' }, - candidate: { status: 'blocked' }, - }) - expect(recovered.activationResult).toEqual(stored) - await restoreKnowledgeCandidateBaseline({ - root, - candidate, - activation: { - activation, - attemptedAt, - identity: 'knowledge:test', - createResult() { - throw new Error('durable conflict result must not be recomputed') - }, - }, - }) - expect( - (await loadKnowledgeImprovementEvents(root, candidate.runId)).filter( - (event) => event.type === 'restore.blocked', - ), - ).toHaveLength(1) - }) - }) - - it('binds already-applied promotion and baseline restore to their exact activation', async () => { - await withKb(async (root) => { - const staged = await improveKnowledgeBase({ - root, - goal: 'Record exact no-op and restore activation outcomes', - runId: 'activation-already-and-restore', - updateKnowledge: async ({ candidateRoot }) => { - await writeFile(join(candidateRoot, 'knowledge', 'candidate.md'), '# Candidate\n') - return { applied: true, summary: 'created measured knowledge' } - }, - evaluate: passingMetric, - }) - const candidate = knowledgeImprovementCandidateRef(staged) - await promoteKnowledgeCandidate({ root, candidate }) - const attemptedAt = '2026-07-17T12:00:00.000Z' - const promoteActivation = knowledgeActivation(candidate, 'activate-candidate') - - const already = await promoteKnowledgeCandidate({ - root, - candidate, - activation: { - activation: promoteActivation, - attemptedAt, - identity: 'knowledge:test', - createResult: (mutation) => - knowledgeActivationResult(promoteActivation, candidate, mutation, attemptedAt), - }, - }) - expect(already.mutation).toEqual({ - target: 'candidate', - beforeHash: candidate.candidateHash, - afterHash: candidate.candidateHash, - changed: false, - transactionId: null, - recovered: false, - }) - expect(already.activationResult?.outcome.status).toBe('already-applied') - await expect( - loadKnowledgeImprovementActivationResult({ - root, - candidate, - activation: promoteActivation, - identity: 'knowledge:test', - }), - ).resolves.toEqual(already.activationResult) - - const restoreActivation = knowledgeActivation(candidate, 'restore-baseline') - const restored = await restoreKnowledgeCandidateBaseline({ - root, - candidate, - activation: { - activation: restoreActivation, - attemptedAt, - identity: 'knowledge:test', - createResult: (mutation) => - knowledgeActivationResult(restoreActivation, candidate, mutation, attemptedAt), - }, - }) - expect(restored.mutation).toMatchObject({ - target: 'baseline', - beforeHash: candidate.candidateHash, - afterHash: candidate.baseHash, - changed: true, - transactionId: expect.any(String), - recovered: false, - }) - expect(restored.activationResult?.outcome.status).toBe('applied') - await expect( - loadKnowledgeImprovementActivationResult({ - root, - candidate, - activation: restoreActivation, - identity: 'knowledge:test', - }), - ).resolves.toEqual(restored.activationResult) - await expect(hashKnowledgeBase(root)).resolves.toBe(candidate.baseHash) - }) - }) - - it('rejects a forged promotion journal entry outside the measured knowledge files', async () => { - await withKb(async (root) => { - const packagePath = join(root, 'package.json') - const originalPackage = '{"private":true}\n' - const forgedPackage = '{"scripts":{"postinstall":"malicious"}}\n' - await writeFile(packagePath, originalPackage) - const staged = await improveKnowledgeBase({ - root, - goal: 'Prepare an exact knowledge-only candidate', - runId: 'forged-promotion-journal', - updateKnowledge: async ({ candidateRoot }) => { - await writeFile(join(candidateRoot, 'knowledge', 'candidate.md'), '# Candidate\n') - return { applied: true, summary: 'created candidate' } - }, - evaluate: passingMetric, - }) - const candidate = knowledgeImprovementCandidateRef(staged) - await expect( - promoteKnowledgeCandidate({ - root, - candidate, - onState() { - throw new Error('leave promotion pending') - }, - }), - ).rejects.toThrow(/leave promotion pending/) - - const transactionRoot = join(root, '.agent-knowledge', 'file-transactions') - const activeTransactions = (await readdir(transactionRoot)).filter((entry) => - entry.startsWith('active-'), - ) - expect(activeTransactions).toHaveLength(1) - const transactionDir = join(transactionRoot, activeTransactions[0]!) - const transactionPath = join(transactionDir, 'transaction.json') - const transaction = JSON.parse(await readFile(transactionPath, 'utf8')) as { - entries: Array<{ - index: number - path: string - beforeHash: string | null - afterHash: string | null - beforeMode?: number - afterMode?: number - }> - } - const index = Math.max(...transaction.entries.map((entry) => entry.index)) + 1 - transaction.entries.push({ - index, - path: 'package.json', - beforeHash: createHash('sha256').update(originalPackage).digest('hex'), - afterHash: createHash('sha256').update(forgedPackage).digest('hex'), - beforeMode: 0o644, - afterMode: 0o644, - }) - await mkdir(join(transactionDir, 'after'), { recursive: true }) - await writeFile(join(transactionDir, 'after', `${index}.bin`), forgedPackage) - await writeFile(transactionPath, `${JSON.stringify(transaction, null, 2)}\n`) - - await expect(promoteKnowledgeCandidate({ root, candidate })).rejects.toThrow( - /unsupported path: package.json/, - ) - await expect(readFile(packagePath, 'utf8')).resolves.toBe(originalPackage) - }) - }) - - it('promotes frozen measured bytes when the mutable workspace changes after approval', async () => { - await withKb(async (root) => { - const source = refundSource() - const staged = await improveKnowledgeBase({ - root, - goal: 'Build billing support refund-policy knowledge', - runId: 'changed-after-approval', - readinessSpecs: [refundSpec], - strict: true, - step: () => ({ - done: true, - sourceTexts: [{ uri: source.uri, text: source.text, title: source.title }], - proposalText: refundProposal(source.id), - }), - }) - const candidate = knowledgeImprovementCandidateRef(staged) - await writeFile( - join(mutableCandidateRoot(root, staged), 'knowledge', 'support', 'refund-policy.md'), - '# changed after approval\n', - ) - - await expect(promoteKnowledgeCandidate({ root, candidate })).resolves.toMatchObject({ - promoted: true, - }) - await expect( - readFile(join(root, 'knowledge', 'support', 'refund-policy.md'), 'utf8'), - ).resolves.toContain('refund within 30 days') - await expect( - readFile(join(root, 'knowledge', 'support', 'refund-policy.md'), 'utf8'), - ).resolves.not.toContain('changed after approval') - }) - }) - - it('resolves and promotes the measured snapshot after mutable workspace removal', async () => { - await withKb(async (root) => { - const staged = await improveKnowledgeBase({ - root, - goal: 'Keep the measured candidate independent from scratch files', - runId: 'removed-mutable-workspace', - updateKnowledge: async ({ candidateRoot }) => { - await writeFile(join(candidateRoot, 'knowledge', 'measured.md'), '# Measured\n') - return { applied: true, summary: 'created measured knowledge' } - }, - evaluate: passingMetric, - }) - const candidate = knowledgeImprovementCandidateRef(staged) - await rm(mutableCandidateRoot(root, staged), { recursive: true, force: true }) - const runDir = knowledgeImprovementRunDir(root, candidate.runId) - let isolatedRoot = '' - - await withKnowledgeImprovementCandidate({ root, candidate }, async (resolved) => { - isolatedRoot = resolved.root - expect(relative(runDir, resolved.root)).toMatch(/^\.\./) - await expect( - readFile(join(resolved.root, 'knowledge', 'measured.md'), 'utf8'), - ).resolves.toBe('# Measured\n') - }) - await expect(stat(isolatedRoot)).rejects.toMatchObject({ code: 'ENOENT' }) - await expect(promoteKnowledgeCandidate({ root, candidate })).resolves.toMatchObject({ - promoted: true, - }) - }) - }) - - it('resolves the exact frozen baseline and candidate from one measured comparison', async () => { - await withKb(async (root) => { - const liveBefore = await hashKnowledgeBase(root) - const staged = await improveKnowledgeBase({ - root, - goal: 'Compare the frozen baseline and candidate bytes', - runId: 'paired-frozen-snapshots', - updateKnowledge: async ({ candidateRoot }) => { - await writeFile(join(candidateRoot, 'knowledge', 'measured.md'), '# Measured\n') - return { applied: true, summary: 'created measured knowledge' } - }, - evaluate: () => ({ ...passingMetric(), dimensions: { quality: 1 } }), - }) - const candidate = knowledgeImprovementCandidateRef(staged) - const isolatedRoots: string[] = [] - - await withKnowledgeImprovementComparison({ root, candidate }, async (comparison) => { - isolatedRoots.push(comparison.baseline.root, comparison.candidate.root) - expect(Object.isFrozen(comparison)).toBe(true) - expect(Object.isFrozen(comparison.baseline)).toBe(true) - expect(Object.isFrozen(comparison.candidate)).toBe(true) - expect(Object.isFrozen(comparison.evaluation)).toBe(true) - expect(Object.isFrozen(comparison.evaluation.provenance)).toBe(true) - expect(Object.isFrozen(comparison.evaluation.dimensions)).toBe(true) - expect(comparison.reference).toEqual(candidate) - expect(comparison.baseline.hash).toBe(candidate.baseHash) - expect(comparison.candidate.hash).toBe(candidate.candidateHash) - await expect(hashKnowledgeBase(comparison.baseline.root)).resolves.toBe(candidate.baseHash) - await expect(hashKnowledgeBase(comparison.candidate.root)).resolves.toBe( - candidate.candidateHash, - ) - await expect( - readFile(join(comparison.baseline.root, 'knowledge', 'measured.md'), 'utf8'), - ).rejects.toMatchObject({ code: 'ENOENT' }) - await expect( - readFile(join(comparison.candidate.root, 'knowledge', 'measured.md'), 'utf8'), - ).resolves.toBe('# Measured\n') - }) - await expect( - withKnowledgeImprovementComparison({ root, candidate }, ({ baseline }) => - writeFile(join(baseline.root, 'knowledge', 'changed.md'), '# Changed\n'), - ), - ).rejects.toThrow(/baseline snapshot changed during use/) - - await expect(hashKnowledgeBase(root)).resolves.toBe(liveBefore) - for (const isolatedRoot of isolatedRoots) { - await expect(stat(isolatedRoot)).rejects.toMatchObject({ code: 'ENOENT' }) - } - }) - }) - - it('rejects a measured snapshot changed while an approved candidate is in use', async () => { - await withKb(async (root) => { - const staged = await improveKnowledgeBase({ - root, - goal: 'Keep approved candidate execution immutable', - runId: 'changed-during-candidate-use', - updateKnowledge: async ({ candidateRoot }) => { - await writeFile(join(candidateRoot, 'knowledge', 'measured.md'), '# Measured\n') - return { applied: true, summary: 'created measured knowledge' } - }, - evaluate: passingMetric, - }) - const candidate = knowledgeImprovementCandidateRef(staged) - - await expect( - withKnowledgeImprovementCandidate({ root, candidate }, async (resolved) => { - await writeFile(join(resolved.root, 'knowledge', 'measured.md'), '# Changed\n') - }), - ).rejects.toThrow(/snapshot changed during use/) - }) - }) - - it('does not return a successful mutation receipt when a state callback changes knowledge', async () => { - await withKb(async (root) => { - const staged = await improveKnowledgeBase({ - root, - goal: 'Reject post-commit knowledge changes', - runId: 'post-commit-change', - updateKnowledge: async ({ candidateRoot }) => { - await writeFile(join(candidateRoot, 'knowledge', 'candidate.md'), '# Candidate\n') - return { applied: true, summary: 'created measured knowledge' } - }, - evaluate: passingMetric, - }) - const candidate = knowledgeImprovementCandidateRef(staged) - - await expect( - promoteKnowledgeCandidate({ - root, - candidate, - async onState() { - await writeFile(join(root, 'knowledge', 'unmeasured.md'), '# Unmeasured\n') - }, - }), - ).rejects.toThrow(/changed before its result was returned/) - await expect(hashKnowledgeBase(root)).rejects.toThrow(/requires its owner to resume/) - await expect(readFile(join(root, 'knowledge', 'unmeasured.md'), 'utf8')).resolves.toBe( - '# Unmeasured\n', - ) - }) - }) - - it('rejects a frozen measured copy changed after approval', async () => { - await withKb(async (root) => { - const source = refundSource() - const staged = await improveKnowledgeBase({ - root, - goal: 'Build billing support refund-policy knowledge', - runId: 'changed-frozen-candidate', - readinessSpecs: [refundSpec], - strict: true, - step: () => ({ - done: true, - sourceTexts: [{ uri: source.uri, text: source.text, title: source.title }], - proposalText: refundProposal(source.id), - }), - }) - const candidate = knowledgeImprovementCandidateRef(staged) - const snapshotRoot = join( - knowledgeImprovementRunDir(root, candidate.runId), - 'candidates', - candidate.candidateId, - 'snapshots', - candidate.candidateHash, - ) - await writeFile( - join(snapshotRoot, 'knowledge', 'support', 'refund-policy.md'), - '# tampered measured copy\n', - ) - - await expect(promoteKnowledgeCandidate({ root, candidate })).rejects.toThrow( - /snapshot changed after approval/, - ) - await expect( - readFile(join(root, 'knowledge', 'support', 'refund-policy.md'), 'utf8'), - ).rejects.toMatchObject({ code: 'ENOENT' }) - }) - }) - - it('promotes candidate deletions exactly', async () => { - await withKb(async (root) => { - const obsoletePage = join(root, 'knowledge', 'obsolete.md') - const obsoleteRaw = join(root, 'raw', 'sources', 'obsolete.txt') - await writeFile(obsoletePage, '# Obsolete\n') - await writeFile(obsoleteRaw, 'obsolete source\n') - const staged = await improveKnowledgeBase({ - root, - goal: 'Remove obsolete knowledge', - runId: 'approved-deletions', - updateKnowledge: async ({ candidateRoot }) => { - await rm(join(candidateRoot, 'knowledge', 'obsolete.md')) - await rm(join(candidateRoot, 'raw', 'sources', 'obsolete.txt')) - return { applied: true, summary: 'removed obsolete knowledge' } - }, - evaluate: passingMetric, - }) - const candidate = knowledgeImprovementCandidateRef(staged) - - await promoteKnowledgeCandidate({ root, candidate }) - - await expect(readFile(obsoletePage, 'utf8')).rejects.toMatchObject({ code: 'ENOENT' }) - await expect(readFile(obsoleteRaw, 'utf8')).rejects.toMatchObject({ code: 'ENOENT' }) - await expect(hashKnowledgeBase(root)).resolves.toBe(candidate.candidateHash) - }) - }) - - it.skipIf(process.platform !== 'linux')('promotes a mode-only candidate exactly', async () => { - await withKb(async (root) => { - const page = join(root, 'knowledge', 'mode-only.md') - await writeFile(page, '# Same bytes\n') - await chmod(page, 0o600) - const staged = await improveKnowledgeBase({ - root, - goal: 'Make the measured page executable', - runId: 'mode-only-promotion', - updateKnowledge: async ({ candidateRoot }) => { - await chmod(join(candidateRoot, 'knowledge', 'mode-only.md'), 0o700) - return { applied: true, summary: 'changed the measured file mode' } - }, - evaluate: passingMetric, - }) - const candidate = knowledgeImprovementCandidateRef(staged) - - expect(candidate.candidateHash).not.toBe(candidate.baseHash) - await expect(promoteKnowledgeCandidate({ root, candidate })).resolves.toMatchObject({ - promoted: true, - }) - expect(await readFile(page, 'utf8')).toBe('# Same bytes\n') - expect((await stat(page)).mode & 0o777).toBe(0o700) - }) - }) - - it('promotes only measured files when the mutable workspace gains an unmeasured entry', async () => { - await withKb(async (root) => { - const staged = await improveKnowledgeBase({ - root, - goal: 'Create one measured page', - runId: 'unsupported-entry', - updateKnowledge: async ({ candidateRoot }) => { - await writeFile(join(candidateRoot, 'knowledge', 'measured.md'), '# Measured\n') - return { applied: true, summary: 'created measured page' } - }, - evaluate: passingMetric, - }) - const candidate = knowledgeImprovementCandidateRef(staged) - await symlink( - join(mutableCandidateRoot(root, staged), 'knowledge', 'measured.md'), - join(mutableCandidateRoot(root, staged), 'knowledge', 'unmeasured.md'), - ) - - await expect(promoteKnowledgeCandidate({ root, candidate })).resolves.toMatchObject({ - promoted: true, - }) - await expect(readFile(join(root, 'knowledge', 'measured.md'), 'utf8')).resolves.toBe( - '# Measured\n', - ) - await expect( - readFile(join(root, 'knowledge', 'unmeasured.md'), 'utf8'), - ).rejects.toMatchObject({ - code: 'ENOENT', - }) - }) - }) - - it('rejects an approval whose measured evidence identity was changed', async () => { - await withKb(async (root) => { - const staged = await improveKnowledgeBase({ - root, - goal: 'Create one measured page', - runId: 'changed-evidence', - updateKnowledge: async (input) => { - const page = join(input.candidateRoot, 'knowledge', 'measured.md') - await mkdir(dirname(page), { recursive: true }) - await writeFile(page, '# Measured\n') - return { applied: true, summary: 'created measured page' } - }, - evaluate: passingMetric, - }) - const candidate = knowledgeImprovementCandidateRef(staged) - const changed = { ...candidate, evidenceHash: '0'.repeat(64) } - - await expect(promoteKnowledgeCandidate({ root, candidate: changed })).rejects.toThrow( - /does not match the measured candidate/, - ) - }) - }) - - it('binds approval to the full measured lifecycle evidence', async () => { - await withKb(async (root) => { - const source = refundSource() - const staged = await improveKnowledgeBase({ - root, - goal: 'Build billing support refund-policy knowledge', - runId: 'full-evidence', - readinessSpecs: [refundSpec], - strict: true, - step: () => ({ - done: true, - sourceTexts: [{ uri: source.uri, text: source.text, title: source.title }], - proposalText: refundProposal(source.id), - }), - }) - const candidate = knowledgeImprovementCandidateRef(staged) - const evidencePath = join( - knowledgeImprovementRunDir(root, candidate.runId), - 'candidates', - candidate.candidateId, - 'evidence.json', - ) - const evidence = JSON.parse(await readFile(evidencePath, 'utf8')) as { - lifecycle: { knowledgeUpdate: { summary: string } } - } - evidence.lifecycle.knowledgeUpdate.summary = 'different measured evidence' - await writeFile(evidencePath, `${JSON.stringify(evidence, null, 2)}\n`) - - await expect(promoteKnowledgeCandidate({ root, candidate })).rejects.toThrow( - /evidence changed after approval/, - ) - }) - }) - - it('fails loudly when persisted improvement state is malformed', async () => { - await withKb(async (root) => { - const runDir = knowledgeImprovementRunDir(root, 'malformed-state') - await mkdir(runDir, { recursive: true }) - await writeFile(join(runDir, 'state.json'), '{broken') - - await expect(loadKnowledgeImprovementState(root, 'malformed-state')).rejects.toThrow() - await writeFile(join(runDir, 'state.json'), '{}') - await expect(loadKnowledgeImprovementState(root, 'malformed-state')).rejects.toThrow() - }) - }) - - it('rejects evaluation copied into mutable run state', async () => { - await withKb(async (root) => { - const staged = await improveKnowledgeBase({ - root, - goal: 'Keep evaluation in immutable evidence only', - runId: 'mutable-evaluation-copy', - evaluate: passingMetric, - }) - const runDir = knowledgeImprovementRunDir(root, staged.runId) - const statePath = join(runDir, 'state.json') - const state = JSON.parse(await readFile(statePath, 'utf8')) as { - candidates: Array> - } - state.candidates[0]!.evaluation = passingMetric() - await writeFile(statePath, `${JSON.stringify(state, null, 2)}\n`) - - await expect(loadKnowledgeImprovementState(root, staged.runId)).rejects.toThrow() - await expect( - withKnowledgeImprovementCandidate( - { root, candidate: knowledgeImprovementCandidateRef(staged) }, - () => undefined, - ), - ).rejects.toThrow() - }) - }) - - it('rejects missing promotion evidence in a native current state', async () => { - await withKb(async (root) => { - const options = { - root, - goal: 'Reject corrupted current promotion state', - runId: 'native-promoted-state', - updateKnowledge: async ({ candidateRoot }: { candidateRoot: string }) => { - await writeFile(join(candidateRoot, 'knowledge', 'current.md'), '# Current\n') - return { applied: true, summary: 'created current page' } - }, - evaluate: passingMetric, - } - const promoted = await improveAndPromote(options) - const runDir = knowledgeImprovementRunDir(root, promoted.runId) - const statePath = join(runDir, 'state.json') - const current = JSON.parse(await readFile(statePath, 'utf8')) as { - candidates: Array> - } - delete current.candidates[0]?.evidenceHash - delete current.candidates[0]?.promotionPlanHash - await writeFile(statePath, `${JSON.stringify(current, null, 2)}\n`) - - await expect(improveKnowledgeBase(options)).rejects.toThrow( - /promoted candidates require content, evidence, and promotion-plan identities/, - ) - }) - }) - - it('encodes external run ids without allowing them to escape the improvement directory', async () => { - await withKb(async (root) => { - const runId = '../../victim' - const improvementsDir = join(root, '.agent-knowledge', 'improvements') - const runDir = knowledgeImprovementRunDir(root, runId) - expect(relative(improvementsDir, runDir)).not.toMatch(/^\.\.(?:\/|$)/) - await expect( - improveKnowledgeBase({ root, goal: 'External run identity', runId }), - ).resolves.toMatchObject({ runId }) - }) - }) - - it('retries a running candidate after a crashed operator run', async () => { - await withKb(async (root) => { - const source = refundSource() - await expect( - improveKnowledgeBase({ - root, - goal: 'Build billing support refund-policy knowledge', - runId: 'crash-resume', - readinessSpecs: [refundSpec], - strict: true, - step: () => { - throw new Error('research worker crashed') - }, - }), - ).rejects.toThrow(/research worker crashed/) - - const resumed = await improveAndPromote({ - root, - goal: 'Build billing support refund-policy knowledge', - runId: 'crash-resume', - readinessSpecs: [refundSpec], - strict: true, - step: () => ({ - done: true, - sourceTexts: [{ uri: source.uri, text: source.text, title: source.title }], - proposalText: refundProposal(source.id), - }), - }) - - expect(resumed.promoted).toBe(true) - expect(resumed.state.candidates).toHaveLength(1) - expect(resumed.candidate?.iteration).toBe(1) - }) - }) - - it('passes the candidate KB root into updateKnowledge callbacks', async () => { - await withKb(async (root) => { - const seen: Array<{ - root: string - baselineRoot: string - candidateRoot: string - runId: string - iteration: number - }> = [] - - const result = await improveKnowledgeBase({ - root, - goal: 'Let a runtime supervisor update the candidate KB', - runId: 'candidate-update-root', - async updateKnowledge(input) { - seen.push({ - root: input.root, - baselineRoot: input.baselineRoot, - candidateRoot: input.candidateRoot, - runId: input.runId, - iteration: input.iteration, - }) - const page = join(input.candidateRoot, 'knowledge', 'runtime-supervisor.md') - await mkdir(dirname(page), { recursive: true }) - await writeFile( - page, - [ - '---', - 'id: runtime-supervisor', - 'title: Runtime Supervisor', - '---', - '# Runtime Supervisor', - 'The runtime supervisor writes to the candidate workspace only.', - ].join('\n'), - ) - return { applied: true, summary: 'candidate workspace updated' } - }, - evaluate: passingMetric, - }) - - expect(result.state.status).toBe('candidate-ready') - expect(seen).toHaveLength(1) - expect(seen[0]!.root).toBe(seen[0]!.candidateRoot) - expect(seen[0]).toMatchObject({ - baselineRoot: root, - runId: 'candidate-update-root', - iteration: 1, - }) - await expect( - readFile(join(root, 'knowledge', 'runtime-supervisor.md'), 'utf8'), - ).rejects.toThrow() - await expect( - readFile( - join(mutableCandidateRoot(root, result), 'knowledge', 'runtime-supervisor.md'), - 'utf8', - ), - ).resolves.toContain('candidate workspace only') - }) - }) - - it('blocks promotion when the live KB changed after candidate creation', async () => { - await withKb(async (root) => { - const source = refundSource() - const first = await improveKnowledgeBase({ - root, - goal: 'Build billing support refund-policy knowledge', - runId: 'conflict', - readinessSpecs: [refundSpec], - strict: true, - step: () => ({ - done: true, - sourceTexts: [{ uri: source.uri, text: source.text, title: source.title }], - proposalText: refundProposal(source.id), - }), - }) - expect(first.state.status).toBe('candidate-ready') - - const concurrentPage = join(root, 'knowledge', 'support', 'concurrent.md') - await mkdir(dirname(concurrentPage), { recursive: true }) - await writeFile( - concurrentPage, - [ - '---', - 'id: concurrent', - 'title: Concurrent Edit', - '---', - '# Concurrent Edit', - 'A different agent edited the live KB.', - ].join('\n'), - ) - - const blocked = await promoteKnowledgeCandidate({ - root, - candidate: knowledgeImprovementCandidateRef(first), - }) - - expect(blocked.blocked).toBe(true) - expect(blocked.state.status).toBe('blocked') - expect(blocked.state.blockedReason).toMatch(/base changed before promotion/) - expect(await readFile(concurrentPage, 'utf8')).toContain('different agent') - }) - }) - - it('fails fast when another active operator owns the run lease', async () => { - await withKb(async (root) => { - let releaseStep!: () => void - let markStarted!: () => void - const started = new Promise((resolve) => { - markStarted = resolve - }) - const released = new Promise((resolve) => { - releaseStep = resolve - }) - const first = improveKnowledgeBase({ - root, - goal: 'Locked run', - runId: 'locked', - updateKnowledge: async () => { - markStarted() - await released - return { applied: true, summary: 'finished' } - }, - evaluate: passingMetric, - }) - await started - - await expect( - improveKnowledgeBase({ - root, - goal: 'Locked run', - runId: 'locked', - readinessSpecs: [refundSpec], - }), - ).rejects.toThrow(/Lock file is already being held/) - releaseStep() - await first - }) - }) - - it('serializes package writers against approved promotion', async () => { - await withKb(async (root) => { - const staged = await improveKnowledgeBase({ - root, - goal: 'Create measured knowledge', - runId: 'writer-race', - updateKnowledge: async ({ candidateRoot }) => { - await writeFile(join(candidateRoot, 'knowledge', 'measured.md'), '# Measured\n') - return { applied: true, summary: 'created measured knowledge' } - }, - evaluate: passingMetric, - }) - const candidate = knowledgeImprovementCandidateRef(staged) - let releaseWriter!: () => void - let markStarted!: () => void - const started = new Promise((resolve) => { - markStarted = resolve - }) - const released = new Promise((resolve) => { - releaseWriter = resolve - }) - const writer = withKnowledgeMutation(root, async () => { - markStarted() - await released - await applyKnowledgeWriteBlocks( - root, - ['---FILE: knowledge/concurrent.md---', '# Concurrent', '---END FILE---'].join('\n'), - ) - }) - await started - - const promotion = promoteKnowledgeCandidate({ root, candidate }) - const beforeRelease = await Promise.race([ - promotion.then( - () => 'settled', - () => 'settled', - ), - new Promise<'waiting'>((resolve) => setTimeout(() => resolve('waiting'), 50)), - ]) - expect(beforeRelease).toBe('waiting') - releaseWriter() - await writer - await expect(promotion).resolves.toMatchObject({ promoted: false, blocked: true }) - await expect(readFile(join(root, 'knowledge', 'concurrent.md'), 'utf8')).resolves.toContain( - 'Concurrent', - ) - await expect(readFile(join(root, 'knowledge', 'measured.md'), 'utf8')).rejects.toMatchObject({ - code: 'ENOENT', - }) - }) - }) - - it('runs retrieval tuning against the updated candidate KB', async () => { - await withKb(async (root) => { - const source = refundSource() - const result = await improveKnowledgeBase({ - root, - goal: 'Build billing support refund-policy knowledge', - runId: 'retrieval', - readinessSpecs: [refundSpec], - strict: true, - step: () => ({ - done: true, - sourceTexts: [{ uri: source.uri, text: source.text, title: source.title }], - proposalText: refundProposal(source.id), - }), - retrieval: { - baseline: { k: 1 }, - scenarios: [ - { - id: 'q-train', - kind: 'retrieval-eval', - query: 'refund policy', - expected: { kind: 'page', pageId: 'refund-policy' }, - }, - ], - holdoutScenarios: [ - { - id: 'q-holdout', - kind: 'retrieval-eval', - query: 'billing refund', - expected: { kind: 'page', pageId: 'refund-policy' }, - }, - ], - searchSpace: { k: [1, 2] }, - retrieve: async ({ k }) => ({ - hits: [ - { pageId: 'distractor', path: 'knowledge/distractor.md', rank: 1 }, - ...(k >= 2 - ? [{ pageId: 'refund-policy', path: 'knowledge/support/refund-policy.md', rank: 2 }] - : []), - ], - }), - targetRecall: 1, - deltaThreshold: 0.01, - populationSize: 1, - maxGenerations: 1, - expectUsage: 'off', - }, - }) - - expect(result.lifecycle?.retrieval?.winnerConfig).toMatchObject({ k: 2 }) - expect(result.lifecycle?.phases.map((phase) => `${phase.phase}:${phase.status}`)).toContain( - 'retrieval-tuning:completed', - ) - }) - }) - - it('rejects a candidate when answer quality fails', async () => { - await withKb(async (root) => { - const source = refundSource() - const result = await improveKnowledgeBase({ - root, - goal: 'Build billing support refund-policy knowledge', - runId: 'answer-quality-failure', - readinessSpecs: [refundSpec], - strict: true, - step: () => ({ - done: true, - sourceTexts: [{ uri: source.uri, text: source.text, title: source.title }], - proposalText: refundProposal(source.id), - }), - evaluateAnswers: () => ({ - passed: false, - metrics: { faithfulness: 0, answer_relevance: 0.8 }, - findings: [ - { - id: 'refund-answer:faithfulness', - kind: 'generator-unsupported-claim', - severity: 'error', - message: 'The answer made an unsupported claim.', - }, - ], - }), - }) - - expect(result.promoted).toBe(false) - expect(result.state.status).toBe('rejected') - expect(result.evaluation?.passed).toBe(false) - expect(result.evaluation?.notes).toContain('answer quality failed') - }) - }) - - it('hashes only promoted KB surfaces, not improvement run scratch space', async () => { - await withKb(async (root) => { - const before = await hashKnowledgeBase(root) - const runDir = knowledgeImprovementRunDir(root, 'scratch') - await mkdir(runDir, { recursive: true }) - await writeFile(join(runDir, 'scratch.txt'), 'scratch state') - await expect(hashKnowledgeBase(root)).resolves.toBe(before) - }) - }) -}) diff --git a/tests/kb-improvement/activation.test.ts b/tests/kb-improvement/activation.test.ts new file mode 100644 index 0000000..c611183 --- /dev/null +++ b/tests/kb-improvement/activation.test.ts @@ -0,0 +1,599 @@ +import { readdir, readFile, rm, writeFile } from 'node:fs/promises' +import { join } from 'node:path' +import { describe, expect, it } from 'vitest' +import { + applyKnowledgeWriteBlocks, + hashKnowledgeBase, + improveKnowledgeBase, + inspectPendingKnowledgeMutation, + type KnowledgeImprovementMutationReceipt, + knowledgeImprovementCandidateRef, + knowledgeImprovementRunDir, + loadKnowledgeImprovementActivationResult, + loadKnowledgeImprovementEvents, + promoteKnowledgeCandidate, + recoverPendingKnowledgeMutation, + restoreKnowledgeCandidateBaseline, +} from '../../src/index' +import { withKnowledgeMutation } from '../../src/mutation-lock' +import { + knowledgeActivation, + knowledgeActivationResult, + passingMetric, + refundProposal, + refundSource, + refundSpec, + withKb, +} from '../support/kb-improvement' + +describe('improveKnowledgeBase', () => { + it('promotes one exact approved candidate without rerunning candidate work', async () => { + await withKb(async (root) => { + const source = refundSource() + let updateCalls = 0 + const staged = await improveKnowledgeBase({ + root, + goal: 'Build billing support refund-policy knowledge', + runId: 'approved-candidate', + readinessSpecs: [refundSpec], + strict: true, + step: () => { + updateCalls += 1 + return { + done: true, + sourceTexts: [{ uri: source.uri, text: source.text, title: source.title }], + proposalText: refundProposal(source.id), + } + }, + }) + const candidate = knowledgeImprovementCandidateRef(staged) + + const promoted = await promoteKnowledgeCandidate({ root, candidate }) + expect(promoted.promoted).toBe(true) + expect(promoted.state.promotedCandidateId).toBe(candidate.candidateId) + expect(updateCalls).toBe(1) + await expect( + readFile(join(root, 'knowledge', 'support', 'refund-policy.md'), 'utf8'), + ).resolves.toContain('refund within 30 days') + + const repeated = await promoteKnowledgeCandidate({ root, candidate }) + expect(repeated.promoted).toBe(true) + expect(repeated.state.promotedCandidateId).toBe(candidate.candidateId) + expect(repeated.mutation).toEqual({ + target: 'candidate', + beforeHash: candidate.candidateHash, + afterHash: candidate.candidateHash, + changed: false, + transactionId: null, + recovered: false, + }) + expect(updateCalls).toBe(1) + }) + }) + + it('finishes an interrupted restore without repeating the mutation', async () => { + await withKb(async (root) => { + const staged = await improveKnowledgeBase({ + root, + goal: 'Create knowledge that can be restored after interruption', + runId: 'interrupted-restore', + updateKnowledge: async ({ candidateRoot }) => { + await writeFile(join(candidateRoot, 'knowledge', 'candidate.md'), '# Candidate\n') + return { applied: true, summary: 'created measured knowledge' } + }, + evaluate: passingMetric, + }) + const candidate = knowledgeImprovementCandidateRef(staged) + await promoteKnowledgeCandidate({ root, candidate }) + + await expect( + restoreKnowledgeCandidateBaseline({ + root, + candidate, + onState() { + throw new Error('operator stopped after restore state persistence') + }, + }), + ).rejects.toThrow(/operator stopped/) + + const recovered = await restoreKnowledgeCandidateBaseline({ root, candidate }) + expect(recovered).toMatchObject({ + promoted: false, + blocked: false, + state: { status: 'candidate-ready' }, + mutation: { + target: 'baseline', + beforeHash: candidate.candidateHash, + afterHash: candidate.baseHash, + changed: true, + transactionId: expect.any(String), + recovered: true, + }, + }) + expect(await hashKnowledgeBase(root)).toBe(candidate.baseHash) + const events = await loadKnowledgeImprovementEvents(root, candidate.runId) + expect(events.filter((event) => event.type === 'candidate.restored')).toHaveLength(1) + const transactions = await readdir(join(root, '.agent-knowledge', 'file-transactions')) + expect(transactions.filter((entry) => entry.startsWith('active-'))).toEqual([]) + + await expect(restoreKnowledgeCandidateBaseline({ root, candidate })).resolves.toMatchObject({ + promoted: false, + blocked: false, + }) + const repeatedEvents = await loadKnowledgeImprovementEvents(root, candidate.runId) + expect(repeatedEvents.filter((event) => event.type === 'candidate.restored')).toHaveLength(1) + }) + }) + + it('finishes an interrupted approved promotion without rerunning candidate work', async () => { + await withKb(async (root) => { + const source = refundSource() + let updateCalls = 0 + const staged = await improveKnowledgeBase({ + root, + goal: 'Build billing support refund-policy knowledge', + runId: 'interrupted-promotion', + readinessSpecs: [refundSpec], + strict: true, + step: () => { + updateCalls += 1 + return { + done: true, + sourceTexts: [{ uri: source.uri, text: source.text, title: source.title }], + proposalText: refundProposal(source.id), + } + }, + }) + const candidate = knowledgeImprovementCandidateRef(staged) + + await expect( + promoteKnowledgeCandidate({ + root, + candidate, + onState() { + throw new Error('operator stopped after state persistence') + }, + }), + ).rejects.toThrow(/operator stopped/) + + const runDir = knowledgeImprovementRunDir(root, candidate.runId) + await rm(join(runDir, 'candidates', candidate.candidateId, 'snapshots'), { + recursive: true, + force: true, + }) + await rm(join(runDir, 'candidates', candidate.candidateId, 'evidence.json'), { + force: true, + }) + await expect( + applyKnowledgeWriteBlocks( + root, + ['---FILE: knowledge/unapproved.md---', '# Unapproved', '---END FILE---'].join('\n'), + ), + ).rejects.toThrow(/requires its owner to resume/) + const recovered = await promoteKnowledgeCandidate({ root, candidate }) + + expect(recovered.promoted).toBe(true) + expect(recovered.mutation).toEqual({ + target: 'candidate', + beforeHash: candidate.baseHash, + afterHash: candidate.candidateHash, + changed: true, + transactionId: expect.any(String), + recovered: true, + }) + expect(updateCalls).toBe(1) + const events = await loadKnowledgeImprovementEvents(root, candidate.runId) + expect(events.filter((event) => event.type === 'candidate.promoted')).toHaveLength(1) + const transactionEntries = await readdir(join(root, '.agent-knowledge', 'file-transactions')) + expect(transactionEntries.filter((entry) => entry.startsWith('active-'))).toEqual([]) + }) + }) + + it('persists an approved result before closing its file transaction and never reapplies it', async () => { + await withKb(async (root) => { + const staged = await improveKnowledgeBase({ + root, + goal: 'Keep approval results recoverable with their knowledge mutation', + runId: 'protected-activation-recovery', + updateKnowledge: async ({ candidateRoot }) => { + await writeFile(join(candidateRoot, 'knowledge', 'candidate.md'), '# Candidate\n') + return { applied: true, summary: 'created measured knowledge' } + }, + evaluate: passingMetric, + }) + const candidate = knowledgeImprovementCandidateRef(staged) + const activation = knowledgeActivation(candidate, 'activate-candidate') + const attemptedAt = '2026-07-17T12:00:00.000Z' + const transactionRoot = join(root, '.agent-knowledge', 'file-transactions') + let interruptedMutation: KnowledgeImprovementMutationReceipt | undefined + + await expect( + promoteKnowledgeCandidate({ + root, + candidate, + activation: { + activation, + attemptedAt, + identity: 'knowledge:test', + createResult(mutation) { + interruptedMutation = mutation + throw new Error('activation result store unavailable') + }, + }, + }), + ).rejects.toThrow(/activation result store unavailable/) + expect(interruptedMutation).toMatchObject({ + target: 'candidate', + beforeHash: candidate.baseHash, + afterHash: candidate.candidateHash, + changed: true, + transactionId: expect.any(String), + recovered: false, + }) + await expect(hashKnowledgeBase(root)).rejects.toThrow(/requires its owner to resume/) + await expect(readFile(join(root, 'knowledge', 'candidate.md'), 'utf8')).resolves.toBe( + '# Candidate\n', + ) + expect( + (await readdir(transactionRoot)).filter((entry) => entry.startsWith('active-')), + ).toHaveLength(1) + const pending = await inspectPendingKnowledgeMutation(root) + expect(pending).toMatchObject({ + transactionId: interruptedMutation?.transactionId, + recoveryOwner: `knowledge-improvement-activation:${activation.digest}`, + }) + await expect( + recoverPendingKnowledgeMutation(root, { + transactionId: pending!.transactionId, + action: 'apply', + }), + ).rejects.toThrow(/must be resumed by 'knowledge-improvement-activation:sha256:/) + await expect(promoteKnowledgeCandidate({ root, candidate })).rejects.toThrow( + /must be resumed by 'knowledge-improvement-activation:sha256:/, + ) + await expect( + loadKnowledgeImprovementActivationResult({ + root, + candidate, + activation, + identity: 'knowledge:test', + }), + ).resolves.toBeNull() + await expect( + improveKnowledgeBase({ + root, + goal: 'Keep approval results recoverable with their knowledge mutation', + runId: 'protected-activation-recovery', + }), + ).rejects.toThrow(/requires its owner to resume/) + + const recovered = await promoteKnowledgeCandidate({ + root, + candidate, + activation: { + activation, + attemptedAt, + identity: 'knowledge:test', + createResult: (mutation) => + knowledgeActivationResult(activation, candidate, mutation, attemptedAt), + }, + }) + expect(recovered.mutation).toEqual({ + ...interruptedMutation, + recovered: true, + }) + expect(recovered.activationResult?.outcome.status).toBe('applied') + expect( + (await readdir(transactionRoot)).filter((entry) => entry.startsWith('active-')), + ).toEqual([]) + await expect( + loadKnowledgeImprovementActivationResult({ + root, + candidate, + activation, + identity: 'knowledge:test', + }), + ).resolves.toEqual(recovered.activationResult) + + await restoreKnowledgeCandidateBaseline({ root, candidate }) + await expect(hashKnowledgeBase(root)).resolves.toBe(candidate.baseHash) + const retried = await promoteKnowledgeCandidate({ + root, + candidate, + activation: { + activation, + attemptedAt, + identity: 'knowledge:test', + createResult() { + throw new Error('durable activation must not be recomputed') + }, + }, + }) + expect(retried.activationResult).toEqual(recovered.activationResult) + await expect(hashKnowledgeBase(root)).resolves.toBe(candidate.baseHash) + }) + }) + + it('repairs blocked run state and its event from a durable conflict result', async () => { + await withKb(async (root) => { + const staged = await improveKnowledgeBase({ + root, + goal: 'Keep conflict results and run state consistent', + runId: 'activation-conflict-recovery', + updateKnowledge: async ({ candidateRoot }) => { + await writeFile(join(candidateRoot, 'knowledge', 'candidate.md'), '# Candidate\n') + return { applied: true, summary: 'created measured knowledge' } + }, + evaluate: passingMetric, + }) + const candidate = knowledgeImprovementCandidateRef(staged) + await promoteKnowledgeCandidate({ root, candidate }) + await writeFile(join(root, 'knowledge', 'candidate.md'), '# Concurrent change\n') + const activation = knowledgeActivation(candidate, 'restore-baseline') + const attemptedAt = '2026-07-17T12:00:00.000Z' + + await expect( + restoreKnowledgeCandidateBaseline({ + root, + candidate, + activation: { + activation, + attemptedAt, + identity: 'knowledge:test', + createResult: (mutation) => + knowledgeActivationResult(activation, candidate, mutation, attemptedAt), + }, + onState() { + throw new Error('operator stopped before conflict event persistence') + }, + }), + ).rejects.toThrow(/operator stopped before conflict event persistence/) + const stored = await loadKnowledgeImprovementActivationResult({ + root, + candidate, + activation, + identity: 'knowledge:test', + }) + expect(stored?.outcome.status).toBe('conflict') + expect( + (await loadKnowledgeImprovementEvents(root, candidate.runId)).filter( + (event) => event.type === 'restore.blocked', + ), + ).toHaveLength(0) + + const recovered = await restoreKnowledgeCandidateBaseline({ + root, + candidate, + activation: { + activation, + attemptedAt, + identity: 'knowledge:test', + createResult() { + throw new Error('durable conflict result must not be recomputed') + }, + }, + }) + expect(recovered).toMatchObject({ + promoted: false, + blocked: true, + state: { status: 'blocked' }, + candidate: { status: 'blocked' }, + }) + expect(recovered.activationResult).toEqual(stored) + await restoreKnowledgeCandidateBaseline({ + root, + candidate, + activation: { + activation, + attemptedAt, + identity: 'knowledge:test', + createResult() { + throw new Error('durable conflict result must not be recomputed') + }, + }, + }) + expect( + (await loadKnowledgeImprovementEvents(root, candidate.runId)).filter( + (event) => event.type === 'restore.blocked', + ), + ).toHaveLength(1) + }) + }) + + it('binds already-applied promotion and baseline restore to their exact activation', async () => { + await withKb(async (root) => { + const staged = await improveKnowledgeBase({ + root, + goal: 'Record exact no-op and restore activation outcomes', + runId: 'activation-already-and-restore', + updateKnowledge: async ({ candidateRoot }) => { + await writeFile(join(candidateRoot, 'knowledge', 'candidate.md'), '# Candidate\n') + return { applied: true, summary: 'created measured knowledge' } + }, + evaluate: passingMetric, + }) + const candidate = knowledgeImprovementCandidateRef(staged) + await promoteKnowledgeCandidate({ root, candidate }) + const attemptedAt = '2026-07-17T12:00:00.000Z' + const promoteActivation = knowledgeActivation(candidate, 'activate-candidate') + + const already = await promoteKnowledgeCandidate({ + root, + candidate, + activation: { + activation: promoteActivation, + attemptedAt, + identity: 'knowledge:test', + createResult: (mutation) => + knowledgeActivationResult(promoteActivation, candidate, mutation, attemptedAt), + }, + }) + expect(already.mutation).toEqual({ + target: 'candidate', + beforeHash: candidate.candidateHash, + afterHash: candidate.candidateHash, + changed: false, + transactionId: null, + recovered: false, + }) + expect(already.activationResult?.outcome.status).toBe('already-applied') + await expect( + loadKnowledgeImprovementActivationResult({ + root, + candidate, + activation: promoteActivation, + identity: 'knowledge:test', + }), + ).resolves.toEqual(already.activationResult) + + const restoreActivation = knowledgeActivation(candidate, 'restore-baseline') + const restored = await restoreKnowledgeCandidateBaseline({ + root, + candidate, + activation: { + activation: restoreActivation, + attemptedAt, + identity: 'knowledge:test', + createResult: (mutation) => + knowledgeActivationResult(restoreActivation, candidate, mutation, attemptedAt), + }, + }) + expect(restored.mutation).toMatchObject({ + target: 'baseline', + beforeHash: candidate.candidateHash, + afterHash: candidate.baseHash, + changed: true, + transactionId: expect.any(String), + recovered: false, + }) + expect(restored.activationResult?.outcome.status).toBe('applied') + await expect( + loadKnowledgeImprovementActivationResult({ + root, + candidate, + activation: restoreActivation, + identity: 'knowledge:test', + }), + ).resolves.toEqual(restored.activationResult) + await expect(hashKnowledgeBase(root)).resolves.toBe(candidate.baseHash) + }) + }) + + it('does not return a successful mutation receipt when a state callback changes knowledge', async () => { + await withKb(async (root) => { + const staged = await improveKnowledgeBase({ + root, + goal: 'Reject post-commit knowledge changes', + runId: 'post-commit-change', + updateKnowledge: async ({ candidateRoot }) => { + await writeFile(join(candidateRoot, 'knowledge', 'candidate.md'), '# Candidate\n') + return { applied: true, summary: 'created measured knowledge' } + }, + evaluate: passingMetric, + }) + const candidate = knowledgeImprovementCandidateRef(staged) + + await expect( + promoteKnowledgeCandidate({ + root, + candidate, + async onState() { + await writeFile(join(root, 'knowledge', 'unmeasured.md'), '# Unmeasured\n') + }, + }), + ).rejects.toThrow(/changed before its result was returned/) + await expect(hashKnowledgeBase(root)).rejects.toThrow(/requires its owner to resume/) + await expect(readFile(join(root, 'knowledge', 'unmeasured.md'), 'utf8')).resolves.toBe( + '# Unmeasured\n', + ) + }) + }) + + it('fails fast when another active operator owns the run lease', async () => { + await withKb(async (root) => { + let releaseStep!: () => void + let markStarted!: () => void + const started = new Promise((resolve) => { + markStarted = resolve + }) + const released = new Promise((resolve) => { + releaseStep = resolve + }) + const first = improveKnowledgeBase({ + root, + goal: 'Locked run', + runId: 'locked', + updateKnowledge: async () => { + markStarted() + await released + return { applied: true, summary: 'finished' } + }, + evaluate: passingMetric, + }) + await started + + await expect( + improveKnowledgeBase({ + root, + goal: 'Locked run', + runId: 'locked', + readinessSpecs: [refundSpec], + }), + ).rejects.toThrow(/Lock file is already being held/) + releaseStep() + await first + }) + }) + + it('serializes package writers against approved promotion', async () => { + await withKb(async (root) => { + const staged = await improveKnowledgeBase({ + root, + goal: 'Create measured knowledge', + runId: 'writer-race', + updateKnowledge: async ({ candidateRoot }) => { + await writeFile(join(candidateRoot, 'knowledge', 'measured.md'), '# Measured\n') + return { applied: true, summary: 'created measured knowledge' } + }, + evaluate: passingMetric, + }) + const candidate = knowledgeImprovementCandidateRef(staged) + let releaseWriter!: () => void + let markStarted!: () => void + const started = new Promise((resolve) => { + markStarted = resolve + }) + const released = new Promise((resolve) => { + releaseWriter = resolve + }) + const writer = withKnowledgeMutation(root, async () => { + markStarted() + await released + await applyKnowledgeWriteBlocks( + root, + ['---FILE: knowledge/concurrent.md---', '# Concurrent', '---END FILE---'].join('\n'), + ) + }) + await started + + const promotion = promoteKnowledgeCandidate({ root, candidate }) + const beforeRelease = await Promise.race([ + promotion.then( + () => 'settled', + () => 'settled', + ), + new Promise<'waiting'>((resolve) => setTimeout(() => resolve('waiting'), 50)), + ]) + expect(beforeRelease).toBe('waiting') + releaseWriter() + await writer + await expect(promotion).resolves.toMatchObject({ promoted: false, blocked: true }) + await expect(readFile(join(root, 'knowledge', 'concurrent.md'), 'utf8')).resolves.toContain( + 'Concurrent', + ) + await expect(readFile(join(root, 'knowledge', 'measured.md'), 'utf8')).rejects.toMatchObject({ + code: 'ENOENT', + }) + }) + }) +}) diff --git a/tests/kb-improvement/candidate.test.ts b/tests/kb-improvement/candidate.test.ts new file mode 100644 index 0000000..cc0d407 --- /dev/null +++ b/tests/kb-improvement/candidate.test.ts @@ -0,0 +1,410 @@ +import { mkdir, readFile, stat, writeFile } from 'node:fs/promises' +import { dirname, join } from 'node:path' +import { describe, expect, it } from 'vitest' +import { + buildEvalKnowledgeBundle, + buildKnowledgeIndex, + evaluateKnowledgeBaseReadiness, + hashKnowledgeBase, + improveKnowledgeBase, + knowledgeImprovementRunDir, +} from '../../src/index' +import { + improveAndPromote, + mutableCandidateRoot, + passingMetric, + refundProposal, + refundSource, + refundSpec, + withEmptyRoot, + withKb, +} from '../support/kb-improvement' + +describe('improveKnowledgeBase', () => { + it('leaves the live knowledge base unchanged unless promotion is explicit', async () => { + await withEmptyRoot(async (root) => { + const before = await hashKnowledgeBase(root) + await expect(stat(join(root, '.agent-knowledge'))).rejects.toMatchObject({ code: 'ENOENT' }) + const result = await improveKnowledgeBase({ + root, + goal: 'Create a measured candidate without applying it', + runId: 'propose-only-default', + updateKnowledge: async ({ candidateRoot }) => { + await writeFile(join(candidateRoot, 'knowledge', 'candidate.md'), '# Candidate\n') + return { applied: true, summary: 'created candidate' } + }, + evaluate: passingMetric, + }) + + expect(result.promoted).toBe(false) + expect(result.state.status).toBe('candidate-ready') + expect(result.candidate?.candidateHash).not.toBe(before) + await expect(hashKnowledgeBase(root)).resolves.toBe(before) + await expect(readFile(join(root, 'knowledge', 'candidate.md'), 'utf8')).rejects.toMatchObject( + { + code: 'ENOENT', + }, + ) + await expect(stat(join(root, 'knowledge'))).rejects.toMatchObject({ code: 'ENOENT' }) + await expect(stat(join(root, 'raw'))).rejects.toMatchObject({ code: 'ENOENT' }) + await expect( + readFile(join(root, '.agent-knowledge', 'sources.json'), 'utf8'), + ).rejects.toMatchObject({ code: 'ENOENT' }) + }) + }) + + it('rejects evaluator results without provenance', async () => { + await withKb(async (root) => { + await expect( + improveKnowledgeBase({ + root, + goal: 'Reject an anonymous evaluation', + runId: 'anonymous-evaluator', + updateKnowledge: async ({ candidateRoot }) => { + await writeFile(join(candidateRoot, 'knowledge', 'candidate.md'), '# Candidate\n') + return { applied: true, summary: 'created candidate' } + }, + evaluate: (() => ({ score: 1, passed: true })) as never, + }), + ).rejects.toThrow(/provenance/) + }) + }) + + it('evaluates root-level KB readiness without running an improvement loop', async () => { + await withKb(async (root) => { + const empty = await evaluateKnowledgeBaseReadiness({ + root, + goal: 'Build billing support refund-policy knowledge', + readinessSpecs: [refundSpec], + strict: true, + }) + + expect(empty.ready).toBe(false) + expect(empty.dimensions.blocking_readiness).toBe(0) + expect(empty.summary).toContain('blocking readiness requirement') + + const source = refundSource() + const improved = await improveAndPromote({ + root, + goal: 'Build billing support refund-policy knowledge', + runId: 'readiness-evaluator', + readinessSpecs: [refundSpec], + strict: true, + step: () => ({ + done: true, + sourceTexts: [{ uri: source.uri, text: source.text, title: source.title }], + proposalText: refundProposal(source.id), + }), + }) + expect(improved.promoted).toBe(true) + + const ready = await evaluateKnowledgeBaseReadiness({ + root, + goal: 'Build billing support refund-policy knowledge', + readinessSpecs: [refundSpec], + strict: true, + }) + + expect(ready.ready).toBe(true) + expect(ready.summary).toBe('knowledge base passed readiness checks') + expect(ready.dimensions).toMatchObject({ + validation: 1, + kb_quality: 1, + blocking_readiness: 1, + }) + }) + }) + + it('promotes a candidate KB after readiness separates weak from strong', async () => { + await withKb(async (root) => { + const emptyIndex = await buildKnowledgeIndex(root) + const emptyReadiness = buildEvalKnowledgeBundle({ + taskId: 'support-agent', + index: emptyIndex, + specs: [refundSpec], + }) + expect(emptyReadiness.report.blockingMissingRequirements).toHaveLength(1) + + const source = refundSource() + const result = await improveAndPromote({ + root, + goal: 'Build billing support refund-policy knowledge', + runId: 'refund-improvement', + readinessSpecs: [refundSpec], + strict: true, + step: () => ({ + done: true, + sourceTexts: [{ uri: source.uri, text: source.text, title: source.title }], + proposalText: refundProposal(source.id), + }), + }) + + expect(result.promoted).toBe(true) + expect(result.state.status).toBe('promoted') + expect(result.evaluation?.score).toBe(1) + expect( + result.lifecycle?.phases.some( + (phase) => phase.phase === 'knowledge-update' && phase.status === 'completed', + ), + ).toBe(true) + + const promotedIndex = await buildKnowledgeIndex(root) + expect(promotedIndex.pages.map((page) => page.id)).toContain('refund-policy') + const promotedReadiness = buildEvalKnowledgeBundle({ + taskId: 'support-agent', + index: promotedIndex, + specs: [refundSpec], + }) + expect(promotedReadiness.report.blockingMissingRequirements).toHaveLength(0) + }) + }) + + it('resumes a candidate workspace after an external edit and promotes the edited artifact', async () => { + await withKb(async (root) => { + const source = refundSource() + const first = await improveKnowledgeBase({ + root, + goal: 'Build billing support refund-policy knowledge', + runId: 'resume-edit', + readinessSpecs: [refundSpec], + strict: true, + step: () => ({ + done: true, + sourceTexts: [{ uri: source.uri, text: source.text, title: source.title }], + proposalText: refundProposal(source.id), + }), + }) + + expect(first.promoted).toBe(false) + expect(first.state.status).toBe('candidate-ready') + const candidateRoot = mutableCandidateRoot(root, first) + + const editedPage = join(candidateRoot!, 'knowledge', 'support', 'refund-escalation.md') + await mkdir(dirname(editedPage), { recursive: true }) + await writeFile( + editedPage, + [ + '---', + 'id: refund-escalation', + 'title: Refund Escalation', + 'sources:', + ` - ${source.id}`, + '---', + '# Refund Escalation', + 'Escalate refund requests that lack receipt proof.', + ].join('\n'), + ) + + const resumed = await improveAndPromote({ + root, + goal: 'Build billing support refund-policy knowledge', + runId: 'resume-edit', + readinessSpecs: [refundSpec], + strict: true, + }) + + expect(resumed.promoted).toBe(true) + const promoted = await readFile( + join(root, 'knowledge', 'support', 'refund-escalation.md'), + 'utf8', + ) + expect(promoted).toContain('Escalate refund requests') + }) + }) + + it('retries a running candidate after a crashed operator run', async () => { + await withKb(async (root) => { + const source = refundSource() + await expect( + improveKnowledgeBase({ + root, + goal: 'Build billing support refund-policy knowledge', + runId: 'crash-resume', + readinessSpecs: [refundSpec], + strict: true, + step: () => { + throw new Error('research worker crashed') + }, + }), + ).rejects.toThrow(/research worker crashed/) + + const resumed = await improveAndPromote({ + root, + goal: 'Build billing support refund-policy knowledge', + runId: 'crash-resume', + readinessSpecs: [refundSpec], + strict: true, + step: () => ({ + done: true, + sourceTexts: [{ uri: source.uri, text: source.text, title: source.title }], + proposalText: refundProposal(source.id), + }), + }) + + expect(resumed.promoted).toBe(true) + expect(resumed.state.candidates).toHaveLength(1) + expect(resumed.candidate?.iteration).toBe(1) + }) + }) + + it('passes the candidate KB root into updateKnowledge callbacks', async () => { + await withKb(async (root) => { + const seen: Array<{ + root: string + baselineRoot: string + candidateRoot: string + runId: string + iteration: number + }> = [] + + const result = await improveKnowledgeBase({ + root, + goal: 'Let a runtime supervisor update the candidate KB', + runId: 'candidate-update-root', + async updateKnowledge(input) { + seen.push({ + root: input.root, + baselineRoot: input.baselineRoot, + candidateRoot: input.candidateRoot, + runId: input.runId, + iteration: input.iteration, + }) + const page = join(input.candidateRoot, 'knowledge', 'runtime-supervisor.md') + await mkdir(dirname(page), { recursive: true }) + await writeFile( + page, + [ + '---', + 'id: runtime-supervisor', + 'title: Runtime Supervisor', + '---', + '# Runtime Supervisor', + 'The runtime supervisor writes to the candidate workspace only.', + ].join('\n'), + ) + return { applied: true, summary: 'candidate workspace updated' } + }, + evaluate: passingMetric, + }) + + expect(result.state.status).toBe('candidate-ready') + expect(seen).toHaveLength(1) + expect(seen[0]!.root).toBe(seen[0]!.candidateRoot) + expect(seen[0]).toMatchObject({ + baselineRoot: root, + runId: 'candidate-update-root', + iteration: 1, + }) + await expect( + readFile(join(root, 'knowledge', 'runtime-supervisor.md'), 'utf8'), + ).rejects.toThrow() + await expect( + readFile( + join(mutableCandidateRoot(root, result), 'knowledge', 'runtime-supervisor.md'), + 'utf8', + ), + ).resolves.toContain('candidate workspace only') + }) + }) + + it('runs retrieval tuning against the updated candidate KB', async () => { + await withKb(async (root) => { + const source = refundSource() + const result = await improveKnowledgeBase({ + root, + goal: 'Build billing support refund-policy knowledge', + runId: 'retrieval', + readinessSpecs: [refundSpec], + strict: true, + step: () => ({ + done: true, + sourceTexts: [{ uri: source.uri, text: source.text, title: source.title }], + proposalText: refundProposal(source.id), + }), + retrieval: { + baseline: { k: 1 }, + scenarios: [ + { + id: 'q-train', + kind: 'retrieval-eval', + query: 'refund policy', + expected: { kind: 'page', pageId: 'refund-policy' }, + }, + ], + holdoutScenarios: [ + { + id: 'q-holdout', + kind: 'retrieval-eval', + query: 'billing refund', + expected: { kind: 'page', pageId: 'refund-policy' }, + }, + ], + searchSpace: { k: [1, 2] }, + retrieve: async ({ k }) => ({ + hits: [ + { pageId: 'distractor', path: 'knowledge/distractor.md', rank: 1 }, + ...(k >= 2 + ? [{ pageId: 'refund-policy', path: 'knowledge/support/refund-policy.md', rank: 2 }] + : []), + ], + }), + targetRecall: 1, + deltaThreshold: 0.01, + populationSize: 1, + maxGenerations: 1, + expectUsage: 'off', + }, + }) + + expect(result.lifecycle?.retrieval?.winnerConfig).toMatchObject({ k: 2 }) + expect(result.lifecycle?.phases.map((phase) => `${phase.phase}:${phase.status}`)).toContain( + 'retrieval-tuning:completed', + ) + }) + }) + + it('rejects a candidate when answer quality fails', async () => { + await withKb(async (root) => { + const source = refundSource() + const result = await improveKnowledgeBase({ + root, + goal: 'Build billing support refund-policy knowledge', + runId: 'answer-quality-failure', + readinessSpecs: [refundSpec], + strict: true, + step: () => ({ + done: true, + sourceTexts: [{ uri: source.uri, text: source.text, title: source.title }], + proposalText: refundProposal(source.id), + }), + evaluateAnswers: () => ({ + passed: false, + metrics: { faithfulness: 0, answer_relevance: 0.8 }, + findings: [ + { + id: 'refund-answer:faithfulness', + kind: 'generator-unsupported-claim', + severity: 'error', + message: 'The answer made an unsupported claim.', + }, + ], + }), + }) + + expect(result.promoted).toBe(false) + expect(result.state.status).toBe('rejected') + expect(result.evaluation?.passed).toBe(false) + expect(result.evaluation?.notes).toContain('answer quality failed') + }) + }) + + it('hashes only promoted KB surfaces, not improvement run scratch space', async () => { + await withKb(async (root) => { + const before = await hashKnowledgeBase(root) + const runDir = knowledgeImprovementRunDir(root, 'scratch') + await mkdir(runDir, { recursive: true }) + await writeFile(join(runDir, 'scratch.txt'), 'scratch state') + await expect(hashKnowledgeBase(root)).resolves.toBe(before) + }) + }) +}) diff --git a/tests/kb-improvement/integrity.test.ts b/tests/kb-improvement/integrity.test.ts new file mode 100644 index 0000000..8c9ad3d --- /dev/null +++ b/tests/kb-improvement/integrity.test.ts @@ -0,0 +1,343 @@ +import { createHash } from 'node:crypto' +import { + mkdir, + mkdtemp, + readdir, + readFile, + rename, + rm, + stat, + symlink, + writeFile, +} from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { dirname, join, relative } from 'node:path' +import { describe, expect, it } from 'vitest' +import { + improveKnowledgeBase, + knowledgeImprovementCandidateRef, + knowledgeImprovementRunDir, + loadKnowledgeImprovementState, + promoteKnowledgeCandidate, + withKnowledgeImprovementCandidate, +} from '../../src/index' +import { + improveAndPromote, + passingMetric, + refundProposal, + refundSource, + refundSpec, + withEmptyRoot, + withKb, +} from '../support/kb-improvement' + +describe('improveKnowledgeBase', () => { + it('does not create improvement state through a symbolic-link metadata directory', async () => { + await withEmptyRoot(async (root) => { + const outside = await mkdtemp(join(tmpdir(), 'agent-knowledge-improvement-outside-')) + try { + await symlink(outside, join(root, '.agent-knowledge')) + await expect( + improveKnowledgeBase({ + root, + goal: 'Reject redirected improvement state', + runId: 'redirected-state', + }), + ).rejects.toThrow(/unsafe directory/) + await expect(stat(join(outside, 'improvements'))).rejects.toMatchObject({ code: 'ENOENT' }) + } finally { + await rm(outside, { recursive: true, force: true }) + } + }) + }) + + it.skipIf(process.platform !== 'linux')( + 'rejects a run directory swapped while candidate work is active', + async () => { + await withKb(async (root) => { + const outside = await mkdtemp(join(tmpdir(), 'agent-knowledge-run-outside-')) + const runId = 'run-directory-swap' + const runDir = knowledgeImprovementRunDir(root, runId) + try { + await expect( + improveKnowledgeBase({ + root, + goal: 'Reject redirected run state', + runId, + async updateKnowledge({ candidateRoot }) { + await writeFile(join(candidateRoot, 'knowledge', 'candidate.md'), '# Candidate\n') + await rename(runDir, `${runDir}-original`) + await symlink(outside, runDir) + return { applied: true, summary: 'candidate written before redirect' } + }, + evaluate: passingMetric, + }), + ).rejects.toThrow(/unsafe directory|changed during use/) + await expect(readFile(join(outside, 'state.json'), 'utf8')).rejects.toMatchObject({ + code: 'ENOENT', + }) + } finally { + await rm(outside, { recursive: true, force: true }) + } + }) + }, + ) + + it('rejects a forged promotion journal entry outside the measured knowledge files', async () => { + await withKb(async (root) => { + const packagePath = join(root, 'package.json') + const originalPackage = '{"private":true}\n' + const forgedPackage = '{"scripts":{"postinstall":"malicious"}}\n' + await writeFile(packagePath, originalPackage) + const staged = await improveKnowledgeBase({ + root, + goal: 'Prepare an exact knowledge-only candidate', + runId: 'forged-promotion-journal', + updateKnowledge: async ({ candidateRoot }) => { + await writeFile(join(candidateRoot, 'knowledge', 'candidate.md'), '# Candidate\n') + return { applied: true, summary: 'created candidate' } + }, + evaluate: passingMetric, + }) + const candidate = knowledgeImprovementCandidateRef(staged) + await expect( + promoteKnowledgeCandidate({ + root, + candidate, + onState() { + throw new Error('leave promotion pending') + }, + }), + ).rejects.toThrow(/leave promotion pending/) + + const transactionRoot = join(root, '.agent-knowledge', 'file-transactions') + const activeTransactions = (await readdir(transactionRoot)).filter((entry) => + entry.startsWith('active-'), + ) + expect(activeTransactions).toHaveLength(1) + const transactionDir = join(transactionRoot, activeTransactions[0]!) + const transactionPath = join(transactionDir, 'transaction.json') + const transaction = JSON.parse(await readFile(transactionPath, 'utf8')) as { + entries: Array<{ + index: number + path: string + beforeHash: string | null + afterHash: string | null + beforeMode?: number + afterMode?: number + }> + } + const index = Math.max(...transaction.entries.map((entry) => entry.index)) + 1 + transaction.entries.push({ + index, + path: 'package.json', + beforeHash: createHash('sha256').update(originalPackage).digest('hex'), + afterHash: createHash('sha256').update(forgedPackage).digest('hex'), + beforeMode: 0o644, + afterMode: 0o644, + }) + await mkdir(join(transactionDir, 'after'), { recursive: true }) + await writeFile(join(transactionDir, 'after', `${index}.bin`), forgedPackage) + await writeFile(transactionPath, `${JSON.stringify(transaction, null, 2)}\n`) + + await expect(promoteKnowledgeCandidate({ root, candidate })).rejects.toThrow( + /unsupported path: package.json/, + ) + await expect(readFile(packagePath, 'utf8')).resolves.toBe(originalPackage) + }) + }) + + it('rejects a measured snapshot changed while an approved candidate is in use', async () => { + await withKb(async (root) => { + const staged = await improveKnowledgeBase({ + root, + goal: 'Keep approved candidate execution immutable', + runId: 'changed-during-candidate-use', + updateKnowledge: async ({ candidateRoot }) => { + await writeFile(join(candidateRoot, 'knowledge', 'measured.md'), '# Measured\n') + return { applied: true, summary: 'created measured knowledge' } + }, + evaluate: passingMetric, + }) + const candidate = knowledgeImprovementCandidateRef(staged) + + await expect( + withKnowledgeImprovementCandidate({ root, candidate }, async (resolved) => { + await writeFile(join(resolved.root, 'knowledge', 'measured.md'), '# Changed\n') + }), + ).rejects.toThrow(/snapshot changed during use/) + }) + }) + + it('rejects a frozen measured copy changed after approval', async () => { + await withKb(async (root) => { + const source = refundSource() + const staged = await improveKnowledgeBase({ + root, + goal: 'Build billing support refund-policy knowledge', + runId: 'changed-frozen-candidate', + readinessSpecs: [refundSpec], + strict: true, + step: () => ({ + done: true, + sourceTexts: [{ uri: source.uri, text: source.text, title: source.title }], + proposalText: refundProposal(source.id), + }), + }) + const candidate = knowledgeImprovementCandidateRef(staged) + const snapshotRoot = join( + knowledgeImprovementRunDir(root, candidate.runId), + 'candidates', + candidate.candidateId, + 'snapshots', + candidate.candidateHash, + ) + await writeFile( + join(snapshotRoot, 'knowledge', 'support', 'refund-policy.md'), + '# tampered measured copy\n', + ) + + await expect(promoteKnowledgeCandidate({ root, candidate })).rejects.toThrow( + /snapshot changed after approval/, + ) + await expect( + readFile(join(root, 'knowledge', 'support', 'refund-policy.md'), 'utf8'), + ).rejects.toMatchObject({ code: 'ENOENT' }) + }) + }) + + it('rejects an approval whose measured evidence identity was changed', async () => { + await withKb(async (root) => { + const staged = await improveKnowledgeBase({ + root, + goal: 'Create one measured page', + runId: 'changed-evidence', + updateKnowledge: async (input) => { + const page = join(input.candidateRoot, 'knowledge', 'measured.md') + await mkdir(dirname(page), { recursive: true }) + await writeFile(page, '# Measured\n') + return { applied: true, summary: 'created measured page' } + }, + evaluate: passingMetric, + }) + const candidate = knowledgeImprovementCandidateRef(staged) + const changed = { ...candidate, evidenceHash: '0'.repeat(64) } + + await expect(promoteKnowledgeCandidate({ root, candidate: changed })).rejects.toThrow( + /does not match the measured candidate/, + ) + }) + }) + + it('binds approval to the full measured lifecycle evidence', async () => { + await withKb(async (root) => { + const source = refundSource() + const staged = await improveKnowledgeBase({ + root, + goal: 'Build billing support refund-policy knowledge', + runId: 'full-evidence', + readinessSpecs: [refundSpec], + strict: true, + step: () => ({ + done: true, + sourceTexts: [{ uri: source.uri, text: source.text, title: source.title }], + proposalText: refundProposal(source.id), + }), + }) + const candidate = knowledgeImprovementCandidateRef(staged) + const evidencePath = join( + knowledgeImprovementRunDir(root, candidate.runId), + 'candidates', + candidate.candidateId, + 'evidence.json', + ) + const evidence = JSON.parse(await readFile(evidencePath, 'utf8')) as { + lifecycle: { knowledgeUpdate: { summary: string } } + } + evidence.lifecycle.knowledgeUpdate.summary = 'different measured evidence' + await writeFile(evidencePath, `${JSON.stringify(evidence, null, 2)}\n`) + + await expect(promoteKnowledgeCandidate({ root, candidate })).rejects.toThrow( + /evidence changed after approval/, + ) + }) + }) + + it('fails loudly when persisted improvement state is malformed', async () => { + await withKb(async (root) => { + const runDir = knowledgeImprovementRunDir(root, 'malformed-state') + await mkdir(runDir, { recursive: true }) + await writeFile(join(runDir, 'state.json'), '{broken') + + await expect(loadKnowledgeImprovementState(root, 'malformed-state')).rejects.toThrow() + await writeFile(join(runDir, 'state.json'), '{}') + await expect(loadKnowledgeImprovementState(root, 'malformed-state')).rejects.toThrow() + }) + }) + + it('rejects evaluation copied into mutable run state', async () => { + await withKb(async (root) => { + const staged = await improveKnowledgeBase({ + root, + goal: 'Keep evaluation in immutable evidence only', + runId: 'mutable-evaluation-copy', + evaluate: passingMetric, + }) + const runDir = knowledgeImprovementRunDir(root, staged.runId) + const statePath = join(runDir, 'state.json') + const state = JSON.parse(await readFile(statePath, 'utf8')) as { + candidates: Array> + } + state.candidates[0]!.evaluation = passingMetric() + await writeFile(statePath, `${JSON.stringify(state, null, 2)}\n`) + + await expect(loadKnowledgeImprovementState(root, staged.runId)).rejects.toThrow() + await expect( + withKnowledgeImprovementCandidate( + { root, candidate: knowledgeImprovementCandidateRef(staged) }, + () => undefined, + ), + ).rejects.toThrow() + }) + }) + + it('rejects missing promotion evidence in a native current state', async () => { + await withKb(async (root) => { + const options = { + root, + goal: 'Reject corrupted current promotion state', + runId: 'native-promoted-state', + updateKnowledge: async ({ candidateRoot }: { candidateRoot: string }) => { + await writeFile(join(candidateRoot, 'knowledge', 'current.md'), '# Current\n') + return { applied: true, summary: 'created current page' } + }, + evaluate: passingMetric, + } + const promoted = await improveAndPromote(options) + const runDir = knowledgeImprovementRunDir(root, promoted.runId) + const statePath = join(runDir, 'state.json') + const current = JSON.parse(await readFile(statePath, 'utf8')) as { + candidates: Array> + } + delete current.candidates[0]?.evidenceHash + delete current.candidates[0]?.promotionPlanHash + await writeFile(statePath, `${JSON.stringify(current, null, 2)}\n`) + + await expect(improveKnowledgeBase(options)).rejects.toThrow( + /promoted candidates require content, evidence, and promotion-plan identities/, + ) + }) + }) + + it('encodes external run ids without allowing them to escape the improvement directory', async () => { + await withKb(async (root) => { + const runId = '../../victim' + const improvementsDir = join(root, '.agent-knowledge', 'improvements') + const runDir = knowledgeImprovementRunDir(root, runId) + expect(relative(improvementsDir, runDir)).not.toMatch(/^\.\.(?:\/|$)/) + await expect( + improveKnowledgeBase({ root, goal: 'External run identity', runId }), + ).resolves.toMatchObject({ runId }) + }) + }) +}) diff --git a/tests/kb-improvement/promotion.test.ts b/tests/kb-improvement/promotion.test.ts new file mode 100644 index 0000000..819b24c --- /dev/null +++ b/tests/kb-improvement/promotion.test.ts @@ -0,0 +1,396 @@ +import { chmod, mkdir, readFile, rm, stat, symlink, writeFile } from 'node:fs/promises' +import { dirname, join, relative } from 'node:path' +import { describe, expect, it } from 'vitest' +import { + hashKnowledgeBase, + improveKnowledgeBase, + knowledgeImprovementCandidateRef, + knowledgeImprovementRunDir, + loadKnowledgeImprovementEvents, + promoteKnowledgeCandidate, + restoreKnowledgeCandidateBaseline, + withKnowledgeImprovementCandidate, + withKnowledgeImprovementComparison, +} from '../../src/index' +import { + mutableCandidateRoot, + passingMetric, + refundProposal, + refundSource, + refundSpec, + withEmptyRoot, + withKb, +} from '../support/kb-improvement' + +describe('improveKnowledgeBase', () => { + it('does not reapply a promoted candidate when the improvement run is reopened', async () => { + await withEmptyRoot(async (root) => { + const options = { + root, + goal: 'Keep candidate generation separate from activation', + runId: 'reopen-promoted-run', + updateKnowledge: async ({ candidateRoot }: { candidateRoot: string }) => { + await writeFile(join(candidateRoot, 'knowledge', 'candidate.md'), '# Candidate\n') + return { applied: true, summary: 'created candidate' } + }, + evaluate: passingMetric, + } + const staged = await improveKnowledgeBase(options) + const candidate = knowledgeImprovementCandidateRef(staged) + await promoteKnowledgeCandidate({ root, candidate }) + await rm(join(root, 'knowledge', 'candidate.md')) + const liveBeforeReopen = await hashKnowledgeBase(root) + + await expect(improveKnowledgeBase(options)).rejects.toThrow(/promoted knowledge base changed/) + expect(await hashKnowledgeBase(root)).toBe(liveBeforeReopen) + await expect(readFile(join(root, 'knowledge', 'candidate.md'), 'utf8')).rejects.toMatchObject( + { code: 'ENOENT' }, + ) + }) + }) + + it('restores the exact frozen baseline and can apply the same candidate again', async () => { + await withKb(async (root) => { + const originalPage = join(root, 'knowledge', 'original.md') + const originalSource = join(root, 'raw', 'sources', 'original.txt') + await writeFile(originalPage, '# Original\n') + await writeFile(originalSource, 'original evidence\n') + const baseHash = await hashKnowledgeBase(root) + const staged = await improveKnowledgeBase({ + root, + goal: 'Replace the original knowledge with a measured candidate', + runId: 'restore-baseline', + updateKnowledge: async ({ candidateRoot }) => { + await writeFile(join(candidateRoot, 'knowledge', 'original.md'), '# Changed\n') + await writeFile(join(candidateRoot, 'knowledge', 'candidate.md'), '# Candidate\n') + await rm(join(candidateRoot, 'raw', 'sources', 'original.txt')) + return { applied: true, summary: 'changed, added, and deleted measured files' } + }, + evaluate: passingMetric, + }) + const candidate = knowledgeImprovementCandidateRef(staged) + + const promoted = await promoteKnowledgeCandidate({ root, candidate }) + expect(promoted).toMatchObject({ + promoted: true, + blocked: false, + }) + expect(promoted.mutation).toEqual({ + target: 'candidate', + beforeHash: baseHash, + afterHash: candidate.candidateHash, + changed: true, + transactionId: expect.any(String), + recovered: false, + }) + expect(await hashKnowledgeBase(root)).toBe(candidate.candidateHash) + + const restored = await restoreKnowledgeCandidateBaseline({ root, candidate }) + expect(restored).toMatchObject({ + promoted: false, + blocked: false, + state: { status: 'candidate-ready' }, + candidate: { status: 'candidate-ready' }, + }) + expect(restored.mutation).toEqual({ + target: 'baseline', + beforeHash: candidate.candidateHash, + afterHash: baseHash, + changed: true, + transactionId: expect.any(String), + recovered: false, + }) + expect(restored.state.promotedCandidateId).toBeUndefined() + expect(await hashKnowledgeBase(root)).toBe(baseHash) + await expect(readFile(originalPage, 'utf8')).resolves.toBe('# Original\n') + await expect(readFile(originalSource, 'utf8')).resolves.toBe('original evidence\n') + await expect(readFile(join(root, 'knowledge', 'candidate.md'), 'utf8')).rejects.toMatchObject( + { + code: 'ENOENT', + }, + ) + + await expect(promoteKnowledgeCandidate({ root, candidate })).resolves.toMatchObject({ + promoted: true, + blocked: false, + }) + expect(await hashKnowledgeBase(root)).toBe(candidate.candidateHash) + const events = await loadKnowledgeImprovementEvents(root, candidate.runId) + expect(events.filter((event) => event.type === 'candidate.promoted')).toHaveLength(1) + expect(events.filter((event) => event.type === 'candidate.restored')).toHaveLength(1) + }) + }) + + it('blocks restore without overwriting knowledge changed after promotion', async () => { + await withKb(async (root) => { + const page = join(root, 'knowledge', 'candidate.md') + const staged = await improveKnowledgeBase({ + root, + goal: 'Preserve concurrent edits during restore', + runId: 'restore-conflict', + updateKnowledge: async ({ candidateRoot }) => { + await writeFile(join(candidateRoot, 'knowledge', 'candidate.md'), '# Candidate\n') + return { applied: true, summary: 'created measured knowledge' } + }, + evaluate: passingMetric, + }) + const candidate = knowledgeImprovementCandidateRef(staged) + await promoteKnowledgeCandidate({ root, candidate }) + await writeFile(page, '# Concurrent change\n') + + const restored = await restoreKnowledgeCandidateBaseline({ root, candidate }) + + expect(restored).toMatchObject({ + promoted: false, + blocked: true, + state: { status: 'blocked' }, + candidate: { status: 'blocked' }, + }) + expect(restored.state.promotedCandidateId).toBeUndefined() + await expect(readFile(page, 'utf8')).resolves.toBe('# Concurrent change\n') + }) + }) + + it('promotes frozen measured bytes when the mutable workspace changes after approval', async () => { + await withKb(async (root) => { + const source = refundSource() + const staged = await improveKnowledgeBase({ + root, + goal: 'Build billing support refund-policy knowledge', + runId: 'changed-after-approval', + readinessSpecs: [refundSpec], + strict: true, + step: () => ({ + done: true, + sourceTexts: [{ uri: source.uri, text: source.text, title: source.title }], + proposalText: refundProposal(source.id), + }), + }) + const candidate = knowledgeImprovementCandidateRef(staged) + await writeFile( + join(mutableCandidateRoot(root, staged), 'knowledge', 'support', 'refund-policy.md'), + '# changed after approval\n', + ) + + await expect(promoteKnowledgeCandidate({ root, candidate })).resolves.toMatchObject({ + promoted: true, + }) + await expect( + readFile(join(root, 'knowledge', 'support', 'refund-policy.md'), 'utf8'), + ).resolves.toContain('refund within 30 days') + await expect( + readFile(join(root, 'knowledge', 'support', 'refund-policy.md'), 'utf8'), + ).resolves.not.toContain('changed after approval') + }) + }) + + it('resolves and promotes the measured snapshot after mutable workspace removal', async () => { + await withKb(async (root) => { + const staged = await improveKnowledgeBase({ + root, + goal: 'Keep the measured candidate independent from scratch files', + runId: 'removed-mutable-workspace', + updateKnowledge: async ({ candidateRoot }) => { + await writeFile(join(candidateRoot, 'knowledge', 'measured.md'), '# Measured\n') + return { applied: true, summary: 'created measured knowledge' } + }, + evaluate: passingMetric, + }) + const candidate = knowledgeImprovementCandidateRef(staged) + await rm(mutableCandidateRoot(root, staged), { recursive: true, force: true }) + const runDir = knowledgeImprovementRunDir(root, candidate.runId) + let isolatedRoot = '' + + await withKnowledgeImprovementCandidate({ root, candidate }, async (resolved) => { + isolatedRoot = resolved.root + expect(relative(runDir, resolved.root)).toMatch(/^\.\./) + await expect( + readFile(join(resolved.root, 'knowledge', 'measured.md'), 'utf8'), + ).resolves.toBe('# Measured\n') + }) + await expect(stat(isolatedRoot)).rejects.toMatchObject({ code: 'ENOENT' }) + await expect(promoteKnowledgeCandidate({ root, candidate })).resolves.toMatchObject({ + promoted: true, + }) + }) + }) + + it('resolves the exact frozen baseline and candidate from one measured comparison', async () => { + await withKb(async (root) => { + const liveBefore = await hashKnowledgeBase(root) + const staged = await improveKnowledgeBase({ + root, + goal: 'Compare the frozen baseline and candidate bytes', + runId: 'paired-frozen-snapshots', + updateKnowledge: async ({ candidateRoot }) => { + await writeFile(join(candidateRoot, 'knowledge', 'measured.md'), '# Measured\n') + return { applied: true, summary: 'created measured knowledge' } + }, + evaluate: () => ({ ...passingMetric(), dimensions: { quality: 1 } }), + }) + const candidate = knowledgeImprovementCandidateRef(staged) + const isolatedRoots: string[] = [] + + await withKnowledgeImprovementComparison({ root, candidate }, async (comparison) => { + isolatedRoots.push(comparison.baseline.root, comparison.candidate.root) + expect(Object.isFrozen(comparison)).toBe(true) + expect(Object.isFrozen(comparison.baseline)).toBe(true) + expect(Object.isFrozen(comparison.candidate)).toBe(true) + expect(Object.isFrozen(comparison.evaluation)).toBe(true) + expect(Object.isFrozen(comparison.evaluation.provenance)).toBe(true) + expect(Object.isFrozen(comparison.evaluation.dimensions)).toBe(true) + expect(comparison.reference).toEqual(candidate) + expect(comparison.baseline.hash).toBe(candidate.baseHash) + expect(comparison.candidate.hash).toBe(candidate.candidateHash) + await expect(hashKnowledgeBase(comparison.baseline.root)).resolves.toBe(candidate.baseHash) + await expect(hashKnowledgeBase(comparison.candidate.root)).resolves.toBe( + candidate.candidateHash, + ) + await expect( + readFile(join(comparison.baseline.root, 'knowledge', 'measured.md'), 'utf8'), + ).rejects.toMatchObject({ code: 'ENOENT' }) + await expect( + readFile(join(comparison.candidate.root, 'knowledge', 'measured.md'), 'utf8'), + ).resolves.toBe('# Measured\n') + }) + await expect( + withKnowledgeImprovementComparison({ root, candidate }, ({ baseline }) => + writeFile(join(baseline.root, 'knowledge', 'changed.md'), '# Changed\n'), + ), + ).rejects.toThrow(/baseline snapshot changed during use/) + + await expect(hashKnowledgeBase(root)).resolves.toBe(liveBefore) + for (const isolatedRoot of isolatedRoots) { + await expect(stat(isolatedRoot)).rejects.toMatchObject({ code: 'ENOENT' }) + } + }) + }) + + it('promotes candidate deletions exactly', async () => { + await withKb(async (root) => { + const obsoletePage = join(root, 'knowledge', 'obsolete.md') + const obsoleteRaw = join(root, 'raw', 'sources', 'obsolete.txt') + await writeFile(obsoletePage, '# Obsolete\n') + await writeFile(obsoleteRaw, 'obsolete source\n') + const staged = await improveKnowledgeBase({ + root, + goal: 'Remove obsolete knowledge', + runId: 'approved-deletions', + updateKnowledge: async ({ candidateRoot }) => { + await rm(join(candidateRoot, 'knowledge', 'obsolete.md')) + await rm(join(candidateRoot, 'raw', 'sources', 'obsolete.txt')) + return { applied: true, summary: 'removed obsolete knowledge' } + }, + evaluate: passingMetric, + }) + const candidate = knowledgeImprovementCandidateRef(staged) + + await promoteKnowledgeCandidate({ root, candidate }) + + await expect(readFile(obsoletePage, 'utf8')).rejects.toMatchObject({ code: 'ENOENT' }) + await expect(readFile(obsoleteRaw, 'utf8')).rejects.toMatchObject({ code: 'ENOENT' }) + await expect(hashKnowledgeBase(root)).resolves.toBe(candidate.candidateHash) + }) + }) + + it.skipIf(process.platform !== 'linux')('promotes a mode-only candidate exactly', async () => { + await withKb(async (root) => { + const page = join(root, 'knowledge', 'mode-only.md') + await writeFile(page, '# Same bytes\n') + await chmod(page, 0o600) + const staged = await improveKnowledgeBase({ + root, + goal: 'Make the measured page executable', + runId: 'mode-only-promotion', + updateKnowledge: async ({ candidateRoot }) => { + await chmod(join(candidateRoot, 'knowledge', 'mode-only.md'), 0o700) + return { applied: true, summary: 'changed the measured file mode' } + }, + evaluate: passingMetric, + }) + const candidate = knowledgeImprovementCandidateRef(staged) + + expect(candidate.candidateHash).not.toBe(candidate.baseHash) + await expect(promoteKnowledgeCandidate({ root, candidate })).resolves.toMatchObject({ + promoted: true, + }) + expect(await readFile(page, 'utf8')).toBe('# Same bytes\n') + expect((await stat(page)).mode & 0o777).toBe(0o700) + }) + }) + + it('promotes only measured files when the mutable workspace gains an unmeasured entry', async () => { + await withKb(async (root) => { + const staged = await improveKnowledgeBase({ + root, + goal: 'Create one measured page', + runId: 'unsupported-entry', + updateKnowledge: async ({ candidateRoot }) => { + await writeFile(join(candidateRoot, 'knowledge', 'measured.md'), '# Measured\n') + return { applied: true, summary: 'created measured page' } + }, + evaluate: passingMetric, + }) + const candidate = knowledgeImprovementCandidateRef(staged) + await symlink( + join(mutableCandidateRoot(root, staged), 'knowledge', 'measured.md'), + join(mutableCandidateRoot(root, staged), 'knowledge', 'unmeasured.md'), + ) + + await expect(promoteKnowledgeCandidate({ root, candidate })).resolves.toMatchObject({ + promoted: true, + }) + await expect(readFile(join(root, 'knowledge', 'measured.md'), 'utf8')).resolves.toBe( + '# Measured\n', + ) + await expect( + readFile(join(root, 'knowledge', 'unmeasured.md'), 'utf8'), + ).rejects.toMatchObject({ + code: 'ENOENT', + }) + }) + }) + + it('blocks promotion when the live KB changed after candidate creation', async () => { + await withKb(async (root) => { + const source = refundSource() + const first = await improveKnowledgeBase({ + root, + goal: 'Build billing support refund-policy knowledge', + runId: 'conflict', + readinessSpecs: [refundSpec], + strict: true, + step: () => ({ + done: true, + sourceTexts: [{ uri: source.uri, text: source.text, title: source.title }], + proposalText: refundProposal(source.id), + }), + }) + expect(first.state.status).toBe('candidate-ready') + + const concurrentPage = join(root, 'knowledge', 'support', 'concurrent.md') + await mkdir(dirname(concurrentPage), { recursive: true }) + await writeFile( + concurrentPage, + [ + '---', + 'id: concurrent', + 'title: Concurrent Edit', + '---', + '# Concurrent Edit', + 'A different agent edited the live KB.', + ].join('\n'), + ) + + const blocked = await promoteKnowledgeCandidate({ + root, + candidate: knowledgeImprovementCandidateRef(first), + }) + + expect(blocked.blocked).toBe(true) + expect(blocked.state.status).toBe('blocked') + expect(blocked.state.blockedReason).toMatch(/base changed before promotion/) + expect(await readFile(concurrentPage, 'utf8')).toContain('different agent') + }) + }) +}) diff --git a/tests/support/kb-improvement.ts b/tests/support/kb-improvement.ts new file mode 100644 index 0000000..52f1e6e --- /dev/null +++ b/tests/support/kb-improvement.ts @@ -0,0 +1,201 @@ +import { mkdtemp, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { + type AgentImprovementActivation, + type AgentImprovementActivationResult, + canonicalCandidateDigest, + type Sha256Digest, +} from '@tangle-network/agent-interface' +import { + defineReadinessSpec, + improveKnowledgeBase, + initKnowledgeBase, + type KnowledgeImprovementCandidateRef, + type KnowledgeImprovementMutationReceipt, + knowledgeImprovementCandidateRef, + knowledgeImprovementRunDir, + promoteKnowledgeCandidate, + sha256, + stableId, +} from '../../src/index' + +export async function withKb(fn: (root: string) => Promise): Promise { + const root = await mkdtemp(join(tmpdir(), 'agent-knowledge-improve-')) + try { + await initKnowledgeBase(root) + await fn(root) + } finally { + await rm(root, { recursive: true, force: true }) + } +} + +export async function withEmptyRoot(fn: (root: string) => Promise): Promise { + const root = await mkdtemp(join(tmpdir(), 'agent-knowledge-improve-empty-')) + try { + await fn(root) + } finally { + await rm(root, { recursive: true, force: true }) + } +} + +export function mutableCandidateRoot( + root: string, + result: { runId: string; candidate?: { candidateId: string } }, +): string { + if (!result.candidate) throw new Error('knowledge improvement result has no candidate') + return join( + knowledgeImprovementRunDir(root, result.runId), + 'candidates', + result.candidate.candidateId, + 'workspace', + ) +} + +export const refundSpec = defineReadinessSpec({ + id: 'refund-policy', + description: 'Refund policy support knowledge', + query: 'refund policy customer billing refund', + requiredFor: ['support-agent'], + importance: 'blocking', + minHits: 1, + minSources: 1, +}) + +export function refundSource() { + const text = 'The billing support refund policy allows refunds within 30 days with receipt proof.' + const uri = 'research://refund-policy' + return { + uri, + text, + title: 'Refund Policy Source', + id: stableId('src', `${sha256(text)}:${uri}`), + } +} + +export function refundProposal(sourceId: string, extra = ''): string { + return [ + '---FILE: knowledge/support/refund-policy.md---', + '---', + 'id: refund-policy', + 'title: Refund Policy', + 'sources:', + ` - ${sourceId}`, + '---', + '# Refund Policy', + 'Billing support can grant a customer refund within 30 days when receipt proof is present.', + extra, + '---END FILE---', + ].join('\n') +} + +export function passingMetric() { + return { + score: 1, + passed: true, + provenance: { + evaluator: 'agent-knowledge-test', + version: '1', + method: 'deterministic' as const, + }, + } +} + +function candidateDigest(seed: string): Sha256Digest { + return canonicalCandidateDigest({ seed }) +} + +function canonicalDocument>( + material: T, +): T & { digest: Sha256Digest } { + return { ...material, digest: canonicalCandidateDigest(material) } +} + +export function knowledgeActivation( + candidate: KnowledgeImprovementCandidateRef, + intent: AgentImprovementActivation['intent'], + identity = 'knowledge:test', +): AgentImprovementActivation { + const expectedBaseHash = + intent === 'activate-candidate' ? candidate.baseHash : candidate.candidateHash + return canonicalDocument({ + kind: 'agent-improvement-activation' as const, + proposalDigest: candidateDigest('proposal'), + reviewDigest: candidateDigest('review'), + experimentDigest: candidateDigest('experiment'), + candidateBundleDigest: candidateDigest('candidate-bundle'), + intent, + targets: [ + { + surface: 'knowledge' as const, + identity, + expectedBaseDigest: `sha256:${expectedBaseHash}` as Sha256Digest, + }, + ] as AgentImprovementActivation['targets'], + fundingOwner: 'tenant:test', + authorizedBy: 'reviewer:test', + authorizedAt: '2026-07-17T00:00:00.000Z', + expiresAt: '2026-07-18T00:00:00.000Z', + }) +} + +export function knowledgeActivationResult( + activation: AgentImprovementActivation, + candidate: KnowledgeImprovementCandidateRef, + mutation: KnowledgeImprovementMutationReceipt, + attemptedAt: string, + identity = 'knowledge:test', +): AgentImprovementActivationResult { + const desiredHash = + activation.intent === 'activate-candidate' ? candidate.candidateHash : candidate.baseHash + const outcome: AgentImprovementActivationResult['outcome'] = mutation.changed + ? { + status: 'applied', + transactionId: mutation.transactionId!, + targets: [ + { + surface: 'knowledge', + identity, + beforeDigest: `sha256:${mutation.beforeHash}`, + afterDigest: `sha256:${mutation.afterHash}`, + }, + ], + } + : mutation.afterHash === desiredHash + ? { + status: 'already-applied', + targets: [ + { + surface: 'knowledge', + identity, + currentDigest: `sha256:${mutation.afterHash}`, + }, + ], + } + : { + status: 'conflict', + targets: [ + { + surface: 'knowledge', + identity, + currentDigest: `sha256:${mutation.afterHash}`, + }, + ], + } + return canonicalDocument({ + kind: 'agent-improvement-activation-result' as const, + idempotencyKey: activation.digest, + attemptedAt, + completedAt: attemptedAt, + outcome, + }) +} + +export async function improveAndPromote(options: Parameters[0]) { + const staged = await improveKnowledgeBase(options) + const promoted = await promoteKnowledgeCandidate({ + root: options.root, + candidate: knowledgeImprovementCandidateRef(staged), + }) + return { ...promoted, evaluation: staged.evaluation, lifecycle: staged.lifecycle } +} From 7db5ec9a5f5a52d0a3aa58b581f6a83e35069f71 Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Sat, 18 Jul 2026 11:38:58 -0600 Subject: [PATCH 2/5] fix(release): accept holdout scenario evidence --- src/release.ts | 27 ++++++++++----------------- tests/release.test.ts | 32 +++++++++++++++++++++++++------- 2 files changed, 35 insertions(+), 24 deletions(-) diff --git a/src/release.ts b/src/release.ts index 0fcdf13..cef84cc 100644 --- a/src/release.ts +++ b/src/release.ts @@ -1,4 +1,5 @@ import { + type DatasetScenario, evaluateReleaseConfidence, type GateDecision, type ReleaseConfidenceScorecard, @@ -17,12 +18,8 @@ export interface KnowledgeReleaseReport { } /** - * Campaign-native release report. The caller (a consumer's KB self-improvement - * loop) supplies the candidate/baseline `RunRecord[]` (e.g. via - * `campaignToRunRecords`) + optional per-instance `ReleaseTraceEvidence` + the - * gate decision; this folds them into a `ReleaseConfidenceScorecard` + a - * `KnowledgeRelease`. Release confidence is computed from run records + traces, - * independent of any optimizer result shape. + * Build a knowledge release report from candidate and baseline run records, + * optional trace evidence, and an optional decision record. */ export interface KnowledgeReleaseInput { candidateId: string @@ -31,14 +28,11 @@ export interface KnowledgeReleaseInput { baselineRuns?: RunRecord[] traces?: ReleaseTraceEvidence[] gateDecision?: GateDecision | null + /** Scenario corpus used to prove train and holdout split coverage. */ + scenarios?: readonly DatasetScenario[] /** - * True when a held-out split was evaluated (drives the holdout threshold). - * - * Constraint: the substrate gate keys the holdout requirement off a scenario - * corpus, which this run-only report does not carry. With `hasHoldout: true` - * the gate fails closed (`missing_holdout_split`) even when holdout RunRecords - * are supplied. Callers that gate on a real held-out split should drive - * `evaluateReleaseConfidence` directly with a dataset/scenarios. + * Require both a holdout scenario and a holdout run. + * Provide `scenarios` with at least one `split: 'holdout'` item when true. */ hasHoldout?: boolean /** Candidate is the search-best variant — a promotion precondition. Default true. */ @@ -56,13 +50,12 @@ export function knowledgeReleaseReport(input: KnowledgeReleaseInput): KnowledgeR baselineId: input.baselineId ?? 'baseline', traces: input.traces ?? [], runs: runRecords, + scenarios: input.scenarios, gateDecision: input.gateDecision ?? null, thresholds: { requireCorpus: false, - // The report gates on RunRecord-level outcomes, not on a separate scenario - // corpus (KnowledgeReleaseInput carries no dataset/scenarios). The scenario - // floor must therefore be 0 — otherwise the corpus axis fails closed on a - // dimension the report has no way to satisfy, masking the run-based gate. + // This report has run records but no scenario corpus, so it cannot require + // a positive scenario count. minScenarioCount: 0, requireHoldout: input.hasHoldout ?? false, minHoldoutRuns: input.hasHoldout ? 1 : 0, diff --git a/tests/release.test.ts b/tests/release.test.ts index acd14fe..3fab616 100644 --- a/tests/release.test.ts +++ b/tests/release.test.ts @@ -85,11 +85,7 @@ describe('knowledgeReleaseReport', () => { expect(withoutHoldout.release.promoted).toBe(true) }) - it('hasHoldout=true enforces a holdout requirement that fails closed without a holdout split', () => { - // hasHoldout flips the gate's holdout requirement on. The substrate gate keys - // the holdout split off a scenario corpus, which this report does not carry, - // so a holdout-declared release fails closed (missing_holdout_split) — a - // documented constraint of the run-only release path. + it('hasHoldout=true fails closed without a holdout scenario', () => { const declared = knowledgeReleaseReport({ candidateId: 'cand-A', candidateRuns: [ @@ -99,14 +95,36 @@ describe('knowledgeReleaseReport', () => { hasHoldout: true, promotedIsBest: true, }) - // The holdout RunRecord IS counted... expect(declared.scorecard.metrics.holdoutRuns).toBe(1) - // ...but the scenario-level holdout split requirement still blocks promotion. expect(declared.scorecard.issues.map((i) => i.code)).toContain('missing_holdout_split') expect(declared.scorecard.status).toBe('fail') expect(declared.release.promoted).toBe(false) }) + it('accepts matching holdout scenarios and runs', () => { + const declared = knowledgeReleaseReport({ + candidateId: 'cand-A', + candidateRuns: [ + run({ runId: 'r1', splitTag: 'search', score: 0.9, candidateId: 'cand-A' }), + run({ runId: 'r2', splitTag: 'holdout', score: 0.88, candidateId: 'cand-A' }), + ], + scenarios: [ + { id: 'search-case', payload: {}, split: 'train' }, + { id: 'holdout-case', payload: {}, split: 'holdout' }, + ], + hasHoldout: true, + promotedIsBest: true, + }) + + expect(declared.scorecard.metrics.holdoutRuns).toBe(1) + expect(declared.scorecard.metrics.splitCounts.holdout).toBe(1) + expect(declared.scorecard.issues.map((issue) => issue.code)).not.toContain( + 'missing_holdout_split', + ) + expect(declared.scorecard.status).not.toBe('fail') + expect(declared.release.promoted).toBe(true) + }) + it('rejects a malformed run record (validateRunRecord fails closed)', () => { const bad = { runId: 'r1', splitTag: 'search' } as unknown as RunRecord expect(() => knowledgeReleaseReport({ candidateId: 'cand-A', candidateRuns: [bad] })).toThrow() From 4e8dd2e4dd30cbfc514eb167872dd08b8912d078 Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Sat, 18 Jul 2026 11:39:07 -0600 Subject: [PATCH 3/5] fix(sources): correct HTTP provenance contract --- src/sources/http.ts | 32 ++++++-------------------------- src/sources/types.ts | 15 ++++++++------- tests/http-cache.test.ts | 40 +++++++++++++++++++++------------------- 3 files changed, 35 insertions(+), 52 deletions(-) diff --git a/src/sources/http.ts b/src/sources/http.ts index e7bbb93..e344074 100644 --- a/src/sources/http.ts +++ b/src/sources/http.ts @@ -3,35 +3,15 @@ import { dirname, join } from 'node:path' import { sha256 } from '../ids' /** - * Polite HTTP fetcher used by every shipped source. + * Polite HTTP fetcher shared by remote sources. * - * Three invariants this enforces — each was a bug found while wiring real - * authorities; do not regress: - * - * 1. Per-host throttling. Cornell LII serves under 1 req/s/origin - * politely and will start serving block pages above that. The lock - * is per-host (`hostThrottle`) rather than per-source so that two - * independent sources targeting the same authority still cooperate. - * - * 2. On-disk content cache keyed by URL. Production sources are called - * from a cron loop; without a cache, every run re-hits the same - * pages and inflates change-detection false-positives (the authority - * occasionally serves slightly different boilerplate). The cache is - * content-addressed by URL, not by ETag — authorities like IRS.gov - * do not consistently send ETag/Last-Modified. - * - * 3. Block-page detection on success. A 200 with a captcha body still - * means "we couldn't authenticate." Sources downstream rely on - * `verifiable` to refuse promotion — losing that signal because the - * fetcher said "well, the status code was 200" is the bug class - * this exists to prevent. - * - * @stable + * Requests to one origin share a throttle, responses are cached by URL, + * and successful status codes are still checked for block pages. */ /** User-Agent string sent on every outbound request. */ export const POLITE_USER_AGENT = - 'agent-knowledge/0.2.0 (+https://github.com/tangle-network/agent-knowledge)' + 'agent-knowledge (+https://github.com/tangle-network/agent-knowledge)' /** Minimum gap between successive requests to the same origin (ms). */ export const MIN_REQUEST_GAP_MS = 1_000 @@ -45,7 +25,7 @@ export interface PoliteFetchOptions { signal?: AbortSignal cacheDir?: string /** - * Cache age beyond which we re-fetch. Default 1 hour — long enough to + * Cache age beyond which we re-fetch. Default 1 hour, long enough to * batch a cron sweep across many selectors, short enough that hourly * authoritative-page changes get picked up next tick. */ @@ -81,7 +61,7 @@ export interface PoliteFetchResult { /** * Fetch one URL with per-host throttling, on-disk cache, and block-page - * detection. Never throws on network/HTTP failure — returns a result with + * detection. Never throws on network/HTTP failure. It returns a result with * `verifiable: false` and `unverifiableReason` set so the caller can decide * whether to skip, retry, or surface. * diff --git a/src/sources/types.ts b/src/sources/types.ts index c1da1f7..38b7789 100644 --- a/src/sources/types.ts +++ b/src/sources/types.ts @@ -15,9 +15,9 @@ * Sources MUST be pure with respect to local filesystem state outside the * cache directory the caller hands them — they read remote authorities and * return data. They MUST mark `verifiable: false` on any fragment they could - * not authenticate (block page, 4xx, parse failure) rather than silently - * substituting empty/partial content. The control loop downstream uses - * `verifiable` to refuse promotion of un-grounded content. + * not fetch and extract (block page, 4xx, parse failure) rather than silently + * substituting empty or partial content. The control loop downstream uses + * `verifiable` to refuse promotion of unusable content. * * @stable */ @@ -79,10 +79,11 @@ export interface FragmentProvenance { */ jurisdiction?: string /** - * True iff the source could authenticate the fetched content (HTTP 200, - * expected selectors present, parse succeeded). False on any block page, - * rate-limit response, 4xx/5xx, or selector miss. Consumers MUST refuse - * to promote `verifiable: false` fragments into citable knowledge. + * True iff the configured URL returned an acceptable response and the + * expected content was extracted. False on a block page, rate-limit + * response, 4xx/5xx, or selector miss. This is not publisher authentication + * or cryptographic content verification. Consumers MUST refuse to promote + * `verifiable: false` fragments into citable knowledge. */ verifiable: boolean /** If `verifiable === false`, the reason — surfaced to operators. */ diff --git a/tests/http-cache.test.ts b/tests/http-cache.test.ts index 3960f6f..1df889f 100644 --- a/tests/http-cache.test.ts +++ b/tests/http-cache.test.ts @@ -3,19 +3,7 @@ import { tmpdir } from 'node:os' import { join } from 'node:path' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { sha256 } from '../src/ids' -import { __resetHttpThrottle, politeFetch } from '../src/sources/index' - -/** - * Bug class each test defends against: - * - * - 4xx swallowed as verifiable ⇒ downstream eval gates promote - * un-grounded fragments. - * - cache write missing ⇒ cron tick re-hits the authority every loop. - * - cache TTL ignored ⇒ stale fragments persist after authority change. - * - throttle not actually serialising ⇒ second request fires before - * 1 req/s gap, Cornell starts block-paging. - * - block-page heuristic miss ⇒ verifiable=true on captcha snapshots. - */ +import { __resetHttpThrottle, POLITE_USER_AGENT, politeFetch } from '../src/sources/index' let cacheDir: string const originalFetch = globalThis.fetch @@ -45,6 +33,21 @@ function html(body: string, status = 200, headers: Record = {}): } describe('politeFetch', () => { + it('identifies the package without embedding a stale release version', async () => { + let headers: Headers | undefined + mockFetch((_url, init) => { + headers = new Headers(init?.headers) + return html(`${'X'.repeat(500)}`) + }) + + await politeFetch('https://headers.test/source') + + expect(POLITE_USER_AGENT).toBe( + 'agent-knowledge (+https://github.com/tangle-network/agent-knowledge)', + ) + expect(headers?.get('User-Agent')).toBe(POLITE_USER_AGENT) + }) + it('returns verifiable=true for a normal 200', async () => { mockFetch(() => html(`${'X'.repeat(500)}`, 200, { @@ -100,7 +103,7 @@ describe('politeFetch', () => { }) it('respects cache TTL — expired entry re-fetches', async () => { - // Plant a stale cache file directly: TTL of 1ms ensures it's stale. + // Plant a stale cache file directly so the test controls its age. const url = 'https://www.law.cornell.edu/uscode/text/18/1836' const key = sha256(url) const path = join(cacheDir, 'http', key.slice(0, 2), `${key}.json`) @@ -117,7 +120,7 @@ describe('politeFetch', () => { verifiable: true, }), ) - // Force the mtime to be 1 day old so any positive TTL ≤ 1d will reject it. + // A one-day-old mtime must exceed the configured one-minute TTL. const { utimes } = await import('node:fs/promises') const dayAgo = new Date(Date.now() - 24 * 60 * 60 * 1000) await utimes(path, dayAgo, dayAgo) @@ -143,8 +146,7 @@ describe('politeFetch', () => { timestamps.push(Date.now()) return html(`${'X'.repeat(500)}`) }) - // Two distinct URLs on the same host bypass the URL cache but should - // still be throttled by the host gate. + // Distinct URLs bypass the URL cache but still share the host throttle. const t0 = Date.now() await Promise.all([ politeFetch('https://throttle.test/a'), @@ -152,7 +154,7 @@ describe('politeFetch', () => { ]) const gap = (timestamps[1] ?? 0) - (timestamps[0] ?? 0) expect(gap).toBeGreaterThanOrEqual(900) // some leeway for timer precision - // Sanity: throttle is on a per-host basis — total elapsed at least gap. + // Total elapsed time must include the per-host delay. expect(Date.now() - t0).toBeGreaterThanOrEqual(900) }, 10_000) @@ -171,7 +173,7 @@ describe('politeFetch', () => { const url = 'https://www.law.cornell.edu/uscode/text/18/1836' await politeFetch(url, { cacheDir }) - // Re-mock to ensure the next call would 500 if it weren't served from cache. + // A network fetch would return 500, so success proves the cache was used. mockFetch(() => html('boom', 500)) const second = await politeFetch(url, { cacheDir }) expect(second.status).toBe(200) From bef787bdd3dcf0bcaca6dd4488e703e3d0b373a1 Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Sat, 18 Jul 2026 11:39:22 -0600 Subject: [PATCH 4/5] chore(package): simplify dependencies and documentation --- AGENTS.md | 50 +- CHANGELOG.md | 20 + README.md | 50 +- docs/eval/investment-material-facts.md | 57 ++- docs/eval/rag-eval-roadmap.md | 11 +- docs/results/adaptive.md | 26 +- docs/results/claim-grounding.md | 22 +- docs/results/cost-quality.md | 8 +- docs/results/investment-thesis.md | 112 ++--- docs/results/research-driving.md | 108 ++--- docs/two-agent-research-ab.md | 188 ++++---- package.json | 11 +- pnpm-lock.yaml | 633 ++++++++++++++++--------- src/claim-grounding.ts | 5 +- src/file-transaction.ts | 2 +- src/freshness.ts | 28 +- src/memory/experiment/metrics.ts | 2 +- src/memory/holdout.ts | 23 +- src/memory/types.ts | 25 +- src/mutation-lock.ts | 2 +- src/research-driving-driver.ts | 6 +- src/two-agent-research-loop.ts | 5 - tests/changes.test.ts | 13 - tests/freshness.test.ts | 9 - tests/sources-live.test.ts | 28 +- tests/sources-mocked.test.ts | 16 +- tests/sources-types.test.ts | 12 - tsconfig.json | 1 + 28 files changed, 804 insertions(+), 669 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 08fbddd..0f4be62 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -2,22 +2,31 @@ Use this package when an agent needs persistent, source-grounded knowledge that improves over time. -## Repo layering — this package depends on agent-eval, never the reverse +## Package layering ``` -agent-knowledge (this repo) ─┐ - ├──► agent-eval (substrate — the bottom) -agent-runtime ───────────────┘ +agent-runtime + | +agent-knowledge + / \ +agent-eval agent-interface ``` -**Rule: agent-knowledge depends on agent-eval. agent-eval MUST NOT import from agent-knowledge.** No upward imports, no peerDependency declaration in agent-eval pointing at agent-knowledge. Substrate primitives that agent-knowledge needs (`AnalystFinding`, `RunRecord`, optimizer types, release-confidence types) live in agent-eval; this repo consumes them. +Imports flow downward in this diagram. +`agent-knowledge` may import `agent-eval` and `agent-interface`, but it must not import `agent-runtime`. +Agent-powered work enters this package through callbacks. +`agent-runtime` owns live agent execution and composes this package when a complete agent workflow is needed. +`agent-eval` must not import `agent-knowledge`. +Shared run and experiment types that knowledge needs live in `agent-eval`. Types that stay in THIS repo because they're knowledge-domain-shaped: - `KbStore`, `KnowledgeFragment`, `KnowledgeChange` - `KnowledgeDiscoveryDispatcher`, source adapters (`createCornellLiiSource`, `createIrsPublicationsSource`) - Freshness store + change-detection primitives -**The test for "where does a type live?"** — does this concept make sense WITHOUT persistent knowledge or sourced fragments? If yes, it's substrate (agent-eval). If no, it's a knowledge-domain concept (stays here). +**The test for "where does a type live?"** +If the concept makes sense without persistent knowledge or sourced fragments, it belongs in `agent-eval` or `agent-interface`. +Otherwise, it stays in this package. ## Rules @@ -72,36 +81,42 @@ Use `knowledgeReleaseReport()` before promotion. It folds the candidate and base ## Integration Boundaries -- Use `KbStore` for storage. Implement D1 in the consuming app when needed. -- Use `KnowledgeDiscoveryDispatcher` for research workers. Production apps should wire this to their own swarm/fleet runtime. +- Use `KbStore` for storage. Applications may provide any durable backend that implements it. +- Use `KnowledgeDiscoveryDispatcher` for research workers. Applications should connect it to their own runtime. - Do not bypass `lint` or `validate` before using generated knowledge in an agent. ## Pluggable Sources + Freshness + Changes Agents that need to stay current against external authorities should compose: -- `createCornellLiiSource({ selectors })` — US Code + Wex from law.cornell.edu. -- `createIrsPublicationsSource({ publications, revenueProcedures })` — IRS index + named pubs. -- `createStateSosSource({ state, baseUrl, entities })` — generic state SOS adapter. +- `createCornellLiiSource({ selectors })`: US Code and Wex from law.cornell.edu. +- `createIrsPublicationsSource({ publications, revenueProcedures })`: IRS index and named publications. +- `createStateSosSource({ state, baseUrl, entities })`: generic state SOS adapter. -Every fetch returns `KnowledgeFragment[]` with `provenance.verifiable` indicating whether the authority was successfully authenticated. Refuse to cite fragments with `verifiable: false`. +Every fetch returns `KnowledgeFragment[]` with `provenance.verifiable` indicating whether the configured URL returned an acceptable response and the expected content was extracted. +This flag does not authenticate the publisher or cryptographically prove the content. +Refuse to cite fragments with `verifiable: false`. Track per-tenant freshness with `createFileSystemFreshnessStore({ root })` and re-fetch only when `stale({ workspaceId, sourceId, ttlMs })` returns true. -Diff snapshots with `detectChanges(prev, next)`. Each `KnowledgeChange` carries `affectedDimensions` — pass those to your eval scheduler to re-run only the relevant campaigns. +Diff snapshots with `detectChanges(prev, next)`. +Each `KnowledgeChange` carries `affectedDimensions`; pass those to your eval scheduler to run only the relevant campaigns again. ## Authorship -Do not add `Co-Authored-By:` trailers (or any other AI-attribution lines) to commits, PR descriptions, or other artifacts in this repo. Author = the human running the session. Applies to every contributor, including AI agents and subagents — do not include the default Claude Code template trailer. +Do not add `Co-Authored-By:` trailers or other AI-attribution lines to commits, PR descriptions, or repository artifacts. +The author is the human running the session. ## Comment & doc discipline (no historical narrative) -Comments describe **what the code does and why** — never what it used to do, what it replaced, which audit found a bug, or what the prior version looked like. History belongs in commit messages and PR descriptions, not the source tree. +Comments describe **what the code does and why**. +They must not describe what code used to do, what it replaced, which audit found a bug, or what a prior version looked like. +History belongs in commit messages and PR descriptions. - Bad: `// replaces the inline retry loop`, `// fix for the silent-zero bug`, `// the 2yr rewrite added this`, `// audit fix` -- Good: `// value: null when retries exhaust — callers must inspect succeeded` +- Good: `// value is null when retries exhaust; callers must inspect succeeded` -Applies to docstrings, README sections, SKILL.md, AGENTS.md, CLAUDE.md — anywhere the source tree carries prose. +This applies anywhere the repository carries prose. ## No fallbacks. Fail loud. @@ -110,4 +125,3 @@ Sloppy fallbacks corrupt every signal downstream. No silent zeros, no `?? defaul External-boundary calls (LLM, network, FS, subprocess) return *typed outcomes* (`{ succeeded, value, error }`). Callers MUST inspect `succeeded` before using `value`. Named, opted-in fallback rotations (`policy.fallbackModels: [...]`) are fine; deep `?? "kimi"` helpers are not. Full doctrine: `~/dotfiles/claude/AGENTS.md` → "No fallbacks. Fail loud." - diff --git a/CHANGELOG.md b/CHANGELOG.md index 78a81ae..d28ba5e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,25 @@ # Changelog +## 4.1.0 + +### Added + +- Added optional scenario input to `knowledgeReleaseReport()` so a required holdout can prove both scenario and run coverage. + +### Changed + +- Split knowledge-base improvement and RAG evaluation internals into focused modules while preserving public exports and implementation behavior. +- Split the knowledge improvement tests into candidate, promotion, activation, and integrity suites with shared setup in one support module. +- Removed unused development dependencies and internal-only exports. +- Updated the test runner to Vitest 4 and Node type definitions to 26; TypeScript remains on 5.9 because tsup's declaration bundler does not yet support TypeScript 7. +- Declared Node types explicitly in TypeScript configuration instead of relying on ambient type discovery. +- Corrected the package dependency guide and RAG roadmap to reflect runtime-owned agent execution through `runKnowledgeImprovementJob()`. +- Removed historical commentary and em dashes from repository documentation. + +### Fixed + +- Replaced the stale versioned HTTP user agent with a stable package identity and added request-header coverage. + ## 4.0.1 ### Changed diff --git a/README.md b/README.md index bbdd55e..946f758 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,7 @@ This package turns raw sources and generated markdown knowledge into a versionab - [Benchmark runner](#benchmark-runner): BEIR/MTEB/qrels, RAG answer, hallucination, KB-improvement cases - [Agent-Eval integration](#agent-eval-integration): retrieval eval, readiness bundles, and release reports - [Memory adapters](#memory-adapters): Mem0, Graphiti, Neo4j, branching, and automatic improvement -- [Research loop](#research-loop): `runKnowledgeResearchLoop` plus the control-loop adapter +- [Research loops](#research-loop): direct and worker/reviewer knowledge growth - [Runtime integration](#runtime-integration): how agent runners plug into the pure KB loop - [Pluggable knowledge sources](#pluggable-knowledge-sources): live authorities → eval re-runs @@ -31,7 +31,7 @@ Two ways in, depending on what you're doing: - **Author / inspect a KB by hand** → the [CLI](#cli) (`init` → `source-add` → `index` → `search` → `lint`). Fastest way to see the shape on disk. - **Drive it from an agent** → pick the primitive by intent: - *"Does the agent have enough context to run?"* → [`buildEvalKnowledgeBundle`](#agent-eval-integration) (block / ask / acquire before execution). - - *"Grow the KB as a researcher"* → [`runKnowledgeResearchLoop`](#research-loop) (deterministic mechanics; your agent owns judgment) or [`runTwoAgentResearchLoop`](#two-agent-research-loop) (researcher proposes, verifier checks + fills gaps). + - *"Grow the KB as a researcher"* → [`runKnowledgeResearchLoop`](#research-loop) (deterministic mechanics; your agent owns judgment) or [`runVerifiedResearchLoop`](#verified-research-loop) (researcher proposes, reviewer checks and fills gaps). - *"Spawn live agents to improve a KB"* → pass an `updateKnowledge` callback to `improveKnowledgeBase`; runtime-backed supervisors live in `@tangle-network/agent-runtime`. - *"Tune retrieval for a knowledge base"* → `runRetrievalImprovementLoop` in the [Agent-Eval integration](#agent-eval-integration) section. - *"Run an operator-grade KB improvement cycle"* → `improveKnowledgeBase` in the [Agent-Eval integration](#agent-eval-integration) section. @@ -419,6 +419,17 @@ To answer whether a candidate knowledge base improves agent task success, run an Use `knowledgeReleaseReport()` before promotion: pass the candidate and baseline `RunRecord[]` (plus optional `ReleaseTraceEvidence` and the gate decision) and it folds them into a `ReleaseConfidenceScorecard` and a `KnowledgeRelease` using `agent-eval`'s release gates and `RunRecord` validation. +When holdout evidence is required, pass both the holdout-tagged run records and the scenario split: + +```ts +const release = knowledgeReleaseReport({ + candidateId, + candidateRuns, + scenarios, + hasHoldout: true, +}) +``` + Use `buildEvalKnowledgeBundle()` before execution when the question is whether the agent has enough task-world context to run: @@ -805,15 +816,13 @@ await runAgentControlLoop({ }) ``` -## Two-agent research loop +## Verified research loop -`runTwoAgentResearchLoop()` is the offline sibling of `runKnowledgeResearchLoop` -with a differentiated worker/driver split over ONE knowledge base: the `worker` -does primary research (discovers sources, proposes pages for the open gaps); the -`driver` verifies each candidate source before it commits, optionally gap-fills -with its own pass (`driverResearches: true`), and gates on the readiness check. -Both are yours (no creds). The loop owns the deterministic mechanics (indexing, -applying write blocks, scoring readiness) and stops once no blocking gap remains. +`runVerifiedResearchLoop()` coordinates a worker and reviewer over one knowledge base. +The worker discovers sources and proposes pages for open gaps. +The reviewer checks each source before admission and can run its own gap-filling pass with `driverResearches: true`. +Callers provide both roles and any credentials. +The loop owns indexing, write-block application, readiness scoring, and stopping when no blocking gap remains. An equal-compute comparison across 9 ML topics using `glm-5.2` is documented in [docs/two-agent-research-ab.md](docs/two-agent-research-ab.md). @@ -824,10 +833,10 @@ The document includes limitations and reproduction steps. ```ts import { defineReadinessSpec, - runTwoAgentResearchLoop, + runVerifiedResearchLoop, } from '@tangle-network/agent-knowledge' -await runTwoAgentResearchLoop({ +await runVerifiedResearchLoop({ root: './kb', goal: 'Build a grounded onboarding wiki for billing support', readinessSpecs: [defineReadinessSpec({ @@ -836,14 +845,14 @@ await runTwoAgentResearchLoop({ query: 'refund policy customer request', requiredFor: ['support-agent'], })], - // WORKER: primary research targeting `ctx.gaps`. Returns a ResearchContribution. + // The worker researches the current gaps. async worker({ gaps, index }) { return { sources: [/* AddSourceTextInput for the open gaps */], proposalText: '/* ---FILE: knowledge/…--- write-protocol blocks */', } }, - // DRIVER: verifies each candidate source before it commits, then gates. + // The reviewer checks each candidate source before admission. driver: { verifySource(source, { gaps }) { return source.uri ? { accept: true } : { accept: false, reason: 'no uri' } @@ -915,11 +924,14 @@ three primitives that bridge "live authority" → "eval re-runs": - `KnowledgeSource`: pluggable contract (`fetch(opts) → KnowledgeFragment[]`). Every fragment carries `provenance` (URL, source-attested timestamp, - jurisdiction, `verifiable` flag) and `dimensionHints` (which eval - dimensions a change in this fragment should re-score). + jurisdiction, and fetch/extraction status) and `dimensionHints` (which eval + dimensions a change in this fragment should re-score). The `verifiable` + flag rejects failed, blocked, or malformed responses; it does not + cryptographically authenticate the publisher. - `KnowledgeFreshnessStore`: per-`(workspaceId, sourceId)` last-refresh - tracker. Filesystem adapter ships in-package; D1 / Postgres adapter - scaffold is shipped as `createD1FreshnessStoreStub(adapter)`. + tracker. The package ships a filesystem implementation and + `createD1FreshnessStoreStub(adapter)`, a complete bridge for an + application-owned D1, PostgreSQL, SQLite, or equivalent adapter. - `detectChanges(prev, next)`: diffs two fragment snapshots, emits `KnowledgeChange[]` tagged with the affected eval dimensions so a cron scheduler knows exactly which campaigns to re-run. @@ -998,4 +1010,4 @@ async function tick({ workspaceId, prevSnapshots }: { Polite-by-default: every HTTP fetch carries the package User-Agent, is throttled to 1 req/sec/origin, caches successful responses to disk, and marks `verifiable: false` on block pages / 4xx rather than promoting -un-grounded content. See `src/sources/http.ts` for the invariants. +unusable content. See `src/sources/http.ts` for the checks. diff --git a/docs/eval/investment-material-facts.md b/docs/eval/investment-material-facts.md index e5f70f0..d4c0c03 100644 --- a/docs/eval/investment-material-facts.md +++ b/docs/eval/investment-material-facts.md @@ -1,10 +1,10 @@ -# Held-out investment-research eval set — material facts + provenance +# Held-out investment-research eval set: material facts + provenance This is the answer key and the provenance ledger for `tests/eval/investment-thesis-set.ts`. **What the set measures.** Give a research loop a company + ticker + an as-of **cutoff** date and ask it to write an investment thesis. Then grade that thesis -against the held-out **material facts** below — facts the loop never saw. A high +against the held-out **material facts** below, facts the loop never saw. A high score means the thesis surfaced the buried, material, non-obvious drivers a thorough analyst would flag and a single ticker search would miss; it is **not** teaching-to-the-test, because the answer key is firewalled from every loop and @@ -20,14 +20,13 @@ the grader is a `$0`, model-free substring check (`gradeFactAgainstText`). guessed (see the drop log). 3. **Knowable at the cutoff.** Every value was disclosed in, or computable from, a filing available on or before the cutoff. The eventual collapse is **not** a - checklist item — it is recorded as `knownOutcome`, for the reader only, and is - never graded. + checklist item. It is recorded as `knownOutcome` for the reader and is never graded. All five primary documents were fetched live from `https://www.sec.gov/Archives/` during curation (a `curl` with a descriptive `User-Agent`, per SEC fair-access rules). Every dollar figure below was read directly out of the de-tagged filing -text. Provenance is verifiable: each `sourceUrl` contains the company's SEC CIK, -and `tests/eval/investment-thesis-set.test.ts` asserts that invariant. +text. Each `sourceUrl` contains the company's SEC CIK, and +`tests/eval/investment-thesis-set.test.ts` checks that invariant. --- @@ -47,7 +46,7 @@ set was curated (June 2026); `investment-thesis-set.test.ts` asserts this. --- -## SIVB — SVB Financial Group (cutoff 2023-02-24) +## SIVB: SVB Financial Group (cutoff 2023-02-24) Source: [FY2022 10-K (`sivb-20221231.htm`)](https://www.sec.gov/Archives/edgar/data/719739/000071973923000021/sivb-20221231.htm) **Known outcome (not graded):** FDIC receivership March 10, 2023; holding-company Chapter 11 March 17, 2023. @@ -59,14 +58,14 @@ Source: [FY2022 10-K (`sivb-20221231.htm`)](https://www.sec.gov/Archives/edgar/d | SIVB/f3 | concentration | Run-prone uninsured deposit base | "estimated uninsured deposits in U.S. offices that exceed the FDIC insurance limit were **$151.5 billion**" | | SIVB/f4 | margin-trend | Cheap deposits fleeing → funding cost set to rise | "Noninterest-bearing demand deposits to total deposits decreased by **20 percentage points to 47 percent**" | | SIVB/f5 | concentration | Single-client-type (innovation-economy) deposit + credit base | 10-K frames the franchise around "the innovation economy" (technology, life-science, venture) | -| SIVB/f6 | off-balance-sheet | AFS loss in AOCI — the visible, smaller tip | "Available-for-sale securities, at fair value (cost of $ **28,602**) **26,069**" → ~$2.5B AFS loss in AOCI | +| SIVB/f6 | off-balance-sheet | AFS loss in AOCI; the visible, smaller tip | "Available-for-sale securities, at fair value (cost of $ **28,602**) **26,069**" → ~$2.5B AFS loss in AOCI | The decisive, non-obvious fact is SIVB/f1+f2: an interest-rate loss roughly equal to all of equity, sitting in the footnotes because HTM accounting keeps it out of both earnings and book equity. A ticker search shows a profitable bank; the filing shows a mark-to-market hole the size of its capital. -## BBBY — Bed Bath & Beyond Inc. (cutoff 2022-04-21) +## BBBY: Bed Bath & Beyond Inc. (cutoff 2022-04-21) Source: [FY2021 10-K (`bbby-20220226.htm`)](https://www.sec.gov/Archives/edgar/data/886158/000088615822000047/bbby-20220226.htm) **Known outcome (not graded):** Chapter 11 on April 23, 2023; equity wiped out. @@ -80,11 +79,11 @@ Source: [FY2021 10-K (`bbby-20220226.htm`)](https://www.sec.gov/Archives/edgar/d | BBBY/f5 | margin-trend | Inventory building into a demand decline | "Merchandise inventories **1,725,410** 1,671,909" ($ thousands) | The non-obvious fact is BBBY/f1+f2+f4 together: in FY2021 the company **lost $560M, -generated only $18M of operating cash, and still spent $575M buying back stock** — +generated only $18M of operating cash, and still spent $575M buying back stock**, returning more cash than it had. The buyback, not the income statement alone, is why a $1.3B equity base became $174M. -## CVNA — Carvana Co. (cutoff 2023-02-23) +## CVNA: Carvana Co. (cutoff 2023-02-23) Source: [FY2022 10-K (`cvna-20221231.htm`)](https://www.sec.gov/Archives/edgar/data/1690820/000169082023000052/cvna-20221231.htm) **Known outcome (not graded):** Stock fell ~98% from its 2021 peak; a 2023 debt exchange cut and extended obligations, narrowly avoiding bankruptcy. @@ -97,18 +96,18 @@ Source: [FY2022 10-K (`cvna-20221231.htm`)](https://www.sec.gov/Archives/edgar/d | CVNA/f4 | governance | Recurring related-party leases with the founder's family | Related-Party note: DriveTime, controlled by "Ernest Garcia II, Ernest Garcia III, and entities controlled by one or both of them" | | CVNA/f5 | liquidity | A wide loss showing unit economics had not turned | "Net loss $ (**2,894**)" ($ millions) | -The non-obvious facts are CVNA/f2 (interest expense up 2.8x — the debt was now -expensive, not just large) and CVNA/f4 (the controlling Garcia family on both +The non-obvious facts are CVNA/f2 (interest expense up 2.8x, making the debt expensive +as well as large) and CVNA/f4 (the controlling Garcia family on both sides of material leases via DriveTime), neither of which a ticker quote shows. -## PTON — Peloton Interactive, Inc. (cutoff 2022-09-07) +## PTON: Peloton Interactive, Inc. (cutoff 2022-09-07) Source: [FY2022 10-K (`pton-20220630.htm`)](https://www.sec.gov/Archives/edgar/data/1639825/000163982522000117/pton-20220630.htm) **Known outcome (not graded):** Stock fell ~95% from its 2021 peak; founder-CEO departed; mass layoffs and a multi-year turnaround. | ID | Lens | Material fact | Value read from the filing | |---|---|---|---| -| PTON/f1 | margin-trend | Hardware gross margin turned **negative** | Connected Fitness "Gross Margin decreased to (**11**)" percent — losing money per unit sold | +| PTON/f1 | margin-trend | Hardware gross margin turned **negative** | Connected Fitness "Gross Margin decreased to (**11**)" percent; losing money per unit sold | | PTON/f2 | liquidity | Inventory glut as pandemic demand normalized | "Inventories, net **1,104.5** 937" ($ millions) | | PTON/f3 | governance | Dual-class super-voting control | "Class B common stock has **20 votes per share** and our Class A common stock has one vote per share" | | PTON/f4 | liquidity | An order-of-magnitude wider loss | "Net loss $ (**2,827**)" ($ millions) | @@ -116,11 +115,11 @@ Source: [FY2022 10-K (`pton-20220630.htm`)](https://www.sec.gov/Archives/edgar/d | PTON/f6 | leverage | Locked-in purchase commitments into falling demand | "purchase commitments related to the manufacture of Peloton products were estimated to be approximately **$334**" million | The non-obvious fact is PTON/f1: revenue was still large, but the **hardware was -sold below cost** (−11% gross margin) — the unit economics, not just the growth -rate, had broken. PTON/f6 compounds it: the company was contractually obliged to +sold below cost** (−11% gross margin). The unit economics had broken despite the +top-line growth. PTON/f6 compounds it: the company was contractually obliged to buy more inventory it could not sell. -## SI — Silvergate Capital Corporation (cutoff 2022-02-28) +## SI: Silvergate Capital Corporation (cutoff 2022-02-28) Source: [FY2021 10-K (`si-20211231.htm`)](https://www.sec.gov/Archives/edgar/data/1312109/000131210922000051/si-20211231.htm) **Known outcome (not graded):** After the FTX collapse triggered a deposit run, Silvergate announced a voluntary wind-down and liquidation of Silvergate Bank in March 2023. @@ -134,7 +133,7 @@ Source: [FY2021 10-K (`si-20211231.htm`)](https://www.sec.gov/Archives/edgar/dat | SI/f5 | concentration | The moat AND the funding are the same crypto-only bet | "Silvergate Exchange Network ('SEN'), our proprietary ... payment network for participants in the digital currency industry" | The non-obvious fact is SI/f1+f2 together: **99.5% noninterest-bearing deposits, -~58% from crypto exchanges** — a funding base with no contractual term and a +~58% from crypto exchanges**, a funding base with no contractual term and a single correlated counterparty type. A ticker quote shows a fast-growing, low-cost-funding bank; the filing shows a bank that could be emptied in days if crypto sentiment turned. @@ -176,8 +175,8 @@ should not be read as a general "investment-research quality" number. The biases 3. **Sector skew toward financials + distressed consumer.** Two of five are banks (SIVB, SI). The two banks are deliberately given **different** buried-risk - lenses — SIVB is an interest-rate / duration / off-balance-sheet story, SI is a - single-industry deposit-concentration story — so they are not redundant, but + lenses, SIVB is an interest-rate / duration / off-balance-sheet story, SI is a + single-industry deposit-concentration story, so they are not redundant, but the set still over-indexes on balance-sheet fragility and under-tests, e.g., technology platform risk, supply-chain concentration, or accounting-policy aggressiveness in a healthy grower. @@ -197,33 +196,33 @@ should not be read as a general "investment-research quality" number. The biases required a second independent source to establish was dropped (below) rather than sourced to the filing alone. -## Drop log — items considered and NOT included (honesty over coverage) +## Drop log: excluded items These were candidate facts I could not independently ground to a primary source available at the cutoff, so I **dropped them rather than guess**: -- **First Republic Bank (FRC)** — dropped as a company entirely. First Republic +- **First Republic Bank (FRC)**: dropped as a company entirely. First Republic was a state-chartered bank that filed its annual reports with the **FDIC**, not on SEC EDGAR, so its 10-K is not at a `sec.gov/Archives` URL and I could not give - it the same clean, CIK-verifiable provenance as the other five. Its widely cited + it the same CIK-matched provenance as the other five. Its widely cited figures (~$15B HTM-style loss, ~$119.5B uninsured deposits) are real but are best - sourced from the FDIC OIG Material Loss Review, a **post**-cutoff document — using + sourced from the FDIC OIG Material Loss Review, a **post**-cutoff document, using it would violate rule 3. Replaced with Silvergate, whose 10-K is on EDGAR. -- **SIVB exact total unrealized-loss footnote line** — I report the HTM gap as the +- **SIVB exact total unrealized-loss footnote line**: I report the HTM gap as the arithmetic difference of two figures printed on the balance sheet ($91,321M − $76,169M), which is exact and on-cutoff. I did **not** include a separately-quoted "$15.1B net unrealized loss" sentence because I did not locate that exact phrasing in the de-tagged text; quoting a number I could not point to verbatim would break rule 2. The computed value is conservative and checkable. -- **CVNA negative gross-profit-per-unit** — a frequently-cited Carvana red flag, +- **CVNA negative gross-profit-per-unit**: a frequently-cited Carvana red flag, but the per-unit figure I could find cleanly was a derived/analyst number, not a single line item I could quote verbatim from the 10-K at the cutoff. Dropped in favor of the directly-quoted total debt, interest expense, ADESA price, related party, and net loss. -- **PTON / SI specific debt-covenant or going-concern language** — I searched for +- **PTON / SI specific debt-covenant or going-concern language**: I searched for explicit "substantial doubt / going concern" wording in both filings and did **not** find it at these cutoffs (it came later). I did not invent it. The facts included are the ones actually present in the cutoff-date document. @@ -248,5 +247,5 @@ for (const company of investmentThesisSet) { ``` The grader is deterministic and model-free, so the same thesis always scores the -same and the answer key never reaches a model the loop could observe — the same +same and the answer key never reaches a model the loop could observe, the same firewall the deep-question exam (`tests/loops/held-out-exam.ts`) uses. diff --git a/docs/eval/rag-eval-roadmap.md b/docs/eval/rag-eval-roadmap.md index 2255060..81ad45c 100644 --- a/docs/eval/rag-eval-roadmap.md +++ b/docs/eval/rag-eval-roadmap.md @@ -12,7 +12,7 @@ SOTA RAG evaluation requires retrieval quality, context quality, generated-answe | [TruLens RAG Triad](https://www.trulens.org/getting_started/core_concepts/rag_triad/) | The minimal end-to-end triad is context relevance, groundedness, and answer relevance. | | [RAGChecker](https://papers.nips.cc/paper_files/paper/2024/hash/27245589131d17368cccdfa990cbf16e-Abstract-Datasets_and_Benchmarks_Track.html) | Fine-grained diagnosis should separate retrieval misses, noisy context, and unsupported generated claims. | | [BEIR](https://arxiv.org/abs/2104.08663) / TREC-style retrieval | Retrieval still needs classical rank metrics: Recall@k, Precision@k, MRR, MAP, and nDCG. | -| [CRAG](https://arxiv.org/abs/2406.04744) | Real RAG evals must include long-tail, dynamic, multi-hop, and unanswerable questions, not only easy static facts. | +| [CRAG](https://arxiv.org/abs/2406.04744) | Real RAG evals must include long-tail, dynamic, multi-hop, and unanswerable questions alongside easy static facts. | | [DeepEval RAG metrics](https://deepeval.com/docs/metrics-faithfulness) | Production tools converge on faithfulness, answer relevance, and context relevance as generator/retriever checks. | ## Current Repo Status @@ -30,12 +30,13 @@ Done: - `normalizeExternalRagScores()` and the row exporters make Ragas, DeepEval, TruLens, RAGChecker, and custom evaluator outputs pluggable instead of hard dependencies. - `scoreKnowledgeBaseIndex()` validates generic wiki/KB health: citation coverage, source-backed pages, stale sources, duplicate source hashes, and lint/validation errors. - `calibrateRagAnswerJudge()` enforces the strong-vs-weak metric check before trusting a RAG answer metric. +- `runKnowledgeImprovementJob()` in `@tangle-network/agent-runtime` connects runtime backends and worker factories to candidate KB creation, readiness checks, frozen comparisons, spend measurement, and explicit activation. +- `agent-knowledge` remains runtime-free; applications with their own agent runner pass an `updateKnowledge` callback directly. Not done: - Slice-level reporting for freshness, distractors, multi-hop, and long-tail cases. -- Packaged runtime adapters for browser/coding/research agents. - The lifecycle API accepts those agents as hooks today; it does not hardcode provider-specific workers. +- A maintained, public benchmark pack with at least 100 labeled scenarios and published baseline results. ## Completion Criteria @@ -95,7 +96,7 @@ Ship criteria: - Every failed eval has one primary failure class. - At least 95 percent of generated claims can be mapped to supporting context, contradicted context, or no context. -- Reports show metrics by slice and by failure class, not only the aggregate score. +- Reports show metrics by slice and failure class as well as the aggregate score. ### Phase 4: Production Loop @@ -115,5 +116,5 @@ Ship criteria: 1. Add slice-level aggregation helpers for freshness, distractors, multi-hop, long-tail, and unanswerable cases. 2. Add forbidden/stale source targets to retrieval scenarios. -3. Add packaged adapters that turn `agent-runtime` browser, research, and coding agents into `runRagKnowledgeImprovementLoop()` hooks. +3. Add a maintained benchmark pack with at least 100 labeled scenarios and a reproducible baseline report. 4. Add a CLI command that runs the lifecycle loop and writes a reproducible report under `.agent-knowledge/eval/`. diff --git a/docs/results/adaptive.md b/docs/results/adaptive.md index 8533536..6f93c56 100644 --- a/docs/results/adaptive.md +++ b/docs/results/adaptive.md @@ -1,9 +1,9 @@ # Adaptive topology: spend the LLM verifier only when it pays The cost/quality A/B (`docs/results/cost-quality.md`) found the LLM relevance -verifier's cleanliness win is dominated by **de-duplication** — which a +verifier's cleanliness win is dominated by **de-duplication**, which a deterministic content-hash / canonical-URL check captures at ~none of the LLM -premium — and that an LLM check only earns its dollar on the off-scope tail. The +premium, and that an LLM check only earns its dollar on the off-scope tail. The production move it named was: do the cheap deterministic work first, reserve the LLM for the ambiguous tail. `createAdaptiveResearchDriver` (`src/adaptive-driver.ts`) is that driver, and this is its measurement. @@ -19,7 +19,7 @@ stopping at the first that decides: github, …) with a substantial body is **kept**; an obvious spam/listicle title or a too-thin body is **dropped**; everything else is **ambiguous**. 3. **LLM escalation ($).** Only ambiguous survivors reach the shipped LLM - relevance verifier (`createVerifyingResearchDriver`) — one call each. + relevance verifier (`createVerifyingResearchDriver`), one call each. ## Live frontier, n=5 topics (glm-5.2) @@ -39,24 +39,24 @@ is the per-arm `RouterClient.usage()` diff (#36). Total spend for the run: | **total** | **25** | **25** | **12 / 25 / $0.0261** | **20 / 6 / $0.0068** | **Cost.** Adaptive cuts LLM verifier calls **76%** (25 → 6) and dollars **74%** -($0.0261 → $0.0068). On 3 of the 5 topics it spent **zero** LLM calls — every +($0.0261 → $0.0068). On 3 of the 5 topics it spent **zero** LLM calls, every unique survivor was on an authoritative host, so the $0 stages decided everything. -## The honest reading: adaptive is a frontier POINT, not a free lunch +## Interpretation: adaptive occupies one cost-quality point Admitted-source counts (lower = cleaner KB): **single 25, adaptive 20, full-LLM 12**. Adaptive sits **between** the two: - It removes **5 of the 13 sources** the full-LLM judge rejects that the single-agent loop keeps (every one of them a real duplicate caught by the $0 - dedup stage — exactly the de-dup-dominated win the cost/quality result + dedup stage, exactly the de-dup-dominated win the cost/quality result predicted). - It does **NOT** match full-LLM cleanliness. The remaining 8 sources full-LLM rejects, adaptive keeps. The cause is structural and visible in the trace: on this topic set every non-duplicate survivor landed on an authoritative host (arxiv / github / official docs), so the heuristic **kept** it without ever - asking the LLM — and the LLM, when full-LLM did ask it, judged several of + asking the LLM, and the LLM, when full-LLM did ask it, judged several of those same authoritative pages not-quite-on-topic and dropped them. The host prior is coarser than the relevance judge. @@ -65,19 +65,19 @@ de-dup half of full-LLM's cleanliness for ~26% of full-LLM's dollars, and gives up the relevance-judgment half.** Whether that is the right point depends on the cost of a kept-but-marginal source. If a slightly-off-topic authoritative page is cheap to carry, adaptive dominates. If every admitted source must clear a -relevance bar, the host heuristic is too permissive and you want the full LLM — +relevance bar, the host heuristic is too permissive and you want the full LLM, or a tightened heuristic. -## Where the heuristic is weak — stated plainly +## Where the heuristic is weak: stated plainly The escalation count is the diagnostic. On 3 of 5 topics it was **zero**: the heuristic never deferred to the LLM, so on those topics adaptive is a **pure host/title/length rule**, and its cleanliness is exactly that rule's -cleanliness — no smarter than "trust arxiv/github, drop spam." That is fine when +cleanliness, no smarter than "trust arxiv/github, drop spam." That is fine when the worker's sources are dominated by authoritative hosts (as here), but it means the LLM's relevance judgment is contributing nothing on those topics, by construction. The two topics where adaptive *did* escalate (grouped-query -attention, LoRA) are where some survivors were on unknown hosts — and there the +attention, LoRA) are where some survivors were on unknown hosts, and there the 3 LLM calls per topic are the off-scope tail the verifier is actually for. The heuristic would mis-route in two directions a richer worker would expose, @@ -95,12 +95,12 @@ neither seen on this authoritative-host-heavy set: The deployable recommendation from the cost/quality result was "deterministic dedup first, reserve the LLM for the tail." This driver ships that and the measurement confirms the **cost** half cleanly (76% fewer calls, 74% cheaper) -and qualifies the **quality** half honestly: adaptive captures the de-dup +and qualifies the **quality** half accurately: adaptive captures the de-dup cleanliness (the dominant, free win) but not the LLM's relevance cleanliness, because the host heuristic resolves authoritative survivors without asking. For a worker whose sources are mostly authoritative, adaptive is the right frontier point. For one whose junk is on-topic-looking pages on unknown hosts, the -ambiguous tail grows and adaptive converges toward full-LLM cost — which is the +ambiguous tail grows and adaptive converges toward full-LLM cost, which is the correct behavior: it pays for the LLM exactly when the cheap signals can't decide. diff --git a/docs/results/claim-grounding.md b/docs/results/claim-grounding.md index f262d9f..a5606b0 100644 --- a/docs/results/claim-grounding.md +++ b/docs/results/claim-grounding.md @@ -1,22 +1,22 @@ # Claim-grounding: the band where the verifier earns its dollar `docs/results/cost-quality.md` found the relevance verifier's cleanliness win is -**dominated by de-duplication** — a deterministic content-hash captures most of it +**dominated by de-duplication**, a deterministic content-hash captures most of it at ~none of the LLM premium. So the open question was: is there an error band where -a verifier earns its cost — something a hash AND a relevance judge both miss? +a verifier earns its cost, something a hash AND a relevance judge both miss? **Yes: misattributed citations.** A source that is on-topic, unique, and real, but whose cited CLAIM does not appear in the page (the LLM wrote a plausible sentence and hung a real URL off it). De-dup passes it (it's unique). A relevance judge passes it (the page is on-topic). Only checking the claim against the fetched text -catches it — and that check is **deterministic text presence, $0 inference**. +catches it, and that check is **deterministic text presence, $0 inference**. ## The mode Each proposed source now carries the specific claim it is cited for (`withCitedClaim` → `metadata.citedClaim`). The verifier (`createClaimGroundingVerifier`) runs `groundClaimInText(claim, pageText)` over the -`htmlToText` output of the page the worker actually fetched — verbatim, normalized +`htmlToText` output of the page the worker actually fetched, verbatim, normalized (punctuation/whitespace-insensitive), or a ≥70% content-word overlap close paraphrase. A claim that isn't present is rejected as **misattributed**. The oracle is text presence, not a model call, so it composes with the LLM relevance verifier @@ -31,25 +31,25 @@ proposals. Cost diffed per arm with the #36 `RouterClient.usage()` instrumentati | n=5 topics | misattributions caught | marginal $ | $/topic | per-$ caught | |---|---|---|---|---| -| no-verifier | 0 / 5 | $0.0000 | — | — | +| no-verifier | 0 / 5 | $0.0000 | n/a | n/a | | relevance (LLM judge) | **4 / 5** | $0.0157 | ~$0.0031 | 254 | | claim-grounding (text) | **5 / 5** | **$0.0000** | $0 | ∞ | Per-topic (caught relevance / grounding): self-speculative decoding 1/1, rotary position embeddings 1/1, grouped-query attention 1/1, **KV-cache quantization 0/1**, -LoRA 1/1. (An earlier 3-topic run missed self-speculative decoding instead — the +LoRA 1/1. (An earlier 3-topic run missed self-speculative decoding instead, the miss moves around; it is not a fixed topic.) **Reading.** Claim-grounding catches every misattribution at **$0**; the relevance judge catches most but **misses one in five at ~$0.003/topic**. The miss is the point: the relevance verifier only ever sees the page text, never the cited claim, -so it is *structurally blind* to misattribution. It catches one only by accident — +so it is *structurally blind* to misattribution. It catches one only by accident: when the fabricated claim happens to also make the page read off-topic (e.g. a "12-billion-parameter draft transformer" claim on a rotary-embeddings page). When the fabrication stays on-topic (the KV-cache case), the judge waves it through. So on THIS band the verifier-per-dollar comparison inverts the cost/quality result: -there, the LLM verifier bought a dedup-shaped gain a free hash already captures — +there, the LLM verifier bought a dedup-shaped gain a free hash already captures; expensive for what a cheap rule does. Here the cheap, deterministic check **dominates** the expensive judge: it catches strictly more (5/5 vs 4/5) at strictly less ($0 vs $0.0157). The verifier earns its dollar on misattribution; it does not on @@ -63,7 +63,7 @@ de-duplication. (and the offline relevance stand-in) accept them. The error is in the *claim*, not the *topic*. - **Executable ground truth.** The check is presence/close-paraphrase of the claim - in the fetched text — deployable in production with no oracle and no model call. + in the fetched text, deployable in production with no oracle and no model call. The offline arm proves the floor with a controlled 4-source pool (2 grounded, 2 misattributed): claim-grounding admits **0/2** misattributions and keeps **2/2** @@ -78,9 +78,9 @@ grounded sources, while relevance and no-verifier both admit **2/2**. - **Planted misattributions, not naturally-occurring ones.** Like the cost/quality offline floor, the misattribution is injected so the band is measurable. It models the real LLM citation-fabrication failure but does not measure its base rate in the - wild — that needs a corpus of model-written citations checked by hand. + wild, that needs a corpus of model-written citations checked by hand. - **The grounding oracle is conservative.** A real paraphrase whose inflected words - differ from the page ("drafts" vs "draft") can score below 0.7 and be rejected — + differ from the page ("drafts" vs "draft") can score below 0.7 and be rejected, a false-positive misattribution flag. `minOverlap` tunes this; the worker should cite the page's own key terms (the `createClaimDecorator` extractor is told to). diff --git a/docs/results/cost-quality.md b/docs/results/cost-quality.md index 6a1d7f6..d0afcf8 100644 --- a/docs/results/cost-quality.md +++ b/docs/results/cost-quality.md @@ -2,7 +2,7 @@ Live 9-topic A/B (glm-5.2, budget B ≤ 4 passes/arm), measured per arm with the router-client instrumentation. The original A/B reported only admitted-sources at -"equal passes" — which charged the two-agent verify step as one pass while it is +"equal passes", which charged the two-agent verify step as one pass while it is actually N `verifySource` LLM calls. Pricing the calls shows what that hid. | per topic (mean) | two-agent | single-agent | ratio | @@ -11,7 +11,7 @@ actually N `verifySource` LLM calls. Pricing the calls shows what that hid. | tokens (in+out) | ~4,900 | ~530 | ~9× | | cost (USD) | ~$0.0072 | ~$0.0013 | ~5.5× | | latency (wall) | ~37 s | ~11 s | ~3.4× | -| cleanliness Δ (single − two admitted) | — | — | +1.56, 95% CI [0.33, 2.67] | +| cleanliness Δ (single − two admitted) | n/a | n/a | +1.56, 95% CI [0.33, 2.67] | Per-topic Δ (single − two admitted) this run: self-speculative decoding +4, grouped-query attention 0, rotary position embeddings +1, KV-cache quantization @@ -19,9 +19,9 @@ grouped-query attention 0, rotary position embeddings +1, KV-cache quantization descent **−1**. Coverage 1.00 every topic, both arms. **Reading.** The verifier buys ~1.5–2.7 fewer junk sources for roughly **5× the -dollars, 9× the tokens, and 3× the latency** — and on two topics it admitted *more* +dollars, 9× the tokens, and 3× the latency**, and on two topics it admitted *more* than the single agent (the cleanliness signal is real but noisier than the +2.3 / +2.7 of earlier runs). The cleanliness gain is dominated by de-duplication, so the -honest production move is a deterministic content-hash / canonical-URL dedup, which +production move is a deterministic content-hash / canonical-URL dedup, which captures most of the cleanliness at ~none of this premium; reserve an LLM check for the off-scope tail. This is the cost half the "equal passes" framing left out. diff --git a/docs/results/investment-thesis.md b/docs/results/investment-thesis.md index 841c391..493feed 100644 --- a/docs/results/investment-thesis.md +++ b/docs/results/investment-thesis.md @@ -1,25 +1,25 @@ -# From paper-retrieval to investigation — a research eval where one search is not enough, a metric that discriminates depth, and a 3-arm topology A/B +# Investment-research depth eval and three-arm comparison *Tangle Network · `agent-knowledge`* -## Verdict (BLUF) +## Verdict -We moved the research eval off ML-paper retrieval — where a **single** web search +We moved the research eval off ML-paper retrieval, where a **single** web search already returns the answer, so the only thing the metric could measure was -*collection* — onto **investment research**, where the material facts are buried in +*collection*, onto **investment research**, where the material facts are buried in the footnotes of an SEC filing and genuinely require *investigation* to surface. On this harder domain we (1) **calibrated** a `$0`, model-free metric and proved it -discriminates research depth — a shallow ticker summary scores **1/27 (4%)**, a -deep filings-grounded thesis scores **27/27 (100%)**, a **+96-point** gap — and then +discriminates research depth, a shallow ticker summary scores **1/27 (4%)**, a +deep filings-grounded thesis scores **27/27 (100%)**, a **+96-point** gap, and then (2) ran **three research topologies head-to-head** on the same five held-out companies at matched compute, grading each one's knowledge base against a firewalled checklist of **27 buried material facts** a ticker search misses. -**The honest A/B verdict: the research-DRIVING loop surfaced the most buried facts -(16/27, 59%), +5 over blind collection (11/27, 41%) — but at n=5 companies that lift +**A/B result: the research-driving loop surfaced the most buried facts +(16/27, 59%), +5 over blind collection (11/27, 41%), but at n=5 companies that lift is NOT statistically clean: the 95% confidence interval crosses zero (P(Δ≤0)=0.08).** -The verify/dedup loop did **not** beat collection at all (10/27, 37% — a wash, -slightly worse). So no topology *significantly* beats collection here either — +The verify/dedup loop did **not** beat collection at all (10/27, 37%, a wash, +slightly worse). No approach *significantly* beats collection here. **driving is the only arm that even points the right way, and it points there suggestively, not significantly.** What the reframe *did* buy is a domain and a meter where the question is finally well-posed: the metric is no longer measuring whether a @@ -27,7 +27,7 @@ single search ran, but whether investigation reached the buried fact. | arm | what the coordinator does | material facts | cost (5 cos.) | chats | searches | tokens | |---|---|---|---|---|---|---| -| **A · collection** | nothing — accepts every source (1 agent collects) | 11/27 (41%) | $0.082 | 10 | 20 | 64,248 | +| **A · collection** | nothing; accepts every source (1 agent collects) | 11/27 (41%) | $0.082 | 10 | 20 | 64,248 | | **B · verify/dedup** | LLM gates each source for relevance, rejects off-topic | 10/27 (37%) | $0.125 | 56 | 52 | 84,678 | | **C · driving (deepen)** | extracts claims, demands corroboration, asks deep follow-ups | **16/27 (59%)** | $0.157 | 33 | 28 | 124,387 | @@ -35,17 +35,17 @@ Cost is real provenance, not an estimate: every `$`/call/token is diffed from `RouterClient.usage()` per company (the router's own `usage` field, priced at glm-5.2 rates), so "driving cost 1.9× collection" is measured, not modelled. -## 1. The domain reframe — why we left ML-paper retrieval +## 1. The domain reframe: why we left ML-paper retrieval The companion paper's depth eval (`docs/two-agent-research-ab.md` §9) measures a -research loop on **20 deep questions across 5 ML topics**. That apparatus is sound — +research loop on **20 deep questions across 5 ML topics**. That method is sound: its grader scores 0/20 on a one-line topic definition and 20/20 on a mechanism-rich paragraph, so it *can* tell depth from surface. But the topology A/B on top of it -came back a clean null (driving 16/20 @ budget 4, 13/20 @ budget 6 — the winner +came back a clean null (driving 16/20 @ budget 4, 13/20 @ budget 6, the winner *flips with the compute budget*, and the within-arm swing is as large as the between-arm gap). The autopsy named the cause: **on an ML topic a single good search already collects the answer.** Every arm finished in one effective round because the -generic "one source closes the gap" readiness was met by the *first* fetch — so the +generic "one source closes the gap" readiness was met by the *first* fetch, so the driving driver, whose entire mechanism is steering a *second* round, never got to act. When one search suffices, there is no investigation for a smarter coordinator to do, and the metric can only reward collection. That is the failure-mode-in-spirit the @@ -55,12 +55,12 @@ easy for topology to matter. So we changed the domain to one where a single search **provably cannot** suffice. **Investment research**: give a loop a company + ticker + an as-of cutoff date and ask for a thesis; grade it on the buried, material, non-obvious drivers a thorough analyst -flags and a ticker search misses. The decisive facts here live in 10-K footnotes — an +flags and a ticker search misses. The decisive facts here live in 10-K footnotes, an HTM securities mark roughly equal to a bank's entire equity (SIVB), a deposit base that is 97% uninsured, a negative per-unit gross margin, a related-party lease. A web search for the company name returns "profitable regional bank" or "high-growth auto e-commerce"; the filing shows the mark-to-market hole the size of capital. **The -answer is not collectable in one fetch — it has to be investigated for.** That is the +answer is not collectable in one fetch, it has to be investigated for.** That is the property the ML domain lacked, and it is what makes a topology A/B finally meaningful: if a smarter coordinator can ever beat blind collection, a domain where the answer is buried is where it has the room to. @@ -69,7 +69,7 @@ The held-out set is **5 companies, 27 material facts**, every fact read live out the primary SEC EDGAR 10-K during curation, every dollar figure quoted from the de-tagged filing text, every value knowable *at the cutoff* (the eventual collapse is recorded for the reader only and is **never graded**). The companies skew distressed / -downside-risk — a documented curation bias, not a hidden one (full provenance, the +downside-risk, a documented curation bias, not a hidden one (full provenance, the drop log, and the per-fact keyword groups: `docs/eval/investment-material-facts.md`): | ticker | company | as-of cutoff | CIK | facts | @@ -80,25 +80,25 @@ drop log, and the per-fact keyword groups: `docs/eval/investment-material-facts. | PTON | Peloton | 2022-09-07 | 1639825 | 6 | | SI | Silvergate | 2022-02-28 | 1312109 | 5 | -## 2. Calibration — does the metric discriminate depth? (the gate the ML exam passed only weakly) +## 2. Calibration: does the metric discriminate depth? (the gate the ML exam passed only weakly) A topology A/B is meaningless unless the metric grading it can tell a *deep* thesis -from a *shallow* one. If it can't, it is measuring word-collection — the exact failure -the reframe set out to escape — and any A/B on top of it is noise. So **before** the +from a *shallow* one. If it can't, it is measuring word-collection, the exact failure +the reframe set out to escape, and any A/B on top of it is noise. So **before** the A/B, we ran a calibration gate (`$0`, offline, the binding result): For each of the 5 companies we hand-wrote two theses and scored each with the metric's pure core (`materialFactsSurfacedInText`): -- **shallow** — a one-paragraph ticker summary: what the company does, a vibe on the +- **shallow**: a one-paragraph ticker summary: what the company does, a vibe on the stock, generic macro/competition risk. The kind a single name-search returns. Names none of the buried, filing-level facts. -- **deep** — a filings-grounded analyst memo naming the buried drivers (the duration +- **deep**: a filings-grounded analyst memo naming the buried drivers (the duration loss, the buyback drain, the negative unit margin, the deposit concentration, the related party) in independent prose, with the real numbers. The grader is the same `$0`, model-free, case-insensitive substring check the held-out -checklist ships (`gradeFactAgainstText`); the checklist is firewalled — read only by +checklist ships (`gradeFactAgainstText`); the checklist is firewalled, read only by the metric, never shown to a loop. | ticker | shallow | deep | gap | @@ -110,20 +110,20 @@ the metric, never shown to a loop. | SI | 0/5 (0%) | 5/5 (100%) | +100pp | | **total** | **1/27 (4%)** | **27/27 (100%)** | **+96pp** | -**The metric is VALID:** it cleanly separates a shallow ticker-summary from a deep, -filings-grounded thesis — a +96-point aggregate gap, every company clearing the bars +**Calibration result:** the metric separates a shallow ticker-summary from a deep, +filings-grounded thesis, a +96-point aggregate gap, every company clearing the bars (shallow `< 30%`, deep `> 70%`) with wide margin. Two guards make the gap real and not a teaching-to-the-test artifact: - **Anti-circularity.** The gate asserts **no deep thesis verbatim-embeds any - checklist `evidence` string** — the deep theses state the same publicly-documented + checklist `evidence` string**, the deep theses state the same publicly-documented facts in independent analyst prose. A high deep score is the meter catching real, independently-phrased depth, not an answer-key echo. -- **The one honest shallow hit.** The single shallow surface (SIVB 1/6) is SIVB/f5 — - the innovation-economy / venture-client concentration — which fires on "technology" +- **The one shallow hit.** The single shallow surface (SIVB 1/6) is SIVB/f5: + the innovation-economy / venture-client concentration, which fires on "technology" / "venture" in the SVB summary. That genuinely *is* the least-buried of SVB's six facts (a ticker search does return "SVB banks tech and venture startups"). We kept it - as a 4% leak rather than tighten the group, because it honestly reflects one + as a 4% leak rather than tighten the group, because it reflects one near-surface fact while the five truly buried ones (the ~$15B HTM loss, the equity-sized mark, the $151.5B uninsured base, the 20-point deposit-mix shift, the AFS/AOCI loss) stay unsurfaced. The firewall holds (17% << 30%). @@ -132,38 +132,38 @@ Calibration also did its job as more than a rubber stamp: the first pass had the **Silvergate shallow thesis scoring 3/5 (60%)**, blowing the bar, because three fact groups accepted bare crypto-bank vocabulary (`crypto`, `grew rapidly`, `proprietary` / `payment network`) that any one-line summary trips. We tightened those three to require -the *buried* signal — the deposit-**concentration** framing, the specific **$14.3B** -figure, the **SEN** (Silvergate Exchange Network) name — and dropped the generic vocab. +the *buried* signal, the deposit-**concentration** framing, the specific **$14.3B** +figure, the **SEN** (Silvergate Exchange Network) name, and dropped the generic vocab. The deep thesis still hits all three; the shallow one no longer does. The metric found a way it could be fooled and we closed it before any A/B depended on it. **This is the gate the ML exam passed only in spirit.** The ML grader discriminated -0/20 vs 20/20 too — but on a domain where the deep answer was *collectable* in one +0/20 vs 20/20 too, but on a domain where the deep answer was *collectable* in one search, so the discrimination measured grammar, not investigation. Here the +96-point gap is over facts that are *buried by construction*, so a high score is reachable only by reaching into the filing. Same shape of result, materially harder domain. -## 3. The 3-arm A/B — what each topology surfaced, head-to-head +## 3. The 3-arm A/B: what each topology surfaced, head-to-head For each company the loop is told **only** `{company, ticker, cik, cutoff}` plus a generic set of analyst-lens readiness specs (balance-sheet risk, concentration, -leverage, margins, liquidity, governance, regulatory — the *lenses* and where they -live, the latest SEC 10-K — **not the answers**). It researches the company *as of* the +leverage, margins, liquidity, governance, regulatory, the *lenses* and where they +live, the latest SEC 10-K, **not the answers**). It researches the company *as of* the cutoff over web + SEC EDGAR (both public), writes a thesis, and we grade the resulting -KB with `materialFactsSurfaced` — the firewalled checklist the loop never sees. +KB with `materialFactsSurfaced`, the firewalled checklist the loop never sees. **Compute is matched by construction.** All three arms run the *same* web worker, the *same* 3-round budget, the *same* worker config (`resultsPerQuery: 3, queriesPerGap: 1, -maxSourcesPerRound: 6`). The **only** thing that varies is the driver — the coordinator +maxSourcesPerRound: 6`). The **only** thing that varies is the driver, the coordinator between the worker and the knowledge base: -- **A · collection** (`createCollectionResearchDriver`) — an inert rubber stamp: +- **A · collection** (`createCollectionResearchDriver`), an inert rubber stamp: accepts every source, gates nothing, steers only with the loop's built-in open-gap list. The driver adds **zero** router calls. This is the blind-collection floor. -- **B · verify/dedup** (`createVerifyingResearchDriver`) — an LLM relevance gate: one +- **B · verify/dedup** (`createVerifyingResearchDriver`), an LLM relevance gate: one chat call per candidate source to accept-or-reject for on-topic relevance and near-duplication. The worker ADDS; the driver GATES. -- **C · driving** (`createResearchDrivingDriver`) — extracts each source's claims, +- **C · driving** (`createResearchDrivingDriver`), extracts each source's claims, tracks how many *independent* sources corroborate each, and synthesizes deep follow-up sub-questions (comparative / mechanism / gap / contradiction) it folds into the worker's next prompt to push depth and demand corroboration. @@ -200,13 +200,13 @@ Every interval crosses zero. **Driving vs collection is the closest to clean coin flip. This is the project's well-documented small-n mirage: exciting deltas born at n=5 do not survive a paired bootstrap. -## 4. Autopsy — the two things worth understanding +## 4. Autopsy: the two things worth understanding ### 4.1 Why driving wins where it wins Driving's mechanism is multi-round: extract claims from round 1, then steer the worker in rounds 2–3 to corroborate the weak ones and chase the deep questions. It helps most -where the **first** fetch lands real filing data the driver can build on — the two +where the **first** fetch lands real filing data the driver can build on, the two banks, where SEC bank-call-report / 10-K data is dense and reachable. SIVB jumps from 2 buried facts (collection) to 5 (driving): the duration loss, the deposit concentration, the AFS/AOCI mark all surface once the driver demands the balance-sheet detail a second @@ -230,21 +230,21 @@ ROUND 1: accepted=0 rejected=3 writtenPages=0 ROUND 2: accepted=0 rejected=2 writtenPages=0 ``` -The verifier was **correct on the merits** — those are aggregators, not the primary +The verifier was **correct on the merits**, those are aggregators, not the primary filing, and last10k showed a post-cutoff filing. But the worker never surfaced the EDGAR primary for BBBY, so a strict primary-only gate left the KB **empty**. Collection accepts the same aggregator pages and scores 4/5 on BBBY; driving accepts them and scores 5/5. **The gate's strictness is a liability when the worker's sourcing is -imperfect:** it throws away the only evidence the loop had. This is a genuine topology -trade-off worth naming, not a harness break — the verify arm surfaced facts fine on -CVNA (2), PTON (4), SI (4), so the harness works. +imperfect:** it throws away the only evidence the loop had. This is a real coordination +trade-off. The verify arm still surfaced facts on CVNA (2), PTON (4), and SI (4), +which rules out a general execution failure. -### 4.3 The empty-thesis caveat (honest) +### 4.3 The empty-thesis caveat Some runs show `thesis=0ch` (empty synthesis) yet still score facts. That is expected: -`materialFactsSurfaced` grades the **whole KB** (the worker's fetched `knowledge/*.md` -pages), not only the final thesis page. glm-5.2 occasionally spends its entire output -budget on hidden reasoning and returns empty visible content on the synthesis call — a +`materialFactsSurfaced` grades the **whole KB**, including the worker's fetched +`knowledge/*.md` pages and the final thesis page. glm-5.2 occasionally spends its entire output +budget on hidden reasoning and returns empty visible content on the synthesis call, a known reasoning-model behavior we floor at 1200 tokens but can't fully eliminate. The score still reflects what the loop fetched, so an empty thesis does not invalidate a run; it just means the synthesis prose was lost while the curated evidence was not. @@ -252,7 +252,7 @@ run; it just means the synthesis prose was lost while the curated evidence was n ## 5. What this does and does not establish - **Does**: the metric *discriminates depth on a domain where one search is not - enough* (calibration: 1/27 shallow vs 27/27 deep, +96pp) — so the topology A/B on top + enough* (calibration: 1/27 shallow vs 27/27 deep, +96pp), so the topology A/B on top of it is finally well-posed, unlike the ML retrieval exam where one search already collected the answer. At matched compute on the 5-company held-out set, the research-driving topology surfaced the most buried material facts (16/27 vs 11/27 for @@ -272,10 +272,10 @@ run; it just means the synthesis prose was lost while the curated evidence was n ## 6. Reproduce ```bash -# The calibration gate that must pass FIRST ($0, offline) — the metric is valid. +# The calibration gate that must pass FIRST ($0, offline): the metric is valid. pnpm test investment-calibration -# The offline task wiring ($0) — proves page → index → grade end-to-end. +# The offline task wiring ($0): proves page → index → grade end-to-end. pnpm test investment-thesis-task # The full live 3-arm A/B (costs ~$0.36 total at 5 companies × 3 arms × 3 rounds). @@ -291,12 +291,12 @@ AGENT_KNOWLEDGE_LIVE=1 IT_LIVE_TICKERS=CVNA IT_LIVE_ARMS=collection \ `IT_LIVE_ARMS` (`|`-separated subset of `collection|verify|driving`) and `IT_LIVE_TICKERS` scope the run; `IT_LIVE_ROUNDS` sets the per-arm round budget -(default 3 — driving needs > 1). The smoke (one cheap glm-5.2 call) runs once before any +(default 3, driving needs > 1). The smoke (one cheap glm-5.2 call) runs once before any arm, so a bad key fails fast, before the burn. --- -*Run provenance. Calibration: `$0`, offline, reproducible — numbers from +*Run provenance. Calibration: `$0`, offline, reproducible, numbers from `tests/eval/investment-calibration.test.ts` (shallow `< 30%`, deep `> 70%`, gap `> 40pp` per company and aggregate; all pass). A/B: 5 companies × 3 arms × 3 rounds = 15 live company-runs, glm-5.2 over the Tangle router, ~37.5 min wall, $0.36 total; diff --git a/docs/results/research-driving.md b/docs/results/research-driving.md index 01c969a..ac752e2 100644 --- a/docs/results/research-driving.md +++ b/docs/results/research-driving.md @@ -1,10 +1,10 @@ -# Does driving research DEEPER beat just collecting it? A held-out deep-question A/B — and an honest null +# Research-driving versus collection: a held-out deep-question result *Tangle Network · `agent-knowledge`* -## Verdict (BLUF) +## Verdict -We built a research driver whose job is to push a web-research loop **deeper** — +We built a research driver whose job is to push a web-research loop **deeper**: extract each source's claims, demand a second independent source for every claim, generate comparative / mechanism / contradiction sub-questions, and steer the worker to chase them. The hypothesis: a KB built this way answers **more held-out @@ -13,7 +13,7 @@ verifier. On a **firewalled exam of 20 deep questions across 5 ML topics**, graded with a $0 deterministic check the loop never sees, at equal compute, **the driving loop -did NOT reliably beat plain collection — and cost ~12–16× more.** The verdict +did NOT reliably beat plain collection, and cost ~12–16× more.** The verdict flips with the compute budget, which is the tell that it is **noise, not signal**: | arm | answered @ B=4 | answered @ B=6 | cost (5 topics) | tokens | @@ -24,20 +24,20 @@ flips with the compute budget, which is the tell that it is **noise, not signal* Driving "wins" at B=4 (16 > 15 > 13) and "loses" at B=6 (13 < 15), while the single-agent arm itself swings 15→13 and driving swings 16→13 on the *same* exam. -At n=5 topics a ±1–3 question difference is within the run-to-run web variance — +At n=5 topics a ±1–3 question difference is within the run-to-run web variance; the arms are not separated by topology, they are separated by **which pages the web returned that minute**. The one thing that is stable and large is the **cost**: driving spends an order of magnitude more for no reliable quality gain. **Why** (the autopsy that explains it, §4): every arm finished in **one effective -round** (`passes=2` on every topic, every budget). The generic readiness gate — -"one source closes the gap" — is satisfied by the first round's fetch, so the +round** (`passes=2` on every topic, every budget). The generic readiness gate, +"one source closes the gap", is satisfied by the first round's fetch, so the loop *stops before the driving driver ever steers a second round*. Driving's entire mechanism is multi-round (extract → demand corroboration → re-search). So -the headline A/B under-tests it. **We then gave it its fairest test** — a -controlled probe (§5) that *forces* 3 rounds so the driver actually steers — and +the headline A/B under-tests it. **We then ran a controlled test**, a +controlled probe (§5) that *forces* 3 rounds so the driver actually steers, and it **still does not win: 8 vs 8 against a blind worker, at ~9× the cost.** The -negative result is robust, not a gate artifact. +same result remains after controlling the stopping condition. This is a real negative result, and we report it as one. @@ -45,7 +45,7 @@ This is a real negative result, and we report it as one. The prior A/B in this repo ([two-agent-research-ab.md](../two-agent-research-ab.md)) measured **cleanliness** -— how *few* sources a verifier admits at equal coverage. That is the right metric +- how *few* sources a verifier admits at equal coverage. That is the right metric for a verifier whose job is to filter. It is the **wrong** metric for the driving driver, whose thesis is the opposite: it is not trying to admit fewer sources, it is trying to build a KB that *answers more*. Measuring it on admitted-count would @@ -60,21 +60,21 @@ So we need a metric for research **quality**, not hygiene. We use: [`tests/loops/held-out-exam.ts`](../../tests/loops/held-out-exam.ts). 5 ML topics, 4 questions each = **20 deep questions**. Each is a *depth* question by -construction — comparative, mechanism-level, or contradiction-aware — chosen so a +construction, comparative, mechanism-level, or contradiction-aware, chosen so a single web search for the topic name does **not** surface the answer: -- *speculative decoding* — the rejection-sampling acceptance rule and why output +- *speculative decoding*, the rejection-sampling acceptance rule and why output is lossless; how self-speculative/Medusa (no draft model) trades off vs the two-model scheme; what bounds the speedup; why one verify pass is ~free. -- *LoRA* — the `W = W₀ + BA` update and which matrices train; QLoRA's 4-bit +- *LoRA*, the `W = W₀ + BA` update and which matrices train; QLoRA's 4-bit tradeoff; why LoRA adds zero inference latency (merge); the very-low-rank claim. -- *grouped-query attention* — the MQA↔GQA↔MHA spectrum of KV heads; the real +- *grouped-query attention*, the MQA↔GQA↔MHA spectrum of KV heads; the real bottleneck (KV cache / memory bandwidth, not FLOPs); uptraining from a checkpoint; the MQA quality cost GQA recovers. -- *RLHF/PPO* — the KL-to-reference penalty and what it prevents; the pairwise +- *RLHF/PPO*, the KL-to-reference penalty and what it prevents; the pairwise preference reward model + Bradley-Terry loss; DPO's reward-model-free insight; PPO's clipped surrogate. -- *mixture-of-experts* — top-k routing and sparse activation; load imbalance + the +- *mixture-of-experts*, top-k routing and sparse activation; load imbalance + the auxiliary balancing loss; params-decoupled-from-FLOPs; the memory cost at inference. @@ -83,7 +83,7 @@ any loop.** A loop is told only the topic name and the *same generic readiness specs* every arm gets ("what X is and how it works" / "results, mechanisms, trade-offs"). It researches blind. **After** it finishes, we grade the KB it built against the held-out questions with a **$0 deterministic substring grader** (no -LLM — so the exam cannot leak into a model the loop observes), where each question +LLM, so the exam cannot leak into a model the loop observes), where each question carries the specific load-bearing answer tokens as keyword groups (a number, a name, a mechanism phrase), with synonym groups so a faithful page in its own words still grades as answered. @@ -95,21 +95,21 @@ still grades as answered. | a one-line topic-definition snippet (what a single search returns) | **0 / 20** | | a deep, mechanism-rich paragraph | **20 / 20** | -So a high score is only reachable by depth — the firewall is real, and the gap +So a high score is only reachable by depth, the firewall is real, and the gap between arms (if any) would be real depth, not grader slack. ## 3. The three arms, at equal compute [`tests/loops/research-driving-ab.test.ts`](../../tests/loops/research-driving-ab.test.ts). All three arms run the **same** real web worker -([`createWebResearchWorker`](../../src/web-research-worker.ts) — glm-5.2 query-gen +([`createWebResearchWorker`](../../src/web-research-worker.ts), glm-5.2 query-gen → live `/v1/search` → `politeFetch` → `htmlToText`). They differ **only** in the driver: -- **(A) single-agent collection** — the worker alone, no driver. It collects. -- **(B) verify/dedup** — [`createVerifyingResearchDriver`](../../src/web-research-worker.ts): +- **(A) single-agent collection**: the worker alone, no driver. It collects. +- **(B) verify/dedup**: [`createVerifyingResearchDriver`](../../src/web-research-worker.ts): a second glm-5.2 pass filters each source for relevance / near-duplicates. -- **(C) DRIVING** — [`createResearchDrivingDriver`](../../src/research-driving-driver.ts): +- **(C) DRIVING**: [`createResearchDrivingDriver`](../../src/research-driving-driver.ts): the driver extracts each source's claims (glm-5.2), tracks independent-source support + contradictions, and folds comparative / mechanism / gap / contradiction sub-questions into the worker's next prompt. @@ -118,10 +118,10 @@ driver: single-agent iteration is 1 worker pass; a two-agent round is 1 worker + 1 driver pass = 2. Each arm gets the same pass ceiling B; the single-agent arm gets more iterations to spend the budget the two-agent arms burn on their driver. Cost is -read per-arm from `RouterClient.usage()` — measured dollars/tokens/calls, not +read per-arm from `RouterClient.usage()`, measured dollars/tokens/calls, not estimates. -## 4. Result — driving does not reliably win, and the verdict flips with budget +## 4. Result: driving does not reliably win, and the verdict flips with budget Per-topic held-out questions answered (out of 4 each), at both budgets: @@ -134,15 +134,15 @@ Per-topic held-out questions answered (out of 4 each), at both budgets: | mixture-of-experts | 4 / 4 | 4 / 3 | 4 / 3 | | **total /20** | **13 / 15** | **15 / 15** | **16 / 13** | -The driving arm answers **16** at B=4 and **13** at B=6 — a 3-question swing on +The driving arm answers **16** at B=4 and **13** at B=6, a 3-question swing on the *same* exam, the same arm, just a different compute ceiling and a different minute of web results. Single swings 13→15; verify is flat 15→15. **The within-arm swing is as large as the between-arm gap**, which is the signature of a null: whatever separates a "win" from a "loss" here is web variance, not the driver. The **cost**, by contrast, is stable and large. Driving spends **~$0.084–0.089** -across 5 topics vs **~$0.005–0.007** for single-agent — **12–16× the dollars** and -**~24× the tokens** (~70k vs ~2.5k) — because it runs a claim-extraction LLM call +across 5 topics vs **~$0.005–0.007** for single-agent, **12–16× the dollars** and +**~24× the tokens** (~70k vs ~2.5k), because it runs a claim-extraction LLM call on every fetched source. For that 12–16× it buys no reliable held-out-question gain over plain collection. @@ -151,18 +151,18 @@ gain over plain collection. The decisive diagnostic is `passes=2` on **every** topic at **every** budget: the two-agent loop ran exactly **one** worker round before the readiness gate reported done, even with B=6 budget for three rounds. The generic specs require one source -to close a gap, and the worker's first-round fetch closes them — so the loop stops +to close a gap, and the worker's first-round fetch closes them, so the loop stops before round 2. The driving driver's mechanism is *multi-round*: round 1 extracts claims and flags the weakly-supported ones; round 2+ is where it steers the worker to corroborate and go deeper. **It never got a round 2.** So in this setting driving's only active effect was a one-round claim-extraction tax with no chance to -use what it extracted — which is exactly what the numbers show: same answers as +use what it extracted, which is exactly what the numbers show: same answers as collection, much higher cost. That makes the equal-compute/generic-gate A/B a test of the wrong thing for the driving thesis. The fair test has to *force* multiple rounds. -## 5. Controlled multi-round probe — the driving thesis's fairest test +## 5. Controlled multi-round probe To isolate "does depth-steering help **when it actually runs**?", we raise the readiness bar so the gate stays unmet and the loop runs the full round budget, @@ -172,23 +172,23 @@ worker **re-searches the same gaps blind** (a no-op driver that accepts every source and never steers). If driving's steering has any value, this is where it shows. -**Result — 3 rounds, 3 topics, driving steers vs blind re-search:** +**Result, 3 rounds, 3 topics, driving steers vs blind re-search:** | topic | driving (steered) answered / cost | blind (no steer) answered / cost | |---|---|---| -| speculative decoding | 2/4 — $0.032 | **4/4 — $0.004** | -| LoRA | 3/4 — $0.028 | 3/4 — $0.003 | -| RLHF / PPO | **3/4** — $0.033 | 1/4 — $0.003 | -| **total /12** | **8/12 — $0.093** | **8/12 — $0.010** | +| speculative decoding | 2/4; $0.032 | **4/4; $0.004** | +| LoRA | 3/4; $0.028 | 3/4; $0.003 | +| RLHF / PPO | **3/4**; $0.033 | 1/4; $0.003 | +| **total /12** | **8/12; $0.093** | **8/12; $0.010** | -**Steering does not help: 8 vs 8, at ~9× the cost.** Given its fairest test — the -full multi-round regime its mechanism was designed for — the driving driver ties a +**Steering does not help: 8 vs 8, at ~9× the cost.** In the controlled test, the +full multi-round regime its mechanism was designed for, the driving driver ties a blind worker that just re-searches the same gaps three times. It is **better on RLHF** (3 vs 1, the one topic where chasing corroboration found a page blind -re-search missed) and **worse on speculative decoding** (2 vs 4 — steering pulled +re-search missed) and **worse on speculative decoding** (2 vs 4, steering pulled the worker *off* the pages that answered the exam and toward corroborating a narrower claim set). Those cancel. So the depth-steering does change *what* gets -fetched, but not *how many* held-out questions get answered — and it pays ~9× the +fetched, but not *how many* held-out questions get answered, and it pays ~9× the dollars for the privilege. The headline null (§4) is therefore **not** merely a gate artifact: even forced to run, driving does not beat blind collection on research quality at this n. @@ -198,13 +198,13 @@ research quality at this n. - **Small n, high web variance.** n = 5 topics; one live run per arm per budget. The §4 budget-flip is itself the evidence that the per-run magnitudes are variance-bound. We did not run a paired bootstrap because the within-arm swing - already exceeds the between-arm gap — the honest read is "no separation," and a + already exceeds the between-arm gap, the supported conclusion is "no separation," and a significance test on a known-null at n=5 would dress it up, not clarify it. -- **The gate, not the driver, ended the headline loop (§4) — but the probe +- **The gate, not the driver, ended the headline loop (§4), but the probe controls for it.** The headline A/B's generic one-source readiness gate closed every loop after one round, so on its own it would only show "driving adds a one-round extraction tax." The §5 probe removes that confound by forcing 3 - rounds, and driving still ties blind (8 vs 8) — so the null survives the fix, + rounds, and driving still ties blind (8 vs 8), so the null survives the fix, it is not an artifact of the permissive gate. - **The worker is shared and shallow.** All arms use the same ~500-line direct pipeline (query-gen → search → fetch), not an `AgentProfile` on a harness. A @@ -220,15 +220,15 @@ research quality at this n. ## 7. What this says, plainly Adding a "drive it deeper" agent did **not** make the research measurably better at -answering hard, held-out questions — at equal compute (§4) *and* forced to run its -full multi-round mechanism (§5), on this worker, at n=5 — and it cost 9–16× more. +answering hard, held-out questions, at equal compute (§4) *and* forced to run its +full multi-round mechanism (§5), on this worker, at n=5, and it cost 9–16× more. The steering changes *what* gets fetched (it helped on RLHF, hurt on speculative decoding) but not *how many* held-out questions get answered. The most durable -thing the experiment produced is the **measurement apparatus**: a firewalled +thing the experiment produced is the **measurement method**: a firewalled deep-question exam with a $0 deterministic grader that *can* tell depth from surface (0/20 vs 20/20), reusable for any future research-quality claim in this -repo. The driving thesis — that pursuing depth + corroboration beats plain -collection — is, on the evidence here, **not supported**; the cheaper paths +repo. The driving thesis, that pursuing depth + corroboration beats plain +collection, is, on the evidence here, **not supported**; the cheaper paths (collect, or dedup) match it. Where it might still earn its cost: a worker rich enough that "go corroborate this claim" reaches a page blind re-search can't (the RLHF case), measured at an n large enough to separate that from variance. @@ -242,24 +242,24 @@ cd agent-knowledge && pnpm install # offline: the exam wiring + the $0 grader (no credentials) pnpm exec vitest run tests/loops/research-driving-ab.test.ts -# the live 3-arm A/B — real web search + glm-5.2, per-arm cost reported +# the live 3-arm A/B: real web search + glm-5.2, per-arm cost reported export TANGLE_API_KEY= AGENT_KNOWLEDGE_LIVE=1 RQ_LIVE_BUDGET=4 \ pnpm exec vitest run tests/loops/research-driving-ab.test.ts -t "3-arm A/B" # re-run at RQ_LIVE_BUDGET=6 to see the verdict flip (the §4 variance point) -# the controlled multi-round probe — forces N rounds so driving actually steers +# the controlled multi-round probe: forces N rounds so driving actually steers AGENT_KNOWLEDGE_LIVE=1 RQ_PROBE=1 RQ_PROBE_ROUNDS=3 TANGLE_API_KEY=<…> \ pnpm exec vitest run tests/loops/research-driving-ab.test.ts -t "multi-round probe" # (~$0.10 for the 5-topic A/B at one budget; ~$0.10 for the 3-topic probe) ``` `RQ_LIVE_TOPICS` takes a `|`-separated subset of the exam topic names to run a -cheaper slice. The exam is held out by construction — no flag shows it to a loop. +cheaper slice. The exam is held out by construction, no flag shows it to a loop. -**Source:** the exam + grader — +**Source:** the exam and grader: [`tests/loops/held-out-exam.ts`](../../tests/loops/held-out-exam.ts); -the 3-arm A/B + multi-round probe — +the 3-arm A/B and multi-round probe: [`tests/loops/research-driving-ab.test.ts`](../../tests/loops/research-driving-ab.test.ts); -the driving driver under test — +the driving driver under test: [`src/research-driving-driver.ts`](../../src/research-driving-driver.ts). diff --git a/docs/two-agent-research-ab.md b/docs/two-agent-research-ab.md index 925e8db..6780a12 100644 --- a/docs/two-agent-research-ab.md +++ b/docs/two-agent-research-ab.md @@ -10,34 +10,34 @@ fixed. A *worker* agent searches the web and proposes sources for the knowledge base's open gaps; a *driver* agent vets each proposed source before it commits, fills gaps the worker missed, and decides when the base is complete. Over 9 machine-learning topics at equal compute, the two-agent loop admitted **2.3–2.7 -fewer sources per topic at identical coverage** — 95% bootstrap intervals +fewer sources per topic at identical coverage**, 95% bootstrap intervals [1.78, 2.89] and [2.22, 3.00] across two independent runs, both above zero. The effect is real and reproduces. But the mechanism is not the one we set out to -test: reading the rejection logs, most of the gain is **de-duplication** — the -same paper fetched from arXiv, OpenReview, and the NeurIPS proceedings — not the +test: reading the rejection logs, most of the gain is **de-duplication**, the +same paper fetched from arXiv, OpenReview, and the NeurIPS proceedings, not the relevance filtering we expected. Pricing the verifier's calls (we added per-arm router-usage instrumentation) shows the cleanliness costs roughly **5× the -dollars, 9× the tokens, and 3× the latency** of the single agent — and that the +dollars, 9× the tokens, and 3× the latency** of the single agent, and that the original "equal passes" framing hid this, because it charged the verify step as one pass while it is actually one LLM call per proposed source. Since the win is de-dup-dominated, a deterministic content hash recovers most of the cleanliness at -~none of the premium. We then asked the sharper question — is there an error band -where an LLM verifier *does* earn its dollar? — and found two, on opposite sides of +~none of the premium. We then asked the sharper question, is there an error band +where an LLM verifier *does* earn its dollar?, and found two, on opposite sides of the ledger. **Misattributed citations** (an on-topic, unique, real source whose cited claim never appears in the page) are caught by a $0 deterministic text-presence check that the LLM relevance judge misses 1 in 5 times, because the judge structurally never sees the claim. And we built the deployable shape the -cost result implied — an **adaptive driver** that runs free dedup, then free +cost result implied, an **adaptive driver** that runs free dedup, then free heuristic triage, and escalates to the LLM only on the ambiguous tail: it cuts LLM verifier calls 76% and dollars 74%, recovering the de-dup half of the -verifier's cleanliness while honestly giving up the relevance-judgment half on a +verifier's cleanliness while giving up the relevance-judgment half on a source pool dominated by authoritative hosts. The verifier earns its dollar on misattribution, not on de-duplication; the right production loop spends it only where the cheap signals can't decide. Finally we ask the harder question this whole -metric can't reach — a filter agent can only make the base *carry less*, never -*answer more* — by building the opposite agent (a **driving** driver that chases +metric can't reach, a filter agent can only make the base *carry less*, never +*answer more*, by building the opposite agent (a **driving** driver that chases depth and corroboration instead of pruning) and the opposite metric (a firewalled -exam of 20 held-out **deep questions**, $0-graded). It is an honest null: driving +exam of 20 held-out **deep questions**, $0-graded). The result is null: driving does **not** reliably beat plain collection at answering hard questions, and costs **12–16×** more; the verdict flips with the compute budget, the signature of web variance rather than a real topology effect, and it still ties a blind worker even @@ -46,15 +46,15 @@ when forced to run its full multi-round mechanism (§9). ## 1. Setup A research agent building a knowledge base accumulates sources. A single agent both -finds sources and, implicitly, decides which to keep — it grades its own work. The +finds sources and, implicitly, decides which to keep, it grades its own work. The hypothesis is the usual one for verification: separating the producer (find sources) from the checker (keep the good ones) should yield a cleaner result, for the same reason a second pair of eyes catches typos the author misses. The trap in any "more agents help" claim is compute. Two agents that simply do more -work will of course produce more — that is a bigger budget, not a finding. So the -comparison must hold total compute fixed and ask whether the *topology* — splitting -find from check — beats spending the same compute on a single agent that just finds +work will of course produce more, that is a bigger budget, not a finding. So the +comparison must hold total compute fixed and ask whether the *topology*, splitting +find from check, beats spending the same compute on a single agent that just finds more. And once topology shows an effect, the second question is what it costs: a cleaner base bought at 5× the inference is a different product decision than one bought for free. @@ -65,9 +65,9 @@ bought for free. The loop has two roles (`src/two-agent-research-loop.ts`, `runTwoAgentResearchLoop`): -- **Worker** — primary research. Each round it reads the open gaps and proposes +- **Worker**: primary research. Each round it reads the open gaps and proposes sources to close them (`ResearchWorker: (ctx: { gaps, steer }) => proposals`). -- **Driver** — does three things (`ResearchDriver`): `verifySource` vets each +- **Driver**: does three things (`ResearchDriver`): `verifySource` vets each proposed source before it commits (dedup against the base, then reject sources that aren't relevant); `research` runs the driver's own gap-fill pass over gaps the worker missed; `foldGaps` turns the still-open gaps into a `steer` string for @@ -80,7 +80,7 @@ reports no blocking gaps left. Note what the driver→worker hand-off is and isn't: the driver *steers* the worker by handing it the remaining readiness gaps (`foldGaps`), which is a deterministic -formatting of unmet requirements — not an LLM authoring a fresh instruction. The +formatting of unmet requirements, not an LLM authoring a fresh instruction. The driver's LLM work is in `verifySource` (one call per proposed source) and its own `research` pass. This matters for §4.2: the verify step is N calls, not one, and the cost framing turns on that. @@ -90,14 +90,14 @@ The readiness gate is `scoreKnowledgeReadiness` (from `agent-eval`). It scores 'blocking'` requirements gate. Coverage below is the fraction of a topic's blocking requirements met. -### 2.2 What the agents are — an honest note +### 2.2 Implementation note The agents in the live run are **not** `AgentProfile`s on a coding harness. The worker is a hand-wired pipeline (`src/web-research-worker.ts`, `createWebResearchWorker`): glm-5.2 turns the gaps into search queries → a real web search over the Tangle router (`POST /v1/search`) → each hit is fetched with the repo's `politeFetch` and reduced to text with `htmlToText` → citing pages are -proposed. It talks to the router directly through `createTangleRouterClient` — no +proposed. It talks to the router directly through `createTangleRouterClient`, no claude-code / opencode / sandbox harness, and no dynamic harness selection. The driver (`createVerifyingResearchDriver`) is one glm-5.2 chat call per source. @@ -109,7 +109,7 @@ Agent profiles and sandbox-backed runners now live in `agent-runtime`; this pack Compute is counted in agent passes. A two-agent round = 1 worker pass + 1 verify pass = 2 passes; a single-agent iteration = 1 pass. Both arms gate on the *same* readiness criterion and stop as soon as it is met, so neither is starved. -We budget-match by passes, not rounds — the single-agent loop gets more rounds to +We budget-match by passes, not rounds, the single-agent loop gets more rounds to spend the compute the two-agent loop spends on verification. The harness asserts, per topic, that the two-agent loop spent no more passes than the single-agent loop and that both stayed under the ceiling; if that ever fails the comparison has @@ -137,7 +137,7 @@ results / trade-offs). Seven are "narrow-scope-inside-a-broad-space" (e.g. broad space to leak in; two are clean controls (*the transformer architecture*, *gradient descent*). -## 3. Result 1 — cleanliness: the verifier admits fewer sources at equal coverage +## 3. Result 1: cleanliness: the verifier admits fewer sources at equal coverage The cleanliness signal is the **admitted-source count**: on live data there is no oracle, so "fewer sources admitted at equal coverage" is the measurable proxy for @@ -157,7 +157,7 @@ oracle, so "fewer sources admitted at equal coverage" is the measurable proxy fo | **mean Δ** | | **2.33** | **2.67** | | **95% CI** (paired bootstrap) | | **[1.78, 2.89]** | **[2.22, 3.00]** | -Coverage was **1.00 on every topic, both arms, both runs** — the verifier never +Coverage was **1.00 on every topic, both arms, both runs**, the verifier never cost completeness. Both bootstrap intervals (`pairedBootstrap`, from `agent-eval`) are above zero. The effect reproduces; its exact magnitude varies run-to-run with what the web returns (one topic swung Δ = 0→1→3 across separate runs during @@ -166,10 +166,10 @@ development). A third, cost-instrumented run (the one priced in §4.2) was noisier: mean Δ **+1.56, 95% CI [0.33, 2.67]**, with two topics where the two-agent loop admitted *more* than the single agent (KV-cache quantization −1, gradient descent −1). The interval still -clears zero, but it is the lower-bound run — a reminder that the magnitude is +clears zero, but it is the lower-bound run, a reminder that the magnitude is web-variance-bound, while the sign is stable. -## 4. Result 2 — what the verifier does, and what it costs +## 4. Result 2: what the verifier does, and what it costs ### 4.1 Mostly de-duplication @@ -181,18 +181,18 @@ We classified each rejection by the verifier's own stated reason: | off-scope (broad space leaked in) | 3 | 2 | | junk page (aggregator / marketing / explainer) | 3 | 0 | -The dominant mechanism is **de-duplication** — canonical papers mirrored across -arXiv, OpenReview, and the NeurIPS proceedings — and it fires regardless of band. +The dominant mechanism is **de-duplication**, canonical papers mirrored across +arXiv, OpenReview, and the NeurIPS proceedings, and it fires regardless of band. The off-scope rejection we set out to measure is real (on *self-speculative decoding* the verifier correctly dropped three general *speculative decoding* papers that use a *separate* draft model) but it is the minority, and it does not concentrate on the narrow topics as hypothesized: narrow mean Δ = 2.14 vs clean -3.00. The strong form of our hypothesis — narrow-in-broad pays more — is **refuted +3.00. The strong form of our hypothesis, narrow-in-broad pays more, is **refuted in magnitude, confirmed only in mechanism**. The practical reading: most of the win is "you fetched the same PDF three times," which a content hash catches for free. The LLM's distinctive contribution is the -page that *looks* on-topic but isn't — the self-speculative-vs-separate-draft +page that *looks* on-topic but isn't, the self-speculative-vs-separate-draft distinction a string match would miss. ### 4.2 The inference premium (and what "equal passes" hid) @@ -207,59 +207,59 @@ distinction a string match would miss. | tokens (in+out) | ~4,900 | ~530 | ~9× | | cost (USD) | ~$0.0072 | ~$0.0013 | ~5.5× | | latency (wall) | ~37 s | ~11 s | ~3.4× | -| cleanliness Δ (single − two admitted) | — | — | +1.56, 95% CI [0.33, 2.67] | +| cleanliness Δ (single − two admitted) | n/a | n/a | +1.56, 95% CI [0.33, 2.67] | The verifier buys ~1.5–2.7 fewer junk sources for roughly **5× the dollars, 9× the tokens, and 3× the latency**. Since the cleanliness gain is de-dup-dominated -(§4.1), the honest production move is a deterministic content-hash / canonical-URL +(§4.1), the production move is a deterministic content-hash / canonical-URL dedup, which captures most of the cleanliness at ~none of this premium, reserving an LLM check only for the off-scope tail. This is the cost half the "equal passes" -framing left out — and the rest of the paper is what we built once we saw it. +framing left out, and the rest of the paper is what we built once we saw it. -## 5. Result 3 — the two bands where an LLM verifier does, and doesn't, earn its dollar +## 5. Result 3: the two bands where an LLM verifier does, and doesn't, earn its dollar If de-dup is free and dominates the win, when is the LLM verifier worth its 5×? We found two bands, and they cut in opposite directions. -### 5.1 Misattributed citations — the cheap check beats the expensive judge +### 5.1 Misattributed citations: the cheap check beats the expensive judge `docs/results/claim-grounding.md`. A source can be on-topic, unique, and real, yet -the cited *claim* never appears in the page — the LLM wrote a plausible sentence and +the cited *claim* never appears in the page, the LLM wrote a plausible sentence and hung a real URL off it. De-dup passes it (unique). A relevance judge passes it (the -page is on-topic). Only checking the claim against the fetched text catches it — and +page is on-topic). Only checking the claim against the fetched text catches it, and that check is **deterministic text presence, $0 inference**. Each proposed source now carries the claim it is cited for (`withCitedClaim` → `metadata.citedClaim`). The claim-grounding verifier (`createClaimGroundingVerifier`, `src/claim-grounding.ts`) runs `groundClaimInText(claim, pageText)` over the -`htmlToText` output of the page the worker actually fetched — verbatim, normalized +`htmlToText` output of the page the worker actually fetched, verbatim, normalized (punctuation/whitespace-insensitive), or a ≥70% content-word overlap close paraphrase. A claim that isn't present is rejected as misattributed. The oracle is text presence, not a model call, so it composes with the LLM relevance verifier or runs alone at zero cost. -Live A/B (glm-5.2, real web fetch, one planted misattribution per topic — a real -fetched page plus a deliberately-wrong claim — over three verifier arms on the same +Live A/B (glm-5.2, real web fetch, one planted misattribution per topic, a real +fetched page plus a deliberately-wrong claim, over three verifier arms on the same proposals): | n=5 topics | misattributions caught | marginal $ | per-$ caught | |---|---|---|---| -| no-verifier | 0 / 5 | $0.0000 | — | +| no-verifier | 0 / 5 | $0.0000 | n/a | | relevance (LLM judge) | 4 / 5 | $0.0157 | 254 | | claim-grounding (text) | **5 / 5** | **$0.0000** | ∞ | -The relevance judge catches one only by accident — when the fabricated claim also +The relevance judge catches one only by accident, when the fabricated claim also makes the page read off-topic (a "12-billion-parameter draft transformer" claim on a rotary-embeddings page). When the fabrication stays on-topic (the KV-cache case), the judge waves it through, because the relevance verifier only ever sees the page text, -never the cited claim — it is **structurally blind** to misattribution. On this band +never the cited claim, it is **structurally blind** to misattribution. On this band the verifier-per-dollar comparison inverts §4.2: the cheap, deterministic check catches strictly more (5/5 vs 4/5) at strictly less ($0 vs $0.0157). The offline floor confirms the wiring: on a controlled 4-source pool (2 grounded, 2 misattributed), claim-grounding admits **0/2** misattributions and keeps **2/2** grounded, while relevance and no-verifier both admit **2/2**. -### 5.2 Adaptive topology — pay the LLM only on the ambiguous tail +### 5.2 Adaptive topology: pay the LLM only on the ambiguous tail `docs/results/adaptive.md`. The deployable shape §4.2 implied: do the free deterministic work first, reserve the LLM for what the cheap signals can't decide. @@ -275,7 +275,7 @@ decides: github, …) with a substantial body is **kept**; an obvious spam/listicle title or a too-thin body is **dropped**; everything else is **ambiguous**. 3. **LLM escalation ($).** Only ambiguous survivors reach the shipped LLM relevance - verifier — one call each. + verifier, one call each. Live frontier, n=5 topics, glm-5.2, same fetched proposals gated through all three drivers (plus one planted tracking-decorated mirror of the first source, so the @@ -291,12 +291,12 @@ dedup stage has a real duplicate to catch). Total spend $0.033: | **total** | **25** | **25** | **12 / 25 / $0.0261** | **20 / 6 / $0.0068** | Adaptive cuts LLM verifier calls **76%** (25 → 6) and dollars **74%** ($0.0261 → -$0.0068). On 3 of the 5 topics it spent **zero** LLM calls — every unique survivor +$0.0068). On 3 of the 5 topics it spent **zero** LLM calls, every unique survivor was on an authoritative host, so the $0 stages decided everything. It is a frontier point, not a free lunch. Admitted counts (lower = cleaner): single 25, **adaptive 20**, full-LLM 12. Adaptive removes the 5 real duplicates the $0 dedup -catches — exactly the de-dup-dominated win — but keeps the 8 sources full-LLM rejects +catches, exactly the de-dup-dominated win, but keeps the 8 sources full-LLM rejects on relevance, because on this authoritative-host-heavy set the heuristic resolved every non-duplicate survivor without ever asking the LLM, and the host prior is coarser than the relevance judge. So adaptive **recovers the deterministic de-dup @@ -304,26 +304,26 @@ half of full-LLM's cleanliness for ~26% of its dollars, and gives up the relevance-judgment half**. The escalation count is the diagnostic: on the 3 topics where it was zero, adaptive *is* a pure host/title/length rule and the LLM contributes nothing by construction; on the 2 topics with unknown-host survivors -(grouped-query attention, LoRA) it escalated 3 calls each — the off-scope tail the +(grouped-query attention, LoRA) it escalated 3 calls each, the off-scope tail the verifier is actually for. ## 6. Discussion The three results compose into one rule. The LLM verifier's headline cleanliness win is real (§3) but **de-dup-dominated** (§4.1) and **expensive** (§4.2, ~5×/9×/3×), so -spending an LLM call on every source is the wrong default — a free content hash buys +spending an LLM call on every source is the wrong default, a free content hash buys most of it. The verifier earns its 5× exactly where the cheap signals are blind: on **misattribution** (§5.1), where a $0 text-presence check beats the LLM judge outright because the judge never sees the claim; and on the **off-scope tail** (§5.2), where a page looks on-topic, is unique, and isn't fabricated, so only a relevance judgment can settle it. The deployable loop therefore stratifies by cost: free dedup, -free claim-grounding, free heuristic triage, then an LLM call only on what survives — +free claim-grounding, free heuristic triage, then an LLM call only on what survives: which is what the adaptive driver ships. Two cross-cutting lessons. First, **the accounting unit decides the verdict**: charging the verify step as one pass made the topology look near-free; pricing it per LLM call (§4.2) is what surfaced the 5× and motivated everything after it. Second, -**the same verifier inverts in value across bands** — on de-dup the LLM is expensive +**the same verifier inverts in value across bands**, on de-dup the LLM is expensive for what a hash does; on misattribution a deterministic check is free for what the LLM can't do; on the off-scope tail the LLM is the only thing that works. "Add a verifier" is not a setting; it is a cost-stratified decision per error type. @@ -344,7 +344,7 @@ is not a setting; it is a cost-stratified decision per error type. - **Planted error bands.** The misattributions (§5.1) and the adaptive duplicate (§5.2) are injected so the band is measurable. They model the real LLM citation-fabrication and mirror-host failures but do not measure their base rate in - the wild — that needs a hand-checked corpus of model-written citations. + the wild, that needs a hand-checked corpus of model-written citations. - **Adaptive's quality is host-prior-bound.** On an authoritative-host-heavy source pool the heuristic resolves everything and the LLM's relevance judgment contributes nothing; a richer worker (good sources on unknown hosts, junk on on-topic-looking @@ -356,20 +356,20 @@ is not a setting; it is a cost-stratified decision per error type. - **High web variance.** One live run per topic per result; numbers move with what search returns. -## 8. A simpler loop — built, not deferred +## 8. A simpler loop: built, not deferred The original write-up named two simplifications as future work. Both are now built and measured; this is what changed. -1. **Deterministic dedup before the LLM, LLM only on the tail — shipped.** The +1. **Deterministic dedup before the LLM, LLM only on the tail, shipped.** The adaptive driver (`src/adaptive-driver.ts`, §5.2) does exactly this: free canonical-URL / content-hash dedup, free host/title/length triage, LLM relevance only on the ambiguous survivors. Measured: **76% fewer LLM calls, 74% cheaper**, recovering the de-dup half of the verifier's cleanliness. The remaining gap to - full-LLM is the relevance-judgment half, kept honest in §5.2 — adaptive is a + full-LLM is the relevance-judgment half, measured separately in §5.2, adaptive is a frontier point you choose by how much a kept-but-marginal source costs you, not a strict improvement. -2. **A free check the LLM judge can't replicate — shipped.** Claim-grounding +2. **A free check the LLM judge can't replicate, shipped.** Claim-grounding (`src/claim-grounding.ts`, §5.1) adds the one verification an LLM relevance judge is structurally blind to: does the cited claim actually appear in the page? It catches 5/5 planted misattributions at **$0**, vs the judge's 4/5 at ~$0.003/topic. @@ -377,19 +377,19 @@ measured; this is what changed. What is still **not** built remains the worker: the live worker is a ~500-line hand-wired pipeline (query-gen, search, fetch, propose) against the router directly, where the runtime integration's pattern is to author an agent profile -and run it on a harness with a web-search tool — reusable and harness-agnostic. The +and run it on a harness with a web-search tool, reusable and harness-agnostic. The direct pipeline is cheaper to run today (no harness, no creds beyond the router) but it is the loop's main remaining piece of duplication, and the obvious next step if this loop graduates from experiment to production. -## 9. From hygiene to depth — a research-DRIVING loop, measured on held-out deep questions +## 9. From hygiene to depth: a research-DRIVING loop, measured on held-out deep questions -Everything above measures one thing: **source hygiene** — how *few* sources a verifier +Everything above measures one thing: **source hygiene**, how *few* sources a verifier admits at equal coverage. That is the right metric for a verifier whose only job is to filter, and it is why the win turned out to be de-duplication (§4.1): the most a filter-only verifier can do is reject. By construction it cannot make the knowledge base *answer more*; it can only make it *carry less*. So "the verifier mostly deduplicates" is -not a disappointing finding about *this* verifier — it is the ceiling of what *any* +not a disappointing finding about *this* verifier, it is the ceiling of what *any* admit-or-reject step can do. To ask whether a second agent can improve the research itself, you have to change both the agent and the metric. @@ -398,17 +398,17 @@ So we built the opposite agent and gave it the opposite metric. The **driving dr extracts each fetched source's claims, demands a second independent source for every claim, generates comparative / mechanism / contradiction sub-questions, and steers the worker to chase them in the next round. Its thesis is that *driving the research deeper* -— not pruning it — builds a knowledge base that answers harder questions. We measure it -not on admitted-count (which would score its whole point as a regression — it admits +- not pruning it, builds a knowledge base that answers harder questions. We measure it +not on admitted-count (which would score its whole point as a regression, it admits *more*) but on a **firewalled exam of 20 deep questions across 5 ML topics** (`tests/loops/held-out-exam.ts`), graded with a $0 deterministic substring grader the -loop never sees. The questions are depth questions by construction — the grader scores +loop never sees. The questions are depth questions by construction, the grader scores **0/20** on a one-line topic definition and **20/20** on a mechanism-rich paragraph, so a high score is reachable only by depth, not by grader slack. All three arms run the same real web worker and differ only in the driver: (A) plain collection, (B) verify/dedup, (C) driving. -**The honest verdict: driving does NOT reliably beat plain collection, and costs +**Result: driving does NOT reliably beat plain collection, and costs ~12–16× more.** The tell is that the winner flips with the compute budget, on the same exam: @@ -420,55 +420,55 @@ exam: Driving "wins" at B=4 (16 > 15 > 13) and "loses" at B=6 (13 < 15), while the single-agent arm itself swings 15→13 across the two budgets. At n=5 topics a ±1–3 -question difference is inside the run-to-run web variance — the **within-arm swing is as +question difference is inside the run-to-run web variance, the **within-arm swing is as large as the between-arm gap**, which is the signature of a null. What is stable and large is the cost: an order of magnitude more dollars and ~24× the tokens, for the claim-extraction LLM call driving runs on every fetched source. The autopsy explains it: every arm finished in **one effective round** -(`passes=2` on every topic at every budget). The generic readiness gate — "one source -closes the gap" — is met by the first fetch, so the loop stops *before* the driving +(`passes=2` on every topic at every budget). The generic readiness gate, "one source +closes the gap", is met by the first fetch, so the loop stops *before* the driving driver ever steers a second round, and its entire mechanism is multi-round. So we gave -it its fairest test: a controlled probe that *forces* three rounds so the driver -actually steers. It **still ties** a blind worker that just re-searches the same gaps — -**8/12 vs 8/12, at ~9× the cost**. Driving was better on RLHF (3 vs 1 — chasing +it a controlled test: a probe that *forces* three rounds so the driver +actually steers. It **still ties** a blind worker that just re-searches the same gaps: +**8/12 vs 8/12, at ~9× the cost**. Driving was better on RLHF (3 vs 1, chasing corroboration reached a page blind search missed) and worse on speculative decoding (2 -vs 4 — steering pulled the worker *off* the pages that answered the exam); the two +vs 4, steering pulled the worker *off* the pages that answered the exam); the two cancel. The null survives the fix, so it is not an artifact of a permissive gate. -The durable output is the apparatus, not the agent: a firewalled deep-question exam with +The durable output is the evaluation method, not the agent: a firewalled deep-question exam with a $0 grader that can tell depth from surface, reusable for any future research-quality claim. Full result, per-topic tables, and the probe: [`docs/results/research-driving.md`](results/research-driving.md). The two findings compose into one rule for "should I add a second research agent?": a **filter** agent measured on hygiene buys de-dup cleanliness you can get cheaper from a hash (§3–§4); a **driver** agent measured on depth buys nothing reliable over plain collection at this n and worker, for 9–16× the cost (§9). Neither earns a blanket "yes"; -both earn a narrow, cost-stratified one — the verifier on misattribution and the +both earn a narrow, cost-stratified one, the verifier on misattribution and the off-scope tail (§5), the driver only where a richer worker makes "go corroborate this" reach a page collection can't. -### 9.1 The domain was too easy — re-running the A/B where one search is not enough +### 9.1 The domain was too easy: re-running the A/B where one search is not enough The §9 null has a structural cause, not a measurement one: **on an ML topic a single good search already collects the answer.** Every arm finished in one effective round -because the first fetch met the readiness gate, so the driving driver — whose mechanism -is steering a *second* round — never acted. When one search suffices, there is no +because the first fetch met the readiness gate, so the driving driver, whose mechanism +is steering a *second* round, never acted. When one search suffices, there is no investigation for a smarter coordinator to do, and the metric can only reward collection. To ask whether topology *can ever* beat blind collection, you have to move to a domain where the answer is buried and a single fetch provably cannot surface it. -So we did. We ported the whole apparatus — firewalled checklist, `$0` model-free -grader, matched-compute 3-arm A/B — onto **investment research**: give a loop a company +So we did. We ported the full method, firewalled checklist, `$0` model-free +grader, matched-compute 3-arm A/B, onto **investment research**: give a loop a company + ticker + an as-of cutoff and grade the thesis on the buried, material 10-K-footnote facts a ticker search misses (an HTM mark the size of a bank's equity, a 97%-uninsured deposit base, a negative per-unit margin). First we *calibrated* the new metric and -proved it discriminates depth on this harder domain — a shallow ticker summary scores +proved it discriminates depth on this harder domain, a shallow ticker summary scores **1/27 (4%)**, a deep filings-grounded thesis **27/27 (100%)**, a **+96-point** gap. Then the live 3-arm A/B over 5 held-out companies: **driving surfaced the most buried -facts (16/27, 59%) vs blind collection (11/27, 41%)** for ~1.9× the cost — the lift is +facts (16/27, 59%) vs blind collection (11/27, 41%)** for ~1.9× the cost, the lift is real and points the right way, but at n=5 the paired-bootstrap CI still crosses zero (P(Δ≤0)=0.08), and verify did not beat collection (10/27). So the verdict survives the -domain change: **no topology *significantly* beats collection — but on a domain where +domain change: **no topology *significantly* beats collection, but on a domain where the answer must be investigated for, driving is the only arm that even leans positive, and it does so suggestively, not significantly.** Full reframe, calibration, and per-company A/B: [`docs/results/investment-thesis.md`](results/investment-thesis.md). @@ -484,7 +484,7 @@ multi-topic burn. git clone https://github.com/tangle-network/agent-knowledge cd agent-knowledge && pnpm install -# offline A/B — deterministic, no credentials (a controlled lower bound that +# offline A/B: deterministic, no credentials (a controlled lower bound that # exercises the same harness against a planted source pool) pnpm exec vitest run tests/loops/research-loop-equal-compute.test.ts @@ -492,29 +492,29 @@ pnpm exec vitest run tests/loops/research-loop-equal-compute.test.ts pnpm exec vitest run tests/loops/claim-grounding-ab.test.ts -t "offline" pnpm exec vitest run tests/loops/adaptive-ab.test.ts -# the live cleanliness sweep — real web search + a real glm-5.2 verifier, with +# the live cleanliness sweep: real web search + a real glm-5.2 verifier, with # per-arm cost reported (~$0.20 for 9 topics) export TANGLE_API_KEY= AGENT_KNOWLEDGE_LIVE=1 \ AGENT_KNOWLEDGE_LIVE_GOALS="self-speculative decoding|grouped-query attention|rotary position embeddings|KV-cache quantization|LoRA|ring attention|constitutional AI|the transformer architecture|gradient descent" \ pnpm exec vitest run tests/loops/research-loop-equal-compute.test.ts -# live misattribution band — three verifier arms over the same proposals +# live misattribution band: three verifier arms over the same proposals AGENT_KNOWLEDGE_LIVE=1 TANGLE_API_KEY=<…> \ CLAIM_GROUNDING_LIVE_GOALS='self-speculative decoding|rotary position embeddings|grouped-query attention|KV-cache quantization|LoRA' \ pnpm exec vitest run tests/loops/claim-grounding-ab.test.ts -t "three verifier arms" -# live adaptive frontier — single / full-LLM / adaptive on the same fetched proposals +# live adaptive frontier: single / full-LLM / adaptive on the same fetched proposals AGENT_KNOWLEDGE_LIVE=1 TANGLE_API_KEY=<…> \ ADAPTIVE_LIVE_GOALS="self-speculative decoding|rotary position embeddings|grouped-query attention|KV-cache quantization|LoRA fine-tuning" \ pnpm exec vitest run tests/loops/adaptive-ab.test.ts -t "three-topology" -# the research-DRIVING 3-arm A/B (§9) — collect / verify / drive, graded on the +# the research-DRIVING 3-arm A/B (§9): collect / verify / drive, graded on the # held-out deep-question exam; re-run at RQ_LIVE_BUDGET=6 to see the verdict flip AGENT_KNOWLEDGE_LIVE=1 RQ_LIVE_BUDGET=4 TANGLE_API_KEY=<…> \ pnpm exec vitest run tests/loops/research-driving-ab.test.ts -t "3-arm A/B" -# the controlled multi-round probe — forces 3 rounds so the driver actually steers +# the controlled multi-round probe: forces 3 rounds so the driver actually steers AGENT_KNOWLEDGE_LIVE=1 RQ_PROBE=1 RQ_PROBE_ROUNDS=3 TANGLE_API_KEY=<…> \ pnpm exec vitest run tests/loops/research-driving-ab.test.ts -t "multi-round probe" ``` @@ -523,17 +523,17 @@ AGENT_KNOWLEDGE_LIVE=1 RQ_PROBE=1 RQ_PROBE_ROUNDS=3 TANGLE_API_KEY=<…> \ topic list; the live arms run the loops on each at equal compute and report the paired bootstrap and per-arm cost. -**Source:** the loop — [`src/two-agent-research-loop.ts`](../src/two-agent-research-loop.ts); -the live worker + verifier + cost instrumentation — [`src/web-research-worker.ts`](../src/web-research-worker.ts); -the misattribution check — [`src/claim-grounding.ts`](../src/claim-grounding.ts); -the adaptive driver — [`src/adaptive-driver.ts`](../src/adaptive-driver.ts); -the driving driver — [`src/research-driving-driver.ts`](../src/research-driving-driver.ts); -the held-out exam + $0 grader — [`tests/loops/held-out-exam.ts`](../tests/loops/held-out-exam.ts); -the A/B harnesses — [`tests/loops/`](../tests/loops/). +**Source:** the loop, [`src/two-agent-research-loop.ts`](../src/two-agent-research-loop.ts); +the live worker + verifier + cost instrumentation, [`src/web-research-worker.ts`](../src/web-research-worker.ts); +the misattribution check, [`src/claim-grounding.ts`](../src/claim-grounding.ts); +the adaptive driver, [`src/adaptive-driver.ts`](../src/adaptive-driver.ts); +the driving driver, [`src/research-driving-driver.ts`](../src/research-driving-driver.ts); +the held-out exam + $0 grader, [`tests/loops/held-out-exam.ts`](../tests/loops/held-out-exam.ts); +the A/B harnesses, [`tests/loops/`](../tests/loops/). Per-result detail: [`docs/results/cost-quality.md`](results/cost-quality.md), [`docs/results/claim-grounding.md`](results/claim-grounding.md), [`docs/results/adaptive.md`](results/adaptive.md), [`docs/results/research-driving.md`](results/research-driving.md), -[`docs/results/investment-thesis.md`](results/investment-thesis.md) (§9.1 — the domain reframe + calibration + 3-arm A/B). +[`docs/results/investment-thesis.md`](results/investment-thesis.md) (§9.1, the domain reframe + calibration + 3-arm A/B). diff --git a/package.json b/package.json index c4c2bc0..3ffe774 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@tangle-network/agent-knowledge", - "version": "4.0.1", + "version": "4.1.0", "description": "Source-grounded, eval-gated knowledge growth primitives for agents.", "homepage": "https://github.com/tangle-network/agent-knowledge#readme", "repository": { @@ -78,21 +78,18 @@ "devDependencies": { "@biomejs/biome": "^2.5.4", "@neo4j-labs/agent-memory": "0.4.0", - "@tangle-network/sandbox": "^0.9.7", - "@types/node": "^25.6.0", + "@types/node": "^26.1.1", "@types/proper-lockfile": "4.1.4", "mem0ai": "3.1.0", "tsup": "^8.0.0", - "tsx": "^4.23.1", "typescript": "^5.7.0", - "vitest": "^3.0.0" + "vitest": "^4.1.10" }, "pnpm": { "minimumReleaseAge": 4320, "minimumReleaseAgeExclude": [ "@tangle-network/agent-eval", - "@tangle-network/agent-interface", - "@tangle-network/sandbox" + "@tangle-network/agent-interface" ], "overrides": { "hono": "4.12.30", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 78194ea..440f089 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -31,12 +31,9 @@ importers: '@neo4j-labs/agent-memory': specifier: 0.4.0 version: 0.4.0 - '@tangle-network/sandbox': - specifier: ^0.9.7 - version: 0.9.7(viem@2.48.8(typescript@5.9.3)(zod@4.4.3)) '@types/node': - specifier: ^25.6.0 - version: 25.6.0 + specifier: ^26.1.1 + version: 26.1.1 '@types/proper-lockfile': specifier: 4.1.4 version: 4.1.4 @@ -45,16 +42,13 @@ importers: version: 3.1.0(@anthropic-ai/sdk@0.40.1)(@azure/identity@4.13.1)(@azure/search-documents@12.2.0)(@cloudflare/workers-types@4.20260702.1)(@google/genai@1.52.0)(@langchain/core@1.2.3(@opentelemetry/api@1.9.1)(ws@8.21.0))(@mistralai/mistralai@1.15.1)(@qdrant/js-client-rest@1.18.0(typescript@5.9.3))(@supabase/supabase-js@2.110.7)(@types/jest@29.5.14)(@types/pg@8.11.0)(better-sqlite3@12.11.1)(cloudflare@4.5.0)(compromise@14.16.0)(groq-sdk@0.3.0)(mongodb@7.2.0)(natural@8.1.1(@opentelemetry/api@1.9.1))(ollama@0.5.18)(pg@8.22.0)(redis@5.12.1(@opentelemetry/api@1.9.1))(ws@8.21.0) tsup: specifier: ^8.0.0 - version: 8.5.1(postcss@8.5.13)(tsx@4.23.1)(typescript@5.9.3)(yaml@2.8.4) - tsx: - specifier: ^4.23.1 - version: 4.23.1 + version: 8.5.1(postcss@8.5.19)(tsx@4.23.1)(typescript@5.9.3)(yaml@2.8.4) typescript: specifier: ^5.7.0 version: 5.9.3 vitest: - specifier: ^3.0.0 - version: 3.2.4(@types/node@25.6.0)(tsx@4.23.1)(yaml@2.8.4) + specifier: ^4.1.10 + version: 4.1.10(@opentelemetry/api@1.9.1)(@types/node@26.1.1)(vite@7.3.2(@types/node@26.1.1)(tsx@4.23.1)(yaml@2.8.4)) packages: @@ -672,126 +666,251 @@ packages: cpu: [arm] os: [android] + '@rollup/rollup-android-arm-eabi@4.62.2': + resolution: {integrity: sha512-6o7ZLZK+BeenkZCFNDXqpbjw9bD6nuWonvS/lwQJp7NoVVxm6p3qE7qQ5jGuBjiFsgvqjD8mZAU5oWxTmbOeOg==} + cpu: [arm] + os: [android] + '@rollup/rollup-android-arm64@4.60.2': resolution: {integrity: sha512-OqZTwDRDchGRHHm/hwLOL7uVPB9aUvI0am/eQuWMNyFHf5PSEQmyEeYYheA0EPPKUO/l0uigCp+iaTjoLjVoHg==} cpu: [arm64] os: [android] + '@rollup/rollup-android-arm64@4.62.2': + resolution: {integrity: sha512-BaH7BllCACHoH1LguOU56UItGfUWjujlO65kS9LAodViaN4bwIKd7oeW/ZHJ/4ljr/7MIiENnNy3HJ0zXv8Zkw==} + cpu: [arm64] + os: [android] + '@rollup/rollup-darwin-arm64@4.60.2': resolution: {integrity: sha512-UwRE7CGpvSVEQS8gUMBe1uADWjNnVgP3Iusyda1nSRwNDCsRjnGc7w6El6WLQsXmZTbLZx9cecegumcitNfpmA==} cpu: [arm64] os: [darwin] + '@rollup/rollup-darwin-arm64@4.62.2': + resolution: {integrity: sha512-v39RCCvj4He82I9sFmk+M1VZ0PLM9sfsLVikjfx2hYBNALhrrOR2D3JjQA6AhlaSOgcR+RzrKY7e1+bT6SUO/A==} + cpu: [arm64] + os: [darwin] + '@rollup/rollup-darwin-x64@4.60.2': resolution: {integrity: sha512-gjEtURKLCC5VXm1I+2i1u9OhxFsKAQJKTVB8WvDAHF+oZlq0GTVFOlTlO1q3AlCTE/DF32c16ESvfgqR7343/g==} cpu: [x64] os: [darwin] + '@rollup/rollup-darwin-x64@4.62.2': + resolution: {integrity: sha512-yl0y2vq3S3lHeuXhEdss6TWfKW8vkujImO12tn4ZkG/4oghr09LvdYm2RElVjokTQiUvDUGXLGsYeLqUMCKpGA==} + cpu: [x64] + os: [darwin] + '@rollup/rollup-freebsd-arm64@4.60.2': resolution: {integrity: sha512-Bcl6CYDeAgE70cqZaMojOi/eK63h5Me97ZqAQoh77VPjMysA/4ORQBRGo3rRy45x4MzVlU9uZxs8Uwy7ZaKnBw==} cpu: [arm64] os: [freebsd] + '@rollup/rollup-freebsd-arm64@4.62.2': + resolution: {integrity: sha512-tT4pvt4qXD+vEoezupCWi+a1F0vvDiksiHc+PxRlYTOH1I6/X4id9jPxTP+Fg+545euaFT1jJVs4CEdHZAU1vw==} + cpu: [arm64] + os: [freebsd] + '@rollup/rollup-freebsd-x64@4.60.2': resolution: {integrity: sha512-LU+TPda3mAE2QB0/Hp5VyeKJivpC6+tlOXd1VMoXV/YFMvk/MNk5iXeBfB4MQGRWyOYVJ01625vjkr0Az98OJQ==} cpu: [x64] os: [freebsd] + '@rollup/rollup-freebsd-x64@4.62.2': + resolution: {integrity: sha512-6nU5F2wCW+qvCBhTn1pdIU3bzsIoF7EUwsCDRxilWGprQR6yd508YnH9+OKFCwpfS8pjZqDUmnCAr7exax0XCg==} + cpu: [x64] + os: [freebsd] + '@rollup/rollup-linux-arm-gnueabihf@4.60.2': resolution: {integrity: sha512-2QxQrM+KQ7DAW4o22j+XZ6RKdxjLD7BOWTP0Bv0tmjdyhXSsr2Ul1oJDQqh9Zf5qOwTuTc7Ek83mOFaKnodPjg==} cpu: [arm] os: [linux] + '@rollup/rollup-linux-arm-gnueabihf@4.62.2': + resolution: {integrity: sha512-n1GJHPOvpIfhi3TmrCeh6S6URt9BFCt0KQE3qvexyGCTAKpR4Lg+eWvNZEqu7epxwus/8ElT3hacYEucm49SZg==} + cpu: [arm] + os: [linux] + '@rollup/rollup-linux-arm-musleabihf@4.60.2': resolution: {integrity: sha512-TbziEu2DVsTEOPif2mKWkMeDMLoYjx95oESa9fkQQK7r/Orta0gnkcDpzwufEcAO2BLBsD7mZkXGFqEdMRRwfw==} cpu: [arm] os: [linux] + '@rollup/rollup-linux-arm-musleabihf@4.62.2': + resolution: {integrity: sha512-JqgflS8wEB+UXV/vS1RpRbifGBeN4D5lz8D8oOFbFZw4vedvdOgCFAjfBmIMdW3yL10XpQQ0Ambepw6MXrhOnA==} + cpu: [arm] + os: [linux] + '@rollup/rollup-linux-arm64-gnu@4.60.2': resolution: {integrity: sha512-bO/rVDiDUuM2YfuCUwZ1t1cP+/yqjqz+Xf2VtkdppefuOFS2OSeAfgafaHNkFn0t02hEyXngZkxtGqXcXwO8Rg==} cpu: [arm64] os: [linux] + '@rollup/rollup-linux-arm64-gnu@4.62.2': + resolution: {integrity: sha512-wnFJkogWvN4jm/hQRF2UBaeUmk20j5+DmHvoyWii2b8HJDyvz1MF2OU/6ynXt2KR63rbZLWkFpoytpdc/yBuSA==} + cpu: [arm64] + os: [linux] + '@rollup/rollup-linux-arm64-musl@4.60.2': resolution: {integrity: sha512-hr26p7e93Rl0Za+JwW7EAnwAvKkehh12BU1Llm9Ykiibg4uIr2rbpxG9WCf56GuvidlTG9KiiQT/TXT1yAWxTA==} cpu: [arm64] os: [linux] + '@rollup/rollup-linux-arm64-musl@4.62.2': + resolution: {integrity: sha512-HVu2bp0zhvJ8xHEV9+UUs7S90VadmBSY3LcIMvozbPo4AuMGDWlz3ymHLHZPX4hR67TKTt8Qp5PJ5RBg/i+RMQ==} + cpu: [arm64] + os: [linux] + '@rollup/rollup-linux-loong64-gnu@4.60.2': resolution: {integrity: sha512-pOjB/uSIyDt+ow3k/RcLvUAOGpysT2phDn7TTUB3n75SlIgZzM6NKAqlErPhoFU+npgY3/n+2HYIQVbF70P9/A==} cpu: [loong64] os: [linux] + '@rollup/rollup-linux-loong64-gnu@4.62.2': + resolution: {integrity: sha512-mQqqAV8QaoSgr9I2fKDLY2BAVvmKjWoGiu/cSYQonsLvtqwEn1E4QYfnCOcp5zoEqNhsDYin1s6jx/VJmrxlZg==} + cpu: [loong64] + os: [linux] + '@rollup/rollup-linux-loong64-musl@4.60.2': resolution: {integrity: sha512-2/w+q8jszv9Ww1c+6uJT3OwqhdmGP2/4T17cu8WuwyUuuaCDDJ2ojdyYwZzCxx0GcsZBhzi3HmH+J5pZNXnd+Q==} cpu: [loong64] os: [linux] + '@rollup/rollup-linux-loong64-musl@4.62.2': + resolution: {integrity: sha512-IxKLoxCQ2IWi6bT2akyDUBGsOImDKB+sPp4EsTmwFQ/fMwpCKm8uLSSgP/Kx/QYUgKis6SEZ5/Nlhup0DIA0PQ==} + cpu: [loong64] + os: [linux] + '@rollup/rollup-linux-ppc64-gnu@4.60.2': resolution: {integrity: sha512-11+aL5vKheYgczxtPVVRhdptAM2H7fcDR5Gw4/bTcteuZBlH4oP9f5s9zYO9aGZvoGeBpqXI/9TZZihZ609wKw==} cpu: [ppc64] os: [linux] + '@rollup/rollup-linux-ppc64-gnu@4.62.2': + resolution: {integrity: sha512-Mk5ha2RQSgyFfmYYLkBpPnUk8D8FriBxesO1u9O75X0mHgXL1UQcH5Itl2lurWL2tj0RxV9b9tJgipac0hRY9A==} + cpu: [ppc64] + os: [linux] + '@rollup/rollup-linux-ppc64-musl@4.60.2': resolution: {integrity: sha512-i16fokAGK46IVZuV8LIIwMdtqhin9hfYkCh8pf8iC3QU3LpwL+1FSFGej+O7l3E/AoknL6Dclh2oTdnRMpTzFQ==} cpu: [ppc64] os: [linux] + '@rollup/rollup-linux-ppc64-musl@4.62.2': + resolution: {integrity: sha512-CjvEnqJL/0/TQ3TXX3OPIJ/kmBellrWd4heXUmHeJlTnmwjKpSJzoehLaL6Xk0ZnMHBu9dZuFADNOrtjF4v+2w==} + cpu: [ppc64] + os: [linux] + '@rollup/rollup-linux-riscv64-gnu@4.60.2': resolution: {integrity: sha512-49FkKS6RGQoriDSK/6E2GkAsAuU5kETFCh7pG4yD/ylj9rKhTmO3elsnmBvRD4PgJPds5W2PkhC82aVwmUcJ7A==} cpu: [riscv64] os: [linux] + '@rollup/rollup-linux-riscv64-gnu@4.62.2': + resolution: {integrity: sha512-1SiZbzwdkaDURsew/tSOrooKiYy7EQGT6m8ufavAi9NEyQb/6VuIxFXAL1fqa4iZe3g4NbNk4P7J32z2tw5Mgg==} + cpu: [riscv64] + os: [linux] + '@rollup/rollup-linux-riscv64-musl@4.60.2': resolution: {integrity: sha512-mjYNkHPfGpUR00DuM1ZZIgs64Hpf4bWcz9Z41+4Q+pgDx73UwWdAYyf6EG/lRFldmdHHzgrYyge5akFUW0D3mQ==} cpu: [riscv64] os: [linux] + '@rollup/rollup-linux-riscv64-musl@4.62.2': + resolution: {integrity: sha512-nQts12zJ3NQRoE6uYljOH89v7szzLDvG2JD/vsX+vGXU8w/At1GowTZ5/7qeFQ8m7L55rpR8Okugnuo5bgjy2Q==} + cpu: [riscv64] + os: [linux] + '@rollup/rollup-linux-s390x-gnu@4.60.2': resolution: {integrity: sha512-ALyvJz965BQk8E9Al/JDKKDLH2kfKFLTGMlgkAbbYtZuJt9LU8DW3ZoDMCtQpXAltZxwBHevXz5u+gf0yA0YoA==} cpu: [s390x] os: [linux] + '@rollup/rollup-linux-s390x-gnu@4.62.2': + resolution: {integrity: sha512-E9/ll019jhPIJgpzfZoIkBGhcz+kKNgVWYRY0zr9srBdPPFVpvOKW8VaJKUbeK+eZXyQF9ltME+Kk6affeaPgg==} + cpu: [s390x] + os: [linux] + '@rollup/rollup-linux-x64-gnu@4.60.2': resolution: {integrity: sha512-UQjrkIdWrKI626Du8lCQ6MJp/6V1LAo2bOK9OTu4mSn8GGXIkPXk/Vsp4bLHCd9Z9Iz2OTEaokUE90VweJgIYQ==} cpu: [x64] os: [linux] + '@rollup/rollup-linux-x64-gnu@4.62.2': + resolution: {integrity: sha512-5BqxR/pshjey51iliyzTD5Xi3EN0aLmQ2lZ3lvefVV9c82BvrLo2/6OT55iifpWBufs6kdwWbuOKS841DrmK9A==} + cpu: [x64] + os: [linux] + '@rollup/rollup-linux-x64-musl@4.60.2': resolution: {integrity: sha512-bTsRGj6VlSdn/XD4CGyzMnzaBs9bsRxy79eTqTCBsA8TMIEky7qg48aPkvJvFe1HyzQ5oMZdg7AnVlWQSKLTnw==} cpu: [x64] os: [linux] + '@rollup/rollup-linux-x64-musl@4.62.2': + resolution: {integrity: sha512-uNN83XxQrRAh/w0/pmAfibcwyb6YWt4gP+dpnQKPVJshAloQ785ii8CT8ZCIxkGg9opVsvAlGhFitSm6D1Jjpg==} + cpu: [x64] + os: [linux] + '@rollup/rollup-openbsd-x64@4.60.2': resolution: {integrity: sha512-6d4Z3534xitaA1FcMWP7mQPq5zGwBmGbhphh2DwaA1aNIXUu3KTOfwrWpbwI4/Gr0uANo7NTtaykFyO2hPuFLg==} cpu: [x64] os: [openbsd] + '@rollup/rollup-openbsd-x64@4.62.2': + resolution: {integrity: sha512-srjEIxSH3LRnJN6THczDHWQplqEMFiAJrTab0msUryh9kwNpkICf3Ea6q6MN/2cZwRFUNx5w+h6Hpi4QuHS6Zg==} + cpu: [x64] + os: [openbsd] + '@rollup/rollup-openharmony-arm64@4.60.2': resolution: {integrity: sha512-NetAg5iO2uN7eB8zE5qrZ3CSil+7IJt4WDFLcC75Ymywq1VZVD6qJ6EvNLjZ3rEm6gB7XW5JdT60c6MN35Z85Q==} cpu: [arm64] os: [openharmony] + '@rollup/rollup-openharmony-arm64@4.62.2': + resolution: {integrity: sha512-8hOJnxgbyObnCm5AlRA3A931xX19xq80RjVTKgJOvEKWqJruP/Uf12IbAOaDjjEXYRewwHLfmF0YRIdK3OwKWA==} + cpu: [arm64] + os: [openharmony] + '@rollup/rollup-win32-arm64-msvc@4.60.2': resolution: {integrity: sha512-NCYhOotpgWZ5kdxCZsv6Iudx0wX8980Q/oW4pNFNihpBKsDbEA1zpkfxJGC0yugsUuyDZ7gL37dbzwhR0VI7pQ==} cpu: [arm64] os: [win32] + '@rollup/rollup-win32-arm64-msvc@4.62.2': + resolution: {integrity: sha512-mmF4AY1i0hG/bLWUctUq59gtmgaSIRa3cu/A3JFRp/sCNEme2bgDEiDS22P9FbnJB8NJNF4jPJiSP5RHQpUTDg==} + cpu: [arm64] + os: [win32] + '@rollup/rollup-win32-ia32-msvc@4.60.2': resolution: {integrity: sha512-RXsaOqXxfoUBQoOgvmmijVxJnW2IGB0eoMO7F8FAjaj0UTywUO/luSqimWBJn04WNgUkeNhh7fs7pESXajWmkg==} cpu: [ia32] os: [win32] + '@rollup/rollup-win32-ia32-msvc@4.62.2': + resolution: {integrity: sha512-DZgkknc6jhHrk46V25vbAM0zZkyP0nSDkJB8/dRkLTxv470dOmWDqGoEJl/9A0dFfS7yE3REOwNDxpHwSLSt0Q==} + cpu: [ia32] + os: [win32] + '@rollup/rollup-win32-x64-gnu@4.60.2': resolution: {integrity: sha512-qdAzEULD+/hzObedtmV6iBpdL5TIbKVztGiK7O3/KYSf+HIzU257+MX1EXJcyIiDbMAqmbwaufcYPvyRryeZtA==} cpu: [x64] os: [win32] + '@rollup/rollup-win32-x64-gnu@4.62.2': + resolution: {integrity: sha512-T6xr6ucWSFto+VGajA8YH26LdpHRuP4YLHEKAtCWvJDOlnmWcDZVCI2Jmjr+IFHDlt2zRaTAKE4tfjTaWLgJBg==} + cpu: [x64] + os: [win32] + '@rollup/rollup-win32-x64-msvc@4.60.2': resolution: {integrity: sha512-Nd/SgG27WoA9e+/TdK74KnHz852TLa94ovOYySo/yMPuTmpckK/jIF2jSwS3g7ELSKXK13/cVdmg1Z/DaCWKxA==} cpu: [x64] os: [win32] + '@rollup/rollup-win32-x64-msvc@4.62.2': + resolution: {integrity: sha512-BfzEnDJOt9T8M989/lA37EcJgat01wLRnoi5dQf3QzOH7jzpqTAzdDbVfRljVr5r+jzKqpbHeyOfAaXxAd0PAA==} + cpu: [x64] + os: [win32] + '@scure/base@1.2.6': resolution: {integrity: sha512-g/nm5FgUa//MCj1gV09zTJTaM6KBAHqLN907YVQqf7zC49+DcO4B1so4ZX07Ef10Twr6nuqYEH9GEggFXA4Fmg==} @@ -904,6 +1023,9 @@ packages: '@types/estree@1.0.8': resolution: {integrity: sha512-dWHzHa2WqEXI/O1E9OjrocMTKJl2mSrEolh1Iomrv6U+JuNwaHXsXx9bLu5gG7BUWFIN0skIQJQ/L1rIex4X6w==} + '@types/estree@1.0.9': + resolution: {integrity: sha512-GhdPgy1el4/ImP05X05Uw4cw2/M93BCUmnEvWZNStlCzEKME4Fkk+YpoA5OiHNQmoS7Cafb8Xa3Pya8m1Qrzeg==} + '@types/istanbul-lib-coverage@2.0.6': resolution: {integrity: sha512-2QF/t/auWm0lsy8XtKVPG19v3sSOQlJe/YHZgfjb/KBBHOGSV+J2q/S671rcq9uTBrLAXmZpqJiaQbMT+zNU1w==} @@ -922,11 +1044,8 @@ packages: '@types/node@18.19.130': resolution: {integrity: sha512-GRaXQx6jGfL8sKfaIDD6OupbIHBr9jv7Jnaml9tB7l4v068PAOXqfcujMMo5PhbIs6ggR1XODELqahT2R8v0fg==} - '@types/node@25.6.0': - resolution: {integrity: sha512-+qIYRKdNYJwY3vRCZMdJbPLJAtGjQBudzZzdzwQYkEPQd+PJGixUL5QfvCLDaULoLv+RhT3LDkwEfKaAkgSmNQ==} - - '@types/node@25.9.5': - resolution: {integrity: sha512-OScDchr2fwuUmWdf4kZ9h7PcJiYDVInhJizG/biAq3cAvqwYktuy/TYGGdZNMtNTFUP7rnb0NU4TUdm82kt4Rg==} + '@types/node@26.1.1': + resolution: {integrity: sha512-nxAkRSVkN1Y0JC1W8ky/fTfkGsMmcrRsbx+3XoZE+rMOX71kLYTV7fLXpqud1GpbpP5TuffXFqfX7fH2GgZREw==} '@types/pg@8.11.0': resolution: {integrity: sha512-sDAlRiBNthGjNFfvt0k6mtotoVYVQ63pA8R4EMWka7crawSR60waVYR0HAgmPRs/e2YaeJTD/43OoZ3PFw80pw==} @@ -959,34 +1078,34 @@ packages: resolution: {integrity: sha512-JVUD8X2tfDMWjcjLs4yVxxVrS8yR5vnh386GAXT9Qj79nBxxXSaHFQZg5FweLmT8HlPQ3kii6noUB+Z9RN7DvQ==} engines: {node: '>=22.0.0'} - '@vitest/expect@3.2.4': - resolution: {integrity: sha512-Io0yyORnB6sikFlt8QW5K7slY4OjqNX9jmJQ02QDda8lyM6B5oNgVWoSoKPac8/kgnCUzuHQKrSLtu/uOqqrig==} + '@vitest/expect@4.1.10': + resolution: {integrity: sha512-YsCn+qAk1GWjQOWFEsEcL2gNQ0zmVmQu3T03qP6UyjhtmdtwtbuI+DASn/7iQB3HGTXkdBwGddzxPlmiql5vlA==} - '@vitest/mocker@3.2.4': - resolution: {integrity: sha512-46ryTE9RZO/rfDd7pEqFl7etuyzekzEhUbTW3BvmeO/BcCMEgq59BKhek3dXDWgAj4oMK6OZi+vRr1wPW6qjEQ==} + '@vitest/mocker@4.1.10': + resolution: {integrity: sha512-v0xaezt+DKEmKfaxg133ldzADrwLGd7Ze1MfQQTYfvs8OqZIwbxyxaYURivwV7sWy5fqn3rH5uOrSp07bp44Ow==} peerDependencies: msw: ^2.4.9 - vite: ^5.0.0 || ^6.0.0 || ^7.0.0-0 + vite: ^6.0.0 || ^7.0.0 || ^8.0.0 peerDependenciesMeta: msw: optional: true vite: optional: true - '@vitest/pretty-format@3.2.4': - resolution: {integrity: sha512-IVNZik8IVRJRTr9fxlitMKeJeXFFFN0JaB9PHPGQ8NKQbGpfjlTx9zO4RefN8gp7eqjNy8nyK3NZmBzOPeIxtA==} + '@vitest/pretty-format@4.1.10': + resolution: {integrity: sha512-W1HsjSH4MXQ9YfmmhLAoIYf1HRfekQCGngeIgcei6MP5QQGWUe0gkopdZQaVCFO+JDJMrAJGwa5pRpNpvy4P8Q==} - '@vitest/runner@3.2.4': - resolution: {integrity: sha512-oukfKT9Mk41LreEW09vt45f8wx7DordoWUZMYdY/cyAk7w5TWkTRCNZYF7sX7n2wB7jyGAl74OxgwhPgKaqDMQ==} + '@vitest/runner@4.1.10': + resolution: {integrity: sha512-IKI6kpIH+LmpROplyLwBBaCfMgOZOMsygVa6BARD6ahA04VRuJSa6OaVG7kRvSEMD870Vd91rSSw0eegtWyLGg==} - '@vitest/snapshot@3.2.4': - resolution: {integrity: sha512-dEYtS7qQP2CjU27QBC5oUOxLE/v5eLkGqPE0ZKEIDGMs4vKWe7IjgLOeauHsR0D5YuuycGRO5oSRXnwnmA78fQ==} + '@vitest/snapshot@4.1.10': + resolution: {integrity: sha512-xRkfOT1qpTAi/Ti4Y1LtfRc3kEuqxGw59eN2jN9pRWMtS/XDevekhcFSqvQqjUNGksfjMJu3Y+oJ+4Ypn2OaJw==} - '@vitest/spy@3.2.4': - resolution: {integrity: sha512-vAfasCOe6AIK70iP5UD11Ac4siNUNJ9i/9PZ3NKx07sG6sUxeag1LWdNrMWeKKYBLlzuK+Gn65Yd5nyL6ds+nw==} + '@vitest/spy@4.1.10': + resolution: {integrity: sha512-PLf/Ugvoq5wO/b4rwYCR1h2PSIdXz7wnkQFMiUpLdtM7l6pqVFcQIBEHyT1+l+cj7mNwAfZHzqXqDyjvOuwbDw==} - '@vitest/utils@3.2.4': - resolution: {integrity: sha512-fB2V0JFrQSMsCo9HiSq3Ezpdv4iYaXRG1Sx8edX3MwxfyNn83mKiGzOcH+Fkxt4MHxr3y42fQi1oeAInqgX2QA==} + '@vitest/utils@4.1.10': + resolution: {integrity: sha512-fy9am/HWxbaGt/Sawrp90vt6Y6jQwf1RX77cz3uwoJwJVMli/e1IEwRPnMNJ7vKfPTwo0diXifkpPvwH9v7nGA==} abitype@1.2.3: resolution: {integrity: sha512-Ofer5QUnuUdTFsBRwARMoWKOH1ND5ehwYhJ3OJ/BQO+StkwQjHw0XyVh4vDttzHB7QOFhPHa/o413PJ82gU/Tg==} @@ -1102,8 +1221,8 @@ packages: resolution: {integrity: sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==} engines: {node: '>= 0.4'} - chai@5.3.3: - resolution: {integrity: sha512-4zNhdJD/iOjSH0A05ea+Ke6MU5mmpQcbQsSOkgdaUMJ9zTlDTD/GYlwohmIE2u0gaxHYiVHEn1Fw9mZ/ktJWgw==} + chai@6.2.2: + resolution: {integrity: sha512-NUPRluOfOiTKBKvWPtSD4PhFvWCqOi0BGStNWs57X9js7XGTprSmFoz5F0tWhR4WPjNeR9jXqdC7/UpSJTnlRg==} engines: {node: '>=18'} chalk@4.1.2: @@ -1113,10 +1232,6 @@ packages: charenc@0.0.2: resolution: {integrity: sha512-yrLQ/yVUFXkzg7EDQsPieE/53+0RlaWTs+wBrvW36cyilJ2SaDWfl4Yj7MtLTXleV9uEKefbAGUPv2/iWSooRA==} - check-error@2.1.3: - resolution: {integrity: sha512-PAJdDJusoxnwm1VwW07VWwUN1sl7smmC3OKggvndJFadxxDRyFJBX/ggnu/KE4kQAB7a3Dp8f/YXC1FlUprWmA==} - engines: {node: '>= 16'} - chokidar@4.0.3: resolution: {integrity: sha512-Qgzu8kfBvo+cA4962jnP1KkS6Dop5NS6g7R5LFYJr4b8Ub94PPQXUksCw9PvXoeXPRRddRNC5C1JQUR2SMGtnA==} engines: {node: '>= 14.16.0'} @@ -1165,6 +1280,9 @@ packages: resolution: {integrity: sha512-5IKcdX0nnYavi6G7TtOhwkYzyjfJlatbjMjuLSfE2kYT5pMDOilZ4OvMhi637CcDICTmz3wARPoyhqyX1Y+XvA==} engines: {node: ^14.18.0 || >=16.10.0} + convert-source-map@2.0.0: + resolution: {integrity: sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg==} + crypt@0.0.2: resolution: {integrity: sha512-mCxBlsHFYh9C+HVpiEacem8FEBnMXgU9gy4zmNC+SXAZNB/1idgp/aulFJ4FgCi7GPEVbfyng092GqL2k2rmow==} @@ -1185,10 +1303,6 @@ packages: resolution: {integrity: sha512-aW35yZM6Bb/4oJlZncMH2LCoZtJXTRxES17vE3hoRiowU2kWHaJKFkSBDnDR+cm9J+9QhXmREyIfv0pji9ejCQ==} engines: {node: '>=10'} - deep-eql@5.0.2: - resolution: {integrity: sha512-h5k/5U50IJJFpzfL6nO9jaaumfjO/f2NjK/oYB2Djzm4p9L+3T9qWpZqZ2hAbLPuuYq9wrU08WQyBTL5GbPk5Q==} - engines: {node: '>=6'} - deep-extend@0.6.0: resolution: {integrity: sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==} engines: {node: '>=4.0.0'} @@ -1246,8 +1360,8 @@ packages: resolution: {integrity: sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==} engines: {node: '>= 0.4'} - es-module-lexer@1.7.0: - resolution: {integrity: sha512-jEQoCwk8hyb2AZziIOLhDqpm5+2ww5uIE6lkO/6jcOCusfk6LhMHpXXfBLXTZ7Ydyt0j4VoUQv6uGNYbdW+kBA==} + es-module-lexer@2.3.1: + resolution: {integrity: sha512-shc1dbU90Yl/xq1QrC7QRtfcwURZuVRfPhZbDoldJ1cn1gzDvBaBWlv0eFolj5+0znnPJz5TXLxsN77X/12KTA==} es-object-atoms@1.1.2: resolution: {integrity: sha512-HWcBoN6NileqtSydK2FqHbS/LoDd2pqrnQHLyJzBj4kOp/ky2MWMN694xOfkK8/SnUsW2DH7EfyVlydKCsm1Zw==} @@ -1292,8 +1406,8 @@ packages: resolution: {integrity: sha512-XYfuKMvj4O35f/pOXLObndIRvyQ+/+6AhODh+OKWj9S9498pHHn/IMszH+gt0fBCRWMNfk1ZSp5x3AifmnI2vg==} engines: {node: '>=6'} - expect-type@1.3.0: - resolution: {integrity: sha512-knvyeauYhqjOYvQ66MznSMs83wmHrCycNEN6Ao+2AeYEfxUIkuiVxdEa1qlGEPK+We3n0THiDciYSsCcgW/DoA==} + expect-type@1.4.0: + resolution: {integrity: sha512-KfYbmpRm0VbLjEvVa9yGwCi9GI34xvi7A/HXYWQO65CSD2u3MczUJSuwXKFIxlGsgBQizV9q5J9NHj4VG0n+pA==} engines: {node: '>=12.0.0'} expect@29.7.0: @@ -1506,9 +1620,6 @@ packages: js-tokens@4.0.0: resolution: {integrity: sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==} - js-tokens@9.0.1: - resolution: {integrity: sha512-mxa9E9ITFOt0ban3j6L5MpjwegGz6lBQmM1IJkWeBZGcMxto50+eWdjC/52xDbS2vy0k7vIMK0Fe2wfL9OQSpQ==} - json-bigint@1.0.0: resolution: {integrity: sha512-SiPv/8VpZuWbvLSMtTDU8hEfrZWg/mH/nV/b4o0CYbSxu1UIQPLdwKOCIyLQX+VIPO5vrLX3i8qtqFyhdPSUSQ==} @@ -1581,9 +1692,6 @@ packages: long@5.3.2: resolution: {integrity: sha512-mNAgZ1GmyNhD7AuqnTG3/VQ26o760+ZYBPKjPvugO8+nLbYfX6TVpJPseBvopbdY+qpZ/lKUnmEc1LeZYS3QAA==} - loupe@3.2.1: - resolution: {integrity: sha512-CdzqowRJCeLU72bHvWqwRBBlLcMEtIvGrlvef74kMnV2AolS9Y8xUv1I0U/MNAWMhBlKIoyuEgoJ0t/bbwHbLQ==} - magic-string@0.30.21: resolution: {integrity: sha512-vd2F4YUyEXKGcLHoq+TEyCjxueSeHnFxyyjNp80yg0XV4vUhnDer/lvvlqM/arB5bXQN5K2/3oinyCRyx8T2CQ==} @@ -1770,8 +1878,8 @@ packages: mz@2.7.0: resolution: {integrity: sha512-z81GNO7nnYMEhrGh9LeymoE4+Yr0Wn5McHIZMK5cfQCl+NDX08sCZgUc9/6MHni9IWuFLm1Z3HTCXu2z9fN62Q==} - nanoid@3.3.12: - resolution: {integrity: sha512-ZB9RH/39qpq5Vu6Y+NmUaFhQR6pp+M2Xt76XBnEwDaGcVAqhlvxrl3B2bKS5D3NH3QR76v3aSrKaF/Kiy7lEtQ==} + nanoid@3.3.16: + resolution: {integrity: sha512-bzlKTyNJ7+LdGIIwy8ijFpIqEQIvafahV7eYykJ8Cvh42EdJeODoJ6gUJXpQJvej1BddH8OqTXZNE/KfbWAu8Q==} engines: {node: ^10 || ^12 || ^13.7 || ^14 || >=15.0.1} hasBin: true @@ -1811,6 +1919,10 @@ packages: obuf@1.1.2: resolution: {integrity: sha512-PX1wu0AmAdPqOL1mWhqmlOd8kOIZQwGZw6rh7uby9fTc5lhaOWFLX3I6R1hrF9k3zUY40e6igsLGkDXK92LJNg==} + obug@2.1.4: + resolution: {integrity: sha512-4a+OsYv9UktOJKE+l1A4OufDgdRF9PifWj+tJnHURo/P+WOxpG4GzUFL9qCalmWauao6ogiG+QvnCovwPoyAWA==} + engines: {node: '>=12.20.0'} + ollama@0.5.18: resolution: {integrity: sha512-lTFqTf9bo7Cd3hpF6CviBe/DEhewjoZYd9N/uCe7O20qYTvGqrNOFOBDj3lbZgFWHUgDv5EeyusYxsZSLS8nvg==} @@ -1863,10 +1975,6 @@ packages: pathe@2.0.3: resolution: {integrity: sha512-WUjGcAqP1gQacoQe+OBJsFA7Ld4DyXuUIjZ5cc75cLHvJ7dtNsTugphxIADwspS+AraAUePCKrSVtPLFj/F88w==} - pathval@2.0.1: - resolution: {integrity: sha512-//nshmD55c46FuFw26xV/xFAaB5HF9Xdap7HJBBnrKdAd6/GxDBaNA1870O79+9ueg61cZLSVc+OaFlfmObYVQ==} - engines: {node: '>= 14.16'} - pg-cloudflare@1.4.0: resolution: {integrity: sha512-Vo7z/6rrQYxpNRylp4Tlob2elzbh+N/MOQbxFVWCxS7oEx6jF53GTJFxK2WWpKuBRkmiin4Mt+xofFDjx09R0A==} @@ -1920,6 +2028,10 @@ packages: resolution: {integrity: sha512-QP88BAKvMam/3NxH6vj2o21R6MjxZUAd6nlwAS/pnGvN9IVLocLHxGYIzFhg6fUQ+5th6P4dv4eW9jX3DSIj7A==} engines: {node: '>=12'} + picomatch@4.0.5: + resolution: {integrity: sha512-RvwwcruNjI1ncT5xRakeyS9Lf8lcItv34KD+aif+VH9kduAyfYBipGh12274xtenIPZ119/R9BdTBa8gAwSh0A==} + engines: {node: '>=12'} + pirates@4.0.7: resolution: {integrity: sha512-TfySrs/5nm8fQJDcBDuUng3VOUKsd7S+zqvbOTiGXHfxX4wK31ard+hoNuvkicM/2YFzlpDgABOevKSsB4G/FA==} engines: {node: '>= 6'} @@ -1945,8 +2057,8 @@ packages: yaml: optional: true - postcss@8.5.13: - resolution: {integrity: sha512-qif0+jGGZoLWdHey3UFHHWP0H7Gbmsk8T5VEqyYFbWqPr1XqvLGBbk/sl8V5exGmcYJklJOhOQq1pV9IcsiFag==} + postcss@8.5.19: + resolution: {integrity: sha512-Mz8SaolMd8nB+G13WkORcxQKHZ/NE4xXevtkJHVuG+guo9/wYKlIMTKAqGdEmYOXR2ijPjTYNHssizdaVSUNdQ==} engines: {node: ^10 || ^12 || >=14} postgres-array@2.0.0: @@ -2048,6 +2160,11 @@ packages: engines: {node: '>=18.0.0', npm: '>=8.0.0'} hasBin: true + rollup@4.62.2: + resolution: {integrity: sha512-RFnrW4lhXA3s3eqHDZvN654g8OTjzRfqpIRJYczCGB6HzphckVAi/Qh4tbPUbRuDi7s1Llv8g/NspLkttY3gTA==} + engines: {node: '>=18.0.0', npm: '>=8.0.0'} + hasBin: true + run-applescript@7.1.0: resolution: {integrity: sha512-DPe5pVFaAsinSaV6QjQ6gdiedWDcRCbUuiQfQa2wmWV7+xC9bGulGI8+TdRmoFkAPaBXk8CrAbnlY2ISniJ47Q==} engines: {node: '>=18'} @@ -2105,8 +2222,8 @@ packages: stackback@0.0.2: resolution: {integrity: sha512-1XMJE5fQo1jGH6Y/7ebnwPOBEkIEnT4QF32d5R1+VXdXveM0IBMJt8zfaxX1P3QhVwrYe+576+jkANtSS2mBbw==} - std-env@3.10.0: - resolution: {integrity: sha512-5GS12FdOZNliM5mAOxFRg7Ir0pWz8MdpYm6AY6VPkGpbA7ZzmbzNcBJQ0GPvvyWgcY7QAhCgf9Uy89I03faLkg==} + std-env@4.2.0: + resolution: {integrity: sha512-oCUKSupKTHX53EyjDtuZQ64pjLJ6yYCtpmEw0goYxtjG9KpbRe8KAsl2tBUGU9DyMcJ0RwJ8GqJAFzMXcXW1Rw==} stopwords-iso@1.1.0: resolution: {integrity: sha512-I6GPS/E0zyieHehMRPQcqkiBMJKGgLta+1hREixhoLPqEA0AlVFiC43dl8uPpmkkeRdDMzYRWFWk5/l9x7nmNg==} @@ -2119,9 +2236,6 @@ packages: resolution: {integrity: sha512-4gB8na07fecVVkOI6Rs4e7T6NOTki5EmL7TUduTs6bu3EdnSycntVJ4re8kgZA+wx9IueI2Y11bfbgwtzuE0KQ==} engines: {node: '>=0.10.0'} - strip-literal@3.1.0: - resolution: {integrity: sha512-8r3mkIM/2+PpjHoOtiAW8Rg3jJLHaV7xPwG+YRGrv6FP0wwk/toTpATxWYOW0BKdWwl82VT2tFYi5DlROa0Mxg==} - sucrase@3.35.1: resolution: {integrity: sha512-DhuTmvZWux4H1UOnWMB3sk0sbaCVOoQZjv8u1rDoTV0HTdGem9hkAZtl4JZy8P2z4Bg0nT+YMeOFyVr4zcG5Tw==} engines: {node: '>=16 || 14 >=14.17'} @@ -2158,20 +2272,20 @@ packages: tinyexec@0.3.2: resolution: {integrity: sha512-KQQR9yN7R5+OSwaK0XQoj22pwHoTlgYqmUscPYoknOoWCWfj/5/ABTMRi69FrKU5ffPVh5QcFikpWJI/P1ocHA==} + tinyexec@1.2.4: + resolution: {integrity: sha512-SHf/r48b7vOrjve9PxJo3MN5v5yuyjHvdUcrQffT3WXMUfnGmHDVbC4k3sHJaJTgZCwpUplIaAo5ANtMyp3YHg==} + engines: {node: '>=18'} + tinyglobby@0.2.16: resolution: {integrity: sha512-pn99VhoACYR8nFHhxqix+uvsbXineAasWm5ojXoN8xEwK5Kd3/TrhNn1wByuD52UxWRLy8pu+kRMniEi6Eq9Zg==} engines: {node: '>=12.0.0'} - tinypool@1.1.1: - resolution: {integrity: sha512-Zba82s87IFq9A9XmjiX5uZA/ARWDrB03OHlq+Vw1fSdt0I+4/Kutwy8BP4Y/y/aORMo61FQ0vIb5j44vSo5Pkg==} - engines: {node: ^18.0.0 || >=20.0.0} - - tinyrainbow@2.0.0: - resolution: {integrity: sha512-op4nsTR47R6p0vMUUoYl/a+ljLFVtlfaXkLQmqfLR1qHma1h/ysYk4hEXZ880bf2CYgTskvTa/e196Vd5dDQXw==} - engines: {node: '>=14.0.0'} + tinyglobby@0.2.17: + resolution: {integrity: sha512-wXR/dYpcqKmfWpEdZjiKJOwCNFndD0DMnrW/cYjVGttEkBfVgcLFHoNrlj47mjOVic9yyNu65alsgF4NQyTa2g==} + engines: {node: '>=12.0.0'} - tinyspy@4.0.4: - resolution: {integrity: sha512-azl+t0z7pw/z958Gy9svOTuzqIk6xq+NSheJzn5MMWtWTFywIacg2wUlzKFGtt3cthx0r2SxMK0yzJOR0IES7Q==} + tinyrainbow@3.1.0: + resolution: {integrity: sha512-Bf+ILmBgretUrdJxzXM0SgXLZ3XfiaUuOj/IKQHuTXip+05Xn+uyEYdVg0kYDipTBcLrCVyUzAPz7QmArb0mmw==} engines: {node: '>=14.0.0'} to-regex-range@5.0.1: @@ -2236,11 +2350,8 @@ packages: undici-types@5.26.5: resolution: {integrity: sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==} - undici-types@7.19.2: - resolution: {integrity: sha512-qYVnV5OEm2AW8cJMCpdV20CDyaN3g0AjDlOGf1OW4iaDEx8MwdtChUp4zu4H0VP3nDRF/8RKWH+IPp9uW0YGZg==} - - undici-types@7.24.6: - resolution: {integrity: sha512-WRNW+sJgj5OBN4/0JpHFqtqzhpbnV0GuB+OozA9gCL7a993SmU+1JBZCzLNxYsbMfIeDL+lTsphD5jN5N+n0zg==} + undici-types@8.3.0: + resolution: {integrity: sha512-j375ScV60dom+YkPFIfTLcOiPxkN/buHz5GobjLhixFuANaNs3C9l4GmrWqejgXWJ7BbJcFYpTEUkS1Ge8bpZQ==} undici@6.27.0: resolution: {integrity: sha512-YmfV3YnEDzXRC5lZ2jWtWWHKGUm1zIt8AhesR1tens+HTNv+YZlN/dp6G727LOvMJ8xjP9Be7Y2Sdr96LDm+pg==} @@ -2265,11 +2376,6 @@ packages: typescript: optional: true - vite-node@3.2.4: - resolution: {integrity: sha512-EbKSKh+bh1E1IFxeO0pg1n4dvoOTt0UDiXMd/qn++r98+jPO1xtJilvXldeuQ8giIB5IkpjCgMleHMNEsGH6pg==} - engines: {node: ^18.0.0 || ^20.0.0 || >=22.0.0} - hasBin: true - vite@7.3.2: resolution: {integrity: sha512-Bby3NOsna2jsjfLVOHKes8sGwgl4TT0E6vvpYgnAYDIF/tie7MRaFthmKuHx1NSXjiTueXH3do80FMQgvEktRg==} engines: {node: ^20.19.0 || >=22.12.0} @@ -2310,26 +2416,39 @@ packages: yaml: optional: true - vitest@3.2.4: - resolution: {integrity: sha512-LUCP5ev3GURDysTWiP47wRRUpLKMOfPh+yKTx3kVIEiu5KOMeqzpnYNsKyOoVrULivR8tLcks4+lga33Whn90A==} - engines: {node: ^18.0.0 || ^20.0.0 || >=22.0.0} + vitest@4.1.10: + resolution: {integrity: sha512-R9jUTe5S4Qb0HCd4TNqpC7oGcrMssMRGXLW80ubjWsW9VH5GF8y1Y0SFLY9AbqSk6nt0PnOx4H4WNJYZ13GUPw==} + engines: {node: ^20.0.0 || ^22.0.0 || >=24.0.0} hasBin: true peerDependencies: '@edge-runtime/vm': '*' - '@types/debug': ^4.1.12 - '@types/node': ^18.0.0 || ^20.0.0 || >=22.0.0 - '@vitest/browser': 3.2.4 - '@vitest/ui': 3.2.4 + '@opentelemetry/api': ^1.9.0 + '@types/node': ^20.0.0 || ^22.0.0 || >=24.0.0 + '@vitest/browser-playwright': 4.1.10 + '@vitest/browser-preview': 4.1.10 + '@vitest/browser-webdriverio': 4.1.10 + '@vitest/coverage-istanbul': 4.1.10 + '@vitest/coverage-v8': 4.1.10 + '@vitest/ui': 4.1.10 happy-dom: '*' jsdom: '*' + vite: ^6.0.0 || ^7.0.0 || ^8.0.0 peerDependenciesMeta: '@edge-runtime/vm': optional: true - '@types/debug': + '@opentelemetry/api': optional: true '@types/node': optional: true - '@vitest/browser': + '@vitest/browser-playwright': + optional: true + '@vitest/browser-preview': + optional: true + '@vitest/browser-webdriverio': + optional: true + '@vitest/coverage-istanbul': + optional: true + '@vitest/coverage-v8': optional: true '@vitest/ui': optional: true @@ -2776,7 +2895,7 @@ snapshots: '@jest/schemas': 29.6.3 '@types/istanbul-lib-coverage': 2.0.6 '@types/istanbul-reports': 3.0.4 - '@types/node': 25.9.5 + '@types/node': 26.1.1 '@types/yargs': 17.0.35 chalk: 4.1.2 @@ -2894,78 +3013,153 @@ snapshots: '@rollup/rollup-android-arm-eabi@4.60.2': optional: true + '@rollup/rollup-android-arm-eabi@4.62.2': + optional: true + '@rollup/rollup-android-arm64@4.60.2': optional: true + '@rollup/rollup-android-arm64@4.62.2': + optional: true + '@rollup/rollup-darwin-arm64@4.60.2': optional: true + '@rollup/rollup-darwin-arm64@4.62.2': + optional: true + '@rollup/rollup-darwin-x64@4.60.2': optional: true + '@rollup/rollup-darwin-x64@4.62.2': + optional: true + '@rollup/rollup-freebsd-arm64@4.60.2': optional: true + '@rollup/rollup-freebsd-arm64@4.62.2': + optional: true + '@rollup/rollup-freebsd-x64@4.60.2': optional: true + '@rollup/rollup-freebsd-x64@4.62.2': + optional: true + '@rollup/rollup-linux-arm-gnueabihf@4.60.2': optional: true + '@rollup/rollup-linux-arm-gnueabihf@4.62.2': + optional: true + '@rollup/rollup-linux-arm-musleabihf@4.60.2': optional: true + '@rollup/rollup-linux-arm-musleabihf@4.62.2': + optional: true + '@rollup/rollup-linux-arm64-gnu@4.60.2': optional: true + '@rollup/rollup-linux-arm64-gnu@4.62.2': + optional: true + '@rollup/rollup-linux-arm64-musl@4.60.2': optional: true + '@rollup/rollup-linux-arm64-musl@4.62.2': + optional: true + '@rollup/rollup-linux-loong64-gnu@4.60.2': optional: true + '@rollup/rollup-linux-loong64-gnu@4.62.2': + optional: true + '@rollup/rollup-linux-loong64-musl@4.60.2': optional: true + '@rollup/rollup-linux-loong64-musl@4.62.2': + optional: true + '@rollup/rollup-linux-ppc64-gnu@4.60.2': optional: true + '@rollup/rollup-linux-ppc64-gnu@4.62.2': + optional: true + '@rollup/rollup-linux-ppc64-musl@4.60.2': optional: true + '@rollup/rollup-linux-ppc64-musl@4.62.2': + optional: true + '@rollup/rollup-linux-riscv64-gnu@4.60.2': optional: true + '@rollup/rollup-linux-riscv64-gnu@4.62.2': + optional: true + '@rollup/rollup-linux-riscv64-musl@4.60.2': optional: true + '@rollup/rollup-linux-riscv64-musl@4.62.2': + optional: true + '@rollup/rollup-linux-s390x-gnu@4.60.2': optional: true + '@rollup/rollup-linux-s390x-gnu@4.62.2': + optional: true + '@rollup/rollup-linux-x64-gnu@4.60.2': optional: true + '@rollup/rollup-linux-x64-gnu@4.62.2': + optional: true + '@rollup/rollup-linux-x64-musl@4.60.2': optional: true + '@rollup/rollup-linux-x64-musl@4.62.2': + optional: true + '@rollup/rollup-openbsd-x64@4.60.2': optional: true + '@rollup/rollup-openbsd-x64@4.62.2': + optional: true + '@rollup/rollup-openharmony-arm64@4.60.2': optional: true + '@rollup/rollup-openharmony-arm64@4.62.2': + optional: true + '@rollup/rollup-win32-arm64-msvc@4.60.2': optional: true + '@rollup/rollup-win32-arm64-msvc@4.62.2': + optional: true + '@rollup/rollup-win32-ia32-msvc@4.60.2': optional: true + '@rollup/rollup-win32-ia32-msvc@4.62.2': + optional: true + '@rollup/rollup-win32-x64-gnu@4.60.2': optional: true + '@rollup/rollup-win32-x64-gnu@4.62.2': + optional: true + '@rollup/rollup-win32-x64-msvc@4.60.2': optional: true + '@rollup/rollup-win32-x64-msvc@4.62.2': + optional: true + '@scure/base@1.2.6': {} '@scure/base@2.2.0': {} @@ -3111,6 +3305,8 @@ snapshots: '@types/estree@1.0.8': {} + '@types/estree@1.0.9': {} + '@types/istanbul-lib-coverage@2.0.6': {} '@types/istanbul-lib-report@3.0.3': @@ -3128,24 +3324,20 @@ snapshots: '@types/node-fetch@2.6.13': dependencies: - '@types/node': 25.6.0 + '@types/node': 26.1.1 form-data: 4.0.6 '@types/node@18.19.130': dependencies: undici-types: 5.26.5 - '@types/node@25.6.0': - dependencies: - undici-types: 7.19.2 - - '@types/node@25.9.5': + '@types/node@26.1.1': dependencies: - undici-types: 7.24.6 + undici-types: 8.3.0 '@types/pg@8.11.0': dependencies: - '@types/node': 25.9.5 + '@types/node': 26.1.1 pg-protocol: 1.15.0 pg-types: 4.1.0 @@ -3179,47 +3371,46 @@ snapshots: transitivePeerDependencies: - supports-color - '@vitest/expect@3.2.4': + '@vitest/expect@4.1.10': dependencies: + '@standard-schema/spec': 1.1.0 '@types/chai': 5.2.3 - '@vitest/spy': 3.2.4 - '@vitest/utils': 3.2.4 - chai: 5.3.3 - tinyrainbow: 2.0.0 + '@vitest/spy': 4.1.10 + '@vitest/utils': 4.1.10 + chai: 6.2.2 + tinyrainbow: 3.1.0 - '@vitest/mocker@3.2.4(vite@7.3.2(@types/node@25.6.0)(tsx@4.23.1)(yaml@2.8.4))': + '@vitest/mocker@4.1.10(vite@7.3.2(@types/node@26.1.1)(tsx@4.23.1)(yaml@2.8.4))': dependencies: - '@vitest/spy': 3.2.4 + '@vitest/spy': 4.1.10 estree-walker: 3.0.3 magic-string: 0.30.21 optionalDependencies: - vite: 7.3.2(@types/node@25.6.0)(tsx@4.23.1)(yaml@2.8.4) + vite: 7.3.2(@types/node@26.1.1)(tsx@4.23.1)(yaml@2.8.4) - '@vitest/pretty-format@3.2.4': + '@vitest/pretty-format@4.1.10': dependencies: - tinyrainbow: 2.0.0 + tinyrainbow: 3.1.0 - '@vitest/runner@3.2.4': + '@vitest/runner@4.1.10': dependencies: - '@vitest/utils': 3.2.4 + '@vitest/utils': 4.1.10 pathe: 2.0.3 - strip-literal: 3.1.0 - '@vitest/snapshot@3.2.4': + '@vitest/snapshot@4.1.10': dependencies: - '@vitest/pretty-format': 3.2.4 + '@vitest/pretty-format': 4.1.10 + '@vitest/utils': 4.1.10 magic-string: 0.30.21 pathe: 2.0.3 - '@vitest/spy@3.2.4': - dependencies: - tinyspy: 4.0.4 + '@vitest/spy@4.1.10': {} - '@vitest/utils@3.2.4': + '@vitest/utils@4.1.10': dependencies: - '@vitest/pretty-format': 3.2.4 - loupe: 3.2.1 - tinyrainbow: 2.0.0 + '@vitest/pretty-format': 4.1.10 + convert-source-map: 2.0.0 + tinyrainbow: 3.1.0 abitype@1.2.3(typescript@5.9.3)(zod@4.4.3): optionalDependencies: @@ -3324,13 +3515,7 @@ snapshots: es-errors: 1.3.0 function-bind: 1.1.2 - chai@5.3.3: - dependencies: - assertion-error: 2.0.1 - check-error: 2.1.3 - deep-eql: 5.0.2 - loupe: 3.2.1 - pathval: 2.0.1 + chai@6.2.2: {} chalk@4.1.2: dependencies: @@ -3339,8 +3524,6 @@ snapshots: charenc@0.0.2: {} - check-error@2.1.3: {} - chokidar@4.0.3: dependencies: readdirp: 4.1.2 @@ -3387,6 +3570,8 @@ snapshots: consola@3.4.2: {} + convert-source-map@2.0.0: {} + crypt@0.0.2: {} data-uri-to-buffer@4.0.1: {} @@ -3399,8 +3584,6 @@ snapshots: dependencies: mimic-response: 3.1.0 - deep-eql@5.0.2: {} - deep-extend@0.6.0: {} default-browser-id@5.0.1: {} @@ -3445,7 +3628,7 @@ snapshots: es-errors@1.3.0: {} - es-module-lexer@1.7.0: {} + es-module-lexer@2.3.1: {} es-object-atoms@1.1.2: dependencies: @@ -3515,12 +3698,13 @@ snapshots: '@esbuild/win32-arm64': 0.28.1 '@esbuild/win32-ia32': 0.28.1 '@esbuild/win32-x64': 0.28.1 + optional: true escape-string-regexp@2.0.0: {} estree-walker@3.0.3: dependencies: - '@types/estree': 1.0.8 + '@types/estree': 1.0.9 event-target-shim@5.0.1: {} @@ -3532,7 +3716,7 @@ snapshots: expand-template@2.0.3: {} - expect-type@1.3.0: {} + expect-type@1.4.0: {} expect@29.7.0: dependencies: @@ -3548,6 +3732,10 @@ snapshots: optionalDependencies: picomatch: 4.0.4 + fdir@6.5.0(picomatch@4.0.5): + optionalDependencies: + picomatch: 4.0.5 + fetch-blob@3.2.0: dependencies: node-domexception: 1.0.0 @@ -3758,7 +3946,7 @@ snapshots: jest-util@29.7.0: dependencies: '@jest/types': 29.6.3 - '@types/node': 25.9.5 + '@types/node': 26.1.1 chalk: 4.1.2 ci-info: 3.9.0 graceful-fs: 4.2.11 @@ -3772,8 +3960,6 @@ snapshots: js-tokens@4.0.0: {} - js-tokens@9.0.1: {} - json-bigint@1.0.0: dependencies: bignumber.js: 9.3.1 @@ -3833,8 +4019,6 @@ snapshots: long@5.3.2: {} - loupe@3.2.1: {} - magic-string@0.30.21: dependencies: '@jridgewell/sourcemap-codec': 1.5.5 @@ -3951,7 +4135,7 @@ snapshots: object-assign: 4.1.1 thenify-all: 1.6.0 - nanoid@3.3.12: {} + nanoid@3.3.16: {} napi-build-utils@2.0.0: {} @@ -4003,6 +4187,8 @@ snapshots: obuf@1.1.2: {} + obug@2.1.4: {} + ollama@0.5.18: dependencies: whatwg-fetch: 3.6.20 @@ -4070,8 +4256,6 @@ snapshots: pathe@2.0.3: {} - pathval@2.0.1: {} - pg-cloudflare@1.4.0: optional: true @@ -4125,6 +4309,8 @@ snapshots: picomatch@4.0.4: {} + picomatch@4.0.5: {} + pirates@4.0.7: {} pkg-types@1.3.1: @@ -4133,17 +4319,17 @@ snapshots: mlly: 1.8.2 pathe: 2.0.3 - postcss-load-config@6.0.1(postcss@8.5.13)(tsx@4.23.1)(yaml@2.8.4): + postcss-load-config@6.0.1(postcss@8.5.19)(tsx@4.23.1)(yaml@2.8.4): dependencies: lilconfig: 3.1.3 optionalDependencies: - postcss: 8.5.13 + postcss: 8.5.19 tsx: 4.23.1 yaml: 2.8.4 - postcss@8.5.13: + postcss@8.5.19: dependencies: - nanoid: 3.3.12 + nanoid: 3.3.16 picocolors: 1.1.1 source-map-js: 1.2.1 @@ -4207,7 +4393,7 @@ snapshots: '@protobufjs/path': 1.1.2 '@protobufjs/pool': 1.1.0 '@protobufjs/utf8': 1.1.2 - '@types/node': 25.9.5 + '@types/node': 26.1.1 long: 5.3.2 proxy-from-env@2.1.0: {} @@ -4284,6 +4470,37 @@ snapshots: '@rollup/rollup-win32-x64-msvc': 4.60.2 fsevents: 2.3.3 + rollup@4.62.2: + dependencies: + '@types/estree': 1.0.9 + optionalDependencies: + '@rollup/rollup-android-arm-eabi': 4.62.2 + '@rollup/rollup-android-arm64': 4.62.2 + '@rollup/rollup-darwin-arm64': 4.62.2 + '@rollup/rollup-darwin-x64': 4.62.2 + '@rollup/rollup-freebsd-arm64': 4.62.2 + '@rollup/rollup-freebsd-x64': 4.62.2 + '@rollup/rollup-linux-arm-gnueabihf': 4.62.2 + '@rollup/rollup-linux-arm-musleabihf': 4.62.2 + '@rollup/rollup-linux-arm64-gnu': 4.62.2 + '@rollup/rollup-linux-arm64-musl': 4.62.2 + '@rollup/rollup-linux-loong64-gnu': 4.62.2 + '@rollup/rollup-linux-loong64-musl': 4.62.2 + '@rollup/rollup-linux-ppc64-gnu': 4.62.2 + '@rollup/rollup-linux-ppc64-musl': 4.62.2 + '@rollup/rollup-linux-riscv64-gnu': 4.62.2 + '@rollup/rollup-linux-riscv64-musl': 4.62.2 + '@rollup/rollup-linux-s390x-gnu': 4.62.2 + '@rollup/rollup-linux-x64-gnu': 4.62.2 + '@rollup/rollup-linux-x64-musl': 4.62.2 + '@rollup/rollup-openbsd-x64': 4.62.2 + '@rollup/rollup-openharmony-arm64': 4.62.2 + '@rollup/rollup-win32-arm64-msvc': 4.62.2 + '@rollup/rollup-win32-ia32-msvc': 4.62.2 + '@rollup/rollup-win32-x64-gnu': 4.62.2 + '@rollup/rollup-win32-x64-msvc': 4.62.2 + fsevents: 2.3.3 + run-applescript@7.1.0: {} safe-buffer@5.2.1: {} @@ -4324,7 +4541,7 @@ snapshots: stackback@0.0.2: {} - std-env@3.10.0: {} + std-env@4.2.0: {} stopwords-iso@1.1.0: {} @@ -4334,10 +4551,6 @@ snapshots: strip-json-comments@2.0.1: {} - strip-literal@3.1.0: - dependencies: - js-tokens: 9.0.1 - sucrase@3.35.1: dependencies: '@jridgewell/gen-mapping': 0.3.13 @@ -4383,16 +4596,19 @@ snapshots: tinyexec@0.3.2: {} + tinyexec@1.2.4: {} + tinyglobby@0.2.16: dependencies: fdir: 6.5.0(picomatch@4.0.4) picomatch: 4.0.4 - tinypool@1.1.1: {} - - tinyrainbow@2.0.0: {} + tinyglobby@0.2.17: + dependencies: + fdir: 6.5.0(picomatch@4.0.5) + picomatch: 4.0.5 - tinyspy@4.0.4: {} + tinyrainbow@3.1.0: {} to-regex-range@5.0.1: dependencies: @@ -4410,7 +4626,7 @@ snapshots: tslib@2.8.1: {} - tsup@8.5.1(postcss@8.5.13)(tsx@4.23.1)(typescript@5.9.3)(yaml@2.8.4): + tsup@8.5.1(postcss@8.5.19)(tsx@4.23.1)(typescript@5.9.3)(yaml@2.8.4): dependencies: bundle-require: 5.1.0(esbuild@0.27.7) cac: 6.7.14 @@ -4421,7 +4637,7 @@ snapshots: fix-dts-default-cjs-exports: 1.0.1 joycon: 3.1.1 picocolors: 1.1.1 - postcss-load-config: 6.0.1(postcss@8.5.13)(tsx@4.23.1)(yaml@2.8.4) + postcss-load-config: 6.0.1(postcss@8.5.19)(tsx@4.23.1)(yaml@2.8.4) resolve-from: 5.0.0 rollup: 4.60.2 source-map: 0.7.6 @@ -4430,7 +4646,7 @@ snapshots: tinyglobby: 0.2.16 tree-kill: 1.2.2 optionalDependencies: - postcss: 8.5.13 + postcss: 8.5.19 typescript: 5.9.3 transitivePeerDependencies: - jiti @@ -4443,6 +4659,7 @@ snapshots: esbuild: 0.28.1 optionalDependencies: fsevents: 2.3.3 + optional: true tunnel-agent@0.6.0: dependencies: @@ -4456,9 +4673,7 @@ snapshots: undici-types@5.26.5: {} - undici-types@7.19.2: {} - - undici-types@7.24.6: {} + undici-types@8.3.0: {} undici@6.27.0: {} @@ -4485,81 +4700,47 @@ snapshots: - utf-8-validate - zod - vite-node@3.2.4(@types/node@25.6.0)(tsx@4.23.1)(yaml@2.8.4): - dependencies: - cac: 6.7.14 - debug: 4.4.3 - es-module-lexer: 1.7.0 - pathe: 2.0.3 - vite: 7.3.2(@types/node@25.6.0)(tsx@4.23.1)(yaml@2.8.4) - transitivePeerDependencies: - - '@types/node' - - jiti - - less - - lightningcss - - sass - - sass-embedded - - stylus - - sugarss - - supports-color - - terser - - tsx - - yaml - - vite@7.3.2(@types/node@25.6.0)(tsx@4.23.1)(yaml@2.8.4): + vite@7.3.2(@types/node@26.1.1)(tsx@4.23.1)(yaml@2.8.4): dependencies: esbuild: 0.27.7 - fdir: 6.5.0(picomatch@4.0.4) - picomatch: 4.0.4 - postcss: 8.5.13 - rollup: 4.60.2 - tinyglobby: 0.2.16 + fdir: 6.5.0(picomatch@4.0.5) + picomatch: 4.0.5 + postcss: 8.5.19 + rollup: 4.62.2 + tinyglobby: 0.2.17 optionalDependencies: - '@types/node': 25.6.0 + '@types/node': 26.1.1 fsevents: 2.3.3 tsx: 4.23.1 yaml: 2.8.4 - vitest@3.2.4(@types/node@25.6.0)(tsx@4.23.1)(yaml@2.8.4): - dependencies: - '@types/chai': 5.2.3 - '@vitest/expect': 3.2.4 - '@vitest/mocker': 3.2.4(vite@7.3.2(@types/node@25.6.0)(tsx@4.23.1)(yaml@2.8.4)) - '@vitest/pretty-format': 3.2.4 - '@vitest/runner': 3.2.4 - '@vitest/snapshot': 3.2.4 - '@vitest/spy': 3.2.4 - '@vitest/utils': 3.2.4 - chai: 5.3.3 - debug: 4.4.3 - expect-type: 1.3.0 + vitest@4.1.10(@opentelemetry/api@1.9.1)(@types/node@26.1.1)(vite@7.3.2(@types/node@26.1.1)(tsx@4.23.1)(yaml@2.8.4)): + dependencies: + '@vitest/expect': 4.1.10 + '@vitest/mocker': 4.1.10(vite@7.3.2(@types/node@26.1.1)(tsx@4.23.1)(yaml@2.8.4)) + '@vitest/pretty-format': 4.1.10 + '@vitest/runner': 4.1.10 + '@vitest/snapshot': 4.1.10 + '@vitest/spy': 4.1.10 + '@vitest/utils': 4.1.10 + es-module-lexer: 2.3.1 + expect-type: 1.4.0 magic-string: 0.30.21 + obug: 2.1.4 pathe: 2.0.3 - picomatch: 4.0.4 - std-env: 3.10.0 + picomatch: 4.0.5 + std-env: 4.2.0 tinybench: 2.9.0 - tinyexec: 0.3.2 - tinyglobby: 0.2.16 - tinypool: 1.1.1 - tinyrainbow: 2.0.0 - vite: 7.3.2(@types/node@25.6.0)(tsx@4.23.1)(yaml@2.8.4) - vite-node: 3.2.4(@types/node@25.6.0)(tsx@4.23.1)(yaml@2.8.4) + tinyexec: 1.2.4 + tinyglobby: 0.2.17 + tinyrainbow: 3.1.0 + vite: 7.3.2(@types/node@26.1.1)(tsx@4.23.1)(yaml@2.8.4) why-is-node-running: 2.3.0 optionalDependencies: - '@types/node': 25.6.0 + '@opentelemetry/api': 1.9.1 + '@types/node': 26.1.1 transitivePeerDependencies: - - jiti - - less - - lightningcss - msw - - sass - - sass-embedded - - stylus - - sugarss - - supports-color - - terser - - tsx - - yaml web-streams-polyfill@3.3.3: {} diff --git a/src/claim-grounding.ts b/src/claim-grounding.ts index 1998aae..2894e19 100644 --- a/src/claim-grounding.ts +++ b/src/claim-grounding.ts @@ -239,8 +239,7 @@ export interface ClaimGroundingDriverOptions extends GroundClaimOptions { * What to do when a proposal carries NO cited claim. `'reject'` (default) is * fail-closed: in claim-grounding mode every source must declare what it is * cited for, so an un-annotated source is treated as ungrounded. `'accept'` - * lets un-annotated sources through to the relevance verifier (if any) — - * useful when mixing annotated and legacy proposals. + * lets unannotated sources through to the relevance verifier, if present. */ onMissingClaim?: 'reject' | 'accept' } @@ -268,7 +267,6 @@ export function createClaimGroundingVerifier(options: ClaimGroundingDriverOption reason: 'no cited claim: claim-grounding mode requires every source to declare its claim', } } - // accept-on-missing: fall through to the relevance verifier (or accept). return options.relevanceVerifier ? options.relevanceVerifier(source, ctx) : { accept: true } } @@ -288,7 +286,6 @@ export function createClaimGroundingVerifier(options: ClaimGroundingDriverOption } } - // Claim is grounded. Compose the relevance verifier if one was provided. if (options.relevanceVerifier) return options.relevanceVerifier(source, ctx) return { accept: true } } diff --git a/src/file-transaction.ts b/src/file-transaction.ts index fdb1b52..2eb8f17 100644 --- a/src/file-transaction.ts +++ b/src/file-transaction.ts @@ -332,7 +332,7 @@ export async function applyKnowledgeFileTransaction(input: { ) } -export async function assertKnowledgeFileTransactionApplied( +async function assertKnowledgeFileTransactionApplied( root: string, transaction: KnowledgeFileTransaction, ): Promise { diff --git a/src/freshness.ts b/src/freshness.ts index 9f6264c..22b1450 100644 --- a/src/freshness.ts +++ b/src/freshness.ts @@ -8,7 +8,7 @@ import { withKnowledgeMutation, withKnowledgeRead } from './mutation-lock' * Knowledge freshness store: tracks when each `(workspaceId, sourceId)` pair * was last successfully refreshed, and reports staleness against a TTL. * - * The contract is intentionally minimal — just enough to drive a cron loop: + * The contract is intentionally minimal and supports scheduled refresh loops: * * ```ts * const store = createFileSystemFreshnessStore({ root: '.agent-knowledge' }) @@ -21,20 +21,16 @@ import { withKnowledgeMutation, withKnowledgeRead } from './mutation-lock' * } * ``` * - * Per-tenant isolation is enforced by `workspaceId` keying — there is no + * Per-tenant isolation is enforced by `workspaceId` keying. There is no * global mutable state across workspaces. * * Two adapters ship in-package: * - * - `createFileSystemFreshnessStore` — JSON file under the knowledge root, + * - `createFileSystemFreshnessStore`: JSON file under the knowledge root, * mirrors the layout convention already used by `sources.json`. - * - `createD1FreshnessStoreStub` — adapter scaffold for Cloudflare D1 / - * Postgres. Production consumers should implement the `D1Adapter` - * interface inside their own app; this stub exists to anchor the shape. - * - * @stable contract — interface is frozen at 0.x within this major. - * @stable filesystem adapter - * @experimental D1 stub — interface will evolve as real consumers wire it. + * - `createD1FreshnessStoreStub`: complete bridge from the `D1Adapter` port + * to this package's freshness interface. The port can be backed by D1, + * PostgreSQL, SQLite, or another application-owned store. */ /** Identity for one freshness record. */ @@ -45,7 +41,7 @@ export interface FreshnessKey { /** TTL bound for staleness checks. */ export interface FreshnessTtl extends FreshnessKey { - /** Milliseconds — `Date.now() - last() > ttlMs` ⇒ stale. */ + /** Milliseconds; the record is stale when `Date.now() - last() > ttlMs`. */ ttlMs: number /** Injected clock for deterministic tests; defaults to system time. */ now?: Date @@ -65,7 +61,7 @@ export interface KnowledgeFreshnessStore { mark(input: FreshnessMark): Promise /** True iff `last(key)` is null or older than `ttlMs`. */ stale(input: FreshnessTtl): Promise - /** All records for a workspace — useful for dashboards / debugging. */ + /** All records for a workspace. */ list(workspaceId: string): Promise } @@ -98,7 +94,7 @@ const freshnessFileSchema = z /** * Filesystem-backed implementation. Single JSON file per knowledge root, - * indexed by `${workspaceId}::${sourceId}`. Reads parse on every call — + * indexed by `${workspaceId}::${sourceId}`. Reads parse on every call; * cron tick rate is well below the cost of one JSON parse. * * Writes share the package-wide filesystem lock, so multiple workers cannot @@ -160,10 +156,8 @@ export function createFileSystemFreshnessStore( } /** - * D1 / Postgres adapter scaffold. Production consumers implement - * `D1Adapter` against their own driver (better-sqlite3, postgres, - * Cloudflare D1 binding, ...). This factory wires the adapter to the - * `KnowledgeFreshnessStore` interface. + * Bridge an application-owned database adapter to `KnowledgeFreshnessStore`. + * The adapter may use D1, PostgreSQL, SQLite, or another durable store. * * The expected schema: * diff --git a/src/memory/experiment/metrics.ts b/src/memory/experiment/metrics.ts index 77afdef..ebf3496 100644 --- a/src/memory/experiment/metrics.ts +++ b/src/memory/experiment/metrics.ts @@ -122,7 +122,7 @@ export function mean(values: readonly number[]): number { return values.length === 0 ? 0 : values.reduce((sum, value) => sum + value, 0) / values.length } -export function format(value: number): string { +function format(value: number): string { return Number.isFinite(value) ? value.toFixed(4) : '0.0000' } diff --git a/src/memory/holdout.ts b/src/memory/holdout.ts index d1a8b89..9eb8ee7 100644 --- a/src/memory/holdout.ts +++ b/src/memory/holdout.ts @@ -342,29 +342,28 @@ export interface RetrievalHoldoutOffPolicyOptions { } export interface RetrievalHoldoutOffPolicyResult { - /** One trajectory per (configHash, sessionIdHash) — never per call. */ + /** One trajectory per (configHash, sessionIdHash), never per call. */ trajectories: OffPolicyTrajectory[] /** 1:1 with `trajectories` (same order): the diagnostic view of each converted session. */ sessions: RetrievalHoldoutSessionSummary[] - /** Sessions surfaced but NOT converted (mixed exposure, bypass-only) — counted, never hidden. */ + /** Sessions not converted because exposure was mixed or retrieval was bypassed. */ excluded: RetrievalHoldoutSessionSummary[] /** Events with no sessionIdHash: outside session randomization, cannot join a reward. */ unattributableEvents: number } /** - * Maps holdout logs onto agent-eval's `OffPolicyTrajectory`, ONE TRAJECTORY PER SESSION, so - * EXP-007's analysis consumes the substrate's `inverseProbabilityWeighting` / - * `selfNormalizedImportanceWeighting` / `doublyRobust` estimators directly. This matches the - * PREREG estimator (session-sticky self-normalized IPW): the design randomizes per SESSION — - * the arm coin is flipped once (P(holdout) = epsilon) and the sticky target is drawn once, + * Maps retrieval-dropout events onto one `OffPolicyTrajectory` per session for use with + * `inverseProbabilityWeighting`, `selfNormalizedImportanceWeighting`, or `doublyRobust`. + * Randomization occurs once per session: the arm is selected with P(dropout) = epsilon and + * the sticky target is drawn once, * uniformly over watchlist ∩ E at the session's first intersecting call. * * Session-level action space and behavior probabilities (they sum to 1 by construction): - * - full delivery (control arm observed): `1 − epsilon`; + * - full delivery: `1 - epsilon`; * - drop candidate i of k = |watchlist ∩ E_first|: `epsilon / k` each (the draw event's logged * `dropPropensity`); - * - sessions whose eligibility sets never intersect the watchlist: probability 1 — full + * - sessions whose eligibility sets never intersect the watchlist: probability 1, because full * delivery was certain in either arm. * * Per-call events are repeated observations WITHIN one session-level randomization; per-call @@ -373,14 +372,12 @@ export interface RetrievalHoldoutOffPolicyResult { * absent from E, and adapter bypass calls, fold into the session summary (callCount / * bypassCallCount) instead of generating independent propensities. * - * Join contract: `rewards` is keyed by `sessionIdHash` (sha256(sessionId) first 16 hex — compute + * Join contract: `rewards` is keyed by `sessionIdHash` (sha256(sessionId) first 16 hex; compute * the same prefix over the outcome table's session ids, or log with * `includePlaintextIdentifiers: true` and join on raw ids upstream). Events are grouped per * (configHash, sessionIdHash) so different experiment configs are never pooled. Mixed-exposure * sessions (arm or target disagreement, e.g. after registry eviction) are excluded from the - * trajectories and surfaced in `excluded` for the analysis to count. - * Pre-registration for EXP-007 stays in the research repo by design; the manifest vocabulary for - * it is agent-eval's `HypothesisManifest` / `signManifest` / `evaluateHypothesis` exports. + * trajectories and surfaced in `excluded` for analysis. */ export function toOffPolicyTrajectory( events: RetrievalHoldoutEvent[], diff --git a/src/memory/types.ts b/src/memory/types.ts index c6d4b9b..4366227 100644 --- a/src/memory/types.ts +++ b/src/memory/types.ts @@ -110,11 +110,11 @@ export interface AgentMemoryAdapter { close?(): Promise } -// Randomized retrieval holdout (epsilon-dropout) for per-item treatment-effect logging. -// Default-off: nothing in this module runs unless a consumer passes a RetrievalHoldoutConfig. -// The library never does I/O here; persistence is the consumer's job via onEvent. -// Design + estimator + sample-size analysis: research repo, -// projects/probabilistic-agent-optimization/notes/2026-07-03-DRAFT-o3-holdout-design.md (O3 / EXP-007). +/** + * Optional session-level retrieval dropout for estimating whether delivered memories affect + * task outcomes. The feature is disabled unless configured, and consumers persist events + * through `onEvent`. + */ export interface RetrievalHoldoutConfig { /** Per-session probability that one eligible watchlist item is suppressed. 0 logs the full schema without ever dropping. */ @@ -132,20 +132,20 @@ export interface RetrievalHoldoutConfig { * sessionIdHash/scopeHash, so PII-bearing identifiers (tenantId/userId/tags) never reach a * consumer-controlled sink unless the consumer explicitly owns that decision. Note that * replaying assignment draws from logs alone needs the plaintext sessionId, so - * privacy-default logs require the consumer's own sessionId mapping for replay audits. + * privacy-preserving logs require the consumer's own sessionId mapping for replay. */ includePlaintextIdentifiers?: boolean /** * Cap on tracked sessions per experiment config in the sticky wrapper's registry. - * Exists so tests can exercise eviction; production should keep the default (10,000). + * The default is 10,000. */ maxTrackedSessions?: number /** * Uniform-[0,1) generator keyed by a string. Defaults to a sha256-derived deterministic - * generator so every assignment is replayable from the logged keys alone (design rule D5). + * generator so every assignment is replayable from the logged keys alone. */ rng?: (key: string) => number - /** Receives one event per retrieval call, INCLUDING no-drop calls: control-arm membership is half the data. */ + /** Receives one event per retrieval call, including calls where nothing is dropped. */ onEvent: (event: RetrievalHoldoutEvent) => void } @@ -164,19 +164,18 @@ export interface RetrievalHoldoutEvent { eventId: string ts: string adapterId?: string - /** Plaintext session id — emitted ONLY when config.includePlaintextIdentifiers is true. */ + /** Plaintext session id, emitted only when `includePlaintextIdentifiers` is true. */ sessionId?: string /** Consumer-supplied experiment/outcome join id (scope.tags.taskId); deliberately plaintext. */ taskId?: string /** 1-based call counter within the session; 0 when the call is outside session randomization. */ callIndex: number /** - * sha256(sessionId) prefix — the default privacy-preserving session join key AND the seed-key - * reference for the assignment draws (previously named rngKey; identical derivation, deduped). + * sha256(sessionId) prefix used as the privacy-preserving join key and assignment seed. */ sessionIdHash?: string queryHash?: string - /** Verbatim scope — emitted ONLY when config.includePlaintextIdentifiers is true (PII risk). */ + /** Verbatim scope, emitted only when `includePlaintextIdentifiers` is true. */ scope?: AgentMemoryScope /** sha256 prefix of the canonical-JSON scope (keys sorted, undefined stripped). */ scopeHash?: string diff --git a/src/mutation-lock.ts b/src/mutation-lock.ts index 59532a2..5593ad8 100644 --- a/src/mutation-lock.ts +++ b/src/mutation-lock.ts @@ -33,7 +33,7 @@ export interface KnowledgeMutationLock { assertOwned(): void } -export interface KnowledgeMutationRecovery { +interface KnowledgeMutationRecovery { transactionId: string purpose: string direction: 'apply' | 'rollback' diff --git a/src/research-driving-driver.ts b/src/research-driving-driver.ts index 3e820c0..cf371d3 100644 --- a/src/research-driving-driver.ts +++ b/src/research-driving-driver.ts @@ -486,8 +486,7 @@ export function createResearchDrivingDriver( } } - // MECHANISM questions: for the best-supported claims, probe the failure - // boundary — under what precise condition does the asserted effect break. + // Probe where the best-supported claims stop holding. for (const claim of [...ledger] .sort((a, b) => b.supportingHosts.size - a.supportingHosts.size) .slice(0, 2)) { @@ -501,8 +500,7 @@ export function createResearchDrivingDriver( ) } - // COMPARATIVE questions: pair the two most-supported claims and ask how their - // tradeoffs differ. + // Compare tradeoffs between the two best-supported claims. const ranked = [...ledger].sort((a, b) => b.supportingHosts.size - a.supportingHosts.size) if (ranked.length >= 2 && ranked[0] && ranked[1]) { out.push( diff --git a/src/two-agent-research-loop.ts b/src/two-agent-research-loop.ts index 331d6e6..d9582a9 100644 --- a/src/two-agent-research-loop.ts +++ b/src/two-agent-research-loop.ts @@ -475,11 +475,6 @@ export function sourceMatchesGaps( return hits } -// ── Deprecated aliases ─────────────────────────────────────────────────────────── -// The old "TwoAgent" names described the MECHANISM (a proposer worker + a verifier driver) -// instead of the VALUE (research whose sources are verified before admission). Kept as -// aliases so existing importers do not break; prefer the `Verified*` names. - /** @deprecated Renamed to {@link runVerifiedResearchLoop}. */ export const runTwoAgentResearchLoop = runVerifiedResearchLoop /** @deprecated Renamed to {@link VerifiedResearchLoopOptions}. */ diff --git a/tests/changes.test.ts b/tests/changes.test.ts index c98b2da..46487a4 100644 --- a/tests/changes.test.ts +++ b/tests/changes.test.ts @@ -3,19 +3,6 @@ import { detectChanges } from '../src/changes' import { sha256 } from '../src/ids' import type { KnowledgeFragment } from '../src/sources/types' -/** - * Bug class each test defends against: - * - * - body-hash compared against itself ⇒ modifications go undetected. - * - unverifiable fragment treated as authoritative ⇒ false `removed` - * events fire when a captcha snapshot is compared to a real one. - * - dimension union dropping deduplication ⇒ eval scheduler re-runs the - * same campaign N times when a fragment hints overlap. - * - `filterDimensions` not narrowing the result ⇒ cron schedules - * campaigns it shouldn't. - * - duplicate ids silently shadowing without warning ⇒ upstream bugs - * get masked. - */ function fragment( id: string, body: string, diff --git a/tests/freshness.test.ts b/tests/freshness.test.ts index e061173..fd5db55 100644 --- a/tests/freshness.test.ts +++ b/tests/freshness.test.ts @@ -9,15 +9,6 @@ import { type FreshnessRecord, } from '../src/freshness' -/** - * Bug class each test defends against: - * - * - filesystem store reading stale in-memory state ⇒ cron re-fetches - * even after a successful mark. - * - tenants leaking across workspaces ⇒ multi-tenant data-isolation bug. - * - TTL miscompare (e.g. `>=` vs `>`) ⇒ off-by-one in cron scheduling. - * - D1 stub interface drift breaking production callers. - */ async function withTempRoot(fn: (root: string) => Promise): Promise { const root = await mkdtemp(join(tmpdir(), 'agent-knowledge-freshness-')) try { diff --git a/tests/sources-live.test.ts b/tests/sources-live.test.ts index 73bf9a4..b51ef51 100644 --- a/tests/sources-live.test.ts +++ b/tests/sources-live.test.ts @@ -10,31 +10,9 @@ import { } from '../src/sources/index' /** - * Live HTTP tests against real authorities (Cornell LII, IRS.gov, CA SOS). - * - * Gated on `AGENT_KNOWLEDGE_RUN_NETWORK_TESTS=1` because network tests in - * sandboxes without outbound connectivity (some CI setups) would otherwise - * be FALSE FAILURES rather than environmental skips. CI passes the flag. - * - * Rate-limit / block-page behaviour: the source contract guarantees - * `verifiable: false` with a reason rather than throwing. The tests below - * therefore SKIP (not fail) when an authority is unreachable or serving a - * block page — the unit-test layer already validates the success path on - * synthetic HTML; what these tests are checking is "the live shape we - * built against is still the live shape." That signal is preserved by - * skipping rather than failing in transient adverse conditions. - * - * Bug class each test defends against: - * - * - Cornell LII HTML re-skinning that breaks the section-text selector - * ⇒ statute body extraction silently returns navigation text. - * - IRS Drupal upgrade that changes the publications-index table markup - * ⇒ change detection floods the cron with phantom removals. - * - State SOS swapping CMS ⇒ wrong jurisdiction tag would feed into - * `KnowledgeChange.affectedDimensions` and re-run the wrong evals. - * - * Each test uses a 30s timeout, a per-test fresh cache dir, and resets - * the in-process throttle so order-of-execution doesn't matter. + * Optional network compatibility checks for Cornell LII, IRS.gov, and CA SOS. + * Each check uses a fresh cache and returns early when the authority reports + * that its response cannot be verified. */ const LIVE_ENABLED = process.env.AGENT_KNOWLEDGE_RUN_NETWORK_TESTS === '1' diff --git a/tests/sources-mocked.test.ts b/tests/sources-mocked.test.ts index 25936af..664bf59 100644 --- a/tests/sources-mocked.test.ts +++ b/tests/sources-mocked.test.ts @@ -9,21 +9,7 @@ import { createStateSosSource, } from '../src/sources/index' -/** - * Each source parses the live HTML shape of its authority. These tests - * mock fetch with HTML snippets that match the live structure (verified - * against real Cornell LII / IRS / CA SOS pages 2026-05-14) so the - * parsing logic is exercised without depending on network. - * - * Bug class each test defends against: - * - * - Cornell LII parser pulling navigation chrome into the statute body. - * - IRS parser missing the "Publication N (YYYY)" revision marker so - * `sourceUpdatedAt` falls back to fetch time and change detection - * stops noticing year-flips. - * - state-sos id-selector mishandling sibling tags so body extraction - * silently returns empty. - */ +/** Representative authority HTML keeps parser tests deterministic and offline. */ let cacheDir: string const originalFetch = globalThis.fetch diff --git a/tests/sources-types.test.ts b/tests/sources-types.test.ts index 0676c1b..4e2b4d8 100644 --- a/tests/sources-types.test.ts +++ b/tests/sources-types.test.ts @@ -8,18 +8,6 @@ import { looksLikeBlockPage, } from '../src/sources/index' -/** - * Pure-unit checks. No network. Bug class each test defends against: - * - * - factories returning wrong source id ⇒ freshness store keys break - * across releases. - * - block-page heuristic missing common interstitials ⇒ verifiable=true - * when it should be false, corrupting change detection. - * - htmlToText eating
separators ⇒ statute subsection structure - * collapses into one paragraph. - * - extractLinks accepting wrong-pattern hrefs ⇒ IRS index parser - * would catalogue ads / navigation links as publications. - */ describe('source factories', () => { it('cornell-lii default id is stable', () => { const source = createCornellLiiSource({ selectors: [{ kind: 'uscode', path: '18/1836' }] }) diff --git a/tsconfig.json b/tsconfig.json index a8b383f..e26f10d 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -4,6 +4,7 @@ "module": "ESNext", "moduleResolution": "bundler", "lib": ["ES2022"], + "types": ["node"], "strict": true, "esModuleInterop": true, "skipLibCheck": true, From 8d7569673d6ebf8df0ab72c0437a2b209ab310d4 Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Sat, 18 Jul 2026 11:46:16 -0600 Subject: [PATCH 5/5] docs(sources): preserve HTTP fetch invariants --- src/sources/http.ts | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/sources/http.ts b/src/sources/http.ts index e344074..fada4cc 100644 --- a/src/sources/http.ts +++ b/src/sources/http.ts @@ -5,8 +5,11 @@ import { sha256 } from '../ids' /** * Polite HTTP fetcher shared by remote sources. * - * Requests to one origin share a throttle, responses are cached by URL, - * and successful status codes are still checked for block pages. + * Independent sources share a per-origin throttle because rate-limited sites + * may return block pages instead of 429 responses. Responses are cached by URL + * because many publishers omit reliable ETag and Last-Modified headers. Bodies + * are checked even after a 2xx response because captcha and block pages often + * use successful status codes. */ /** User-Agent string sent on every outbound request. */