diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..5f9afd4 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,20 @@ +name: test + +# The first check surface this repo has had (#5 review, 2026-09-25). Ranking and migration +# are where a memory store fails silently -- nothing throws, the store just stops surfacing +# what it holds -- so they are pinned in tests/ and run on every PR. +on: + pull_request: + push: + branches: [main] + +jobs: + test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: 22 + - run: npm ci + - run: npm test diff --git a/package.json b/package.json index 97ae5cb..b1887a7 100644 --- a/package.json +++ b/package.json @@ -19,6 +19,7 @@ "scripts": { "build": "tsc", "dev": "tsc --watch", + "test": "npm run build && node --test tests/*.test.mjs", "postinstall": "npm run build || true", "prepublishOnly": "npm run build" }, diff --git a/src/db.ts b/src/db.ts index 9598aee..1627753 100644 --- a/src/db.ts +++ b/src/db.ts @@ -78,6 +78,7 @@ CREATE TABLE IF NOT EXISTS observations ( cwd TEXT, tags TEXT, content_hash TEXT, + shape_hash TEXT, scored_by TEXT, event_session_id TEXT ); @@ -477,6 +478,18 @@ export function openDatabase(path?: string): Database.Database { db.exec(`UPDATE patterns SET last_seen = COALESCE(updated_at, created_at, datetime('now')) WHERE last_seen IS NULL`); } catch { /* column already exists */ } + // observations.shape_hash — the repetition discount's key (see memory.ts `shapeHash`). + // The warning above is not historical: adding the column to SCHEMA alone does nothing for + // an existing database, because CREATE TABLE IF NOT EXISTS will not widen a table it + // already made, and insertObservation names the column — so prepareStatements throws + // `no such column: shape_hash` on every existing db and capture dies silently, exactly as + // it did for four days in 2026-06/07. Nullable ADD COLUMN, no backfill: rows written before + // the discount existed keep the salience they were given, and NULL is the honest value for + // "this row's shape was never computed". + try { + db.exec(`ALTER TABLE observations ADD COLUMN shape_hash TEXT`); + } catch { /* column already exists */ } + // Migration: base_salience — immutable IMPORTANCE, decoupled from the decaying `salience`. // decayObservations drives `salience` toward 0 for old rows, but search ranks by salience, so // important old memories became unfindable. Search now ranks by base_salience (importance); @@ -533,6 +546,9 @@ export function openDatabase(path?: string): Database.Database { // Index AFTER the ALTER, for the same reason base_salience's index is here: on a // pre-migration db the SCHEMA block runs before the column exists. db.exec(`CREATE INDEX IF NOT EXISTS idx_obs_content_hash ON observations(content_hash)`); + // Same rule, fourth time in this file: an index cannot precede its column. idx_obs_shape + // is created here, after the shape_hash ALTER above, never in SCHEMA. + db.exec(`CREATE INDEX IF NOT EXISTS idx_obs_shape ON observations(shape_hash)`); // Migration: seen_set.last_seen — enables recency-windowed novelty (prune stale tokens so novelty // doesn't saturate to 0 as the set grows). Backfill from first_seen. @@ -566,8 +582,14 @@ export function prepareStatements(db: Database.Database) { insertObservation: db.prepare(` INSERT INTO observations (session_id, tool_name, input_summary, output_summary, surprise, novelty, arousal, reward, conflict, salience, base_salience, cwd, tags, - content_hash, scored_by) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + content_hash, scored_by, shape_hash) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + `), + + // How many observations already share this SHAPE. Drives the repetition discount: + // the 684th copy of a wake prompt is not 684 times as important as the first. + countShape: db.prepare(` + SELECT COUNT(*) AS n FROM observations WHERE shape_hash = ? `), // Same insert WITH the event's own timestamp. Kept as a second statement rather diff --git a/src/memory.ts b/src/memory.ts index 0a38613..4adb504 100644 --- a/src/memory.ts +++ b/src/memory.ts @@ -16,6 +16,29 @@ import { createHash } from 'node:crypto'; /** Stable content hash for dedup — backs the observations.content_hash column (Kimi #4). */ const contentHash = (s: string): string => createHash('sha256').update(s).digest('hex'); +/** + * A hash of an observation's SHAPE — its text with the volatile parts masked. + * + * content_hash is exact, and exactness is useless against boilerplate that carries an id: + * measured 2026-09-25, the hestia wake prompt appears 684 times in one store with 684 + * DISTINCT content hashes, because each carries a fresh notice id and timestamp. The scorer + * saw novelty 0.7 on every one of them and filed all 684 at base_salience 0.9 — 37% of the + * whole store, one prompt, all of it ranked as if newly informative. + * + * Masking ids, hashes, timestamps and bare numbers gives repeated boilerplate a stable + * identity, so the discount below can see that it is the same thing again. + */ +const shapeHash = (toolName: string, input: string): string => { + const norm = (input || '') + .replace(/\b[0-9a-f]{8,}\b/gi, '#') // ids, shas, uuids + .replace(/\d{4}-\d{2}-\d{2}[T ][\d:.]+Z?/g, '#') // timestamps + .replace(/\b\d+\b/g, '#') // bare numbers + .replace(/\s+/g, ' ') + .trim() + .slice(0, 400); + return contentHash(toolName + '\x00' + norm); +}; + /** * Which scorer generation wrote a row — stamped into observations.scored_by. * @@ -183,6 +206,18 @@ export class SNARCMemory { const stored = scores.salience >= this.scorer.threshold; if (stored) { const tags = extractTags(toolName, inputSummary, outputSummary); + // REPETITION DISCOUNT. The Nth identical-shaped observation carries less information + // than the first; base_salience is what retrieval ranks by, so that is what falls. + // log2 rather than 1/n: the 2nd copy still matters, the 700th does not, and nothing + // is ever discounted to zero because the event did happen. + const shape = shapeHash(toolName, inputSummary); + let priorShapes = 0; + try { + priorShapes = (this.stmts.countShape.get(shape) as any)?.n ?? 0; + } catch { /* pre-migration database — no discount, same as before */ } + const discounted = priorShapes > 0 + ? Math.max(0.05, scores.salience / (1 + Math.log2(1 + priorShapes))) + : scores.salience; this.stmts.insertObservation.run( this.sessionId, toolName, @@ -194,11 +229,12 @@ export class SNARCMemory { scores.reward, scores.conflict, scores.salience, - scores.salience, // base_salience — immutable importance; `salience` (prev col) decays, this doesn't + discounted, // base_salience — importance. Discounted by how often this SHAPE recurred. cwd, JSON.stringify(tags), contentHash(toolName + '\x00' + inputSummary + '\x00' + outputSummary), SCORER_VERSION, + shape, ); } @@ -331,10 +367,24 @@ export class SNARCMemory { } } catch { /* FTS query syntax error — skip */ } - // Sort: patterns first (higher value), then by salience + // Sort: patterns and identity ABOVE raw observations, then by strength. + // + // The intent above ("patterns first") was right and the comparator did the opposite: + // `a.tier - b.tier` ascending puts Tier 1 first, and because Tier 1 holds 1,861 rows + // against Tier 2's 86, observations always filled `limit` and the slice below then + // discarded every pattern. Measured 2026-09-25: `search('prior art')` returned 5 results, + // 0 of them patterns, while the prepared statement for Tier 2 returned the "check for + // prior art: look for open PRs" pattern on the same query. The tier was fetched, ranked + // last, and thrown away — every time, for every query, while the tool advertised + // "search across all tiers". + const rank = (t: number) => (t === 2 ? 0 : t === 3 ? 1 : 2); results.sort((a, b) => { - if (a.tier !== b.tier) return a.tier - b.tier; // lower tier = higher value - return (b.salience || 0) - (a.salience || 0); + const ra = rank(a.tier), rb = rank(b.tier); + if (ra !== rb) return ra - rb; + // Patterns carry `confidence`, observations carry `salience`; compare like with like. + const sa = a.salience ?? a.confidence ?? 0; + const sb = b.salience ?? b.confidence ?? 0; + return sb - sa; }); return results.slice(0, limit); @@ -391,19 +441,66 @@ export class SNARCMemory { // Tier 2 patterns — INFERRED, only high-confidence (>= 0.6) // Exclude proposed_identity — those need human review before injection + // + // RANKED BY WHAT A PATTERN TEACHES, NOT BY HOW OFTEN ITS SHAPE RECURRED. getAllPatterns + // orders by `frequency DESC`, and the most frequent shapes are the conversational + // scaffolding: "Conversation → user_prompt → Conversation" at frequency 434. Every + // consolidated engineering lesson has frequency 1. So before this, the three briefing + // slots were structurally guaranteed to hold tautologies — measured on this database + // 2026-09-25, all three were Conversation/user_prompt cycles — while 80 deep_* patterns + // ("check for prior art: look for open PRs that touch it", "take a letter's date from + // the file's mtime") were never once surfaced. The seat then re-derived several of them + // at cost in a single day. Frequency is a measure of repetition, not of information. + const briefingRank = (p: any): number => { + const byKind: Record = { + deep_insight: 0, deep_error_fix: 0, deep_decision: 1, deep_workflow: 1, + concept_cluster: 3, tool_sequence: 4, + }; + return byKind[p.kind] ?? 2; + }; + // A tool_sequence made only of conversational turns encodes nothing: it says that talking + // is followed by talking. Drop it rather than rank it, so it cannot crowd a slot. + const isEmptySequence = (p: any): boolean => { + if (p.kind !== 'tool_sequence') return false; + const steps = String(p.summary || '').replace(/^.*workflow:\s*/i, '').split('→'); + return steps.every((s: string) => /^(conversation|user_prompt)$/i.test(s.trim())); + }; const patterns = this.getPatterns() - .filter((p: any) => p.confidence >= 0.6 && p.kind !== 'proposed_identity'); + .filter((p: any) => p.confidence >= 0.6 && p.kind !== 'proposed_identity') + .filter((p: any) => !isEmptySequence(p)) + .sort((a: any, b: any) => briefingRank(a) - briefingRank(b) || b.confidence - a.confidence); if (patterns.length > 0) { lines.push('Inferred patterns (heuristic — may not be accurate):'); - for (const p of patterns.slice(0, 3)) { + for (const p of patterns.slice(0, 5)) { lines.push(` - [${p.kind}] ${p.summary} (confidence: ${p.confidence.toFixed(2)})`); this.logRetrieval(cwd, 'briefing', 'pattern', p.confidence, `${p.summary} ${p.detail || ''}`); } } // Tier 1 observations — OBSERVED, above median salience (>= 0.35) - const recent = this.stmts.getRecentObservations.all(20) as any[]; - const highSalience = recent.filter((o: any) => o.salience >= 0.35); + // DEDUPED, because the loudest observations are the ones the harness repeats. A hestia + // mesh wake injects a near-identical prompt every time and it records at salience 1.000, + // so without this the three observation slots hold three copies of the same wake text — + // measured 2026-09-25. Keep the first of each near-duplicate and let the next distinct + // thing have the slot. + const recent = this.stmts.getRecentObservations.all(40) as any[]; + const seen = new Set(); + const highSalience = recent + .filter((o: any) => o.salience >= 0.35) + .filter((o: any) => { + // Normalise before comparing: the SAME wake text is recorded once as a user_prompt + // and again as a Conversation carrying a "[Human] " role tag, so a raw prefix key + // sees two distinct strings and keeps both. + const key = String(o.input_summary || '') + .replace(/^\s*\[[^\]]{1,20}\]\s*/, '') + .replace(/\s+/g, ' ') + .trim() + .slice(0, 60) + .toLowerCase(); + if (seen.has(key)) return false; + seen.add(key); + return true; + }); if (highSalience.length > 0) { lines.push('Recent observations (directly recorded):'); for (const o of highSalience.slice(0, 3)) { diff --git a/tests/ranking-and-migration.test.mjs b/tests/ranking-and-migration.test.mjs new file mode 100644 index 0000000..1af3387 --- /dev/null +++ b/tests/ranking-and-migration.test.mjs @@ -0,0 +1,195 @@ +// Regression pins for #5 (GPT seat's HOLD, 2026-09-25): the three memory behaviours this PR +// changes are load-bearing, so each is pinned against a THROWAWAY database in a temp dir. +// +// 1. migration of a pre-existing DB (no shape_hash column) -- and the ordering rule that +// the index on the new column is created only AFTER the guarded ALTER +// 2. repeated-shape discount, without changing the first occurrence +// 3. a matched Tier-2 pattern outranks Tier-1 observations in search +// 4. confidence-vs-salience secondary ordering among patterns +// 5. conversational-sequence filtering in the briefing (and wake-prompt dedupe) +// +// SNARC_DIST selects the build under test (default: this checkout's dist/). Pointing it at a +// build of the pre-PR code is how these were proven red-before. +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { mkdtempSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { pathToFileURL } from 'node:url'; +import Database from 'better-sqlite3'; + +const DIST = resolve(process.env.SNARC_DIST || new URL('../dist', import.meta.url).pathname); +const { SNARCMemory } = await import(pathToFileURL(join(DIST, 'src/memory.js')).href); + +// A standalone store: not under `/projects//`, so the cross-shard root claim store +// is never opened (db.ts openRootClaims returns null) and nothing outside the temp dir is touched. +function freshPath() { + return join(mkdtempSync(join(tmpdir(), 'snarc-test-')), 'snarc.db'); +} +function memoryAt(path) { + const m = new SNARCMemory(path); + m.initSession('test-session', '/tmp'); + return m; +} +const stmts = (m) => m.stmts; // plain JS: the private field is reachable, and names the real statement +function addPattern(m, kind, summary, frequency, confidence, detail = '') { + stmts(m).upsertPattern.run(kind, summary, detail, frequency, '[]', confidence); +} +function columns(path, table) { + const db = new Database(path, { readonly: true }); + try { return db.prepare(`PRAGMA table_info(${table})`).all().map((c) => c.name); } + finally { db.close(); } +} +function indexes(path, table) { + const db = new Database(path, { readonly: true }); + try { return db.prepare(`PRAGMA index_list(${table})`).all().map((i) => i.name); } + finally { db.close(); } +} + +// --------------------------------------------------------------------------------------------- +// 1. Migration of a pre-existing database. +// --------------------------------------------------------------------------------------------- +test('1. a pre-shape_hash database opens, gains the column and its index, and keeps its rows', () => { + const path = freshPath(); + // Build a store, then turn it into a PRE-PR store: drop the index and the column. What is left + // is the observations table as every existing snarc.db has it. + let m = memoryAt(path); + m.capture('Bash', 'git status in the repo', 'clean', '/tmp'); + m.close(); + { + const db = new Database(path); + db.exec('DROP INDEX IF EXISTS idx_obs_shape'); + db.exec('ALTER TABLE observations DROP COLUMN shape_hash'); + db.close(); + } + assert.ok(!columns(path, 'observations').includes('shape_hash'), 'fixture must lack the column'); + + // Reopen with the code under test. If the index on shape_hash were created in SCHEMA (before + // the guarded ALTER), this open would throw "no such column: shape_hash" -- the 2026-06/07 + // four-day capture outage shape. + m = memoryAt(path); + try { + assert.ok(columns(path, 'observations').includes('shape_hash'), 'migration added shape_hash'); + assert.ok(indexes(path, 'observations').includes('idx_obs_shape'), 'index created after the ALTER'); + const rows = stmts(m).getRecentObservations.all(10); + assert.equal(rows.length, 1, 'the pre-existing row survived'); + assert.equal(rows[0].shape_hash, null, 'no backfill: an old row honestly has no shape'); + // Capture must still work: insertObservation names shape_hash, so a missing column would + // throw here and capture would die silently in the hook. + const r = m.capture('Bash', 'git log --oneline', 'a1b2c3d commit', '/tmp'); + assert.equal(r.stored, true); + assert.equal(stmts(m).getRecentObservations.all(10).length, 2); + } finally { m.close(); } +}); + +// --------------------------------------------------------------------------------------------- +// 2. Repeated-shape discount, first occurrence unchanged. +// --------------------------------------------------------------------------------------------- +test('2. a repeated SHAPE is discounted; the first occurrence keeps its full salience', () => { + const m = memoryAt(freshPath()); + try { + // Same boilerplate, different volatile parts (id, sha, timestamp, count) -> distinct + // content_hash, one shape. This is the hestia wake prompt pattern. + const wake = (id, sha, ts, n) => + `You are Claude, woken by the mesh. notice id=${id} sha ${sha} queued_at=${ts} pending ${n}`; + m.capture('user_prompt', wake(13020, 'ffb18188781595dc', '2026-09-25T10:00:00Z', 3), '', '/tmp'); + const first = stmts(m).getRecentObservations.all(1)[0]; + m.capture('user_prompt', wake(13021, '0a1b2c3d4e5f6789', '2026-09-25T11:30:00Z', 7), '', '/tmp'); + const rows = stmts(m).getRecentObservations.all(10); + const second = rows.find((r) => r.id !== first.id); + const firstAfter = rows.find((r) => r.id === first.id); + + assert.equal(first.base_salience, first.salience, 'first occurrence is not discounted'); + assert.equal(firstAfter.base_salience, first.base_salience, 'a later copy does not rewrite the first'); + assert.ok(second.base_salience < second.salience, + `second copy discounted: base ${second.base_salience} vs salience ${second.salience}`); + assert.ok(second.base_salience >= 0.05, 'never discounted to zero -- the event happened'); + + // A DIFFERENT shape is not discounted. + m.capture('Bash', 'npm run build', 'ok', '/tmp'); + const other = stmts(m).getRecentObservations.all(1)[0]; + assert.equal(other.base_salience, other.salience, 'a distinct shape keeps full salience'); + } finally { m.close(); } +}); + +// --------------------------------------------------------------------------------------------- +// 3. A matched Tier-2 pattern outranks Tier-1 observations. +// --------------------------------------------------------------------------------------------- +test('3. search: a matched pattern is returned ahead of observations that fill the limit', () => { + const m = memoryAt(freshPath()); + try { + // More matching observations than the limit, as in the live store (1,861 obs vs 86 patterns), + // each at salience 0.9 -- HIGHER than the pattern's 0.7 confidence, so the pattern must win + // on tier, not on strength. captureContext bypasses the scorer, which (correctly) drops + // repeated same-tool captures below its threshold after the first two. + const topics = ['auth', 'billing', 'cache', 'deploy', 'export', 'fonts', 'gateway', 'hooks']; + for (const t of topics) { + m.captureContext('decision', `looked for prior art on the ${t} refactor before starting`, '/tmp', 0.9); + } + addPattern(m, 'deep_workflow', + 'Before fixing a bug, check for prior art: look for open PRs that touch it', 1, 0.7); + const res = m.search('prior', 5); + assert.equal(res.length, 5); + assert.equal(res[0].tier, 2, `first result is the pattern, got ${JSON.stringify(res.map((r) => r.tier))}`); + assert.ok(res.some((r) => r.tier === 2), 'the pattern survives the limit slice'); + } finally { m.close(); } +}); + +// --------------------------------------------------------------------------------------------- +// 4. Secondary ordering compares like with like: patterns by confidence. +// --------------------------------------------------------------------------------------------- +test('4. search: patterns are ordered by confidence (they carry no salience)', () => { + const m = memoryAt(freshPath()); + try { + // Inserted LOW first, so any ordering that ignores confidence (reading `salience`, which + // patterns do not have) leaves insertion/FTS order and puts the weak one first. + addPattern(m, 'deep_insight', 'widget calibration drifts under load (weak signal)', 1, 0.61); + addPattern(m, 'deep_insight', 'widget calibration must be re-run after a driver update', 1, 0.95); + const res = m.search('widget', 10).filter((r) => r.tier === 2); + assert.equal(res.length, 2); + assert.deepEqual(res.map((r) => r.confidence), [0.95, 0.61]); + } finally { m.close(); } +}); + +// --------------------------------------------------------------------------------------------- +// 5. Briefing: conversational tool_sequences are filtered; lessons are surfaced. +// --------------------------------------------------------------------------------------------- +test('5. briefing: talk-follows-talk sequences are dropped and a lesson takes the slot', () => { + const m = memoryAt(freshPath()); + try { + // The live shape: conversational scaffolding at huge frequency, every lesson at frequency 1. + addPattern(m, 'tool_sequence', 'Recurring workflow: Conversation → user_prompt → Conversation', 434, 0.9); + addPattern(m, 'tool_sequence', 'Recurring workflow: user_prompt → Conversation → user_prompt', 300, 0.9); + addPattern(m, 'tool_sequence', 'Recurring workflow: Conversation → Conversation → Conversation', 200, 0.9); + addPattern(m, 'deep_workflow', 'Take a letter\'s date from the file mtime, not the wake stamp', 1, 0.75); + const b = m.getSessionBriefing('/tmp'); + assert.match(b, /file mtime/, 'the lesson is surfaced'); + assert.doesNotMatch(b, /Conversation → user_prompt/, 'a talk-only sequence is not surfaced'); + assert.doesNotMatch(b, /Conversation → Conversation/); + } finally { m.close(); } +}); + +test('5b. briefing: a real tool sequence is kept; only all-conversational ones are dropped', () => { + const m = memoryAt(freshPath()); + try { + addPattern(m, 'tool_sequence', 'Recurring workflow: Read → Edit → Bash', 12, 0.9); + addPattern(m, 'tool_sequence', 'Recurring workflow: Conversation → user_prompt → Conversation', 434, 0.9); + const b = m.getSessionBriefing('/tmp'); + assert.match(b, /Read → Edit → Bash/); + assert.doesNotMatch(b, /Conversation → user_prompt/); + } finally { m.close(); } +}); + +test('5c. briefing: the same wake text recorded twice (role-tagged and not) fills one slot', () => { + const m = memoryAt(freshPath()); + try { + const wake = 'You are Claude (claude-code) on CBP, woken by the hestia member mesh. Pending notices'; + m.captureContext('user_prompt', wake, '/tmp', 1.0); + m.captureContext('Conversation', `[Human] ${wake}`, '/tmp', 1.0); + m.captureContext('decision', 'chose to pin the migration order in a test', '/tmp', 0.9); + const b = m.getSessionBriefing('/tmp'); + const copies = b.split('\n').filter((l) => l.includes('woken by the hestia member mesh')).length; + assert.equal(copies, 1, `wake text appears ${copies}x:\n${b}`); + assert.match(b, /pin the migration order/, 'the next distinct thing gets the slot'); + } finally { m.close(); } +});