-
Notifications
You must be signed in to change notification settings - Fork 13
Expand file tree
/
Copy pathengine.cjs
More file actions
5241 lines (4811 loc) · 249 KB
/
Copy pathengine.cjs
File metadata and controls
5241 lines (4811 loc) · 249 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
// SPDX-License-Identifier: AGPL-3.0-or-later
/**
* Constellation Engine v0.4 — R2: Tube Diameter Decay
* Star-map engine core: topological memory network
*
* R2 changes: Hebb strengthening on render, differential decay,
* endangered-node dreamCollide priority, identity/principle immunity
*
* API: remember / rememberRaw / render / forget / dream
* Storage: better-sqlite3 + sqlite-vec + BGE-M3 (via Mímir daemon)
* LLM: OpenAI-compatible endpoint
*/
const Database = require('better-sqlite3');
const path = require('path');
const fs = require('fs');
let liveBus = null;
try { liveBus = require('./src/live-bus.cjs'); } catch {}
const DB_PATH = path.join(__dirname, 'constellation.db');
const SCHEMA_PATH = path.join(__dirname, 'schema.sql');
const EMBED_DIM = 1024;
// Star map ownership (B6 migration 2026-04-21; Plan C2 ALS rewire 2026-04-25).
// Stamp on every nodes/edges INSERT. Resolved via this._resolveOwnerStamp(),
// which prefers ALS identity (set by main.js installing _identityResolver), then
// falls back to legacy this._currentUserOwnerId field (for direct CJS callers
// like scripts/* and CRON_INSTRUCTIONS), then to 'self'.
// Hardcoded 'self' — see src/user-identity.js STAR_MAP_OWNER for rationale.
const STAR_MAP_OWNER_ID_DEFAULT = 'self';
const TAXONOMY_PATH = path.join(__dirname, 'config', 'node_taxonomy.json');
// ── Tag taxonomy (loaded once at startup) ──
let _taxonomy = null;
try {
const taxRaw = JSON.parse(fs.readFileSync(TAXONOMY_PATH, 'utf8'));
const tier1 = new Set();
const allTags = new Set();
for (const cat of ['knowledge', 'behavior', 'system']) {
for (const d of (taxRaw.domains?.[cat] || [])) tier1.add(d.tier1_tag);
}
for (const [, tags] of Object.entries(taxRaw.tier2_tags || {})) {
if (Array.isArray(tags)) tags.forEach(t => allTags.add(t));
}
(taxRaw.tier3_tags?.tags || []).forEach(t => allTags.add(t));
(taxRaw.tier4_tags?.tags || []).forEach(t => allTags.add(t));
tier1.forEach(t => allTags.add(t));
// Build Tier 2 → Tier 1 reverse mapping for auto-inference
const tier2ToTier1 = {};
for (const [tier1Tag, tags] of Object.entries(taxRaw.tier2_tags || {})) {
if (Array.isArray(tags)) tags.forEach(t => { tier2ToTier1[t] = tier1Tag; });
}
_taxonomy = { tier1, allTags, behavior: new Set(taxRaw.ir_routing?.behavior_domains || []), tier2ToTier1 };
} catch (e) {
// taxonomy optional — runs without validation if missing
}
// LLM config for envelope generation
// Default: route through local OAuth gateway (same as runtime)
// Override with CONSTELLATION_LLM_* env vars if needed
const LLM_PROVIDER = process.env.CONSTELLATION_LLM_PROVIDER || 'openai-compat';
const LLM_BASE_URL = process.env.CONSTELLATION_LLM_URL || 'http://127.0.0.1:3456';
const LLM_API_KEY = process.env.CONSTELLATION_LLM_KEY || 'constellation-local';
const LLM_MODEL = process.env.CONSTELLATION_LLM_MODEL || '';
const CONSOLIDATION_MODEL = process.env.CONSTELLATION_CONSOLIDATION_MODEL || '';
const CONSOLIDATION_COSINE_THRESHOLD = 0.65; // cosine sim above this → send to consolidation model (lowered 0.70→0.65 2026-05-18 to catch real-duplicate band 0.65-0.70)
const CONSOLIDATION_ENABLED = process.env.CONSTELLATION_CONSOLIDATION !== '0';
// ── Timeline Merge (A4) — 4th consolidation verdict: same-topic arc merging ──
// Config source: config.json engine.timelineMerge.{enabled,maxSections,maxChars,windowDays,minGapHours}
// Defaults mirror locked R1 decisions: Q1=b(config flag), Q2=a(30d), Q4=a(6 sections)
let _engineConfigFile = null;
try { _engineConfigFile = JSON.parse(fs.readFileSync(path.join(__dirname, 'config.json'), 'utf8')); } catch {}
const _tmCfg = _engineConfigFile?.engine?.timelineMerge || {};
const TIMELINE_MERGE_ENABLED = _tmCfg.enabled !== false;
const TIMELINE_MERGE_MAX_SECTIONS = Number.isFinite(_tmCfg.maxSections) ? _tmCfg.maxSections : 6;
const TIMELINE_MERGE_MAX_CHARS = Number.isFinite(_tmCfg.maxChars) ? _tmCfg.maxChars : 12000;
const TIMELINE_MERGE_MIN_GAP_HOURS = Number.isFinite(_tmCfg.minGapHours) ? _tmCfg.minGapHours : 2;
function extractFirstJsonObject(text) {
const raw = String(text || '').trim();
if (!raw) return null;
let start = -1;
let depth = 0;
let inString = false;
let escaped = false;
for (let i = 0; i < raw.length; i++) {
const ch = raw[i];
if (start < 0) {
if (ch === '{') {
start = i;
depth = 1;
}
continue;
}
if (escaped) {
escaped = false;
continue;
}
if (ch === '\\') {
escaped = true;
continue;
}
if (ch === '"') {
inString = !inString;
continue;
}
if (inString) continue;
if (ch === '{') depth++;
else if (ch === '}') {
depth--;
if (depth === 0) return raw.slice(start, i + 1);
}
}
return null;
}
// Multi-SA edge whitelist — used by _callConsolidationJudge to validate the judge LLM's EDGE_TYPE output.
// Source of truth: engine-output/architecture-research/PLAN-MULTI-SA-REACTIVATION.md §4.1
// 23 types across 3 channels. Hallucinated types are dropped + logged; optional fallback to
// FALLBACK_COARSE (5 coarse types) keeps the connection signal recoverable.
const CONSOLIDATION_EDGE_WHITELIST = new Set([
// Knowledge channel (epistemic)
'causal', 'contrastive', 'hierarchical',
'supports', 'contradicts', 'causes',
'extends', 'synthesizes', 'challenges',
'contextualizes', 'contrasts',
// Language channel (stylistic / associative / narrative)
'associative', 'temporal',
'inspires', 'parallels', 'exemplifies', 'complements',
// Scaffold channel (procedural / structural) — formerly "Reflex", renamed 2026-04-16
'enables', 'triggers', 'depends_on',
'contains', 'supersedes', 'builds_on',
// Mímir tension-resolution: a synthesis node points to the two contradicting endpoints.
'resolves',
]);
// Edge Evolution v1 (2026-04-26): closed 35-fine-type subset per coarse type.
// Used by addEdges to validate optional fine_type, by updateEdgeFineType to gate refines,
// and exported on the class for worker reuse. NEVER read by Multi-SA channel routing —
// that path keys off edge_type (5 coarse) only. See architecture-research/2026-04-26-*.
const FINE_TYPES_BY_COARSE = {
causal: ['enables', 'prevents', 'requires', 'triggers', 'undermines', 'mitigates', 'explains'],
contrastive: ['contradicts', 'challenges', 'refines', 'narrows', 'generalizes', 'tension', 'alternative'],
hierarchical: ['contains', 'specializes', 'exemplifies', 'aggregates', 'decomposes', 'is_a', 'part_of'],
associative: ['co_occurs', 'reminiscent_of', 'inspires', 'resonates', 'parallels', 'evokes', 'contextualizes'],
temporal: ['precedes', 'follows', 'concurrent', 'triggers_next', 'culminates_in', 'preempts', 'recurs'],
};
const ALLOWED_FINE_SOURCE_PREFIXES = ['autonomous:mimir-', 'consolidation', 'manual'];
function _isFineSourceAllowed(src) {
if (!src) return false;
return ALLOWED_FINE_SOURCE_PREFIXES.some(p => src.startsWith(p));
}
// Per-type fusion cosine thresholds (from taxonomy attribute matrix)
// Higher threshold = harder to trigger fusion. Types not listed use default 0.70.
const FUSION_THRESHOLD_BY_TYPE = {
'identity': Infinity, // never fuse
'milestone': Infinity, // never fuse
'principle': Infinity, // never fuse (revision only)
'diary': Infinity, // never fuse (each entry is unique)
'experiment': Infinity, // never fuse (each experiment is independent)
'relationship': Infinity, // never fuse (in-place update only)
'profile-dim': Infinity, // never fuse — dialectic adds new dim, never overwrites (master plan §7)
'reflection': Infinity, // Mímir reflection — preserve each synthesis individually
'tension-resolution': Infinity, // Mímir tension synthesis — preserve each resolution individually
'social-rule': 0.90, // very high threshold
'language-template': 0.90, // very high threshold
'theory': 0.85, // low-frequency fusion
'reading-note': 0.85, // low-frequency fusion
'action': 0.85, // low-frequency fusion (step iteration)
'general-knowledge': 0.85, // almost never fuse
'introspection': 0.80, // moderate threshold
'decision': 0.75, // same decision chain can fuse
'engineering': 0.70, // standard — frequent supersede
'observation': 0.70, // standard — time-sensitive
'conversation-insight': 0.70, // standard
'interaction': 0.70, // explicit: INDEPENDENT-only via ALLOWED_OPS; kept at default so judge can confirm edge type
'knowledge': 0.70, // default
};
// Per-type allowed consolidation operations (Section 19.1 of master plan)
// Types not listed allow all operations. null = type uses Infinity threshold (never reaches judge).
const ALLOWED_OPS_BY_TYPE = {
'social-rule': ['INDEPENDENT'],
'language-template': ['INDEPENDENT'],
'general-knowledge': ['INDEPENDENT'],
'theory': ['FUSE', 'SUPERSEDE', 'TIMELINE_MERGE', 'INDEPENDENT'],
'reading-note': ['FUSE', 'TIMELINE_MERGE', 'INDEPENDENT'],
'introspection': ['FUSE', 'TIMELINE_MERGE', 'INDEPENDENT'],
'decision': ['SUPERSEDE', 'INDEPENDENT'],
'engineering': ['FUSE', 'SUPERSEDE', 'TIMELINE_MERGE', 'INDEPENDENT'],
'observation': ['FUSE', 'SUPERSEDE', 'TIMELINE_MERGE', 'INDEPENDENT'],
'conversation-insight': ['FUSE', 'SUPERSEDE', 'TIMELINE_MERGE', 'INDEPENDENT'],
'knowledge': ['FUSE', 'SUPERSEDE', 'INDEPENDENT'],
'action': ['INDEPENDENT'],
'interaction': ['INDEPENDENT'],
'profile-dim': ['INDEPENDENT'],
'reflection': ['INDEPENDENT'],
'tension-resolution': ['INDEPENDENT'],
'self_act': ['FUSE', 'TIMELINE_MERGE', 'INDEPENDENT'],
};
// Sources that are user-authored (not autonomous/debrief). Auto-supersede + dialectic
// supersede are blocked when target.source matches one of these AND superseder is not
// session-debrief / autonomous:mimir-*. Master plan §10: profile-update cluster collapse.
const USER_AUTHORED_SOURCE_PREFIXES = [
'session-write', 'manual', 'telegram:', 'dashboard:', 'cron:user',
];
const USER_AUTHORED_SOURCE_EXACT = new Set([
'session-write', 'manual', 'foreign:dashboard',
'telegram', 'dashboard', 'cron', // bare forms — telegram.js/dashboard.js/cron.js write these without colon
]);
function _isUserAuthoredSource(source) {
if (!source) return true; // NULL legacy rows treated as user-authored — defensive
if (USER_AUTHORED_SOURCE_EXACT.has(source)) return true;
return USER_AUTHORED_SOURCE_PREFIXES.some(p => source.startsWith(p));
}
class ConstellationEngine {
constructor(dbPath = DB_PATH) {
this.db = new Database(dbPath);
this.db.pragma('journal_mode = WAL');
this.db.pragma('busy_timeout = 5000'); // 5s — Mímir batch writes can hold lock for 1-3s; better to wait than throw
this.db.pragma('synchronous = NORMAL'); // fsync on commit only — prevents corruption from partial writes while avoiding FULL I/O cost
this._adjCache = null; // adjacency list cache for render() BFS
this._adjCacheVersion = 0; // incremented by remember()/dream() to invalidate cache
this._consolidationStats = { fuse: 0, supersede: 0, independent: 0, checked: 0, unfusable_skipped: 0, no_neighbors: 0, below_threshold: 0 };
this._consolidationLastHeartbeat = Date.now();
this._renderNodeFn = null; // lazy-loaded narrative-ir renderNode function
this._currentUserOwnerId = null; // legacy per-call fallback (CJS scripts only). ALS path preferred.
this._identityResolver = null; // installed by main.js: () => getStarMapOwnerId(getCurrentIdentity())
this.llmRouter = null; // installed by main.js after LLMRouter constructed; cold-start routes via setLLMRouter()
// access_count buffered increments — flushed every 30s out-of-band to avoid render-time lock contention
this._accessBumps = new Map(); // node_id -> count
this._accessFlushTimer = setInterval(() => this._flushAccessBumps(), 30_000);
if (this._accessFlushTimer.unref) this._accessFlushTimer.unref();
this._lastConsolidationSummary = { ...this._consolidationStats };
this._consolidationSummaryTimer = setInterval(() => {
try {
const cur = this._consolidationStats;
const prev = this._lastConsolidationSummary || {};
const delta = {
fuse: (cur.fuse || 0) - (prev.fuse || 0),
supersede: (cur.supersede || 0) - (prev.supersede || 0),
timeline_merge: (cur.timelineMerge || 0) - (prev.timelineMerge || 0),
independent: (cur.independent || 0) - (prev.independent || 0),
checked: (cur.checked || 0) - (prev.checked || 0),
};
const total = delta.fuse + delta.supersede + delta.timeline_merge + delta.independent + delta.checked;
if (total > 0 && liveBus) {
liveBus.safeEmit('engine.consolidation.summary', delta);
}
this._lastConsolidationSummary = { ...cur };
} catch {}
}, 60_000);
if (this._consolidationSummaryTimer.unref) this._consolidationSummaryTimer.unref();
// Boot banner — confirm consolidation wiring is loaded
const _consEnabled = process.env.CONSTELLATION_CONSOLIDATION !== '0';
console.log(`[Consolidation] ✓ ready (enabled=${_consEnabled}, silent path logged per-check)`);
// Cold-start dispatcher state (Phase 9.5).
// Architecture A: bootstrap loop lives in engine.cjs, not in mimir-js. The
// tick reads gate state and routes to bootstrap controller XOR steady-state
// v3 picker. mimir-js v1 hardcodes autonomy off, so the steady-state branch
// is a no-op until the picker is ported. State is engine-local; cross-restart
// continuity comes from engine_meta rows.
this._coldStart = {
lastPhase: null, // 'bootstrap' | 'steady' | 'expired' | null
tickCount: 0,
tickInProgress: false, // re-entrancy guard
lastTickAt: 0,
messagesCountCache: { value: null, ts: 0, ttlMs: 60_000 },
messagesCountResolver: null, // installed by main.js: () => convStore.db.prepare(...).get().c
mimirActionsResolver: null, // installed by main.js: ({sinceMs, limit}) => rows
llmExpansionInflight: false, // Q5/H3: only one LLM expansion job at a time
};
this._init();
// Verify daemon embed endpoint is reachable (non-blocking)
fetch(`http://127.0.0.1:${process.env.MIMIR_PORT || 18810}/embed`, {
method: 'POST', headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ text: 'ping' }),
}).then(() => console.log('[Engine] BGE-M3 embedder ready (via daemon)'))
.catch(() => console.warn('[Engine] Warning: daemon /embed not reachable yet'));
// Cold-start dispatcher tick (Phase 9.5). 60s cadence: gate evaluation is
// cheap, but the bootstrap controller fires LLM/HTTP work, so we keep it
// unhurried. ENGINE_COLD_START=0 disables (kill-switch); env unset = on.
if (process.env.ENGINE_COLD_START !== '0') {
this._coldStartTimer = setInterval(() => {
this._coldStartTick().catch(e => {
console.warn('[ColdStart] tick error:', e.message);
});
}, 60_000);
if (this._coldStartTimer.unref) this._coldStartTimer.unref();
}
// Lever B (2026-05-18) — periodic consolidation re-sweep.
// Fires every 6h, judges up to 30 pairs in the cosSim ∈ [type_threshold, 0.95)
// band that the write-time top-6 KNN missed or that drifted in after the
// threshold drop. Kill-switch: ENGINE_CONSOLIDATION_RESWEEP=0.
if (process.env.ENGINE_CONSOLIDATION_RESWEEP !== '0' && CONSOLIDATION_ENABLED) {
this._consolidationResweepTimer = setInterval(() => {
this._consolidationResweep({ windowDays: 30, maxPairs: 30 })
.catch(e => console.warn('[Resweep] tick err:', e.message));
}, 6 * 3600 * 1000);
if (this._consolidationResweepTimer.unref) this._consolidationResweepTimer.unref();
}
}
// ─── Owner scoping (B6 owner_id migration 2026-04-21, gated by ENGINE_OWNER_SCOPE) ──
// Phase B reader filter. Default OFF = legacy behaviour. When ON, accepts rows
// owned by current user OR shared ('*') OR legacy NULL (defensive).
_ownerScopeOn() {
return process.env.ENGINE_OWNER_SCOPE === '1';
}
/**
* Resolve the owner_id stamp for star-map writes (Plan C2, 2026-04-25).
* Precedence: ALS identity (via _identityResolver) → legacy _currentUserOwnerId
* → STAR_MAP_OWNER_ID_DEFAULT. Returns the *string* stamp, never null.
*/
_resolveOwnerStamp() {
if (this._identityResolver) {
try {
const stamp = this._identityResolver();
if (stamp) return stamp;
} catch { /* fall through to legacy */ }
}
return this._currentUserOwnerId || STAR_MAP_OWNER_ID_DEFAULT;
}
_activeOwner() {
return this._resolveOwnerStamp();
}
/**
* Master plan §10 — Mímir-driven supersede must NOT clobber user-written nodes.
* Block ONLY when an autonomous:mimir-* writer would supersede a user-authored target.
* All legacy paths (consolidation `knowledge`, `inference`, manual edges, debrief) pass through
* unchanged; only Mímir's own writes are constrained.
* Returns true = allowed, false = blocked.
*/
_isSupersedeAllowed(supersederSource, targetNodeId) {
if (!supersederSource || !supersederSource.startsWith('autonomous:mimir-')) {
return true; // legacy / human / debrief writers unaffected
}
try {
const row = this.db.prepare("SELECT source FROM nodes WHERE id = ?").get(targetNodeId);
if (!row) return true; // missing target — let SQL handle the no-op
// Mímir is allowed to supersede other Mímir output and engine-authored nodes
// (knowledge/inference/reflection). Block only on user-authored target.
return !_isUserAuthoredSource(row.source);
} catch {
return true; // never break writes on a guard-lookup failure
}
}
/**
* Post-fetch row filter. Use after SELECT * (which includes owner_id).
* For non-* projections, see _ownerSqlClause().
*/
_filterByOwner(rows) {
if (!this._ownerScopeOn() || !rows) return rows;
const owner = this._activeOwner();
const ok = (r) => !r || r.owner_id == null || r.owner_id === owner || r.owner_id === '*';
if (Array.isArray(rows)) return rows.filter(ok);
return ok(rows) ? rows : null;
}
/**
* SQL fragment + params for in-query filtering. Always returns a string fragment
* starting with " AND " (or empty string when off) so it can splice into existing WHERE.
* Pass an alias (e.g. 'n') when the query uses table aliases.
*/
_ownerSqlClause(alias = null) {
if (!this._ownerScopeOn()) return { sql: '', params: [] };
const col = alias ? `${alias}.owner_id` : 'owner_id';
return { sql: ` AND (${col} = ? OR ${col} = '*' OR ${col} IS NULL)`, params: [this._activeOwner()] };
}
/**
* Bi-temporal read filter (Phase 1b, 2026-04-27). Returns a SQL fragment that
* restricts edge reads to rows currently valid (`valid_to IS NULL`). State filter
* is intentionally NOT bundled — callers keep their own `state='active'` clauses,
* so dormant/diagnostic readers can drop just the bi-temporal half.
* Env: MIMIR_BITEMPORAL_READ_FILTER=off disables (default on).
* No bound params — the column is null-checked, not compared.
*/
_bitemporalSqlClause(alias = null) {
if (process.env.MIMIR_BITEMPORAL_READ_FILTER === 'off') return { sql: '', params: [] };
const col = alias ? `${alias}.valid_to` : 'valid_to';
return { sql: ` AND ${col} IS NULL`, params: [] };
}
/**
* Zombie-edge defense (r14, 2026-05-13). Canonical invariant:
* an edge is "live" iff state='active' AND both endpoint nodes are
* state='active' AND superseded_at IS NULL.
*
* Single SQL fragment so every count / read enforces the same rule and
* stays in sync with the cascade in _applySupersede/_applyFuse. Without it,
* conn_count drifts from /api/graph/node which drifts from sa.js, and the
* user sees "node has 3 edges but show-edges renders 0" type symptoms.
*
* Returns: SQL fragment to AND into an edges-table SELECT/UPDATE.
* `edgeAlias` is the edges-table alias; defaults to `'edges'` for the bare
* table case. Never call with `''` — the `nodes` table has a `source` column,
* so bare `source`/`target` in the EXISTS subquery would resolve to the inner
* `nodes.source` instead of the outer edge endpoint, making EXISTS always
* false (root cause of the 2026-05-14 r21 edges:0 dashboard regression).
*/
_validEdgeEndpointsSql(edgeAlias = 'edges') {
const ep = `${edgeAlias}.`;
return (
` AND EXISTS (SELECT 1 FROM nodes ns WHERE ns.id = ${ep}source AND ns.state = 'active' AND ns.superseded_at IS NULL)` +
` AND EXISTS (SELECT 1 FROM nodes nt WHERE nt.id = ${ep}target AND nt.state = 'active' AND nt.superseded_at IS NULL)`
);
}
/**
* Reactivate edges that were dormanted by an earlier endpoint going away, but
* whose endpoints are *now* both active again. Called from every node
* insert/replace path so a re-remember of a node that was previously
* superseded restores its edge web in one shot.
*/
_reactivateNodeEdges(nodeId) {
return this.db.prepare(`
UPDATE edges SET state = 'active'
WHERE state = 'dormant'
AND (source = ? OR target = ?)
AND source IN (SELECT id FROM nodes WHERE state = 'active' AND superseded_at IS NULL)
AND target IN (SELECT id FROM nodes WHERE state = 'active' AND superseded_at IS NULL)
`).run(nodeId, nodeId).changes;
}
/**
* Sweep edges still flagged active whose endpoints have drifted out of the
* live set (deleted, superseded outside a FUSE/SUPERSEDE path, or otherwise
* dormanted). Idempotent. Called once on engine boot and from the dream
* cycle so manual deletes / migration backfills don't leave zombies.
* Returns the number of edges dormanted (0 on a clean DB).
*/
_sweepOrphanEdges() {
try {
const res = this.db.prepare(`
UPDATE edges SET state = 'dormant'
WHERE state = 'active'
AND (
source NOT IN (SELECT id FROM nodes WHERE state = 'active' AND superseded_at IS NULL)
OR target NOT IN (SELECT id FROM nodes WHERE state = 'active' AND superseded_at IS NULL)
)
`).run();
if (res.changes > 0) {
console.log(`[Engine] _sweepOrphanEdges: dormanted ${res.changes} edges with dead endpoints`);
this._adjCacheVersion++;
}
return res.changes;
} catch (err) {
console.warn(`[Engine] _sweepOrphanEdges failed: ${err.message}`);
return 0;
}
}
/**
* Schema migration runner (2026-05-03). Reads scripts/migrations/NNNN-*.sql in
* sorted order and applies any with version > MAX(schema_version.version) inside
* a transaction. The schema.sql baseline counts as v1, so 0001-baseline.sql is
* a no-op marker that just stamps version=1.
*
* On failure, throws an Error with .migrationFailure=true so src/main.js can
* exit(78) and let electron/main.js show a recovery dialog instead of the
* generic crash modal. We never auto-rollback partial migrations — better to
* leave the user with a clear "migration X failed, see logs" than to mask
* data loss with a silent retry.
*/
_runMigrations() {
const migrationsDir = path.join(__dirname, 'scripts', 'migrations');
if (!fs.existsSync(migrationsDir)) {
// Packaging bug if this fires in production — fresh installs need the
// baseline marker. Loud warn so it shows up in launcher log capture.
console.warn(`[Engine] scripts/migrations/ missing at ${migrationsDir} — skipping migration chain (packaging issue?)`);
return;
}
// The runner owns schema_version. schema.sql intentionally does NOT create
// it — keeps the bookkeeping responsibility in one place.
this.db.exec(`
CREATE TABLE IF NOT EXISTS schema_version (
version INTEGER PRIMARY KEY,
applied_at DATETIME DEFAULT CURRENT_TIMESTAMP,
description TEXT
)
`);
let currentVersion = 0;
try {
const row = this.db.prepare('SELECT COALESCE(MAX(version), 0) AS v FROM schema_version').get();
currentVersion = Number(row?.v) || 0;
} catch (e) {
const err = new Error(`schema_version read failed: ${e.message}`);
err.migrationFailure = true;
throw err;
}
const files = fs.readdirSync(migrationsDir)
.filter(f => /^\d{4}-.+\.sql$/.test(f))
.sort();
for (const f of files) {
const m = f.match(/^(\d{4})-(.+)\.sql$/);
const version = parseInt(m[1], 10);
if (version <= currentVersion) continue;
const description = m[2].replace(/\.sql$/, '').replace(/[-_]/g, ' ');
const sql = fs.readFileSync(path.join(migrationsDir, f), 'utf-8');
const apply = this.db.transaction(() => {
this.db.exec(sql);
this.db.prepare(
'INSERT INTO schema_version (version, description) VALUES (?, ?)'
).run(version, description);
});
try {
console.log(`[Engine] Applying migration ${f}...`);
apply();
console.log(`[Engine] Migration ${f} applied (v${version}).`);
} catch (e) {
const err = new Error(`Migration ${f} failed: ${e.message}`);
err.migrationFailure = true;
err.migrationFile = f;
err.migrationVersion = version;
err.original = e;
throw err;
}
}
}
_init() {
// Load sqlite-vec extension
const sqliteVec = require('sqlite-vec');
sqliteVec.load(this.db);
// Apply schema (exec handles multiple statements including BEGIN...END triggers)
const schema = fs.readFileSync(SCHEMA_PATH, 'utf-8');
this.db.exec(schema);
// Schema migration chain (2026-05-03). schema.sql is the v0.1.0 baseline; every
// future schema change ships as a numbered file in scripts/migrations/. Throws
// a tagged error on failure so src/main.js can exit(78) → electron/main.js
// surfaces a recovery dialog instead of a generic crash modal.
this._runMigrations();
// Cold-start substrate (Phase 9.0): stamp first_run_at exactly once.
// INSERT OR IGNORE preserves the original epoch across restarts. Cold-start
// gate (engine.cjs Phase 9.5) reads this to compute the 30d hard exit.
// autonomy_enabled_at stamped separately on first autonomy toggle.
try {
this.db.prepare(
"INSERT OR IGNORE INTO engine_meta (key, value) VALUES ('first_run_at', CAST(strftime('%s','now')*1000 AS TEXT))"
).run();
} catch (e) {
console.warn('[Engine] first_run_at stamp failed:', e.message);
}
// Create vec0 virtual table — migrate from 384d to 1024d if needed
try {
this.db.exec(`CREATE VIRTUAL TABLE node_embeddings USING vec0(id integer primary key, embedding float[${EMBED_DIM}])`);
} catch (e) {
if (e.message.includes('already exists')) {
// Check if dimension mismatch (migration from MiniLM 384d to BGE-M3 1024d)
try {
const row = this.db.prepare('SELECT embedding FROM node_embeddings LIMIT 1').get();
if (row && row.embedding && row.embedding.length !== EMBED_DIM * 4) {
console.log(`[Engine] Migrating vec0 table from ${row.embedding.length / 4}d to ${EMBED_DIM}d...`);
this.db.exec('DROP TABLE node_embeddings');
this.db.exec(`CREATE VIRTUAL TABLE node_embeddings USING vec0(id integer primary key, embedding float[${EMBED_DIM}])`);
console.log(`[Engine] vec0 table recreated with ${EMBED_DIM}d. Nodes need re-embedding.`);
}
} catch (migErr) {
// Empty table or other issue — table exists and is fine
console.log('[Engine] vec0 table exists, no migration needed or table empty.');
}
} else {
throw e;
}
}
// Optimization 3: ensure (target, state) composite index exists for incoming-edge queries
this.db.exec("CREATE INDEX IF NOT EXISTS idx_edges_target_state ON edges(target, state)");
// Consolidation verdict log (additive — separate from nodes.superseded_*; survives node dormancy)
// Captures every FUSE/SUPERSEDE/TIMELINE_MERGE/INDEPENDENT decision with both ids + verdict + ts.
// Read by dashboard Recent Activity ("old → new" jumps) and audit/replay tooling.
this.db.exec(`
CREATE TABLE IF NOT EXISTS consolidation_log (
id INTEGER PRIMARY KEY AUTOINCREMENT,
verdict TEXT NOT NULL,
new_node_id TEXT,
old_node_id TEXT,
new_l0 TEXT,
old_l0 TEXT,
cosine REAL,
reason TEXT,
created_at TEXT NOT NULL DEFAULT (datetime('now'))
)
`);
this.db.exec("CREATE INDEX IF NOT EXISTS idx_consolidation_log_created ON consolidation_log(created_at DESC)");
// Phase 9.3 — Launcher OS-notification outbox.
// Internal callers (bootstrap fetch/outreach, Mímir error surfaces) push
// a row; the Electron launcher polls /api/launcher/notifications/dequeue
// every 15s and fires a real OS notification per row, then deletes it.
// delivered_at is set the moment we hand it to the launcher (poll model);
// rows older than 24h get reaped on engine boot to keep the table small.
this.db.exec(`
CREATE TABLE IF NOT EXISTS notification_outbox (
id INTEGER PRIMARY KEY AUTOINCREMENT,
kind TEXT NOT NULL,
title TEXT NOT NULL,
body TEXT NOT NULL,
deeplink TEXT,
created_at TEXT NOT NULL DEFAULT (datetime('now')),
delivered_at TEXT
)
`);
this.db.exec("CREATE INDEX IF NOT EXISTS idx_notification_outbox_created ON notification_outbox(delivered_at, created_at)");
try {
this.db.prepare("DELETE FROM notification_outbox WHERE created_at < datetime('now','-24 hours')").run();
} catch {}
// ── Edge Evolution v1 (2026-04-26) ──
// Dual-storage fine_type richness layer. edge_type stays 5-coarse (Multi-SA channel
// routing depends on it); fine_type/fine_confidence/fine_source are additive richness
// populated by Curation mode + consolidation. NEVER read by SA channel routing.
// See engine-output/architecture-research/2026-04-26-edge-evolution-deployment-plan.md
for (const stmt of [
"ALTER TABLE edges ADD COLUMN fine_type TEXT DEFAULT NULL",
"ALTER TABLE edges ADD COLUMN fine_confidence REAL DEFAULT NULL",
"ALTER TABLE edges ADD COLUMN fine_source TEXT DEFAULT NULL",
]) {
try { this.db.exec(stmt); } catch (e) {
if (!String(e.message).includes('duplicate column')) throw e;
}
}
this.db.exec("CREATE INDEX IF NOT EXISTS idx_edges_fine_type ON edges(fine_type) WHERE fine_type IS NOT NULL");
// ── Event time column (2026-04-26) ──
// event_at = source-time the content describes (e.g. yesterday's diary written today).
// Distinct from created_at (wall-clock write time). NULL = caller didn't specify;
// dashboard falls back to created_at for display in that case.
try { this.db.exec("ALTER TABLE nodes ADD COLUMN event_at TEXT DEFAULT NULL"); }
catch (e) { if (!String(e.message).includes('duplicate column')) throw e; }
this.db.exec("CREATE INDEX IF NOT EXISTS idx_nodes_event_at ON nodes(event_at) WHERE event_at IS NOT NULL");
// ── Memory Migration Importer batch tag (2026-04-29) ──
// Stamped by scripts/tools/migrate_memory.py; NULL = organic node.
// Drives SA pool soft-suppression (mimir_daemon: 0.4x while access_count<5)
// and rollback (--rollback-batch). Schema mirrored in OSS schema.sql.
try { this.db.exec("ALTER TABLE nodes ADD COLUMN imported_batch_id TEXT"); }
catch (e) { if (!String(e.message).includes('duplicate column')) throw e; }
this.db.exec("CREATE INDEX IF NOT EXISTS idx_nodes_imported_batch ON nodes(imported_batch_id) WHERE imported_batch_id IS NOT NULL");
// fine_type proposals table — when LLM suggests a fine_type outside the 35-subset,
// we collect it here for periodic dictionary expansion review (user approves manually).
// Lives in constellation.db because it's a star-map taxonomy artifact.
// Audit (mimir_edge_changes) and action cooldowns (mimir_edge_action_cooldowns)
// live in conversations.db alongside other mimir_* worker-owned tables.
this.db.exec(`
CREATE TABLE IF NOT EXISTS fine_type_proposals (
id INTEGER PRIMARY KEY AUTOINCREMENT,
coarse_type TEXT NOT NULL,
proposed_fine TEXT NOT NULL,
count INTEGER NOT NULL DEFAULT 1,
first_seen TEXT NOT NULL DEFAULT (datetime('now')),
last_seen TEXT NOT NULL DEFAULT (datetime('now')),
approved INTEGER NOT NULL DEFAULT 0,
example_edge_ids TEXT,
UNIQUE(coarse_type, proposed_fine)
)
`);
// Drop legacy idx_edges_state_strength — it lured the planner into full-partition
// scans (37x BFS slowdown at 187K edges). All BFS queries now pin source/target
// indexes explicitly via INDEXED BY, so this index has no legitimate user.
try { this.db.exec('DROP INDEX IF EXISTS idx_edges_state_strength'); } catch {}
// Without sqlite_stat1 the planner picked idx_edges_state_strength over
// idx_edges_source for BFS — 37x slowdown at 187K edges. Run ANALYZE on
// first init (and any time stats go missing) so OSS user avoid this.
try {
const hasStats = this.db.prepare("SELECT name FROM sqlite_master WHERE name='sqlite_stat1'").get();
if (!hasStats) {
const t0 = Date.now();
this.db.exec('ANALYZE');
console.log(`[Engine] ANALYZE done in ${Date.now() - t0}ms (planner stats seeded)`);
}
} catch (e) {
console.warn('[Engine] ANALYZE skipped:', e.message);
}
// ── Ratatoskr pulse_hint_log ──
// Append-only envelope log for L0 self-touch pulse kinds (task / cognitive).
// Used by writers to surface task-completion signals + cognitive observations
// back to dashboards and Anamnesis elide.
this.db.exec(`
CREATE TABLE IF NOT EXISTS pulse_hint_log (
id INTEGER PRIMARY KEY AUTOINCREMENT,
received_at INTEGER NOT NULL,
kind TEXT NOT NULL,
source_hint TEXT,
owner_id TEXT,
target_kind TEXT,
target_id TEXT,
payload TEXT,
severity TEXT,
processed_at INTEGER,
processed_by TEXT
)
`);
this.db.exec("CREATE INDEX IF NOT EXISTS idx_phl_unprocessed ON pulse_hint_log(processed_at) WHERE processed_at IS NULL");
this.db.exec("CREATE INDEX IF NOT EXISTS idx_phl_target ON pulse_hint_log(target_kind, target_id)");
this.db.exec("CREATE INDEX IF NOT EXISTS idx_phl_received ON pulse_hint_log(received_at)");
// Boot-time TTL prune: pulse_hint_log is an audit log, useful only for the
// Anamnesis elide window (~24h) plus debugging breathing room. Anything
// older than 30 days is dead weight — drop it once per boot.
try {
const cutoff = Date.now() - 30 * 24 * 3600 * 1000;
const r = this.db.prepare("DELETE FROM pulse_hint_log WHERE received_at < ?").run(cutoff);
if (r.changes > 0) console.log(`[pulse_hint_log] pruned ${r.changes} row(s) older than 30d`);
} catch (e) { /* boot prune best-effort */ }
// ── Sleipnir (2026-04-29) — experiential anchor pattern v2 ──
// Plan: engine-output/architecture-research/2026-04-29-experiential-anchor-planning-v2.md
//
// Three tables:
// 1. exploration_trail — raw grep/read/web/autonomy events (TTL 7d)
// 2. experiential_pending_review — LLM-aggregated proposals waiting decision (cap 200 FIFO)
// 3. sleipnir_metrics — hourly tallies for dashboard panel
// Plus: nodes.subtype column for exploration_anchor classification
// (factual / navigational / conceptual) and task_trail subtype.
// 1. nodes.subtype — fine-grained classification under subkind='exploration_anchor'
try {
this.db.exec("ALTER TABLE nodes ADD COLUMN subtype TEXT");
console.log("[sleipnir] added nodes.subtype column");
} catch (e) { /* already exists — idempotent */ }
// 2. exploration_trail — raw events before LLM aggregation
this.db.exec(`
CREATE TABLE IF NOT EXISTS exploration_trail (
id INTEGER PRIMARY KEY AUTOINCREMENT,
occurred_at INTEGER NOT NULL,
caller_kind TEXT NOT NULL,
caller_session TEXT,
cron_name TEXT,
source_kind TEXT NOT NULL,
region TEXT,
query TEXT,
finding TEXT,
signature TEXT,
gate_decision TEXT,
metadata TEXT,
promoted INTEGER DEFAULT 0
)
`);
this.db.exec("CREATE INDEX IF NOT EXISTS idx_extrl_signature ON exploration_trail(signature, occurred_at)");
this.db.exec("CREATE INDEX IF NOT EXISTS idx_extrl_region ON exploration_trail(region)");
this.db.exec("CREATE INDEX IF NOT EXISTS idx_extrl_occurred ON exploration_trail(occurred_at)");
this.db.exec("CREATE INDEX IF NOT EXISTS idx_extrl_promoted ON exploration_trail(promoted, occurred_at) WHERE promoted = 1");
// Aggregator cursor — set when the LLM has consumed a row into a candidate.
try { this.db.exec("ALTER TABLE exploration_trail ADD COLUMN processed_at INTEGER"); } catch { /* idempotent */ }
this.db.exec("CREATE INDEX IF NOT EXISTS idx_extrl_unprocessed ON exploration_trail(promoted, processed_at) WHERE promoted = 1 AND processed_at IS NULL");
// Step 6 Plan A (2026-04-29) — raw text capture for hybrid storage. Caller
// populates these from tool result; aggregator passes through to
// experiential_pending_review; promoter splits raw_excerpt into chunks
// for experiential_raw side table.
try { this.db.exec("ALTER TABLE exploration_trail ADD COLUMN raw_excerpt TEXT"); } catch { /* idempotent */ }
try { this.db.exec("ALTER TABLE exploration_trail ADD COLUMN raw_line_range TEXT"); } catch { /* idempotent */ }
try { this.db.exec("ALTER TABLE exploration_trail ADD COLUMN raw_file_path TEXT"); } catch { /* idempotent */ }
// 3. experiential_pending_review — aggregator output queue
this.db.exec(`
CREATE TABLE IF NOT EXISTS experiential_pending_review (
review_id TEXT PRIMARY KEY,
proposed_at INTEGER NOT NULL,
proposed_by TEXT,
candidate_id TEXT,
l0 TEXT,
l1 TEXT,
l2 TEXT,
subtype TEXT,
trail_ids TEXT,
resolver_verdict TEXT,
cos_dedup_score REAL,
state TEXT DEFAULT 'pending',
expires_at INTEGER,
notes TEXT
)
`);
// Step 5: persist candidate embedding so IR injection can do cosine match
// without re-embedding at every turn. Step 4 dedup writes it, Step 5 reads.
try { this.db.exec("ALTER TABLE experiential_pending_review ADD COLUMN embedding BLOB"); } catch { /* idempotent */ }
// Step 6: decay channels — touch counter, last refresh, effective strength.
// effective_strength starts at confidence and decays adaptively (half-life
// inversely proportional to conf). Anchors below MIN_STRENGTH get aged out.
try { this.db.exec("ALTER TABLE experiential_pending_review ADD COLUMN last_refreshed_at INTEGER"); } catch { /* idempotent */ }
try { this.db.exec("ALTER TABLE experiential_pending_review ADD COLUMN refresh_count INTEGER DEFAULT 0"); } catch { /* idempotent */ }
try { this.db.exec("ALTER TABLE experiential_pending_review ADD COLUMN effective_strength REAL"); } catch { /* idempotent */ }
try { this.db.exec("ALTER TABLE experiential_pending_review ADD COLUMN region TEXT"); } catch { /* idempotent */ }
this.db.exec("CREATE INDEX IF NOT EXISTS idx_epr_state ON experiential_pending_review(state, proposed_at)");
this.db.exec("CREATE INDEX IF NOT EXISTS idx_epr_expires ON experiential_pending_review(expires_at) WHERE expires_at IS NOT NULL");
this.db.exec("CREATE INDEX IF NOT EXISTS idx_epr_accepted ON experiential_pending_review(state, subtype) WHERE state = 'accepted'");
this.db.exec("CREATE INDEX IF NOT EXISTS idx_epr_strength ON experiential_pending_review(state, effective_strength) WHERE state = 'accepted'");
// 4. sleipnir_metrics — hourly tallies (silent_drop / trail_only / promote / task_trail)
this.db.exec(`
CREATE TABLE IF NOT EXISTS sleipnir_metrics (
bucket_hour INTEGER PRIMARY KEY,
silent_drop INTEGER DEFAULT 0,
trail_only INTEGER DEFAULT 0,
promote INTEGER DEFAULT 0,
task_trail INTEGER DEFAULT 0,
redaction_hits INTEGER DEFAULT 0,
caller_subagent INTEGER DEFAULT 0
)
`);
// Boot-time TTL prune for exploration_trail: 7 days for non-promoted,
// 30 days for promoted (kept longer since they may feed aggregator retries).
try {
const cutoff7d = Date.now() - 7 * 24 * 3600 * 1000;
const cutoff30d = Date.now() - 30 * 24 * 3600 * 1000;
const r1 = this.db.prepare("DELETE FROM exploration_trail WHERE promoted = 0 AND occurred_at < ?").run(cutoff7d);
const r2 = this.db.prepare("DELETE FROM exploration_trail WHERE promoted = 1 AND occurred_at < ?").run(cutoff30d);
if (r1.changes + r2.changes > 0) console.log(`[sleipnir] pruned ${r1.changes} non-promoted (>7d) + ${r2.changes} promoted (>30d) trail rows`);
} catch (e) { /* boot prune best-effort */ }
// Boot-time prune for experiential_pending_review: drop expired
try {
const r = this.db.prepare("DELETE FROM experiential_pending_review WHERE expires_at IS NOT NULL AND expires_at < ?").run(Date.now());
if (r.changes > 0) console.log(`[sleipnir] pruned ${r.changes} expired pending review row(s)`);
} catch (e) { /* boot prune best-effort */ }
// FIFO cap on experiential_pending_review: keep only newest 200 pending
try {
const r = this.db.prepare(`
DELETE FROM experiential_pending_review
WHERE state = 'pending' AND review_id IN (
SELECT review_id FROM experiential_pending_review
WHERE state = 'pending'
ORDER BY proposed_at DESC
LIMIT -1 OFFSET 200
)
`).run();
if (r.changes > 0) console.log(`[sleipnir] FIFO-capped pending_review: dropped ${r.changes} oldest`);
} catch (e) { /* best-effort */ }
// ── Sleipnir Step 6 (2026-04-29) — hybrid promotion ──
// Plan: engine-output/architecture-research/2026-04-29-sleipnir-step6-hybrid-planning.md
// experiential_raw — chunked raw excerpts (kept out of nodes.l2 to
// avoid cosine pollution / BLOB inflation)
// sleipnir_promote_log — promote audit + daily-cap accounting
// Plus 6 ALTER columns on experiential_pending_review (raw side metadata +
// promoted_node_id link + accepted_expires_at TTL).
this.db.exec(`
CREATE TABLE IF NOT EXISTS experiential_raw (
node_id TEXT NOT NULL,
chunk_idx INTEGER NOT NULL DEFAULT 0,
total_chunks INTEGER NOT NULL DEFAULT 1,
source_kind TEXT,
file_path TEXT,
line_range TEXT,
byte_offset INTEGER,
raw_text TEXT NOT NULL,
created_at INTEGER NOT NULL,
PRIMARY KEY (node_id, chunk_idx),
FOREIGN KEY (node_id) REFERENCES nodes(id) ON DELETE CASCADE
)
`);
this.db.exec("CREATE INDEX IF NOT EXISTS idx_experiential_raw_node ON experiential_raw(node_id)");
this.db.exec("CREATE INDEX IF NOT EXISTS idx_experiential_raw_kind ON experiential_raw(source_kind, created_at DESC)");
try { this.db.exec("ALTER TABLE experiential_pending_review ADD COLUMN raw_excerpt TEXT"); } catch { /* idempotent */ }
try { this.db.exec("ALTER TABLE experiential_pending_review ADD COLUMN raw_line_range TEXT"); } catch { /* idempotent */ }
try { this.db.exec("ALTER TABLE experiential_pending_review ADD COLUMN raw_file_path TEXT"); } catch { /* idempotent */ }
try { this.db.exec("ALTER TABLE experiential_pending_review ADD COLUMN promoted_node_id TEXT"); } catch { /* idempotent */ }
try { this.db.exec("ALTER TABLE experiential_pending_review ADD COLUMN promoted_at INTEGER"); } catch { /* idempotent */ }
try { this.db.exec("ALTER TABLE experiential_pending_review ADD COLUMN accepted_expires_at INTEGER"); } catch { /* idempotent */ }
this.db.exec("CREATE INDEX IF NOT EXISTS idx_epr_promoted ON experiential_pending_review(promoted_node_id) WHERE promoted_node_id IS NOT NULL");
this.db.exec(`
CREATE TABLE IF NOT EXISTS sleipnir_promote_log (
id INTEGER PRIMARY KEY AUTOINCREMENT,
review_id TEXT NOT NULL,
promoted_node_id TEXT,
decision TEXT NOT NULL,
reason TEXT,
cos_max_neighbor REAL,
edges_written INTEGER,
raw_chunks INTEGER,
created_at INTEGER NOT NULL
)
`);
this.db.exec("CREATE INDEX IF NOT EXISTS idx_sleipnir_promote_log_recent ON sleipnir_promote_log(created_at DESC)");
// ─── V5b Phase 11 (OSS migration): persona / outreach substrate ────────
// Schema parity with main arch (Plan §6 Phase 7). All idempotent. Critic
// gate (Phase 11.3) and review-queue UI (Phase 11.4) consume these tables.
// OSS posture: post/reply default-OFF; review_queue is the only path until
// a Critic LLM is configured AND `direct_send_enabled=1` is flipped.
try { this.db.exec("ALTER TABLE nodes ADD COLUMN persona_id TEXT"); } catch { /* idempotent */ }
try { this.db.exec("ALTER TABLE nodes ADD COLUMN external_source_uri TEXT"); } catch { /* idempotent */ }
try { this.db.exec("ALTER TABLE edges ADD COLUMN persona_id TEXT"); } catch { /* idempotent */ }
this.db.exec(`
CREATE TABLE IF NOT EXISTS personas (
owner_id TEXT NOT NULL,
id TEXT NOT NULL,
display_name TEXT NOT NULL,
voice_rubric TEXT,
voice_exemplars TEXT,
created_at INTEGER NOT NULL,
active INTEGER DEFAULT 1,
PRIMARY KEY (owner_id, id)
)
`);
this.db.exec(`
CREATE TABLE IF NOT EXISTS persona_caps (
owner_id TEXT NOT NULL,
persona_id TEXT NOT NULL,
platform TEXT NOT NULL,
action TEXT NOT NULL,
daily_cap INTEGER NOT NULL,
quiet_start_hour INTEGER,
quiet_end_hour INTEGER,
quiet_tz TEXT,
direct_send_enabled INTEGER DEFAULT 1,
PRIMARY KEY (owner_id, persona_id, platform, action)
)
`);
try { this.db.exec("ALTER TABLE persona_caps ADD COLUMN direct_send_enabled INTEGER DEFAULT 1"); } catch { /* idempotent */ }
// r20 Option B: direct_send is permanently ON in OSS — the review-queue
// workflow was removed (panel + endpoints + write paths). Hard-lock every
// boot so legacy 0-rows (and any rogue writes) snap back to 1. The Critic
// gate still runs and unsafe drafts still drop.
try { this.db.prepare("UPDATE persona_caps SET direct_send_enabled = 1 WHERE direct_send_enabled != 1").run(); } catch { /* table may not exist on very old DBs */ }
this.db.exec(`
CREATE TABLE IF NOT EXISTS outreach_review_queue (
id INTEGER PRIMARY KEY AUTOINCREMENT,
owner_id TEXT NOT NULL,
persona_id TEXT NOT NULL,
platform TEXT NOT NULL,
action TEXT NOT NULL,
draft_text TEXT NOT NULL,
draft_hash TEXT NOT NULL,
parent_ref TEXT,
critic_result TEXT,
created_at INTEGER NOT NULL,
approved_at INTEGER,
rejected_at INTEGER,
sent_at INTEGER
)
`);
// Critic verdict log — one row per criticGate(Async) call. Drives the
// auto-demotion sweep (counts pass / reject / drop rates over a window).
// Errors / timeouts / unavailable are recorded but excluded from rate calc.
this.db.exec(`
CREATE TABLE IF NOT EXISTS mimir_critic_log (
id INTEGER PRIMARY KEY AUTOINCREMENT,
owner_id TEXT NOT NULL,
persona_id TEXT,
platform TEXT,
action TEXT,
ts INTEGER NOT NULL,
kind TEXT NOT NULL,
stage INTEGER,
reason TEXT,
latency_ms INTEGER,
meta TEXT
)
`);
this.db.exec("CREATE INDEX IF NOT EXISTS idx_critic_log_lookup ON mimir_critic_log(owner_id, persona_id, platform, action, ts)");
this.db.exec("CREATE INDEX IF NOT EXISTS idx_critic_log_kind ON mimir_critic_log(kind, ts)");
this.db.exec(`
CREATE TABLE IF NOT EXISTS outreach_target_lock (
source_url TEXT NOT NULL,
persona_id TEXT NOT NULL,
acquired_at INTEGER NOT NULL,
ttl_s INTEGER DEFAULT 3600,
PRIMARY KEY (source_url, persona_id)
)
`);
this.db.exec(`
CREATE TABLE IF NOT EXISTS mimir_outreach_audit (