From cf46c5a507cc6b00730fd977213b6e38be23aef4 Mon Sep 17 00:00:00 2001 From: Renato <229318195+SignalLayerLabs@users.noreply.github.com> Date: Fri, 9 Oct 2026 08:48:46 +0200 Subject: [PATCH 1/3] Keep personal knowledge lookup memory within the runtime budget --- MANIFEST.sha256 | 9 ++- .../packed-knowledge-checkpoints.md | 37 ++++++++++++ tests/test_sovereign_compact_string_index.mjs | 36 ++++++++++++ tests/test_sovereign_index_string_memory.mjs | 56 +++++++++++++++++++ worker/src/knowledge-archive.js | 52 +++++++++++++---- world/compact-string-index.js | 47 ++++++++++++++++ world/epistemics.js | 7 ++- 7 files changed, 228 insertions(+), 16 deletions(-) create mode 100644 tests/test_sovereign_compact_string_index.mjs create mode 100644 tests/test_sovereign_index_string_memory.mjs create mode 100644 world/compact-string-index.js diff --git a/MANIFEST.sha256 b/MANIFEST.sha256 index b019413..3711427 100644 --- a/MANIFEST.sha256 +++ b/MANIFEST.sha256 @@ -37,7 +37,7 @@ eee99e85b57fec9864f92915a28eb75033e54da4fdb3c97e8de2f499cffefaf4 docs/operation d21ac14687f4f46b05b61622da37c7556c2d843c956215ac9e82bb9e4ce69398 docs/operations/heartbeat-recovery.md 5a82eb9533678d61622a462bd9de4ad34a3c306cb5fc3129dbeefc1088a67754 docs/operations/memory-path-v2.md 455a149710f27cb6c5ee900d3989fdf5f17bea24d90cd479f4c409db0ef5d96f docs/operations/memory-pressure-recovery.md -d8bc46174615c8473aa07f9acbc8dd56f5e8b32cd0d93f6d24ac0c9f0acb07b3 docs/operations/packed-knowledge-checkpoints.md +7a3ace622c725954082d1314c68801724451ca07c3ff87576b218b6c57a2c0cc docs/operations/packed-knowledge-checkpoints.md 9b52f43e56fa717704a1ce1e92b781fcf54ddc2fe2b48080da2ab4e3dcdff8af docs/operations/websocket-heartbeat-recovery.md 0ecf68ef4eb4b31275e584f0ebd9fc1fa72893c5ae514e14f4996d1eb1654589 docs/reference/repository-map.md 9025958e141161d60f59cb801b9aeae31cafca5538b59c94e0042a1a7bd330ef functions/_lib/auth.js @@ -144,6 +144,7 @@ de8f675a1e0a3cb96705cf72bdeb2aca9fa0867c3b7246853bc63821c777bc31 tests/test_sov 415105e34c5ebae6e0fe4dd0455e156a232b0e2c476d73844e4d3d12449aad71 tests/test_sovereign_cognition_scheduler.mjs 77716f2c424d56e2d7ac8966628706853b94e6e4da1debd18fbd180ed37f380b tests/test_sovereign_cognition_state.mjs a9e8632aa631a2da14cd8ab6637e7f20d187b22fe8159da15776ffe43a006a8d tests/test_sovereign_cold_cpu_window.mjs +e65e8c757b0e2ef6859848b158fd961fbe3baa813675d9a3cd7183074c3da4ff tests/test_sovereign_compact_string_index.mjs 2d64a32a4e5a8375c7558b3859cc5636af13b0866204506bde8c3455092f7f0a tests/test_sovereign_conservation.mjs 96186ae225e60cd3ccef5076bf9e17d813654bb7e25e3738449b66e41117d7c6 tests/test_sovereign_construction_demand.mjs 408ca361ae15be807f3f3e7a9507b98d2243c9c6b79864f9e49d72172d899e48 tests/test_sovereign_construction_physics_upgrade.mjs @@ -163,6 +164,7 @@ a71ce1ffa138b92138755cc64eb828f2e145e78c4b369b1eda2911078700a003 tests/test_sov 849b9fcdf721848651b67ca829e802616791c296b95cc1cc874e4f1be4a022f4 tests/test_sovereign_graphics2.mjs e0d0c6bcba695b34d69aa3c4ceeae502a80b3dc61843ef12f3cf555e883d5a66 tests/test_sovereign_history.mjs ca70bc968311b97bf9676c762c25a41c5d682d7753d32ea1ecd0108b5eea6e78 tests/test_sovereign_impact_physics_upgrade.mjs +5070dc74b7b3fe9d8e5e6b3a73e18b877ff1223159fa6d4f860a7cd878f57308 tests/test_sovereign_index_string_memory.mjs 07d222fb857cbb7b9e3927e20527ad101d5f9d97a8a3569f323d9dae7227ff39 tests/test_sovereign_kernel_integration.mjs 8a5829515cfcbf398495e02158f555ba28d3ca3618132c54a28fb8eb0ab2e880 tests/test_sovereign_knowledge_archive.mjs d21f8b3342cd4f196ac9a7c050c50a02a827d238c8aa41dc39bdaa6ae6728158 tests/test_sovereign_knowledge_encoding.mjs @@ -217,7 +219,7 @@ fe8914a4a9cb0b62c9dc0f499ee2e9bc7a61e31a7fd29ee3bae1bbf958405429 tests/test_sov f345373ad773f9a2feaf38a7337d2514bf3cc3538bedcf79fff836d76911ed1e tests/test_sovereign_websocket_signal.mjs 6d2cf0bac9610895650c096698c50691c863dde56fd8f262af4646907a630a09 worker/src/archive-digest.js 20517cc692843f2190984462fc7cb147caf27401ac72541b66ddf0cbd5ecf90d worker/src/index.js -c5f54fe81bb3edac9c2384607e6298dc579d9d6f29778eea59d2090ae4e97b24 worker/src/knowledge-archive.js +5262b8cd32219aa876912fa9cab2329e722be52aff45449afe42123f5b5511cf worker/src/knowledge-archive.js 4b00db1fd2286f8d9dd4b522a9eb1111bd24b4e1befc705b5d6c18b45a4b3c0b worker/src/neuron-governor.js fa52d714982a95ef708fc8958b9dc2569f0a03b814a76e1cb89866bb6d3b3974 worker/src/persistence.js 805f6cb28bb8b6d3a0badd83c93bccd9671fa01a3b4b92b7042b0743325ac243 worker/src/vendor/fflate.LICENSE @@ -232,13 +234,14 @@ c5ce7c1f20314fdb3ee302545401010fd1800d62739a77519684068e135c22cd world/biology. cd7a7faa5ea4f4e790d76d89b7d0a1e0783eeb1a81ed0e0248d7ec090787cc1a world/cognition-queue.js f9697932e6e6728f0374858b2f19e27275204a449d2c1f7268e94dfedffecaed world/cognition-state.js cfc8bcbcbcd47b2e9b4a52b838a1bb7837947d98077bf7f45a5d8536725c2446 world/cognition.js +8f5f49b4fa80dc4ecc2011b963128db8afc7943a903bf5a79f3ca82ffb35217d world/compact-string-index.js f652161b6761b77228a0a6abae5527e5e19fc2481aba0aeaa2ed5934bf68f3ff world/constants.js d89131a70e43eddbb1cc0f4a699e48766b90eca3b7f56edeba92d8bd22ebaf3e world/designs.js 848a0d0eaefef35306d30b50a30b4b22400aa26fffad3c88efe9b62ec5eed12f world/destruction.js 3bb66b15a8796801310e798ea0e12961ae0744536d7048142e2d452a0b64e18b world/disease.js 9e5c628bce5573ba65aaeaddfbe97b11a9d22af4020669cef76eddb4da9a1973 world/engine.js 6c718ab427ac70d5846320bc668aabb005b9ef3f517c799ed8035fa1d0a96d0a world/environment.js -e5b8c100507052d613aab8e67eaebfe7905ea097bd5d47701870e213fcff09ca world/epistemics.js +2d7adc7d7106f94e96a81935d5b7f0fe675e8399034597e50e183099a4dc1068 world/epistemics.js 5420339655dd40b2b20a14576e9e0379be6d8d97668810ba941e26a542ca741b world/evidence-pool.js ad4e953667ba12b6acce5b7407436458e4edada4d2dd3b584d854af8d65f21cf world/genesis.js 7f8a07c4dd491f6eee3baf10ea55c4dad637f47ebf68d9be89f62c1766cfc658 world/genetics.js diff --git a/docs/operations/packed-knowledge-checkpoints.md b/docs/operations/packed-knowledge-checkpoints.md index 4334c90..b0336a9 100644 --- a/docs/operations/packed-knowledge-checkpoints.md +++ b/docs/operations/packed-knowledge-checkpoints.md @@ -234,3 +234,40 @@ See [Cloudflare pricing](https://developers.cloudflare.com/durable-objects/platf Account request, CPU, storage and row limits still apply. This change neither upgrades the plan nor promises unlimited retention or availability on a finite Free account. + +## Shared strings and compact exact lookup directories + +The main deployment from PR 59 failed its production quiet gate with HTTP 503. +A single post-failure diagnostic on October 9 recorded an actual isolate memory +reset. Cold recovery contained 875,469 knowledge records, 875,471 sources, 50,255 +memories and 12,023 objects. Earlier sparse native fixtures did not reproduce +the repeated concept/entity strings and large personal membership directories. + +A bounded pool now shares detached immutable concept/entity/fallback/known-entity +strings (65,536 entries and 2 Mi code units maximum). It contains neither personal +membership nor evidence. Exact per-Citizen lookup directories retain Uint32 +positions into their existing string arrays rather than full Map/Set buckets. +Hash collisions compare the complete original key; append, deletion, forgetting, +first-active duplicate selection and non-string keys preserve their semantics. +No canonical knowledge, object, source, chronology or private membership is removed. +The packed-v2 durable format and backward readers are unchanged. + +The controlled native fixture reconstructs the public physical shape and 875,469 +private knowledge records, 50,255 initial memories and 108 Citizens. An additional +3,585 inert object-shaped records brings object count to 12,023; these are synthetic +records because the public projection omits depleted objects. The old reader +exhausts a 96 MiB V8 old-space cap during cold recovery. String sharing alone passes +the smaller fixture but exhausts an 80 MiB old-space cap in the first physical phase +with all objects. Shared strings plus numeric lookup directories pass cold recovery, +thirty chronological minutes and checkpoint publication under 80 MiB old space +and 4 MiB semi space: 4 moved Citizens, 72 changed actions, 100 changed bodies, all 108 +IDs retained. Sampled JavaScript heap plus ArrayBuffer backing peaks at 112.48 MiB. +Encoding takes 4.66 seconds; first physical phase takes 11.83 seconds. A subsequent +cold restart recovers minute 177,183 with 876,938 knowledge records and both sampled +original private sources intact. + +These are local controlled measurements, not an exact reproduction of Cloudflare's +128 MiB account/runtime enforcement or a guarantee of indefinite capacity. Private +sources, cold lookup semantics, collision/deletion handling and the existing +known-entity read-count gate have regression coverage. Production must pass all +unchanged gates and a separate ten-minute quiet proof before stability clearance. diff --git a/tests/test_sovereign_compact_string_index.mjs b/tests/test_sovereign_compact_string_index.mjs new file mode 100644 index 0000000..4171afa --- /dev/null +++ b/tests/test_sovereign_compact_string_index.mjs @@ -0,0 +1,36 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import {CompactStringIndex} from '../world/compact-string-index.js'; +import {sqliteStorage,wake} from './helpers/sovereign-sqlite.mjs'; +import {forget,knowledgeEntry,knowsEntity} from '../world/epistemics.js'; + +test('compact lookup preserves exact collision, deletion, append and non-string Map semantics',()=>{ + const entries=[];const index=new CompactStringIndex(entries); + const expected=new Map(); + for(let i=0;i<12000;i++){const key=`entity:${i%9000}`;entries.push(key);index.set(key,i);expected.set(key,i);} + for(const [key,value] of expected)assert.equal(index.get(key),value); + for(let i=0;i<9000;i+=3){assert.equal(index.delete(`entity:${i}`),true);expected.delete(`entity:${i}`);} + for(let i=0;i<9000;i++){assert.equal(index.has(`entity:${i}`),expected.has(`entity:${i}`));assert.equal(index.get(`entity:${i}`),expected.get(`entity:${i}`));} + const object={};for(const key of [null,undefined,false,0,NaN,object]){entries.push(key);index.set(key,entries.length-1);assert.equal(index.get(key),entries.length-1);} + assert.equal(index.has({}),false);assert.equal(index.delete('absent'),false); + assert.ok(index.table.byteLength<=entries.length*16,'lookup must use a compact numeric directory'); +}); + +test('compact archive lookup retains the first active duplicate through changes and cold recovery',async t=>{ + const storage=sqliteStorage();t.after(()=>storage.db.close());const {instance}=await wake(storage);const c=instance.world.citizens[0]; + const entry=(concept,eventId)=>({concept,active:true,confidence:.8,provenance:[{kind:'observation',eventId}]}); + c.knowledge=[entry('duplicate','source:a'),entry('duplicate','source:b'),entry('unrelated','source:c')]; + await instance.persist({forceSeal:true});let r=(await wake(storage)).instance.world.citizens[0]; + assert.equal(knowledgeEntry(r,'duplicate').provenance[0].eventId,'source:a'); + forget(r,'duplicate');assert.equal(knowledgeEntry(r,'duplicate').provenance[0].eventId,'source:b'); + r.knowledge[1].concept='changed';assert.equal(knowledgeEntry(r,'duplicate'),null);assert.equal(knowledgeEntry(r,'changed').provenance[0].eventId,'source:b'); + assert.equal(knowledgeEntry(r,'unrelated').provenance[0].eventId,'source:c'); +}); + +test('compact entity membership stays private and indexes append and replacement without granting unknown IDs',()=>{ + const a={knownEntityIds:Array.from({length:12000},(_,i)=>`entity:${i}`)},b={knownEntityIds:['entity:5']}; + assert.equal(knowsEntity(a,'entity:11999'),true);assert.equal(knowsEntity(a,'entity:12000'),false); + a.knownEntityIds.push('entity:12000');assert.equal(knowsEntity(a,'entity:12000'),true);assert.equal(knowsEntity(b,'entity:12000'),false); + a.knownEntityIds=['entity:other'];assert.equal(knowsEntity(a,'entity:5'),false);assert.equal(knowsEntity(a,'entity:other'),true); + a.knownEntityIds.length=0;assert.equal(knowsEntity(a,'entity:other'),false); +}); diff --git a/tests/test_sovereign_index_string_memory.mjs b/tests/test_sovereign_index_string_memory.mjs new file mode 100644 index 0000000..a464aa8 --- /dev/null +++ b/tests/test_sovereign_index_string_memory.mjs @@ -0,0 +1,56 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import {spawnSync} from 'node:child_process'; +import {KnowledgeArchive} from '../worker/src/knowledge-archive.js'; +import {sqliteStorage,wake} from './helpers/sovereign-sqlite.mjs'; +import {knows,forget,knowsEntity} from '../world/epistemics.js'; + +test('shared immutable index strings preserve private membership, ordering and sources after cold recovery',async t=>{ + const storage=sqliteStorage();t.after(()=>storage.db.close());const {instance}=await wake(storage); + const [a,b]=instance.world.citizens; + const entry=(concept,eventId)=>({concept,active:true,confidence:.8,provenance:[{kind:'observation',eventId,evidence:{entityId:'object:shared-physical-id'}}]}); + a.knowledge=[entry('concept:shared-physical-id','private:a'),entry('concept:only-a','private:secret')]; + b.knowledge=[entry('concept:shared-physical-id','private:b')]; + a.knownEntityIds=['object:shared-physical-id','object:only-a'];b.knownEntityIds=['object:shared-physical-id']; + const expected=JSON.stringify([a.knowledge,b.knowledge]); + await instance.persist({forceSeal:true});const {instance:restored}=await wake(storage); + const [ra,rb]=restored.world.citizens; + assert.equal(JSON.stringify([ra.knowledge,rb.knowledge]),expected); + assert.deepEqual(ra.knownEntityIds,a.knownEntityIds);assert.deepEqual(rb.knownEntityIds,b.knownEntityIds); + assert.ok(restored.knowledgeArchive.stats().indexStringPoolHits>=3,'cold duplicates should reuse detached strings'); + assert.equal(knows(rb,'concept:only-a'),false);assert.equal(knowsEntity(rb,'object:only-a'),false); + forget(ra,'concept:shared-physical-id'); + assert.equal(knows(ra,'concept:shared-physical-id'),false);assert.equal(knows(rb,'concept:shared-physical-id'),true); + assert.equal(rb.knowledge[0].provenance[0].eventId,'private:b'); +}); + +test('index string sharing has finite entry and character bounds without changing uncached values',()=>{ + const archive=new KnowledgeArchive(null,{maxIndexStrings:3,maxIndexStringUnits:40}); + for(let i=0;i<100;i++)assert.equal(archive.internIndexString(`entity:${i}`),`entity:${i}`); + const oversized='private-long-meaning:'.repeat(200); + assert.equal(archive.internIndexString(oversized),oversized); + assert.equal(archive.internIndexString(null),null);assert.equal(archive.internIndexString(undefined),undefined); + assert.ok(archive.stats().indexStringPoolEntries<=3);assert.ok(archive.stats().indexStringPoolUnits<=40); + archive.dispose();assert.equal(archive.stats().indexStringPoolEntries,0);assert.equal(archive.stats().indexStringPoolUnits,0); +}); + +test('actual repeated physical-ID shapes retain substantially less heap when shared',()=>{ + const probe=spawnSync(process.execPath,['--expose-gc','--input-type=module','-e',` + import v8 from 'node:v8'; + import {KnowledgeArchive} from './worker/src/knowledge-archive.js'; + function measure(shared){ + global.gc();const before=v8.getHeapStatistics().used_heap_size; + const archive=new KnowledgeArchive(null,{maxIndexStrings:shared?65536:0}); + const values=[]; + for(let i=0;i<120000;i++)values.push(archive.internIndexString('kobject:'+String(i%512).padStart(8,'0'))); + global.gc();const retained=v8.getHeapStatistics().used_heap_size-before; + if(values[119999]!=='kobject:'+String(119999%512).padStart(8,'0'))throw new Error('index value changed'); + return retained; + } + const unshared=measure(false),shared=measure(true); + console.log(JSON.stringify({unshared,shared})); + `],{encoding:'utf8'}); + assert.equal(probe.status,0,probe.stderr); + const {unshared,shared}=JSON.parse(probe.stdout); + assert.ok(sharedtypeof key==='string'&&/^(0|[1-9][0-9]*)$/.test(key); // Parsed strings can retain their large Citizen JSON backing buffer in V8. @@ -49,7 +51,8 @@ function indexColumn(source=[],boolean=false){ // Immutable compressed pages are packed once into durable bins. The guarded // checkpoint owns the bin descriptors and logical order; caches are disposable. export class KnowledgeArchive { - constructor(sql,{hydrateEntry=()=>{},maxBackingBytes=BACKING_BYTES,maxBinCacheBytes=BIN_CACHE_BYTES}={}){ + constructor(sql,{hydrateEntry=()=>{},maxBackingBytes=BACKING_BYTES,maxBinCacheBytes=BIN_CACHE_BYTES, + maxIndexStrings=INDEX_STRINGS,maxIndexStringUnits=INDEX_STRING_UNITS}={}){ this.sql=sql;this.bins=new Map();this.binCache=new Map();this.binCacheBytes=0;this.maxBinCacheBytes=maxBinCacheBytes; this.sqlRowsRead=0;this.sqlRowsWritten=0; this.hydrateEntry=hydrateEntry;this.maxBackingBytes=maxBackingBytes; @@ -59,12 +62,15 @@ export class KnowledgeArchive { this.dirtyCodeUnits=0;this.pageRefs=new Map();this.pageCounts=new Map();this.pageReads=0; this.deadPages=new Set(); this.entityMatchCache=new Map();this.entityMatchIndices=0; + this.indexStringPool=new Map();this.indexStringPoolUnits=0;this.indexStringPoolHits=0; + this.maxIndexStrings=maxIndexStrings;this.maxIndexStringUnits=maxIndexStringUnits; } dispose(){ this.pages.clear();this.cache.clear();this.serializedCache.clear();this.stores.clear();this.citizenStores=new WeakMap(); this.pageRefs.clear();this.pageCounts.clear(); this.deadPages.clear(); this.entityMatchCache.clear();this.entityMatchIndices=0; + this.indexStringPool.clear();this.indexStringPoolUnits=0;this.indexStringPoolHits=0; this.bins.clear();this.binCache.clear();this.binCacheBytes=0; this.backingBytes=0;this.cachedCodeUnits=0;this.serializedCodeUnits=0;this.dirtyCodeUnits=0; this.hydrateEntry=()=>{}; @@ -73,7 +79,27 @@ export class KnowledgeArchive { cachedPages:this.cache.size,pageReads:this.pageReads,livePages:this.pageRefs.size, serializedCodeUnits:this.serializedCodeUnits,compressedBytes:this.backingBytes+this.binCacheBytes, pendingCompressedBytes:this.backingBytes,cachedBinBytes:this.binCacheBytes,archivedCompressedBytes:[...this.bins.values()].reduce((n,b)=>n+b.size,0), - maxCompressedBytes:this.maxBackingBytes,storage:this.sql?'packed-sqlite':'compressed-ram',sqlRowsRead:this.sqlRowsRead,sqlRowsWritten:this.sqlRowsWritten};} + maxCompressedBytes:this.maxBackingBytes,storage:this.sql?'packed-sqlite':'compressed-ram',sqlRowsRead:this.sqlRowsRead,sqlRowsWritten:this.sqlRowsWritten, + indexStringPoolEntries:this.indexStringPool.size,indexStringPoolUnits:this.indexStringPoolUnits,indexStringPoolHits:this.indexStringPoolHits};} + internIndexString(value){ + if(typeof value!=='string')return value; + const prior=this.indexStringPool.get(value); + if(prior!==undefined){this.indexStringPoolHits++;return prior;} + const owned=ownedString(value); + // Cache only immutable primitive values, never personal membership or + // evidence. Eviction cannot change knowledge, sources, or array order. + if(this.maxIndexStrings<1||owned.length>4096||owned.length>this.maxIndexStringUnits)return owned; + this.indexStringPool.set(owned,owned);this.indexStringPoolUnits+=owned.length; + while(this.indexStringPool.size>this.maxIndexStrings||this.indexStringPoolUnits>this.maxIndexStringUnits){ + const first=this.indexStringPool.keys().next().value; + this.indexStringPoolUnits-=first.length;this.indexStringPool.delete(first); + } + return owned; + } + shareCitizenEntityIds(citizen){ + if(Array.isArray(citizen.knownEntityIds))for(let i=0;iArray.isArray(value)?value.map(ownedString):ownedString(value)); + store.concepts=saved.concepts.map(value=>this.internIndexString(value)); + store.lookupIndex=new CompactStringIndex(store.concepts,n); + store.entities=saved.entities.map(value=>Array.isArray(value)?value.map(id=>this.internIndexString(id)):this.internIndexString(value)); for(const field of ['rows','offsets','sourceCounts'])store[field]=indexColumn(saved[field]); store.active=indexColumn(saved.active,true); - store.fallbackIds=new Map(saved.fallbackIds.map(([index,id])=>[index,ownedString(id)])); + store.fallbackIds=new Map(saved.fallbackIds.map(([index,id])=>[index,this.internIndexString(id)])); for(let i=0;i=page.ends.length||legacy&&legacyPages.get(store.rows[i])[offset]!==store.concepts[i]|| @@ -335,7 +364,7 @@ export class KnowledgeArchive { class ArchivedKnowledge { constructor(archive){ this.archive=archive;this.concepts=[];this.active=indexColumn([],true);this.entities=[]; - this.rows=indexColumn();this.offsets=indexColumn();this.lookupIndex=new Map();this.fallbackIds=new Map();this.dirty=new Map(); + this.rows=indexColumn();this.offsets=indexColumn();this.lookupIndex=new CompactStringIndex(this.concepts);this.fallbackIds=new Map();this.dirty=new Map(); this.activeCount=0;this.sourceCount=0;this.sourceCounts=indexColumn();this.views=new Map(); this.entityCache=new Map();this.entityCacheIndices=0; const store=this; @@ -372,10 +401,12 @@ class ArchivedKnowledge { indexEntry(index,entry){ const was=this.active[index],concept=this.concepts[index]; const oldEntities=this.entities[index]; + const wasIndexed=this.lookupIndex.get(concept)===index; + if(wasIndexed)this.lookupIndex.delete(concept); if(was)this.activeCount--; - this.concepts[index]=ownedString(entry.concept);this.active[index]=entry.active!==false; - if(!entry.concept&&entry.id)this.fallbackIds.set(index,ownedString(entry.id));else this.fallbackIds.delete(index); - const ids=[...new Set((entry.provenance||[]).map(source=>source.evidence?.entityId).filter(Boolean))].map(ownedString); + this.concepts[index]=this.archive.internIndexString(entry.concept);this.active[index]=entry.active!==false; + if(!entry.concept&&entry.id)this.fallbackIds.set(index,this.archive.internIndexString(entry.id));else this.fallbackIds.delete(index); + const ids=[...new Set((entry.provenance||[]).map(source=>source.evidence?.entityId).filter(Boolean))].map(id=>this.archive.internIndexString(id)); const oldIds=Array.isArray(oldEntities)?oldEntities:oldEntities==null?[]:[oldEntities]; if(was!==this.active[index]||oldIds.length!==ids.length||oldIds.some((id,i)=>id!==ids[i])){ for(const id of new Set([...oldIds,...ids])){ @@ -387,8 +418,7 @@ class ArchivedKnowledge { const sources=entry.provenance?.length||0; this.sourceCount+=sources-(this.sourceCounts[index]||0);this.sourceCounts[index]=sources; if(this.active[index])this.activeCount++; - if(concept!==undefined&&this.lookupIndex.get(concept)===index){ - this.lookupIndex.delete(concept); + if(wasIndexed){ if(concept!==entry.concept||!this.active[index]){ const replacement=this.concepts.findIndex((id,i)=>id===concept&&this.active[i]); if(replacement>=0)this.lookupIndex.set(concept,replacement); diff --git a/world/compact-string-index.js b/world/compact-string-index.js new file mode 100644 index 0000000..5568b15 --- /dev/null +++ b/world/compact-string-index.js @@ -0,0 +1,47 @@ +// A disposable exact lookup directory into an existing canonical array. String +// keys stay in that array; buckets retain only their numeric positions. Callers +// delete an indexed key before changing its canonical value. +const DELETED=0xffffffff; +function hash(value){ + let h=2166136261; + for(let i=0;i>>0; +} +export class CompactStringIndex { + constructor(entries,expectedSize=0){ + this.entries=entries;this.table=new Uint32Array(16);this.size=0;this.used=0;this.other=new Map(); + while(this.table.length*.65=DELETED-1||this.entries[index]!==key) + throw new Error('compact_string_index_position_invalid'); + if((this.used+1)>this.table.length*.65)this.resize(this.size Date: Fri, 9 Oct 2026 08:59:33 +0200 Subject: [PATCH 2/3] Stop repeated harvesting from growing object and knowledge directories --- MANIFEST.sha256 | 5 +-- .../packed-knowledge-checkpoints.md | 19 +++++++++++ tests/test_sovereign_gather_stability.mjs | 33 +++++++++++++++++++ world/engine.js | 25 +++++++++++++- 4 files changed, 79 insertions(+), 3 deletions(-) create mode 100644 tests/test_sovereign_gather_stability.mjs diff --git a/MANIFEST.sha256 b/MANIFEST.sha256 index 3711427..57bb7d1 100644 --- a/MANIFEST.sha256 +++ b/MANIFEST.sha256 @@ -37,7 +37,7 @@ eee99e85b57fec9864f92915a28eb75033e54da4fdb3c97e8de2f499cffefaf4 docs/operation d21ac14687f4f46b05b61622da37c7556c2d843c956215ac9e82bb9e4ce69398 docs/operations/heartbeat-recovery.md 5a82eb9533678d61622a462bd9de4ad34a3c306cb5fc3129dbeefc1088a67754 docs/operations/memory-path-v2.md 455a149710f27cb6c5ee900d3989fdf5f17bea24d90cd479f4c409db0ef5d96f docs/operations/memory-pressure-recovery.md -7a3ace622c725954082d1314c68801724451ca07c3ff87576b218b6c57a2c0cc docs/operations/packed-knowledge-checkpoints.md +4dea7fa2a9005ef605efae2719b7b8771c36266deb69502bd4de218dba4f8560 docs/operations/packed-knowledge-checkpoints.md 9b52f43e56fa717704a1ce1e92b781fcf54ddc2fe2b48080da2ab4e3dcdff8af docs/operations/websocket-heartbeat-recovery.md 0ecf68ef4eb4b31275e584f0ebd9fc1fa72893c5ae514e14f4996d1eb1654589 docs/reference/repository-map.md 9025958e141161d60f59cb801b9aeae31cafca5538b59c94e0042a1a7bd330ef functions/_lib/auth.js @@ -160,6 +160,7 @@ b3a62f70768edc4cf2c0a2d509c7e8055ad2002c3e1f79f292d728c39c64a3db tests/test_sov 2b2d7007ed708eab76efc151f59e30d5239fcee65b83e0fd92b02add258ad5c0 tests/test_sovereign_entity_knowledge_cache.mjs bbe9b971d7162e2e44d9a01fc31587df88cd715c84b4c63fdf06bf879aa8293b tests/test_sovereign_epistemics.mjs a71ce1ffa138b92138755cc64eb828f2e145e78c4b369b1eda2911078700a003 tests/test_sovereign_exposure_upgrade.mjs +1600de4ecd28f5355a2c7044901eacd2544550db98f0e546b5cd8d62f14f3bb1 tests/test_sovereign_gather_stability.mjs 831ecc901261fe003f15b3f2a2ab2f2a41273bb1fcbb14fa2377d99d1efe8899 tests/test_sovereign_genesis.mjs 849b9fcdf721848651b67ca829e802616791c296b95cc1cc874e4f1be4a022f4 tests/test_sovereign_graphics2.mjs e0d0c6bcba695b34d69aa3c4ceeae502a80b3dc61843ef12f3cf555e883d5a66 tests/test_sovereign_history.mjs @@ -239,7 +240,7 @@ f652161b6761b77228a0a6abae5527e5e19fc2481aba0aeaa2ed5934bf68f3ff world/constant d89131a70e43eddbb1cc0f4a699e48766b90eca3b7f56edeba92d8bd22ebaf3e world/designs.js 848a0d0eaefef35306d30b50a30b4b22400aa26fffad3c88efe9b62ec5eed12f world/destruction.js 3bb66b15a8796801310e798ea0e12961ae0744536d7048142e2d452a0b64e18b world/disease.js -9e5c628bce5573ba65aaeaddfbe97b11a9d22af4020669cef76eddb4da9a1973 world/engine.js +dbe4ca572f9253b1f1219ee3f096d43f3ca765e963863a72908a8f214cbcfe3a world/engine.js 6c718ab427ac70d5846320bc668aabb005b9ef3f517c799ed8035fa1d0a96d0a world/environment.js 2d7adc7d7106f94e96a81935d5b7f0fe675e8399034597e50e183099a4dc1068 world/epistemics.js 5420339655dd40b2b20a14576e9e0379be6d8d97668810ba941e26a542ca741b world/evidence-pool.js diff --git a/docs/operations/packed-knowledge-checkpoints.md b/docs/operations/packed-knowledge-checkpoints.md index b0336a9..d38fce8 100644 --- a/docs/operations/packed-knowledge-checkpoints.md +++ b/docs/operations/packed-knowledge-checkpoints.md @@ -271,3 +271,22 @@ These are local controlled measurements, not an exact reproduction of Cloudflare sources, cold lookup semantics, collision/deletion handling and the existing known-entity read-count gate have regression coverage. Production must pass all unchanged gates and a separate ten-minute quiet proof before stability clearance. + +## Repeated harvest growth + +Compact caches alone cannot make continuous allocation sustainable. The duration +audit found that every GATHER action created another physical object, possession +and personal entity ID, prompting all nearby Citizens to acquire another concept. +Repeated ordinary harvests now add matter to one compatible personal stack from +the same deposit. Existing objects/IDs and original provenance remain intact. +Every harvest still emits its own causal receipt; the stack points to the latest +physical event. Reserved, heated, transformed, damaged, foreign and different-source +objects are excluded. Fractional availability caps the harvested quantity exactly; +a depleted source cannot manufacture the former minimum 0.1 kg of matter. + +A thousand repeated harvests retain one new object and one new personal entity, +with all thousand distinct action receipts, 2,000 kg gathered and 3,000 kg remaining +in the source. This prevents this routine activity from growing object/concept +directories with elapsed time. Other genuine discoveries, births, transformations +and preserved historical records still consume finite resources; account-level +Free quotas remain external capacity constraints. diff --git a/tests/test_sovereign_gather_stability.mjs b/tests/test_sovereign_gather_stability.mjs new file mode 100644 index 0000000..187ec96 --- /dev/null +++ b/tests/test_sovereign_gather_stability.mjs @@ -0,0 +1,33 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import {createSovereignGenesis} from '../world/genesis.js'; +import {startAction,interruptAction} from '../world/actions.js'; +import {advanceWorldTo} from '../world/engine.js'; +import {verifyLedger} from '../world/ledger.js'; + +function fixture(){const world=createSovereignGenesis({seed:37,realEpochMs:0}),c=world.citizens[0],d=world.resourceDeposits.find(x=>x.type==='timber');world.citizens=[c];c.position={...d.position};d.renewPerDay=0;d.quantity=5000;return {world,c,d};} +function gather(f,quantity=2){const {world,c,d}=f;interruptAction(world,c,'controlled fixture');Object.assign(c.body,{hydration:100,calories:100,health:100,sleepPressure:10});const a=startAction(world,c,{type:'GATHER',durationMinutes:1,targetId:d.id,payload:{quantity},purpose:'self-directed'},world.clock.worldMinute);advanceWorldTo(world,a.endsWorldMinute*1000);return a;} + +test('a thousand repeated gathers retain one compatible stack, every harvest receipt, matter and existing IDs',()=>{ + const f=fixture(),ids=f.world.objects.map(o=>o.id),count=ids.length,initialEntities=new Set([...ids,...f.world.resourceDeposits.map(d=>d.id),...f.world.buildings.map(b=>b.id),f.c.id]); + for(let i=0;i<1000;i++)gather(f); + const stacks=f.world.objects.filter(o=>o.holderId===f.c.id&&o.provenance?.depositId===f.d.id); + assert.equal(stacks.length,1);assert.equal(f.world.objects.length,count+1);assert.equal(stacks[0].quantity,2000);assert.equal(f.d.quantity,3000); + assert.ok(ids.every(id=>f.world.objects.some(o=>o.id===id)));assert.equal(new Set(f.c.possessions).size,f.c.possessions.length); + assert.deepEqual(f.c.knownEntityIds.filter(id=>!initialEntities.has(id)),[stacks[0].id],'harvests must introduce only one new personal entity membership'); + const receipts=f.world.ledger.filter(e=>e.type==='RESOURCE_GATHERED');assert.equal(receipts.length,1000);assert.equal(new Set(receipts.map(e=>e.causes[0])).size,1000); + assert.equal(stacks[0].provenance.actionId,receipts[0].causes[0]);assert.equal(stacks[0].lastPhysicalEventId,receipts.at(-1).id);assert.equal(verifyLedger(f.world),true); +}); + +test('gathering never refills transformed, reserved, foreign or different-source objects',()=>{ + for(const change of [{holderId:'other'},{reservedProjectId:'project:x'},{condition:.5},{temperatureC:80},{geometry:{lengthM:1}},{properties:{processed:true}},{provenance:{type:'GATHERED',depositId:'other'}}]){ + const f=fixture();gather(f);const original=f.world.objects.find(o=>o.holderId===f.c.id&&o.provenance?.depositId===f.d.id);Object.assign(original,change);const quantity=original.quantity; + gather(f);assert.equal(original.quantity,quantity);assert.equal(f.world.objects.filter(o=>o.kind==='gathered_material').length,2); + } +}); + +test('depleted and fractional deposits cannot manufacture matter or go negative',()=>{ + const f=fixture();f.d.quantity=.03;gather(f,2); + const stack=f.world.objects.find(o=>o.holderId===f.c.id&&o.provenance?.depositId===f.d.id);assert.equal(stack.quantity,.03);assert.equal(f.d.quantity,0); + gather(f,2);assert.equal(stack.quantity,.03);assert.equal(f.d.quantity,0);assert.equal(f.world.ledger.filter(e=>e.type==='RESOURCE_GATHERED').length,1); +}); diff --git a/world/engine.js b/world/engine.js index f02429d..3e6be68 100644 --- a/world/engine.js +++ b/world/engine.js @@ -55,7 +55,30 @@ function runReflex(world,citizen,at){ type:'OBSERVE',durationMinutes:10,purpose:'orientation' },at); } -function gather(world,c,a,at){const d=world.resourceDeposits.find(x=>x.id===a.targetId);if(!d)return;const q=Math.max(.1,Math.min(Number(a.payload?.quantity)||1,d.quantity));d.quantity-=q;recordHarvest(world,d,q,at);const o={id:stableId('obj',d.id,c.id,world.objects.length,at),kind:'gathered_material',material:d.type,quantity:q,massPerUnitKg:1,properties:null,holderId:c.id,position:{...c.position},condition:1,provenance:{type:'GATHERED',depositId:d.id,actionId:a.id}};world.objects.push(o);c.possessions.push(o.id);if(!knowsEntity(c,o.id))c.knownEntityIds.push(o.id);appendEvent(world,'RESOURCE_GATHERED',c.id,{objectId:o.id,depositId:d.id,quantity:q},[a.id],at);} +function gather(world,c,a,at){ + const d=world.resourceDeposits.find(x=>x.id===a.targetId); + if(!d||!(d.quantity>0))throw new Error('gather_source_unavailable'); + const q=Math.min(Math.max(.1,Number(a.payload?.quantity)||1),d.quantity); + // Repeated harvests add homogeneous matter to a compatible personal stack. + // Existing objects and their original provenance remain intact; each harvest + // still gets its own causal receipt. Processed/reserved matter never absorbs it. + let o=world.objects.find(x=>x.kind==='gathered_material'&&x.holderId===c.id&& + x.material===d.type&&x.massPerUnitKg===1&&x.condition===1&& + !x.reservedProjectId&&!x.burning&&!x.geometry&&x.properties==null&& + x.temperatureC==null&&x.provenance?.type==='GATHERED'&&x.provenance.depositId===d.id); + d.quantity-=q;recordHarvest(world,d,q,at); + if(o){o.quantity+=q;o.position={...c.position};} + else{ + o={id:stableId('obj',d.id,c.id,world.objects.length,at),kind:'gathered_material',material:d.type, + quantity:q,massPerUnitKg:1,properties:null,holderId:c.id,position:{...c.position},condition:1, + provenance:{type:'GATHERED',depositId:d.id,actionId:a.id}}; + world.objects.push(o); + } + if(!c.possessions.includes(o.id))c.possessions.push(o.id); + if(!knowsEntity(c,o.id))c.knownEntityIds.push(o.id); + const event=appendEvent(world,'RESOURCE_GATHERED',c.id,{objectId:o.id,depositId:d.id,quantity:q},[a.id],at); + o.lastPhysicalEventId=event.id; +} function consumeFromDeposit(world,c,a,type,amount,restore,at){const d=world.resourceDeposits.find(x=>x.id===a.targetId&&x.type===type);if(!d||d.quantityx.id===a.targetId&&x.quantity>0&&(x.holderId===null||x.holderId===undefined)); From 7057c0993728a96c8cf2e5effd4ee537efe10423 Mon Sep 17 00:00:00 2001 From: Renato <229318195+SignalLayerLabs@users.noreply.github.com> Date: Fri, 9 Oct 2026 09:06:35 +0200 Subject: [PATCH 3/3] Preserve causal contributions when extending a material stack --- MANIFEST.sha256 | 6 +++--- docs/operations/packed-knowledge-checkpoints.md | 3 ++- tests/test_sovereign_gather_stability.mjs | 8 ++++++++ world/engine.js | 4 +++- 4 files changed, 16 insertions(+), 5 deletions(-) diff --git a/MANIFEST.sha256 b/MANIFEST.sha256 index 57bb7d1..928c4cc 100644 --- a/MANIFEST.sha256 +++ b/MANIFEST.sha256 @@ -37,7 +37,7 @@ eee99e85b57fec9864f92915a28eb75033e54da4fdb3c97e8de2f499cffefaf4 docs/operation d21ac14687f4f46b05b61622da37c7556c2d843c956215ac9e82bb9e4ce69398 docs/operations/heartbeat-recovery.md 5a82eb9533678d61622a462bd9de4ad34a3c306cb5fc3129dbeefc1088a67754 docs/operations/memory-path-v2.md 455a149710f27cb6c5ee900d3989fdf5f17bea24d90cd479f4c409db0ef5d96f docs/operations/memory-pressure-recovery.md -4dea7fa2a9005ef605efae2719b7b8771c36266deb69502bd4de218dba4f8560 docs/operations/packed-knowledge-checkpoints.md +e22d0d74080e440ba093beb670b2f1b45d2cd9fe58c2fd8b29b1887859bc0e8d docs/operations/packed-knowledge-checkpoints.md 9b52f43e56fa717704a1ce1e92b781fcf54ddc2fe2b48080da2ab4e3dcdff8af docs/operations/websocket-heartbeat-recovery.md 0ecf68ef4eb4b31275e584f0ebd9fc1fa72893c5ae514e14f4996d1eb1654589 docs/reference/repository-map.md 9025958e141161d60f59cb801b9aeae31cafca5538b59c94e0042a1a7bd330ef functions/_lib/auth.js @@ -160,7 +160,7 @@ b3a62f70768edc4cf2c0a2d509c7e8055ad2002c3e1f79f292d728c39c64a3db tests/test_sov 2b2d7007ed708eab76efc151f59e30d5239fcee65b83e0fd92b02add258ad5c0 tests/test_sovereign_entity_knowledge_cache.mjs bbe9b971d7162e2e44d9a01fc31587df88cd715c84b4c63fdf06bf879aa8293b tests/test_sovereign_epistemics.mjs a71ce1ffa138b92138755cc64eb828f2e145e78c4b369b1eda2911078700a003 tests/test_sovereign_exposure_upgrade.mjs -1600de4ecd28f5355a2c7044901eacd2544550db98f0e546b5cd8d62f14f3bb1 tests/test_sovereign_gather_stability.mjs +201948179849116d2cd0877c4ae1425605fdd6b6ccc638255e18467fc6fb30cb tests/test_sovereign_gather_stability.mjs 831ecc901261fe003f15b3f2a2ab2f2a41273bb1fcbb14fa2377d99d1efe8899 tests/test_sovereign_genesis.mjs 849b9fcdf721848651b67ca829e802616791c296b95cc1cc874e4f1be4a022f4 tests/test_sovereign_graphics2.mjs e0d0c6bcba695b34d69aa3c4ceeae502a80b3dc61843ef12f3cf555e883d5a66 tests/test_sovereign_history.mjs @@ -240,7 +240,7 @@ f652161b6761b77228a0a6abae5527e5e19fc2481aba0aeaa2ed5934bf68f3ff world/constant d89131a70e43eddbb1cc0f4a699e48766b90eca3b7f56edeba92d8bd22ebaf3e world/designs.js 848a0d0eaefef35306d30b50a30b4b22400aa26fffad3c88efe9b62ec5eed12f world/destruction.js 3bb66b15a8796801310e798ea0e12961ae0744536d7048142e2d452a0b64e18b world/disease.js -dbe4ca572f9253b1f1219ee3f096d43f3ca765e963863a72908a8f214cbcfe3a world/engine.js +cc384e3f39c59f8bcaf79f2c2e4457fd3070778d3fe04bcf6d34497154395179 world/engine.js 6c718ab427ac70d5846320bc668aabb005b9ef3f517c799ed8035fa1d0a96d0a world/environment.js 2d7adc7d7106f94e96a81935d5b7f0fe675e8399034597e50e183099a4dc1068 world/epistemics.js 5420339655dd40b2b20a14576e9e0379be6d8d97668810ba941e26a542ca741b world/evidence-pool.js diff --git a/docs/operations/packed-knowledge-checkpoints.md b/docs/operations/packed-knowledge-checkpoints.md index d38fce8..6975b7c 100644 --- a/docs/operations/packed-knowledge-checkpoints.md +++ b/docs/operations/packed-knowledge-checkpoints.md @@ -280,7 +280,8 @@ and personal entity ID, prompting all nearby Citizens to acquire another concept Repeated ordinary harvests now add matter to one compatible personal stack from the same deposit. Existing objects/IDs and original provenance remain intact. Every harvest still emits its own causal receipt; the stack points to the latest -physical event. Reserved, heated, transformed, damaged, foreign and different-source +physical event, whose causes also retain the prior stack contribution (or the +original action for a legacy stack). Reserved, heated, transformed, damaged, foreign and different-source objects are excluded. Fractional availability caps the harvested quantity exactly; a depleted source cannot manufacture the former minimum 0.1 kg of matter. diff --git a/tests/test_sovereign_gather_stability.mjs b/tests/test_sovereign_gather_stability.mjs index 187ec96..a99961f 100644 --- a/tests/test_sovereign_gather_stability.mjs +++ b/tests/test_sovereign_gather_stability.mjs @@ -16,6 +16,7 @@ test('a thousand repeated gathers retain one compatible stack, every harvest rec assert.ok(ids.every(id=>f.world.objects.some(o=>o.id===id)));assert.equal(new Set(f.c.possessions).size,f.c.possessions.length); assert.deepEqual(f.c.knownEntityIds.filter(id=>!initialEntities.has(id)),[stacks[0].id],'harvests must introduce only one new personal entity membership'); const receipts=f.world.ledger.filter(e=>e.type==='RESOURCE_GATHERED');assert.equal(receipts.length,1000);assert.equal(new Set(receipts.map(e=>e.causes[0])).size,1000); + for(let i=1;io.holderId===f.c.id&&o.provenance?.depositId===f.d.id);assert.equal(stack.quantity,.03);assert.equal(f.d.quantity,0); gather(f,2);assert.equal(stack.quantity,.03);assert.equal(f.d.quantity,0);assert.equal(f.world.ledger.filter(e=>e.type==='RESOURCE_GATHERED').length,1); }); + +test('reused legacy material retains its original action as the causal anchor',()=>{ + const f=fixture(),first=gather(f),stack=f.world.objects.find(o=>o.holderId===f.c.id&&o.provenance?.depositId===f.d.id); + delete stack.lastPhysicalEventId;const second=gather(f); + const receipt=f.world.ledger.findLast(e=>e.type==='RESOURCE_GATHERED'); + assert.equal(stack.quantity,4);assert.deepEqual(receipt.causes,[second.id,first.id]);assert.equal(stack.provenance.actionId,first.id); +}); diff --git a/world/engine.js b/world/engine.js index 3e6be68..6fb707b 100644 --- a/world/engine.js +++ b/world/engine.js @@ -66,6 +66,7 @@ function gather(world,c,a,at){ x.material===d.type&&x.massPerUnitKg===1&&x.condition===1&& !x.reservedProjectId&&!x.burning&&!x.geometry&&x.properties==null&& x.temperatureC==null&&x.provenance?.type==='GATHERED'&&x.provenance.depositId===d.id); + const previousCause=o?.lastPhysicalEventId||o?.provenance?.actionId; d.quantity-=q;recordHarvest(world,d,q,at); if(o){o.quantity+=q;o.position={...c.position};} else{ @@ -76,7 +77,8 @@ function gather(world,c,a,at){ } if(!c.possessions.includes(o.id))c.possessions.push(o.id); if(!knowsEntity(c,o.id))c.knownEntityIds.push(o.id); - const event=appendEvent(world,'RESOURCE_GATHERED',c.id,{objectId:o.id,depositId:d.id,quantity:q},[a.id],at); + const causes=previousCause?[a.id,previousCause]:[a.id]; + const event=appendEvent(world,'RESOURCE_GATHERED',c.id,{objectId:o.id,depositId:d.id,quantity:q},causes,at); o.lastPhysicalEventId=event.id; } function consumeFromDeposit(world,c,a,type,amount,restore,at){const d=world.resourceDeposits.find(x=>x.id===a.targetId&&x.type===type);if(!d||d.quantity