Stabilize knowledge memory and repeated harvest growth

MANIFEST.sha256

@@ -37,7 +37,7 @@ eee99e85b57fec9864f92915a28eb75033e54da4fdb3c97e8de2f499cffefaf4  docs/operation
 d21ac14687f4f46b05b61622da37c7556c2d843c956215ac9e82bb9e4ce69398  docs/operations/heartbeat-recovery.md
 5a82eb9533678d61622a462bd9de4ad34a3c306cb5fc3129dbeefc1088a67754  docs/operations/memory-path-v2.md
 455a149710f27cb6c5ee900d3989fdf5f17bea24d90cd479f4c409db0ef5d96f  docs/operations/memory-pressure-recovery.md
-d8bc46174615c8473aa07f9acbc8dd56f5e8b32cd0d93f6d24ac0c9f0acb07b3  docs/operations/packed-knowledge-checkpoints.md
+e22d0d74080e440ba093beb670b2f1b45d2cd9fe58c2fd8b29b1887859bc0e8d  docs/operations/packed-knowledge-checkpoints.md
 9b52f43e56fa717704a1ce1e92b781fcf54ddc2fe2b48080da2ab4e3dcdff8af  docs/operations/websocket-heartbeat-recovery.md
 0ecf68ef4eb4b31275e584f0ebd9fc1fa72893c5ae514e14f4996d1eb1654589  docs/reference/repository-map.md
 9025958e141161d60f59cb801b9aeae31cafca5538b59c94e0042a1a7bd330ef  functions/_lib/auth.js
@@ -144,6 +144,7 @@ de8f675a1e0a3cb96705cf72bdeb2aca9fa0867c3b7246853bc63821c777bc31  tests/test_sov
 415105e34c5ebae6e0fe4dd0455e156a232b0e2c476d73844e4d3d12449aad71  tests/test_sovereign_cognition_scheduler.mjs
 77716f2c424d56e2d7ac8966628706853b94e6e4da1debd18fbd180ed37f380b  tests/test_sovereign_cognition_state.mjs
 a9e8632aa631a2da14cd8ab6637e7f20d187b22fe8159da15776ffe43a006a8d  tests/test_sovereign_cold_cpu_window.mjs
+e65e8c757b0e2ef6859848b158fd961fbe3baa813675d9a3cd7183074c3da4ff  tests/test_sovereign_compact_string_index.mjs
 2d64a32a4e5a8375c7558b3859cc5636af13b0866204506bde8c3455092f7f0a  tests/test_sovereign_conservation.mjs
 96186ae225e60cd3ccef5076bf9e17d813654bb7e25e3738449b66e41117d7c6  tests/test_sovereign_construction_demand.mjs
 408ca361ae15be807f3f3e7a9507b98d2243c9c6b79864f9e49d72172d899e48  tests/test_sovereign_construction_physics_upgrade.mjs
@@ -159,10 +160,12 @@ b3a62f70768edc4cf2c0a2d509c7e8055ad2002c3e1f79f292d728c39c64a3db  tests/test_sov
 2b2d7007ed708eab76efc151f59e30d5239fcee65b83e0fd92b02add258ad5c0  tests/test_sovereign_entity_knowledge_cache.mjs
 bbe9b971d7162e2e44d9a01fc31587df88cd715c84b4c63fdf06bf879aa8293b  tests/test_sovereign_epistemics.mjs
 a71ce1ffa138b92138755cc64eb828f2e145e78c4b369b1eda2911078700a003  tests/test_sovereign_exposure_upgrade.mjs
+201948179849116d2cd0877c4ae1425605fdd6b6ccc638255e18467fc6fb30cb  tests/test_sovereign_gather_stability.mjs
 831ecc901261fe003f15b3f2a2ab2f2a41273bb1fcbb14fa2377d99d1efe8899  tests/test_sovereign_genesis.mjs
 849b9fcdf721848651b67ca829e802616791c296b95cc1cc874e4f1be4a022f4  tests/test_sovereign_graphics2.mjs
 e0d0c6bcba695b34d69aa3c4ceeae502a80b3dc61843ef12f3cf555e883d5a66  tests/test_sovereign_history.mjs
 ca70bc968311b97bf9676c762c25a41c5d682d7753d32ea1ecd0108b5eea6e78  tests/test_sovereign_impact_physics_upgrade.mjs
+5070dc74b7b3fe9d8e5e6b3a73e18b877ff1223159fa6d4f860a7cd878f57308  tests/test_sovereign_index_string_memory.mjs
 07d222fb857cbb7b9e3927e20527ad101d5f9d97a8a3569f323d9dae7227ff39  tests/test_sovereign_kernel_integration.mjs
 8a5829515cfcbf398495e02158f555ba28d3ca3618132c54a28fb8eb0ab2e880  tests/test_sovereign_knowledge_archive.mjs
 d21f8b3342cd4f196ac9a7c050c50a02a827d238c8aa41dc39bdaa6ae6728158  tests/test_sovereign_knowledge_encoding.mjs
@@ -217,7 +220,7 @@ fe8914a4a9cb0b62c9dc0f499ee2e9bc7a61e31a7fd29ee3bae1bbf958405429  tests/test_sov
 f345373ad773f9a2feaf38a7337d2514bf3cc3538bedcf79fff836d76911ed1e  tests/test_sovereign_websocket_signal.mjs
 6d2cf0bac9610895650c096698c50691c863dde56fd8f262af4646907a630a09  worker/src/archive-digest.js
 20517cc692843f2190984462fc7cb147caf27401ac72541b66ddf0cbd5ecf90d  worker/src/index.js
-c5f54fe81bb3edac9c2384607e6298dc579d9d6f29778eea59d2090ae4e97b24  worker/src/knowledge-archive.js
+5262b8cd32219aa876912fa9cab2329e722be52aff45449afe42123f5b5511cf  worker/src/knowledge-archive.js
 4b00db1fd2286f8d9dd4b522a9eb1111bd24b4e1befc705b5d6c18b45a4b3c0b  worker/src/neuron-governor.js
 fa52d714982a95ef708fc8958b9dc2569f0a03b814a76e1cb89866bb6d3b3974  worker/src/persistence.js
 805f6cb28bb8b6d3a0badd83c93bccd9671fa01a3b4b92b7042b0743325ac243  worker/src/vendor/fflate.LICENSE
@@ -232,13 +235,14 @@ c5ce7c1f20314fdb3ee302545401010fd1800d62739a77519684068e135c22cd  world/biology.
 cd7a7faa5ea4f4e790d76d89b7d0a1e0783eeb1a81ed0e0248d7ec090787cc1a  world/cognition-queue.js
 f9697932e6e6728f0374858b2f19e27275204a449d2c1f7268e94dfedffecaed  world/cognition-state.js
 cfc8bcbcbcd47b2e9b4a52b838a1bb7837947d98077bf7f45a5d8536725c2446  world/cognition.js
+8f5f49b4fa80dc4ecc2011b963128db8afc7943a903bf5a79f3ca82ffb35217d  world/compact-string-index.js
 f652161b6761b77228a0a6abae5527e5e19fc2481aba0aeaa2ed5934bf68f3ff  world/constants.js
 d89131a70e43eddbb1cc0f4a699e48766b90eca3b7f56edeba92d8bd22ebaf3e  world/designs.js
 848a0d0eaefef35306d30b50a30b4b22400aa26fffad3c88efe9b62ec5eed12f  world/destruction.js
 3bb66b15a8796801310e798ea0e12961ae0744536d7048142e2d452a0b64e18b  world/disease.js
-9e5c628bce5573ba65aaeaddfbe97b11a9d22af4020669cef76eddb4da9a1973  world/engine.js
+cc384e3f39c59f8bcaf79f2c2e4457fd3070778d3fe04bcf6d34497154395179  world/engine.js
 6c718ab427ac70d5846320bc668aabb005b9ef3f517c799ed8035fa1d0a96d0a  world/environment.js
-e5b8c100507052d613aab8e67eaebfe7905ea097bd5d47701870e213fcff09ca  world/epistemics.js
+2d7adc7d7106f94e96a81935d5b7f0fe675e8399034597e50e183099a4dc1068  world/epistemics.js
 5420339655dd40b2b20a14576e9e0379be6d8d97668810ba941e26a542ca741b  world/evidence-pool.js
 ad4e953667ba12b6acce5b7407436458e4edada4d2dd3b584d854af8d65f21cf  world/genesis.js
 7f8a07c4dd491f6eee3baf10ea55c4dad637f47ebf68d9be89f62c1766cfc658  world/genetics.js

docs/operations/packed-knowledge-checkpoints.md

@@ -234,3 +234,60 @@ See [Cloudflare pricing](https://developers.cloudflare.com/durable-objects/platf
 Account request, CPU, storage and row limits still apply. This change neither
 upgrades the plan nor promises unlimited retention or availability on a finite
 Free account.
+
+## Shared strings and compact exact lookup directories
+
+The main deployment from PR 59 failed its production quiet gate with HTTP 503.
+A single post-failure diagnostic on October 9 recorded an actual isolate memory
+reset. Cold recovery contained 875,469 knowledge records, 875,471 sources, 50,255
+memories and 12,023 objects. Earlier sparse native fixtures did not reproduce
+the repeated concept/entity strings and large personal membership directories.
+
+A bounded pool now shares detached immutable concept/entity/fallback/known-entity
+strings (65,536 entries and 2 Mi code units maximum). It contains neither personal
+membership nor evidence. Exact per-Citizen lookup directories retain Uint32
+positions into their existing string arrays rather than full Map/Set buckets.
+Hash collisions compare the complete original key; append, deletion, forgetting,
+first-active duplicate selection and non-string keys preserve their semantics.
+No canonical knowledge, object, source, chronology or private membership is removed.
+The packed-v2 durable format and backward readers are unchanged.
+
+The controlled native fixture reconstructs the public physical shape and 875,469
+private knowledge records, 50,255 initial memories and 108 Citizens. An additional
+3,585 inert object-shaped records brings object count to 12,023; these are synthetic
+records because the public projection omits depleted objects. The old reader
+exhausts a 96 MiB V8 old-space cap during cold recovery. String sharing alone passes
+the smaller fixture but exhausts an 80 MiB old-space cap in the first physical phase
+with all objects. Shared strings plus numeric lookup directories pass cold recovery,
+thirty chronological minutes and checkpoint publication under 80 MiB old space
+and 4 MiB semi space: 4 moved Citizens, 72 changed actions, 100 changed bodies, all 108
+IDs retained. Sampled JavaScript heap plus ArrayBuffer backing peaks at 112.48 MiB.
+Encoding takes 4.66 seconds; first physical phase takes 11.83 seconds. A subsequent
+cold restart recovers minute 177,183 with 876,938 knowledge records and both sampled
+original private sources intact.
+
+These are local controlled measurements, not an exact reproduction of Cloudflare's
+128 MiB account/runtime enforcement or a guarantee of indefinite capacity. Private
+sources, cold lookup semantics, collision/deletion handling and the existing
+known-entity read-count gate have regression coverage. Production must pass all
+unchanged gates and a separate ten-minute quiet proof before stability clearance.
+
+## Repeated harvest growth
+
+Compact caches alone cannot make continuous allocation sustainable. The duration
+audit found that every GATHER action created another physical object, possession
+and personal entity ID, prompting all nearby Citizens to acquire another concept.
+Repeated ordinary harvests now add matter to one compatible personal stack from
+the same deposit. Existing objects/IDs and original provenance remain intact.
+Every harvest still emits its own causal receipt; the stack points to the latest
+physical event, whose causes also retain the prior stack contribution (or the
+original action for a legacy stack). Reserved, heated, transformed, damaged, foreign and different-source
+objects are excluded. Fractional availability caps the harvested quantity exactly;
+a depleted source cannot manufacture the former minimum 0.1 kg of matter.
+
+A thousand repeated harvests retain one new object and one new personal entity,
+with all thousand distinct action receipts, 2,000 kg gathered and 3,000 kg remaining
+in the source. This prevents this routine activity from growing object/concept
+directories with elapsed time. Other genuine discoveries, births, transformations
+and preserved historical records still consume finite resources; account-level
+Free quotas remain external capacity constraints.

tests/test_sovereign_compact_string_index.mjs

@@ -0,0 +1,36 @@
+import test from 'node:test';
+import assert from 'node:assert/strict';
+import {CompactStringIndex} from '../world/compact-string-index.js';
+import {sqliteStorage,wake} from './helpers/sovereign-sqlite.mjs';
+import {forget,knowledgeEntry,knowsEntity} from '../world/epistemics.js';
+
+test('compact lookup preserves exact collision, deletion, append and non-string Map semantics',()=>{
+  const entries=[];const index=new CompactStringIndex(entries);
+  const expected=new Map();
+  for(let i=0;i<12000;i++){const key=`entity:${i%9000}`;entries.push(key);index.set(key,i);expected.set(key,i);}
+  for(const [key,value] of expected)assert.equal(index.get(key),value);
+  for(let i=0;i<9000;i+=3){assert.equal(index.delete(`entity:${i}`),true);expected.delete(`entity:${i}`);}
+  for(let i=0;i<9000;i++){assert.equal(index.has(`entity:${i}`),expected.has(`entity:${i}`));assert.equal(index.get(`entity:${i}`),expected.get(`entity:${i}`));}
+  const object={};for(const key of [null,undefined,false,0,NaN,object]){entries.push(key);index.set(key,entries.length-1);assert.equal(index.get(key),entries.length-1);}
+  assert.equal(index.has({}),false);assert.equal(index.delete('absent'),false);
+  assert.ok(index.table.byteLength<=entries.length*16,'lookup must use a compact numeric directory');
+});
+
+test('compact archive lookup retains the first active duplicate through changes and cold recovery',async t=>{
+  const storage=sqliteStorage();t.after(()=>storage.db.close());const {instance}=await wake(storage);const c=instance.world.citizens[0];
+  const entry=(concept,eventId)=>({concept,active:true,confidence:.8,provenance:[{kind:'observation',eventId}]});
+  c.knowledge=[entry('duplicate','source:a'),entry('duplicate','source:b'),entry('unrelated','source:c')];
+  await instance.persist({forceSeal:true});let r=(await wake(storage)).instance.world.citizens[0];
+  assert.equal(knowledgeEntry(r,'duplicate').provenance[0].eventId,'source:a');
+  forget(r,'duplicate');assert.equal(knowledgeEntry(r,'duplicate').provenance[0].eventId,'source:b');
+  r.knowledge[1].concept='changed';assert.equal(knowledgeEntry(r,'duplicate'),null);assert.equal(knowledgeEntry(r,'changed').provenance[0].eventId,'source:b');
+  assert.equal(knowledgeEntry(r,'unrelated').provenance[0].eventId,'source:c');
+});
+
+test('compact entity membership stays private and indexes append and replacement without granting unknown IDs',()=>{
+  const a={knownEntityIds:Array.from({length:12000},(_,i)=>`entity:${i}`)},b={knownEntityIds:['entity:5']};
+  assert.equal(knowsEntity(a,'entity:11999'),true);assert.equal(knowsEntity(a,'entity:12000'),false);
+  a.knownEntityIds.push('entity:12000');assert.equal(knowsEntity(a,'entity:12000'),true);assert.equal(knowsEntity(b,'entity:12000'),false);
+  a.knownEntityIds=['entity:other'];assert.equal(knowsEntity(a,'entity:5'),false);assert.equal(knowsEntity(a,'entity:other'),true);
+  a.knownEntityIds.length=0;assert.equal(knowsEntity(a,'entity:other'),false);
+});

tests/test_sovereign_gather_stability.mjs

@@ -0,0 +1,41 @@
+import test from 'node:test';
+import assert from 'node:assert/strict';
+import {createSovereignGenesis} from '../world/genesis.js';
+import {startAction,interruptAction} from '../world/actions.js';
+import {advanceWorldTo} from '../world/engine.js';
+import {verifyLedger} from '../world/ledger.js';
+
+function fixture(){const world=createSovereignGenesis({seed:37,realEpochMs:0}),c=world.citizens[0],d=world.resourceDeposits.find(x=>x.type==='timber');world.citizens=[c];c.position={...d.position};d.renewPerDay=0;d.quantity=5000;return {world,c,d};}
+function gather(f,quantity=2){const {world,c,d}=f;interruptAction(world,c,'controlled fixture');Object.assign(c.body,{hydration:100,calories:100,health:100,sleepPressure:10});const a=startAction(world,c,{type:'GATHER',durationMinutes:1,targetId:d.id,payload:{quantity},purpose:'self-directed'},world.clock.worldMinute);advanceWorldTo(world,a.endsWorldMinute*1000);return a;}
+
+test('a thousand repeated gathers retain one compatible stack, every harvest receipt, matter and existing IDs',()=>{
+  const f=fixture(),ids=f.world.objects.map(o=>o.id),count=ids.length,initialEntities=new Set([...ids,...f.world.resourceDeposits.map(d=>d.id),...f.world.buildings.map(b=>b.id),f.c.id]);
+  for(let i=0;i<1000;i++)gather(f);
+  const stacks=f.world.objects.filter(o=>o.holderId===f.c.id&&o.provenance?.depositId===f.d.id);
+  assert.equal(stacks.length,1);assert.equal(f.world.objects.length,count+1);assert.equal(stacks[0].quantity,2000);assert.equal(f.d.quantity,3000);
+  assert.ok(ids.every(id=>f.world.objects.some(o=>o.id===id)));assert.equal(new Set(f.c.possessions).size,f.c.possessions.length);
+  assert.deepEqual(f.c.knownEntityIds.filter(id=>!initialEntities.has(id)),[stacks[0].id],'harvests must introduce only one new personal entity membership');
+  const receipts=f.world.ledger.filter(e=>e.type==='RESOURCE_GATHERED');assert.equal(receipts.length,1000);assert.equal(new Set(receipts.map(e=>e.causes[0])).size,1000);
+  for(let i=1;i<receipts.length;i++)assert.ok(receipts[i].causes.includes(receipts[i-1].id),'every stack addition must preserve its preceding physical contribution');
+  assert.equal(stacks[0].provenance.actionId,receipts[0].causes[0]);assert.equal(stacks[0].lastPhysicalEventId,receipts.at(-1).id);assert.equal(verifyLedger(f.world),true);
+});
+
+test('gathering never refills transformed, reserved, foreign or different-source objects',()=>{
+  for(const change of [{holderId:'other'},{reservedProjectId:'project:x'},{condition:.5},{temperatureC:80},{geometry:{lengthM:1}},{properties:{processed:true}},{provenance:{type:'GATHERED',depositId:'other'}}]){
+    const f=fixture();gather(f);const original=f.world.objects.find(o=>o.holderId===f.c.id&&o.provenance?.depositId===f.d.id);Object.assign(original,change);const quantity=original.quantity;
+    gather(f);assert.equal(original.quantity,quantity);assert.equal(f.world.objects.filter(o=>o.kind==='gathered_material').length,2);
+  }
+});
+
+test('depleted and fractional deposits cannot manufacture matter or go negative',()=>{
+  const f=fixture();f.d.quantity=.03;gather(f,2);
+  const stack=f.world.objects.find(o=>o.holderId===f.c.id&&o.provenance?.depositId===f.d.id);assert.equal(stack.quantity,.03);assert.equal(f.d.quantity,0);
+  gather(f,2);assert.equal(stack.quantity,.03);assert.equal(f.d.quantity,0);assert.equal(f.world.ledger.filter(e=>e.type==='RESOURCE_GATHERED').length,1);
+});
+
+test('reused legacy material retains its original action as the causal anchor',()=>{
+  const f=fixture(),first=gather(f),stack=f.world.objects.find(o=>o.holderId===f.c.id&&o.provenance?.depositId===f.d.id);
+  delete stack.lastPhysicalEventId;const second=gather(f);
+  const receipt=f.world.ledger.findLast(e=>e.type==='RESOURCE_GATHERED');
+  assert.equal(stack.quantity,4);assert.deepEqual(receipt.causes,[second.id,first.id]);assert.equal(stack.provenance.actionId,first.id);
+});