feat(plugin): weighted compressor selection with per-message depth tracking

The compressor now scores compartments using a weighted formula
(0.7 × age + 0.3 × inverse compression depth) instead of always
picking the oldest. Per-message compression depth is tracked in a
new compression_depth table and incremented after each successful
compression pass, preventing the earliest compartments from being
re-compressed indefinitely while untouched middle history exists.

Includes a 3× token expansion guard that falls back to the oldest
contiguous subset when scored picks span too wide a range.

Author: ualtinok

packages/plugin/src/features/magic-context/compression-depth-storage.test.ts

@@ -0,0 +1,72 @@
+/// <reference types="bun-types" />
+
+import { afterEach, describe, expect, it } from "bun:test";
+import { mkdtempSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import {
+    clearCompressionDepth,
+    closeDatabase,
+    getAverageCompressionDepth,
+    getMaxCompressionDepth,
+    incrementCompressionDepth,
+    openDatabase,
+} from "./storage";
+
+const tempDirs: string[] = [];
+const originalXdgDataHome = process.env.XDG_DATA_HOME;
+
+function makeTempDir(prefix: string): string {
+    const dir = mkdtempSync(join(tmpdir(), prefix));
+    tempDirs.push(dir);
+    return dir;
+}
+
+function useTempDataHome(prefix: string): void {
+    process.env.XDG_DATA_HOME = makeTempDir(prefix);
+}
+
+afterEach(() => {
+    closeDatabase();
+    process.env.XDG_DATA_HOME = originalXdgDataHome;
+
+    for (const dir of tempDirs) {
+        rmSync(dir, { recursive: true, force: true });
+    }
+    tempDirs.length = 0;
+});
+
+describe("compression-depth-storage", () => {
+    it("increments depth across a range and treats missing ordinals as zero in averages", () => {
+        useTempDataHome("compression-depth-range-");
+        const db = openDatabase();
+
+        incrementCompressionDepth(db, "ses-depth", 2, 4);
+
+        expect(getAverageCompressionDepth(db, "ses-depth", 1, 4)).toBe(0.75);
+        expect(getAverageCompressionDepth(db, "ses-depth", 2, 4)).toBe(1);
+        expect(getMaxCompressionDepth(db, "ses-depth")).toBe(1);
+    });
+
+    it("accumulates repeated compression passes", () => {
+        useTempDataHome("compression-depth-repeat-");
+        const db = openDatabase();
+
+        incrementCompressionDepth(db, "ses-depth", 1, 3);
+        incrementCompressionDepth(db, "ses-depth", 2, 4);
+
+        expect(getAverageCompressionDepth(db, "ses-depth", 1, 4)).toBe(1.5);
+        expect(getMaxCompressionDepth(db, "ses-depth")).toBe(2);
+    });
+
+    it("clears session depth rows", () => {
+        useTempDataHome("compression-depth-clear-");
+        const db = openDatabase();
+
+        incrementCompressionDepth(db, "ses-depth", 1, 5);
+        clearCompressionDepth(db, "ses-depth");
+
+        expect(getAverageCompressionDepth(db, "ses-depth", 1, 5)).toBe(0);
+        expect(getMaxCompressionDepth(db, "ses-depth")).toBe(0);
+    });
+});

packages/plugin/src/features/magic-context/compression-depth-storage.ts

@@ -0,0 +1,105 @@
+import type { Database } from "bun:sqlite";
+
+type PreparedStatement = ReturnType<Database["prepare"]>;
+
+const incrementDepthStatements = new WeakMap<Database, PreparedStatement>();
+const totalDepthStatements = new WeakMap<Database, PreparedStatement>();
+const maxDepthStatements = new WeakMap<Database, PreparedStatement>();
+const clearDepthStatements = new WeakMap<Database, PreparedStatement>();
+
+interface TotalDepthRow {
+    total_depth: number;
+}
+
+interface MaxDepthRow {
+    max_depth: number;
+}
+
+function getIncrementDepthStatement(db: Database): PreparedStatement {
+    let stmt = incrementDepthStatements.get(db);
+    if (!stmt) {
+        stmt = db.prepare(
+            "INSERT INTO compression_depth (session_id, message_ordinal, depth) VALUES (?, ?, 1) ON CONFLICT(session_id, message_ordinal) DO UPDATE SET depth = depth + 1",
+        );
+        incrementDepthStatements.set(db, stmt);
+    }
+    return stmt;
+}
+
+function getTotalDepthStatement(db: Database): PreparedStatement {
+    let stmt = totalDepthStatements.get(db);
+    if (!stmt) {
+        stmt = db.prepare(
+            "SELECT COALESCE(SUM(depth), 0) AS total_depth FROM compression_depth WHERE session_id = ? AND message_ordinal BETWEEN ? AND ?",
+        );
+        totalDepthStatements.set(db, stmt);
+    }
+    return stmt;
+}
+
+function getMaxDepthStatement(db: Database): PreparedStatement {
+    let stmt = maxDepthStatements.get(db);
+    if (!stmt) {
+        stmt = db.prepare(
+            "SELECT COALESCE(MAX(depth), 0) AS max_depth FROM compression_depth WHERE session_id = ?",
+        );
+        maxDepthStatements.set(db, stmt);
+    }
+    return stmt;
+}
+
+function getClearDepthStatement(db: Database): PreparedStatement {
+    let stmt = clearDepthStatements.get(db);
+    if (!stmt) {
+        stmt = db.prepare("DELETE FROM compression_depth WHERE session_id = ?");
+        clearDepthStatements.set(db, stmt);
+    }
+    return stmt;
+}
+
+export function incrementCompressionDepth(
+    db: Database,
+    sessionId: string,
+    startOrdinal: number,
+    endOrdinal: number,
+): void {
+    if (endOrdinal < startOrdinal) {
+        return;
+    }
+
+    db.transaction(() => {
+        const stmt = getIncrementDepthStatement(db);
+        for (let ordinal = startOrdinal; ordinal <= endOrdinal; ordinal += 1) {
+            stmt.run(sessionId, ordinal);
+        }
+    })();
+}
+
+export function getAverageCompressionDepth(
+    db: Database,
+    sessionId: string,
+    startOrdinal: number,
+    endOrdinal: number,
+): number {
+    if (endOrdinal < startOrdinal) {
+        return 0;
+    }
+
+    const row = getTotalDepthStatement(db).get(
+        sessionId,
+        startOrdinal,
+        endOrdinal,
+    ) as TotalDepthRow | null;
+    const totalDepth = typeof row?.total_depth === "number" ? row.total_depth : 0;
+    const messageCount = endOrdinal - startOrdinal + 1;
+    return totalDepth / messageCount;
+}
+
+export function getMaxCompressionDepth(db: Database, sessionId: string): number {
+    const row = getMaxDepthStatement(db).get(sessionId) as MaxDepthRow | null;
+    return typeof row?.max_depth === "number" ? row.max_depth : 0;
+}
+
+export function clearCompressionDepth(db: Database, sessionId: string): void {
+    getClearDepthStatement(db).run(sessionId);
+}

packages/plugin/src/features/magic-context/message-index.ts

@@ -6,6 +6,7 @@ import {
 } from "../../hooks/magic-context/read-session-chunk";
 import type { RawMessage } from "../../hooks/magic-context/read-session-raw";
 import { removeSystemReminders } from "../../shared/system-directive";
+import { clearCompressionDepth } from "./compression-depth-storage";
 
 type PreparedStatement = ReturnType<Database["prepare"]>;
 
@@ -122,6 +123,7 @@ export function clearIndexedMessages(db: Database, sessionId: string): void {
     db.transaction(() => {
         getDeleteFtsStatement(db).run(sessionId);
         getDeleteIndexStatement(db).run(sessionId);
+        clearCompressionDepth(db, sessionId);
     })();
 }
 

packages/plugin/src/features/magic-context/storage-db.test.ts

@@ -51,7 +51,7 @@ describe("storage-db", () => {
             expect(isDatabasePersisted(db)).toBe(true);
         });
 
-        it("#when called first time #then creates all 5 required tables", () => {
+        it("#when called first time #then creates required tables", () => {
             useTempDataHome("storage-db-tables-");
 
             const db = openDatabase();
@@ -61,7 +61,13 @@ describe("storage-db", () => {
                 .all() as Array<{ name: string }>;
             const tableNames = tables.map((t) => t.name);
             expect(tableNames).toEqual(
-                expect.arrayContaining(["tags", "pending_ops", "source_contents", "session_meta"]),
+                expect.arrayContaining([
+                    "tags",
+                    "pending_ops",
+                    "source_contents",
+                    "compression_depth",
+                    "session_meta",
+                ]),
             );
         });
 
@@ -80,6 +86,7 @@ describe("storage-db", () => {
                     "idx_pending_ops_session",
                     "idx_source_contents_session",
                     "idx_compartments_session",
+                    "idx_compression_depth_session",
                     "idx_session_facts_session",
                     "idx_session_notes_session",
                 ]),

packages/plugin/src/features/magic-context/storage-db.ts

@@ -60,6 +60,15 @@ export function initializeDatabase(db: Database): void {
       created_at INTEGER NOT NULL,
       UNIQUE(session_id, sequence)
     );
+    CREATE INDEX IF NOT EXISTS idx_compartments_session ON compartments(session_id);
+
+    CREATE TABLE IF NOT EXISTS compression_depth (
+      session_id TEXT NOT NULL,
+      message_ordinal INTEGER NOT NULL,
+      depth INTEGER NOT NULL DEFAULT 0,
+      PRIMARY KEY(session_id, message_ordinal)
+    );
+    CREATE INDEX IF NOT EXISTS idx_compression_depth_session ON compression_depth(session_id);
 
     CREATE TABLE IF NOT EXISTS session_facts (
       id INTEGER PRIMARY KEY AUTOINCREMENT,
@@ -245,7 +254,6 @@ CREATE INDEX IF NOT EXISTS idx_dream_queue_pending ON dream_queue(started_at, en
       created_at INTEGER NOT NULL
     );
 
-    CREATE INDEX IF NOT EXISTS idx_compartments_session ON compartments(session_id);
     CREATE INDEX IF NOT EXISTS idx_session_facts_session ON session_facts(session_id);
     CREATE INDEX IF NOT EXISTS idx_recomp_compartments_session ON recomp_compartments(session_id);
     CREATE INDEX IF NOT EXISTS idx_recomp_facts_session ON recomp_facts(session_id);

packages/plugin/src/features/magic-context/storage-meta-session.ts

@@ -1,4 +1,5 @@
 import type { Database } from "bun:sqlite";
+import { clearCompressionDepth } from "./compression-depth-storage";
 import { clearIndexedMessages } from "./message-index";
 import {
     BOOLEAN_META_KEYS,
@@ -70,6 +71,7 @@ export function clearSession(db: Database, sessionId: string): void {
         db.prepare("DELETE FROM tags WHERE session_id = ?").run(sessionId);
         db.prepare("DELETE FROM session_meta WHERE session_id = ?").run(sessionId);
         db.prepare("DELETE FROM compartments WHERE session_id = ?").run(sessionId);
+        clearCompressionDepth(db, sessionId);
         db.prepare("DELETE FROM session_facts WHERE session_id = ?").run(sessionId);
         db.prepare("DELETE FROM session_notes WHERE session_id = ?").run(sessionId);
         db.prepare("DELETE FROM recomp_compartments WHERE session_id = ?").run(sessionId);

packages/plugin/src/features/magic-context/storage-meta.test.ts

@@ -83,7 +83,7 @@ describe("storage-meta", () => {
             //#then
             // 2 transactions: outer clearSession + nested clearIndexedMessages
             expect(db.transaction).toHaveBeenCalledTimes(2);
-            expect(db.prepare).toHaveBeenCalledTimes(11);
+            expect(db.prepare).toHaveBeenCalledTimes(12);
         });
     });
 });

packages/plugin/src/features/magic-context/storage.ts

@@ -7,6 +7,12 @@ export {
     replaceAllCompartmentState,
     type SessionFact,
 } from "./compartment-storage";
+export {
+    clearCompressionDepth,
+    getAverageCompressionDepth,
+    getMaxCompressionDepth,
+    incrementCompressionDepth,
+} from "./compression-depth-storage";
 export {
     clearIndexedMessages,
     deleteIndexedMessage,