diff --git a/.github/release-notes/v1.2.2.md b/.github/release-notes/v1.2.2.md new file mode 100644 index 000000000..c604f0062 --- /dev/null +++ b/.github/release-notes/v1.2.2.md @@ -0,0 +1,22 @@ +# Memmy v1.2.2 + +## Highlights + +- Improved memory initialization and turn capture by moving bounded work off the model path and exposing clearer background-capture diagnostics. +- Improved bundled Memory runtime upgrades by selecting runtimes by build identity and validating the installed runtime across macOS and Windows. +- Added long-document reading with bounded chunks and continuation offsets for PDF and Office files. +- Improved agent-source ingestion, auto-injection, and skill distribution with broader regression coverage and clearer lifecycle handling. +- Fixed LM Studio and OpenAI-compatible JSON response-format incompatibilities by retrying with text mode when needed. + +## Agent and Desktop improvements + +- Refreshed installed Claude Code, Codex, and Cursor hooks when their bundled revision or on-disk files are stale, and record the revision only after a successful install. +- Kept queued memory processing moving after session close and retained end-topic turns that contain feedback or instructions for L1 capture. +- Improved filesystem and document tools with consistent line numbering, continued long-document reads, and expanded agent-runtime regression coverage. +- Tightened analytics boundaries by avoiding raw search queries, memory text, and identifiers in CLI command actions while reporting stored-memory and source-layer events consistently. + +## Upgrade notes + +- Memmy Desktop and Agent are version `1.2.2`. +- The bundled Memory service, viewer, and CLI remain independently versioned. +- Memory protocol version remains `1`. diff --git a/.github/workflows/memory-release.yml b/.github/workflows/memory-release.yml index 03d5ec56f..71c26a54e 100644 --- a/.github/workflows/memory-release.yml +++ b/.github/workflows/memory-release.yml @@ -6,7 +6,7 @@ on: version: description: Memory version (X.Y.Z) required: true - default: 2.1.3 + default: 2.1.4 push: tags: - "memory-v*" @@ -26,7 +26,9 @@ jobs: - run: npm ci - run: npm run typecheck -w @memmy/memory - run: npm test -w @memmy/memory - - run: npx vitest run App/shell/desktop/tests/packaged-runtime-boundary.test.ts App/shell/desktop/tests/runtime-services.test.ts + - run: npm run build -w @memmy/migrations + - run: npm run build -w @memmy/local-api-contracts + - run: npx vitest run App/shell/desktop/tests/packaged-runtime-boundary.test.ts App/shell/desktop/tests/runtime-services.test.ts App/shell/desktop/tests/runtime-memory-upgrade.test.ts tests/packaged-runtime-config.test.mjs runtime: needs: verify diff --git a/.github/workflows/windows-memory-validation.yml b/.github/workflows/windows-memory-validation.yml index 2a02aff15..465b3006b 100644 --- a/.github/workflows/windows-memory-validation.yml +++ b/.github/workflows/windows-memory-validation.yml @@ -8,7 +8,11 @@ on: - "Memory/**" - "AgentSourceCore/**" - "App/shell/desktop/build/**" + - "App/shell/desktop/src/main/runtime-services.ts" + - "App/shell/desktop/tests/runtime-memory-upgrade.test.ts" - "App/shell/desktop/tests/windows-standard-upgrade-check.test.ts" + - "scripts/internal/shared/memory-runtime-build-identity.mjs" + - "scripts/internal/win/create-memory-runtime-manifest.mjs" - "package.json" - "package-lock.json" workflow_dispatch: @@ -57,11 +61,19 @@ jobs: run: npm ci - name: Build shared Agent source contracts run: npm run build -w @memmy/agent-source-core + - name: Build desktop startup dependencies + run: | + npm run build -w @memmy/migrations + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + npm run build -w @memmy/local-api-contracts + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } - name: Test Windows Memory contracts and temporary upgrade layouts run: >- node node_modules/vitest/vitest.mjs run tests/windows-memory-workflow.test.ts Memory/tests/runtime-installer-windows.test.ts + Memory/tests/runtime-installer.test.ts + App/shell/desktop/tests/runtime-memory-upgrade.test.ts App/shell/desktop/tests/windows-standard-upgrade-check.test.ts --reporter=default --reporter=json --outputFile=windows-memory-unit-results.json - name: Test the native temporary Memory task diff --git a/App/backend/src/adapters/inbound/local-api/tests/agent-sources-route.test.ts b/App/backend/src/adapters/inbound/local-api/tests/agent-sources-route.test.ts index f2b420568..3440ab189 100644 --- a/App/backend/src/adapters/inbound/local-api/tests/agent-sources-route.test.ts +++ b/App/backend/src/adapters/inbound/local-api/tests/agent-sources-route.test.ts @@ -1039,6 +1039,9 @@ function createFakeAgentSourceService(): AgentSourceService { async uninstallPlugin() { return undefined; }, + async isInstalledHookCurrent() { + return true; + }, async detectMemoryPluginConflicts() { return []; } diff --git a/App/backend/src/adapters/outbound/skill-writer/claude-code/target.ts b/App/backend/src/adapters/outbound/skill-writer/claude-code/target.ts index fcf45fc11..b114faa71 100644 --- a/App/backend/src/adapters/outbound/skill-writer/claude-code/target.ts +++ b/App/backend/src/adapters/outbound/skill-writer/claude-code/target.ts @@ -3,6 +3,7 @@ import { mkdir, readFile, rename, rm, stat, writeFile } from "node:fs/promises"; import { homedir } from "node:os"; import { basename, dirname, join } from "node:path"; import { createNodeHookCommand } from "../hook-command.js"; +import { isInstalledMemmyHookCurrent, memmyHookRevisionField } from "../hook-revision.js"; import { readMemmyMemoryServiceConfig } from "../memmy-runtime-config.js"; import { removeMemmySkillDirectory, replaceMemmySkillDirectory } from "../skill-directory.js"; import { renderMemmyPluginSkillManifest } from "../templates/memmy-plugin.js"; @@ -85,6 +86,19 @@ export function createClaudeCodeSkillTarget(deps: CreateClaudeCodeSkillTargetDep return (await readTextFile(join(root, TARGET_FILE_NAME))).includes(START_MARKER); }, + async isInstalledHookCurrent() { + const root = await this.resolveRootDirectory(); + if (!root) return false; + const hookDirectory = join(root, HOOK_DIRECTORY_NAME); + return isInstalledMemmyHookCurrent({ + source: CLAUDE_CODE_TARGET_ID, + mode: "claude-code", + hookScriptPath: join(hookDirectory, HOOK_SCRIPT_FILE_NAME), + bridgePath: join(hookDirectory, WORKSPACE_BRIDGE_FILE_NAME), + configPath: join(hookDirectory, HOOK_CONFIG_FILE_NAME) + }); + }, + async installPlugin(_targetId) { const root = await this.resolveRootDirectory(); if (!root) { @@ -93,11 +107,13 @@ export function createClaudeCodeSkillTarget(deps: CreateClaudeCodeSkillTargetDep const hookDirectory = join(root, HOOK_DIRECTORY_NAME); const hookScriptPath = join(hookDirectory, HOOK_SCRIPT_FILE_NAME); + const hookConfigPath = join(hookDirectory, HOOK_CONFIG_FILE_NAME); + const hookConfig = { + memmy_config_path: memmyConfigPath, + ...(await readMemmyMemoryServiceConfig(memmyConfigPath)) + }; await mkdir(hookDirectory, { recursive: true }); - await writeFileAtomically( - join(hookDirectory, HOOK_CONFIG_FILE_NAME), - `${JSON.stringify({ memmy_config_path: memmyConfigPath, ...(await readMemmyMemoryServiceConfig(memmyConfigPath)) }, null, 2)}\n` - ); + await writeFileAtomically(hookConfigPath, `${JSON.stringify(hookConfig, null, 2)}\n`); await writeFileAtomically( hookScriptPath, renderMemmyResumeHookScript({ source: CLAUDE_CODE_TARGET_ID, mode: "claude-code" }) @@ -118,6 +134,10 @@ export function createClaudeCodeSkillTarget(deps: CreateClaudeCodeSkillTargetDep upsertMarkerBlock(await readTextFile(filePath), renderMemmySkillBootstrapManifest(manifest)) ); await replaceMemmySkillDirectory(root, manifest); + await writeFileAtomically( + hookConfigPath, + `${JSON.stringify({ ...hookConfig, ...(await memmyHookRevisionField(CLAUDE_CODE_TARGET_ID, "claude-code")) }, null, 2)}\n` + ); }, async uninstallPlugin(_targetId) { diff --git a/App/backend/src/adapters/outbound/skill-writer/claude-code/tests/target.test.ts b/App/backend/src/adapters/outbound/skill-writer/claude-code/tests/target.test.ts index ee697e306..b33faf7f7 100644 --- a/App/backend/src/adapters/outbound/skill-writer/claude-code/tests/target.test.ts +++ b/App/backend/src/adapters/outbound/skill-writer/claude-code/tests/target.test.ts @@ -119,6 +119,30 @@ describe("claude code skill target", () => { } }); + it("marks a hook without a stored revision as outdated and refreshes it in place", async () => { + const { rootDirectory, memmyConfigPath } = createFixture(); + const target = createClaudeCodeSkillTarget({ rootDirectory, memmyConfigPath }); + const configPath = join(rootDirectory, "hooks", "memmy-memory-config.json"); + + await target.installPlugin?.("claude_code"); + const installed = JSON.parse(readFileSync(configPath, "utf8")) as { hook_revision?: string }; + expect(installed.hook_revision).toEqual(expect.stringMatching(/^[a-f0-9]{64}$/)); + await expect(target.isInstalledHookCurrent?.()).resolves.toBe(true); + + writeFileSync(configPath, `${JSON.stringify({ ...installed, hook_revision: "stale" }, null, 2)}\n`, "utf8"); + await expect(target.isInstalledHookCurrent?.()).resolves.toBe(false); + + writeFileSync(join(rootDirectory, "hooks", "memmy-resume-hook.mjs"), "old hook\n", "utf8"); + writeFileSync(configPath, `${JSON.stringify(installed, null, 2)}\n`, "utf8"); + await expect(target.isInstalledHookCurrent?.()).resolves.toBe(false); + rmSync(join(rootDirectory, "hooks", "memmy-resume-hook.mjs")); + writeFileSync(configPath, `${JSON.stringify(installed, null, 2)}\n`, "utf8"); + await expect(target.isInstalledHookCurrent?.()).resolves.toBe(false); + + await target.installPlugin?.("claude_code"); + await expect(target.isInstalledHookCurrent?.()).resolves.toBe(true); + }); + it("installs a UserPromptSubmit hook that blocks resume commands with top L1 candidates", async () => { const { rootDirectory, memmyConfigPath } = createFixture(); let requestBody: Record | undefined; diff --git a/App/backend/src/adapters/outbound/skill-writer/codex/target.ts b/App/backend/src/adapters/outbound/skill-writer/codex/target.ts index 54e75627c..a292d4d33 100644 --- a/App/backend/src/adapters/outbound/skill-writer/codex/target.ts +++ b/App/backend/src/adapters/outbound/skill-writer/codex/target.ts @@ -4,6 +4,7 @@ import { homedir } from "node:os"; import { basename, dirname, join } from "node:path"; import { resolveCodexHomeDirectory } from "../../agent-paths.js"; import { createNodeHookCommand } from "../hook-command.js"; +import { isInstalledMemmyHookCurrent, memmyHookRevisionField } from "../hook-revision.js"; import { readMemmyMemoryServiceConfig } from "../memmy-runtime-config.js"; import { removeMemmyResumeSkillDirectory, @@ -91,6 +92,19 @@ export function createCodexSkillTarget(deps: CreateCodexSkillTargetDeps = {}): S return (await readTextFile(join(root, TARGET_FILE_NAME))).includes(START_MARKER); }, + async isInstalledHookCurrent() { + const root = await this.resolveRootDirectory(); + if (!root) return false; + const hookDirectory = join(root, HOOK_DIRECTORY_NAME); + return isInstalledMemmyHookCurrent({ + source: CODEX_TARGET_ID, + mode: "codex", + hookScriptPath: join(hookDirectory, HOOK_SCRIPT_FILE_NAME), + bridgePath: join(hookDirectory, WORKSPACE_BRIDGE_FILE_NAME), + configPath: join(hookDirectory, HOOK_CONFIG_FILE_NAME) + }); + }, + async installPlugin(_targetId) { const root = await this.resolveRootDirectory(); if (!root) { @@ -99,11 +113,13 @@ export function createCodexSkillTarget(deps: CreateCodexSkillTargetDeps = {}): S const hookDirectory = join(root, HOOK_DIRECTORY_NAME); const hookScriptPath = join(hookDirectory, HOOK_SCRIPT_FILE_NAME); + const hookConfigPath = join(hookDirectory, HOOK_CONFIG_FILE_NAME); + const hookConfig = { + memmy_config_path: memmyConfigPath, + ...(await readMemmyMemoryServiceConfig(memmyConfigPath)) + }; await mkdir(hookDirectory, { recursive: true }); - await writeFileAtomically( - join(hookDirectory, HOOK_CONFIG_FILE_NAME), - `${JSON.stringify({ memmy_config_path: memmyConfigPath, ...(await readMemmyMemoryServiceConfig(memmyConfigPath)) }, null, 2)}\n` - ); + await writeFileAtomically(hookConfigPath, `${JSON.stringify(hookConfig, null, 2)}\n`); await writeFileAtomically(hookScriptPath, renderMemmyResumeHookScript({ source: CODEX_TARGET_ID, mode: "codex" })); await writeFileAtomically( join(hookDirectory, WORKSPACE_BRIDGE_FILE_NAME), @@ -127,6 +143,10 @@ export function createCodexSkillTarget(deps: CreateCodexSkillTargetDeps = {}): S hooksFilePath, hookCommand }); + await writeFileAtomically( + hookConfigPath, + `${JSON.stringify({ ...hookConfig, ...(await memmyHookRevisionField(CODEX_TARGET_ID, "codex")) }, null, 2)}\n` + ); }, async uninstallPlugin(_targetId) { diff --git a/App/backend/src/adapters/outbound/skill-writer/codex/tests/target.test.ts b/App/backend/src/adapters/outbound/skill-writer/codex/tests/target.test.ts index 5ee80fafe..5a3e6855e 100644 --- a/App/backend/src/adapters/outbound/skill-writer/codex/tests/target.test.ts +++ b/App/backend/src/adapters/outbound/skill-writer/codex/tests/target.test.ts @@ -127,6 +127,27 @@ describe("codex skill target", () => { } }); + it("marks a hook without a stored revision as outdated and refreshes it in place", async () => { + const { rootDirectory, memmyConfigPath } = createFixture(); + const target = createCodexSkillTarget({ rootDirectory, memmyConfigPath, trustHooks: noOpTrustHooks }); + const configPath = join(rootDirectory, "hooks", "memmy-memory-config.json"); + + await target.installPlugin?.("codex"); + const installed = JSON.parse(readFileSync(configPath, "utf8")) as { hook_revision?: string }; + expect(installed.hook_revision).toEqual(expect.stringMatching(/^[a-f0-9]{64}$/)); + await expect(target.isInstalledHookCurrent?.()).resolves.toBe(true); + + writeFileSync(configPath, `${JSON.stringify({ ...installed, hook_revision: "stale" }, null, 2)}\n`, "utf8"); + await expect(target.isInstalledHookCurrent?.()).resolves.toBe(false); + + writeFileSync(join(rootDirectory, "hooks", "memmy-resume-hook.mjs"), "old hook\n", "utf8"); + writeFileSync(configPath, `${JSON.stringify(installed, null, 2)}\n`, "utf8"); + await expect(target.isInstalledHookCurrent?.()).resolves.toBe(false); + + await target.installPlugin?.("codex"); + await expect(target.isInstalledHookCurrent?.()).resolves.toBe(true); + }); + it("persists trust for the installed user-level hooks before installation completes", async () => { const { rootDirectory, memmyConfigPath } = createFixture(); let trustOptions: TrustMemmyCodexHooksOptions | undefined; diff --git a/App/backend/src/adapters/outbound/skill-writer/cursor/target.ts b/App/backend/src/adapters/outbound/skill-writer/cursor/target.ts index 3620be095..28e6e96aa 100644 --- a/App/backend/src/adapters/outbound/skill-writer/cursor/target.ts +++ b/App/backend/src/adapters/outbound/skill-writer/cursor/target.ts @@ -3,6 +3,7 @@ import { mkdir, readFile, rename, rm, writeFile } from "node:fs/promises"; import { homedir } from "node:os"; import { basename, dirname, join } from "node:path"; import { createNodeHookCommand } from "../hook-command.js"; +import { isInstalledMemmyHookCurrent, memmyHookRevisionField } from "../hook-revision.js"; import { readMemmyMemoryServiceConfig } from "../memmy-runtime-config.js"; import { removeMemmyResumeSkillDirectory, @@ -59,16 +60,29 @@ export function createCursorSkillTarget(deps: CreateCursorSkillTargetDeps = {}): return (await readTextFile(join(cursorRootDirectory, "skills", "memmy-memory", "SKILL.md"))).includes("name: memmy-memory"); }, + async isInstalledHookCurrent() { + const hookDirectory = join(cursorRootDirectory, HOOK_DIRECTORY_NAME); + return isInstalledMemmyHookCurrent({ + source: CURSOR_TARGET_ID, + mode: "cursor", + hookScriptPath: join(hookDirectory, HOOK_SCRIPT_FILE_NAME), + bridgePath: join(hookDirectory, WORKSPACE_BRIDGE_FILE_NAME), + configPath: join(hookDirectory, HOOK_CONFIG_FILE_NAME) + }); + }, + async installPlugin(_targetId) { await mkdir(cursorRootDirectory, { recursive: true }); const hookDirectory = join(cursorRootDirectory, HOOK_DIRECTORY_NAME); const hookScriptPath = join(hookDirectory, HOOK_SCRIPT_FILE_NAME); + const hookConfigPath = join(hookDirectory, HOOK_CONFIG_FILE_NAME); + const hookConfig = { + memmy_config_path: memmyConfigPath, + ...(await readMemmyMemoryServiceConfig(memmyConfigPath)) + }; await mkdir(hookDirectory, { recursive: true }); - await writeFileAtomically( - join(hookDirectory, HOOK_CONFIG_FILE_NAME), - `${JSON.stringify({ memmy_config_path: memmyConfigPath, ...(await readMemmyMemoryServiceConfig(memmyConfigPath)) }, null, 2)}\n` - ); + await writeFileAtomically(hookConfigPath, `${JSON.stringify(hookConfig, null, 2)}\n`); await writeFileAtomically( hookScriptPath, renderMemmyResumeHookScript({ source: CURSOR_TARGET_ID, mode: "cursor" }) @@ -83,6 +97,10 @@ export function createCursorSkillTarget(deps: CreateCursorSkillTargetDeps = {}): const manifest = renderMemmyPluginSkillManifest(_targetId); await replaceMemmySkillDirectory(cursorRootDirectory, manifest); await replaceMemmyResumeSkillDirectory(cursorRootDirectory, CURSOR_TARGET_ID); + await writeFileAtomically( + hookConfigPath, + `${JSON.stringify({ ...hookConfig, ...(await memmyHookRevisionField(CURSOR_TARGET_ID, "cursor")) }, null, 2)}\n` + ); }, async uninstallPlugin(_targetId) { diff --git a/App/backend/src/adapters/outbound/skill-writer/cursor/tests/target.test.ts b/App/backend/src/adapters/outbound/skill-writer/cursor/tests/target.test.ts index 13adeca05..b321c58dc 100644 --- a/App/backend/src/adapters/outbound/skill-writer/cursor/tests/target.test.ts +++ b/App/backend/src/adapters/outbound/skill-writer/cursor/tests/target.test.ts @@ -103,6 +103,31 @@ describe("cursor skill target", () => { } }); + it("marks a hook without a stored revision as outdated and refreshes it in place", async () => { + const { rootDirectory, memmyConfigPath } = createFixture(); + const target = createCursorSkillTarget({ rootDirectory, memmyConfigPath }); + const configPath = join(rootDirectory, "hooks", "memmy-memory-config.json"); + + await target.installPlugin?.("cursor"); + const installed = JSON.parse(readFileSync(configPath, "utf8")) as { hook_revision?: string }; + expect(installed.hook_revision).toEqual(expect.stringMatching(/^[a-f0-9]{64}$/)); + await expect(target.isInstalledHookCurrent?.()).resolves.toBe(true); + + writeFileSync(configPath, `${JSON.stringify({ ...installed, hook_revision: "stale" }, null, 2)}\n`, "utf8"); + await expect(target.isInstalledHookCurrent?.()).resolves.toBe(false); + + writeFileSync(join(rootDirectory, "hooks", "memmy-resume-hook.mjs"), "old hook\n", "utf8"); + writeFileSync(configPath, `${JSON.stringify(installed, null, 2)}\n`, "utf8"); + await expect(target.isInstalledHookCurrent?.()).resolves.toBe(false); + + delete installed.hook_revision; + writeFileSync(configPath, `${JSON.stringify(installed, null, 2)}\n`, "utf8"); + await expect(target.isInstalledHookCurrent?.()).resolves.toBe(false); + + await target.installPlugin?.("cursor"); + await expect(target.isInstalledHookCurrent?.()).resolves.toBe(true); + }); + it("installs a resume Skill and reads the current memmyMemory storage instead of legacy storage", async () => { const { rootDirectory, memmyConfigPath } = createFixture(); let requestBody: Record | undefined; diff --git a/App/backend/src/adapters/outbound/skill-writer/hook-revision.test.ts b/App/backend/src/adapters/outbound/skill-writer/hook-revision.test.ts new file mode 100644 index 000000000..19f58e21b --- /dev/null +++ b/App/backend/src/adapters/outbound/skill-writer/hook-revision.test.ts @@ -0,0 +1,44 @@ +/** Hook revision tests. */ +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { currentMemmyHookRevision, hookRevision, isInstalledMemmyHookCurrent } from "./hook-revision.js"; + +let tempDir: string | undefined; + +afterEach(() => { + if (tempDir) { + rmSync(tempDir, { recursive: true, force: true }); + tempDir = undefined; + } +}); + +describe("hook revision", () => { + it("changes when the hook script or workspace bridge changes", () => { + const current = hookRevision("hook script", "workspace bridge"); + + expect(hookRevision("hook script", "workspace bridge")).toBe(current); + expect(hookRevision("updated hook script", "workspace bridge")).not.toBe(current); + expect(hookRevision("hook script", "updated workspace bridge")).not.toBe(current); + }); + + it("treats a current fingerprint with leftover hook files as outdated", async () => { + tempDir = mkdtempSync(join(tmpdir(), "memmy-hook-revision-")); + const revision = await currentMemmyHookRevision("cursor", "cursor"); + const hookScriptPath = join(tempDir, "memmy-resume-hook.mjs"); + const bridgePath = join(tempDir, "memmy-workspace-bridge.mjs"); + const configPath = join(tempDir, "memmy-memory-config.json"); + writeFileSync(hookScriptPath, "old hook\n", "utf8"); + writeFileSync(bridgePath, "old bridge\n", "utf8"); + writeFileSync(configPath, `${JSON.stringify({ hook_revision: revision }, null, 2)}\n`, "utf8"); + + await expect(isInstalledMemmyHookCurrent({ + source: "cursor", + mode: "cursor", + hookScriptPath, + bridgePath, + configPath + })).resolves.toBe(false); + }); +}); diff --git a/App/backend/src/adapters/outbound/skill-writer/hook-revision.ts b/App/backend/src/adapters/outbound/skill-writer/hook-revision.ts new file mode 100644 index 000000000..5c9e4d218 --- /dev/null +++ b/App/backend/src/adapters/outbound/skill-writer/hook-revision.ts @@ -0,0 +1,95 @@ +/** Hook revision module. */ +import { createHash } from "node:crypto"; +import { readFile } from "node:fs/promises"; +import { renderMemmyResumeHookScript, type MemmyResumeHookMode } from "./templates/memmy-resume-hook.js"; +import { loadMemmyWorkspaceBridgeRuntimeAsset } from "./workspace-bridge/runtime-loader.js"; + +export const HOOK_REVISION_FIELD = "hook_revision"; + +const currentRevisionByKey = new Map>(); + +/** Fingerprint of the hook script and workspace bridge this package would install. */ +export function hookRevision(script: string, bridge: string): string { + return createHash("sha256").update(script).update("\0").update(bridge).digest("hex"); +} + +/** Returns the cached fingerprint for one agent hook bundle. */ +export function currentMemmyHookRevision(source: string, mode: MemmyResumeHookMode): Promise { + const key = `${mode}\0${source}`; + const cached = currentRevisionByKey.get(key); + if (cached) return cached; + + const pending = (async () => { + const script = renderMemmyResumeHookScript({ source, mode }); + const bridge = await loadMemmyWorkspaceBridgeRuntimeAsset(); + return hookRevision(script, bridge); + })().catch((error: unknown) => { + currentRevisionByKey.delete(key); + throw error; + }); + currentRevisionByKey.set(key, pending); + return pending; +} + +/** Config field written beside an installed hook. */ +export async function memmyHookRevisionField( + source: string, + mode: MemmyResumeHookMode +): Promise> { + return { [HOOK_REVISION_FIELD]: await currentMemmyHookRevision(source, mode) }; +} + +export interface InstalledMemmyHookPaths { + source: string; + mode: MemmyResumeHookMode; + hookScriptPath: string; + bridgePath: string; + configPath: string; +} + +/** + * An installed hook is current only after the on-disk script and bridge match + * this bundle and the config records that same fingerprint. A fingerprint written + * before those files were replaced is an interrupted install. + */ +export async function isInstalledMemmyHookCurrent(input: InstalledMemmyHookPaths): Promise { + const [script, bridge, configText] = await Promise.all([ + readOptionalText(input.hookScriptPath), + readOptionalText(input.bridgePath), + readOptionalText(input.configPath) + ]); + if (!script.trim() || !bridge.trim()) return false; + const stored = readStoredHookRevision(configText); + if (!stored) return false; + const current = await currentMemmyHookRevision(input.source, input.mode); + return stored === current && hookRevision(script, bridge) === current; +} + +function readStoredHookRevision(configText: string): string | null { + if (!configText.trim()) return null; + try { + const parsed = JSON.parse(configText) as unknown; + if (!isRecord(parsed)) return null; + const value = parsed[HOOK_REVISION_FIELD]; + return typeof value === "string" && value.trim() ? value : null; + } catch { + return null; + } +} + +async function readOptionalText(filePath: string): Promise { + try { + return await readFile(filePath, "utf8"); + } catch (error) { + if (isNodeError(error) && error.code === "ENOENT") return ""; + throw error; + } +} + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +function isNodeError(error: unknown): error is NodeJS.ErrnoException { + return error instanceof Error && "code" in error; +} diff --git a/App/backend/src/adapters/outbound/skill-writer/types.ts b/App/backend/src/adapters/outbound/skill-writer/types.ts index 9ec21586a..a1c8cd20e 100644 --- a/App/backend/src/adapters/outbound/skill-writer/types.ts +++ b/App/backend/src/adapters/outbound/skill-writer/types.ts @@ -25,5 +25,6 @@ export interface SkillTarget { isInstalled(targetId: string): Promise; installPlugin?(targetId: string): Promise; uninstallPlugin?(targetId: string): Promise; + isInstalledHookCurrent?(): Promise; detectMemoryPluginConflict?(): Promise; } diff --git a/App/backend/src/analytics/analytics-transport.ts b/App/backend/src/analytics/analytics-transport.ts index 115c00387..6708ea247 100644 --- a/App/backend/src/analytics/analytics-transport.ts +++ b/App/backend/src/analytics/analytics-transport.ts @@ -103,9 +103,13 @@ export function resolveAnalyticsEnvParams(options: { }; } +/** GA4 Measurement Protocol accepts only string and number param values. A boolean drops the whole event. */ export function compactAnalyticsParams(params: AnalyticsParams = {}): AnalyticsParams { return Object.fromEntries( - Object.entries(params).filter(([, value]) => value !== undefined && value !== null && value !== ""), + Object.entries(params).flatMap(([key, value]) => { + if (value === undefined || value === null || value === "") return []; + return [[key, typeof value === "boolean" ? (value ? 1 : 0) : value]]; + }), ) as AnalyticsParams; } diff --git a/App/backend/src/analytics/memory-add-analytics.ts b/App/backend/src/analytics/memory-add-analytics.ts index 5cd6a62be..47fec4d58 100644 --- a/App/backend/src/analytics/memory-add-analytics.ts +++ b/App/backend/src/analytics/memory-add-analytics.ts @@ -43,6 +43,8 @@ export type MemoryDesktopScanAddBaseInput = { adapterId: string; /** Present for agent-source scan/import paths; omitted when unavailable. */ scanMode?: MemoryDesktopAddScanMode; + /** Defaults to L1. Skill imports pass Skill so they are not counted as conversation memory. */ + layer?: "L1" | "Skill"; conversationId?: string | null; turnId?: string | null; }; @@ -60,7 +62,7 @@ export function buildMemoryDesktopScanAddParams(input: MemoryDesktopScanAddBaseI adapter_id: input.adapterId, storage_backend: MEMORY_DESKTOP_ADD_STORAGE_BACKEND, mode: MEMORY_DESKTOP_ADD_MODE_AGENT_SOURCE_SCAN, - layer: MEMORY_DESKTOP_ADD_LAYER_L1, + layer: input.layer ?? MEMORY_DESKTOP_ADD_LAYER_L1, ...(input.scanMode ? { scan_mode: input.scanMode } : {}), ...(sessionIdHash ? { session_id_hash: sessionIdHash } : {}), ...(turnIdHash ? { turn_id_hash: turnIdHash } : {}), diff --git a/App/backend/src/analytics/tests/memory-add-analytics.test.ts b/App/backend/src/analytics/tests/memory-add-analytics.test.ts index e4505c826..62fd84232 100644 --- a/App/backend/src/analytics/tests/memory-add-analytics.test.ts +++ b/App/backend/src/analytics/tests/memory-add-analytics.test.ts @@ -25,6 +25,12 @@ describe("memory-add-analytics", () => { session_id_hash: hashAnalyticsId("conv-1"), turn_id_hash: hashAnalyticsId("cursor:abc"), }); + expect(buildMemoryDesktopScanAddParams({ + adapterId: "agent-source:cursor", + layer: "Skill", + conversationId: "skill:review-code", + turnId: "skill:review-code:v2", + }).layer).toBe("Skill"); }); it("tracks started/succeeded/failed desktop add events", async () => { @@ -83,13 +89,13 @@ describe("memory-add-analytics", () => { adapter_id: "agent-source:cursor", }); expect(body.events[1]?.params).toMatchObject({ - success: true, + success: 1, stored_count: 1, duration_ms: 12, scan_mode: "full", }); expect(body.events[2]?.params).toMatchObject({ - success: false, + success: 0, error_code: "boom", duration_ms: 3, scan_mode: "incremental", diff --git a/App/backend/src/project-version.ts b/App/backend/src/project-version.ts index 2323f01a3..915a5e82a 100644 --- a/App/backend/src/project-version.ts +++ b/App/backend/src/project-version.ts @@ -1,2 +1,2 @@ /** Generated from the root package.json by scripts/sync-project-version.mjs. */ -export const MEMMY_VERSION = "1.2.1"; +export const MEMMY_VERSION = "1.2.2"; diff --git a/App/backend/src/services/agent-source-auto-inject-service.ts b/App/backend/src/services/agent-source-auto-inject-service.ts index f5ac09fc4..c97eed893 100644 --- a/App/backend/src/services/agent-source-auto-inject-service.ts +++ b/App/backend/src/services/agent-source-auto-inject-service.ts @@ -1,5 +1,5 @@ /** Agent source auto inject service module. */ -import type { AgentSourceAutoInjectResult, ScanPreferences } from "@memmy/local-api-contracts"; +import type { AgentSourceAutoInjectResult, AgentSourceView, ScanPreferences } from "@memmy/local-api-contracts"; import type { PermissionManager } from "../permission/index.js"; import type { AgentSourceService } from "./agent-source-service.js"; @@ -16,13 +16,14 @@ const AUTO_INJECT_AGENT_SOURCE_IDS = new Set([ "qwenwork" ]); const HOOK_OR_PLUGIN_AGENT_SOURCE_IDS = new Set(["cursor", "claude_code", "codex", "opencode", "openclaw", "hermes", "deepseek_harness"]); +const UPDATABLE_HOOK_AGENT_SOURCE_IDS = new Set(["cursor", "claude_code", "codex"]); export interface AgentSourceAutoInjectService { runOnce(): Promise; } export interface CreateAgentSourceAutoInjectServiceOptions { - agentSources: Pick; + agentSources: Pick; permissionManager: Pick; getScanPreferences: () => ScanPreferences; } @@ -45,23 +46,41 @@ export function createAgentSourceAutoInjectService( }; } - const preferences = options.getScanPreferences(); - if (!preferences.autoInjectSkill) { - return { - ok: true, - skipped: true, - reason: "auto_inject_disabled", - installed: [], - failed: [] - }; - } - running = true; try { const sources = await options.agentSources.list(); const installed: string[] = []; const failed: Array<{ sourceId: string; reason: string }> = []; + for (const source of sources) { + if (!isUpdatableInstalledHook(source)) continue; + if (!(await options.permissionManager.canWriteAgentSkill({ agentSourceId: source.sourceId }))) { + continue; + } + + try { + if (await options.agentSources.isInstalledHookCurrent(source.sourceId)) continue; + await options.agentSources.installPlugin(source.sourceId, { installType: "auto_inject" }); + installed.push(source.sourceId); + } catch (error) { + failed.push({ + sourceId: source.sourceId, + reason: error instanceof Error ? error.message : String(error) + }); + } + } + + const preferences = options.getScanPreferences(); + if (!preferences.autoInjectSkill) { + return { + ok: true, + skipped: true, + reason: "auto_inject_disabled", + installed, + failed + }; + } + for (const source of sources) { if (!AUTO_INJECT_AGENT_SOURCE_IDS.has(source.sourceId) || !source.builtin || !source.available || source.status !== "not_connected") { continue; @@ -98,3 +117,10 @@ export function createAgentSourceAutoInjectService( } }; } + +function isUpdatableInstalledHook(source: AgentSourceView): boolean { + return UPDATABLE_HOOK_AGENT_SOURCE_IDS.has(source.sourceId) + && source.builtin + && source.available + && source.status === "plugin_installed"; +} diff --git a/App/backend/src/services/agent-source-service.ts b/App/backend/src/services/agent-source-service.ts index 521de8d24..cbecc0dfe 100644 --- a/App/backend/src/services/agent-source-service.ts +++ b/App/backend/src/services/agent-source-service.ts @@ -103,6 +103,7 @@ export interface AgentSourceService { uninstallSkill(sourceId: string): Promise; installPlugin(sourceId: string, action?: AgentSourcePluginActionInput): Promise; uninstallPlugin(sourceId: string, action?: AgentSourcePluginActionInput): Promise; + isInstalledHookCurrent(sourceId: string): Promise; detectMemoryPluginConflicts(): Promise; } @@ -452,6 +453,10 @@ export function createAgentSourceService(options: CreateAgentSourceServiceOption } }, + async isInstalledHookCurrent(sourceId) { + return options.skillDistributionService.isInstalledHookCurrent?.(sourceId) ?? true; + }, + async uninstallPlugin(sourceId, action = {}) { const startedAt = Date.now(); const installType = normalizePluginInstallType(action.installType); @@ -729,7 +734,7 @@ async function ingestPersistentStagedSource( const ingestion = await ingestPersistentSource(options, store, sourceId, scanOptions, []); const scannedAt = now(); options.agentSourceRepository.setLastScannedAt(sourceId, scannedAt); - const skillResult = await ingestSourceSkills(options, sourceId, scanOptions, store); + const skillResult = await ingestSourceSkills(options, sourceId, scanOptions, store, mode); if (stage.errors.length === 0 && ingestion.errors.length === 0 && skillResult.errorCount === 0 && !ingestion.hasUncommittedSkips) { updatePersistentWatermark(options, sourceId, mode, ingestion.latestSeenAt, scannedAt, since); } @@ -907,6 +912,14 @@ async function ingestPersistentSource( } const scanMode = persistentScanMode(store.getSourceState(sourceId)?.mode ?? scanOptions.mode); if (hasStagedSourceTurn(turn.messages[0]) || sourceId === "codex") { + let addAnalyticsBase: { + adapterId: string; + conversationId: string; + turnId: string; + scanMode?: MemoryDesktopAddScanMode; + } | undefined; + let addStartedAt = 0; + let reportedAdd = false; try { const sourceTurn = sourceTurnFromMessages(turn.messages); if (!sourceTurn) { @@ -918,6 +931,13 @@ async function ingestPersistentSource( continue; } const legacyImportTurnId = legacyImportTurnIdFromMessages(sourceId, turn.conversationId, turn.messages); + addAnalyticsBase = { + adapterId: `agent-source:${sourceId}`, + conversationId: turn.conversationId, + turnId: sourceTurn.turnId, + ...(scanMode ? { scanMode } : {}) + }; + addStartedAt = Date.now(); const result = await options.memoryClient.completeSourceTurn({ ...buildSourceTurnRequest(sourceTurn, "agent_source_scan"), ...(legacyImportTurnId ? { legacyImportTurnId } : {}), @@ -935,6 +955,13 @@ async function ingestPersistentSource( if (result.status === "stored") { memoryIdCount += ids.length; memoryIds.push(...ids.slice(0, Math.max(0, 1000 - memoryIds.length))); + options.memoryAddAnalytics?.trackAddStarted(addAnalyticsBase); + options.memoryAddAnalytics?.trackAddSucceeded({ + ...addAnalyticsBase, + durationMs: Date.now() - addStartedAt, + storedCount: Math.max(ids.length, 1) + }); + reportedAdd = true; } else { deduped += turn.messages.length; } @@ -945,6 +972,14 @@ async function ingestPersistentSource( const reason = error instanceof Error ? error.message : "native turn ingestion failed"; skipPersistentTurn(store, sourceId, turn.conversationId, reason); noteUncommittedSkip(reason); + if (addAnalyticsBase && !reportedAdd) { + options.memoryAddAnalytics?.trackAddStarted(addAnalyticsBase); + options.memoryAddAnalytics?.trackAddFailed({ + ...addAnalyticsBase, + durationMs: Date.now() - addStartedAt, + error + }); + } } processed += turn.messages.length; emitAddProgress("Capturing conversation turns"); @@ -1241,7 +1276,7 @@ async function ingestCollectedSource( ) { updateScanWatermark(options, collected, scanOptions, scannedAt); } - errors.push(...(await ingestSourceSkills(options, collected.sourceId, scanOptions)).errors); + errors.push(...(await ingestSourceSkills(options, collected.sourceId, scanOptions, undefined, collected.scanMode ?? scanOptions.mode)).errors); return { result: { sourceId: collected.sourceId, @@ -1265,7 +1300,8 @@ async function ingestSourceSkills( options: CreateAgentSourceServiceOptions, sourceId: string, scanOptions: AgentSourceScanOptions, - store?: AppAgentSourceScanStore + store?: AppAgentSourceScanStore, + resolvedScanMode?: AgentSourceScanMode ): Promise { if (!options.skillDistributionService.listSkills) return { errors: [], errorCount: 0, memoryIdCount: 0 }; @@ -1281,8 +1317,18 @@ async function ingestSourceSkills( return { errors: [], errorCount: 0, memoryIdCount: 0 }; } + const scanMode = persistentScanMode(resolvedScanMode ?? scanOptions.mode); for (const skill of skills) { scanOptions.signal?.throwIfAborted(); + const turnId = `skill:${skill.sourceSkillId}:${skill.sourceSkillVersion}`; + const addAnalyticsBase = { + adapterId: `agent-source:${sourceId}`, + conversationId: `skill:${skill.sourceSkillId}`, + turnId, + layer: "Skill" as const, + ...(scanMode ? { scanMode } : {}) + }; + const addStartedAt = Date.now(); try { const added = await options.memoryClient.addMemory({ requestId: `agent-source-skill:${sourceId}:${skill.sourceSkillId}:${skill.sourceContentHash}`, @@ -1292,7 +1338,7 @@ async function ingestSourceSkills( title: skill.title, tags: ["agent-source", "cross-agent-skill", sourceId], source: sourceId, - turnId: `skill:${skill.sourceSkillId}:${skill.sourceSkillVersion}`, + turnId, createdAt: skill.updatedAt, sourceAgentId: sourceId, sourceSkillId: skill.sourceSkillId, @@ -1300,6 +1346,14 @@ async function ingestSourceSkills( sourceSkillVersion: skill.sourceSkillVersion, sourceContentHash: skill.sourceContentHash }); + if (added.duplicate !== true && added.status !== "deleted") { + options.memoryAddAnalytics?.trackAddStarted(addAnalyticsBase); + options.memoryAddAnalytics?.trackAddSucceeded({ + ...addAnalyticsBase, + durationMs: Date.now() - addStartedAt, + storedCount: 1 + }); + } memoryIdCount += 1; store?.saveResult({ sourceId, conversationId: `skill:${skill.sourceSkillId}`, memoryId: added.id }); } catch (error) { @@ -1309,6 +1363,12 @@ async function ingestSourceSkills( `skill:${skill.sourceSkillId}`, error instanceof Error ? error.message : "Agent Skill import failed" ); + options.memoryAddAnalytics?.trackAddStarted(addAnalyticsBase); + options.memoryAddAnalytics?.trackAddFailed({ + ...addAnalyticsBase, + durationMs: Date.now() - addStartedAt, + error + }); } } return { errors, errorCount, memoryIdCount }; diff --git a/App/backend/src/services/ingestion-service.ts b/App/backend/src/services/ingestion-service.ts index 827099691..169713030 100644 --- a/App/backend/src/services/ingestion-service.ts +++ b/App/backend/src/services/ingestion-service.ts @@ -318,6 +318,14 @@ async function processNativeConversation( emitIngestionProgress(ctx, stats); continue; } + const addAnalyticsBase = { + adapterId: `agent-source:${ctx.sourceId}`, + conversationId: turn.conversationId, + turnId: sourceTurn.turnId, + ...(ctx.scanMode ? { scanMode: ctx.scanMode } : {}) + }; + const addStartedAt = Date.now(); + let reportedAdd = false; try { const legacyImportTurnId = legacyImportTurnIdFromMessages(ctx.sourceId, turn.conversationId, turn.messages); const result = await options.memoryClient.completeSourceTurn({ @@ -338,6 +346,13 @@ async function processNativeConversation( stats.written += turn.messages.length; stats.writtenMemories += ids.length; stats.memoryIds.push(...ids); + options.memoryAddAnalytics?.trackAddStarted(addAnalyticsBase); + options.memoryAddAnalytics?.trackAddSucceeded({ + ...addAnalyticsBase, + durationMs: Date.now() - addStartedAt, + storedCount: Math.max(ids.length, 1) + }); + reportedAdd = true; } else { stats.deduped += turn.messages.length; if (result.status === "existing") stats.dedupedMemories += 1; @@ -350,6 +365,14 @@ async function processNativeConversation( stats.failed += turn.messages.length; stats.failedMemories += 1; reportItemSkip(options, ctx, turn.conversationId, error instanceof Error ? error.message : "native turn ingestion failed"); + if (!reportedAdd) { + options.memoryAddAnalytics?.trackAddStarted(addAnalyticsBase); + options.memoryAddAnalytics?.trackAddFailed({ + ...addAnalyticsBase, + durationMs: Date.now() - addStartedAt, + error + }); + } } emitIngestionProgress(ctx, stats); } diff --git a/App/backend/src/services/skill-distribution-service.ts b/App/backend/src/services/skill-distribution-service.ts index a849bc27e..1bb35d937 100644 --- a/App/backend/src/services/skill-distribution-service.ts +++ b/App/backend/src/services/skill-distribution-service.ts @@ -25,6 +25,7 @@ export interface SkillDistributionService { uninstall(sourceId: string): Promise; installPlugin(sourceId: string): Promise; uninstallPlugin(sourceId: string): Promise; + isInstalledHookCurrent?(sourceId: string): Promise; detectMemoryPluginConflicts?(): Promise; } @@ -101,6 +102,12 @@ export function createSkillDistributionService( await target.uninstall(sourceId); }, + async isInstalledHookCurrent(sourceId) { + const target = options.targetRegistry.get(sourceId); + if (!target?.isInstalledHookCurrent) return true; + return target.isInstalledHookCurrent(); + }, + async detectMemoryPluginConflicts() { const conflicts: MemoryPluginConflict[] = []; for (const target of options.targetRegistry.list()) { diff --git a/App/backend/src/services/tests/agent-source-auto-inject-service.test.ts b/App/backend/src/services/tests/agent-source-auto-inject-service.test.ts index 04821d248..a79b18a5b 100644 --- a/App/backend/src/services/tests/agent-source-auto-inject-service.test.ts +++ b/App/backend/src/services/tests/agent-source-auto-inject-service.test.ts @@ -108,6 +108,157 @@ describe("agent source auto inject service", () => { releaseInstall(); await running; }); + + it("refreshes an outdated installed hook while auto inject is disabled", async () => { + const calls: string[] = []; + const service = createAgentSourceAutoInjectService({ + agentSources: { + ...createAgentSources(calls), + async list() { + return [source("cursor", "plugin_installed", true)]; + }, + async isInstalledHookCurrent() { + return false; + } + }, + permissionManager: { async canWriteAgentSkill() { return true; } }, + getScanPreferences: () => ({ ...enabledPreferences, autoInjectSkill: false }) + }); + + await expect(service.runOnce()).resolves.toEqual({ + ok: true, + skipped: true, + reason: "auto_inject_disabled", + installed: ["cursor"], + failed: [] + }); + expect(calls).toEqual(["plugin:cursor:auto_inject"]); + }); + + it("leaves a current hook in place and still installs a newly found agent", async () => { + const calls: string[] = []; + const service = createAgentSourceAutoInjectService({ + agentSources: { + ...createAgentSources(calls), + async list() { + return [ + source("cursor", "plugin_installed", true), + source("claude_code", "plugin_installed", true), + source("codex", "plugin_installed", true), + source("workbuddy", "not_connected", true) + ]; + }, + async isInstalledHookCurrent(sourceId) { + return sourceId !== "codex"; + } + }, + permissionManager: { async canWriteAgentSkill() { return true; } }, + getScanPreferences: () => enabledPreferences + }); + + await expect(service.runOnce()).resolves.toEqual({ + ok: true, + skipped: false, + installed: ["codex", "workbuddy"], + failed: [] + }); + expect(calls).toEqual([ + "plugin:codex:auto_inject", + "skill:workbuddy" + ]); + }); + + it("does not refresh installed plugins that are not hooks", async () => { + const calls: string[] = []; + const service = createAgentSourceAutoInjectService({ + agentSources: { + ...createAgentSources(calls), + async list() { + return [ + source("hermes", "plugin_installed", true), + source("opencode", "plugin_installed", true) + ]; + }, + async isInstalledHookCurrent() { + return false; + } + }, + permissionManager: { async canWriteAgentSkill() { return true; } }, + getScanPreferences: () => enabledPreferences + }); + + await expect(service.runOnce()).resolves.toEqual({ + ok: true, + skipped: false, + installed: [], + failed: [] + }); + expect(calls).toEqual([]); + }); + + it("skips an outdated hook when the agent is unavailable or not writable", async () => { + const calls: string[] = []; + const service = createAgentSourceAutoInjectService({ + agentSources: { + ...createAgentSources(calls), + async list() { + return [ + { ...source("cursor", "plugin_installed", true), available: false }, + source("claude_code", "plugin_installed", true) + ]; + }, + async isInstalledHookCurrent() { + return false; + } + }, + permissionManager: { + async canWriteAgentSkill(input: { agentSourceId: string }) { + return input.agentSourceId !== "claude_code"; + } + }, + getScanPreferences: () => ({ ...enabledPreferences, autoInjectSkill: false }) + }); + + await expect(service.runOnce()).resolves.toEqual({ + ok: true, + skipped: true, + reason: "auto_inject_disabled", + installed: [], + failed: [] + }); + expect(calls).toEqual([]); + }); + + it("records a hook refresh failure and continues with other agents", async () => { + const calls: string[] = []; + const service = createAgentSourceAutoInjectService({ + agentSources: { + ...createAgentSources(calls), + async list() { + return [ + source("cursor", "plugin_installed", true), + source("pi", "not_connected", true) + ]; + }, + async isInstalledHookCurrent() { + return false; + }, + async installPlugin() { + throw new Error("install failed"); + } + }, + permissionManager: { async canWriteAgentSkill() { return true; } }, + getScanPreferences: () => enabledPreferences + }); + + await expect(service.runOnce()).resolves.toEqual({ + ok: true, + skipped: false, + installed: ["pi"], + failed: [{ sourceId: "cursor", reason: "install failed" }] + }); + expect(calls).toEqual(["skill:pi"]); + }); }); function createAgentSources(calls: string[]) { @@ -130,6 +281,9 @@ function createAgentSources(calls: string[]) { }, async installPlugin(sourceId: string, action?: { installType?: string }) { calls.push(`plugin:${sourceId}:${action?.installType ?? "manual"}`); + }, + async isInstalledHookCurrent() { + return true; } }; } diff --git a/App/backend/src/services/tests/agent-source-service.test.ts b/App/backend/src/services/tests/agent-source-service.test.ts index 977643968..69515fb4a 100644 --- a/App/backend/src/services/tests/agent-source-service.test.ts +++ b/App/backend/src/services/tests/agent-source-service.test.ts @@ -175,6 +175,64 @@ describe("agent source service", () => { ]); }); + it("emits add analytics only when a scanned skill is newly stored", async () => { + const events: Array<{ name: string; payload: Record }> = []; + const memoryClient = createMockMemoryClient(); + const service = createService({ + adapters: [createFakeAdapter("cursor", [])], + memoryClient: { + ...memoryClient, + async addMemory(input, context) { + if (input.sourceSkillId === "duplicate-skill") { + return { ...(await memoryClient.addMemory(input, context)), duplicate: true }; + } + if (input.sourceSkillId === "deleted-skill") { + return { ...(await memoryClient.addMemory(input, context)), status: "deleted" as const }; + } + if (input.sourceSkillId === "failing-skill") throw new Error("skill import failed"); + return memoryClient.addMemory(input, context); + } + }, + memoryAddAnalytics: createAddAnalyticsRecorder(events), + skillDistributionService: { + async listSkills() { + return ["fresh-skill", "duplicate-skill", "deleted-skill", "failing-skill"].map((sourceSkillId) => ({ + sourceAgentId: "cursor", + sourceSkillId, + sourceSkillPath: `/tmp/cursor/skills/${sourceSkillId}/SKILL.md`, + sourceSkillVersion: "v1", + sourceContentHash: `${sourceSkillId}-hash`, + title: sourceSkillId, + content: sourceSkillId, + updatedAt: "2026-05-28T09:00:00.000Z" + })); + }, + async install() {}, + async uninstall() {}, + async installPlugin() {}, + async uninstallPlugin() {} + } + }); + + await service.scanOne("cursor", { mode: "incremental" }); + + expect(events.map((event) => event.name)).toEqual(["started", "succeeded", "started", "failed"]); + expect(events[1]?.payload).toMatchObject({ + adapterId: "agent-source:cursor", + conversationId: "skill:fresh-skill", + turnId: "skill:fresh-skill:v1", + layer: "Skill", + scanMode: "incremental", + storedCount: 1 + }); + expect(events[3]?.payload).toMatchObject({ + conversationId: "skill:failing-skill", + layer: "Skill", + scanMode: "incremental", + error: expect.any(Error) + }); + }); + it("completes the scan and advances checkpoints when every memory is skipped", async () => { const repository = createRepository(); const messages = createCompleteMemoryMessages("cursor", 1, "2026-05-28T10:00:00.000Z"); @@ -1389,12 +1447,12 @@ describe("agent source service", () => { status_before: "not_connected", status_after: "plugin_installed", install_type: "manual", - success: true, + success: 1, }); expect(analytics.events[1]?.params).toMatchObject({ source_id: "cursor", status_after: "not_connected", - success: true, + success: 1, }); expect(analytics.events[2]?.params).toMatchObject({ source_id: "openclaw", @@ -1404,6 +1462,33 @@ describe("agent source service", () => { }); }); + it("reports whether an installed hook matches the bundled revision", async () => { + const fallback = createService(); + const service = createService({ + skillDistributionService: { + async install() { + return undefined; + }, + async uninstall() { + return undefined; + }, + async installPlugin() { + return undefined; + }, + async uninstallPlugin() { + return undefined; + }, + async isInstalledHookCurrent(sourceId) { + return sourceId !== "cursor"; + } + } + }); + + await expect(fallback.isInstalledHookCurrent("cursor")).resolves.toBe(true); + await expect(service.isInstalledHookCurrent("cursor")).resolves.toBe(false); + await expect(service.isInstalledHookCurrent("claude_code")).resolves.toBe(true); + }); + it("emits skill install analytics", async () => { const repository = createRepository(); repository.upsertSource({ @@ -1446,7 +1531,7 @@ describe("agent source service", () => { permission: "scan_only", status_before: "not_connected", status_after: "skill_installed", - success: true, + success: 1, }, }); }); @@ -1492,7 +1577,7 @@ describe("agent source service", () => { status_before: "not_connected", status_after: "not_connected", install_type: "auto_inject", - success: false, + success: 0, error_code: "install failed", }, }); @@ -1576,6 +1661,49 @@ describe("agent source service", () => { expect(events).toEqual([]); }); + it("emits add analytics for a newly stored native persistent scan turn", async () => { + const events: Array<{ name: string; payload: Record }> = []; + tempDir = mkdtempSync(join(tmpdir(), "persistent-native-add-analytics-")); + const service = createService({ + adapters: [createFakeAdapter("cursor", stagedNativeMessages("cursor"))], + scanStoreDirectory: join(tempDir, "scans"), + memoryClient: createReadyMemoryClient(), + memoryAddAnalytics: createAddAnalyticsRecorder(events) + }); + + const result = await service.scanOne("cursor", { scanJobId: "job-native-add", mode: "initial_subset" }); + + expect(result.errors).toEqual([]); + expect(events.map((event) => event.name)).toEqual(["started", "succeeded"]); + expect(events[1]?.payload).toMatchObject({ + adapterId: "agent-source:cursor", + conversationId: "cursor-conv-1", + turnId: "native-turn", + scanMode: "initial_subset", + storedCount: 1 + }); + }); + + it("does not emit add analytics when a native persistent scan turn already exists", async () => { + const events: Array<{ name: string; payload: Record }> = []; + tempDir = mkdtempSync(join(tmpdir(), "persistent-native-existing-analytics-")); + const memoryClient = createReadyMemoryClient(); + memoryClient.completeSourceTurn = vi.fn().mockResolvedValue({ + status: "existing", + result: { l1MemoryIds: ["l1-existing"] } + }); + const service = createService({ + adapters: [createFakeAdapter("cursor", stagedNativeMessages("cursor"))], + scanStoreDirectory: join(tempDir, "scans"), + memoryClient, + memoryAddAnalytics: createAddAnalyticsRecorder(events) + }); + + await service.scanOne("cursor", { scanJobId: "job-native-existing", mode: "incremental" }); + + expect(events).toEqual([]); + }); + it("does not emit add analytics for already checkpointed persistent scan turns", async () => { const events: Array<{ name: string; payload: Record }> = []; tempDir = mkdtempSync(join(tmpdir(), "persistent-scan-skip-analytics-")); @@ -1820,6 +1948,39 @@ function createMessage(sourceId: string, index: number): ConversationMessage { }; } +function stagedNativeMessages(sourceId: string): ConversationMessage[] { + const sourceTurn = { + source: sourceId, + conversationId: `${sourceId}-conv-1`, + turnId: "native-turn", + sequence: 1, + startedAt: "2026-05-28T10:00:00.000Z", + completedAt: "2026-05-28T10:01:00.000Z", + completionEvidence: "task_complete:native-turn", + query: "Run tests", + answer: "Tests passed", + status: "succeeded", + toolCalls: [], + toolResults: [] + }; + return [ + { + ...createMessage(sourceId, 1), + role: "user", + content: sourceTurn.query, + createdAt: sourceTurn.startedAt, + rawMeta: { sourceTurnId: "native-turn", sourceTurnState: "complete" } + }, + { + ...createMessage(sourceId, 2), + role: "assistant", + content: sourceTurn.answer, + createdAt: sourceTurn.completedAt, + rawMeta: { sourceTurnId: "native-turn", sourceTurnState: "complete", sourceTurn } + } + ]; +} + function createCompleteMemoryMessages( sourceId: string, count: number, diff --git a/App/backend/src/services/tests/ingestion-service.test.ts b/App/backend/src/services/tests/ingestion-service.test.ts index b862844bc..3fc59ab1d 100644 --- a/App/backend/src/services/tests/ingestion-service.test.ts +++ b/App/backend/src/services/tests/ingestion-service.test.ts @@ -47,6 +47,54 @@ describe("native Codex ingestion", () => { expect(markSeen).toHaveBeenCalledTimes(2); }); + it("emits add analytics only when a native scan turn is newly stored", async () => { + const events: Array<{ name: string; payload: Record }> = []; + const analytics = { + trackAddStarted(input: Record) { + events.push({ name: "started", payload: { ...input } }); + }, + trackAddSucceeded(input: Record) { + events.push({ name: "succeeded", payload: { ...input } }); + }, + trackAddFailed(input: Record) { + events.push({ name: "failed", payload: { ...input } }); + } + }; + const stored = vi.fn().mockResolvedValue({ status: "stored", result: { l1MemoryIds: ["l1-a", "l1-b"] } }); + await createService({ completeSourceTurn: stored }, {}, undefined, analytics).ingest( + toAsyncIterable(nativeMessages()), + { sourceId: "codex", scanMode: "initial_subset" } + ); + expect(events.map((event) => event.name)).toEqual(["started", "succeeded"]); + expect(events[1]?.payload).toMatchObject({ + adapterId: "agent-source:codex", + conversationId: "native-conversation", + turnId: "native-turn", + scanMode: "initial_subset", + storedCount: 2 + }); + + events.length = 0; + const existing = vi.fn().mockResolvedValue({ status: "existing", result: { l1MemoryIds: ["l1-a"] } }); + await createService({ completeSourceTurn: existing }, {}, undefined, analytics).ingest( + toAsyncIterable(nativeMessages()), + { sourceId: "codex", scanMode: "incremental" } + ); + expect(events).toEqual([]); + + const failed = vi.fn().mockRejectedValue(new Error("native write failed")); + await createService({ completeSourceTurn: failed }, {}, undefined, analytics).ingest( + toAsyncIterable(nativeMessages()), + { sourceId: "codex", scanMode: "full" } + ); + expect(events.map((event) => event.name)).toEqual(["started", "failed"]); + expect(events[1]?.payload).toMatchObject({ + adapterId: "agent-source:codex", + turnId: "native-turn", + scanMode: "full" + }); + }); + it("asks a full scan to capture history that completed before activation", async () => { const completeSourceTurn = vi.fn().mockResolvedValue({ status: "stored", result: { l1MemoryIds: ["l1-historical"] } }); await createService({ completeSourceTurn }).ingest(toAsyncIterable(nativeMessages()), { sourceId: "codex", scanMode: "full" }); diff --git a/App/backend/src/services/tests/skill-distribution-service.test.ts b/App/backend/src/services/tests/skill-distribution-service.test.ts index b5abf7768..7bd2fdd51 100644 --- a/App/backend/src/services/tests/skill-distribution-service.test.ts +++ b/App/backend/src/services/tests/skill-distribution-service.test.ts @@ -125,6 +125,27 @@ describe("skill distribution service", () => { expect(calls).toEqual(["cursor"]); }); + it("reads hook currency from targets that publish it", async () => { + const current = createSkillDistributionService({ + targetRegistry: createSkillTargetRegistry([ + createFakeTarget({ isInstalledHookCurrent: async () => true }) + ]) + }); + const outdated = createSkillDistributionService({ + targetRegistry: createSkillTargetRegistry([ + createFakeTarget({ isInstalledHookCurrent: async () => false }) + ]) + }); + const untouched = createSkillDistributionService({ + targetRegistry: createSkillTargetRegistry([createFakeTarget({})]) + }); + + await expect(current.isInstalledHookCurrent?.("cursor")).resolves.toBe(true); + await expect(outdated.isInstalledHookCurrent?.("cursor")).resolves.toBe(false); + await expect(untouched.isInstalledHookCurrent?.("cursor")).resolves.toBe(true); + await expect(untouched.isInstalledHookCurrent?.("missing")).resolves.toBe(true); + }); + it("delegates native plugin installation to plugin-capable targets", async () => { const calls: string[] = []; const service = createSkillDistributionService({ @@ -224,6 +245,7 @@ function createFakeTarget(overrides: { uninstall?: (targetId: string) => void; installPlugin?: (targetId: string) => void; uninstallPlugin?: (targetId: string) => void; + isInstalledHookCurrent?: () => Promise; detectMemoryPluginConflict?: () => MemoryPluginConflict | null; }): SkillTarget { return { @@ -253,6 +275,7 @@ function createFakeTarget(overrides: { : undefined, detectMemoryPluginConflict: overrides.detectMemoryPluginConflict ? async () => overrides.detectMemoryPluginConflict?.() ?? null - : undefined + : undefined, + isInstalledHookCurrent: overrides.isInstalledHookCurrent }; } diff --git a/App/backend/src/tests/project-version.test.ts b/App/backend/src/tests/project-version.test.ts index 4dce59640..1b1634caa 100644 --- a/App/backend/src/tests/project-version.test.ts +++ b/App/backend/src/tests/project-version.test.ts @@ -11,7 +11,7 @@ describe("project version", () => { ); expect(MEMMY_VERSION).toBe(rootManifest.version); - expect(MEMMY_VERSION).toBe("1.2.1"); + expect(MEMMY_VERSION).toBe("1.2.2"); expect(MEMMY_VERSION).toMatch(/^\d+\.\d+\.\d+(?:[-+][0-9A-Za-z.-]+)?$/); expect(readFileSync(resolve(repoRoot, "App/backend/src/project-version.ts"), "utf8")) .toContain(`MEMMY_VERSION = ${JSON.stringify(rootManifest.version)}`); diff --git a/App/frontend/desktop/src/analytics/cloud-analytics.ts b/App/frontend/desktop/src/analytics/cloud-analytics.ts index b40f65a9b..70ae5dabe 100644 --- a/App/frontend/desktop/src/analytics/cloud-analytics.ts +++ b/App/frontend/desktop/src/analytics/cloud-analytics.ts @@ -145,7 +145,11 @@ function flushNow(): Promise { function compactParams(params: CloudAnalyticsParams): CloudAnalyticsParams { return Object.fromEntries( - Object.entries(params).filter(([, value]) => value !== undefined && value !== null && value !== "") + Object.entries(params).flatMap(([key, value]) => { + if (value === undefined || value === null || value === "") return []; + // GA4 Measurement Protocol accepts only string and number values. + return [[key, typeof value === "boolean" ? (value ? 1 : 0) : value]]; + }) ) as CloudAnalyticsParams; } diff --git a/App/frontend/desktop/src/analytics/tests/cloud-analytics.test.ts b/App/frontend/desktop/src/analytics/tests/cloud-analytics.test.ts index fb17e7178..cd20362e0 100644 --- a/App/frontend/desktop/src/analytics/tests/cloud-analytics.test.ts +++ b/App/frontend/desktop/src/analytics/tests/cloud-analytics.test.ts @@ -85,6 +85,17 @@ describe("cloud-analytics", () => { }); }); + it("encodes boolean event parameters as GA4 numeric values", async () => { + setDesktopAnalyticsClientId("cid-boolean"); + trackCloudAnalyticsEvent("feature_flag_seen", { enabled: true, dismissed: false }); + await flushDesktopCloudAnalytics(); + + const body = JSON.parse(String(fetchMock.mock.calls[0]?.[1]?.body)) as { + events: Array<{ params: Record }>; + }; + expect(body.events[0]?.params).toMatchObject({ enabled: 1, dismissed: 0 }); + }); + it("always supplies the required byok mode for a signed-out event", async () => { setDesktopAnalyticsClientId("cid-1"); trackCloudAnalyticsEvent("welcome_viewed"); diff --git a/App/memmy-agent/package-lock.json b/App/memmy-agent/package-lock.json index 2480c9f56..08f557291 100644 --- a/App/memmy-agent/package-lock.json +++ b/App/memmy-agent/package-lock.json @@ -1,12 +1,12 @@ { "name": "memmy-agent", - "version": "1.2.1", + "version": "1.2.2", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "memmy-agent", - "version": "1.2.1", + "version": "1.2.2", "dependencies": { "@anthropic-ai/sdk": "^0.100.1", "@aws-sdk/client-bedrock-runtime": "^3.1061.0", diff --git a/App/memmy-agent/package.json b/App/memmy-agent/package.json index 3ecb9b8bd..509cdfea9 100644 --- a/App/memmy-agent/package.json +++ b/App/memmy-agent/package.json @@ -1,6 +1,6 @@ { "name": "memmy-agent", - "version": "1.2.1", + "version": "1.2.2", "description": "TypeScript refactor of memmy's agent runtime.", "type": "module", "main": "./dist/index.js", diff --git a/App/memmy-agent/src/analytics/cloud-analytics.ts b/App/memmy-agent/src/analytics/cloud-analytics.ts index b8480729f..93437737a 100644 --- a/App/memmy-agent/src/analytics/cloud-analytics.ts +++ b/App/memmy-agent/src/analytics/cloud-analytics.ts @@ -111,9 +111,13 @@ export function resolveAnalyticsEnvParams(options: { }; } +/** GA4 Measurement Protocol accepts only string and number param values. A boolean drops the whole event. */ export function compactAnalyticsParams(params: AnalyticsParams = {}): AnalyticsParams { return Object.fromEntries( - Object.entries(params).filter(([, value]) => value !== undefined && value !== null && value !== ""), + Object.entries(params).flatMap(([key, value]) => { + if (value === undefined || value === null || value === "") return []; + return [[key, typeof value === "boolean" ? (value ? 1 : 0) : value]]; + }), ) as AnalyticsParams; } diff --git a/App/memmy-agent/src/core/agent-runtime/loop.ts b/App/memmy-agent/src/core/agent-runtime/loop.ts index 9b379fc5d..750a149c8 100644 --- a/App/memmy-agent/src/core/agent-runtime/loop.ts +++ b/App/memmy-agent/src/core/agent-runtime/loop.ts @@ -3892,7 +3892,7 @@ export class AgentLoop { session: ctx.session, sessionKey: ctx.sessionKey, reason: "system_prompt_build", - spec: { hostProjectId: ctx.sessionProjectId, workspace: sessionWorkspace }, + spec: { hostProjectId: ctx.sessionProjectId, workspace: sessionWorkspace, turnId: ctx.turnId, abortSignal: ctx.abortSignal }, metadata: { lifecycle: "system_prompt" }, })); const compactionOptions: { @@ -4266,6 +4266,7 @@ export class AgentLoop { sessionBindingOverride?: WebuiSessionBinding | null; } = {}, ): Promise { + turnId ??= firstString(msg.metadata?.turn_id, msg.metadata?.turnId) ?? cryptoRandomId(); const rawChatId = String(msg.chatId ?? ""); const separator = rawChatId.indexOf(":"); const channel = separator >= 0 ? rawChatId.slice(0, separator) : "cli"; @@ -4330,7 +4331,7 @@ export class AgentLoop { session, sessionKey: key, reason: "system_prompt_build", - spec: { hostProjectId: sessionBinding.projectId, workspace: sessionWorkspace }, + spec: { hostProjectId: sessionBinding.projectId, workspace: sessionWorkspace, turnId, abortSignal }, metadata: { lifecycle: "system_prompt" }, })); if (this.restoreRuntimeCheckpoint(session)) this.sessions.save(session); @@ -4453,6 +4454,7 @@ export class AgentLoop { sessionKey: key, pendingQueue, abortSignal, + turnId, tools, sessionWorkspace, hostProjectId: sessionBinding.projectId, diff --git a/App/memmy-agent/src/core/agent-runtime/runner.ts b/App/memmy-agent/src/core/agent-runtime/runner.ts index 7a28486c0..bfe6ef224 100644 --- a/App/memmy-agent/src/core/agent-runtime/runner.ts +++ b/App/memmy-agent/src/core/agent-runtime/runner.ts @@ -1491,6 +1491,7 @@ export class AgentRunner { } async run(spec: AgentRunSpec): Promise { + spec.turnId ??= crypto.randomUUID(); spec.provider ??= this.provider; if (!spec.provider) throw new Error("AgentRunSpec.provider is required"); const hook = spec.hook ?? new AgentHook(); diff --git a/App/memmy-agent/src/core/agent-runtime/tools/filesystem.ts b/App/memmy-agent/src/core/agent-runtime/tools/filesystem.ts index 671a0564b..b80a31c1f 100644 --- a/App/memmy-agent/src/core/agent-runtime/tools/filesystem.ts +++ b/App/memmy-agent/src/core/agent-runtime/tools/filesystem.ts @@ -32,6 +32,14 @@ function normalizeNewlines(text: string): string { return text.replace(/\r\n/g, "\n"); } +/** Same line split as apply_patch: a final newline does not create an extra blank line. */ +function splitFileLines(text: string): string[] { + if (!text) return []; + const lines = normalizeNewlines(text).split("\n"); + if (lines.at(-1) === "") lines.pop(); + return lines; +} + function lineNumbered(lines: string[], start: number): string { const width = String(start + lines.length - 1).length; return lines.map((line, index) => `${String(start + index).padStart(width, " ")}| ${line}`).join("\n"); @@ -83,37 +91,48 @@ function filterExtractedPages(text: string, pages?: string | null): string { .join("\n\n"); } -async function formatDocumentRead(target: string, pages?: string | null, maxChars = 128_000): Promise { - const ext = path.extname(target).toLowerCase(); - const extracted = await extractText(target); - if (extracted == null) return `Error: Unsupported file format: ${ext}`; - if (extracted.startsWith("[error:")) return `Error reading ${ext.toUpperCase()} file: ${extracted}`; - const paged = ext === ".pdf" ? filterExtractedPages(extracted, pages) : extracted; - if (paged.startsWith("Error:")) return paged; - if (!paged) return `(${ext.toUpperCase().replace(".", "")} has no extractable text: ${target})`; - if (paged.length > maxChars) return `${paged.slice(0, maxChars)}\n\n(Document text truncated at ~128K chars)`; - return paged; +interface DocumentReadOptions { + char_offset?: number; + char_limit?: number; } -async function readPdf(target: string, pages?: string | null, maxChars = 128_000): Promise { - const extracted = await extractText(target); - if (extracted == null) return "Error: PDF reading requires PDF text extraction support."; - if (extracted.startsWith("[error:")) return `Error reading PDF: ${extracted}`; - const paged = filterExtractedPages(extracted, pages); - if (paged.startsWith("Error:")) return paged; - if (!paged) return `(PDF has no extractable text: ${target})`; - if (paged.length > maxChars) return `${paged.slice(0, maxChars)}\n\n(PDF text truncated at ~128K chars)`; - return paged; +function paginateDocument(text: string, options: DocumentReadOptions, maxChars: number): string { + const offset = options.char_offset ?? 0; + const limit = options.char_limit ?? 32_000; + if (!Number.isSafeInteger(offset) || offset < 0 || offset > text.length) { + return `Error reading document: char_offset must be between 0 and ${text.length}.`; + } + if (!Number.isSafeInteger(limit) || limit < 1) { + return "Error reading document: char_limit must be a positive integer."; + } + // Reserve space for the continuation marker inside the runtime's result budget. + const budget = Math.max(2, maxChars - 512); + let end = Math.min(text.length, offset + Math.min(limit, budget)); + if (end < text.length && /[\uD800-\uDBFF]/.test(text[end - 1]) && /[\uDC00-\uDFFF]/.test(text[end])) { + // A one-unit request must still make progress over a surrogate pair. + end += end - offset === 1 ? 1 : -1; + } + const status = end < text.length + ? `More content remains. Continue with char_offset=${end}. Keep the same path and pages; do not treat this excerpt as the full document.` + : "End of document (selected pages, if any)."; + return `${text.slice(offset, end)}\n\n[Document read: chars ${offset}-${end} of ${text.length} (UTF-16, end exclusive). ${status}]`; } -async function readOfficeDoc(target: string, maxChars = 128_000): Promise { +async function formatDocumentRead( + target: string, + pages?: string | null, + maxChars = 128_000, + options: DocumentReadOptions = {}, +): Promise { const ext = path.extname(target).toLowerCase(); - const extracted = await extractText(target); + // Page filtering and continuation must operate on the complete extraction, not its preview. + const extracted = await extractText(target, { maxChars: null }); if (extracted == null) return `Error: Unsupported file format: ${ext}`; if (extracted.startsWith("[error:")) return `Error reading ${ext.toUpperCase()} file: ${extracted}`; - if (!extracted) return `(${ext.toUpperCase().replace(".", "")} has no extractable text: ${target})`; - if (extracted.length > maxChars) return `${extracted.slice(0, maxChars)}\n\n(Document text truncated at ~128K chars)`; - return extracted; + const paged = ext === ".pdf" ? filterExtractedPages(extracted, pages) : extracted; + if (paged.startsWith("Error:")) return paged; + if (!paged) return `(${ext.toUpperCase().replace(".", "")} has no extractable text: ${target})`; + return paginateDocument(paged, options, maxChars); } function splitLinesKeepEnds(text: string): string[] { @@ -575,7 +594,10 @@ export class ReadFileTool extends Tool { "Images return visual content for analysis. Supports PDF, DOCX, XLSX, PPTX documents. " + "Use find_files/list_dir first when the path is uncertain. " + "Read the relevant range before editing so replacements or patches are based on current content. " + - "Use offset and limit for large text files. Use force=true to re-read content even if unchanged." + "Use offset and limit for text file lines. For documents use pages for PDF page ranges, " + + "and char_offset/char_limit to read extracted text in chunks (also within a large PDF page). " + + "Follow the returned char_offset with the same path/pages until the required content is read; " + + "an excerpt is not the full document. Use force=true to re-read text even if unchanged." ); } @@ -586,7 +608,9 @@ export class ReadFileTool extends Tool { path: { type: "string" }, offset: { type: "integer", minimum: 1 }, limit: { type: "integer", minimum: 1 }, - pages: { type: "string" }, + pages: { type: "string", description: "PDF page or inclusive page range, e.g. 15 or 15-20." }, + char_offset: { type: "integer", minimum: 0, description: "Documents only: zero-based UTF-16 offset in extracted text after PDF page selection. Use the returned continuation offset." }, + char_limit: { type: "integer", minimum: 1, description: "Documents only: requested chunk size in UTF-16 units (default 32000; capped to the output budget)." }, force: { type: "boolean" }, }, required: ["path"], @@ -605,7 +629,7 @@ export class ReadFileTool extends Tool { return resolveFileStates(this.fileStateOwner, this.fallbackFileStates, this.requestContext); } - async execute(params: { path?: string; offset?: number; limit?: number; pages?: string; force?: boolean } = {}): Promise { + async execute(params: { path?: string; offset?: number; limit?: number; pages?: string; force?: boolean } & DocumentReadOptions = {}): Promise { const requested = params.path; if (!requested) return "Error reading file: Unknown path"; if (isBlockedDevicePath(requested)) { @@ -622,9 +646,8 @@ export class ReadFileTool extends Tool { const stat = await fs.stat(target); if (!stat.isFile()) return `Error reading file: not a file: ${requested}`; const ext = path.extname(target).toLowerCase(); - if (ext === ".pdf") return this.readPdfFile(target, params.pages ?? null); - if ([".docx", ".xlsx", ".pptx"].includes(ext)) return this.readOfficeDocument(target); - if (DOCUMENT_EXTS.has(ext)) return await formatDocumentRead(target, params.pages, ReadFileTool.MAX_CHARS); + if (ext === ".pdf") return await this.readPdfFile(target, params.pages ?? null, params); + if (DOCUMENT_EXTS.has(ext)) return await this.readOfficeDocument(target, params); if (IMAGE_EXTS.has(ext)) { const data = await fs.readFile(target); const mime = lookup(target) || "image/png"; @@ -638,7 +661,7 @@ export class ReadFileTool extends Tool { } const content = await fs.readFile(target, "utf8"); if (!content) return "Empty file"; - const lines = normalizeNewlines(content).split("\n"); + const lines = splitFileLines(content); const offset = params.offset ?? 1; if (offset > lines.length) return `Error reading file: offset ${offset} beyond end (${lines.length} lines)`; const limit = params.limit ?? null; @@ -665,12 +688,12 @@ export class ReadFileTool extends Tool { } } - readPdfFile(target: string, pages?: string | null): Promise { - return readPdf(target, pages, ReadFileTool.MAX_CHARS); + readPdfFile(target: string, pages?: string | null, options: DocumentReadOptions = {}): Promise { + return formatDocumentRead(target, pages, ReadFileTool.MAX_CHARS, options); } - readOfficeDocument(target: string): Promise { - return readOfficeDoc(target, ReadFileTool.MAX_CHARS); + readOfficeDocument(target: string, options: DocumentReadOptions = {}): Promise { + return formatDocumentRead(target, null, ReadFileTool.MAX_CHARS, options); } } diff --git a/App/memmy-agent/src/memmy-memory/client.ts b/App/memmy-agent/src/memmy-memory/client.ts index 89381f4e6..bb1a6ef95 100644 --- a/App/memmy-agent/src/memmy-memory/client.ts +++ b/App/memmy-agent/src/memmy-memory/client.ts @@ -28,6 +28,8 @@ export class MemmyMemoryHttpError extends Error { type FetchLike = typeof fetch; +export type MemoryRequestOptions = { signal?: AbortSignal; timeoutMs?: number }; + export const DEFAULT_MEMOS_MEMORY_TIMEOUT_MS = 60_000; /** @@ -70,6 +72,7 @@ export class MemmyMemoryClient { body?: any; headers?: Record; timeoutMs?: number; + signal?: AbortSignal; } = {}): Promise { const headers: Record = { accept: "application/json", ...(opts.headers ?? {}) }; headers["x-memmy-time-zone"] = this.timeZone; @@ -82,7 +85,9 @@ export class MemmyMemoryClient { method, headers, body: opts.body === undefined ? undefined : JSON.stringify(opts.body), - signal: AbortSignal.timeout(opts.timeoutMs ?? this.timeoutMs), + signal: opts.signal + ? AbortSignal.any([opts.signal, AbortSignal.timeout(opts.timeoutMs ?? this.timeoutMs)]) + : AbortSignal.timeout(opts.timeoutMs ?? this.timeoutMs), }); const text = await response.text(); const parsed = text.trim() ? safeJsonParse(text) : null; @@ -94,36 +99,36 @@ export class MemmyMemoryClient { return parsed as T; } - get(path: string, query?: Record): Promise { - return this.request("GET", path, { query }); + get(path: string, query?: Record, options: MemoryRequestOptions = {}): Promise { + return this.request("GET", path, { ...options, query }); } - post(path: string, body: any = {}): Promise { - return this.request("POST", path, { body }); + post(path: string, body: any = {}, options: MemoryRequestOptions = {}): Promise { + return this.request("POST", path, { ...options, body }); } - async health(): Promise { - return MemoryHealthSnapshotSchema.parse(await this.get("/api/v1/health")); + async health(options: MemoryRequestOptions = {}): Promise { + return MemoryHealthSnapshotSchema.parse(await this.get("/api/v1/health", undefined, options)); } - openSession(body: JsonRecord & MemmyMemoryRequestEnvelope): Promise { - return this.post("/api/v1/sessions/open", body); + openSession(body: JsonRecord & MemmyMemoryRequestEnvelope, options: MemoryRequestOptions = {}): Promise { + return this.post("/api/v1/sessions/open", body, options); } - closeSession(sessionId: string, body: MemmyMemoryRequestEnvelope): Promise { - return this.post(`/api/v1/sessions/${encodeURIComponent(sessionId)}/close`, body); + closeSession(sessionId: string, body: MemmyMemoryRequestEnvelope, options: MemoryRequestOptions = {}): Promise { + return this.post(`/api/v1/sessions/${encodeURIComponent(sessionId)}/close`, body, options); } - startTurn(turnId: string, body: JsonRecord & MemmyMemoryRequestEnvelope): Promise { + startTurn(turnId: string, body: JsonRecord & MemmyMemoryRequestEnvelope, options: MemoryRequestOptions = {}): Promise { return this.post("/api/v1/turns/start", { ...body, turnId, query: body.query, - }); + }, options); } - completeTurn(turnId: string, body: JsonRecord & MemmyMemoryRequestEnvelope): Promise { - return this.post(`/api/v1/turns/${encodeURIComponent(turnId)}/complete`, body); + completeTurn(turnId: string, body: JsonRecord & MemmyMemoryRequestEnvelope, options: MemoryRequestOptions = {}): Promise { + return this.post(`/api/v1/turns/${encodeURIComponent(turnId)}/complete`, body, options); } search(body: JsonRecord & MemmyMemoryRequestEnvelope): Promise { @@ -136,36 +141,40 @@ export class MemmyMemoryClient { async l3WorldModelTraceHead( sessionId: string, - envelope: L3WorldModelRequestEnvelope + envelope: L3WorldModelRequestEnvelope, + options: MemoryRequestOptions = {} ): Promise { const transport = l3WorldModelGetTransport(envelope); const value = await this.request( "GET", `/api/v1/sessions/${encodeURIComponent(sessionId)}/l3-world-model-trace-head`, - { query: transport.query, headers: transport.headers } + { ...options, query: transport.query, headers: transport.headers } ); return L3WorldModelTraceHeadResponseSchema.parse(value); } async l3WorldModelBoundary( sessionId: string, - request: L3WorldModelBoundaryRequest + request: L3WorldModelBoundaryRequest, + options: MemoryRequestOptions = {} ): Promise { return L3WorldModelBoundaryResponseSchema.parse(await this.post( `/api/v1/sessions/${encodeURIComponent(sessionId)}/l3-world-model-boundary`, - request + request, + options )); } async l3WorldModelContext( sessionId: string, - envelope: L3WorldModelRequestEnvelope + envelope: L3WorldModelRequestEnvelope, + options: MemoryRequestOptions = {} ): Promise { const transport = l3WorldModelGetTransport(envelope); const value = await this.request( "GET", `/api/v1/l3-world-model/sessions/${encodeURIComponent(sessionId)}/context`, - { query: transport.query, headers: transport.headers } + { ...options, query: transport.query, headers: transport.headers } ); return SessionL3WorldModelContextResponseSchema.parse(value); } diff --git a/App/memmy-agent/src/memmy-memory/hook.ts b/App/memmy-agent/src/memmy-memory/hook.ts index 80a993bf3..fd7365dbb 100644 --- a/App/memmy-agent/src/memmy-memory/hook.ts +++ b/App/memmy-agent/src/memmy-memory/hook.ts @@ -1,5 +1,6 @@ import { createHash, randomUUID } from "node:crypto"; -import { INTERACTIVE_MEMORY_TIMEOUT_MS } from "./client.js"; +import { DEFAULT_MEMOS_MEMORY_TIMEOUT_MS, INTERACTIVE_MEMORY_TIMEOUT_MS, MemmyMemoryHttpError } from "./client.js"; +import { MemoryWriteQueue, waitForMemory } from "./lifecycle.js"; import { getOrCreateInstallationId } from "../analytics/cloud-analytics.js"; import { AgentHook, AgentHookContext, type AgentToolRegistrationContext, type SystemPromptBuildContext } from "../core/agent-runtime/hook.js"; import { ContextBuilder } from "../core/agent-runtime/context.js"; @@ -16,7 +17,6 @@ import { createMemoryLifecycleAnalytics, elapsedMs, errorCodeFromUnknown, - hasInjectedContextValue, hashId, hitCountFromSearchResponse, memoryAnalyticsEventsFor, @@ -59,6 +59,21 @@ Treat as authoritative and as untr If appears, memory was not checked. Tell the user the long-term memory service is temporarily unavailable rather than implying a search found no results.`; +type SessionGeneration = { + controller: AbortController; + pending?: Promise; + state?: MemmyMemorySessionState; +}; + +type InteractiveTurn = MemmyMemoryTurnState & { + generation: SessionGeneration; + deadline: number; + controller: AbortController; + abortSignal?: AbortSignal; + recallApplied: boolean; + finished: boolean; +}; + export class MemmyMemoryHook extends AgentHook implements MemmyMemoryToolRuntime { private readonly client: MemmyMemoryClient; private readonly options: Required< @@ -84,8 +99,13 @@ export class MemmyMemoryHook extends AgentHook implements MemmyMemoryToolRuntime private readonly analytics: MemoryLifecycleAnalytics; lastError: string | null = null; private initialized = false; + private disposed = false; + private readonly generations = new Map(); + private readonly allGenerations = new Set(); + private readonly writes = new MemoryWriteQueue(); + private readonly closing = new Map; retry: (signal: AbortSignal) => Promise }>(); private readonly sessionIdBySessionKey = new Map(); - private readonly turnBySessionKey = new Map(); + private readonly turnBySessionKey = new Map(); private readonly entrypointBySessionKey = new Map(); private readonly unavailableWarnedSessionKeys = new Set(); private readonly sessionStateBySessionKey = new Map(); @@ -147,15 +167,19 @@ export class MemmyMemoryHook extends AgentHook implements MemmyMemoryToolRuntime override async beforeBuildSystemPrompt(ctx: AgentHookContext): Promise { const sessionKey = this.sessionKeyFromContext(ctx); - if (!sessionKey) return; + if (!sessionKey || this.disposed) return; + const turn = this.interactiveTurn(ctx, sessionKey); try { - await this.withInteractiveDeadline(async () => { - await this.ensureSession(ctx, sessionKey); - const state = this.sessionStateBySessionKey.get(sessionKey); - if (state?.protocol === "v2") await this.loadL3Context(sessionKey, state); + await this.withTurnDeadline(turn, "prompt", async (signal) => { + await this.ensureSession(ctx, sessionKey, turn.generation); + signal.throwIfAborted(); + this.assertCurrentTurn(turn); + const state = turn.generation.state; + if (state?.protocol === "v2") await this.loadL3Context(sessionKey, state, turn, signal); }); this.clearMemoryUnavailable(sessionKey); } catch (error) { + if (!this.isCurrentTurn(turn)) return; this.rememberUnavailableL3(sessionKey); this.warnMemoryUnavailable(sessionKey, "recall", error); } @@ -163,44 +187,87 @@ export class MemmyMemoryHook extends AgentHook implements MemmyMemoryToolRuntime override async sessionStart(ctx: AgentHookContext): Promise { const sessionKey = this.sessionKeyFromContext(ctx); - if (!sessionKey) return; - try { - await this.withInteractiveDeadline(() => this.ensureSession(ctx, sessionKey)); - this.clearMemoryUnavailable(sessionKey); - } catch (error) { - this.warnMemoryUnavailable(sessionKey, "session-start", error); + if (!sessionKey || this.disposed) return; + const generation = this.generation(sessionKey); + // Warm the shared initialization without adding a second foreground budget. + void this.ensureSession(ctx, sessionKey, generation).catch((error) => { + if (this.generations.get(sessionKey) === generation) { + this.warnMemoryUnavailable(sessionKey, "session-start", error); + } + }); + } + + private generation(sessionKey: string): SessionGeneration { + let generation = this.generations.get(sessionKey); + if (!generation) { + generation = { controller: new AbortController() }; + this.generations.set(sessionKey, generation); + this.allGenerations.add(generation); + } + return generation; + } + + private interactiveTurn(ctx: AgentHookContext, sessionKey: string): InteractiveTurn { + const turnId = stringOrUndefined(ctx.spec?.turnId) ?? randomUUID(); + const previous = this.turnBySessionKey.get(sessionKey); + if (previous?.turnId === turnId && !previous.finished) return previous; + previous?.controller.abort(new Error("memory turn superseded")); + const turn: InteractiveTurn = { + sessionKey, turnId, sessionId: "", userText: "", messageStartIndex: 0, + generation: this.generation(sessionKey), + deadline: performance.now() + INTERACTIVE_MEMORY_TIMEOUT_MS, + controller: new AbortController(), + abortSignal: ctx.spec?.abortSignal ?? undefined, + recallApplied: false, + finished: false, + }; + this.turnBySessionKey.set(sessionKey, turn); + return turn; + } + + private isCurrentTurn(turn: InteractiveTurn): boolean { + return !this.disposed && !turn.controller.signal.aborted && !turn.abortSignal?.aborted + && this.generations.get(turn.sessionKey) === turn.generation + && this.turnBySessionKey.get(turn.sessionKey) === turn; + } + + private assertCurrentTurn(turn: InteractiveTurn): void { + if (!this.isCurrentTurn(turn) || turn.finished || performance.now() >= turn.deadline) { + throw new Error("memory turn expired or cancelled"); } } - /** - * Bounds a memory call that sits between the user and their reply. - * - * These hooks are already written to continue without memory, so the only - * thing a long wait buys is a stalled send: the desktop client abandons a - * message after 30 seconds, while the memory client alone allows 60. Stop - * waiting well before that. The request itself is left running, so a slow - * service still warms the session for the next turn. - */ - private async withInteractiveDeadline(operation: () => Promise): Promise { - let timer: ReturnType | undefined; + private forgetTurn(turn: InteractiveTurn): void { + if (this.turnBySessionKey.get(turn.sessionKey) === turn) this.turnBySessionKey.delete(turn.sessionKey); + turn.finished = true; + turn.controller.abort(new Error("memory turn finished")); + } + + private async withTurnDeadline(turn: InteractiveTurn, phase: string, operation: (signal: AbortSignal) => Promise): Promise { + const startedAt = performance.now(); + let status = "failed"; try { - return await Promise.race([ - operation(), - new Promise((_resolve, reject) => { - timer = setTimeout( - () => reject(new Error(`memory did not answer within ${INTERACTIVE_MEMORY_TIMEOUT_MS}ms`)), - INTERACTIVE_MEMORY_TIMEOUT_MS, - ); - }), + this.assertCurrentTurn(turn); + const signal = AbortSignal.any([ + turn.controller.signal, turn.generation.controller.signal, + ...(turn.abortSignal ? [turn.abortSignal] : []), ]); + const result = await waitForMemory(operation, turn.deadline - performance.now(), signal); + status = "succeeded"; + return result; } finally { - if (timer) clearTimeout(timer); + this.analytics.track("memory_interactive_wait_finished", { + ...this.turnAnalyticsParams(turn), phase, + status: turn.abortSignal?.aborted || turn.controller.signal.aborted ? "cancelled" + : performance.now() >= turn.deadline ? "timed_out" : status, + duration_ms: Math.round(performance.now() - startedAt), + }); } } override async beforeRun(ctx: AgentHookContext): Promise { const sessionKey = this.sessionKeyFromContext(ctx); - if (!sessionKey) return; + if (!sessionKey || this.disposed) return; const messages = ctx.messages ?? ctx.spec?.initialMessages ?? []; const internalTurnContext = ctx.spec?.internalTurnContext; const isGoalContinuation = internalTurnContext?.kind === "goal_continuation"; @@ -208,72 +275,77 @@ export class MemmyMemoryHook extends AgentHook implements MemmyMemoryToolRuntime ? internalTurnContext.objective.trim() : ""; if (isGoalContinuation && !internalObjective) return; + const turn = this.interactiveTurn(ctx, sessionKey); + turn.userText = isGoalContinuation ? internalObjective : lastUserText(messages); + turn.messageStartIndex = messages.length; try { - const sessionId = await this.ensureSession(ctx, sessionKey); - const turnId = stringOrUndefined(ctx.spec?.turnId) ?? randomUUID(); - const userText = isGoalContinuation ? internalObjective : lastUserText(messages); - const turn: MemmyMemoryTurnState = { - sessionKey, - sessionId, - turnId, - userText, - messageStartIndex: messages.length, - }; - this.turnBySessionKey.set(sessionKey, turn); - - const events = this.eventsFor(sessionKey, ctx); - this.analytics.track(events.turnStarted, this.turnAnalyticsParams(turn)); - - const retrievalLayerLabel = this.options.retrievalLayers === null - ? "all" - : this.options.retrievalLayers.length > 0 - ? this.options.retrievalLayers.join("+") - : "none"; - const searchBase = this.memoryOpParams(turn, MEMORY_OP_MODES.turnStart, retrievalLayerLabel, sessionKey, ctx); - this.analytics.track(events.searchStarted, searchBase); - const searchStartedAt = Date.now(); - try { - const response = await this.client.startTurn(turnId, compact({ - ...this.requestEnvelope(sessionKey, ctx), - sessionId, - query: userText || "(conversation continued)", - layers: this.options.retrievalLayers ?? undefined, - })); - turn.episodeId = stringOrUndefined(response?.episodeId); - turn.sourceMemoryIds = arrayOfStrings(response?.sourceMemoryIds); - turn.hasInjectedContext = hasInjectedContextValue(response?.injectedContext); - turn.sourceMemoryCount = sourceMemoryCountFromResponse(response); - this.injectMemoryContext(messages, response?.injectedContext); - turn.messageStartIndex = messages.length; - this.analytics.track(events.searchSucceeded, { - ...this.memoryOpParams(turn, MEMORY_OP_MODES.turnStart, retrievalLayerLabel, sessionKey, ctx), - duration_ms: elapsedMs(searchStartedAt), - success: true, - hit_count: hitCountFromSearchResponse(response), - }); - } catch (error) { - this.analytics.track(events.searchFailed, { - ...searchBase, - duration_ms: elapsedMs(searchStartedAt), - success: false, - error_code: errorCodeFromUnknown(error), - }); - this.analytics.track(events.turnFailed, { - ...this.turnAnalyticsParams(turn), - has_injected_context: false, - source_memory_count: 0, - tool_call_count: 0, - status: "failed", - phase: "start", - error_code: errorCodeFromUnknown(error), - }); - throw error; - } + await this.withTurnDeadline(turn, "recall", async (signal) => { + const sessionId = await this.ensureSession(ctx, sessionKey, turn.generation); + signal.throwIfAborted(); + this.assertCurrentTurn(turn); + turn.sessionId = sessionId; + const turnId = turn.turnId; + const userText = turn.userText; + + const events = this.eventsFor(sessionKey, ctx); + this.analytics.track(events.turnStarted, this.turnAnalyticsParams(turn)); + + const retrievalLayerLabel = this.options.retrievalLayers === null + ? "all" + : this.options.retrievalLayers.length > 0 + ? this.options.retrievalLayers.join("+") + : "none"; + const searchBase = this.memoryOpParams(turn, MEMORY_OP_MODES.turnStart, retrievalLayerLabel, sessionKey, ctx); + this.analytics.track(events.searchStarted, searchBase); + const searchStartedAt = Date.now(); + try { + const response = await this.client.startTurn(turnId, compact({ + ...this.requestEnvelope(sessionKey, ctx), + sessionId, + query: userText || "(conversation continued)", + layers: this.options.retrievalLayers ?? undefined, + }), { signal }); + signal.throwIfAborted(); + this.assertCurrentTurn(turn); + turn.recallApplied = this.injectMemoryContext(messages, response?.injectedContext); + turn.episodeId = stringOrUndefined(response?.episodeId); + turn.sourceMemoryIds = turn.recallApplied ? arrayOfStrings(response?.sourceMemoryIds) : []; + turn.hasInjectedContext = turn.recallApplied; + turn.sourceMemoryCount = turn.recallApplied ? sourceMemoryCountFromResponse(response) : 0; + turn.messageStartIndex = messages.length; + this.analytics.track(events.searchSucceeded, { + ...this.memoryOpParams(turn, MEMORY_OP_MODES.turnStart, retrievalLayerLabel, sessionKey, ctx), + duration_ms: elapsedMs(searchStartedAt), + success: true, + hit_count: hitCountFromSearchResponse(response), + }); + } catch (error) { + this.analytics.track(events.searchFailed, { + ...searchBase, + duration_ms: elapsedMs(searchStartedAt), + success: false, + error_code: errorCodeFromUnknown(error), + }); + this.analytics.track(events.turnFailed, { + ...this.turnAnalyticsParams(turn), + has_injected_context: false, + source_memory_count: 0, + tool_call_count: 0, + status: "failed", + phase: "start", + error_code: errorCodeFromUnknown(error), + }); + throw error; + } + }); this.clearMemoryUnavailable(sessionKey); } catch (error) { - this.turnBySessionKey.delete(sessionKey); - this.warnMemoryUnavailable(sessionKey, "recall", error); - this.injectMemoryUnavailableNotice(messages); + if (this.isCurrentTurn(turn)) { + this.warnMemoryUnavailable(sessionKey, "recall", error); + this.injectMemoryUnavailableNotice(messages); + } + } finally { + turn.finished = true; } } @@ -281,11 +353,11 @@ export class MemmyMemoryHook extends AgentHook implements MemmyMemoryToolRuntime const sessionKey = this.sessionKeyFromContext(ctx); if (!sessionKey) return; const turn = this.turnBySessionKey.get(sessionKey); - if (!turn) return; + if (!turn || (ctx.spec?.turnId && ctx.spec.turnId !== turn.turnId)) return; try { const status = statusFromResult(result, ctx); if (status === "cancelled") { - this.turnBySessionKey.delete(sessionKey); + this.forgetTurn(turn); return; } const messages = Array.isArray(result?.messages) ? result.messages : []; @@ -304,133 +376,192 @@ export class MemmyMemoryHook extends AgentHook implements MemmyMemoryToolRuntime status === "failed" ? failedTurnText(result, ctx) : undefined, ); if (!turn.userText.trim() || !answer) { - this.turnBySessionKey.delete(sessionKey); + this.forgetTurn(turn); return; } - const baseParams = { - ...this.turnAnalyticsParams(turn), - has_injected_context: Boolean(turn.hasInjectedContext), - source_memory_count: turn.sourceMemoryCount ?? 0, - tool_call_count: toolCalls.length, - status, - }; - const events = this.eventsFor(sessionKey, ctx); - const addBase = this.memoryOpParams(turn, MEMORY_OP_MODES.turnComplete, "L1", sessionKey, ctx); - this.analytics.track(events.addStarted, addBase); - const addStartedAt = Date.now(); - try { - const response = await this.client.completeTurn(turn.turnId, compact({ - ...this.requestEnvelope(sessionKey, ctx), - requestId: completeRequestId(turn.turnId, status, turn.userText, answer), - sessionId: turn.sessionId, - episodeId: turn.episodeId, - query: turn.userText, - answer, - reasoningSummary, - toolCalls, - toolResults, - sourceMemoryIds: turn.sourceMemoryIds, - usage: result?.usage ?? ctx.usage, - status, - })); - turn.rawTurnId = stringOrUndefined(response?.rawTurnId) ?? turn.rawTurnId; - turn.l1MemoryId = stringOrUndefined(response?.l1MemoryId) ?? turn.l1MemoryId; - const l1MemoryIds = arrayOfStrings(response?.l1MemoryIds); - if (!turn.l1MemoryId && l1MemoryIds?.length) turn.l1MemoryId = l1MemoryIds[0]; - this.analytics.track(events.addSucceeded, { - ...addBase, - duration_ms: elapsedMs(addStartedAt), - success: true, - stored_count: storedCountFromCompleteTurn(response), - }); - this.analytics.track(events.turnCompleted, baseParams); - this.turnBySessionKey.delete(sessionKey); - } catch (error) { - this.analytics.track(events.addFailed, { - ...addBase, - duration_ms: elapsedMs(addStartedAt), - success: false, - error_code: errorCodeFromUnknown(error), - }); - this.analytics.track(events.turnFailed, { - ...baseParams, - status: "failed", - phase: "complete", - error_code: errorCodeFromUnknown(error), - }); - throw error; - } - this.clearMemoryUnavailable(sessionKey); + const capture = structuredClone({ + status, answer, reasoningSummary, toolCalls, toolResults, + usage: result?.usage ?? ctx.usage, + }); + const captureContext = new AgentHookContext({ + sessionKey, + spec: { workspace: this.workspaceFromContext(ctx), hostProjectId: this.hostProjectIdFromContext(ctx) }, + metadata: { ...ctx.metadata }, + }); + this.forgetTurn(turn); + this.writes.enqueue(turn.generation, Buffer.byteLength(JSON.stringify(capture)) + Buffer.byteLength(turn.userText), async (signal) => { + turn.sessionId = await this.ensureSession(captureContext, sessionKey, turn.generation); + signal.throwIfAborted(); + await this.captureTurn(turn, captureContext, capture, signal); + }, (error) => this.reportBackgroundFailure(sessionKey, "write", error)); } catch (error) { + this.forgetTurn(turn); this.warnMemoryUnavailable(sessionKey, "write", error); } } + /** Explicit drain for shutdown and tests; normal answers never wait on it. */ + async flushPendingWrites(): Promise { await this.writes.flush(); } + + private async captureTurn(turn: InteractiveTurn, ctx: AgentHookContext, capture: JsonRecord, signal: AbortSignal): Promise { + const sessionKey = turn.sessionKey; + const { status, answer, reasoningSummary, toolCalls, toolResults, usage } = capture; + const baseParams = { + ...this.turnAnalyticsParams(turn), + has_injected_context: Boolean(turn.hasInjectedContext), + source_memory_count: turn.sourceMemoryCount ?? 0, + tool_call_count: toolCalls.length, + status, + }; + const events = this.eventsFor(sessionKey, ctx); + const addBase = this.memoryOpParams(turn, MEMORY_OP_MODES.turnComplete, "L1", sessionKey, ctx); + this.analytics.track(events.addStarted, addBase); + const addStartedAt = Date.now(); + try { + const body = compact({ + ...this.envelopeForGeneration(turn.generation, sessionKey, ctx), + requestId: completeRequestId(turn.turnId, status, turn.userText, answer), + sessionId: turn.sessionId, + episodeId: turn.episodeId, + query: turn.userText, answer, reasoningSummary, toolCalls, toolResults, + sourceMemoryIds: turn.sourceMemoryIds ?? [], + recallApplied: turn.recallApplied, + usage, status, + }); + let response: JsonRecord = {}; + for (let attempt = 0; ; attempt += 1) { + signal.throwIfAborted(); + try { + response = await this.client.completeTurn(turn.turnId, body, { signal }); + break; + } catch (error) { + if (signal.aborted || attempt >= 2 || (error instanceof MemmyMemoryHttpError && error.status < 500 && error.status !== 429)) throw error; + await waitForMemory(async (retrySignal) => new Promise((resolve, reject) => { + const onAbort = () => { clearTimeout(timer); reject(retrySignal.reason); }; + const timer = setTimeout(() => { retrySignal.removeEventListener("abort", onAbort); resolve(); }, 250 * (attempt + 1)); + retrySignal.addEventListener("abort", onAbort, { once: true }); + }), 2_000, signal); + } + } + signal.throwIfAborted(); + turn.rawTurnId = stringOrUndefined(response?.rawTurnId) ?? turn.rawTurnId; + turn.l1MemoryId = stringOrUndefined(response?.l1MemoryId) ?? turn.l1MemoryId; + const l1MemoryIds = arrayOfStrings(response?.l1MemoryIds); + if (!turn.l1MemoryId && l1MemoryIds?.length) turn.l1MemoryId = l1MemoryIds[0]; + this.analytics.track(events.addSucceeded, { + ...addBase, + duration_ms: elapsedMs(addStartedAt), + success: true, + stored_count: storedCountFromCompleteTurn(response), + }); + this.analytics.track(events.turnCompleted, baseParams); + + } catch (error) { + this.analytics.track(events.addFailed, { + ...addBase, + duration_ms: elapsedMs(addStartedAt), + success: false, + error_code: errorCodeFromUnknown(error), + }); + this.analytics.track(events.turnFailed, { + ...baseParams, + status: "failed", + phase: "complete", + error_code: errorCodeFromUnknown(error), + }); + throw error; + } + if (this.generations.get(sessionKey) === turn.generation) this.clearMemoryUnavailable(sessionKey); + } + override async afterCompaction(ctx: AgentHookContext): Promise { if (ctx.compaction?.kind !== "token" || ctx.compaction.changed !== true || ctx.compaction.error) return; const sessionKey = this.sessionKeyFromContext(ctx); - if (!sessionKey) return; - const state = this.sessionStateBySessionKey.get(sessionKey); - if (!state || state.protocol !== "v2") return; - try { + if (!sessionKey || this.disposed) return; + const generation = this.generations.get(sessionKey); + if (!generation) return; + this.writes.enqueue(generation, 0, async (signal) => { + const state = generation.state; + if (!state || state.protocol !== "v2") return; const envelope = this.l3Envelope(sessionKey, state); - const head = await this.client.l3WorldModelTraceHead(state.memorySessionId, envelope); + const head = await this.client.l3WorldModelTraceHead(state.memorySessionId, envelope, { signal }); + signal.throwIfAborted(); if (head.throughL1MemoryId) { await this.client.l3WorldModelBoundary(state.memorySessionId, { - ...envelope, - trigger: "token_compaction", - throughL1MemoryId: head.throughL1MemoryId, - }); + ...envelope, trigger: "token_compaction", throughL1MemoryId: head.throughL1MemoryId, + }, { signal }); } - this.clearMemoryUnavailable(sessionKey); - } catch (error) { - this.warnMemoryUnavailable(sessionKey, "recall", error); - } + }, (error) => this.reportBackgroundFailure(sessionKey, "recall", error)); } override async sessionEnd(ctx: AgentHookContext): Promise { const sessionKey = this.sessionKeyFromContext(ctx); if (!sessionKey) return; - try { - const cachedSessionId = this.sessionIdBySessionKey.get(sessionKey) ?? null; - // Only close sessions this hook instance opened. Without a cached id there is - // nothing to close against stock Memory (no close-active API). - if (cachedSessionId) { - const response = await this.client.closeSession( - cachedSessionId, - this.requestEnvelope(sessionKey, ctx), - ); - const closedSessionId = - stringOrUndefined(response?.sessionId) ?? cachedSessionId; - if (closedSessionId && response?.status !== "noop") { + const generation = this.generations.get(sessionKey); + if (!generation) return; + this.generations.delete(sessionKey); + const turn = this.turnBySessionKey.get(sessionKey); + if (turn) this.forgetTurn(turn); + this.sessionIdBySessionKey.delete(sessionKey); + this.sessionStateBySessionKey.delete(sessionKey); + this.entrypointBySessionKey.delete(sessionKey); + this.clearMemoryUnavailable(sessionKey); + if (!this.writes.has(generation) && !generation.pending) generation.controller.abort(new Error("memory session closed")); + // Previously accepted captures run before closing their original remote session. + const close = async (signal: AbortSignal): Promise => { + try { + await generation.pending?.catch(() => {}); + signal.throwIfAborted(); + const cachedSessionId = generation.state?.memorySessionId; + if (!cachedSessionId) return; + const response = await this.client.closeSession(cachedSessionId, + this.envelopeForGeneration(generation, sessionKey, ctx), { signal }); + if (response?.status !== "noop") { const closeTrigger = normalizeSessionCloseTrigger(ctx.reason); - const events = this.eventsFor(sessionKey, ctx); - // Await so /quit and Ctrl+C can flush before process teardown. - await this.analytics.trackAwait(events.sessionClosed, { - entrypoint: this.entrypointFor(sessionKey, ctx), - session_id_hash: hashId(closedSessionId)!, - status: "closed", - ...(closeTrigger ? { close_trigger: closeTrigger } : {}), + this.analytics.track(this.eventsFor(null, ctx).sessionClosed, { + entrypoint: this.entrypointFor(null, ctx), session_id_hash: hashId(cachedSessionId)!, + status: "closed", ...(closeTrigger ? { close_trigger: closeTrigger } : {}), }); } + } finally { + generation.controller.abort(new Error("memory session closed")); + this.allGenerations.delete(generation); } - this.sessionIdBySessionKey.delete(sessionKey); - this.sessionStateBySessionKey.delete(sessionKey); - this.turnBySessionKey.delete(sessionKey); - this.entrypointBySessionKey.delete(sessionKey); - this.clearMemoryUnavailable(sessionKey); + }; + let closeError: unknown; + this.writes.enqueue(generation, 0, close, (error) => { + closeError = error; + this.reportBackgroundFailure(sessionKey, "session-end", error); + }, true); + const closing = { + promise: this.writes.flush(generation).then(() => { if (closeError) throw closeError; }), + retry: (signal: AbortSignal) => waitForMemory(close, DEFAULT_MEMOS_MEMORY_TIMEOUT_MS, signal), + }; + this.closing.set(sessionKey, closing); + void closing.promise.then(() => { + if (this.closing.get(sessionKey) === closing) this.closing.delete(sessionKey); + }).catch(() => {}); + + } + + async dispose(): Promise { + this.disposed = true; + const generations = [...this.allGenerations]; + await Promise.all([...this.generations.keys()].map((sessionKey) => this.sessionEnd(new AgentHookContext({ sessionKey, reason: "dispose" })))); + try { + await waitForMemory(() => this.flushPendingWrites(), 2_000); } catch (error) { - this.warnMemoryUnavailable(sessionKey, "session-end", error); + console.warn("[memmy-memory] Shutdown drain timed out; queued memory writes may not be stored.", error); + } finally { + for (const generation of generations) generation.controller.abort(new Error("memory hook disposed")); + this.writes.stop(); } } - async dispose(): Promise { - const sessionKeys = [...this.sessionIdBySessionKey.keys()]; - await Promise.allSettled(sessionKeys.map((sessionKey) => this.sessionEnd(new AgentHookContext({ - sessionKey, - reason: "dispose", - metadata: { lifecycle: "session" }, - })))); + private envelopeForGeneration(generation: SessionGeneration, sessionKey: string, ctx: AgentHookContext): MemmyMemoryRequestEnvelope { + return generation.state?.protocol === "v2" + ? this.l3Envelope(sessionKey, generation.state) + : this.legacyRequestEnvelope(sessionKey, ctx); } requestEnvelope(sessionKey?: string | null, ctx?: AgentHookContext | null): MemmyMemoryRequestEnvelope { @@ -538,15 +669,36 @@ export class MemmyMemoryHook extends AgentHook implements MemmyMemoryToolRuntime return resolved; } - private async ensureSession(ctx: AgentHookContext, sessionKey: string): Promise { - const cached = this.sessionIdBySessionKey.get(sessionKey); - if (cached) return cached; + private ensureSession(ctx: AgentHookContext, sessionKey: string, generation = this.generation(sessionKey)): Promise { + if (generation.state) return Promise.resolve(generation.state.memorySessionId); + if (generation.pending) return generation.pending; + const pending = waitForMemory((signal) => this.initializeSession(ctx, sessionKey, generation, signal), + DEFAULT_MEMOS_MEMORY_TIMEOUT_MS, generation.controller.signal); + generation.pending = pending; + void pending.finally(() => { + if (generation.pending === pending) generation.pending = undefined; + }).catch(() => {}); + return pending; + } + + private async initializeSession(ctx: AgentHookContext, sessionKey: string, generation: SessionGeneration, signal: AbortSignal): Promise { + const close = this.closing.get(sessionKey); + if (close && this.generations.get(sessionKey) === generation) { + try { await close.promise; } catch { + signal.throwIfAborted(); + close.promise = close.retry(signal); + await close.promise; + } + if (this.closing.get(sessionKey) === close) this.closing.delete(sessionKey); + } + signal.throwIfAborted(); this.entrypointFor(sessionKey, ctx); const workspacePath = this.workspaceFromContext(ctx); const hostProjectId = this.hostProjectIdFromContext(ctx); const health = typeof (this.client as any).health === "function" - ? await this.client.health().catch(() => null) + ? await this.client.health({ signal }).catch(() => { signal.throwIfAborted(); return null; }) : null; + signal.throwIfAborted(); const supportsV2 = health?.features?.l3WorldModelProtocolVersions?.includes(2) === true; let workspaceRoot: string | null = null; let workspaceUri: MemmyMemorySessionState["workspaceUri"] = null; @@ -563,6 +715,7 @@ export class MemmyMemoryHook extends AgentHook implements MemmyMemoryToolRuntime : this.legacyRequestEnvelope(sessionKey, ctx); // Omit stable sessionId: Memory binds via namespace.sessionKey (host key). // After /new closes the prior session, the next open mints a new sessionId. + signal.throwIfAborted(); const response = await this.client.openSession(compact(supportsV2 ? { ...openEnvelope, l3WorldModelProtocolVersion: 2, @@ -572,16 +725,15 @@ export class MemmyMemoryHook extends AgentHook implements MemmyMemoryToolRuntime } : { ...openEnvelope, workspacePath, - })); + }), { signal }); + signal.throwIfAborted(); const resolved = stringOrUndefined(response?.sessionId); if (!resolved) throw new Error("memmy memory openSession did not return sessionId"); - this.sessionIdBySessionKey.set(sessionKey, resolved); const memoryProjectId = supportsV2 ? stringOrUndefined(response?.projectId) ?? null : null; if (workspaceRoot && !memoryProjectId) { - this.sessionIdBySessionKey.delete(sessionKey); throw new Error("memmy memory project session did not return projectId"); } - this.sessionStateBySessionKey.set(sessionKey, { + generation.state = { hostSessionKey: sessionKey, memorySessionId: resolved, memoryProjectId, @@ -590,9 +742,13 @@ export class MemmyMemoryHook extends AgentHook implements MemmyMemoryToolRuntime workspaceUri, workspaceHostId, l3Cache: emptyL3Cache(resolved, memoryProjectId, "empty", ""), - }); + }; + if (this.generations.get(sessionKey) === generation && !this.disposed) { + this.sessionIdBySessionKey.set(sessionKey, resolved); + this.sessionStateBySessionKey.set(sessionKey, generation.state); + } // Only emit opened for a newly created session; resumed opens are continuations. - if (response?.resumed !== true) { + if (response?.resumed !== true && this.generations.get(sessionKey) === generation) { const events = this.eventsFor(sessionKey, ctx); this.analytics.track(events.sessionOpened, { entrypoint: this.entrypointFor(sessionKey, ctx), @@ -603,11 +759,14 @@ export class MemmyMemoryHook extends AgentHook implements MemmyMemoryToolRuntime return resolved; } - private async loadL3Context(sessionKey: string, state: MemmyMemorySessionState): Promise { + private async loadL3Context(sessionKey: string, state: MemmyMemorySessionState, turn: InteractiveTurn, signal: AbortSignal): Promise { const response = await this.client.l3WorldModelContext( state.memorySessionId, this.l3Envelope(sessionKey, state), + { signal }, ); + signal.throwIfAborted(); + this.assertCurrentTurn(turn); const loadedAt = new Date().toISOString(); const current = state.l3Cache; if ( @@ -714,20 +873,21 @@ export class MemmyMemoryHook extends AgentHook implements MemmyMemoryToolRuntime return stringOrUndefined(ctx?.spec?.sessionKey) ?? stringOrUndefined(ctx?.sessionKey) ?? stringOrUndefined(ctx?.session?.key) ?? null; } - private injectMemoryContext(messages: JsonRecord[], injectedContext: any): void { + private injectMemoryContext(messages: JsonRecord[], injectedContext: any): boolean { const markdown = typeof injectedContext === "string" ? injectedContext : typeof injectedContext?.markdown === "string" ? injectedContext.markdown : ""; - if (!markdown.trim()) return; + if (!markdown.trim()) return false; const memoryBlock = renderMemmyMemoryContext(markdown, "turn_start"); for (let index = messages.length - 1; index >= 0; index -= 1) { const message = messages[index]; if (message?.role !== "user") continue; message.content = injectProtocolContent(message.content, memoryBlock); - return; + return true; } + return false; } private injectMemoryUnavailableNotice(messages: JsonRecord[]): void { @@ -740,6 +900,13 @@ export class MemmyMemoryHook extends AgentHook implements MemmyMemoryToolRuntime } } + private reportBackgroundFailure(sessionKey: string, phase: "write" | "recall" | "session-end", error: unknown): void { + this.analytics.track("memory_background_failed", { + session_key_hash: hashId(sessionKey)!, phase, error_code: errorCodeFromUnknown(error), + }); + this.warnMemoryUnavailable(sessionKey, phase, error); + } + private warnMemoryUnavailable( sessionKey: string, phase: "session-start" | "recall" | "write" | "session-end", @@ -750,8 +917,8 @@ export class MemmyMemoryHook extends AgentHook implements MemmyMemoryToolRuntime this.unavailableWarnedSessionKeys.add(sessionKey); console.warn( `[memmy-memory] Memory service unavailable (session "${sessionKey}", ${phase}): ${this.lastError}. ` + - "Continuing without long-term memory recall/write for this session; further failures for this " + - "session are suppressed until the service recovers.", + "Agent execution continues; background memory writes remain best effort. Further warnings " + + "for this session are suppressed until the service recovers.", ); } diff --git a/App/memmy-agent/src/memmy-memory/lifecycle.ts b/App/memmy-agent/src/memmy-memory/lifecycle.ts new file mode 100644 index 000000000..b430e9c71 --- /dev/null +++ b/App/memmy-agent/src/memmy-memory/lifecycle.ts @@ -0,0 +1,82 @@ +/** A transport may ignore cancellation, so callers must also guard commits. */ +export async function waitForMemory( + operation: (signal: AbortSignal) => Promise, + timeoutMs: number, + signal?: AbortSignal | null, +): Promise { + if (signal?.aborted) throw signal.reason ?? new Error("memory operation cancelled"); + if (timeoutMs <= 0) throw new Error("memory interactive deadline exceeded"); + const controller = new AbortController(); + let timer: ReturnType | undefined; + let onAbort: (() => void) | undefined; + const interrupted = new Promise((_resolve, reject) => { + const stop = (reason: unknown): void => { + controller.abort(reason); + reject(reason); + }; + onAbort = () => stop(signal?.reason ?? new Error("memory operation cancelled")); + signal?.addEventListener("abort", onAbort, { once: true }); + timer = setTimeout( + () => stop(new Error(`memory did not answer within ${Math.ceil(timeoutMs)}ms`)), + timeoutMs, + ); + }); + try { + return await Promise.race([operation(controller.signal), interrupted]); + } finally { + if (timer) clearTimeout(timer); + if (onAbort) signal?.removeEventListener("abort", onAbort); + } +} + +/** Bounded, process-local writes. This is not a durable outbox. */ +export class MemoryWriteQueue { + private readonly tails = new Map>(); + private readonly controller = new AbortController(); + private count = 0; + private bytes = 0; + + enqueue( + key: object, + bytes: number, + operation: (signal: AbortSignal) => Promise, + onError: (error: unknown) => void, + closing = false, + ): boolean { + if ( + this.controller.signal.aborted || + (!closing && (this.count >= 100 || this.bytes + bytes > 10 * 1024 * 1024)) + ) { + onError(new Error("memory background queue capacity exceeded; capture was not stored")); + return false; + } + this.count += 1; + this.bytes += bytes; + const deadline = performance.now() + 120_000; + const previous = this.tails.get(key) ?? Promise.resolve(); + const job = previous + .then(async () => { + await waitForMemory(operation, deadline - performance.now(), this.controller.signal); + }) + .catch(onError) + .finally(() => { + this.count -= 1; + this.bytes -= bytes; + if (this.tails.get(key) === job) this.tails.delete(key); + }); + this.tails.set(key, job); + return true; + } + + has(key: object): boolean { + return this.tails.has(key); + } + + stop(): void { + this.controller.abort(new Error("memory background queue stopped")); + } + + async flush(key?: object): Promise { + await Promise.all(key ? [this.tails.get(key)] : [...this.tails.values()]); + } +} diff --git a/App/memmy-agent/src/templates/agent/tool-contract.md b/App/memmy-agent/src/templates/agent/tool-contract.md index 6eaae19ff..b9c939ad5 100644 --- a/App/memmy-agent/src/templates/agent/tool-contract.md +++ b/App/memmy-agent/src/templates/agent/tool-contract.md @@ -29,7 +29,9 @@ records general tool contracts and less obvious usage patterns. - Use `fixed_strings=true` for literal keywords containing regex characters. - Before reading full matches, use `output_mode="count"` to estimate the scale of broad searches. - Use `head_limit` and `offset` to page through large result sets. -- You may skip binary or very large files to keep results readable. +- For long documents, use `read_file` with `char_offset` and `char_limit`; use `pages` to select PDF pages. Follow the returned continuation offset with the same path and page selection until the required content is read. When the task requires the whole document, continue to the end before claiming it is fully read. +- A truncated excerpt does not establish that a fact is absent or unavailable. Retrieve the missing range before drawing conclusions or asking the user to supply information already in the document. +- If command output is truncated while extracting a document, save the extracted text to a file and read the relevant ranges, or use `read_file` on the original document. Polling a completed command does not recover omitted output. ## File and Coding Workflows diff --git a/App/memmy-agent/src/utils/document.ts b/App/memmy-agent/src/utils/document.ts index 82dde0ddc..c97d54dee 100644 --- a/App/memmy-agent/src/utils/document.ts +++ b/App/memmy-agent/src/utils/document.ts @@ -122,17 +122,17 @@ async function extractDocxOpenXml(file: string): Promise { return texts.join(""); }) .filter((line) => line.trim()); - if (paragraphs.length) return truncate(paragraphs.join("\n\n")); + if (paragraphs.length) return paragraphs.join("\n\n"); const texts: string[] = []; collectXmlText(parsed, new Set(["w:t"]), texts); - return truncate(texts.filter(Boolean).join("\n\n")); + return texts.filter(Boolean).join("\n\n"); } async function extractDocx(file: string): Promise { try { const result = await mammoth.extractRawText({ path: file }); const text = result.value.trim(); - if (text) return truncate(text); + if (text) return text; return extractDocxOpenXml(file); } catch (err) { try { @@ -171,7 +171,7 @@ async function extractXlsxOpenXml(file: string): Promise { } if (rows.length) sheets.push(`--- Sheet: Sheet${index + 1} ---\n${rows.join("\n")}`); } - return truncate(sheets.join("\n\n")); + return sheets.join("\n\n"); } async function extractXlsx(file: string): Promise { @@ -188,7 +188,7 @@ async function extractXlsx(file: string): Promise { }); if (rows.length) sheets.push(`--- Sheet: ${sheet.name} ---\n${rows.join("\n")}`); }); - return truncate(sheets.join("\n\n")); + return sheets.join("\n\n"); } catch (err) { try { return await extractXlsxOpenXml(file); @@ -264,7 +264,7 @@ async function extractPptx(file: string): Promise { const cleaned = texts.map((line) => line.trim()).filter(Boolean); if (cleaned.length) slides.push(`--- Slide ${index + 1} ---\n${cleaned.join("\n")}`); } - return truncate(slides.join("\n\n")); + return slides.join("\n\n"); } catch (err) { return `[error: failed to extract PPTX: ${(err as Error).message}]`; } @@ -292,11 +292,11 @@ async function extractPdf(file: string): Promise { .trim(); pages.push(`--- Page ${i} ---\n${text}`); } - return truncate(pages.join("\n\n")); + return pages.join("\n\n"); } catch (err) { const raw = fs.readFileSync(file); const text = raw.toString("latin1").match(/\(([^()\r\n]{2,})\)/g)?.map((m) => m.slice(1, -1)).join("\n") ?? ""; - if (text) return truncate(`--- Page 1 ---\n${text}`); + if (text) return `--- Page 1 ---\n${text}`; return `[error: failed to extract PDF: ${(err as Error).message}]`; } finally { await (doc as any)?.destroy?.(); @@ -305,13 +305,14 @@ async function extractPdf(file: string): Promise { function extractTextFile(file: string): string { try { - return truncate(fs.readFileSync(file, "utf8")); + return fs.readFileSync(file, "utf8"); } catch { - return truncate(fs.readFileSync(file, "latin1")); + return fs.readFileSync(file, "latin1"); } } -export async function extractText(filePath: string): Promise { +/** Keep extraction lossless so callers can select pages before applying an output budget. */ +async function extractFullText(filePath: string): Promise { const file = String(filePath); if (!fs.existsSync(file)) return `[error: file not found: ${file}]`; const ext = path.extname(file).toLowerCase(); @@ -329,3 +330,13 @@ export async function extractText(filePath: string): Promise { if ([".png", ".jpg", ".jpeg", ".gif", ".webp"].includes(ext)) return `[image: ${path.basename(file)}]`; return null; } + +/** By default return a bounded preview; paginated readers explicitly request the full text. */ +export async function extractText( + filePath: string, + options: { maxChars?: number | null } = {}, +): Promise { + const text = await extractFullText(filePath); + if (text == null || text.startsWith("[error:") || options.maxChars === null) return text; + return truncate(text, options.maxChars ?? MAX_TEXT_LENGTH); +} diff --git a/App/memmy-agent/tests/core/agent-runtime/lifecycle-hooks.test.ts b/App/memmy-agent/tests/core/agent-runtime/lifecycle-hooks.test.ts index 6c8da8d69..97b5800aa 100644 --- a/App/memmy-agent/tests/core/agent-runtime/lifecycle-hooks.test.ts +++ b/App/memmy-agent/tests/core/agent-runtime/lifecycle-hooks.test.ts @@ -114,6 +114,27 @@ describe("lifecycle hooks", () => { }); }); + it.each(["ordinary", "system"])("passes one turn identity and cancellation signal through %s prompt and run", async (kind) => { + const contexts: AgentHookContext[] = []; + class IdentityHook extends AgentHook { + override async beforeBuildSystemPrompt(ctx: AgentHookContext): Promise { contexts.push(ctx); } + override async beforeRun(ctx: AgentHookContext): Promise { contexts.push(ctx); } + } + const loop = makeLoop([new IdentityHook()]); + const abort = new AbortController(); + if (kind === "ordinary") { + await loop.processDirect("hello", { sessionKey: "cli:identity", abortSignal: abort.signal }); + } else { + await loop.processSystemMessage({ channel: "system", chatId: "cli:identity", senderId: "system", content: "hello", metadata: {} } as any, + "cli:identity", { turnId: "identity-turn", abortSignal: abort.signal }); + } + expect(contexts).toHaveLength(2); + expect(contexts[0].spec.turnId).toEqual(expect.any(String)); + expect(contexts[0].spec.turnId).toBe(contexts[1].spec.turnId); + expect(contexts[0].spec.abortSignal).toBe(contexts[1].spec.abortSignal); + expect(contexts[0].spec.abortSignal).toBeInstanceOf(AbortSignal); + }); + it("emits sessionStart once for a newly created session", async () => { const hook = new RecordingLifecycleHook(); const loop = makeLoop([hook]); diff --git a/App/memmy-agent/tests/core/agent-runtime/long-document-review.test.ts b/App/memmy-agent/tests/core/agent-runtime/long-document-review.test.ts new file mode 100644 index 000000000..73fd689ab --- /dev/null +++ b/App/memmy-agent/tests/core/agent-runtime/long-document-review.test.ts @@ -0,0 +1,145 @@ +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { AgentRunner, AgentRunSpec } from "../../../src/core/agent-runtime/runner.js"; +import { ReadFileTool } from "../../../src/core/agent-runtime/tools/filesystem.js"; +import { ExecTool } from "../../../src/core/agent-runtime/tools/shell.js"; +import { ToolRegistry } from "../../../src/core/agent-runtime/tools/registry.js"; +import { ExecSessionManager, WriteStdinTool } from "../../../src/core/agent-runtime/tools/exec-session.js"; +import { SESSION_TOOL_RESULT_MAX_CHARS_BY_NAME } from "../../../src/core/agent-runtime/tool-result-budget.js"; +import { LLMProvider, LLMResponse, ToolCallRequest } from "../../../src/providers/base.js"; +import { renderTemplate } from "../../../src/utils/prompt-templates.js"; +import { REPORT_FACTS, reportText, writeDocx } from "../../helpers/long-document-fixtures.js"; + +const roots: string[] = []; +async function workspace() { + const root = await fs.mkdtemp(path.join(os.tmpdir(), "memmy-document-review-")); + roots.push(root); + return root; +} +afterEach(async () => { + for (const root of roots.splice(0)) await fs.rm(root, { recursive: true, force: true }); +}); + +// Scripted decisions validate runtime/tool plumbing, NOT real-model compliance. +class ScriptedReader extends LLMProvider { + calls = 0; + constructor(private readonly decide: (messages: any[], turn: number) => LLMResponse) { super(); } + getDefaultModel() { return "scripted-review-not-a-real-model"; } + async chat(args: any) { return this.decide(args.messages, this.calls++); } + async chatWithRetry(args: any) { return this.chat(args); } +} +function call(name: string, args: any, turn: number) { + return new LLMResponse({ content: null, toolCalls: [new ToolCallRequest({ id: `review-${turn}`, name, arguments: args })] }); +} +function spec(root: string, tools: ToolRegistry) { + return new AgentRunSpec({ + workspace: root, tools, maxIterations: 40, + toolResultMaxCharsByName: SESSION_TOOL_RESULT_MAX_CHARS_BY_NAME, + initialMessages: [ + { role: "system", content: renderTemplate("agent/tool-contract.md") }, + { role: "user", content: "Read the entire synthetic report and return every VERIFIED figure." }, + ], + }); +} +const quote = (text: string) => `'${text.replace(/'/g, "'\\''")}'`; + +describe("long-document review through the real AgentRunner", () => { + it("delivers continuation cursors and all late facts across multiple tool turns", async () => { + const root = await workspace(); + const file = await writeDocx(root, reportText()); + const tools = new ToolRegistry(); + tools.register(new ReadFileTool({ workspace: root })); + const provider = new ScriptedReader((messages, turn) => { + const outputs = messages.filter((m) => m.role === "tool").map((m) => String(m.content)); + if (!outputs.length) return call("read_file", { path: file }, turn); + const last = outputs.at(-1)!; + const next = /Continue with char_offset=(\d+)/.exec(last); + if (next) return call("read_file", { path: file, char_offset: Number(next[1]) }, turn); + expect(last).toContain("End of document"); + const figures = outputs.join("\n").match(/(?:PE|HOLDING|ISSUED|RAISED)=[0-9.%]+/g); + return new LLMResponse({ content: JSON.stringify(figures) }); + }); + const result = await new AgentRunner(provider).run(spec(root, tools)); + expect(JSON.parse(result.finalContent!)).toEqual(REPORT_FACTS); + expect(result.messages.filter((m) => m.role === "tool").length).toBeGreaterThan(8); + expect(result.toolEvents.every((e) => e.status === "ok")).toBe(true); + }); + + it("documents that a model can still stop early despite a continuation marker", async () => { + const root = await workspace(); + const file = await writeDocx(root, reportText()); + const tools = new ToolRegistry(); + tools.register(new ReadFileTool({ workspace: root })); + const provider = new ScriptedReader((_, turn) => turn === 0 + ? call("read_file", { path: file }, turn) + : new LLMResponse({ content: "Stopped without reading the remainder." })); + const result = await new AgentRunner(provider).run(spec(root, tools)); + const outputs = result.messages.filter((m) => m.role === "tool"); + expect(outputs).toHaveLength(1); + expect(outputs[0].content).toContain("Continue with char_offset="); + expect(result.finalContent).toBe("Stopped without reading the remainder."); + }); + + it("recovers exec's omitted middle by saving output and reading the file in ranges", async () => { + const root = await workspace(); + const input = reportText(60_000); + await fs.writeFile(path.join(root, "source.txt"), input); + await fs.writeFile(path.join(root, "emit.cjs"), 'process.stdout.write(require("node:fs").readFileSync("source.txt", "utf8"));'); + const command = `${quote(process.execPath)} emit.cjs`; + const tools = new ToolRegistry(); + tools.register(new ExecTool({ workspace: root })); + tools.register(new ReadFileTool({ workspace: root })); + const provider = new ScriptedReader((messages, turn) => { + const outputs = messages.filter((m) => m.role === "tool").map((m) => String(m.content)); + if (turn === 0) return call("exec", { command }, turn); + if (turn === 1) { + expect(outputs[0]).toContain("truncated"); + expect(outputs[0]).not.toContain(REPORT_FACTS[1]); + return call("exec", { command: `${command} > extracted.txt` }, turn); + } + if (turn === 2) return call("read_file", { path: "extracted.txt", limit: 150 }, turn); + const next = /Use offset=(\d+) to continue/.exec(outputs.at(-1)!); + if (next) return call("read_file", { path: "extracted.txt", offset: Number(next[1]), limit: 150 }, turn); + expect(outputs.at(-1)).toContain("End of file"); + const figures = outputs.slice(2).join("\n").match(/(?:PE|HOLDING|ISSUED|RAISED)=[0-9.%]+/g); + return new LLMResponse({ content: JSON.stringify(figures) }); + }); + const result = await new AgentRunner(provider).run(spec(root, tools)); + expect(JSON.parse(result.finalContent!)).toEqual(REPORT_FACTS); + expect(await fs.readFile(path.join(root, "extracted.txt"), "utf8")).toBe(input); + }); + + it("confirms completed exec sessions cannot replay omitted output", async () => { + const root = await workspace(); + const source = "a".repeat(30_000) + "MISSING_MIDDLE_FACT" + "z".repeat(30_000); + await fs.writeFile(path.join(root, "source.txt"), source); + await fs.writeFile(path.join(root, "emit.cjs"), 'process.stdout.write(require("node:fs").readFileSync("source.txt", "utf8"));'); + const manager = new ExecSessionManager(); + try { + // The manager returns the id even when the completed tool response omits it. + const [id, first] = await manager.start({ command: `${quote(process.execPath)} emit.cjs`, cwd: root, yieldTimeMs: 1000, maxOutputChars: 1000 }); + expect(first.done).toBe(true); + expect(first.truncatedChars).toBeGreaterThan(0); + expect(first.output).not.toContain("MISSING_MIDDLE_FACT"); + const next = await new WriteStdinTool({ manager }).execute({ session_id: id, yield_time_ms: 0 }); + expect(next).not.toContain("MISSING_MIDDLE_FACT"); + expect(next).not.toContain("a".repeat(20)); + expect(next).toContain("exec session not found"); + } finally { + for (const session of manager.sessions.values()) await session.kill(); + } + }); + + it("exposes the remaining oversized single-line text recovery limitation", async () => { + const root = await workspace(); + const file = path.join(root, "single-line-extraction.txt"); + await fs.writeFile(file, "x".repeat(200_000) + "TAIL"); + const result = await new ReadFileTool({ workspace: root }).execute({ path: file }); + // Characterization of a known gap, not evidence that this recovery route is safe. + expect(result.length).toBeGreaterThan(ReadFileTool.MAX_CHARS); + expect(result).not.toContain("Use offset="); + expect(result).toContain("End of file"); + }); +}); diff --git a/App/memmy-agent/tests/core/agent-runtime/tools/filesystem-tools.test.ts b/App/memmy-agent/tests/core/agent-runtime/tools/filesystem-tools.test.ts index fb57219fa..49db204b4 100644 --- a/App/memmy-agent/tests/core/agent-runtime/tools/filesystem-tools.test.ts +++ b/App/memmy-agent/tests/core/agent-runtime/tools/filesystem-tools.test.ts @@ -97,6 +97,30 @@ describe("ReadFileTool", () => { expect(result).toContain("End of file"); }); + it("does not show a trailing newline as an extra blank line", async () => { + const root = workspace(); + const file = path.join(root, "memory.md"); + await fs.writeFile(file, "only one memory line\n"); + + const result = await new ReadFileTool({ workspace: root }).execute({ path: file }); + + expect(result).toContain("1| only one memory line"); + expect(result).not.toMatch(/^2\|/m); + expect(result).toContain("End of file"); + }); + + it("keeps a real blank line before the final newline", async () => { + const root = workspace(); + const file = path.join(root, "memory.md"); + await fs.writeFile(file, "first\n\n"); + + const result = await new ReadFileTool({ workspace: root }).execute({ path: file }); + + expect(result).toContain("1| first"); + expect(result).toMatch(/^2\| $/m); + expect(result).not.toMatch(/^3\|/m); + }); + it("reports empty files", async () => { const root = workspace(); const file = path.join(root, "empty.txt"); diff --git a/App/memmy-agent/tests/core/agent-runtime/tools/read-enhancements.test.ts b/App/memmy-agent/tests/core/agent-runtime/tools/read-enhancements.test.ts index d44fd4a60..b79290a5e 100644 --- a/App/memmy-agent/tests/core/agent-runtime/tools/read-enhancements.test.ts +++ b/App/memmy-agent/tests/core/agent-runtime/tools/read-enhancements.test.ts @@ -45,7 +45,7 @@ function zipTree(tree: Record, filename: string): string { return out; } -function makePdf(root: string, name: string, parts: string[]): string { +function makePdf(root: string, name: string, parts: string[], fontSize = 24): string { const file = path.join(root, name); const escapePdfText = (text: string) => text.replace(/\\/g, "\\\\").replace(/\(/g, "\\(").replace(/\)/g, "\\)"); const objects: string[] = []; @@ -55,7 +55,7 @@ function makePdf(root: string, name: string, parts: string[]): string { parts.forEach((part, index) => { const pageObj = 3 + index * 2; const contentObj = 4 + index * 2; - const stream = `BT /F1 24 Tf 72 720 Td (${escapePdfText(part)}) Tj ET`; + const stream = `BT /F1 ${fontSize} Tf 72 720 Td (${escapePdfText(part)}) Tj ET`; objects[pageObj] = `<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] ` + `/Resources << /Font << /F1 ${fontObj} 0 R >> >> /Contents ${contentObj} 0 R >>`; @@ -351,14 +351,107 @@ describe("ReadFileTool enhanced behavior", () => { expect(await new ReadFileTool({ workspace: root }).execute({ path: file })).toContain("no extractable text"); }); - it("truncates large extracted documents", async () => { - documentMocks.extractText.mockReturnValueOnce("x".repeat(200_000)); + it.each(["docx", "xlsx", "pptx", "pdf"])("reads every character of a long %s through continuation offsets", async (ext) => { const root = tmpRoot(); - const file = path.join(root, "large.docx"); - fs.writeFileSync(file, "PK", "utf8"); - const result = await new ReadFileTool({ workspace: root }).execute({ path: file }); - expect(result.length).toBeLessThanOrEqual(ReadFileTool.MAX_CHARS + 100); - expect(result).toContain("truncated at ~128K chars"); + const file = path.join(root, `large.${ext}`); + fs.writeFileSync(file, "fixture"); + const source = "前文🙂".repeat(60_000) + "TAIL: verified figure 38.95"; + const tool = new ReadFileTool({ workspace: root }); + let offset = 0; + let combined = ""; + for (let count = 0; count < 30; count += 1) { + documentMocks.extractText.mockResolvedValueOnce(source); + const result = await tool.execute({ path: file, char_offset: offset }); + expect(result.length).toBeLessThanOrEqual(ReadFileTool.MAX_CHARS); + const separator = result.lastIndexOf("\n\n[Document read:"); + expect(separator).toBeGreaterThanOrEqual(0); + const body = result.slice(0, separator); + expect(body).not.toMatch(/[\uD800-\uDBFF]$/); + combined += body; + const next = /Continue with char_offset=(\d+)/.exec(result); + if (!next) { + expect(result).toContain("End of document"); + break; + } + expect(Number(next[1])).toBeGreaterThan(offset); + offset = Number(next[1]); + } + expect(combined).toBe(source); + expect(documentMocks.extractText).toHaveBeenLastCalledWith(file, { maxChars: null }); + }); + + it("reads a real DOCX tail beyond both former truncation limits", async () => { + const text = "正文段落 ".repeat(50_000) + "FINAL_VERIFIED_NUMBER_38.95"; + const file = zipTree( + { "word/document.xml": `${text}` }, + "long.docx", + ); + const tool = new ReadFileTool({ workspace: path.dirname(file) }); + let offset = 0; + let combined = ""; + for (let count = 0; count < 10; count += 1) { + const result = await tool.execute({ path: file, char_offset: offset, char_limit: 1_000_000 }); + expect(result.length).toBeLessThanOrEqual(ReadFileTool.MAX_CHARS); + combined += result.slice(0, result.lastIndexOf("\n\n[Document read:")); + const next = /Continue with char_offset=(\d+)/.exec(result); + if (!next) { + expect(result).toContain("End of document"); + break; + } + offset = Number(next[1]); + } + expect(combined === text).toBe(true); + }); + + it("reads a real PDF page after more than 200K extracted characters", async () => { + const root = tmpRoot(); + // Keep every glyph inside the page bounds; PDF.js omits off-page text. + const file = makePdf(root, "long.pdf", [...Array.from({ length: 14 }, () => "a".repeat(16_000)), "PAGE_15_VERIFIED_NUMBER"], 0.02); + const extracted = await documentMocks.extractText(file, { maxChars: null }); + expect(extracted.length).toBeGreaterThan(200_000); + const tool = new ReadFileTool({ workspace: root }); + const result = await tool.execute({ path: file, pages: "15" }); + expect(result).toContain("--- Page 15 ---"); + expect(result).toContain("PAGE_15_VERIFIED_NUMBER"); + expect(result).toContain("End of document"); + }); + + it("makes progress when a one-unit chunk encounters an emoji", async () => { + const root = tmpRoot(); + const file = path.join(root, "emoji.docx"); + fs.writeFileSync(file, "fixture"); + documentMocks.extractText.mockResolvedValueOnce("🙂tail"); + const result = await new ReadFileTool({ workspace: root }).execute({ path: file, char_limit: 1 }); + expect(result).toMatch(/^🙂\n\n/); + expect(result).toContain("Continue with char_offset=2"); + }); + + it("keeps PDF page selection when continuing within a long page", async () => { + const root = tmpRoot(); + const file = path.join(root, "pages.pdf"); + fs.writeFileSync(file, "fixture"); + const source = "--- Page 1 ---\n" + "a".repeat(210_000) + "\n\n--- Page 2 ---\n" + "b".repeat(70_000) + "LATE_FIGURE"; + const tool = new ReadFileTool({ workspace: root }); + documentMocks.extractText.mockResolvedValueOnce(source); + const first = await tool.execute({ path: file, pages: "2", char_limit: 1000 }); + expect(first).toContain("--- Page 2 ---"); + expect(first).toContain("Continue with char_offset=1000"); + expect(first).toContain("Keep the same path and pages"); + documentMocks.extractText.mockResolvedValueOnce(source); + const last = await tool.execute({ path: file, pages: "2", char_offset: 70_000 }); + expect(last).toContain("LATE_FIGURE"); + expect(last).toContain("End of document"); + }); + + it("rejects invalid or out-of-range document cursors", async () => { + const root = tmpRoot(); + const file = path.join(root, "small.docx"); + fs.writeFileSync(file, "fixture"); + const tool = new ReadFileTool({ workspace: root }); + for (const options of [{ char_offset: -1 }, { char_offset: 0.5 }, { char_offset: 20 }, { char_limit: 0 }]) { + documentMocks.extractText.mockResolvedValueOnce("short"); + expect(await tool.execute({ path: file, ...options })).toContain("Error"); + } }); it("does not truncate small extracted documents", async () => { diff --git a/App/memmy-agent/tests/document-parsing.test.ts b/App/memmy-agent/tests/document-parsing.test.ts index 2afa5848d..1ca876b5b 100644 --- a/App/memmy-agent/tests/document-parsing.test.ts +++ b/App/memmy-agent/tests/document-parsing.test.ts @@ -91,6 +91,16 @@ describe("document parsing", () => { expect(await extractText(file)).toBe("Hello, world!\nThis is a test."); }); + it("can extract a complete document while retaining the default preview limit", async () => { + const text = "long paragraph ".repeat(20_000) + "TAIL_AFTER_200K"; + const docx = zipTree( + { "word/document.xml": `${text}` }, + "long.docx", + ); + expect(await extractText(docx)).not.toContain("TAIL_AFTER_200K"); + expect(await extractText(docx, { maxChars: null })).toBe(text); + }); + it("truncates large text files", async () => { const root = tempRoot(); const file = path.join(root, "large.txt"); diff --git a/App/memmy-agent/tests/helpers/long-document-fixtures.ts b/App/memmy-agent/tests/helpers/long-document-fixtures.ts new file mode 100644 index 000000000..e56acfc13 --- /dev/null +++ b/App/memmy-agent/tests/helpers/long-document-fixtures.ts @@ -0,0 +1,52 @@ +import fs from "node:fs/promises"; +import path from "node:path"; +import JSZip from "jszip"; + +export const REPORT_FACTS = ["PE=38.95", "HOLDING=61.35%", "ISSUED=112000000", "RAISED=2110000000"]; + +export function reportText(chars = 260_000): string { + const section = "Synthetic company disclosure. Figures in this fixture are not investment advice.\n"; + const padding = section.repeat(Math.ceil(chars / 4 / section.length)); + return REPORT_FACTS.map((fact, index) => `Section ${index + 1}\n${padding}\nVERIFIED: ${fact}\n`).join("\n"); +} + +export async function writeDocx(root: string, text: string): Promise { + const zip = new JSZip(); + zip.file("[Content_Types].xml", ''); + zip.file("_rels/.rels", ''); + const escape = (value: string) => value.replace(/&/g, "&").replace(//g, ">"); + const body = text.split("\n").map((line) => `${escape(line)}`).join(""); + zip.file("word/document.xml", `${body}`); + const file = path.join(root, "synthetic-report.docx"); + await fs.writeFile(file, await zip.generateAsync({ type: "nodebuffer", compression: "DEFLATE" })); + return file; +} + +export async function writePdf(root: string, pageCount: number): Promise { + const objects: string[] = []; + const font = 3 + pageCount * 2; + objects[1] = "<< /Type /Catalog /Pages 2 0 R >>"; + objects[2] = `<< /Type /Pages /Kids [${Array.from({ length: pageCount }, (_, i) => `${3 + i * 2} 0 R`).join(" ")}] /Count ${pageCount} >>`; + for (let i = 0; i < pageCount; i += 1) { + // 120 visible lines at 4pt: ~15K extracted chars/page, all within page bounds. + const lines = Array.from({ length: 120 }, (_, row) => `Page ${i + 1} row ${row + 1} synthetic report ` + "data ".repeat(19)); + if (i === pageCount - 1) lines[119] = "VERIFIED: " + REPORT_FACTS.join("; "); + const stream = `BT /F1 4 Tf 5 TL 36 760 Td ${lines.map((line, row) => `${row ? "T* " : ""}(${line}) Tj`).join("\n")} ET`; + objects[3 + i * 2] = `<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Resources << /Font << /F1 ${font} 0 R >> >> /Contents ${4 + i * 2} 0 R >>`; + objects[4 + i * 2] = `<< /Length ${Buffer.byteLength(stream)} >>\nstream\n${stream}\nendstream`; + } + objects[font] = "<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>"; + let body = "%PDF-1.4\n"; + const offsets = [0]; + for (let i = 1; i < objects.length; i += 1) { + offsets[i] = Buffer.byteLength(body); + body += `${i} 0 obj\n${objects[i]}\nendobj\n`; + } + const start = Buffer.byteLength(body); + body += `xref\n0 ${objects.length}\n0000000000 65535 f \n`; + for (let i = 1; i < objects.length; i += 1) body += `${String(offsets[i]).padStart(10, "0")} 00000 n \n`; + body += `trailer\n<< /Root 1 0 R /Size ${objects.length} >>\nstartxref\n${start}\n%%EOF\n`; + const file = path.join(root, "synthetic-report.pdf"); + await fs.writeFile(file, body); + return file; +} diff --git a/App/memmy-agent/tests/memmy-memory/client-tools.test.ts b/App/memmy-agent/tests/memmy-memory/client-tools.test.ts index 53a39ea76..c967214d7 100644 --- a/App/memmy-agent/tests/memmy-memory/client-tools.test.ts +++ b/App/memmy-agent/tests/memmy-memory/client-tools.test.ts @@ -75,6 +75,19 @@ describe("MemmyMemoryClient", () => { expect(JSON.parse(String(calls[0].init.body))).toEqual({ requestId: "req-1", sessionId: "s1" }); }); + it("propagates caller cancellation to an in-flight memory request", async () => { + const abort = new AbortController(); + const transport = vi.fn((_url: any, init: any) => new Promise((_resolve, reject) => { + init.signal.addEventListener("abort", () => reject(init.signal.reason), { once: true }); + })); + const client = new MemmyMemoryClient({ baseUrl: "http://memory.test" }, transport as any); + const result = client.startTurn("turn", { sessionId: "session", query: "hello" }, { signal: abort.signal }); + const rejection = expect(result).rejects.toThrow("cancel this turn"); + abort.abort(new Error("cancel this turn")); + await rejection; + expect(transport.mock.calls[0][1].signal.aborted).toBe(true); + }); + it("throws structured HTTP errors", async () => { const client = new MemmyMemoryClient({ baseUrl: "http://memory.test", timeoutMs: 1000 }, vi.fn(async () => response({ error: { message: "bad token" } }, 401)) as any); diff --git a/App/memmy-agent/tests/memmy-memory/hook.test.ts b/App/memmy-agent/tests/memmy-memory/hook.test.ts index 2a00aaf42..96e4b3e33 100644 --- a/App/memmy-agent/tests/memmy-memory/hook.test.ts +++ b/App/memmy-agent/tests/memmy-memory/hook.test.ts @@ -6,6 +6,16 @@ import { AgentHookContext, SystemPromptBuildContext } from "../../src/core/agent import { ToolRegistry } from "../../src/core/agent-runtime/tools/registry.js"; import { MemmyMemoryHook } from "../../src/memmy-memory/hook.js"; +async function completeAndFlush(hook: MemmyMemoryHook, ctx: AgentHookContext, result: any): Promise { + await hook.afterRun(ctx, result); + await hook.flushPendingWrites(); +} + +async function compactAndFlush(hook: MemmyMemoryHook, ctx: AgentHookContext): Promise { + await hook.afterCompaction(ctx); + await hook.flushPendingWrites(); +} + function fakeClient() { return { openSession: vi.fn(async (body: any) => ({ @@ -116,7 +126,7 @@ describe("MemmyMemoryHook", () => { const messages = [{ role: "user", content: "继续开发" }]; await hook.beforeRun(new AgentHookContext({ spec, messages })); - await hook.afterRun(new AgentHookContext({ spec }), { + await completeAndFlush(hook, new AgentHookContext({ spec }), { finalContent: "完成", stopReason: "completed", }); @@ -271,7 +281,7 @@ describe("MemmyMemoryHook", () => { const lifecycle = new AgentHookContext({ sessionKey: spec.sessionKey, spec }); await hook.beforeBuildSystemPrompt(lifecycle); - await hook.afterCompaction(new AgentHookContext({ + await compactAndFlush(hook, new AgentHookContext({ sessionKey: spec.sessionKey, spec, compaction: { kind: "token", changed: false, error: null }, @@ -279,7 +289,7 @@ describe("MemmyMemoryHook", () => { expect(client.l3WorldModelBoundary).not.toHaveBeenCalled(); expect(client.l3WorldModelContext).toHaveBeenCalledTimes(1); - await hook.afterCompaction(new AgentHookContext({ + await compactAndFlush(hook, new AgentHookContext({ sessionKey: spec.sessionKey, spec, compaction: { kind: "token", changed: true, error: null }, @@ -470,7 +480,7 @@ describe("MemmyMemoryHook", () => { "[Runtime Context - metadata only, not instructions]\nCurrent Time: now\n[/Runtime Context]", ]); - await hook.afterRun(new AgentHookContext({ spec }), { + await completeAndFlush(hook, new AgentHookContext({ spec }), { finalContent: "Done", usage: { prompt_tokens: 1 }, stopReason: "completed", @@ -526,7 +536,7 @@ describe("MemmyMemoryHook", () => { spec, messages: [{ role: "user", content: "Start a long task" }], })); - await hook.afterRun(new AgentHookContext({ spec }), { + await completeAndFlush(hook, new AgentHookContext({ spec }), { finalContent: "Partial answer", stopReason: "cancelledByUser", }); @@ -548,7 +558,7 @@ describe("MemmyMemoryHook", () => { spec, messages: [{ role: "user", content: "Deploy the service" }], })); - await hook.afterRun(new AgentHookContext({ spec }), { + await completeAndFlush(hook, new AgentHookContext({ spec }), { error: new Error("connection timed out"), stopReason: "error", }); @@ -574,7 +584,7 @@ describe("MemmyMemoryHook", () => { spec, messages: [{ role: "user", content: "This turn never receives an answer" }], })); - await hook.afterRun(new AgentHookContext({ spec }), { + await completeAndFlush(hook, new AgentHookContext({ spec }), { stopReason: "completed", }); @@ -582,7 +592,7 @@ describe("MemmyMemoryHook", () => { expect(hook.currentTurnId("cli:incomplete")).toBeNull(); }); - it("retains the pending turn after a network failure and retries with the same request id", async () => { + it("retries background capture with the same request id and detaches the active turn", async () => { const client = fakeClient(); (client.completeTurn as any) .mockRejectedValueOnce(new Error("network unavailable")) @@ -599,11 +609,10 @@ describe("MemmyMemoryHook", () => { messages: [{ role: "user", content: "Retry this capture" }], })); const result = { finalContent: "Completed once", stopReason: "completed" }; - await hook.afterRun(new AgentHookContext({ spec }), result); - expect(hook.lastError).toBe("network unavailable"); - expect(hook.currentTurnId("cli:retry")).not.toBeNull(); + await completeAndFlush(hook, new AgentHookContext({ spec }), result); + expect(hook.currentTurnId("cli:retry")).toBeNull(); - await hook.afterRun(new AgentHookContext({ spec }), result); + await completeAndFlush(hook, new AgentHookContext({ spec }), result); expect(client.completeTurn).toHaveBeenCalledTimes(2); expect((client.completeTurn as any).mock.calls[0][1].requestId).toBe( @@ -726,7 +735,7 @@ describe("MemmyMemoryHook", () => { spec, messages: [{ role: "user", content: "Current task" }], })); - await hook.afterRun(new AgentHookContext({ spec }), { + await completeAndFlush(hook, new AgentHookContext({ spec }), { finalContent: "Done with the current task.", messages: [{ role: "tool", @@ -760,7 +769,7 @@ describe("MemmyMemoryHook", () => { const messages = [{ role: "user", content: "Inspect the image" }]; await hook.beforeRun(new AgentHookContext({ spec, messages })); - await hook.afterRun(new AgentHookContext({ spec }), { + await completeAndFlush(hook, new AgentHookContext({ spec }), { finalContent: "Done", messages: [{ role: "tool", @@ -805,7 +814,7 @@ describe("MemmyMemoryHook", () => { ]; await hook.beforeRun(new AgentHookContext({ spec, messages })); - await hook.afterRun(new AgentHookContext({ spec }), { + await completeAndFlush(hook, new AgentHookContext({ spec }), { finalContent: "This machine has 10 CPUs.", messages: [ ...messages, @@ -865,7 +874,7 @@ describe("MemmyMemoryHook", () => { ]; await hook.beforeRun(new AgentHookContext({ spec, messages })); - await hook.afterRun(new AgentHookContext({ spec }), { + await completeAndFlush(hook, new AgentHookContext({ spec }), { finalContent: "Implemented and verified the next stage.", messages: [ ...messages, @@ -949,8 +958,9 @@ describe("MemmyMemoryHook", () => { }), ); await hook.sessionEnd(base); + await hook.flushPendingWrites(); - expect(client.closeSession).toHaveBeenCalledWith("session-generated-1", expect.any(Object)); + expect(client.closeSession).toHaveBeenCalledWith("session-generated-1", expect.any(Object), expect.objectContaining({ signal: expect.any(AbortSignal) })); }); it("skips Memory close when this hook never opened a session", async () => { @@ -1005,7 +1015,7 @@ describe("MemmyMemoryHook", () => { const spec = { sessionKey: "cli:direct", workspace: "/tmp/workspace", contextWindowTokens: 4096 }; await hook.beforeRun(new AgentHookContext({ spec, messages: [{ role: "user", content: "hi" }] })); - await hook.afterRun(new AgentHookContext({ spec }), { finalContent: "Done", stopReason: "completed" }); + await completeAndFlush(hook, new AgentHookContext({ spec }), { finalContent: "Done", stopReason: "completed" }); expect(client.completeTurn).not.toHaveBeenCalled(); vi.restoreAllMocks(); diff --git a/App/memmy-agent/tests/memmy-memory/interactive-deadline.test.ts b/App/memmy-agent/tests/memmy-memory/interactive-deadline.test.ts index a79cc68d2..1953eb1bd 100644 --- a/App/memmy-agent/tests/memmy-memory/interactive-deadline.test.ts +++ b/App/memmy-agent/tests/memmy-memory/interactive-deadline.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it, vi } from "vitest"; import { INTERACTIVE_MEMORY_TIMEOUT_MS } from "../../src/memmy-memory/client.js"; -import { MemmyMemoryHook } from "../../src/memmy-memory/hook.js"; +import { waitForMemory } from "../../src/memmy-memory/lifecycle.js"; describe("interactive memory deadline", () => { it("stays well inside the window the desktop client allows a send", () => { @@ -12,9 +12,8 @@ describe("interactive memory deadline", () => { it("gives up on a hanging memory service instead of holding the reply", async () => { vi.useFakeTimers(); try { - const hook = Object.create(MemmyMemoryHook.prototype) as any; const never = new Promise(() => {}); - const raced = hook.withInteractiveDeadline(() => never); + const raced = waitForMemory(() => never, INTERACTIVE_MEMORY_TIMEOUT_MS); const settled = raced.then(() => "resolved", (error: Error) => error.message); await vi.advanceTimersByTimeAsync(INTERACTIVE_MEMORY_TIMEOUT_MS + 1); @@ -26,14 +25,12 @@ describe("interactive memory deadline", () => { }); it("returns the value untouched when memory answers in time", async () => { - const hook = Object.create(MemmyMemoryHook.prototype) as any; - await expect(hook.withInteractiveDeadline(async () => "session-1")).resolves.toBe("session-1"); + await expect(waitForMemory(async () => "session-1", INTERACTIVE_MEMORY_TIMEOUT_MS)).resolves.toBe("session-1"); }); it("propagates a real failure rather than masking it as a timeout", async () => { - const hook = Object.create(MemmyMemoryHook.prototype) as any; - await expect(hook.withInteractiveDeadline(async () => { + await expect(waitForMemory(async () => { throw new Error("memory refused the connection"); - })).rejects.toThrow("memory refused the connection"); + }, INTERACTIVE_MEMORY_TIMEOUT_MS)).rejects.toThrow("memory refused the connection"); }); }); diff --git a/App/memmy-agent/tests/memmy-memory/runner-deadline.test.ts b/App/memmy-agent/tests/memmy-memory/runner-deadline.test.ts new file mode 100644 index 000000000..c330c3e6e --- /dev/null +++ b/App/memmy-agent/tests/memmy-memory/runner-deadline.test.ts @@ -0,0 +1,328 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { AgentHookContext, SystemPromptBuildContext } from "../../src/core/agent-runtime/hook.js"; +import { AgentRunner, AgentRunSpec } from "../../src/core/agent-runtime/runner.js"; +import { LLMResponse } from "../../src/providers/base.js"; +import { MemmyMemoryHook } from "../../src/memmy-memory/hook.js"; +import { MemoryWriteQueue } from "../../src/memmy-memory/lifecycle.js"; + +vi.mock("../../src/analytics/memory-lifecycle-analytics.js", async (original) => ({ + ...(await original()), + createMemoryLifecycleAnalytics: () => ({ + track: vi.fn(), + trackAwait: vi.fn(async () => {}), + flush: vi.fn(async () => {}), + }), +})); +function deferred() { + let resolve!: (value: T) => void; + const promise = new Promise((done) => { + resolve = done; + }); + return { promise, resolve }; +} +const health = { features: { l3WorldModelProtocolVersions: [2] } }; +const opened = { sessionId: "session-1", projectId: null, resumed: true }; +const l3 = { + sessionId: "session-1", + projectId: null, + memoryId: null, + memoryVersion: null, + renderedContext: "", + sourceMemoryIds: [], +}; +const recalled = { sourceMemoryIds: [], injectedContext: null }; +function fixture() { + const client = { + health: vi.fn(async () => health), + openSession: vi.fn(async () => opened), + l3WorldModelContext: vi.fn(async () => l3), + startTurn: vi.fn(async () => recalled), + completeTurn: vi.fn(async () => ({ rawTurnId: "raw-1", l1MemoryId: "l1-1" })), + closeSession: vi.fn(async () => ({ status: "closed" })), + l3WorldModelTraceHead: vi.fn(async () => ({ throughL1MemoryId: "l1-1" })), + l3WorldModelBoundary: vi.fn(async () => ({})), + }; + const hook = new MemmyMemoryHook(client as any); + const model = vi.fn(async () => new LLMResponse({ content: "回答成功" })); + const runner = new AgentRunner({ chatWithRetry: model } as any); + const spec = new AgentRunSpec({ + sessionKey: "cli:deadline", + turnId: "turn-1", + hook, + messages: [{ role: "user", content: "Inspect the pending memory operation" }], + maxIterations: 1, + }); + return { + client, + hook, + runner, + model, + spec, + ctx: new AgentHookContext({ spec, sessionKey: spec.sessionKey }), + }; +} +beforeEach(() => { + vi.useFakeTimers(); + vi.spyOn(console, "warn").mockImplementation(() => {}); +}); +afterEach(() => { + vi.useRealTimers(); + vi.restoreAllMocks(); +}); + +describe("memory deadlines on the real AgentRunner", () => { + it.each(["health", "openSession"] as const)( + "bounds %s across sessionStart, prompt and Runner, then captures after recovery", + async (stage) => { + const f = fixture(); + const pending = deferred(); + f.client[stage].mockImplementation(() => pending.promise); + await f.hook.sessionStart(f.ctx); + const prep = f.hook.beforeBuildSystemPrompt(f.ctx); + await vi.advanceTimersByTimeAsync(8_000); + await prep; + expect((await f.runner.run(f.spec)).finalContent).toBe("回答成功"); + expect(f.model).toHaveBeenCalledTimes(1); + expect(f.client[stage]).toHaveBeenCalledTimes(1); + expect(f.client.startTurn).not.toHaveBeenCalled(); + expect(f.client.completeTurn).not.toHaveBeenCalled(); + pending.resolve(stage === "health" ? health : opened); + await f.hook.flushPendingWrites(); + expect(f.client[stage]).toHaveBeenCalledTimes(1); + expect(f.client.completeTurn).toHaveBeenCalledTimes(1); + expect((f.client.completeTurn.mock.calls as any)[0][1]).toMatchObject({ + recallApplied: false, + sourceMemoryIds: [], + sessionId: "session-1", + answer: "回答成功", + }); + expect(f.client.l3WorldModelContext).not.toHaveBeenCalled(); + }, + ); + it("shares the remaining two seconds after six seconds of initialization", async () => { + const f = fixture(); + const ready = deferred(); + const recall = deferred(); + f.client.health.mockImplementation(() => ready.promise); + f.client.startTurn.mockImplementation(() => recall.promise); + const prep = f.hook.beforeBuildSystemPrompt(f.ctx); + await vi.advanceTimersByTimeAsync(6_000); + ready.resolve(health); + await prep; + const run = f.runner.run(f.spec); + await vi.advanceTimersByTimeAsync(1_999); + expect(f.model).not.toHaveBeenCalled(); + await vi.advanceTimersByTimeAsync(1); + expect((await run).finalContent).toBe("回答成功"); + const input = JSON.stringify(f.model.mock.calls); + recall.resolve({ injectedContext: "late context", sourceMemoryIds: ["late-memory"] }); + await vi.advanceTimersByTimeAsync(0); + expect(JSON.stringify(f.model.mock.calls)).toBe(input); + await f.hook.flushPendingWrites(); + expect((f.client.completeTurn.mock.calls as any)[0][1].recallApplied).toBe(false); + }); + it.each(["startTurn", "l3WorldModelContext"] as const)( + "ignores late %s results after the budget expires", + async (stage) => { + const f = fixture(); + const pending = deferred(); + f.client[stage].mockImplementation(() => pending.promise); + const prep = f.hook.beforeBuildSystemPrompt(f.ctx); + if (stage === "l3WorldModelContext") await vi.advanceTimersByTimeAsync(8_000); + await prep; + const run = f.runner.run(f.spec); + if (stage === "startTurn") await vi.advanceTimersByTimeAsync(8_000); + const result = await run; + const before = JSON.stringify(result.messages); + pending.resolve({ + ...l3, + memoryId: "late", + renderedContext: "late world model", + injectedContext: "late recall", + sourceMemoryIds: ["late"], + }); + await vi.advanceTimersByTimeAsync(0); + expect(JSON.stringify(result.messages)).toBe(before); + const prompt = new SystemPromptBuildContext({ sessionKey: f.spec.sessionKey }); + f.hook.onBuildSystemPrompt(prompt); + expect(prompt.getSection("memmy-l3-world-model")).toBeNull(); + await f.hook.flushPendingWrites(); + }, + ); + it("cancels beforeRun promptly but lets shared initialization warm the session", async () => { + const f = fixture(); + const ready = deferred(); + f.client.health.mockImplementation(() => ready.promise); + const abort = new AbortController(); + f.spec.abortSignal = abort.signal; + const run = f.runner.run(f.spec); + await vi.advanceTimersByTimeAsync(0); + abort.abort(); + expect((await run).stopReason).toBe("cancelled"); + expect(f.model).not.toHaveBeenCalled(); + expect(f.client.completeTurn).not.toHaveBeenCalled(); + ready.resolve(health); + await vi.advanceTimersByTimeAsync(0); + expect(f.hook.currentSessionId(f.spec.sessionKey)).toBe("session-1"); + expect(f.client.startTurn).not.toHaveBeenCalled(); + }); + it("isolates a cancelled first recall from the next turn", async () => { + const f = fixture(); + const pending = deferred(); + f.client.startTurn.mockImplementationOnce(() => pending.promise); + const abort = new AbortController(); + f.spec.abortSignal = abort.signal; + const first = f.runner.run(f.spec); + await vi.advanceTimersByTimeAsync(0); + abort.abort(); + await first; + const second = await f.runner.run( + new AgentRunSpec({ + ...f.spec, + turnId: "turn-2", + abortSignal: null, + messages: [{ role: "user", content: "Second request" }], + }), + ); + pending.resolve({ injectedContext: "first-turn-only", sourceMemoryIds: ["first"] }); + await vi.advanceTimersByTimeAsync(0); + expect(second.finalContent).toBe("回答成功"); + expect(JSON.stringify(second.messages)).not.toContain("first-turn-only"); + await f.hook.flushPendingWrites(); + expect(f.client.completeTurn).toHaveBeenCalledTimes(1); + expect((f.client.completeTurn.mock.calls as any)[0][0]).toBe("turn-2"); + }); + it("does not let slow completion block answers or overwrite the next turn", async () => { + const f = fixture(); + const pending = deferred(); + f.client.completeTurn.mockImplementationOnce(() => pending.promise); + const first = await f.runner.run(f.spec); + first.messages.length = 0; + const second = await f.runner.run( + new AgentRunSpec({ + ...f.spec, + turnId: "turn-2", + messages: [{ role: "user", content: "Second request" }], + }), + ); + expect(second.finalContent).toBe("回答成功"); + expect(f.model).toHaveBeenCalledTimes(2); + expect(f.client.completeTurn).toHaveBeenCalledTimes(1); + pending.resolve({ rawTurnId: "raw-1" }); + await f.hook.flushPendingWrites(); + const calls = f.client.completeTurn.mock.calls as any; + expect(calls.map((call: any) => call[0])).toEqual(["turn-1", "turn-2"]); + expect(calls[0][1].query).toBe("Inspect the pending memory operation"); + expect(calls[1][1].query).toBe("Second request"); + }); + it("closes a late initialization before opening a replacement session", async () => { + const f = fixture(); + const pending = deferred(); + const operations: string[] = []; + f.client.openSession + .mockImplementationOnce(() => { + operations.push("open-old"); + return pending.promise; + }) + .mockImplementationOnce(async () => { + operations.push("open-new"); + return { ...opened, sessionId: "session-2" }; + }); + f.client.closeSession.mockImplementation(async () => { + operations.push("close-old"); + return { status: "closed" }; + }); + await f.hook.sessionStart(f.ctx); + await vi.advanceTimersByTimeAsync(0); + await f.hook.sessionEnd(f.ctx); + expect(f.hook.currentSessionId(f.spec.sessionKey)).toBeNull(); + await f.hook.sessionStart(f.ctx); + await vi.advanceTimersByTimeAsync(0); + expect(f.client.openSession).toHaveBeenCalledTimes(1); + pending.resolve(opened); + await f.hook.flushPendingWrites(); + await vi.advanceTimersByTimeAsync(0); + expect(operations).toEqual(["open-old", "close-old", "open-new"]); + expect(f.hook.currentSessionId(f.spec.sessionKey)).toBe("session-2"); + }); + it("retries a failed close before allowing a replacement session", async () => { + const f = fixture(); + await f.hook.beforeBuildSystemPrompt(f.ctx); + f.client.closeSession.mockRejectedValueOnce(new Error("close temporarily unavailable")); + await f.hook.sessionEnd(f.ctx); + await f.hook.flushPendingWrites(); + f.client.openSession.mockResolvedValue({ ...opened, sessionId: "session-2" }); + await f.runner.run(new AgentRunSpec({ ...f.spec, turnId: "turn-2" })); + await f.hook.flushPendingWrites(); + expect(f.client.closeSession).toHaveBeenCalledTimes(2); + expect(f.client.openSession).toHaveBeenCalledTimes(2); + expect(f.hook.currentSessionId(f.spec.sessionKey)).toBe("session-2"); + }); + it("does not attribute source IDs when the response contains no injected context", async () => { + const f = fixture(); + f.client.startTurn.mockResolvedValue({ + sourceMemoryIds: ["not-delivered"], + injectedContext: null, + } as any); + await f.runner.run(f.spec); + await f.hook.flushPendingWrites(); + expect((f.client.completeTurn.mock.calls as any)[0][1]).toMatchObject({ + recallApplied: false, + sourceMemoryIds: [], + }); + }); + it("recovers from failed initialization on the next turn", async () => { + const f = fixture(); + f.client.openSession.mockRejectedValueOnce(new Error("offline")); + await f.hook.beforeRun(f.ctx); + expect(f.client.startTurn).not.toHaveBeenCalled(); + await f.runner.run(new AgentRunSpec({ ...f.spec, turnId: "turn-2" })); + expect(f.client.startTurn).toHaveBeenCalledTimes(1); + await f.hook.flushPendingWrites(); + }); + it("does not block the foreground on a compaction boundary", async () => { + const f = fixture(); + const pending = deferred(); + await f.hook.beforeBuildSystemPrompt(f.ctx); + f.client.l3WorldModelTraceHead.mockImplementation(() => pending.promise); + await f.hook.afterCompaction( + new AgentHookContext({ spec: f.spec, compaction: { kind: "token", changed: true } }), + ); + expect((await f.runner.run(f.spec)).finalContent).toBe("回答成功"); + pending.resolve({ throughL1MemoryId: "l1-1" }); + await f.hook.flushPendingWrites(); + expect(f.client.l3WorldModelBoundary).toHaveBeenCalledTimes(1); + expect(f.client.completeTurn).toHaveBeenCalledTimes(1); + }); + it("bounds shutdown and rejects late initialization after disposal", async () => { + const f = fixture(); + const pending = deferred(); + f.client.health.mockImplementation(() => pending.promise); + await f.hook.sessionStart(f.ctx); + const disposal = f.hook.dispose(); + await vi.advanceTimersByTimeAsync(2_000); + await disposal; + pending.resolve(health); + await vi.advanceTimersByTimeAsync(0); + expect(f.client.openSession).not.toHaveBeenCalled(); + expect(f.hook.currentSessionId(f.spec.sessionKey)).toBeNull(); + expect(vi.getTimerCount()).toBe(0); + }); +}); +describe("bounded memory capture queue", () => { + it("limits payloads and expires hung queued tasks with a reported failure", async () => { + const queue = new MemoryWriteQueue(); + const error = vi.fn(); + const pending = deferred(); + expect(queue.enqueue({}, 11 * 1024 * 1024, async () => {}, error)).toBe(false); + const key = {}; + for (let index = 0; index < 100; index += 1) + expect(queue.enqueue(key, 1, () => pending.promise, error)).toBe(true); + expect(queue.enqueue(key, 1, async () => {}, error)).toBe(false); + await vi.advanceTimersByTimeAsync(120_000); + await queue.flush(); + expect(error).toHaveBeenCalledTimes(102); + expect(queue.has(key)).toBe(false); + pending.resolve(undefined); + }); +}); diff --git a/App/shell/desktop/package.json b/App/shell/desktop/package.json index c1eaaa6e4..27e6a9600 100644 --- a/App/shell/desktop/package.json +++ b/App/shell/desktop/package.json @@ -1,6 +1,6 @@ { "name": "@memmy/desktop", - "version": "1.2.1", + "version": "1.2.2", "private": true, "type": "module", "description": "Memmy desktop client.", diff --git a/App/shell/desktop/src/main/runtime-services.ts b/App/shell/desktop/src/main/runtime-services.ts index 332a672ec..a520c1af2 100644 --- a/App/shell/desktop/src/main/runtime-services.ts +++ b/App/shell/desktop/src/main/runtime-services.ts @@ -904,7 +904,7 @@ export async function ensureMemoryService( ); } -/** Only replace a runtime installed by this Desktop, after its migrations finish. */ +/** Replace a Desktop-owned runtime when its version or build changes, after migrations finish. */ async function stopOlderBundledMemoryRuntime( runtimeConfig: PackagedRuntimeConfig, options: StartManagedRuntimeServicesOptions, @@ -929,7 +929,12 @@ async function stopOlderBundledMemoryRuntime( const installedVersion = parseStableMemoryVersion(installed.version); if (!bundledVersion || !installedVersion) return false; const difference = bundledVersion.map((part, index) => part - installedVersion[index]!).find((delta) => delta !== 0) ?? 0; - if (difference <= 0 || bundled.protocolVersion !== SUPPORTED_MEMORY_PROTOCOL_VERSION + const changedBuild = typeof bundled.buildId === "string" && /^[a-f0-9]{64}$/.test(bundled.buildId) + && bundled.buildId !== installed.buildId; + if (difference < 0 || (difference === 0 && !changedBuild) + || (typeof bundled.schemaVersion === "number" && typeof installed.schemaVersion === "number" + && bundled.schemaVersion < installed.schemaVersion) + || bundled.protocolVersion !== SUPPORTED_MEMORY_PROTOCOL_VERSION || installed.protocolVersion !== SUPPORTED_MEMORY_PROTOCOL_VERSION) return false; // The standalone CLI records its own Node executable. Sharing a home or a diff --git a/App/shell/desktop/tests/dev-cli-launcher.test.ts b/App/shell/desktop/tests/dev-cli-launcher.test.ts index ea91c4f7c..5bbafd041 100644 --- a/App/shell/desktop/tests/dev-cli-launcher.test.ts +++ b/App/shell/desktop/tests/dev-cli-launcher.test.ts @@ -151,7 +151,7 @@ fi`; ); expect(init.status, init.stderr || init.stdout).toBe(0); const config = YAML.parse(readFileSync(configPath, "utf8")); - expect(config.memmyMemory.embedding).toEqual({ mode: "local", provider: "local" }); + expect(config.memmyMemory.embedding).toBeUndefined(); const validate = spawnSync( "node", diff --git a/App/shell/desktop/tests/packaged-runtime-boundary.test.ts b/App/shell/desktop/tests/packaged-runtime-boundary.test.ts index 973d49587..125e0d4ce 100644 --- a/App/shell/desktop/tests/packaged-runtime-boundary.test.ts +++ b/App/shell/desktop/tests/packaged-runtime-boundary.test.ts @@ -125,7 +125,7 @@ describe("desktop packaged runtime boundaries", () => { yaml: expect.any(String), zod: expect.any(String) }); - expect(memoryPackage.version).toBe("2.1.3"); + expect(memoryPackage.version).toBe("2.1.4"); expect(memoryPackage.dependencies ?? {}).not.toHaveProperty("@memmy/local-api-contracts"); expect(memoryPackage.dependencies ?? {}).not.toHaveProperty("@memmy/migrations"); expect(memoryPackage.scripts?.prebuild).toBeUndefined(); diff --git a/App/shell/desktop/tests/runtime-memory-upgrade.test.ts b/App/shell/desktop/tests/runtime-memory-upgrade.test.ts index e8d2d67ee..dc7d28e53 100644 --- a/App/shell/desktop/tests/runtime-memory-upgrade.test.ts +++ b/App/shell/desktop/tests/runtime-memory-upgrade.test.ts @@ -21,6 +21,10 @@ afterEach(async () => { async function fixture(options: { version?: string; + buildId?: string; + bundledBuildId?: string; + schemaVersion?: number; + bundledSchemaVersion?: number; delay?: number; independent?: boolean; identity?: boolean; @@ -78,9 +82,10 @@ async function fixture(options: { ].join("\n"); await writeFile(entry, source); await writeFile(join(bundled, "server.cjs"), source); - await writeFile(join(bundled, "memory-runtime.json"), JSON.stringify({ version: "2.1.1", protocolVersion: 1 })); + await writeFile(join(bundled, "memory-runtime.json"), JSON.stringify({ version: "2.1.1", protocolVersion: 1, buildId: options.bundledBuildId, schemaVersion: options.bundledSchemaVersion })); await writeFile(join(serviceHome, "current.json"), JSON.stringify({ version: options.version ?? "2.1.0", protocolVersion: 1, runtimeDir, entrypoint: entry, + buildId: options.buildId, schemaVersion: options.schemaVersion, runtimeExecutable: options.independent ? join(root, "independent-node") : process.execPath, })); await writeFile(cli, [ @@ -163,7 +168,7 @@ describe("bundled Memory upgrades", () => { expect(running.children).toHaveLength(0); expect(running.old.process.exitCode).toBeNull(); expect(await running.version()).toBe("2.1.1"); - }); + }, 15_000); it("waits for a migrating Windows database owner before repairing its launcher", async () => { const running = await fixture({ version: "2.1.1", platform: "win32", delay: 400 }); @@ -240,6 +245,42 @@ describe("bundled Memory upgrades", () => { expect(running.children[0]?.process.pid).not.toBe(running.old.process.pid); }); + it.each([undefined, "a".repeat(64)])("replaces an old same-version Desktop build (%s) after releasing its database", async (buildId) => { + const running = await fixture({ version: "2.1.1", buildId, bundledBuildId: "b".repeat(64), shutdownDelay: 100 }); + await running.ensure(); + expect(running.old.process.exitCode).toBe(0); + expect(running.children).toHaveLength(1); + expect(await running.version()).toBe("2.1.1"); + }); + + it("reuses an identical Desktop build", async () => { + const running = await fixture({ version: "2.1.1", buildId: "a".repeat(64), bundledBuildId: "a".repeat(64) }); + await running.ensure(); + expect(running.old.process.exitCode).toBeNull(); + expect(running.children).toHaveLength(0); + }); + + it("keeps a newer compatible independently installed build", async () => { + const running = await fixture({ version: "2.2.0", buildId: "a".repeat(64), bundledBuildId: "b".repeat(64) }); + await running.ensure(); + expect(running.old.process.exitCode).toBeNull(); + expect(running.children).toHaveLength(0); + }); + + it("does not replace a separately owned same-version build", async () => { + const running = await fixture({ independent: true, version: "2.1.1", buildId: "a".repeat(64), bundledBuildId: "b".repeat(64) }); + await running.ensure(); + expect(running.old.process.exitCode).toBeNull(); + expect(running.children).toHaveLength(0); + }); + + it("does not replace a same-version build with a lower schema", async () => { + const running = await fixture({ version: "2.1.1", buildId: "a".repeat(64), bundledBuildId: "b".repeat(64), schemaVersion: 9, bundledSchemaVersion: 8 }); + await running.ensure(); + expect(running.old.process.exitCode).toBeNull(); + expect(running.children).toHaveLength(0); + }); + it("waits for an older migrating runtime before upgrading and releasing its database lock", async () => { const running = await fixture({ delay: 400 }); await running.ensure(); diff --git a/Memory/package.json b/Memory/package.json index d77297d89..d6d7845ad 100644 --- a/Memory/package.json +++ b/Memory/package.json @@ -1,6 +1,6 @@ { "name": "@memmy/memory", - "version": "2.1.3", + "version": "2.1.4", "private": true, "type": "module", "main": "./dist/src/index.js", diff --git a/Memory/src/agent-source/runtime.ts b/Memory/src/agent-source/runtime.ts index d0bfa9028..755f27014 100644 --- a/Memory/src/agent-source/runtime.ts +++ b/Memory/src/agent-source/runtime.ts @@ -337,7 +337,9 @@ export function createAgentSourceExecutor(options: CreateAgentSourceExecutorOpti sourceId, store, options.resolveAgentSkillRoot, - options.scheduleWorker + options.scheduleWorker, + options.memoryAddAnalytics, + persistentScanMode(mode) ); const sourceErrorCount = stage.scanErrorCount + result.errorCount + skillResult.errorCount; failureCount += sourceErrorCount; @@ -829,6 +831,14 @@ async function ingestStagedMessages( continue; } if (hasStagedSourceTurn(turn.messages[0])) { + let addAnalyticsBase: { + adapterId: string; + conversationId: string; + turnId: string; + scanMode?: MemoryDesktopAddScanMode; + } | undefined; + let addStartedAt = 0; + let reportedAdd = false; try { const sourceTurn = sourceTurnFromMessages(turn.messages); if (!sourceTurn) { @@ -840,6 +850,14 @@ async function ingestStagedMessages( continue; } const legacyImportTurnId = legacyImportTurnIdFromMessages(sourceId, turn.conversationId, turn.messages); + const resolvedScanMode = persistentScanMode(store.getSourceState(sourceId)?.mode ?? scanMode); + addAnalyticsBase = { + adapterId: `agent-source:${sourceId}`, + conversationId: turn.conversationId, + turnId: sourceTurn.turnId, + ...(resolvedScanMode ? { scanMode: resolvedScanMode } : {}) + }; + addStartedAt = Date.now(); const result = service.completeSourceTurn({ ...buildSourceTurnRequest(sourceTurn, "agent_source_scan"), ...(legacyImportTurnId ? { legacyImportTurnId } : {}), @@ -861,7 +879,16 @@ async function ingestStagedMessages( continue; } const ids = result.result?.l1MemoryIds ?? []; - if (result.status === "stored") written += ids.length; + if (result.status === "stored") { + written += ids.length; + memoryAddAnalytics?.trackAddStarted(addAnalyticsBase); + memoryAddAnalytics?.trackAddSucceeded({ + ...addAnalyticsBase, + durationMs: Date.now() - addStartedAt, + storedCount: Math.max(ids.length, 1) + }); + reportedAdd = true; + } if (ids.length === 0) store.saveResult({ sourceId, conversationId: turn.conversationId }); for (const memoryId of ids) store.saveResult({ sourceId, conversationId: turn.conversationId, memoryId }); messageCount += turn.messages.length; @@ -870,6 +897,14 @@ async function ingestStagedMessages( const reason = error instanceof Error ? error.message : "native turn ingestion failed"; recordScanItemSkip(store, sourceId, turn.conversationId, reason); noteUncommittedSkip(reason); + if (addAnalyticsBase && !reportedAdd) { + memoryAddAnalytics?.trackAddStarted(addAnalyticsBase); + memoryAddAnalytics?.trackAddFailed({ + ...addAnalyticsBase, + durationMs: Date.now() - addStartedAt, + error + }); + } } processed += turn.messages.length; onProgress({ sourceId, phase: "add", current: processed, total: store.count(sourceId), message: "Capturing conversation turns" }); @@ -1064,7 +1099,12 @@ async function ingestAgentSkills( sourceId: string, store: MemoryAgentSourceScanStore, resolveAgentSkillRoot: (sourceId: string) => string | null = agentRootDirectory, - scheduleWorker?: () => void + scheduleWorker?: () => void, + memoryAddAnalytics?: Pick< + MemoryDesktopAddAnalytics, + "trackAddStarted" | "trackAddSucceeded" | "trackAddFailed" + >, + scanMode?: MemoryDesktopAddScanMode ): Promise<{ written: number; memoryIdCount: number; errorCount: number; errors: string[] }> { const root = resolveAgentSkillRoot(sourceId); if (!root) return { written: 0, memoryIdCount: 0, errorCount: 0, errors: [] }; @@ -1085,6 +1125,15 @@ async function ingestAgentSkills( const sourceSkillId = relative(skillsRoot, dirname(filePath)).replaceAll("\\", "/"); const requestId = `agent-source-skill:${sourceId}:${sourceSkillId}:${contentHash}`; const fileStat = await stat(filePath); + const turnId = `skill:${sourceSkillId}:${contentHash}`; + const addAnalyticsBase = { + adapterId: `agent-source:${sourceId}`, + conversationId: `skill:${sourceSkillId}`, + turnId, + layer: "Skill" as const, + ...(scanMode ? { scanMode } : {}) + }; + const addStartedAt = Date.now(); try { const added = service.addMemory({ requestId, @@ -1094,7 +1143,7 @@ async function ingestAgentSkills( title: frontmatterValue(content, "name") ?? sourceSkillId, tags: ["agent-source", "cross-agent-skill", sourceId], source: sourceId, - turnId: `skill:${sourceSkillId}:${contentHash}`, + turnId, createdAt: fileStat.mtime.toISOString(), sourceAgentId: sourceId, sourceSkillId, @@ -1103,6 +1152,14 @@ async function ingestAgentSkills( sourceContentHash: contentHash, deferProcessing: true }); + if (added.duplicate !== true && added.status !== "deleted") { + memoryAddAnalytics?.trackAddStarted(addAnalyticsBase); + memoryAddAnalytics?.trackAddSucceeded({ + ...addAnalyticsBase, + durationMs: Date.now() - addStartedAt, + storedCount: 1 + }); + } written += 1; memoryIdCount += 1; store.saveResult({ sourceId, conversationId: `skill:${sourceSkillId}`, memoryId: added.id }); @@ -1117,6 +1174,12 @@ async function ingestAgentSkills( `skill:${sourceSkillId}`, error instanceof Error ? error.message : String(error) ); + memoryAddAnalytics?.trackAddStarted(addAnalyticsBase); + memoryAddAnalytics?.trackAddFailed({ + ...addAnalyticsBase, + durationMs: Date.now() - addStartedAt, + error + }); } } flush(true); diff --git a/Memory/src/cli/analytics.ts b/Memory/src/cli/analytics.ts index 6a1a16c67..b7bd05852 100644 --- a/Memory/src/cli/analytics.ts +++ b/Memory/src/cli/analytics.ts @@ -107,9 +107,13 @@ export function resolveAnalyticsEnvParams(options: { }; } +/** GA4 Measurement Protocol accepts only string and number param values. A boolean drops the whole event. */ export function compactAnalyticsParams(params: AnalyticsParams = {}): AnalyticsParams { return Object.fromEntries( - Object.entries(params).filter(([, value]) => value !== undefined && value !== null && value !== ""), + Object.entries(params).flatMap(([key, value]) => { + if (value === undefined || value === null || value === "") return []; + return [[key, typeof value === "boolean" ? (value ? 1 : 0) : value]]; + }), ) as AnalyticsParams; } @@ -424,6 +428,26 @@ export function resolveEndpointKind(url: string): EndpointKind { return "remote"; } +// Keep in sync with runCommand and mapTopLevelCommand in commands.ts. Only these fixed words are +// reported; other positionals can be search queries, memory text, or ids. +const CLI_COMMAND_ACTIONS = new Map([ + ["init", []], + ["install", []], + ["upgrade", []], + ["stop", []], + ["service", ["start", "stop", "status", "repair-launcher"]], + ["raw", ["GET", "POST", "DELETE"]], + ["serve", []], + ["health", []], + ["reload-config", []], + ["session", ["open", "close"]], + ["turn", ["start", "complete"]], + ["search", []], + ["add", []], + ["get", []], + ["delete", []], +]); + export function resolveCommandIdentity(argv: string[]): { command_group: string; command_action?: string; @@ -456,8 +480,11 @@ export function resolveCommandIdentity(argv: string[]): { }; } - const group = words[0] ?? "help"; - const action = words[1]; + const [first = "", second = ""] = words; + const knownActions = CLI_COMMAND_ACTIONS.get(first); + const group = knownActions ? first : "unknown"; + // raw accepts its method in any case, like runRaw in commands.ts. + const action = knownActions?.includes(group === "raw" ? second.toUpperCase() : second) ? second : undefined; const positionalId = words[2]?.trim() || ""; const has_session_id = diff --git a/Memory/src/cli/legacy-migration.ts b/Memory/src/cli/legacy-migration.ts index 76366f042..42b936bde 100644 --- a/Memory/src/cli/legacy-migration.ts +++ b/Memory/src/cli/legacy-migration.ts @@ -156,15 +156,16 @@ async function importLegacyConfig(source: LegacySource, targetPath: string): Pro const legacy = record(parsed); const llm = record(legacy.llm); const skillEvolver = record(legacy.skillEvolver); - const embedding = record(legacy.embedding); + const rawEmbedding = record(legacy.embedding); const algorithm = record(legacy.algorithm); const summary = mapLlm(llm); const evolution = mapLlm(Object.keys(skillEvolver).length ? skillEvolver : llm); + const embedding = mapEmbedding(rawEmbedding); await mutateMemoryConfig(targetPath, (root) => { const memory = record(root.memmyMemory); const hub = record(legacy.hub); const telemetry = record(legacy.telemetry); - const nextMemory = { + const nextMemory: Record = { ...memory, roleRouting: { ...record(memory.roleRouting), @@ -173,19 +174,20 @@ async function importLegacyConfig(source: LegacySource, targetPath: string): Pro }, summary, evolution, - embedding: mapEmbedding(embedding), + embedding, algorithm: mergeLegacyAlgorithm(record(memory.algorithm), algorithm), ...(Object.keys(telemetry).length ? { telemetry: { ...record(memory.telemetry), ...telemetry } } : {}), ...(Object.keys(hub).length ? { hub: { ...hub, migratedFrom: source.agent } } : {}), migratedFrom: source.agent }; - root.memmyMemory = nextMemory; syncMemoryModelCatalog(root, nextMemory, { roleRouting: nextMemory.roleRouting, summary, evolution, - embedding: nextMemory.embedding + embedding }); + delete nextMemory.embedding; + root.memmyMemory = nextMemory; }); } diff --git a/Memory/src/cli/npm/package.json b/Memory/src/cli/npm/package.json index 1ed540d11..2a1df7e19 100644 --- a/Memory/src/cli/npm/package.json +++ b/Memory/src/cli/npm/package.json @@ -1,6 +1,6 @@ { "name": "@memtensor/memmy-memory-cli", - "version": "2.1.3", + "version": "2.1.4", "description": "Memmy Memory CLI for local agent memory.", "type": "module", "bin": { diff --git a/Memory/src/cli/runtime-installer.ts b/Memory/src/cli/runtime-installer.ts index 7aba2effc..535fb482c 100644 --- a/Memory/src/cli/runtime-installer.ts +++ b/Memory/src/cli/runtime-installer.ts @@ -15,7 +15,8 @@ const INSTALL_LOCK_WRITE_GRACE_MS = 2_000; const SERVICE_STOP_TIMEOUT_MS = 5_000; export const DEFAULT_HEALTH_CHECK_TIMEOUT_MS = 120_000; -export interface RuntimeAssetDescriptor { name: string; sha256: string; size?: number; url?: string; } +export interface RuntimeBuildIdentity { buildId?: string; schemaVersion?: number; } +export interface RuntimeAssetDescriptor extends RuntimeBuildIdentity { name: string; sha256: string; size?: number; url?: string; } export interface MemoryReleaseManifest { version: string; protocolVersion: number; @@ -45,7 +46,7 @@ export interface MemoryRuntimeInstallOptions { preferInstalledCompatible?: boolean; } -export interface InstalledRuntimePointer { +export interface InstalledRuntimePointer extends RuntimeBuildIdentity { version: string; protocolVersion: number; target: string; @@ -69,16 +70,14 @@ export async function installMemoryRuntime(options: MemoryRuntimeInstallOptions } const currentPath = join(serviceHome, "current.json"); const installationPath = join(serviceHome, "installation.json"); - const previous = await readJsonFile(currentPath); - const versionComparison = previous ? compareVersions(manifest.version, previous.version) : 1; - if (previous && options.preferInstalledCompatible && previous.protocolVersion === MEMORY_PROTOCOL_VERSION && versionComparison <= 0) { - return reuseInstalledRuntime(previous, home, serviceHome, options, healthCheckTimeoutMs); - } - if (previous && versionComparison < 0) { - throw new Error(`refusing to downgrade Memory from ${previous.version} to ${manifest.version}`); - } - const runtimeDir = join(runtimeRoot, manifest.version, target); + const identity = parseBuildIdentity(descriptor); + // Keep builds immutable so replacing same-version code never overwrites an + // active process or destroys the previous runtime needed for recovery. + const runtimeDir = identity.buildId + ? join(runtimeRoot, manifest.version, target, identity.buildId) + : join(runtimeRoot, manifest.version, target); const pointer: InstalledRuntimePointer = { + ...identity, version: manifest.version, protocolVersion: manifest.protocolVersion, target, @@ -89,6 +88,11 @@ export async function installMemoryRuntime(options: MemoryRuntimeInstallOptions }; const launcher = launcherPaths(home); if (options.dryRun) { + const previous = await currentInstalledRuntime(home); + if (previous && compareVersions(manifest.version, previous.version) < 0) { + if (options.preferInstalledCompatible) return { ok: true, reused: true, dryRun: true, ...previous }; + throw new Error(`refusing to downgrade Memory from ${previous.version} to ${manifest.version}`); + } return { ok: true, dryRun: true, home, serviceHome, target, manifest, pointer, launcher }; } @@ -97,6 +101,22 @@ export async function installMemoryRuntime(options: MemoryRuntimeInstallOptions let stagedPath: string | undefined; let installedRuntimeCreated = false; try { + // Decide after acquiring the lock: a concurrent installer may have changed + // the pointer while this installer was resolving its release. + const previous = await currentInstalledRuntime(home); + const versionComparison = previous ? compareVersions(manifest.version, previous.version) : 1; + if (previous && options.preferInstalledCompatible && previous.protocolVersion === MEMORY_PROTOCOL_VERSION + && (versionComparison < 0 || (versionComparison === 0 + && (!identity.buildId || identity.buildId === previous.buildId) + && existsSync(previous.entrypoint)))) { + return reuseInstalledRuntime(previous, home, serviceHome, options, healthCheckTimeoutMs); + } + if (previous && versionComparison < 0) { + throw new Error(`refusing to downgrade Memory from ${previous.version} to ${manifest.version}`); + } + if (previous?.schemaVersion && identity.schemaVersion && identity.schemaVersion < previous.schemaVersion) { + throw new Error(`refusing to downgrade Memory schema from ${previous.schemaVersion} to ${identity.schemaVersion}`); + } if (!existsSync(pointer.entrypoint)) { stagedPath = join(runtimeRoot, `.staging-${process.pid}-${Date.now()}`); await mkdir(stagedPath, { recursive: true }); @@ -113,13 +133,13 @@ export async function installMemoryRuntime(options: MemoryRuntimeInstallOptions await mkdir(unpacked, { recursive: true }); extractTarGzip(archivePath, unpacked); } - await validateRuntime(unpacked, manifest.version, target, manifest.protocolVersion); + await validateRuntime(unpacked, manifest.version, target, manifest.protocolVersion, identity); await mkdir(dirname(runtimeDir), { recursive: true }); await rm(runtimeDir, { recursive: true, force: true }); await rename(unpacked, runtimeDir); installedRuntimeCreated = true; } else { - await validateRuntime(runtimeDir, manifest.version, target, manifest.protocolVersion); + await validateRuntime(runtimeDir, manifest.version, target, manifest.protocolVersion, identity); } const switching = !previous || previous.runtimeDir !== runtimeDir; @@ -129,6 +149,7 @@ export async function installMemoryRuntime(options: MemoryRuntimeInstallOptions if (process.platform === "win32") await stopInstalledMemoryService(home); else if (switching && previous) stopUserService(); } + if (switching) await waitForServiceOwnerExit(serviceHome); await writeJsonAtomic(currentPath, pointer); await writeStableLauncher(home, serviceHome, pointer.runtimeExecutable!); if (!options.skipServiceRegistration) registerAndStartUserService(home, serviceHome); @@ -139,14 +160,19 @@ export async function installMemoryRuntime(options: MemoryRuntimeInstallOptions await waitForRuntimeHealth( options.endpoint ?? "http://127.0.0.1:18960", manifest.version, - healthCheckTimeoutMs + healthCheckTimeoutMs, + identity.buildId ); } catch (error) { if (!options.skipServiceRegistration && previous) { if (process.platform === "win32") await stopInstalledMemoryService(home); else stopUserService(); } - if (previous) { + if (previous && identity.schemaVersion && (!previous.schemaVersion || previous.schemaVersion < identity.schemaVersion)) { + // The failed process may already have committed its DB migration. + // Keep the new pointer; rolling back code could strand that database. + throw new Error(`Memory activation failed; kept the runtime supporting schema ${identity.schemaVersion} to avoid a schema downgrade`, { cause: error }); + } else if (previous) { await writeJsonAtomic(currentPath, previous); await writeStableLauncher(home, serviceHome, previous.runtimeExecutable ?? process.execPath); if (!options.skipServiceRegistration) registerAndStartUserService(home, serviceHome); @@ -166,6 +192,7 @@ export async function installMemoryRuntime(options: MemoryRuntimeInstallOptions } await writeJsonAtomic(installationPath, { + ...identity, serviceVersion: manifest.version, protocolVersion: manifest.protocolVersion, target, @@ -200,7 +227,7 @@ export async function startInstalledMemoryService(home = "~/.memmy"): Promise { + const deadline = Date.now() + SERVICE_STOP_TIMEOUT_MS; + while (true) { + const lock = await readJsonFile<{ pid?: number }>(join(serviceHome, "service.lock")); + if (!lock || (Number.isInteger(lock.pid) && !isProcessAlive(lock.pid!))) return; + if (Date.now() >= deadline) throw new Error("Memory service still owns its storage; stop it before activating another runtime"); + await new Promise((resolveDelay) => setTimeout(resolveDelay, 50)); + } +} + +function isProcessAlive(pid: number): boolean { + try { + process.kill(pid, 0); + return true; + } catch (error) { + return isNodeError(error) && error.code === "EPERM"; + } +} + function loopbackEndpoint(value: string): string { const url = new URL(value); if (url.protocol !== "http:" || !["127.0.0.1", "localhost", "[::1]"].includes(url.hostname)) { @@ -394,27 +440,21 @@ async function reuseInstalledRuntime( healthCheckTimeoutMs: number ): Promise> { if (options.dryRun) return { ok: true, reused: true, dryRun: true, ...pointer }; - const installLock = await acquireInstallLock(join(serviceHome, "install.lock")); - try { - // Another installer may have activated a newer runtime while we waited. - pointer = await currentInstalledRuntime(home) ?? pointer; - await validateRuntime(pointer.runtimeDir, pointer.version, pointer.target, pointer.protocolVersion); - const repairLegacyTask = Boolean(options.skipServiceRegistration && isLegacyWindowsTask(home)); - if (process.platform === "win32" && (!options.skipServiceRegistration || repairLegacyTask)) await stopInstalledMemoryService(home); - await writeStableLauncher(home, serviceHome, pointer.runtimeExecutable ?? options.nodeExecutable ?? process.execPath); - if (!options.skipServiceRegistration) registerAndStartUserService(home, serviceHome); - else if (repairLegacyTask) tryRepairLegacyWindowsTaskRegistration(home, serviceHome); - if (!options.skipHealthCheck) { - await waitForRuntimeHealth( - options.endpoint ?? "http://127.0.0.1:18960", - pointer.version, - healthCheckTimeoutMs - ); - } - return { ok: true, reused: true, ...pointer }; - } finally { - await installLock.release(); + await validateRuntime(pointer.runtimeDir, pointer.version, pointer.target, pointer.protocolVersion, pointer); + const repairLegacyTask = Boolean(options.skipServiceRegistration && isLegacyWindowsTask(home)); + if (process.platform === "win32" && (!options.skipServiceRegistration || repairLegacyTask)) await stopInstalledMemoryService(home); + await writeStableLauncher(home, serviceHome, pointer.runtimeExecutable ?? options.nodeExecutable ?? process.execPath); + if (!options.skipServiceRegistration) registerAndStartUserService(home, serviceHome); + else if (repairLegacyTask) tryRepairLegacyWindowsTaskRegistration(home, serviceHome); + if (!options.skipHealthCheck) { + await waitForRuntimeHealth( + options.endpoint ?? "http://127.0.0.1:18960", + pointer.version, + healthCheckTimeoutMs, + pointer.buildId + ); } + return { ok: true, reused: true, ...pointer }; } async function resolveReleaseManifest( @@ -437,10 +477,13 @@ async function resolveReleaseManifest( if (options.runtimeAsset) { const path = resolveHome(options.runtimeAsset); const sha256 = options.runtimeSha256 ?? await sha256File(path); + const result = spawnSync("tar", ["-xOf", path, "./memory-runtime.json"], { encoding: "utf8" }); + if (result.status !== 0) throw new Error(`Cannot read Memory runtime metadata: ${result.stderr}`); + const identity = parseBuildIdentity(JSON.parse(result.stdout) as Record); return { version: options.version ?? MEMORY_SERVICE_VERSION, protocolVersion: MEMORY_PROTOCOL_VERSION, - assets: { [target]: { name: basename(path), sha256, url: pathToFileURL(path).href } } + assets: { [target]: { ...identity, name: basename(path), sha256, url: pathToFileURL(path).href } } }; } if (options.runtimeDirectory) { @@ -460,6 +503,7 @@ async function resolveReleaseManifest( protocolVersion, assets: { [packagedTarget]: { + ...parseBuildIdentity(metadata ?? {}), name: basename(path), sha256: "0".repeat(64), url: pathToFileURL(path).href @@ -491,7 +535,7 @@ function parseReleaseManifest(value: unknown): MemoryReleaseManifest { if (!isRecord(asset) || typeof asset.name !== "string" || !asset.name || typeof asset.sha256 !== "string" || !/^[a-f0-9]{64}$/i.test(asset.sha256)) { throw new Error(`Memory release asset is invalid: ${target}`); } - assets[target] = { name: asset.name, sha256: asset.sha256.toLowerCase(), ...(typeof asset.size === "number" ? { size: asset.size } : {}), ...(typeof asset.url === "string" ? { url: asset.url } : {}) }; + assets[target] = { ...parseBuildIdentity(asset), name: asset.name, sha256: asset.sha256.toLowerCase(), ...(typeof asset.size === "number" ? { size: asset.size } : {}), ...(typeof asset.url === "string" ? { url: asset.url } : {}) }; } return { version: value.version, protocolVersion: value.protocolVersion as number, assets }; } @@ -543,9 +587,24 @@ function sourceUrl(value: string): string { return pathToFileURL(resolveHome(value)).href; } -async function validateRuntime(path: string, version: string, target: string, protocolVersion: number): Promise { +function parseBuildIdentity(value: { buildId?: unknown; schemaVersion?: unknown }): RuntimeBuildIdentity { + if (value.buildId !== undefined && (typeof value.buildId !== "string" || !/^[a-f0-9]{64}$/.test(value.buildId))) { + throw new Error("Memory runtime buildId must be a SHA-256 digest"); + } + if (value.schemaVersion !== undefined && (!Number.isSafeInteger(value.schemaVersion) || Number(value.schemaVersion) < 1)) { + throw new Error("Memory runtime schemaVersion must be a positive integer"); + } + return { + ...(typeof value.buildId === "string" ? { buildId: value.buildId } : {}), + ...(typeof value.schemaVersion === "number" ? { schemaVersion: value.schemaVersion } : {}) + }; +} + +async function validateRuntime(path: string, version: string, target: string, protocolVersion: number, identity: RuntimeBuildIdentity = {}): Promise { const manifest = await readJsonFile>(join(path, "memory-runtime.json")); - if (!manifest || manifest.version !== version || manifest.target !== target || manifest.protocolVersion !== protocolVersion) { + if (!manifest || manifest.version !== version || manifest.target !== target || manifest.protocolVersion !== protocolVersion + || (identity.buildId && manifest.buildId !== identity.buildId) + || (identity.schemaVersion && manifest.schemaVersion !== identity.schemaVersion)) { throw new Error(`Memory runtime metadata is invalid for ${version}-${target}`); } const entrypoint = join(path, "dist", "src", "server", "index.js"); @@ -760,7 +819,8 @@ function runLifecycle(command: string, args: string[], allowFailure = false): vo async function waitForRuntimeHealth( endpoint: string, expectedVersion: string, - timeoutMs: number + timeoutMs: number, + expectedBuildId?: string ): Promise { const deadline = Date.now() + timeoutMs; let lastError = "service did not respond"; @@ -776,11 +836,12 @@ async function waitForRuntimeHealth( health.ok === true && health.protocolVersion === MEMORY_PROTOCOL_VERSION && (health.serviceVersion === expectedVersion || health.version === expectedVersion) + && (!expectedBuildId || health.buildId === expectedBuildId) ) return; if (health.protocolVersion !== MEMORY_PROTOCOL_VERSION) { lastError = `service reported protocol ${String(health.protocolVersion)}`; } else { - lastError = `service reported version ${String(health.serviceVersion ?? health.version)}`; + lastError = `service reported version ${String(health.serviceVersion ?? health.version)}, build ${String(health.buildId)}`; } } else { lastError = `health returned HTTP ${response.status}`; diff --git a/Memory/src/cli/scripts/assemble-release.mjs b/Memory/src/cli/scripts/assemble-release.mjs index 17289d483..19edef8c2 100644 --- a/Memory/src/cli/scripts/assemble-release.mjs +++ b/Memory/src/cli/scripts/assemble-release.mjs @@ -4,6 +4,7 @@ import { createReadStream, existsSync } from "node:fs"; import { copyFile, mkdir, readdir, stat, writeFile } from "node:fs/promises"; import { basename, join, resolve } from "node:path"; import { fileURLToPath } from "node:url"; +import { spawnSync } from "node:child_process"; const scriptDirectory = fileURLToPath(new URL(".", import.meta.url)); const memoryRoot = resolve(scriptDirectory, "../../.."); @@ -27,7 +28,16 @@ for (const source of discovered) { const match = name.match(new RegExp(`^memmy-memory-runtime-${escapeRegExp(version)}-(darwin|linux|windows)-(arm64|x64)\\.tar\\.gz$`)); if (match) { const target = `${match[1]}-${match[2]}`; - assets[target] = { name, sha256, size: (await stat(destination)).size }; + const result = spawnSync("tar", ["-xOf", destination, "./memory-runtime.json"], { encoding: "utf8" }); + if (result.status !== 0) throw new Error(`Cannot read runtime metadata for ${target}: ${result.stderr}`); + const metadata = JSON.parse(result.stdout); + if (metadata.version !== version || metadata.target !== target || metadata.protocolVersion !== 1 + || !/^[a-f0-9]{64}$/.test(metadata.buildId ?? "") + || !Number.isSafeInteger(metadata.schemaVersion) || metadata.schemaVersion < 1) { + throw new Error(`Runtime build identity is invalid for ${target}`); + } + const buildIdentity = { buildId: metadata.buildId, schemaVersion: metadata.schemaVersion }; + assets[target] = { ...buildIdentity, name, sha256, size: (await stat(destination)).size }; } } for (const target of targets) { diff --git a/Memory/src/cli/scripts/build-runtime.mjs b/Memory/src/cli/scripts/build-runtime.mjs index cafdede24..9e1dfba5a 100755 --- a/Memory/src/cli/scripts/build-runtime.mjs +++ b/Memory/src/cli/scripts/build-runtime.mjs @@ -1,4 +1,5 @@ #!/usr/bin/env node +import { memoryRuntimeBuildIdentity } from "../../../../scripts/internal/shared/memory-runtime-build-identity.mjs"; import { createHash } from "node:crypto"; import { createReadStream, existsSync } from "node:fs"; import { cp, mkdir, mkdtemp, readFile, readdir, rename, rm, stat, writeFile } from "node:fs/promises"; @@ -54,7 +55,9 @@ try { await verifyEmbeddingModel(runtimeRoot); } await verifyRuntimeDependencies(runtimeRoot, target, platform, arch, workspaceDependencies); + const buildIdentity = await memoryRuntimeBuildIdentity(repositoryRoot); await writeJson(join(runtimeRoot, "memory-runtime.json"), { + ...buildIdentity, name: "memmy-memory-runtime", version, protocolVersion: 1, @@ -68,7 +71,7 @@ try { await mkdir(outputRoot, { recursive: true }); const assetPath = join(outputRoot, assetName); run("tar", ["-czf", assetPath, "-C", runtimeRoot, "."], repositoryRoot); - const descriptor = { name: assetName, sha256: await sha256File(assetPath), size: (await stat(assetPath)).size }; + const descriptor = { ...buildIdentity, name: assetName, sha256: await sha256File(assetPath), size: (await stat(assetPath)).size }; await updateReleaseManifest(outputRoot, version, target, descriptor); process.stdout.write(`${assetPath}\n`); } finally { diff --git a/Memory/src/cli/setup.ts b/Memory/src/cli/setup.ts index dcc791f06..df8298243 100644 --- a/Memory/src/cli/setup.ts +++ b/Memory/src/cli/setup.ts @@ -1,3 +1,4 @@ +import { syncMemoryModelCatalog } from "../config/model-catalog.js"; import { mutateMemoryConfig } from "../config/writer.js"; import { existsSync, @@ -293,8 +294,8 @@ function setupMemoryConfig( ): void { const app = asRecord(config.app); const appUserId = optionalString(app.userId); - - config.memmyMemory = setupMemmyMemoryConfig(asRecord(config.memmyMemory), { + const existingMemory = asRecord(config.memmyMemory); + const memory = setupMemmyMemoryConfig(existingMemory, { appUserId, accountMode: app.userMode === "account", dbPath: options.dbPath, @@ -302,6 +303,13 @@ function setupMemoryConfig( token: options.token, generateTokenIfMissing: options.generateTokenIfMissing, }); + // Older installs stored embedding settings here. Normalize them into the + // shared model catalog while ensuring the retired runtime field is absent. + if (Object.prototype.hasOwnProperty.call(existingMemory, "embedding")) { + const embedding = setupEmbeddingForCatalog(existingMemory.embedding, app.userMode === "account"); + syncMemoryModelCatalog(config, { ...memory, embedding }, { embedding }); + } + config.memmyMemory = memory; } function setupMemmyMemoryConfig( @@ -316,7 +324,6 @@ function setupMemmyMemoryConfig( } ): Record { const roleRouting = asRecord(existing.roleRouting); - const embedding = asRecord(existing.embedding); const storage = asRecord(existing.storage); const algorithm = asRecord(existing.algorithm); const agentAccess = asRecord(existing.agentAccess); @@ -355,16 +362,35 @@ function setupMemmyMemoryConfig( watchFileChanges: optionalBoolean(agentAccess.watchFileChanges) ?? true, autoInjectSkill: optionalBoolean(agentAccess.autoInjectSkill) ?? false }, - embedding: Object.keys(embedding).length - ? embedding - : { - mode: options.accountMode ? "cloud" : "local", - ...(options.accountMode ? {} : { provider: "local" }) - } }; + delete memmyMemory.embedding; return memmyMemory; } +function setupEmbeddingForCatalog(value: unknown, accountMode: boolean): Record { + const embedding = asRecord(value); + const nested = asRecord(embedding.custom); + const configuredMode = optionalString(embedding.mode); + if (configuredMode === "local" || optionalString(embedding.provider) === "local") { + return { ...embedding, mode: "local" }; + } + if (configuredMode === "custom" || Object.keys(nested).length > 0 || hasEmbeddingConnection(embedding)) { + return { + ...embedding, + ...nested, + mode: "custom", + }; + } + return { + ...embedding, + mode: accountMode ? "cloud" : "local", + }; +} + +function hasEmbeddingConnection(value: Record): boolean { + return ["endpoint", "model", "apiKey", "extraHeaders", "extraBody"].some((key) => key in value); +} + function memoryRoleRouting(value: unknown): "follow" | "fixed" { return value === "fixed" ? "fixed" : "follow"; } diff --git a/Memory/src/config/index.ts b/Memory/src/config/index.ts index 85a96a4f2..a1e0af3f9 100644 --- a/Memory/src/config/index.ts +++ b/Memory/src/config/index.ts @@ -947,19 +947,25 @@ function resolveMemoryEmbedding( hasCatalog: boolean ): Record { const embedding = asRecord(memory.embedding); - const configuredMode = optionalString(embedding.mode); - const embeddingMode = configuredMode === "cloud" - || configuredMode === "local" - || configuredMode === "custom" - ? configuredMode - : mode === "account" && hasCatalog - ? "cloud" - : DEFAULT_MEMMY_CONFIG.embedding.mode; const activeAssignment = mode ? asRecord(asRecord(rootConfig.modelAssignments)[mode]) : {}; const rawAssignedPreset = activeAssignment.embedding; const hasExplicitAssignment = rawAssignedPreset !== undefined && rawAssignedPreset !== null; + const resolved = resolveMemoryAssignment(rootConfig, mode, "embedding"); + const configuredMode = optionalString(embedding.mode); + const embeddingMode = configuredMode === "cloud" + || configuredMode === "local" + || configuredMode === "custom" + ? configuredMode + : rawAssignedPreset === BUILTIN_LOCAL_EMBEDDING_ASSIGNMENT_ID + || (mode === "byok" && hasCatalog && !hasExplicitAssignment) + ? "local" + : hasExplicitAssignment && resolved.ok && embeddingProtocolSupported(resolved.context.protocol) + ? resolved.context.source === "account" ? "cloud" : "custom" + : mode === "account" && hasCatalog + ? "cloud" + : DEFAULT_MEMMY_CONFIG.embedding.mode; if (rawAssignedPreset === BUILTIN_LOCAL_EMBEDDING_ASSIGNMENT_ID) { const assignmentOwner = optionalString(activeAssignment.ownerAccountId); const activeAccountId = optionalString(asRecord(rootConfig.app).userId); @@ -988,8 +994,6 @@ function resolveMemoryEmbedding( }; } - const resolved = resolveMemoryAssignment(rootConfig, mode, "embedding"); - if (mode === "byok" && hasCatalog && !hasExplicitAssignment) { return localEmbeddingConfig(embedding); } @@ -1006,7 +1010,7 @@ function resolveMemoryEmbedding( return localEmbeddingConfig(embedding); } - if (embeddingMode === "custom") { + if (embeddingMode === "custom" && (configuredMode === "custom" || !hasExplicitAssignment)) { const custom = asRecord(embedding.custom); return { ...embedding, diff --git a/Memory/src/model/llm.ts b/Memory/src/model/llm.ts index d727f81b6..03e8dccb1 100644 --- a/Memory/src/model/llm.ts +++ b/Memory/src/model/llm.ts @@ -2,7 +2,7 @@ import { get_encoding } from "tiktoken"; import { MEMORY_SUMMARY_MAX_TOKENS, type LlmConfig } from "../config/index.js"; import { createMemoryLogger, memoryErrorFields } from "../logging/logger.js"; import { resolveMemoryAgentRegion } from "./agent-region.js"; -import { bearer, postJsonWithRetry, trimTrailingSlash } from "./http.js"; +import { bearer, ModelHttpError, postJsonWithRetry, trimTrailingSlash } from "./http.js"; import { HttpByokTokenUsageRecorder, extractModelTokenUsage, @@ -95,6 +95,7 @@ export function createLlmClient(config: LlmConfig, options: CreateLlmClientOptio class HttpLlmClient implements LlmClient { private lastOkAt: string | undefined; private lastError: string | undefined; + private openAiJsonResponseFormat: "json_object" | "text" = "json_object"; private readonly usageRecorder: MemoryTokenUsageSink; constructor(readonly config: LlmConfig, private readonly options: CreateLlmClientOptions = {}) { @@ -334,31 +335,53 @@ class HttpLlmClient implements LlmClient { ? this.config.thinkingBudget : undefined; const agentRegion = resolveMemoryAgentRegion(this.config.sourceProvider); - const response = await postJsonWithRetry({ - actualModelContext: this.config.actualModelContext, - provider: "openai_compatible", - operation: options.operation, - model: this.config.model, - url, - headers: { - ...bearer(this.config.apiKey), - ...(this.config.extraHeaders ?? {}), - ...(agentRegion ? { "X-Agent-Region": agentRegion } : {}) - }, - timeoutMs: options.timeoutMs ?? this.config.timeoutMs, - maxRetries: options.maxRetries ?? this.config.maxRetries, - body: { + const explicitResponseFormat = this.config.extraBody?.response_format; + const automaticResponseFormat = options.jsonMode && !omitJsonMode && explicitResponseFormat === undefined + ? this.openAiJsonResponseFormat + : undefined; + const request = (responseFormat: "json_object" | "text" | undefined) => + postJsonWithRetry({ + actualModelContext: this.config.actualModelContext, + provider: "openai_compatible", + operation: options.operation, model: this.config.model, - messages, - ...(!omitTemperature ? { temperature: options.temperature ?? this.config.temperature } : {}), - max_tokens: options.maxTokens ?? this.config.maxTokens, - stream: false, - ...thinking.fields, - ...(thinkingBudget !== undefined ? { thinking_budget: thinkingBudget } : {}), - ...(options.jsonMode && !omitJsonMode ? { response_format: { type: "json_object" } } : {}), - ...(this.config.extraBody ?? {}) + url, + headers: { + ...bearer(this.config.apiKey), + ...(this.config.extraHeaders ?? {}), + ...(agentRegion ? { "X-Agent-Region": agentRegion } : {}) + }, + timeoutMs: options.timeoutMs ?? this.config.timeoutMs, + maxRetries: options.maxRetries ?? this.config.maxRetries, + body: { + model: this.config.model, + messages, + ...(!omitTemperature ? { temperature: options.temperature ?? this.config.temperature } : {}), + max_tokens: options.maxTokens ?? this.config.maxTokens, + stream: false, + ...thinking.fields, + ...(thinkingBudget !== undefined ? { thinking_budget: thinkingBudget } : {}), + ...(responseFormat ? { response_format: { type: responseFormat } } : {}), + ...(this.config.extraBody ?? {}) + } + }); + let response: OpenAiChatResponse; + try { + response = await request(automaticResponseFormat); + } catch (error) { + if (automaticResponseFormat !== "json_object" || !requiresTextResponseFormat(error)) { + throw error; } - }); + this.openAiJsonResponseFormat = "text"; + logger.warn("json.response_format_fallback", { + operation: options.operation, + provider: "openai_compatible", + model: this.config.model, + responseFormat: "text", + ...memoryErrorFields(error) + }); + response = await request("text"); + } const choice = response.choices?.[0]; const text = choice?.message?.content; if (typeof text !== "string" || !text.trim()) { @@ -591,6 +614,16 @@ class HttpLlmClient implements LlmClient { } } +function requiresTextResponseFormat(error: unknown): boolean { + if (!(error instanceof ModelHttpError) || (error.httpStatus !== 400 && error.httpStatus !== 422)) { + return false; + } + const detail = error.detail.toLowerCase().replace(/['"`]/g, ""); + return /response[_ ]format(?:\.type)?/.test(detail) + && detail.includes("json_schema") + && /\btext\b/.test(detail); +} + function constrainSummaryMessages( messages: LlmMessage[], outputTokens: number, diff --git a/Memory/src/runtime-build.ts b/Memory/src/runtime-build.ts new file mode 100644 index 000000000..32842735f --- /dev/null +++ b/Memory/src/runtime-build.ts @@ -0,0 +1,11 @@ +import { readFileSync } from "node:fs"; + +/** Installed packages carry their build identity; development checkouts do not. */ +export const MEMORY_RUNTIME_BUILD_ID = (() => { + try { + const metadata = JSON.parse(readFileSync(new URL("../../memory-runtime.json", import.meta.url), "utf8")) as { buildId?: unknown }; + return typeof metadata.buildId === "string" && /^[a-f0-9]{64}$/.test(metadata.buildId) ? metadata.buildId : undefined; + } catch { + return undefined; + } +})(); diff --git a/Memory/src/server/desktop-analytics-identity.ts b/Memory/src/server/desktop-analytics-identity.ts new file mode 100644 index 000000000..6eff01564 --- /dev/null +++ b/Memory/src/server/desktop-analytics-identity.ts @@ -0,0 +1,36 @@ +import { existsSync, readFileSync } from "node:fs"; +import { parse as parseYaml } from "yaml"; +import { normalizeAnalyticsUserId } from "../cli/analytics.js"; + +export type DesktopAnalyticsIdentity = { + userId: string | null; + userMode: "account" | "byok" | null; +}; + +/** Desktop login projected into `~/.memmy/config.yaml`. Read on each report so a mode switch is visible without restarting. */ +export function readDesktopAnalyticsIdentity(configPath?: string | null): DesktopAnalyticsIdentity { + const empty: DesktopAnalyticsIdentity = { userId: null, userMode: null }; + if (!configPath) return empty; + try { + if (!existsSync(configPath)) return empty; + const root = asRecord(parseYaml(readFileSync(configPath, "utf8"))); + const app = asRecord(root.app); + const mode = optionalString(app.userMode); + const userMode = mode === "account" || mode === "byok" ? mode : null; + const cloudUuid = optionalString(app.cloudUuid); + const userId = cloudUuid ? normalizeAnalyticsUserId(optionalString(app.userId)) : null; + return { userId, userMode }; + } catch { + return empty; + } +} + +function asRecord(value: unknown): Record { + return typeof value === "object" && value !== null && !Array.isArray(value) + ? value as Record + : {}; +} + +function optionalString(value: unknown): string | undefined { + return typeof value === "string" && value.trim() ? value.trim() : undefined; +} diff --git a/Memory/src/server/http.ts b/Memory/src/server/http.ts index 1d2d0eecd..d4abf53d7 100644 --- a/Memory/src/server/http.ts +++ b/Memory/src/server/http.ts @@ -30,6 +30,7 @@ import { MemoryService } from "../service/memory-service.js"; import { MemoryServiceError, statusForCode } from "../utils/error.js"; import { stableHash } from "../utils/id.js"; import { resolveTimeZone } from "../utils/time.js"; +import { readDesktopAnalyticsIdentity } from "./desktop-analytics-identity.js"; import { createMemoryDesktopAddAnalytics, type MemoryDesktopAddAnalytics, @@ -38,9 +39,10 @@ import { createPluginRuntimeAnalytics, hitCountFromGetResponse, hitCountFromSearchResponse, - storedCountFromAddResponse, trackExternalHookCapture, trackExternalHookRecall, + trackStoredExternalMemoryAdd, + trackStoredExternalSourceTurnCapture, trackExternalToolCall, type PluginRuntimeAnalytics, } from "./plugin-runtime-analytics.js"; @@ -146,8 +148,15 @@ export function createMemoryHttpServer(options: MemoryHttpServerOptions): Server startupFallbackMs: options.workerStartupFallbackMs ?? DEFAULT_WORKER_STARTUP_FALLBACK_MS, postHealthDelayMs: options.workerPostHealthDelayMs ?? DEFAULT_WORKER_POST_HEALTH_DELAY_MS }); - const pluginRuntimeAnalytics = options.pluginRuntimeAnalytics ?? createPluginRuntimeAnalytics(); - const memoryAddAnalytics = options.memoryAddAnalytics ?? createMemoryDesktopAddAnalytics(); + const desktopIdentity = () => readDesktopAnalyticsIdentity(options.configPath); + const pluginRuntimeAnalytics = options.pluginRuntimeAnalytics ?? createPluginRuntimeAnalytics({ + getUserId: () => desktopIdentity().userId, + getUserMode: () => desktopIdentity().userMode, + }); + const memoryAddAnalytics = options.memoryAddAnalytics ?? createMemoryDesktopAddAnalytics({ + getUserId: () => desktopIdentity().userId, + getUserMode: () => desktopIdentity().userMode, + }); const agentSources = options.agentSourceExecutor ?? createAgentSourceExecutor({ service: options.service, configPath: options.configPath, @@ -595,7 +604,8 @@ async function routeRequest( const result = await service.idempotent("sessions.close", request, { sessionId, request }, () => service.closeSession(sessionId, request) ); - scheduleAutoWorkerForEvolution(result, autoWorker); + // Close can queue L3 batches even when no episode closes, so always wake the worker. + autoWorker.schedule(); return publicCloseSessionResponse(result); } @@ -689,7 +699,7 @@ async function routeRequest( const request = strictEnvelopeWithPrincipal({ ...input, namespace }, scopedPrincipal) as unknown as SourceTurnCompleteRequest; requireStringField(request, "query", "source-turn.complete"); requireStringField(request, "answer", "source-turn.complete"); - const result = service.completeSourceTurn({ + const completeSourceTurn = () => service.completeSourceTurn({ namespace: request.namespace, timeZone: request.timeZone, source: request.source, sourceTurn: request.sourceTurn, channel: request.channel, workspacePath: request.workspacePath, ...(request.captureLegacyHistory === true ? { captureLegacyHistory: true } : {}), @@ -702,6 +712,19 @@ async function routeRequest( sourceMemoryIds: request.sourceMemoryIds, usage: request.usage, status: request.status, tags: request.tags, userMemoryCorrection: request.userMemoryCorrection }); + const result = request.channel === "hook" + ? await trackStoredExternalSourceTurnCapture( + pluginRuntimeAnalytics, + { + source: request.source ?? request.sourceTurn?.source, + adapterId: request.adapterId, + namespace: request.namespace, + turnId: request.sourceTurn?.turnId, + }, + request, + completeSourceTurn, + ) + : completeSourceTurn(); if (result.result) scheduleAutoWorkerForEvolution(result.result, autoWorker); return result; } @@ -715,6 +738,7 @@ async function routeRequest( requireStringField(request, "answer", "turn.complete"); const turnId = decodeMatchSegment(turnComplete, 1); const publicRequest: TurnCompleteRequest = { + ...(typeof request.recallApplied === "boolean" ? { recallApplied: request.recallApplied } : {}), requestId: request.requestId, adapterId: request.adapterId, namespace: request.namespace, @@ -815,17 +839,13 @@ async function routeRequest( sourceContentHash: typeof request.sourceContentHash === "string" ? request.sourceContentHash : undefined }; const idempotency = memoryAddIdempotency(publicRequest, path); - const result = await trackExternalToolCall( + const result = await trackStoredExternalMemoryAdd( pluginRuntimeAnalytics, - { ...request, toolName: "memmy_memory_add" }, + { ...request, layer: publicRequest.layer }, () => service.idempotent("memory.add", idempotency.request, idempotency.fingerprint, () => service.addMemory(publicRequest) ), - (addResult) => ({ - stored_count: storedCountFromAddResponse(addResult), - ...(publicRequest.layer ? { layer: publicRequest.layer } : {}), - }), ); if (!publicRequest.deferProcessing) { autoWorker.schedule(); diff --git a/Memory/src/server/index.ts b/Memory/src/server/index.ts index 6ce143515..f14c9440c 100644 --- a/Memory/src/server/index.ts +++ b/Memory/src/server/index.ts @@ -12,6 +12,8 @@ import { closeMemoryHttpServer, listenMemoryHttpServer } from "./http.js"; import { loadCloudServiceEnv } from "../cli/load-env.js"; import { requestMemoryServiceRestart } from "./service-restart.js"; import { MEMORY_PROTOCOL_VERSION, MEMORY_SERVICE_VERSION } from "../version.js"; +import { SCHEMA_VERSION } from "../storage/schema.js"; +import { MEMORY_RUNTIME_BUILD_ID } from "../runtime-build.js"; const logger = createMemoryLogger("server"); @@ -65,6 +67,9 @@ async function runMemoryService(argv: string[], lifecycle: { const sqlitePath = options.dbPath ?? config.storage.sqlitePath; const serviceHome = resolve(dirname(configPath), "memory-service"); logger.info("service.starting", { + serviceVersion: MEMORY_SERVICE_VERSION, + supportedSchemaVersion: SCHEMA_VERSION, + buildId: MEMORY_RUNTIME_BUILD_ID, host, port, mode: config.storage.mode, @@ -129,6 +134,8 @@ async function runMemoryService(argv: string[], lifecycle: { pid: process.pid, endpoint: url, serviceVersion: MEMORY_SERVICE_VERSION, + buildId: MEMORY_RUNTIME_BUILD_ID, + supportedSchemaVersion: SCHEMA_VERSION, protocolVersion: MEMORY_PROTOCOL_VERSION, configPath, sqlitePath, diff --git a/Memory/src/server/memory-add-analytics.ts b/Memory/src/server/memory-add-analytics.ts index 3c2c84e12..08468f47a 100644 --- a/Memory/src/server/memory-add-analytics.ts +++ b/Memory/src/server/memory-add-analytics.ts @@ -42,6 +42,8 @@ export type MemoryDesktopScanAddBaseInput = { adapterId: string; /** Present for agent-source scan/import paths; omitted when unavailable. */ scanMode?: MemoryDesktopAddScanMode; + /** Defaults to L1. Skill imports pass Skill so they are not counted as conversation memory. */ + layer?: "L1" | "Skill"; conversationId?: string | null; turnId?: string | null; }; @@ -59,7 +61,7 @@ export function buildMemoryDesktopScanAddParams(input: MemoryDesktopScanAddBaseI adapter_id: input.adapterId, storage_backend: MEMORY_DESKTOP_ADD_STORAGE_BACKEND, mode: MEMORY_DESKTOP_ADD_MODE_AGENT_SOURCE_SCAN, - layer: MEMORY_DESKTOP_ADD_LAYER_L1, + layer: input.layer ?? MEMORY_DESKTOP_ADD_LAYER_L1, ...(input.scanMode ? { scan_mode: input.scanMode } : {}), ...(sessionIdHash ? { session_id_hash: sessionIdHash } : {}), ...(turnIdHash ? { turn_id_hash: turnIdHash } : {}), diff --git a/Memory/src/server/plugin-runtime-analytics.ts b/Memory/src/server/plugin-runtime-analytics.ts index b46c82fed..a002d7a37 100644 --- a/Memory/src/server/plugin-runtime-analytics.ts +++ b/Memory/src/server/plugin-runtime-analytics.ts @@ -42,12 +42,14 @@ const EXTERNAL_AGENT_SOURCE_IDS = new Set([ "opencode", "openclaw", "hermes", + "deepseek_harness", ]); const HOOK_AGENT_SOURCE_IDS = new Set(["cursor", "claude_code", "codex"]); -const NATIVE_PLUGIN_AGENT_SOURCE_IDS = new Set(["opencode", "openclaw", "hermes"]); +const NATIVE_PLUGIN_AGENT_SOURCE_IDS = new Set(["opencode", "openclaw", "hermes", "deepseek_harness"]); -const PLUGIN_RUNTIME_SOURCE = "memmy-memory"; +/** Ingest stores `memmy-agent`. `memmy-memory` is rejected before the row is written. */ +const PLUGIN_RUNTIME_SOURCE = "memmy-agent"; export type PluginRuntimeAnalytics = CliLifecycleAnalytics; @@ -58,6 +60,9 @@ export type PluginRuntimeBaseParams = AnalyticsParams & { export function createPluginRuntimeAnalytics(options: { getClientId?: () => string | null | undefined; + getInstallationId?: () => string | null | undefined; + getUserId?: () => string | null | undefined; + getUserMode?: () => string | null | undefined; fetchImpl?: typeof fetch; baseUrl?: string | null; } = {}): PluginRuntimeAnalytics { @@ -195,6 +200,14 @@ export function storedCountFromCompleteTurnResponse(response: unknown): number { return typeof record.l1MemoryId === "string" && record.l1MemoryId.trim() ? 1 : 0; } +/** Source-turn replies nest the turn result and repeat its ids on `existing`. */ +export function storedCountFromSourceTurnResponse(response: unknown): number { + if (!response || typeof response !== "object") return 0; + const record = response as { status?: unknown; result?: unknown }; + if (record.status !== "stored") return 0; + return storedCountFromCompleteTurnResponse(record.result); +} + export function toolCallCountFromCompleteTurnRequest(request: unknown): number | undefined { if (!request || typeof request !== "object") return undefined; const toolCalls = (request as { toolCalls?: unknown }).toolCalls; @@ -224,6 +237,16 @@ export function storedCountFromAddResponse(response: unknown): number { return typeof id === "string" && id.trim() ? 1 : 0; } +function isReplayCompleteTurn(response: unknown): boolean { + return Boolean(response && typeof response === "object" && (response as { duplicate?: unknown }).duplicate === true); +} + +function isReplayOrDeletedAdd(response: unknown): boolean { + if (!response || typeof response !== "object") return false; + const record = response as { duplicate?: unknown; status?: unknown }; + return record.duplicate === true || record.status === "deleted"; +} + export async function trackExternalHookRecall( analytics: PluginRuntimeAnalytics, input: PluginRuntimeAttribution, @@ -263,6 +286,11 @@ export async function trackExternalHookRecall( } } +/** + * Reports a legacy `POST /turns/:turnId/complete` write only after a new row is stored. + * Replays (`duplicate: true`) and empty captures emit nothing. + * `started` is emitted with `succeeded` so a replay does not leave an unmatched start. + */ export async function trackExternalHookCapture( analytics: PluginRuntimeAnalytics, input: PluginRuntimeAttribution, @@ -270,40 +298,162 @@ export async function trackExternalHookCapture( run: () => Promise | T, ): Promise { const agentSourceId = resolveExternalAgentSource(input); - if (!agentSourceId) return await run(); - - const baseParams = buildPluginRuntimeBaseParams(agentSourceId, input, "turn_complete"); - if (!baseParams) return await run(); - + const baseParams = agentSourceId + ? buildPluginRuntimeBaseParams(agentSourceId, input, "turn_complete") + : null; const fallbackMode = resolveCaptureFallbackMode(input.turnId); const startedAt = Date.now(); - analytics.track(PLUGIN_RUNTIME_EVENTS.hookCaptureStarted, compactAnalyticsParams({ - ...baseParams, - status: "started", - ...(fallbackMode ? { fallback_mode: fallbackMode } : {}), - })); try { const result = await run(); + const storedCount = isReplayCompleteTurn(result) ? 0 : storedCountFromCompleteTurnResponse(result); + if (!baseParams || storedCount <= 0) return result; const toolCallCount = toolCallCountFromCompleteTurnRequest(request); + analytics.track(PLUGIN_RUNTIME_EVENTS.hookCaptureStarted, compactAnalyticsParams({ + ...baseParams, + status: "started", + ...(fallbackMode ? { fallback_mode: fallbackMode } : {}), + })); analytics.track(PLUGIN_RUNTIME_EVENTS.hookCaptureSucceeded, compactAnalyticsParams({ ...baseParams, status: "succeeded", ...(fallbackMode ? { fallback_mode: fallbackMode } : {}), latency_ms: elapsedMs(startedAt), success: true, - stored_count: storedCountFromCompleteTurnResponse(result), + stored_count: storedCount, ...(typeof toolCallCount === "number" ? { tool_call_count: toolCallCount } : {}), })); return result; } catch (error) { - analytics.track(PLUGIN_RUNTIME_EVENTS.hookCaptureFailed, compactAnalyticsParams({ + if (baseParams) { + analytics.track(PLUGIN_RUNTIME_EVENTS.hookCaptureStarted, compactAnalyticsParams({ + ...baseParams, + status: "started", + ...(fallbackMode ? { fallback_mode: fallbackMode } : {}), + })); + analytics.track(PLUGIN_RUNTIME_EVENTS.hookCaptureFailed, compactAnalyticsParams({ + ...baseParams, + status: "failed", + ...(fallbackMode ? { fallback_mode: fallbackMode } : {}), + latency_ms: elapsedMs(startedAt), + success: false, + error_code: errorCodeFromUnknown(error), + })); + } + throw error; + } +} + +/** + * Reports a hook or native-plugin source-turn write only after a new row is stored. + * Replays (`existing`), skips, and scan-channel calls must not reach this helper. + * `started` is emitted with `succeeded` so a replay does not leave an unmatched start. + */ +export async function trackStoredExternalSourceTurnCapture( + analytics: PluginRuntimeAnalytics, + input: PluginRuntimeAttribution, + request: unknown, + run: () => Promise | T, +): Promise { + const agentSourceId = resolveExternalAgentSource(input); + const baseParams = agentSourceId + ? buildPluginRuntimeBaseParams(agentSourceId, input, "turn_complete") + : null; + const fallbackMode = resolveCaptureFallbackMode(input.turnId); + const startedAt = Date.now(); + try { + const result = await run(); + const storedCount = storedCountFromSourceTurnResponse(result); + if (!baseParams || storedCount <= 0) return result; + const toolCallCount = toolCallCountFromCompleteTurnRequest(request); + analytics.track(PLUGIN_RUNTIME_EVENTS.hookCaptureStarted, compactAnalyticsParams({ ...baseParams, - status: "failed", + status: "started", + ...(fallbackMode ? { fallback_mode: fallbackMode } : {}), + })); + analytics.track(PLUGIN_RUNTIME_EVENTS.hookCaptureSucceeded, compactAnalyticsParams({ + ...baseParams, + status: "succeeded", ...(fallbackMode ? { fallback_mode: fallbackMode } : {}), latency_ms: elapsedMs(startedAt), - success: false, - error_code: errorCodeFromUnknown(error), + success: true, + stored_count: storedCount, + ...(typeof toolCallCount === "number" ? { tool_call_count: toolCallCount } : {}), })); + return result; + } catch (error) { + if (baseParams) { + analytics.track(PLUGIN_RUNTIME_EVENTS.hookCaptureStarted, compactAnalyticsParams({ + ...baseParams, + status: "started", + ...(fallbackMode ? { fallback_mode: fallbackMode } : {}), + })); + analytics.track(PLUGIN_RUNTIME_EVENTS.hookCaptureFailed, compactAnalyticsParams({ + ...baseParams, + status: "failed", + ...(fallbackMode ? { fallback_mode: fallbackMode } : {}), + latency_ms: elapsedMs(startedAt), + success: false, + error_code: errorCodeFromUnknown(error), + })); + } + throw error; + } +} + +/** + * Reports a native-plugin `memmy_memory_add` only after a new memory is stored. + * Duplicate and deleted echoes still return the existing id, and emit nothing. + */ +export async function trackStoredExternalMemoryAdd( + analytics: PluginRuntimeAnalytics, + input: PluginRuntimeAttribution & { layer?: string }, + run: () => Promise | T, +): Promise { + const toolName = "memmy_memory_add"; + const agentSourceId = resolveExternalPluginToolCallSource(input); + const baseParams = agentSourceId + ? buildPluginRuntimeBaseParams(agentSourceId, input, toolName) + : null; + const layer = stringValue(input.layer); + const startedAt = Date.now(); + try { + const result = await run(); + const storedCount = isReplayOrDeletedAdd(result) ? 0 : storedCountFromAddResponse(result); + if (!baseParams || storedCount <= 0) return result; + analytics.track(PLUGIN_RUNTIME_EVENTS.toolCallStarted, compactAnalyticsParams({ + ...baseParams, + status: "started", + tool_name: toolName, + ...(layer ? { layer } : {}), + })); + analytics.track(PLUGIN_RUNTIME_EVENTS.toolCallSucceeded, compactAnalyticsParams({ + ...baseParams, + status: "succeeded", + tool_name: toolName, + ...(layer ? { layer } : {}), + latency_ms: elapsedMs(startedAt), + success: true, + stored_count: storedCount, + })); + return result; + } catch (error) { + if (baseParams) { + analytics.track(PLUGIN_RUNTIME_EVENTS.toolCallStarted, compactAnalyticsParams({ + ...baseParams, + status: "started", + tool_name: toolName, + ...(layer ? { layer } : {}), + })); + analytics.track(PLUGIN_RUNTIME_EVENTS.toolCallFailed, compactAnalyticsParams({ + ...baseParams, + status: "failed", + tool_name: toolName, + ...(layer ? { layer } : {}), + latency_ms: elapsedMs(startedAt), + success: false, + error_code: errorCodeFromUnknown(error), + })); + } throw error; } } diff --git a/Memory/src/service/memory-service.ts b/Memory/src/service/memory-service.ts index 0374a0f5d..bb44dab52 100644 --- a/Memory/src/service/memory-service.ts +++ b/Memory/src/service/memory-service.ts @@ -9,6 +9,7 @@ import { traceMetaFromMemory } from "../algorithm/plugin-algorithms.js"; import { PROJECT_VERSION } from "../cli/project-version.js"; +import { MEMORY_RUNTIME_BUILD_ID } from "../runtime-build.js"; import { MEMORY_CAPABILITIES, MEMORY_PROTOCOL_VERSION, @@ -915,6 +916,7 @@ export class MemoryService { return { ok: schema.version > 0 && ![summary, evolution, embedding].some((model) => Boolean(model.lastError)), serviceVersion: PROJECT_VERSION, + ...(MEMORY_RUNTIME_BUILD_ID ? { buildId: MEMORY_RUNTIME_BUILD_ID } : {}), protocolVersion: MEMORY_PROTOCOL_VERSION, viewerVersion: MEMORY_VIEWER_VERSION, viewerUrl: viewerUrlFromEndpoint(this.viewerEndpoint ?? this.config.storage.endpoint), diff --git a/Memory/src/service/session/session-turn-service.ts b/Memory/src/service/session/session-turn-service.ts index d0cec5955..181d12d5d 100644 --- a/Memory/src/service/session/session-turn-service.ts +++ b/Memory/src/service/session/session-turn-service.ts @@ -288,9 +288,6 @@ function endTopicDecisionFromRawTurn(rawTurn: RawTurnRecord): EndTopicDecision | } function rawTurnIsExcludedFromL1(rawTurn: RawTurnRecord): boolean { - if (endTopicDecisionFromRawTurn(rawTurn)) { - return true; - } const turnStart = isRecord(rawTurn.messagePayload?.turn_start) ? rawTurn.messagePayload.turn_start : undefined; @@ -1736,7 +1733,7 @@ export class SessionTurnService { const requestSourceMemoryIds = normalizeCompleteTurnSourceMemoryIds(request); const sourceMemoryIds = requestSourceMemoryIds.length > 0 ? requestSourceMemoryIds - : turnStartRecall?.injectedMemoryIds ?? []; + : request.recallApplied === false ? [] : turnStartRecall?.injectedMemoryIds ?? []; const completionRequest = sourceMemoryIds === requestSourceMemoryIds ? request : { ...request, sourceMemoryIds }; @@ -3046,7 +3043,9 @@ export class SessionTurnService { Boolean(rawTurn && (rawTurn.id === currentRawTurn.id || !seenRawTurnIds.has(rawTurn.id))) ) .filter((rawTurn) => isRecord(rawTurn.messagePayload?.turn_complete)) - .filter((rawTurn) => !endTopicDecisionFromRawTurn(rawTurn)) + // A bare end-topic command has nothing to remember. Any other end_topic turn can still + // carry feedback or instructions, so it goes through the same capture decision. + .filter((rawTurn) => !explicitEndTopicDecision(rawTurn.userText ?? "")) .filter((rawTurn) => this.deps.llm.isConfigured() || !rawTurnIsExcludedFromL1(rawTurn)) .sort((a, b) => Date.parse(a.createdAt) - Date.parse(b.createdAt)); return rawTurns.flatMap((rawTurn) => diff --git a/Memory/src/storage/repositories.ts b/Memory/src/storage/repositories.ts index cb60646b1..dee1bb81f 100644 --- a/Memory/src/storage/repositories.ts +++ b/Memory/src/storage/repositories.ts @@ -8053,6 +8053,11 @@ function evolutionJobPrioritySql(): string { WHEN job_type IN ('trace_summary', 'import_summary', 'embedding') AND ${onboardingFirstReportTarget} THEN -100 WHEN json_extract(payload_json, '$.source') = 'memory.processing.manual_retry' THEN 0 + WHEN job_type IN ('trace_summary', 'import_summary') + AND json_extract(payload_json, '$.source') IN ( + 'startup.processing_repair', + 'startup.placeholder_summary_repair' + ) THEN 90 WHEN job_type = 'trace_summary' OR (job_type = 'embedding' AND ${interactiveL1Target}) THEN 1 WHEN job_type = 'import_summary' diff --git a/Memory/src/types.ts b/Memory/src/types.ts index 85824a385..b4fc21768 100644 --- a/Memory/src/types.ts +++ b/Memory/src/types.ts @@ -354,6 +354,8 @@ export interface TurnStartRequest extends RequestEnvelope { } export interface TurnCompleteRequest extends RequestEnvelope { + /** False when automatic recall was not delivered to the model (for example, a host timeout). */ + recallApplied?: boolean; sessionId: string; episodeId?: string; query: string; @@ -573,6 +575,7 @@ export interface RawTurnRedactRequest extends RequestEnvelope { export interface HealthResponse { ok: boolean; + buildId?: string; serviceVersion: string; protocolVersion: number; viewerVersion: string; diff --git a/Memory/src/version.ts b/Memory/src/version.ts index 8437b6cf7..bef5b56b4 100644 --- a/Memory/src/version.ts +++ b/Memory/src/version.ts @@ -1,5 +1,5 @@ /** Independent Memory/Local Plugin release identity. */ -export const MEMORY_SERVICE_VERSION = "2.1.3"; +export const MEMORY_SERVICE_VERSION = "2.1.4"; export const MEMORY_VIEWER_VERSION = MEMORY_SERVICE_VERSION; export const MEMORY_PROTOCOL_VERSION = 1; diff --git a/Memory/tests/agent-source-runtime.test.ts b/Memory/tests/agent-source-runtime.test.ts index c0ec084fb..4474711f7 100644 --- a/Memory/tests/agent-source-runtime.test.ts +++ b/Memory/tests/agent-source-runtime.test.ts @@ -610,6 +610,62 @@ describe("standalone Agent source executor", () => { })); }); + it("emits add analytics only when a scanned skill is newly stored", async () => { + const root = tempRoot(); + const codexRoot = join(root, ".codex"); + for (const sourceSkillId of ["deleted-skill", "duplicate-skill", "failing-skill", "fresh-skill"]) { + const skillPath = join(codexRoot, "skills", sourceSkillId, "SKILL.md"); + mkdirSync(join(skillPath, ".."), { recursive: true }); + writeFileSync(skillPath, `---\nname: ${sourceSkillId}\nversion: 1\n---\n\n${sourceSkillId}\n`); + } + vi.stubEnv("CODEX_HOME", codexRoot); + const events: Array<{ name: string; payload: Record }> = []; + const service = { + addMemory: vi.fn((input: { sourceSkillId?: string }) => { + if (input.sourceSkillId === "duplicate-skill") return { id: "dup", duplicate: true, status: "activated" }; + if (input.sourceSkillId === "deleted-skill") return { id: "gone", status: "deleted" }; + if (input.sourceSkillId === "failing-skill") throw new Error("skill import failed"); + return { id: "fresh", status: "activated" }; + }), + enqueuePendingImportSummaries: vi.fn() + } as unknown as MemoryService; + const adapter: SourceAdapter = { + descriptor: { sourceId: "codex", displayName: "Codex", builtin: true, dataPath: codexRoot }, + detect: async () => true, + async *scan() {} + }; + const executor = createAgentSourceExecutor({ + service, + configPath: join(root, "config.yaml"), + statePath: join(root, "agent-sources.json"), + sourceRegistry: createSourceRegistry([adapter]), + memoryAddAnalytics: { + trackAddStarted(input) { events.push({ name: "started", payload: { ...input } }); }, + trackAddSucceeded(input) { events.push({ name: "succeeded", payload: { ...input } }); }, + trackAddFailed(input) { events.push({ name: "failed", payload: { ...input } }); } + } + }); + + await executor.startScan({ sourceId: "codex", mode: "incremental" }); + await waitForScan(executor); + + expect(events.map((event) => event.name)).toEqual(["started", "failed", "started", "succeeded"]); + expect(events[1]?.payload).toMatchObject({ + adapterId: "agent-source:codex", + conversationId: "skill:failing-skill", + layer: "Skill", + scanMode: "incremental" + }); + expect(events[3]?.payload).toMatchObject({ + adapterId: "agent-source:codex", + conversationId: "skill:fresh-skill", + layer: "Skill", + scanMode: "incremental", + storedCount: 1 + }); + await executor.dispose(); + }); + it("installs and removes a real Cursor Hook without Memmy Desktop", async () => { const root = tempRoot(); const cursorRoot = join(root, ".cursor"); diff --git a/Memory/tests/analytics-contract.test.ts b/Memory/tests/analytics-contract.test.ts index 60b7168e9..6fc3e32db 100644 --- a/Memory/tests/analytics-contract.test.ts +++ b/Memory/tests/analytics-contract.test.ts @@ -1,5 +1,9 @@ import { describe, expect, it, vi } from "vitest"; -import { postAnalyticsEvents } from "../src/cli/analytics.js"; +import { + postAnalyticsEvents, + resolveCliAnalyticsParams, + resolveCommandIdentity, +} from "../src/cli/analytics.js"; describe("CLI analytics event contract", () => { it("posts the installation id alongside the existing GA-style event contract", async () => { @@ -31,7 +35,7 @@ describe("CLI analytics event contract", () => { eventName: "memmy_cli_completed", params: expect.objectContaining({ engagement_time_msec: 100, - success: true, + success: 1, user_mode: "byok", source: "memmy-memory", app_env: "prod", @@ -43,3 +47,60 @@ describe("CLI analytics event contract", () => { expect(body.events[0]?.params).not.toHaveProperty("user_id"); }); }); + +describe("CLI analytics command identity", () => { + it.each([ + ["search", "some private query"], + ["add", ""], + ["get", "mem_123"], + ["delete", "mem_123"], + ])("does not report the %s argument as a command action", (group, argument) => { + expect(resolveCommandIdentity([group, argument])).toEqual({ + command_group: group, + has_session_id: false, + has_turn_id: false, + }); + }); + + it("reports only known subcommands as command actions", () => { + expect(resolveCommandIdentity(["session", "close", "session-1"])).toEqual({ + command_group: "session", + command_action: "close", + has_session_id: true, + has_turn_id: false, + }); + expect(resolveCommandIdentity(["turn", "complete", "turn-1"])).toEqual({ + command_group: "turn", + command_action: "complete", + has_session_id: false, + has_turn_id: true, + }); + expect(resolveCommandIdentity(["service", "start"])).toMatchObject({ command_action: "start" }); + expect(resolveCommandIdentity(["raw", "GET", "/api/v1/health"])).toMatchObject({ command_action: "GET" }); + expect(resolveCommandIdentity(["raw", "get", "/api/v1/health"])).toMatchObject({ command_action: "get" }); + expect(resolveCommandIdentity(["session", "whatever-text"])).toEqual({ + command_group: "session", + has_session_id: false, + has_turn_id: false, + }); + }); + + it("does not report an unknown command verbatim", () => { + expect(resolveCommandIdentity(["some free text"])).toEqual({ + command_group: "unknown", + has_session_id: false, + has_turn_id: false, + }); + }); + + it("keeps CLI arguments out of the event params", () => { + const params = resolveCliAnalyticsParams( + ["search", "some private query", "--url", "http://memmy.test"], + { env: {} }, + ); + + expect(params).toMatchObject({ command_group: "search" }); + expect(params).not.toHaveProperty("command_action"); + expect(JSON.stringify(params)).not.toContain("some private query"); + }); +}); diff --git a/Memory/tests/cli-setup.test.ts b/Memory/tests/cli-setup.test.ts index da0e89db4..05ac55649 100644 --- a/Memory/tests/cli-setup.test.ts +++ b/Memory/tests/cli-setup.test.ts @@ -70,12 +70,9 @@ describe("memmy-memory CLI setup commands", () => { enableMemorySearch: true, enableQueryRewrite: false }, - embedding: { - mode: "local", - provider: "local" - } } }); + expect(saved.memmyMemory.embedding).toBeUndefined(); expect(existsSync(dbPath)).toBe(false); }); @@ -429,11 +426,8 @@ describe("memmy-memory CLI setup commands", () => { enableMemorySearch: true, enableQueryRewrite: false }, - embedding: { - mode: "local", - provider: "local" - } }); + expect(saved.memmyMemory.embedding).toBeUndefined(); expect(existsSync(dbPath)).toBe(false); }); @@ -473,7 +467,7 @@ describe("memmy-memory CLI setup commands", () => { expect(saved.memmyMemory.summary.model).toBe("memmy-model"); }); - it("preserves embedding modes during setup", async () => { + it("removes legacy embedding modes during setup", async () => { for (const mode of ["cloud", "local", "custom"]) { const root = tempRoot(); const configPath = join(root, "config.yaml"); @@ -495,21 +489,23 @@ describe("memmy-memory CLI setup commands", () => { }); const saved = YAML.parse(readFileSync(configPath, "utf8")); - expect(saved.memmyMemory.embedding).toEqual({ mode }); + expect(saved.memmyMemory.embedding).toBeUndefined(); } }); - it("preserves embedding connections during setup", async () => { + it("moves embedding connections into the model catalog during setup", async () => { for (const embedding of [ { provider: "openai_compatible", - endpoint: "https://example.com/v1" + endpoint: "https://example.com/v1", + model: "text-embedding-3-small" }, { mode: "custom", custom: { provider: "openai_compatible", - endpoint: "https://example.com/v1" + endpoint: "https://example.com/v1", + model: "text-embedding-3-small" } } ]) { @@ -528,7 +524,8 @@ describe("memmy-memory CLI setup commands", () => { ] }); const saved = YAML.parse(readFileSync(configPath, "utf8")); - expect(saved.memmyMemory.embedding).toEqual(embedding); + expect(saved.memmyMemory.embedding).toBeUndefined(); + expect(saved.modelAssignments.byok.embedding).toEqual(expect.any(String)); } }); diff --git a/Memory/tests/config.test.ts b/Memory/tests/config.test.ts index 0afb417ea..be064876e 100644 --- a/Memory/tests/config.test.ts +++ b/Memory/tests/config.test.ts @@ -434,6 +434,51 @@ describe("memmy memory config", () => { expect(config.embedding.selectionError).toBeUndefined(); }); + it("resolves catalog embedding assignments when the legacy runtime field is absent", () => { + const root = tempRoot(); + const configPath = join(root, "config.yaml"); + writeFileSync(configPath, YAML.stringify({ + providers: { + openai: { + apiKey: "sk-embedding", + endpoints: { + embeddings: { + apiBase: "https://embedding.example/v1", + protocol: "openai-embeddings" + } + } + } + }, + modelPresets: { + embeddings: { + provider: "openai", + endpoint: "embeddings", + model: "text-embedding-3-small", + source: "byok", + capabilities: ["embedding"] + } + }, + modelAssignments: { + byok: { embedding: "embeddings" }, + account: {} + }, + app: { userMode: "byok" }, + memmyMemory: {} + })); + + const { config } = loadMemmyConfig(configPath); + + expect(config.embedding).toMatchObject({ + mode: "custom", + provider: "openai_compatible", + sourceProvider: "openai", + endpoint: "https://embedding.example/v1", + model: "text-embedding-3-small", + apiKey: "sk-embedding" + }); + expect(config.embedding.selectionError).toBeUndefined(); + }); + it("rejects a built-in local Embedding assignment owned by another account", () => { const root = tempRoot(); const configPath = join(root, "config.yaml"); diff --git a/Memory/tests/contract/memory-rest-service.test.ts b/Memory/tests/contract/memory-rest-service.test.ts index 1a4c80ea5..70a011210 100644 --- a/Memory/tests/contract/memory-rest-service.test.ts +++ b/Memory/tests/contract/memory-rest-service.test.ts @@ -438,6 +438,30 @@ describe("MemoryService / REST contract", () => { db.close(); }); + it("forwards recallApplied=false through HTTP without attributing undelivered recall", async () => { + const { db, service } = createTestService(); + const opened = service.openSession({ namespace: { source: "memmy-agent", profileId: "default" } }); + const started = await service.startTurn({ sessionId: opened.sessionId, turnId: "late-recall", query: "Implement the memory timeout repair" }); + db.db.prepare("UPDATE recall_events SET injected_memory_ids_json = ? WHERE id = ?").run('["undelivered-memory"]', started.searchEventId); + const server = createMemoryHttpServer({ service }); + await withServerClosed(server, async () => { + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + const address = server.address(); + if (!address || typeof address === "string") throw new Error("expected TCP address"); + const response = await fetch(`http://127.0.0.1:${address.port}/api/v1/turns/late-recall/complete`, { + method: "POST", headers: { "content-type": "application/json" }, + body: JSON.stringify({ sessionId: opened.sessionId, query: "Implement the memory timeout repair", + answer: "The model answered after memory timed out.", sourceMemoryIds: [], recallApplied: false }), + }); + expect(response.status).toBe(200); + const result = await response.json() as { rawTurnId: string; l1MemoryIds: string[] }; + expect(result.l1MemoryIds).toHaveLength(1); + const row = db.db.prepare("SELECT source_memory_ids_json FROM raw_turns WHERE id = ?").get(result.rawTurnId) as { source_memory_ids_json: string }; + expect(JSON.parse(row.source_memory_ids_json)).toEqual([]); + }); + db.close(); + }); + it("preserves lifecycle routing fields across the REST boundary", async () => { const { db, service } = createTestService(); const analyticsEvents: string[] = []; @@ -688,6 +712,72 @@ describe("MemoryService / REST contract", () => { db.close(); }); + it("auto-drains L3 World Model jobs frozen by a REST session.close with no open episode", async () => { + const { db, service } = createTestService(); + const opened = service.openSession({ + l3WorldModelProtocolVersion: 2, + l3WorldModelTransition: "resume_only", + namespace: { + source: "codex", + profileId: "default", + sessionKey: "auto-worker-l3-close-session", + userId: "auto-worker-l3-close-user" + } + }); + const first = service.completeTurn("turn-auto-worker-l3-close", { + sessionId: opened.sessionId, + query: "Configure nginx TLS for the service", + answer: "Use port 443 and verify the certificate chain." + }); + const ended = service.completeTurn("turn-auto-worker-l3-close-end", { + sessionId: opened.sessionId, + query: "结束会话", + answer: "好的,本话题到这里结束。" + }); + expect(ended.closedEpisodeIds).toEqual([first.episodeId]); + await service.runWorkerOnce(20); + const l3Jobs = () => db.db.prepare( + `SELECT status, attempts + FROM evolution_jobs + WHERE job_type = 'l3_world_model_update'` + ).all() as Array<{ status: string; attempts: number }>; + expect(l3Jobs()).toEqual([]); + + // Keep the startup fallback out of the wait window so only session.close can wake the worker. + const server = createMemoryHttpServer({ service, workerStartupFallbackMs: 60_000 }); + await withServerClosed(server, async () => { + await new Promise((resolve) => { + server.listen(0, "127.0.0.1", resolve); + }); + const address = server.address(); + if (!address || typeof address === "string") { + throw new Error("expected TCP address"); + } + const baseUrl = `http://127.0.0.1:${address.port}/api/v1`; + + const closeResponse = await fetch( + `${baseUrl}/sessions/${encodeURIComponent(opened.sessionId)}/close`, + { + method: "POST", + headers: { "content-type": "application/json" }, + body: "{}" + } + ); + expect(closeResponse.status).toBe(200); + await expect(closeResponse.json()).resolves.toMatchObject({ + ok: true, + sessionId: opened.sessionId, + status: "closed", + closedEpisodeIds: [] + }); + expect(l3Jobs()).toHaveLength(1); + + await waitFor(() => l3Jobs().every((job) => job.status !== "queued" && job.attempts > 0)); + + }); + db.close(); + }); + it("auto-closes idle episodes after a REST turn.complete", async () => { const { db, service } = createTestService(); const idleSession = service.openSession({ diff --git a/Memory/tests/desktop-analytics-identity.test.ts b/Memory/tests/desktop-analytics-identity.test.ts new file mode 100644 index 000000000..b5b8cd5f7 --- /dev/null +++ b/Memory/tests/desktop-analytics-identity.test.ts @@ -0,0 +1,127 @@ +import { mkdtempSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { describe, expect, it, vi } from "vitest"; +import { readDesktopAnalyticsIdentity } from "../src/server/desktop-analytics-identity.js"; +import { createMemoryDesktopAddAnalytics } from "../src/server/memory-add-analytics.js"; +import { + PLUGIN_RUNTIME_EVENTS, + createPluginRuntimeAnalytics, +} from "../src/server/plugin-runtime-analytics.js"; + +describe("desktop analytics identity", () => { + it("reads the logged-in account from the config file on each call", async () => { + const configPath = writeConfig({ + userMode: "account", + userId: "account-user", + cloudUuid: "cloud-1", + }); + expect(readDesktopAnalyticsIdentity(configPath)).toEqual({ + userId: "account-user", + userMode: "account", + }); + + writeConfig({ + userMode: "byok", + userId: "account-user", + cloudUuid: "cloud-1", + }, configPath); + expect(readDesktopAnalyticsIdentity(configPath)).toEqual({ + userId: "account-user", + userMode: "byok", + }); + }); + + it("omits the account id unless cloudUuid is present", () => { + const configPath = writeConfig({ + userMode: "account", + userId: "account-user", + }); + expect(readDesktopAnalyticsIdentity(configPath)).toEqual({ + userId: null, + userMode: "account", + }); + expect(readDesktopAnalyticsIdentity(join(tmpdir(), "missing-memmy-config.yaml"))).toEqual({ + userId: null, + userMode: null, + }); + }); + + it("does not report local-user as an account id", () => { + const configPath = writeConfig({ + userMode: "byok", + userId: "local-user", + cloudUuid: "cloud-1", + }); + expect(readDesktopAnalyticsIdentity(configPath).userId).toBeNull(); + }); + + it("posts hook and scan events with the identity currently in the config", async () => { + const configPath = writeConfig({ + userMode: "account", + userId: "account-user", + cloudUuid: "cloud-1", + }); + const fetchImpl = vi.fn(async () => new Response(JSON.stringify({ code: 0 }), { status: 200 })); + const identity = () => readDesktopAnalyticsIdentity(configPath); + const shared = { + fetchImpl, + baseUrl: "https://cloud.example.com", + getClientId: () => "1234567890.1234567890", + getInstallationId: () => "install-1", + getUserId: () => identity().userId, + getUserMode: () => identity().userMode, + }; + const plugin = createPluginRuntimeAnalytics(shared); + const scan = createMemoryDesktopAddAnalytics(shared); + + await plugin.trackAwait(PLUGIN_RUNTIME_EVENTS.hookCaptureStarted, { source_id: "cursor" }); + scan.trackAddStarted({ adapterId: "agent-source:cursor", scanMode: "incremental" }); + await scan.flush(); + + expect(postedIdentity(fetchImpl, 0)).toMatchObject({ + userId: "account-user", + user_id: "account-user", + user_mode: "account", + }); + expect(postedIdentity(fetchImpl, 1)).toMatchObject({ + userId: "account-user", + user_id: "account-user", + user_mode: "account", + }); + + writeConfig({ userMode: "byok" }, configPath); + await plugin.trackAwait(PLUGIN_RUNTIME_EVENTS.hookCaptureStarted, { source_id: "cursor" }); + expect(postedIdentity(fetchImpl, 2)).toMatchObject({ user_mode: "byok" }); + expect(postedIdentity(fetchImpl, 2).userId).toBeUndefined(); + }); +}); + +function writeConfig( + app: { userMode?: string; userId?: string; cloudUuid?: string }, + configPath = join(mkdtempSync(join(tmpdir(), "memmy-analytics-")), "config.yaml"), +): string { + const lines = ["app:"]; + if (app.userMode) lines.push(` userMode: ${app.userMode}`); + if (app.userId) lines.push(` userId: ${app.userId}`); + if (app.cloudUuid) lines.push(` cloudUuid: ${app.cloudUuid}`); + writeFileSync(configPath, `${lines.join("\n")}\n`); + return configPath; +} + +function postedIdentity(fetchImpl: { mock: { calls: unknown[][] } }, call: number): { + userId?: string; + user_id?: string; + user_mode?: string; +} { + const init = fetchImpl.mock.calls[call]?.[1] as { body?: string } | undefined; + const body = JSON.parse(String(init?.body)) as { + userId?: string; + events: Array<{ params: { user_id?: string; user_mode?: string } }>; + }; + return { + userId: body.userId, + user_id: body.events[0]?.params.user_id, + user_mode: body.events[0]?.params.user_mode, + }; +} diff --git a/Memory/tests/llm-json-mode.test.ts b/Memory/tests/llm-json-mode.test.ts new file mode 100644 index 000000000..c194ac75b --- /dev/null +++ b/Memory/tests/llm-json-mode.test.ts @@ -0,0 +1,88 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import type { LlmConfig } from "../src/config/index.js"; +import { createLlmClient } from "../src/model/llm.js"; + +afterEach(() => { + vi.unstubAllGlobals(); +}); + +describe("OpenAI-compatible JSON response formats", () => { + it("falls back to text when an endpoint rejects json_object and remembers the capability", async () => { + const fetchMock = vi.fn() + .mockResolvedValueOnce(new Response(JSON.stringify({ + error: "'response_format.type' must be 'json_schema' or 'text'" + }), { + status: 400, + headers: { "content-type": "application/json" } + })) + .mockImplementation(async () => jsonResponse({ + choices: [{ message: { content: '{"ok":true}' }, finish_reason: "stop" }] + })); + vi.stubGlobal("fetch", fetchMock); + const client = createLlmClient(llmConfig()); + + await expect(client.completeJson<{ ok: boolean }>( + [{ role: "user", content: "summarize" }], + { operation: "capture.summarize" } + )).resolves.toEqual({ ok: true }); + await expect(client.completeJson<{ ok: boolean }>( + [{ role: "user", content: "reflect" }], + { operation: "capture.alpha.reflection.score.v1" } + )).resolves.toEqual({ ok: true }); + + expect(fetchMock).toHaveBeenCalledTimes(3); + expect(requestBody(fetchMock, 0)).toMatchObject({ + response_format: { type: "json_object" } + }); + expect(requestBody(fetchMock, 1)).toMatchObject({ + response_format: { type: "text" } + }); + expect(requestBody(fetchMock, 2)).toMatchObject({ + response_format: { type: "text" } + }); + }); + + it("does not retry unrelated invalid requests", async () => { + const fetchMock = vi.fn(async () => new Response(JSON.stringify({ + error: { message: "model does not exist" } + }), { + status: 400, + headers: { "content-type": "application/json" } + })); + vi.stubGlobal("fetch", fetchMock); + const client = createLlmClient(llmConfig()); + + await expect(client.completeJson( + [{ role: "user", content: "summarize" }], + { operation: "capture.summarize" } + )).rejects.toThrow("model does not exist"); + expect(fetchMock).toHaveBeenCalledTimes(1); + }); +}); + +function llmConfig(): LlmConfig { + return { + provider: "openai_compatible", + endpoint: "http://127.0.0.1:1234/v1", + model: "local-model", + apiKey: "lm-studio", + enableThinking: false, + temperature: 0, + maxTokens: 512, + timeoutMs: 5_000, + maxRetries: 0, + malformedRetries: 0 + }; +} + +function requestBody(fetchMock: ReturnType>, index: number): Record { + const [, init] = fetchMock.mock.calls[index] as [Parameters[0], RequestInit]; + return JSON.parse(String(init.body)) as Record; +} + +function jsonResponse(body: unknown): Response { + return new Response(JSON.stringify(body), { + status: 200, + headers: { "content-type": "application/json" } + }); +} diff --git a/Memory/tests/memory-add-analytics.test.ts b/Memory/tests/memory-add-analytics.test.ts index c49c38b62..2e9e9f4cf 100644 --- a/Memory/tests/memory-add-analytics.test.ts +++ b/Memory/tests/memory-add-analytics.test.ts @@ -25,6 +25,12 @@ describe("memory-add-analytics", () => { session_id_hash: hashAnalyticsId("conv-1"), turn_id_hash: hashAnalyticsId("cursor:abc"), }); + expect(buildMemoryDesktopScanAddParams({ + adapterId: "agent-source:cursor", + layer: "Skill", + conversationId: "skill:review-code", + turnId: "skill:review-code:v2", + }).layer).toBe("Skill"); }); it("tracks started/succeeded/failed desktop add events", async () => { @@ -82,13 +88,13 @@ describe("memory-add-analytics", () => { adapter_id: "agent-source:cursor", }); expect(body.events[1]?.params).toMatchObject({ - success: true, + success: 1, stored_count: 1, duration_ms: 12, scan_mode: "full", }); expect(body.events[2]?.params).toMatchObject({ - success: false, + success: 0, error_code: "boom", duration_ms: 3, scan_mode: "incremental", diff --git a/Memory/tests/plugin-runtime-source-turn-analytics.test.ts b/Memory/tests/plugin-runtime-source-turn-analytics.test.ts new file mode 100644 index 000000000..f95983ecd --- /dev/null +++ b/Memory/tests/plugin-runtime-source-turn-analytics.test.ts @@ -0,0 +1,243 @@ +import { describe, expect, it, vi } from "vitest"; +import { + PLUGIN_RUNTIME_EVENTS, + createPluginRuntimeAnalytics, + storedCountFromSourceTurnResponse, + trackExternalHookCapture, + trackStoredExternalMemoryAdd, + trackStoredExternalSourceTurnCapture, + type PluginRuntimeAnalytics, +} from "../src/server/plugin-runtime-analytics.js"; + +describe("stored source-turn capture analytics", () => { + it("emits hook capture only after a new source turn is stored", async () => { + const events: Array<{ name: string; params: Record }> = []; + const stored = { + status: "stored", + result: { l1MemoryIds: ["l1-a", "l1-b"], l1MemoryId: "l1-a" }, + }; + const result = await trackStoredExternalSourceTurnCapture( + recorder(events), + { source: "cursor", adapterId: "memmy-cursor-hook", turnId: "turn-1" }, + { toolCalls: [{ id: "call-1" }] }, + () => stored, + ); + + expect(result).toBe(stored); + expect(events.map((event) => event.name)).toEqual([ + PLUGIN_RUNTIME_EVENTS.hookCaptureStarted, + PLUGIN_RUNTIME_EVENTS.hookCaptureSucceeded, + ]); + expect(events[1]?.params).toMatchObject({ + source_id: "cursor", + source_kind: "hook", + adapter_id: "memmy-cursor-hook", + hook_name: "turn_complete", + success: 1, + stored_count: 2, + tool_call_count: 1, + }); + }); + + it("does not emit when the source turn already exists", async () => { + const events: Array<{ name: string; params: Record }> = []; + const existing = { + status: "existing", + result: { l1MemoryIds: ["l1-a"], duplicate: true }, + }; + + await trackStoredExternalSourceTurnCapture( + recorder(events), + { source: "opencode", adapterId: "memmy-opencode-plugin", turnId: "turn-1" }, + {}, + () => existing, + ); + + expect(storedCountFromSourceTurnResponse(existing)).toBe(0); + expect(events).toEqual([]); + }); + + it("emits capture failure only when the write throws", async () => { + const events: Array<{ name: string; params: Record }> = []; + + await expect(trackStoredExternalSourceTurnCapture( + recorder(events), + { source: "deepseek_harness", adapterId: "memmy-deepseek-harness-plugin", turnId: "turn-1" }, + {}, + () => { + throw new Error("capture failed"); + }, + )).rejects.toThrow("capture failed"); + + expect(events.map((event) => event.name)).toEqual([ + PLUGIN_RUNTIME_EVENTS.hookCaptureStarted, + PLUGIN_RUNTIME_EVENTS.hookCaptureFailed, + ]); + expect(events[1]?.params).toMatchObject({ + source_id: "deepseek_harness", + source_kind: "native_plugin", + success: 0, + error_code: "capture failed", + }); + }); +}); + +describe("legacy turn complete capture analytics", () => { + it("emits hook capture only after a new turn stores memories", async () => { + const events: Array<{ name: string; params: Record }> = []; + const stored = { l1MemoryIds: ["l1-a"], l1MemoryId: "l1-a" }; + + const result = await trackExternalHookCapture( + recorder(events), + { source: "cursor", adapterId: "memmy-cursor-hook", turnId: "turn-1" }, + { toolCalls: [{ id: "call-1" }] }, + () => stored, + ); + + expect(result).toBe(stored); + expect(events.map((event) => event.name)).toEqual([ + PLUGIN_RUNTIME_EVENTS.hookCaptureStarted, + PLUGIN_RUNTIME_EVENTS.hookCaptureSucceeded, + ]); + expect(events[1]?.params).toMatchObject({ + source_id: "cursor", + success: 1, + stored_count: 1, + tool_call_count: 1, + }); + }); + + it("does not emit when the completed turn is a replay", async () => { + const events: Array<{ name: string; params: Record }> = []; + + await trackExternalHookCapture( + recorder(events), + { source: "cursor", adapterId: "memmy-cursor-hook", turnId: "turn-1" }, + {}, + () => ({ l1MemoryIds: ["l1-a"], l1MemoryId: "l1-a", duplicate: true }), + ); + + expect(events).toEqual([]); + }); + + it("emits capture failure only when the legacy complete throws", async () => { + const events: Array<{ name: string; params: Record }> = []; + + await expect(trackExternalHookCapture( + recorder(events), + { source: "codex", adapterId: "memmy-codex-hook", turnId: "turn-1" }, + {}, + () => { + throw new Error("complete failed"); + }, + )).rejects.toThrow("complete failed"); + + expect(events.map((event) => event.name)).toEqual([ + PLUGIN_RUNTIME_EVENTS.hookCaptureStarted, + PLUGIN_RUNTIME_EVENTS.hookCaptureFailed, + ]); + expect(events[1]?.params).toMatchObject({ + success: 0, + error_code: "complete failed", + }); + }); +}); + +describe("native plugin memory add analytics", () => { + it("emits tool call analytics only after a new memory is stored", async () => { + const events: Array<{ name: string; params: Record }> = []; + const stored = { id: "mem-1", status: "activated" }; + + const result = await trackStoredExternalMemoryAdd( + recorder(events), + { source: "opencode", adapterId: "memmy-opencode-plugin", layer: "L1" }, + () => stored, + ); + + expect(result).toBe(stored); + expect(events.map((event) => event.name)).toEqual([ + PLUGIN_RUNTIME_EVENTS.toolCallStarted, + PLUGIN_RUNTIME_EVENTS.toolCallSucceeded, + ]); + expect(events[1]?.params).toMatchObject({ + source_id: "opencode", + source_kind: "native_plugin", + tool_name: "memmy_memory_add", + layer: "L1", + success: 1, + stored_count: 1, + }); + }); + + it("does not emit when the added memory already exists or was deleted", async () => { + const events: Array<{ name: string; params: Record }> = []; + + await trackStoredExternalMemoryAdd( + recorder(events), + { source: "hermes", adapterId: "memmy-hermes-plugin" }, + () => ({ id: "mem-1", duplicate: true }), + ); + await trackStoredExternalMemoryAdd( + recorder(events), + { source: "hermes", adapterId: "memmy-hermes-plugin" }, + () => ({ id: "skill-1", status: "deleted" }), + ); + + expect(events).toEqual([]); + }); + + it("emits tool call failure only when the add throws", async () => { + const events: Array<{ name: string; params: Record }> = []; + + await expect(trackStoredExternalMemoryAdd( + recorder(events), + { source: "openclaw", adapterId: "memmy-openclaw-plugin" }, + () => { + throw new Error("add failed"); + }, + )).rejects.toThrow("add failed"); + + expect(events.map((event) => event.name)).toEqual([ + PLUGIN_RUNTIME_EVENTS.toolCallStarted, + PLUGIN_RUNTIME_EVENTS.toolCallFailed, + ]); + expect(events[1]?.params).toMatchObject({ + success: 0, + error_code: "add failed", + }); + }); +}); + +describe("plugin runtime analytics source", () => { + it("posts hook writes with the source ingest stores", async () => { + const fetchImpl = vi.fn(async () => new Response(JSON.stringify({ code: 0 }), { status: 200 })); + const analytics = createPluginRuntimeAnalytics({ + fetchImpl, + baseUrl: "https://cloud.example.com", + getClientId: () => "1234567890.1234567890", + }); + + await analytics.trackAwait(PLUGIN_RUNTIME_EVENTS.hookCaptureStarted, { + source_id: "cursor", + source_kind: "hook", + status: "started", + }); + + const body = JSON.parse(String(fetchImpl.mock.calls[0]?.[1]?.body)) as { + events: Array<{ params: { source?: string } }>; + }; + expect(body.events[0]?.params.source).toBe("memmy-agent"); + }); +}); + +function recorder(events: Array<{ name: string; params: Record }>): PluginRuntimeAnalytics { + return { + track(eventName, params = {}) { + events.push({ name: eventName, params }); + }, + async trackAwait(eventName, params = {}) { + events.push({ name: eventName, params }); + }, + async flush() {}, + }; +} diff --git a/Memory/tests/project-version.test.ts b/Memory/tests/project-version.test.ts index 24c51d578..7ba8f5bf2 100644 --- a/Memory/tests/project-version.test.ts +++ b/Memory/tests/project-version.test.ts @@ -14,7 +14,7 @@ describe("Memory service version", () => { readFileSync(resolve(fileURLToPath(import.meta.url), "../../viewer/package.json"), "utf8") ); - expect(PROJECT_VERSION).toBe("2.1.3"); + expect(PROJECT_VERSION).toBe("2.1.4"); expect(PROJECT_VERSION).toBe(manifest.version); expect(cliManifest.version).toBe(manifest.version); expect(viewerManifest.version).toBe(manifest.version); diff --git a/Memory/tests/runtime-build.test.ts b/Memory/tests/runtime-build.test.ts new file mode 100644 index 000000000..4f8455312 --- /dev/null +++ b/Memory/tests/runtime-build.test.ts @@ -0,0 +1,29 @@ +import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { pathToFileURL } from "node:url"; +import { spawnSync } from "node:child_process"; +import { ModuleKind, ScriptTarget, transpileModule } from "typescript"; +import { afterEach, describe, expect, it } from "vitest"; + +const roots: string[] = []; +afterEach(() => { for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }); }); + +describe("packaged Memory build identity", () => { + it.each([true, false])("reads the installed build identity with a manifest present=%s", (hasManifest) => { + const root = mkdtempSync(join(tmpdir(), "memmy-runtime-build-")); + roots.push(root); + const modulePath = join(root, "dist/src/runtime-build.mjs"); + mkdirSync(join(root, "dist/src"), { recursive: true }); + const source = readFileSync(new URL("../src/runtime-build.ts", import.meta.url), "utf8"); + writeFileSync(modulePath, transpileModule(source, { + compilerOptions: { module: ModuleKind.ESNext, target: ScriptTarget.ES2022 } + }).outputText); + if (hasManifest) writeFileSync(join(root, "memory-runtime.json"), JSON.stringify({ buildId: "a".repeat(64) })); + const child = spawnSync(process.execPath, ["--input-type=module", "--eval", + `const runtime = await import(${JSON.stringify(pathToFileURL(modulePath).href)}); process.stdout.write(runtime.MEMORY_RUNTIME_BUILD_ID ?? "development");` + ], { encoding: "utf8" }); + expect(child.status, child.stderr).toBe(0); + expect(child.stdout).toBe(hasManifest ? "a".repeat(64) : "development"); + }); +}); diff --git a/Memory/tests/runtime-installer.test.ts b/Memory/tests/runtime-installer.test.ts index d4ae0282f..32c3ef973 100644 --- a/Memory/tests/runtime-installer.test.ts +++ b/Memory/tests/runtime-installer.test.ts @@ -126,6 +126,145 @@ describe("standalone Memory runtime installer", () => { const launcher = readFileSync(join(home, "bin", process.platform === "win32" ? "memmy-memory-service.cmd" : "memmy-memory-service"), "utf8"); expect(launcher).toContain("/original/node"); expect(launcher).not.toContain("/desktop/electron"); + }, 15_000); + + it.each([true, false])("replaces changed same-version builds (preferInstalledCompatible=%s) without touching data", async (preferInstalledCompatible) => { + const root = tempRoot(); + const home = join(root, "home"); + const legacy = createRuntimeDirectory(root, "2.1.3"); + const flags = { home, skipServiceRegistration: true, skipHealthCheck: true, agents: [] }; + const old = await installMemoryRuntime({ ...flags, runtimeDirectory: legacy }); + const dbPath = join(home, "memory-service", "memory.sqlite"); + writeFileSync(dbPath, "existing schema 9 database"); + const bundle = createRuntimeDirectory(root, "2.1.3", { buildId: "a".repeat(64), schemaVersion: 9 }); + writeFileSync(join(bundle, "dist/src/server/index.js"), "// schema 9 implementation"); + const upgraded = await installMemoryRuntime({ ...flags, runtimeDirectory: bundle, preferInstalledCompatible }); + expect(upgraded).toMatchObject({ buildId: "a".repeat(64), schemaVersion: 9, upgraded: true }); + expect(upgraded.entrypoint).not.toBe(old.entrypoint); + expect(readFileSync(String(upgraded.entrypoint), "utf8")).toContain("schema 9 implementation"); + expect(readFileSync(String(old.entrypoint), "utf8")).toContain("runtime fixture"); + expect(readFileSync(dbPath, "utf8")).toBe("existing schema 9 database"); + const next = createRuntimeDirectory(root, "2.1.3", { buildId: "b".repeat(64), schemaVersion: 9 }); + const changed = await installMemoryRuntime({ ...flags, runtimeDirectory: next, preferInstalledCompatible }); + expect(changed.entrypoint).not.toBe(upgraded.entrypoint); + const reused = await installMemoryRuntime({ ...flags, runtimeDirectory: next, preferInstalledCompatible: true }); + expect(reused).toMatchObject({ reused: true, buildId: "b".repeat(64) }); + }); + + it("keeps the previous pointer after a changed same-version build fails activation", async () => { + const root = tempRoot(); + const home = join(root, "home"); + const flags = { home, skipServiceRegistration: true, skipHealthCheck: true, agents: [] }; + await installMemoryRuntime({ ...flags, runtimeDirectory: createRuntimeDirectory(root, "2.1.3", { buildId: "a".repeat(64), schemaVersion: 9 }) }); + const before = await currentInstalledRuntime(home); + const fetchSpy = vi.spyOn(globalThis, "fetch").mockRejectedValue(new Error("fixture unavailable")); + try { + await expect(installMemoryRuntime({ + ...flags, runtimeDirectory: createRuntimeDirectory(root, "2.1.3", { buildId: "b".repeat(64), schemaVersion: 9 }), + preferInstalledCompatible: true, skipHealthCheck: false, healthCheckTimeoutMs: 1 + })).rejects.toThrow("failed its activation health check"); + expect(await currentInstalledRuntime(home)).toEqual(before); + expect(readFileSync(before!.entrypoint, "utf8")).toContain("runtime fixture"); + } finally { fetchSpy.mockRestore(); } + }); + + it("repairs an installed build whose files were removed", async () => { + const root = tempRoot(); + const home = join(root, "home"); + const runtimeDirectory = createRuntimeDirectory(root, "2.1.3", { buildId: "a".repeat(64), schemaVersion: 9 }); + const flags = { home, runtimeDirectory, skipServiceRegistration: true, skipHealthCheck: true, agents: [] }; + const installed = await installMemoryRuntime(flags); + rmSync(String(installed.runtimeDir), { recursive: true }); + await installMemoryRuntime({ ...flags, preferInstalledCompatible: true }); + expect(existsSync(String(installed.entrypoint))).toBe(true); + }); + + it("keeps the new runtime after a failed activation may have advanced the schema", async () => { + const root = tempRoot(); + const home = join(root, "home"); + const flags = { home, skipServiceRegistration: true, skipHealthCheck: true, agents: [] }; + await installMemoryRuntime({ ...flags, runtimeDirectory: createRuntimeDirectory(root, "2.1.3") }); + const fetchSpy = vi.spyOn(globalThis, "fetch").mockRejectedValue(new Error("fixture unavailable")); + try { + await expect(installMemoryRuntime({ ...flags, preferInstalledCompatible: true, skipHealthCheck: false, healthCheckTimeoutMs: 1, + runtimeDirectory: createRuntimeDirectory(root, "2.1.3", { buildId: "b".repeat(64), schemaVersion: 9 }) + })).rejects.toThrow("kept the runtime supporting schema 9"); + expect(await currentInstalledRuntime(home)).toMatchObject({ buildId: "b".repeat(64), schemaVersion: 9 }); + } finally { fetchSpy.mockRestore(); } + }); + + it("does not accept health from another same-version build", async () => { + const root = tempRoot(); + const home = join(root, "home"); + const fetchSpy = vi.spyOn(globalThis, "fetch").mockResolvedValue(new Response(JSON.stringify({ ok: true, protocolVersion: 1, serviceVersion: "2.1.3", buildId: "b".repeat(64) }))); + try { + await expect(installMemoryRuntime({ home, skipServiceRegistration: true, healthCheckTimeoutMs: 1, agents: [], + runtimeDirectory: createRuntimeDirectory(root, "2.1.3", { buildId: "a".repeat(64), schemaVersion: 9 }) + })).rejects.toThrow("failed its activation health check"); + expect(await currentInstalledRuntime(home)).toBeUndefined(); + } finally { fetchSpy.mockRestore(); } + }); + + it("waits for the old service to release storage before switching the pointer", async () => { + const root = tempRoot(); + const home = join(root, "home"); + const flags = { home, skipServiceRegistration: true, skipHealthCheck: true, agents: [] }; + await installMemoryRuntime({ ...flags, runtimeDirectory: createRuntimeDirectory(root, "2.1.3") }); + const before = await currentInstalledRuntime(home); + const lockPath = join(home, "memory-service/service.lock"); + writeFileSync(lockPath, JSON.stringify({ pid: process.pid })); + const install = installMemoryRuntime({ ...flags, + runtimeDirectory: createRuntimeDirectory(root, "2.1.3", { buildId: "a".repeat(64), schemaVersion: 9 }) + }); + try { + await new Promise((resolve) => setTimeout(resolve, 100)); + expect(await currentInstalledRuntime(home)).toEqual(before); + } finally { rmSync(lockPath); } + await expect(install).resolves.toMatchObject({ buildId: "a".repeat(64) }); + }); + + it("rechecks the installed version after waiting for another installer", async () => { + const root = tempRoot(); + const home = join(root, "home"); + const flags = { home, skipServiceRegistration: true, skipHealthCheck: true, agents: [] }; + await installMemoryRuntime({ ...flags, runtimeDirectory: createRuntimeDirectory(root, "2.1.3") }); + const before = await currentInstalledRuntime(home); + const lockPath = join(home, "memory-service/install.lock"); + writeFileSync(lockPath, String(process.pid)); + const install = installMemoryRuntime({ ...flags, runtimeDirectory: createRuntimeDirectory(root, "2.1.4") }); + await new Promise((resolve) => setTimeout(resolve, 100)); + const newer = { ...before, version: "3.0.0" }; + writeFileSync(join(home, "memory-service/current.json"), JSON.stringify(newer)); + rmSync(lockPath); + await expect(install).rejects.toThrow("refusing to downgrade Memory from 3.0.0"); + expect(await currentInstalledRuntime(home)).toEqual(newer); + }); + + it("installs a release archive using its advertised build identity", async () => { + const root = tempRoot(); + const identity = { buildId: "a".repeat(64), schemaVersion: 9 }; + const fixture = createRuntimeArchive(root, "2.1.3", identity); + const releaseManifest = join(root, "memory-release.json"); + writeFileSync(releaseManifest, JSON.stringify({ version: "2.1.3", protocolVersion: 1, assets: { + [fixture.target]: { ...identity, name: "runtime.tar.gz", sha256: fixture.sha256, url: fixture.archive } + } })); + const installed = await installMemoryRuntime({ home: join(root, "home"), releaseManifest, + skipServiceRegistration: true, skipHealthCheck: true, agents: [] + }); + expect(installed).toMatchObject(identity); + expect(String(installed.runtimeDir)).toContain(identity.buildId); + }); + + it("rejects same-version schema downgrades before changing the pointer", async () => { + const root = tempRoot(); + const home = join(root, "home"); + const flags = { home, skipServiceRegistration: true, skipHealthCheck: true, agents: [] }; + await installMemoryRuntime({ ...flags, runtimeDirectory: createRuntimeDirectory(root, "2.1.3", { buildId: "a".repeat(64), schemaVersion: 9 }) }); + const before = await currentInstalledRuntime(home); + await expect(installMemoryRuntime({ ...flags, preferInstalledCompatible: true, + runtimeDirectory: createRuntimeDirectory(root, "2.1.3", { buildId: "b".repeat(64), schemaVersion: 8 }) + })).rejects.toThrow("refusing to downgrade Memory schema"); + expect(await currentInstalledRuntime(home)).toEqual(before); }); it("rejects checksum failures without activating the staged runtime", async () => { @@ -467,9 +606,9 @@ function tempRoot(): string { return root; } -function createRuntimeArchive(root: string, version: string): { archive: string; sha256: string; target: string } { +function createRuntimeArchive(root: string, version: string, identity: { buildId?: string; schemaVersion?: number } = {}): { archive: string; sha256: string; target: string } { const target = runtimeTarget(process.platform, process.arch); - const stage = createRuntimeDirectory(root, version); + const stage = createRuntimeDirectory(root, version, identity); const archive = join(root, `memmy-memory-runtime-${version}-${target}.tar.gz`); const packed = spawnSync("tar", ["-czf", archive, "-C", stage, "."], { encoding: "utf8" }); if (packed.status !== 0) throw new Error(packed.stderr || "failed to create runtime fixture"); @@ -477,11 +616,11 @@ function createRuntimeArchive(root: string, version: string): { archive: string; return { archive, sha256, target }; } -function createRuntimeDirectory(root: string, version: string): string { +function createRuntimeDirectory(root: string, version: string, identity: { buildId?: string; schemaVersion?: number } = {}): string { const target = runtimeTarget(process.platform, process.arch); const stage = join(root, `runtime-${version}-${Math.random().toString(36).slice(2)}`); mkdirSync(join(stage, "dist", "src", "server"), { recursive: true }); writeFileSync(join(stage, "dist", "src", "server", "index.js"), "// runtime fixture\n"); - writeFileSync(join(stage, "memory-runtime.json"), `${JSON.stringify({ version, protocolVersion: 1, target })}\n`); + writeFileSync(join(stage, "memory-runtime.json"), `${JSON.stringify({ version, protocolVersion: 1, target, ...identity })}\n`); return stage; } diff --git a/Memory/tests/runtime-release.test.ts b/Memory/tests/runtime-release.test.ts new file mode 100644 index 000000000..5d2198487 --- /dev/null +++ b/Memory/tests/runtime-release.test.ts @@ -0,0 +1,34 @@ +import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { spawnSync } from "node:child_process"; +import { afterEach, expect, it } from "vitest"; + +const roots: string[] = []; +afterEach(() => { for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }); }); + +it("publishes the identity embedded in each runtime archive", () => { + const root = mkdtempSync(join(tmpdir(), "memmy-runtime-release-")); + roots.push(root); + const input = join(root, "input"); + const output = join(root, "output"); + mkdirSync(input); + const targets = ["darwin-arm64", "darwin-x64", "linux-arm64", "linux-x64", "windows-arm64", "windows-x64"]; + for (const [index, target] of targets.entries()) { + const stage = join(root, target); + mkdirSync(stage); + writeFileSync(join(stage, "memory-runtime.json"), JSON.stringify({ + version: "2.1.3", protocolVersion: 1, target, buildId: String(index).repeat(64), schemaVersion: 9 + })); + const packed = spawnSync("tar", ["-czf", join(input, `memmy-memory-runtime-2.1.3-${target}.tar.gz`), "-C", stage, "."], { encoding: "utf8" }); + expect(packed.status, packed.stderr).toBe(0); + } + const assemble = fileURLToPath(new URL("../src/cli/scripts/assemble-release.mjs", import.meta.url)); + const assembled = spawnSync(process.execPath, [assemble, input, output, "2.1.3"], { encoding: "utf8" }); + expect(assembled.status, assembled.stderr).toBe(0); + const manifest = JSON.parse(readFileSync(join(output, "memory-release.json"), "utf8")); + for (const [index, target] of targets.entries()) { + expect(manifest.assets[target]).toMatchObject({ buildId: String(index).repeat(64), schemaVersion: 9 }); + } +}); diff --git a/Memory/tests/service/session/episode-relation.test.ts b/Memory/tests/service/session/episode-relation.test.ts index 804bdcd5b..0b27c7599 100644 --- a/Memory/tests/service/session/episode-relation.test.ts +++ b/Memory/tests/service/session/episode-relation.test.ts @@ -3,11 +3,14 @@ import { afterEach, describe, expect, it } from "vitest"; import { DEFAULT_MEMMY_CONFIG, MemoryDb, - type LlmClient + type LlmClient, + type LlmCompletionOptions, + type LlmMessage } from "../../../src/index.js"; import { Repositories } from "../../../src/storage/repositories.js"; import { accountRuntimeConfig, + createBatchReflectionLlm, createCapturingEmbedder, createMemoryServiceFixture, runWorkerRounds @@ -131,6 +134,44 @@ function createFollowUpRelationClassifierLlm(calls: string[]): LlmClient { }; } +const END_TOPIC_CONVENTION = "以后查看本地项目版本号都按这个步骤来"; + +function createEndTopicCaptureLlm( + calls: string[], + closingText: string, + closingDecision: { l1: unknown; user: unknown } +): LlmClient { + const base = createBatchReflectionLlm([], "captured task turn"); + return { + ...base, + async completeJson>( + messages: LlmMessage[], + options: LlmCompletionOptions + ): Promise { + calls.push(options.operation); + if (options.operation === "retrieval.retrieval.query.extract.v2") { + return { queryVecText: "", keywords: [] } as unknown as T; + } + if (options.operation === "relation.classify.v1") { + return { + relation: "end_topic", + confidence: 1, + reason: "user ends the current topic" + } as unknown as T; + } + const payload = messages.find((message) => message.role === "user")?.content ?? ""; + if ( + options.operation === "capture.summarize" && + messages[0]?.content.includes("Judge L1 and User Memory") && + payload.includes(closingText) + ) { + return closingDecision as unknown as T; + } + return base.completeJson(messages, options); + } + }; +} + describe("MemoryService / session / episode relation", () => { it("closes an explicitly ended topic without capturing the control turn as L1", async () => { const relationCalls: string[] = []; @@ -245,10 +286,11 @@ describe("MemoryService / session / episode relation", () => { expect(completed.l1MemoryIds).toEqual([]); }); - it("commits an LLM end-topic proposal without capturing the control turn as L1", async () => { - const relationCalls: string[] = []; - const { service } = createTestService({ - llm: createRelationClassifierLlm(relationCalls, undefined, "end_topic") + it("commits an LLM end-topic proposal and lets the capture decision drop a closing-only turn", async () => { + const calls: string[] = []; + const closingQuery = "That covers everything for this topic"; + const { db, service } = createTestService({ + llm: createEndTopicCaptureLlm(calls, closingQuery, { l1: null, user: null }) }); const session = service.openSession({ namespace: { @@ -265,10 +307,10 @@ describe("MemoryService / session / episode relation", () => { const started = await service.startTurn({ turnId: "turn-llm-end-topic-close", sessionId: session.sessionId, - query: "That covers everything for this topic" + query: closingQuery }); expect(started.status).toContain("relation:end_topic:proposed"); - expect(relationCalls).toContain("relation.classify.v1"); + expect(calls).toContain("relation.classify.v1"); expect(service.getMemory(first.episodeId)).toMatchObject({ kind: "episode", status: "open" @@ -276,15 +318,105 @@ describe("MemoryService / session / episode relation", () => { const completed = service.completeTurn("turn-llm-end-topic-close", { sessionId: session.sessionId, - query: "That covers everything for this topic", + query: closingQuery, answer: "Understood." }); expect(completed.closedEpisodeIds).toEqual([first.episodeId]); - expect(completed.l1MemoryIds).toEqual([]); + expect(completed.l1MemoryIds).toHaveLength(1); expect(service.getMemory(first.episodeId)).toMatchObject({ kind: "episode", status: "closed" }); + + await runWorkerRounds(service, 4, 50); + + expect(db.db.prepare( + `SELECT status FROM memories WHERE id = ?` + ).get(completed.l1MemoryId)).toEqual({ status: "deleted" }); + const reward = db.db.prepare( + `SELECT payload_json FROM evolution_jobs WHERE episode_id = ? AND job_type = 'reward'` + ).get(first.episodeId) as { payload_json: string } | undefined; + expect(reward).toBeDefined(); + expect(JSON.parse(reward!.payload_json)).toMatchObject({ + l1MemoryId: first.l1MemoryId, + targetKind: "episode" + }); + }); + + it("keeps the feedback and work convention of an LLM end-topic turn in L1 before the episode reward", async () => { + const calls: string[] = []; + const { db, service } = createTestService({ + llm: createEndTopicCaptureLlm(calls, END_TOPIC_CONVENTION, { + l1: { + title: "查看版本号的步骤约定", + summary: END_TOPIC_CONVENTION, + evidence: [{ quote: END_TOPIC_CONVENTION, role: "user", kind: "user_directive" }] + }, + user: { + action: "create", + evidence: [{ quote: END_TOPIC_CONVENTION, type: "User Preference" }], + target: "", + replacement: "" + } + }) + }); + const session = service.openSession({ + namespace: { + source: "codex", + profileId: "jiang", + userId: "user-end-topic-feedback" + } + }); + const first = service.completeTurn("turn-end-topic-feedback-first", { + sessionId: session.sessionId, + query: "查看本地项目的版本号", + answer: "读取 package.json 的 version 字段,当前版本是 1.1.8。" + }); + const closingQuery = `很好,这个步骤很有用,谢谢。${END_TOPIC_CONVENTION}。结束会话`; + const started = await service.startTurn({ + turnId: "turn-end-topic-feedback-close", + sessionId: session.sessionId, + query: closingQuery + }); + expect(started.status).toContain("relation:end_topic:proposed"); + + const completed = service.completeTurn("turn-end-topic-feedback-close", { + sessionId: session.sessionId, + query: closingQuery, + answer: "好的,以后查看本地项目版本号都会按这个步骤来。" + }); + expect(completed.closedEpisodeIds).toEqual([first.episodeId]); + expect(completed.l1MemoryIds).toHaveLength(1); + const episodeEvolutionJobs = () => db.db.prepare( + `SELECT job_type, payload_json + FROM evolution_jobs + WHERE episode_id = ? + AND job_type IN ('reflection', 'reward')` + ).all(first.episodeId) as Array<{ job_type: string; payload_json: string }>; + expect(episodeEvolutionJobs()).toEqual([]); + + await runWorkerRounds(service, 4, 50); + + expect(db.db.prepare( + `SELECT status FROM memories WHERE id = ?` + ).get(completed.l1MemoryId)).toEqual({ status: "activated" }); + expect(db.db.prepare( + `SELECT l1_memory_ids_json FROM episodes WHERE id = ?` + ).get(first.episodeId)).toEqual({ + l1_memory_ids_json: JSON.stringify([first.l1MemoryId, completed.l1MemoryId]) + }); + expect(db.db.prepare( + `SELECT content, source_turn_id FROM user_memories WHERE user_id = ? AND status = 'active'` + ).all("user-end-topic-feedback")).toEqual([{ + content: closingQuery, + source_turn_id: completed.rawTurnId + }]); + const reward = episodeEvolutionJobs().find((job) => job.job_type === "reward"); + expect(reward).toBeDefined(); + expect(JSON.parse(reward!.payload_json)).toMatchObject({ + l1MemoryId: completed.l1MemoryId, + targetKind: "episode" + }); }); it("keeps end-topic start and complete retries idempotent", async () => { diff --git a/Memory/tests/service/session/turn-capture.test.ts b/Memory/tests/service/session/turn-capture.test.ts index ee5ea17aa..b195a0238 100644 --- a/Memory/tests/service/session/turn-capture.test.ts +++ b/Memory/tests/service/session/turn-capture.test.ts @@ -20,6 +20,28 @@ const { afterEach(cleanup); describe("MemoryService / session / turn capture", () => { + it.each([false, undefined])("stores a completed degraded turn and respects recallApplied=%s", async (recallApplied) => { + const { db, service } = createTestService(); + const session = service.openSession({ namespace: { source: "memmy-agent", profileId: "deadline", userId: "deadline-user" } }); + const seed = service.completeTurn("seed", { sessionId: session.sessionId, query: "Inspect the SQLite schema", answer: "The schema has a sessions table." }); + const started = await service.startTurn({ sessionId: session.sessionId, turnId: "degraded", query: "Implement the SQLite deadline fix" }); + db.db.prepare("UPDATE recall_events SET injected_memory_ids_json = ? WHERE id = ?") + .run(JSON.stringify([seed.l1MemoryId]), started.searchEventId); + const request = { adapterId: "memmy-agent", requestId: "deadline-complete", sessionId: session.sessionId, + query: "Implement the SQLite deadline fix", answer: "Implemented and verified the deadline.", sourceMemoryIds: [], + ...(recallApplied === false ? { recallApplied } : {}) }; + const completed = service.completeTurn("degraded", request); + if (recallApplied === false) expect(completed.l1MemoryIds).toHaveLength(1); + const row = db.db.prepare("SELECT source_memory_ids_json, user_text, assistant_text FROM raw_turns WHERE id = ?") + .get(completed.rawTurnId) as { source_memory_ids_json: string; user_text: string; assistant_text: string }; + expect(JSON.parse(row.source_memory_ids_json)).toEqual(recallApplied === false ? [] : [seed.l1MemoryId]); + expect(row.user_text).toBe(request.query); + expect(row.assistant_text).toBe(request.answer); + expect(service.completeTurn("degraded", request)).toMatchObject({ duplicate: true, rawTurnId: completed.rawTurnId }); + expect(db.db.prepare("SELECT COUNT(*) AS count FROM raw_turns WHERE session_id = ? AND turn_id = ?").get(session.sessionId, "degraded")).toEqual({ count: 1 }); + db.close(); + }); + it("preserves an explicit empty turn-start layer selection for evaluation ablations", async () => { const { db, service } = createTestService(); const session = service.openSession({ diff --git a/Memory/tests/service/worker/worker-runtime.test.ts b/Memory/tests/service/worker/worker-runtime.test.ts index 8629dbf42..8cde81de3 100644 --- a/Memory/tests/service/worker/worker-runtime.test.ts +++ b/Memory/tests/service/worker/worker-runtime.test.ts @@ -4,7 +4,12 @@ import { DEFAULT_MEMMY_CONFIG } from "../../../src/config/index.js"; import { MEMORY_BYOK_BUDGET_KV_KEY } from "../../../src/service/memory-token-budget-ledger.js"; import { evolutionJobDedupeKey } from "../../../src/service/worker/job-handlers.js"; import { Repositories } from "../../../src/storage/repositories.js"; -import { accountRuntimeConfig, byokRuntimeConfig, createMemoryServiceFixture } from "../../fixtures/memory-service-fixture.js"; +import { + accountRuntimeConfig, + addAgentSourceImport, + byokRuntimeConfig, + createMemoryServiceFixture +} from "../../fixtures/memory-service-fixture.js"; const { cleanup: cleanupMemoryServiceFixture, @@ -16,6 +21,63 @@ afterEach(() => { }); describe("MemoryService / worker / runtime", () => { + it("runs normal reflection before startup summary repair backlogs", () => { + const { db, service } = createTestService(); + const namespace = { + source: "codex", + profileId: "startup-repair-priority", + userId: "startup-repair-priority-user" + }; + const processingRepair = addAgentSourceImport( + service, + namespace, + "repair an interrupted import summary", + "startup-processing-repair" + ); + const placeholderRepair = addAgentSourceImport( + service, + namespace, + "repair a placeholder import summary", + "startup-placeholder-repair" + ); + db.db.prepare( + `UPDATE evolution_jobs + SET payload_json = json_set(payload_json, '$.source', ?) + WHERE target_memory_id = ?` + ).run("startup.processing_repair", processingRepair.id); + db.db.prepare( + `UPDATE evolution_jobs + SET payload_json = json_set(payload_json, '$.source', ?) + WHERE target_memory_id = ?` + ).run("startup.placeholder_summary_repair", placeholderRepair.id); + db.db.prepare( + `INSERT INTO evolution_jobs ( + id, job_type, status, user_id, episode_id, payload_json, + attempts, max_attempts, created_at, updated_at + ) VALUES (?, 'reflection', 'queued', ?, ?, '{}', 0, 3, ?, ?)` + ).run( + "job_normal_reflection_before_startup_repairs", + namespace.userId, + "episode_normal_reflection_before_startup_repairs", + "2026-01-01T00:00:00.000Z", + "2026-01-01T00:00:00.000Z" + ); + + const runtime = new Repositories(db.db).runtime; + const leased = runtime.leaseQueuedJobs(10, 60, undefined, true).map((job) => job.id); + + expect(leased).toEqual(["job_normal_reflection_before_startup_repairs"]); + runtime.completeJob("job_normal_reflection_before_startup_repairs"); + const repairs = runtime.leaseQueuedJobs(10, 60, undefined, true); + expect(repairs).toHaveLength(2); + expect(repairs.every((job) => job.jobType === "import_summary")).toBe(true); + expect(repairs.map((job) => job.payload.source).sort()).toEqual([ + "startup.placeholder_summary_repair", + "startup.processing_repair" + ]); + db.close(); + }); + it.each([ [{ repairId: "repair-1" }, "episode-1", "decision_repair:repair-1"], [{ feedbackId: "feedback-1" }, "episode-1", "decision_repair:feedback-1"], diff --git a/Memory/viewer/package.json b/Memory/viewer/package.json index 38349b0ae..1c80787d3 100644 --- a/Memory/viewer/package.json +++ b/Memory/viewer/package.json @@ -1,6 +1,6 @@ { "name": "@memmy/memory-viewer", - "version": "2.1.3", + "version": "2.1.4", "private": true, "type": "module" } diff --git a/Migrations/src/migrations/v1.1.9/0001-remove-legacy-memory-embedding.ts b/Migrations/src/migrations/v1.1.9/0001-remove-legacy-memory-embedding.ts new file mode 100644 index 000000000..4c32f156a --- /dev/null +++ b/Migrations/src/migrations/v1.1.9/0001-remove-legacy-memory-embedding.ts @@ -0,0 +1,76 @@ +import { + mutateRuntimeConfig, + mutateRuntimeConfigLockHeld, + type RuntimeConfigDocument, +} from "../../runtime-config-writer.js"; +import { + MigrationError, + type AgentWorkspaceMigrationContext, + type MigrationDefinition, + type MigrationResult, +} from "../../types.js"; + +const MIGRATION_ID = "v1.1.9/0001-remove-legacy-memory-embedding"; + +function isObject(value: unknown): value is RuntimeConfigDocument { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +function removeLegacyEmbedding(config: RuntimeConfigDocument): void { + // Memory setup used to write this field after the model-catalog migration + // had already recorded its completion, so it needs a follow-up cleanup. + const memory = isObject(config.memmyMemory) ? config.memmyMemory : null; + if (memory) delete memory.embedding; +} + +function wrapError(error: unknown): never { + if (error instanceof MigrationError) { + throw new MigrationError(error.code, error.message, { + migrationId: MIGRATION_ID, + scope: "runtime-config", + cause: error.cause, + }); + } + throw new MigrationError( + "migration_config_invalid", + "Unable to remove the legacy memory embedding configuration", + { + migrationId: MIGRATION_ID, + scope: "runtime-config", + cause: error, + }, + ); +} + +async function runMigration( + context: AgentWorkspaceMigrationContext, +): Promise { + try { + const options = { createIfMissing: false as const }; + const result = context.runtimeConfigLock + ? await mutateRuntimeConfigLockHeld(context.runtimeConfigLock, removeLegacyEmbedding, options) + : await mutateRuntimeConfig(context.runtimeConfigFile, removeLegacyEmbedding, options); + if (!result.sourceExists) { + return { scanned: 0, changed: 0, ignored: 0, deferred: true }; + } + return result.changed + ? { scanned: 1, changed: 1, ignored: 0 } + : { scanned: 1, changed: 0, ignored: 1 }; + } catch (error) { + wrapError(error); + } +} + +export const removeLegacyMemoryEmbeddingV119: MigrationDefinition = { + id: MIGRATION_ID, + introducedIn: "1.1.9", + scope: "runtime-config", + description: "Remove the legacy memmyMemory.embedding runtime configuration", + up: runMigration, +}; + +export function removeLegacyMemoryEmbeddingForTest( + context: AgentWorkspaceMigrationContext, +): Promise { + return runMigration(context); +} diff --git a/Migrations/src/registry.ts b/Migrations/src/registry.ts index 50d646e7a..8dc6d57b1 100644 --- a/Migrations/src/registry.ts +++ b/Migrations/src/registry.ts @@ -6,6 +6,7 @@ import { addGoalDagBoundaryV107 } from "./migrations/v1.0.7/0004-add-goal-dag-bo import { repairRuntimeModelCatalogV109 } from "./migrations/v1.0.9/0001-repair-runtime-model-catalog.js"; import { upgradeSummaryTimeoutV112 } from "./migrations/v1.1.2/0001-upgrade-summary-timeout.js"; import { addOpenComputerUseMcpV116 } from "./migrations/v1.1.6/0001-add-open-computer-use-mcp.js"; +import { removeLegacyMemoryEmbeddingV119 } from "./migrations/v1.1.9/0001-remove-legacy-memory-embedding.js"; import { MigrationError, type MigrationDefinition } from "./types.js"; const STABLE_SEMVER_PATTERN = @@ -28,6 +29,7 @@ export const migrations: readonly MigrationDefinition[] = [ repairRuntimeModelCatalogV109, upgradeSummaryTimeoutV112, addOpenComputerUseMcpV116, + removeLegacyMemoryEmbeddingV119, ]; function definitionError(message: string, migrationId: string | null = null): never { diff --git a/Migrations/tests/0001-remove-legacy-memory-embedding.test.ts b/Migrations/tests/0001-remove-legacy-memory-embedding.test.ts new file mode 100644 index 000000000..870b83461 --- /dev/null +++ b/Migrations/tests/0001-remove-legacy-memory-embedding.test.ts @@ -0,0 +1,71 @@ +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import YAML from "yaml"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { removeLegacyMemoryEmbeddingV119 } from "../src/migrations/v1.1.9/0001-remove-legacy-memory-embedding.js"; +import type { MigrationLogger } from "../src/types.js"; + +const temporaryDirectories: string[] = []; + +function logger(): MigrationLogger { + return { info: vi.fn(), warn: vi.fn(), error: vi.fn() }; +} + +async function fixture(config: unknown) { + const root = await fs.mkdtemp(path.join(os.tmpdir(), "memmy-remove-embedding-migration-")); + temporaryDirectories.push(root); + const configPath = path.join(root, "config.yaml"); + await fs.writeFile(configPath, YAML.stringify(config), "utf8"); + return { root, configPath }; +} + +function context(root: string, configPath: string) { + return { + profileWorkspace: root, + sessionsDir: path.join(root, "sessions"), + runtimeConfigFile: configPath, + sessionDagDir: path.join(root, "session-dag"), + logger: logger(), + }; +} + +afterEach(async () => { + await Promise.all( + temporaryDirectories.splice(0).map((directory) => + fs.rm(directory, { recursive: true, force: true }), + ), + ); +}); + +describe("v1.1.9/0001-remove-legacy-memory-embedding", () => { + it("removes memmyMemory.embedding while preserving the rest of the config", async () => { + const { root, configPath } = await fixture({ + app: { userMode: "byok" }, + memmyMemory: { + embedding: { mode: "local", provider: "local" }, + storage: { sqlitePath: "/tmp/memory.sqlite" }, + }, + }); + + await expect(removeLegacyMemoryEmbeddingV119.up(context(root, configPath))).resolves.toEqual({ + scanned: 1, + changed: 1, + ignored: 0, + }); + + const config = YAML.parse(await fs.readFile(configPath, "utf8")); + expect(config.memmyMemory.embedding).toBeUndefined(); + expect(config.memmyMemory.storage).toEqual({ sqlitePath: "/tmp/memory.sqlite" }); + }); + + it("is idempotent when the legacy field is already absent", async () => { + const { root, configPath } = await fixture({ memmyMemory: { enabled: true } }); + + await expect(removeLegacyMemoryEmbeddingV119.up(context(root, configPath))).resolves.toEqual({ + scanned: 1, + changed: 0, + ignored: 1, + }); + }); +}); diff --git a/Migrations/tests/runner.test.ts b/Migrations/tests/runner.test.ts index c14b02385..5bc2d9b83 100644 --- a/Migrations/tests/runner.test.ts +++ b/Migrations/tests/runner.test.ts @@ -113,9 +113,10 @@ describe("migration runner", () => { "v1.0.9/0001-repair-runtime-model-catalog", "v1.1.2/0001-upgrade-summary-timeout", "v1.1.6/0001-add-open-computer-use-mcp", + "v1.1.9/0001-remove-legacy-memory-embedding", ]); expect(first.deferred).toEqual(["v1.0.7/0002-import-legacy-app-state-model-config"]); - expect(first.results).toEqual({ scanned: 6, changed: 3, ignored: 3 }); + expect(first.results).toEqual({ scanned: 7, changed: 3, ignored: 4 }); expect(second).toEqual({ applied: [], skipped: [ @@ -126,6 +127,7 @@ describe("migration runner", () => { "v1.0.9/0001-repair-runtime-model-catalog", "v1.1.2/0001-upgrade-summary-timeout", "v1.1.6/0001-add-open-computer-use-mcp", + "v1.1.9/0001-remove-legacy-memory-embedding", ], deferred: ["v1.0.7/0002-import-legacy-app-state-model-config"], results: { scanned: 0, changed: 0, ignored: 0 }, @@ -194,6 +196,15 @@ describe("migration runner", () => { key: runtimeConfigTargetKey(configPath), }, }, + { + id: "v1.1.9/0001-remove-legacy-memory-embedding", + introducedIn: "1.1.9", + appliedAt: expect.stringMatching(/Z$/), + target: { + type: "runtime-config", + key: runtimeConfigTargetKey(configPath), + }, + }, ]); }); diff --git a/package-lock.json b/package-lock.json index b7319fd43..462fd745d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "memmy-agent", - "version": "1.2.1", + "version": "1.2.2", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "memmy-agent", - "version": "1.2.1", + "version": "1.2.2", "workspaces": [ "Migrations", "AgentSourceCore", @@ -588,7 +588,7 @@ }, "App/shell/desktop": { "name": "@memmy/desktop", - "version": "1.2.1", + "version": "1.2.2", "dependencies": { "@memmy/backend": "0.0.0", "@memmy/desktop-interface": "0.0.0", @@ -744,7 +744,7 @@ }, "Memory": { "name": "@memmy/memory", - "version": "2.1.3", + "version": "2.1.4", "dependencies": { "@huggingface/transformers": "^3.8.0", "@memmy/agent-source-core": "0.0.0", diff --git a/package.json b/package.json index b132ee00c..b867317f5 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "memmy-agent", - "version": "1.2.1", + "version": "1.2.2", "private": true, "type": "module", "description": "Local-first agent memory substrate with desktop and CLI surfaces.", diff --git a/scripts/internal/mac/build-dmg.sh b/scripts/internal/mac/build-dmg.sh index 0e3768217..24cb8e09a 100755 --- a/scripts/internal/mac/build-dmg.sh +++ b/scripts/internal/mac/build-dmg.sh @@ -337,6 +337,7 @@ create_memory_runtime_manifest() { node --input-type=module <<'NODE' import { mkdir, readFile, writeFile } from "node:fs/promises"; import { join } from "node:path"; +import { pathToFileURL } from "node:url"; const rootDir = requiredEnv("ROOT_DIR"); const memoryDir = requiredEnv("MEMORY_DIR"); @@ -345,6 +346,8 @@ const targetCpu = requiredEnv("TARGET_CPU"); const runtimeName = "memmy-memory-runtime"; const memoryPackage = JSON.parse(await readFile(join(memoryDir, "package.json"), "utf8")); const runtimeVersion = memoryPackage.version; +const { memoryRuntimeBuildIdentity } = await import(pathToFileURL(join(rootDir, "scripts/internal/shared/memory-runtime-build-identity.mjs"))); +const buildIdentity = await memoryRuntimeBuildIdentity(rootDir); const rootLock = JSON.parse(await readFile(join(rootDir, "package-lock.json"), "utf8")); const dependencies = { ...(memoryPackage.dependencies ?? {}) }; const workspacePackageDirs = new Map(); @@ -400,6 +403,7 @@ await writeFile(join(runtimeDir, "package.json"), `${JSON.stringify(runtimePacka await writeFile(join(runtimeDir, "package-lock.json"), `${JSON.stringify(runtimeLock, null, 2)}\n`); await writeFile(join(runtimeDir, "memory-runtime.json"), `${JSON.stringify({ version: runtimeVersion, + ...buildIdentity, protocolVersion: 1, target: `darwin-${targetCpu}`, entrypoint: "dist/src/server/index.js", diff --git a/scripts/internal/shared/memory-runtime-build-identity.mjs b/scripts/internal/shared/memory-runtime-build-identity.mjs new file mode 100644 index 000000000..ec4500f99 --- /dev/null +++ b/scripts/internal/shared/memory-runtime-build-identity.mjs @@ -0,0 +1,43 @@ +import { createHash } from "node:crypto"; +import { readFile, readdir } from "node:fs/promises"; +import { join } from "node:path"; + +// Include runtime code, assets and dependency resolution. A source change must +// change identity even when a release forgets to increment Memory's version. +export async function memoryRuntimeBuildIdentity(repoRoot) { + const inputs = [ + "Memory/src", "Memory/viewer/src", "Memory/viewer/public", "Memory/adapters", "AgentSourceCore/src", + "Memory/package.json", "Memory/tsconfig.json", "Memory/tsconfig.base.json", + "Memory/viewer/package.json", "Memory/viewer/vite.config.ts", "Memory/viewer/index.html", + "AgentSourceCore/package.json", "AgentSourceCore/tsconfig.json", "tsconfig.base.json", "package-lock.json" + ]; + const hash = createHash("sha256"); + async function addFile(path) { + const content = await readFile(join(repoRoot, path)); + // Git checkouts on Windows may use CRLF. Normalize text, preserve binaries. + const normalized = /\.(?:[cm]?[jt]sx?|json|mdx?|css|html|ya?ml|sh|ps1|toml|txt)$/.test(path) + ? Buffer.from(content.toString("utf8").replace(/\r\n/g, "\n")) : content; + hash.update(`${path}\0${normalized.length}\0`); + hash.update(normalized); + } + async function addDirectory(path) { + const entries = await readdir(join(repoRoot, path), { withFileTypes: true }); + entries.sort((a, b) => a.name < b.name ? -1 : a.name > b.name ? 1 : 0); + for (const entry of entries) { + if (entry.name.startsWith(".") || ["node_modules", "dist"].includes(entry.name)) continue; + const child = `${path}/${entry.name}`; + // This ignored bridge is generated from sources in the same directory. + if (child === "Memory/src/agent-source/integration/workspace-bridge/memmy-workspace-bridge.mjs") continue; + if (entry.isDirectory()) await addDirectory(child); + else if (entry.isFile()) await addFile(child); + } + } + for (const input of inputs) { + if (input.endsWith("/src") || input.endsWith("/public") || input === "Memory/adapters") await addDirectory(input); + else await addFile(input); + } + const schema = await readFile(join(repoRoot, "Memory/src/storage/schema.ts"), "utf8"); + const schemaVersion = Number(/export const SCHEMA_VERSION = (\d+);/.exec(schema)?.[1]); + if (!Number.isSafeInteger(schemaVersion) || schemaVersion < 1) throw new Error("Memory schema version is missing"); + return { buildId: hash.digest("hex"), schemaVersion }; +} diff --git a/scripts/internal/win/create-memory-runtime-manifest.mjs b/scripts/internal/win/create-memory-runtime-manifest.mjs index 47ea41c1c..4a3848a01 100644 --- a/scripts/internal/win/create-memory-runtime-manifest.mjs +++ b/scripts/internal/win/create-memory-runtime-manifest.mjs @@ -1,7 +1,8 @@ #!/usr/bin/env node import { mkdir, readFile, writeFile } from "node:fs/promises"; -import { dirname } from "node:path"; +import { memoryRuntimeBuildIdentity } from "../shared/memory-runtime-build-identity.mjs"; +import { dirname, resolve } from "node:path"; const [sourcePackagePath, runtimePackagePath, runtimeMetadataPath] = process.argv.slice(2); if (!sourcePackagePath || !runtimePackagePath || !runtimeMetadataPath) { @@ -22,6 +23,7 @@ const runtimePackage = { dependencies, }; const runtimeMetadata = { + ...await memoryRuntimeBuildIdentity(resolve(dirname(sourcePackagePath), "..")), version: sourcePackage.version, protocolVersion: 1, target: "windows-x64", diff --git a/tests/packaged-runtime-config.test.mjs b/tests/packaged-runtime-config.test.mjs index bd34735fb..f27277e9e 100644 --- a/tests/packaged-runtime-config.test.mjs +++ b/tests/packaged-runtime-config.test.mjs @@ -19,6 +19,8 @@ import { } from "../scripts/internal/shared/write-desktop-edition-manifest-lib.mjs"; import { pruneRuntimeEnvFiles } from "../scripts/internal/shared/prune-runtime-env-files-lib.mjs"; +import { memoryRuntimeBuildIdentity } from "../scripts/internal/shared/memory-runtime-build-identity.mjs"; + const roots = []; afterEach(() => { @@ -152,8 +154,23 @@ describe("packaged desktop runtime configuration", () => { expect(existsSync(join(runtime, ".env"))).toBe(false); }); + it("changes Memory build identity when runtime code changes without a version bump", async () => { + const root = fixtureRoot(); + seedMemoryBuildInputs(root); + const before = await memoryRuntimeBuildIdentity(root); + writeFileSync(join(root, "Memory/src/storage/schema.ts"), "export const SCHEMA_VERSION = 9;\r\n"); + expect(await memoryRuntimeBuildIdentity(root)).toEqual(before); + writeFileSync(join(root, "Memory/src/server.ts"), "export const behavior = 'new';\n"); + const changed = await memoryRuntimeBuildIdentity(root); + expect(changed.buildId).not.toBe(before.buildId); + expect(changed.schemaVersion).toBe(9); + writeFileSync(join(root, "Memory/src/storage/schema.ts"), "export const SCHEMA_VERSION = 10;\n"); + expect(await memoryRuntimeBuildIdentity(root)).toMatchObject({ schemaVersion: 10 }); + }); + it("creates a standalone Windows Memory manifest without private workspace dependencies", () => { const root = fixtureRoot(); + seedMemoryBuildInputs(root); const sourcePackage = join(root, "Memory", "package.json"); const runtimePackage = join(root, "runtime", "package.json"); const runtimeMetadata = join(root, "runtime", "memory-runtime.json"); @@ -186,6 +203,8 @@ describe("packaged desktop runtime configuration", () => { dependencies: { zod: "^4.4.3" }, }); expect(JSON.parse(readFileSync(runtimeMetadata, "utf8"))).toEqual({ + buildId: expect.stringMatching(/^[a-f0-9]{64}$/), + schemaVersion: 9, version: "2.1.0", protocolVersion: 1, target: "windows-x64", @@ -196,6 +215,7 @@ describe("packaged desktop runtime configuration", () => { it("creates a standalone macOS Memory manifest and stages its workspace parser package", () => { const root = fixtureRoot(); + seedMemoryBuildInputs(root); const memoryDir = join(root, "Memory"); const runtimeDir = join(root, "runtime"); const coreDir = join(root, "AgentSourceCore"); @@ -225,6 +245,9 @@ describe("packaged desktop runtime configuration", () => { env: { ...process.env, ROOT_DIR: root, MEMORY_DIR: memoryDir, MEMORY_RUNTIME_DIR: runtimeDir, TARGET_CPU: "arm64" }, }); expect(generated.status, generated.stderr).toBe(0); + expect(JSON.parse(readFileSync(join(runtimeDir, "memory-runtime.json"), "utf8"))).toMatchObject({ + buildId: expect.stringMatching(/^[a-f0-9]{64}$/), schemaVersion: 9 + }); expect(JSON.parse(readFileSync(join(runtimeDir, "package.json"), "utf8")).dependencies).toEqual({ "@memmy/agent-source-core": "file:../../../../../../AgentSourceCore", zod: "4.4.3", @@ -619,3 +642,18 @@ function fixtureRoot() { roots.push(root); return root; } + +function seedMemoryBuildInputs(root) { + for (const path of ["Memory/src/storage", "Memory/viewer/src", "Memory/viewer/public", "Memory/adapters", "AgentSourceCore/src"]) { + mkdirSync(join(root, path), { recursive: true }); + } + for (const path of ["Memory/package.json", "Memory/tsconfig.json", "Memory/tsconfig.base.json", + "Memory/viewer/package.json", "AgentSourceCore/package.json", "AgentSourceCore/tsconfig.json", + "tsconfig.base.json", "package-lock.json"]) writeFixtureJson(join(root, path), {}); + writeFileSync(join(root, "Memory/src/storage/schema.ts"), "export const SCHEMA_VERSION = 9;\n"); + writeFileSync(join(root, "Memory/viewer/vite.config.ts"), "export default {};\n"); + writeFileSync(join(root, "Memory/viewer/index.html"), "\n"); + const relative = "scripts/internal/shared/memory-runtime-build-identity.mjs"; + mkdirSync(dirname(join(root, relative)), { recursive: true }); + writeFileSync(join(root, relative), readFileSync(join(dirname(fileURLToPath(import.meta.url)), "..", relative))); +}