Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
45 changes: 45 additions & 0 deletions App/backend/local-api-contracts/src/memory-l3-world-model.ts
Original file line number Diff line number Diff line change
Expand Up @@ -83,6 +83,51 @@ export const L3WorldModelBoundaryResponseSchema = z.object({
}).strict();
export type L3WorldModelBoundaryResponse = z.infer<typeof L3WorldModelBoundaryResponseSchema>;

const AbsolutePathSchema = z.string().min(1).max(4096)
.refine((value) => value.startsWith("/") || /^[A-Za-z]:[\\/]/.test(value), "path must be absolute");

/** Full list of folders the user picked in the desktop app; each push replaces the previous one. */
export const L3UserProjectsReplaceRequestSchema = z.object({
...L3WorldModelRequestEnvelopeShape,
projects: z.array(z.object({
rootPath: AbsolutePathSchema,
name: z.string().min(1).max(512)
}).strict()).max(500),
standaloneRoots: z.array(AbsolutePathSchema).max(20).default([])
}).strict().superRefine(assertEnvelopeSourceConsistency);
export type L3UserProjectsReplaceRequest = z.input<typeof L3UserProjectsReplaceRequestSchema>;

export const L3UserProjectsReplaceResponseSchema = z.object({
count: z.number().int().nonnegative(),
serverTime: z.string().datetime()
}).strict();
export type L3UserProjectsReplaceResponse = z.infer<typeof L3UserProjectsReplaceResponseSchema>;

/** Puts the user's new-L3 batches that ended in dead letter back in the queue. */
export const L3FailedBatchesRetryRequestSchema = z.object({
...L3WorldModelRequestEnvelopeShape
}).strict().superRefine(assertEnvelopeSourceConsistency);
export type L3FailedBatchesRetryRequest = z.input<typeof L3FailedBatchesRetryRequestSchema>;

export const L3FailedBatchesRetryResponseSchema = z.object({
requeued: z.number().int().nonnegative(),
serverTime: z.string().datetime()
}).strict();
export type L3FailedBatchesRetryResponse = z.infer<typeof L3FailedBatchesRetryResponseSchema>;

/** End of a hosted import: freeze what it added to these scanned Sessions into L3 batches. */
export const L3ScanSessionsFreezeRequestSchema = z.object({
...L3WorldModelRequestEnvelopeShape,
sessionIds: z.array(NonEmptyStringSchema).max(10_000)
}).strict().superRefine(assertEnvelopeSourceConsistency);
export type L3ScanSessionsFreezeRequest = z.input<typeof L3ScanSessionsFreezeRequestSchema>;

export const L3ScanSessionsFreezeResponseSchema = z.object({
batches: z.number().int().nonnegative(),
serverTime: z.string().datetime()
}).strict();
export type L3ScanSessionsFreezeResponse = z.infer<typeof L3ScanSessionsFreezeResponseSchema>;

export const SessionL3WorldModelContextResponseSchema = z.object({
schemaVersion: z.literal(2),
projectId: NonEmptyStringSchema.nullable(),
Expand Down
1 change: 1 addition & 0 deletions App/backend/local-api-contracts/src/memory-runtime.ts
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,7 @@ export const JobTypeSchema = z.enum([
"l2_induction",
"l3_abstraction",
"l3_world_model_update",
"l3_unit_update",
"project_environment_profile",
"skill_crystallization",
"skill_trial_resolve",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import {
CloseSessionOutputSchema,
CompleteTurnOutputSchema,
SourceTurnCompleteOutputSchema,
L3ScanSessionsFreezeResponseSchema,
DeleteMemoryOutputSchema,
EmbeddingInferenceOutputSchema,
DeletePanelTaskOutputSchema,
Expand All @@ -32,6 +33,7 @@ import {
RetryMemoryProcessingOutputSchema,
WorkerRunOutputSchema
} from "@memmy/local-api-contracts";
import { randomUUID } from "node:crypto";
import { z, type ZodType } from "zod";
import { MemoryLayerError, MemoryLayerNetworkError } from "./errors.js";
import { buildMemoryLayerUrl, MEMORY_LAYER_PATHS } from "./memory-layer-endpoints.js";
Expand Down Expand Up @@ -206,6 +208,12 @@ export function createHttpMemoryClient(
return request("POST", "completeSourceTurn", SourceTurnCompleteOutputSchema, { body: input, context });
},

async freezeScanSessions(source, sessionIds, context) {
// The L3 envelope resolves the owner the same way the source turns that opened these Sessions did.
const body = { requestId: randomUUID(), adapterId: `agent-source:${source}`, source, sessionIds };
return request("POST", "freezeScanSessions", L3ScanSessionsFreezeResponseSchema, { body, context });
},

async search(input, context) {
return request("POST", "search", SearchOutputSchema, { body: input, context });
},
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ export const MEMORY_LAYER_PATHS = Object.freeze({
startTurn: "/api/v1/turns/start",
completeTurn: "/api/v1/turns/:turnId/complete",
completeSourceTurn: "/api/v1/source-turns/complete",
freezeScanSessions: "/api/v1/l3/scan-sessions/freeze",
search: "/api/v1/memory/search",
embeddingInference: "/api/v1/models/embedding/infer",
addMemory: "/api/v1/memory/add",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,20 @@ describe("HttpMemoryClient", () => {
expect(JSON.parse(String(calls[0]?.init?.body))).toEqual(input);
expect(new Headers(calls[0]?.init?.headers).get("x-memmy-user-id")).toBe("fixture-user");
});
it("hands scanned Sessions to L3 inside a full request envelope for their source", async () => {
const calls: Array<{ url: string; init?: RequestInit }> = [];
const client = createHttpMemoryClient({ baseUrl: "http://memory.test", token: "fixture-token", timeoutMs: 500, maxRetries: 0 }, {
fetchImpl: (async (url, init) => {
calls.push({ url: String(url), init });
return new Response(JSON.stringify({ batches: 2, serverTime: "2099-01-01T00:00:00.000Z" }), { status: 200, headers: { "content-type": "application/json" } });
}) as typeof fetch
});
expect(await client.freezeScanSessions("codex", ["s-1", "s-2"])).toEqual({ batches: 2, serverTime: "2099-01-01T00:00:00.000Z" });
expect(calls[0]?.url).toBe("http://memory.test/api/v1/l3/scan-sessions/freeze");
expect(JSON.parse(String(calls[0]?.init?.body))).toEqual({
requestId: expect.stringMatching(/^[0-9a-f-]{36}$/), adapterId: "agent-source:codex", source: "codex", sessionIds: ["s-1", "s-2"]
});
});
it("only defines path templates for the final memory HTTP APIs", () => {
expect(Object.values(MEMORY_LAYER_PATHS)).toEqual([
"/api/v1/agent-sources",
Expand All @@ -63,6 +77,7 @@ describe("HttpMemoryClient", () => {
"/api/v1/turns/start",
"/api/v1/turns/:turnId/complete",
"/api/v1/source-turns/complete",
"/api/v1/l3/scan-sessions/freeze",
"/api/v1/memory/search",
"/api/v1/models/embedding/infer",
"/api/v1/memory/add",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,7 @@ describe("createMockMemoryClient", () => {
"completeSourceTurn",
"deleteMemory",
"enqueueImportSummaries",
"freezeScanSessions",
"getMemory",
"getMemoryProcessingStatus",
"getMemoryTokenBudget",
Expand Down
2 changes: 2 additions & 0 deletions App/backend/src/adapters/outbound/memory-client/types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -84,6 +84,8 @@ export interface MemoryClient {
completeTurn(input: CompleteTurnInput & { turnId: string }, context?: MemoryRequestContext): Promise<CompleteTurnOutput>;

completeSourceTurn(input: SourceTurnCompleteInput, context?: MemoryRequestContext): Promise<SourceTurnCompleteOutput>;
/** Freeze what an import added to these scanned Sessions into L3 batches. */
freezeScanSessions(source: string, sessionIds: string[], context?: MemoryRequestContext): Promise<{ batches: number }>;

search(input: SearchInput, context?: MemoryRequestContext): Promise<SearchOutput>;
embeddingInference?(input: EmbeddingInferenceInput, options?: { signal?: AbortSignal }): Promise<EmbeddingInferenceOutput>;
Expand Down
38 changes: 26 additions & 12 deletions App/backend/src/services/ingestion-service.ts
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,8 @@ export interface IngestionStats {
dedupedMemories: number;
failedMemories: number;
memoryIds: string[];
/** Memory Sessions this import added turns to; frozen into L3 when the import ends. */
sessionIds?: string[];
/**
* Subset of `memoryIds` written through the legacy add-memory path, which still needs a
* summary job. A native turn is summarized inside the Memory capture transaction, so its
Expand All @@ -77,7 +79,7 @@ export interface IngestionItemSkip {

/** Contract for create ingestion service options. */
export interface CreateIngestionServiceOptions {
memoryClient: Pick<MemoryClient, "addMemory" | "completeSourceTurn">;
memoryClient: Pick<MemoryClient, "addMemory" | "completeSourceTurn"> & Partial<Pick<MemoryClient, "freezeScanSessions">>;
agentSourceRepository: Pick<AgentSourceRepository, "hasSeen" | "markSeen">;
memoryAddAnalytics?: Pick<
MemoryDesktopAddAnalytics,
Expand Down Expand Up @@ -117,20 +119,30 @@ export function createIngestionService(options: CreateIngestionServiceOptions):
};

const conversations = groupByConversation(messages, stats)[Symbol.asyncIterator]();
while (true) {
if (ctx.signal?.aborted) {
break;
try {
while (true) {
if (ctx.signal?.aborted) {
break;
}

const next = await conversations.next();
if (next.done) {
break;
}

stats.conversations += 1;
await processConversation(options, next.value, ctx, stats);
}

const next = await conversations.next();
if (next.done) {
break;
} finally {
// Imported history never ends its Sessions, so hand what this import added to L3 now, also
// when it stopped early. Freezing only writes rows in Memory; a failure must not fail the import.
const sessionIds = [...new Set(stats.sessionIds ?? [])];
if (sessionIds.length > 0 && options.memoryClient.freezeScanSessions) {
await options.memoryClient.freezeScanSessions(ctx.sourceId, sessionIds).catch((error: unknown) => {
console.warn(`[agent-source] l3.freeze_failed ${error instanceof Error ? error.message : String(error)}`);
});
}

stats.conversations += 1;
await processConversation(options, next.value, ctx, stats);
}

return stats;
}
};
Expand Down Expand Up @@ -333,6 +345,8 @@ async function processNativeConversation(
...(legacyImportTurnId ? { legacyImportTurnId } : {}),
...(ctx.scanMode === "initial_subset" || ctx.scanMode === "full" ? { captureLegacyHistory: true } : {})
});
// Stored and already-stored turns both count: a re-import must freeze what a failed freeze left.
if (result.result?.sessionId) (stats.sessionIds ??= []).push(result.result.sessionId);
if (result.status === "pending" || result.status === "conflict") {
failed = true;
stats.failed += turn.messages.length;
Expand Down
35 changes: 35 additions & 0 deletions App/backend/src/services/tests/ingestion-service.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,41 @@ describe("native Codex ingestion", () => {
expect(markSeen).toHaveBeenCalledTimes(2);
});

it("freezes the sessions an import touched once it ends, and a failed freeze does not fail the import", async () => {
const completeSourceTurn = vi.fn().mockResolvedValue({ status: "stored", result: { sessionId: "s-1", l1MemoryIds: ["l1-native"] } });
const freezeScanSessions = vi.fn().mockResolvedValue({ batches: 1 });
await createService({ completeSourceTurn, freezeScanSessions }).ingest(toAsyncIterable(nativeMessages()), { sourceId: "codex" });
expect(freezeScanSessions).toHaveBeenCalledWith("codex", ["s-1"]);

const failing = vi.fn().mockRejectedValue(new Error("memory unavailable"));
const warn = vi.spyOn(console, "warn").mockImplementation(() => undefined);
const stats = await createService({ completeSourceTurn, freezeScanSessions: failing })
.ingest(toAsyncIterable(nativeMessages()), { sourceId: "codex" });
expect(stats.memoryIds).toEqual(["l1-native"]);
expect(warn).toHaveBeenCalledWith(expect.stringContaining("l3.freeze_failed"));
warn.mockRestore();

// Importing the same history again returns "existing"; the freeze is asked again so it can catch up.
const again = vi.fn().mockResolvedValue({ batches: 1 });
const existing = vi.fn().mockResolvedValue({ status: "existing", result: { sessionId: "s-1", l1MemoryIds: ["l1-native"] } });
await createService({ completeSourceTurn: existing, freezeScanSessions: again }).ingest(toAsyncIterable(nativeMessages()), { sourceId: "codex" });
expect(again).toHaveBeenCalledWith("codex", ["s-1"]);
});

it("still freezes what an import added when the import stops with an error", async () => {
const completeSourceTurn = vi.fn().mockResolvedValue({ status: "stored", result: { sessionId: "s-1", l1MemoryIds: ["l1-native"] } });
const freezeScanSessions = vi.fn().mockResolvedValue({ batches: 1 });
async function* brokenSource() {
yield* nativeMessages();
// A later conversation fails to read after the first one was stored.
yield { ...nativeMessages()[0]!, conversationId: "next-conversation", messageId: "next-0" };
throw new Error("source read failed");
}
await expect(createService({ completeSourceTurn, freezeScanSessions }).ingest(brokenSource(), { sourceId: "codex" }))
.rejects.toThrow("source read failed");
expect(freezeScanSessions).toHaveBeenCalledWith("codex", ["s-1"]);
});

it("emits add analytics only when a native scan turn is newly stored", async () => {
const events: Array<{ name: string; payload: Record<string, unknown> }> = [];
const analytics = {
Expand Down
5 changes: 5 additions & 0 deletions App/backend/src/tests/support/mock-memory-client.ts
Original file line number Diff line number Diff line change
Expand Up @@ -141,6 +141,11 @@ export function createMockMemoryClient(options: CreateMockMemoryClientOptions =
};
},

async freezeScanSessions() {
failIfNeeded();
return { batches: 0 };
},

async completeSourceTurn(input) {
failIfNeeded();
const l1MemoryId = randomUUID();
Expand Down
7 changes: 7 additions & 0 deletions App/memmy-agent/src/core/agent-runtime/loop.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1295,6 +1295,13 @@ export class AgentLoop {
return initialization;
}

/** Long-running service only: keep Memory's copy of the desktop project list current. */
startL3UserProjectSync(): void {
const projectStore = this.projectStore;
if (!projectStore) return;
this.memmyMemoryIntegration.startL3UserProjectSync?.(() => projectStore.snapshot());
}

async closeRuntimeTools(): Promise<void> {
this.messageLoopReady = false;
this.runtimeToolsClosed = true;
Expand Down
1 change: 1 addition & 0 deletions App/memmy-agent/src/entrypoints/cli/commands.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1161,6 +1161,7 @@ export async function gateway({
cronService: cron,
projectStore,
});
loop.startL3UserProjectSync?.();
reportStartupStage("agent-loop-created");
if (loop.sessions instanceof SessionManager) {
loop.guiTranscriptMirror = new GuiTranscriptMirror(loop.sessions, canonicalWorkspace);
Expand Down
10 changes: 10 additions & 0 deletions App/memmy-agent/src/memmy-memory/client.ts
Original file line number Diff line number Diff line change
@@ -1,11 +1,14 @@
import type { MemmyMemoryConnection, MemmyMemoryRequestEnvelope, JsonRecord } from "./types.js";
import { normalizeTimeZoneOffset } from "../utils/time-zone.js";
import {
L3UserProjectsReplaceResponseSchema,
L3WorldModelBoundaryResponseSchema,
L3WorldModelTraceHeadResponseSchema,
MemoryHealthSnapshotSchema,
SessionL3WorldModelContextResponseSchema,
l3WorldModelGetTransport,
type L3UserProjectsReplaceRequest,
type L3UserProjectsReplaceResponse,
type L3WorldModelBoundaryRequest,
type L3WorldModelBoundaryResponse,
type L3WorldModelRequestEnvelope,
Expand Down Expand Up @@ -179,6 +182,13 @@ export class MemmyMemoryClient {
return SessionL3WorldModelContextResponseSchema.parse(value);
}

async replaceL3UserProjects(
request: L3UserProjectsReplaceRequest,
options: MemoryRequestOptions = {}
): Promise<L3UserProjectsReplaceResponse> {
return L3UserProjectsReplaceResponseSchema.parse(await this.post("/api/v1/l3/user-projects", request, options));
}

}

function safeJsonParse(text: string): any {
Expand Down
82 changes: 82 additions & 0 deletions App/memmy-agent/src/memmy-memory/l3-user-project-sync.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,82 @@
import { randomUUID } from "node:crypto";
import type { L3UserProjectsReplaceRequest } from "@memmy/local-api-contracts";
import type { WebuiProjectRegistrySnapshot } from "../entrypoints/frontend-bridge/projects.js";
import type { MemmyMemoryClient } from "./client.js";

export const L3_USER_PROJECT_SYNC_INTERVAL_MS = 60_000;
/** An unchanged list is still pushed this often, so Memory gets it back after its data was cleared. */
export const L3_USER_PROJECT_RESYNC_MS = 10 * 60_000;

type Envelope = Omit<L3UserProjectsReplaceRequest, "requestId" | "projects" | "standaloneRoots">;

/**
* Keeps Memory's copy of the user's selected project folders current. It reads
* the registry on a timer and pushes the full list only when it differs from
* the last successful push, so startup, edits made by any process and a Memory
* service that was not reachable yet are all covered by the same loop. An
* unchanged list is pushed again every ten minutes in case Memory lost it.
*/
export class L3UserProjectSync {
private lastPushedKey: string | null = null;
private lastPushedAt = 0;
private inFlight: Promise<boolean> | null = null;
private timer: ReturnType<typeof setInterval> | null = null;

constructor(private readonly options: {
client: Pick<MemmyMemoryClient, "replaceL3UserProjects">;
envelope: Envelope;
readProjects: () => WebuiProjectRegistrySnapshot;
standaloneRoots: () => string[];
intervalMs?: number;
now?: () => number;
}) {}

start(): void {
if (this.timer) return;
void this.syncNow();
this.timer = setInterval(() => void this.syncNow(), this.options.intervalMs ?? L3_USER_PROJECT_SYNC_INTERVAL_MS);
this.timer.unref?.();
}

stop(): void {
if (this.timer) clearInterval(this.timer);
this.timer = null;
}

/** Pushes when the list changed or the last push failed; never throws. A corrupt registry is not pushed. */
syncNow(): Promise<boolean> {
if (this.inFlight) return this.inFlight;
this.inFlight = this.pushIfChanged().finally(() => {
this.inFlight = null;
});
return this.inFlight;
}

private async pushIfChanged(): Promise<boolean> {
let snapshot: WebuiProjectRegistrySnapshot;
try {
snapshot = this.options.readProjects();
} catch {
return false;
}
if (snapshot.state !== "ready") return false;
const projects = snapshot.projects.map((project) => ({ rootPath: project.rootPath, name: project.name }));
const standaloneRoots = this.options.standaloneRoots();
const key = JSON.stringify({ projects, standaloneRoots });
const now = (this.options.now ?? Date.now)();
if (key === this.lastPushedKey && now - this.lastPushedAt < L3_USER_PROJECT_RESYNC_MS) return true;
try {
await this.options.client.replaceL3UserProjects({
...this.options.envelope,
requestId: randomUUID(),
projects,
standaloneRoots
});
this.lastPushedKey = key;
this.lastPushedAt = now;
return true;
} catch {
return false;
}
}
}
Loading
Loading