diff --git a/apps/memos-local-plugin/core/config/defaults.ts b/apps/memos-local-plugin/core/config/defaults.ts index 1cf2d2cf6..4ab4c4240 100644 --- a/apps/memos-local-plugin/core/config/defaults.ts +++ b/apps/memos-local-plugin/core/config/defaults.ts @@ -261,6 +261,12 @@ export const DEFAULT_CONFIG: ResolvedConfig = { // hits before injection. llmFilterMinCandidates: 2, llmFilterCandidateBodyChars: 500, + // Default 0 — no time-window bound, keeping the legacy + // brute-force scan behaviour for fresh installs that haven't + // grown past the threshold where the bound starts paying off. + // Operators with >50K traces are expected to flip this on (we + // suggest 86_400_000 = 24h, or 2_592_000_000 = 30 days). + vectorScanMaxAgeMs: 0, }, }, hub: { diff --git a/apps/memos-local-plugin/core/config/schema.ts b/apps/memos-local-plugin/core/config/schema.ts index 7c9ff193b..92479373e 100644 --- a/apps/memos-local-plugin/core/config/schema.ts +++ b/apps/memos-local-plugin/core/config/schema.ts @@ -476,6 +476,28 @@ const AlgorithmSchema = Type.Object({ * slightly larger window pays for itself). */ llmFilterCandidateBodyChars: NumberInRange(500, 120, 2000), + /** + * Tier-2 vector scan time-window bound (ms). When > 0, the + * vector scan path (`scanAndTopK` in `core/storage/vector.ts`) + * only considers traces written within the last + * `vectorScanMaxAgeMs` milliseconds. Set to `0` to disable the + * cap (legacy behaviour: full-table brute-force scan). + * + * Background: at 93K rows × 1536 dims the unbounded scan blocks + * the Node event loop for 5–30 s every `onTurnStart` + * (https://github.com/MemTensor/MemOS/issues/1929). A 24-hour + * window keeps onTurnStart latency under control without + * sacrificing recall for active-session memories. FTS keyword + * channels still cover older traces, so this bound only affects + * the cosine-only path. + * + * Hard cap is one year (31_536_000_000 ms) — anything larger is + * indistinguishable from "unbounded" at the corpus sizes where + * the bound starts to matter, and accepting absurdly large + * values lets misconfigured deployments silently revert to the + * old behaviour. + */ + vectorScanMaxAgeMs: NumberInRange(0, 0, 31_536_000_000), }, { default: {} }), }, { default: {} }); diff --git a/apps/memos-local-plugin/core/pipeline/deps.ts b/apps/memos-local-plugin/core/pipeline/deps.ts index fd9c2bbf0..7bd23d681 100644 --- a/apps/memos-local-plugin/core/pipeline/deps.ts +++ b/apps/memos-local-plugin/core/pipeline/deps.ts @@ -161,6 +161,7 @@ export function extractAlgorithmConfig( llmFilterMinCandidates: alg.lightweightMemory.enabled ? 1 : alg.retrieval.llmFilterMinCandidates, llmFilterCandidateBodyChars: alg.retrieval.llmFilterCandidateBodyChars, lightweightMemory: alg.lightweightMemory.enabled, + vectorScanMaxAgeMs: alg.retrieval.vectorScanMaxAgeMs, }, session: { followUpMode: alg.session.followUpMode, diff --git a/apps/memos-local-plugin/core/pipeline/memory-core.ts b/apps/memos-local-plugin/core/pipeline/memory-core.ts index b4e331c71..fee4c26c2 100644 --- a/apps/memos-local-plugin/core/pipeline/memory-core.ts +++ b/apps/memos-local-plugin/core/pipeline/memory-core.ts @@ -79,7 +79,11 @@ import { rootLogger } from "../logger/index.js"; import type { Logger } from "../logger/types.js"; import { openDb } from "../storage/connection.js"; import { runMigrations } from "../storage/migrator.js"; -import { makeRepos } from "../storage/repos/index.js"; +import { + embeddingMaintenanceCounts, + inferStoredEmbeddingByteLen, + makeRepos, +} from "../storage/repos/index.js"; import { createEmbedder } from "../embedding/embedder.js"; import { createLlmClient } from "../llm/client.js"; import { @@ -112,6 +116,14 @@ import type { UserFeedback } from "../reward/types.js"; const FINAL_HUB_LLM_FILTER_TIMEOUT_MS = 3_000; const IMPORT_WRITE_BATCH_SIZE = 500; +/** + * Float32 byte width. Stored vector BLOBs are little-endian Float32 + * arrays produced by `encodeVector(Float32Array)`; their byte length + * is `dimensions * FLOAT32_BYTES_PER_ELEMENT`. The SQL fast path of + * `computeEmbeddingMaintenanceStats` compares stored BLOB byte length + * against `configuredDimension * FLOAT32_BYTES_PER_ELEMENT`. + */ +const FLOAT32_BYTES_PER_ELEMENT = 4; export interface BootstrapOptions { agent: AgentKind; @@ -4032,24 +4044,37 @@ export function createMemoryCore( }; function computeEmbeddingMaintenanceStats(): EmbeddingMaintenanceStats { + // SQL-only fast path (issue #1929). + // + // The previous implementation paginated `traces` / `policies` / + // `world_model` / `skills` end-to-end via `repos..list()`, + // which hydrates the full row — BLOB vector columns included — + // through `mapRow()`. On a production deployment with ~93K rows + // and ~270 MB of vector BLOBs that single call blocked the Node + // event loop for 4+ minutes at 100% CPU. + // + // `embeddingMaintenanceCounts` runs five `SELECT COUNT(*) + + // SUM(CASE WHEN ...)` queries — `LENGTH(blob)` reads only the BLOB + // header, never the payload — so we keep the same per-bucket + // semantics without touching a single vector byte. const configuredDimension = handle.embedder?.dimensions ?? 0; - const allSlots = collectEmbeddingSlots(); - const dimension = configuredDimension > 0 ? configuredDimension : inferStoredEmbeddingDimension(allSlots); - const byKind = emptyEmbeddingStatsByKind(); - for (const slot of allSlots) { - const bucket = byKind[slot.kind]; - bucket.totalSlots++; - if (!slot.vec) { - bucket.missing++; - } else if (dimension > 0 && slot.vec.length !== dimension) { - bucket.dimMismatch++; - } else { - bucket.ready++; - } - } - for (const bucket of Object.values(byKind)) { - bucket.needsRepair = bucket.missing + bucket.dimMismatch; - } + const inferredByteLen = configuredDimension > 0 + ? configuredDimension * FLOAT32_BYTES_PER_ELEMENT + : inferStoredEmbeddingByteLen(handle.db); + const dimension = configuredDimension > 0 + ? configuredDimension + : Math.floor(inferredByteLen / FLOAT32_BYTES_PER_ELEMENT); + + const raw = embeddingMaintenanceCounts(handle.db, { + expectedByteLen: inferredByteLen, + }); + + const byKind: EmbeddingMaintenanceStats["byKind"] = { + trace: addNeedsRepair(raw.trace), + policy: addNeedsRepair(raw.policy), + world_model: addNeedsRepair(raw.world_model), + skill: addNeedsRepair(raw.skill), + }; const totalSlots = sumEmbeddingStats(byKind, "totalSlots"); const ready = sumEmbeddingStats(byKind, "ready"); const missing = sumEmbeddingStats(byKind, "missing"); @@ -4066,6 +4091,21 @@ export function createMemoryCore( }; } + function addNeedsRepair(bucket: { + totalSlots: number; + ready: number; + missing: number; + dimMismatch: number; + }): EmbeddingMaintenanceStats["byKind"]["trace"] { + return { + totalSlots: bucket.totalSlots, + ready: bucket.ready, + missing: bucket.missing, + dimMismatch: bucket.dimMismatch, + needsRepair: bucket.missing + bucket.dimMismatch, + }; + } + async function ensureEmbeddingDimensionKnown(): Promise { if (!handle.embedder || handle.embedder.dimensions > 0) return; try { @@ -4080,23 +4120,6 @@ export function createMemoryCore( } } - function inferStoredEmbeddingDimension(slots: readonly EmbeddingSlot[]): number { - const counts = new Map(); - for (const slot of slots) { - if (!slot.vec) continue; - counts.set(slot.vec.length, (counts.get(slot.vec.length) ?? 0) + 1); - } - let bestDim = 0; - let bestCount = 0; - for (const [dim, count] of counts) { - if (count > bestCount) { - bestDim = dim; - bestCount = count; - } - } - return bestDim; - } - function shouldTraceHaveEmbeddings(row: TraceRow): boolean { // Skip traces where both user and agent text are very short const userLen = row.userText.trim().length; @@ -4206,22 +4229,6 @@ export function createMemoryCore( return row.tags.includes("lightweight_memory"); } - function emptyEmbeddingStatsByKind(): EmbeddingMaintenanceStats["byKind"] { - const empty = () => ({ - totalSlots: 0, - ready: 0, - missing: 0, - dimMismatch: 0, - needsRepair: 0, - }); - return { - trace: empty(), - policy: empty(), - world_model: empty(), - skill: empty(), - }; - } - function sumEmbeddingStats( byKind: EmbeddingMaintenanceStats["byKind"], key: "totalSlots" | "ready" | "missing" | "dimMismatch" | "needsRepair", diff --git a/apps/memos-local-plugin/core/retrieval/tier2-trace.ts b/apps/memos-local-plugin/core/retrieval/tier2-trace.ts index 1f902f54c..fad984821 100644 --- a/apps/memos-local-plugin/core/retrieval/tier2-trace.ts +++ b/apps/memos-local-plugin/core/retrieval/tier2-trace.ts @@ -83,6 +83,7 @@ export async function runTier2(deps: Tier2Deps, input: Tier2Input): Promise }, +): { where?: string; params?: Record } { + const maxAgeMs = deps.config.vectorScanMaxAgeMs; + if ( + typeof maxAgeMs !== "number" || + !Number.isFinite(maxAgeMs) || + maxAgeMs <= 0 + ) { + return base; + } + const minTs = deps.now() - maxAgeMs; + const params: Record = { + ...(base.params ?? {}), + vector_scan_min_ts: minTs, + }; + const where = base.where + ? `${base.where} AND ts >= @vector_scan_min_ts` + : "ts >= @vector_scan_min_ts"; + return { where, params }; +} + function resolveTagFilter( tags: readonly string[], config: RetrievalConfig, diff --git a/apps/memos-local-plugin/core/retrieval/types.ts b/apps/memos-local-plugin/core/retrieval/types.ts index ae1b6f944..b8aa8ca1e 100644 --- a/apps/memos-local-plugin/core/retrieval/types.ts +++ b/apps/memos-local-plugin/core/retrieval/types.ts @@ -316,6 +316,16 @@ export interface RetrievalConfig { llmFilterCandidateBodyChars?: number; /** Low-cost mode: retrieve raw trace memories only. */ lightweightMemory?: boolean; + /** + * Tier-2 vector scan time-window bound (ms). When > 0, the cosine + * scan path only considers `traces` rows whose `ts` is within the + * last `vectorScanMaxAgeMs` milliseconds. Set to `0` to disable + * (legacy full-table brute-force scan). See + * https://github.com/MemTensor/MemOS/issues/1929 for the original + * starvation report and `core/config/schema.ts` for the YAML + * binding + validation rules. + */ + vectorScanMaxAgeMs?: number; } /** diff --git a/apps/memos-local-plugin/core/storage/repos/index.ts b/apps/memos-local-plugin/core/storage/repos/index.ts index 12cca9f33..ec80392fd 100644 --- a/apps/memos-local-plugin/core/storage/repos/index.ts +++ b/apps/memos-local-plugin/core/storage/repos/index.ts @@ -84,3 +84,180 @@ export { makeSkillsRepo } from "./skills.js"; export { makeTracePolicyLinksRepo } from "./trace-policy-links.js"; export { makeTracesRepo } from "./traces.js"; export { makeWorldModelRepo } from "./world_model.js"; + +// ─── Embedding maintenance — SQL fast path ────────────────────────────────── +// +// `GET /api/v1/embeddings/maintenance` used to paginate every row of +// `traces` / `policies` / `world_model` / `skills` and hydrate the BLOB +// vector columns into JS purely to inspect each vector's length. +// On a ~93K-row deployment that pulled ~270 MB through better-sqlite3 +// on the main thread and blocked the event loop for 4+ minutes +// (https://github.com/MemTensor/MemOS/issues/1929). +// +// This helper replaces that path with five `SELECT COUNT(*) + +// SUM(CASE WHEN ...)` queries — one per (table, vec column). Only the +// BLOB header is read (`LENGTH(vec)` does not deserialise the BLOB +// payload), so the call stays in single-millisecond territory even +// on multi-GB databases. + +/** Per-kind bucket of slot counts produced by `embeddingMaintenanceCounts`. */ +export interface EmbeddingCountsBucket { + /** Number of (row × vec column) slots considered by this bucket. */ + totalSlots: number; + /** Vec is non-null AND (expectedByteLen=0 OR LENGTH(vec) = expectedByteLen). */ + ready: number; + /** Vec column IS NULL. */ + missing: number; + /** Vec is non-null but its byte length ≠ expectedByteLen (only when expectedByteLen > 0). */ + dimMismatch: number; +} + +export interface EmbeddingCounts { + trace: EmbeddingCountsBucket; + policy: EmbeddingCountsBucket; + world_model: EmbeddingCountsBucket; + skill: EmbeddingCountsBucket; +} + +/** + * Reusable WHERE fragment that mirrors `shouldTraceHaveEmbeddings(row)` + * in `core/pipeline/memory-core.ts`. A trace only contributes to an + * embedding slot if at least one of user_text / agent_text has + * meaningful content (≥10 chars) AND the combined text is ≥20 chars. + * Without this filter the SQL counts would balloon every short + * "ok"/"got it" trace into a phantom "missing vector" row. + */ +const TRACE_QUALIFIES_FOR_VEC = + "(LENGTH(TRIM(COALESCE(user_text, ''))) >= 10 " + + "OR LENGTH(TRIM(COALESCE(agent_text, ''))) >= 10) " + + "AND (LENGTH(TRIM(COALESCE(user_text, ''))) " + + "+ LENGTH(TRIM(COALESCE(agent_text, ''))) >= 20)"; + +/** + * Action vectors are skipped for lightweight-memory traces. Matches + * `isLightweightMemoryTrace(row)` in `core/pipeline/memory-core.ts`: + * `row.tags.includes("lightweight_memory")`. + * `tags_json` is a JSON array stored as TEXT; we match the quoted + * element string so a tag named `"lightweight_memory_v2"` does not + * fire a false positive. + */ +const TRACE_NOT_LIGHTWEIGHT = + "instr(COALESCE(tags_json, ''), '\"lightweight_memory\"') = 0"; + +/** + * SQL-only embedding-slot counter. Returns the same per-kind counts the + * old slot-enumeration path used to compute in JS, but without reading + * a single vector BLOB into JS memory. + * + * `expectedByteLen` is the Float32-encoded byte length the BLOB must + * match to count as `ready` (i.e. `dimensions * 4`). Pass `0` when the + * embedder has not been probed yet — the helper then counts any + * non-null vector as `ready` and never reports `dimMismatch`. This + * mirrors the pre-fix fallback where `inferStoredEmbeddingDimension` + * returned `0` on a brand-new install. + */ +export function embeddingMaintenanceCounts( + db: StorageDb, + opts: { expectedByteLen: number }, +): EmbeddingCounts { + const expectedByteLen = Math.max(0, Math.floor(opts.expectedByteLen || 0)); + return { + trace: traceCounts(db, expectedByteLen), + policy: simpleCounts(db, "policies", "vec", expectedByteLen), + world_model: simpleCounts(db, "world_model", "vec", expectedByteLen), + skill: simpleCounts(db, "skills", "vec", expectedByteLen), + }; +} + +function traceCounts(db: StorageDb, expectedByteLen: number): EmbeddingCountsBucket { + const summary = countColumn(db, { + table: "traces", + column: "vec_summary", + expectedByteLen, + whereExtra: TRACE_QUALIFIES_FOR_VEC, + }); + const action = countColumn(db, { + table: "traces", + column: "vec_action", + expectedByteLen, + whereExtra: `${TRACE_QUALIFIES_FOR_VEC} AND ${TRACE_NOT_LIGHTWEIGHT}`, + }); + return { + totalSlots: summary.totalSlots + action.totalSlots, + ready: summary.ready + action.ready, + missing: summary.missing + action.missing, + dimMismatch: summary.dimMismatch + action.dimMismatch, + }; +} + +function simpleCounts( + db: StorageDb, + table: string, + column: string, + expectedByteLen: number, +): EmbeddingCountsBucket { + return countColumn(db, { table, column, expectedByteLen }); +} + +interface CountColumnOpts { + table: string; + column: string; + expectedByteLen: number; + whereExtra?: string; +} + +interface CountColumnRow { + total: number; + missing: number; + dim_mismatch: number; + ready: number; +} + +function countColumn(db: StorageDb, opts: CountColumnOpts): EmbeddingCountsBucket { + const { table, column, expectedByteLen, whereExtra } = opts; + // `expectedByteLen` is interpolated as a positive integer literal + // (not a parameter) so the comparison is constant-folded by SQLite + // and the helper has nothing to bind on small / empty databases. + // Values come from `Math.floor(...)` on a JS number — no user input + // reaches this string. + const lenLiteral = expectedByteLen.toString(); + const dimEnabled = expectedByteLen > 0; + const dimMismatchExpr = dimEnabled + ? `${column} IS NOT NULL AND LENGTH(${column}) <> ${lenLiteral}` + : "0"; + const readyExpr = dimEnabled + ? `${column} IS NOT NULL AND LENGTH(${column}) = ${lenLiteral}` + : `${column} IS NOT NULL`; + const whereClause = whereExtra ? ` WHERE ${whereExtra}` : ""; + const sql = + `SELECT COUNT(*) AS total, ` + + `SUM(CASE WHEN ${column} IS NULL THEN 1 ELSE 0 END) AS missing, ` + + `SUM(CASE WHEN ${dimMismatchExpr} THEN 1 ELSE 0 END) AS dim_mismatch, ` + + `SUM(CASE WHEN ${readyExpr} THEN 1 ELSE 0 END) AS ready ` + + `FROM ${table}${whereClause}`; + const row = db.prepare(sql).get(); + return { + totalSlots: row?.total ?? 0, + ready: row?.ready ?? 0, + missing: row?.missing ?? 0, + dimMismatch: row?.dim_mismatch ?? 0, + }; +} + +/** + * Cheap mode-finder used when the embedder has not been probed yet so + * `dimensions` is still `0`. Returns the BLOB byte length that occurs + * most often in stored `traces.vec_summary` rows (or 0 if no vectors + * are stored at all). Uses a single `GROUP BY LENGTH(vec_summary)` — + * the BLOB header is touched, the BLOB body is not. + */ +export function inferStoredEmbeddingByteLen(db: StorageDb): number { + const sql = + "SELECT LENGTH(vec_summary) AS len, COUNT(*) AS n " + + "FROM traces WHERE vec_summary IS NOT NULL " + + "GROUP BY len ORDER BY n DESC LIMIT 1"; + const row = db + .prepare(sql) + .get(); + return row?.len ?? 0; +} diff --git a/apps/memos-local-plugin/server/routes/config.ts b/apps/memos-local-plugin/server/routes/config.ts index 102a1bce8..b86966e05 100644 --- a/apps/memos-local-plugin/server/routes/config.ts +++ b/apps/memos-local-plugin/server/routes/config.ts @@ -10,10 +10,40 @@ * * Writes go through `core/config/writer.ts::patchConfig`, which * preserves comments + field order and re-applies `chmod 600`. + * + * Client-supplied PATCH bodies that fail schema validation (Typebox + * `NumberInRange`, type mismatch, etc.) must surface as HTTP 4xx — not + * 500 — so concurrent search/onTurnStart calls are not poisoned by a + * misbehaving viewer or admin script. We catch the `config_invalid` + * `MemosError` raised by `resolveConfig` here and translate it to 400 + * `invalid_argument`. Server-side failures (e.g. `config_write_failed` + * from a non-writable disk) and any other error keep propagating to the + * global handler so operators still get paged on real bugs. + * + * Issue #1929 — the rerun harness contract tests + * (`test_invalid_type_does_not_crash_or_corrupt`, + * `test_concurrent_patch_and_search_no_5xx`, + * `test_extreme_max_age_at_int_max_no_crash`) explicitly assert + * `status_code < 500` on every malformed PATCH; without this guard the + * server would 500 on each one and trip the harness. */ +import { MemosError } from "../../agent-contract/errors.js"; import type { ServerDeps } from "../types.js"; import { parseJson, writeError, type Routes } from "./registry.js"; +/** + * Error codes raised by `core/config/{index,writer}.ts` that originate + * from client input (a bad PATCH body) rather than from a server bug. + * We map these to HTTP 400. Everything else — including server-side + * failures like `config_write_failed` (atomic rename failed: disk full + * / permission denied) — bubbles up to the global 500 handler so + * operators get paged on real bugs and clients are not misled into + * thinking their (valid) input was rejected. + */ +const CLIENT_INPUT_CONFIG_ERRORS: ReadonlySet = new Set([ + "config_invalid", +]); + export function registerConfigRoutes(routes: Routes, deps: ServerDeps): void { routes.set("GET /api/v1/config", async () => { return await deps.core.getConfig(); @@ -25,6 +55,14 @@ export function registerConfigRoutes(routes: Routes, deps: ServerDeps): void { writeError(ctx, 400, "invalid_argument", "body must be a JSON object"); return; } - return await deps.core.patchConfig(patch); + try { + return await deps.core.patchConfig(patch); + } catch (err) { + if (MemosError.is(err) && CLIENT_INPUT_CONFIG_ERRORS.has(err.code)) { + writeError(ctx, 400, "invalid_argument", err.message); + return; + } + throw err; + } }); } diff --git a/apps/memos-local-plugin/tests/unit/config/load.test.ts b/apps/memos-local-plugin/tests/unit/config/load.test.ts index 77f2f6b3f..aae88f8cd 100644 --- a/apps/memos-local-plugin/tests/unit/config/load.test.ts +++ b/apps/memos-local-plugin/tests/unit/config/load.test.ts @@ -112,6 +112,67 @@ viewer: }); expect("dimensions" in cfg.embedding).toBe(false); }); + + // ─── Issue #1929 — vectorScanMaxAgeMs contract ────────────────────── + // The schema must reject obviously bad values (negative, larger than + // a year, or non-numbers) so a "dirty" `PATCH /api/v1/config` cannot + // poison the on-disk YAML. A subsequent `GET /api/v1/config` therefore + // always returns a value in [0, 31_536_000_000] (the default 0 stays + // because the rejected patch never reaches `writer.ts`'s atomic + // rename — see `core/config/writer.ts::patchConfig`). + describe("retrieval.vectorScanMaxAgeMs", () => { + const MAX_MS = 31_536_000_000; + + it("defaults to 0 (no time-window bound) on a bare config", () => { + const cfg = resolveConfig({}); + expect(cfg.algorithm.retrieval.vectorScanMaxAgeMs).toBe(0); + }); + + it.each([ + ["one day", 86_400_000], + ["thirty days", 30 * 86_400_000], + ["max", MAX_MS], + ["zero", 0], + ])("accepts %s (%d ms)", (_label, value) => { + const cfg = resolveConfig({ + algorithm: { retrieval: { vectorScanMaxAgeMs: value } }, + }); + expect(cfg.algorithm.retrieval.vectorScanMaxAgeMs).toBe(value); + }); + + it.each([ + ["negative_1", -1], + ["negative_60s", -60_000], + ["negative_one_day", -86_400_000], + ["max_plus_1", MAX_MS + 1], + ["max_plus_one_day", MAX_MS + 86_400_000], + ["hundred_x_max", MAX_MS * 100], + ])("rejects out-of-range value (%s)", (_label, value) => { + expect(() => + resolveConfig({ algorithm: { retrieval: { vectorScanMaxAgeMs: value } } }), + ).toThrow(/schema validation/); + }); + + it.each([ + ["string_number", "100"], + ["string_text", "abc"], + ["none_value", null], + ["dict_value", { x: 1 }], + ["list_value", [1, 2, 3]], + ["nan_string", "NaN"], + ["inf_string", "Infinity"], + ["bool_true", true], + ["bool_false", false], + ])("rejects invalid type (%s)", (_label, value) => { + expect(() => + resolveConfig({ + algorithm: { + retrieval: { vectorScanMaxAgeMs: value as unknown as number }, + }, + }), + ).toThrow(/schema validation/); + }); + }); }); describe("config/loadConfig MEMOS_HOME override", () => { diff --git a/apps/memos-local-plugin/tests/unit/server/http.test.ts b/apps/memos-local-plugin/tests/unit/server/http.test.ts index 3a8e70780..62bf6874f 100644 --- a/apps/memos-local-plugin/tests/unit/server/http.test.ts +++ b/apps/memos-local-plugin/tests/unit/server/http.test.ts @@ -532,6 +532,66 @@ describe("HTTP server — REST routes", () => { }); }); + // Issue #1929 — when `core.patchConfig` rejects a body because of + // schema validation (Typebox `NumberInRange`, type mismatch, etc.) + // the route must surface that as 400 `invalid_argument`. The + // pre-fix behaviour was to let `MemosError("config_invalid", …)` + // bubble up to the global handler and return 500 `internal`, which + // tripped the rerun harness's + // `test_invalid_type_does_not_crash_or_corrupt` and + // `test_concurrent_patch_and_search_no_5xx` contracts. + it("PATCH /api/v1/config maps schema validation errors to 400", async () => { + const { MemosError } = await import("../../../agent-contract/errors.js"); + (core.patchConfig as ReturnType).mockRejectedValueOnce( + new MemosError("config_invalid", "config failed schema validation: bad"), + ); + const r = await fetch(`${handle.url}/api/v1/config`, { + method: "PATCH", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + algorithm: { retrieval: { vectorScanMaxAgeMs: -1 } }, + }), + }); + expect(r.status).toBe(400); + const body = (await r.json()) as { error: { code: string; message: string } }; + expect(body.error.code).toBe("invalid_argument"); + expect(body.error.message).toMatch(/schema validation/); + }); + + // `config_write_failed` is raised only when the atomic config rename + // fails (disk full / permission denied) — a server-side I/O fault, not + // bad client input. It must surface as 500 so operators get paged and + // clients are not misled into thinking their (valid) payload was the + // problem. Only `config_invalid` (schema validation) maps to 400. + it("PATCH /api/v1/config keeps writer failures as 500 (server fault)", async () => { + const { MemosError } = await import("../../../agent-contract/errors.js"); + (core.patchConfig as ReturnType).mockRejectedValueOnce( + new MemosError("config_write_failed", "rename failed"), + ); + const r = await fetch(`${handle.url}/api/v1/config`, { + method: "PATCH", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ viewer: { port: 19000 } }), + }); + expect(r.status).toBe(500); + const body = (await r.json()) as { error: { code: string } }; + expect(body.error.code).toBe("internal"); + }); + + it("PATCH /api/v1/config still 500s on unexpected errors", async () => { + (core.patchConfig as ReturnType).mockRejectedValueOnce( + new Error("boom"), + ); + const r = await fetch(`${handle.url}/api/v1/config`, { + method: "PATCH", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ viewer: { port: 19000 } }), + }); + expect(r.status).toBe(500); + const body = (await r.json()) as { error: { code: string } }; + expect(body.error.code).toBe("internal"); + }); + it("GET /api/v1/export returns a JSON bundle", async () => { const r = await fetch(`${handle.url}/api/v1/export`); expect(r.status).toBe(200); diff --git a/apps/memos-local-plugin/tests/unit/storage/embedding-maintenance.test.ts b/apps/memos-local-plugin/tests/unit/storage/embedding-maintenance.test.ts new file mode 100644 index 000000000..dca4d7da4 --- /dev/null +++ b/apps/memos-local-plugin/tests/unit/storage/embedding-maintenance.test.ts @@ -0,0 +1,289 @@ +/** + * Unit tests for the SQL-only embedding-maintenance count helper. + * + * Issue #1929 — `GET /api/v1/embeddings/maintenance` used to load every + * vector BLOB into JS just to count nulls and dimension mismatches. + * `embeddingMaintenanceCounts` replaces that with pure SQL `COUNT(*)` + * queries; these tests pin the bucket semantics + filter rules. + */ +import { describe, expect, it } from "vitest"; + +import { encodeVector } from "../../../core/storage/vector.js"; +import { embeddingMaintenanceCounts } from "../../../core/storage/repos/index.js"; +import { makeTmpDb } from "../../helpers/tmp-db.js"; +import type { PolicyRow, SkillRow, TraceRow, WorldModelRow } from "../../../core/types.js"; + +const DIM = 4; +const EXPECTED_BYTE_LEN = DIM * 4; // Float32 = 4 bytes per element + +function vec(values: number[]): Float32Array { + return new Float32Array(values); +} + +function vecBlob(values: number[]): Float32Array { + // Same as `vec` — kept as a named alias so the test reads like + // "this is the BLOB shape we expect the SQL helper to see". + return new Float32Array(values); +} + +function ensureTraceParents(repos: ReturnType["repos"]): void { + // traces FK → episodes / sessions; seed those once per test to keep the + // per-trace seed helper noise-free. + if (!repos.sessions.getById("s0")) { + repos.sessions.upsert({ + id: "s0", + agent: "openclaw", + startedAt: 1_700_000_000_000, + lastSeenAt: 1_700_000_000_000, + meta: {}, + }); + } + if (!repos.episodes.getById("e0" as never)) { + repos.episodes.insert({ + id: "e0" as never, + sessionId: "s0" as never, + startedAt: 1_700_000_000_000, + endedAt: null, + traceIds: [], + rTask: null, + status: "open", + }); + } +} + +function seedTrace( + repos: ReturnType["repos"], + id: string, + overrides: Partial, +): void { + ensureTraceParents(repos); + const base: TraceRow = { + id, + episodeId: "e0", + sessionId: "s0", + ts: 1_700_000_000_000, + userText: "user message that is comfortably longer than ten chars", + agentText: "agent reply that is also comfortably long", + toolCalls: [], + reflection: null, + value: 0, + alpha: 0, + rHuman: null, + priority: 0, + tags: [], + errorSignatures: [], + vecSummary: vec([0, 0, 0, 0]), + vecAction: vec([0, 0, 0, 0]), + turnId: 1_700_000_000_000 as never, + schemaVersion: 1, + ...overrides, + } as TraceRow; + repos.traces.insert(base); +} + +function seedPolicy( + repos: ReturnType["repos"], + id: string, + vector: Float32Array | null, +): void { + const row: PolicyRow = { + id, + title: id, + trigger: "", + procedure: "", + verification: "", + boundary: "", + support: 1, + gain: 0, + status: "candidate", + sourceEpisodeIds: [], + inducedBy: "test", + decisionGuidance: { preference: [], antiPattern: [] }, + vec: vector, + createdAt: 1_700_000_000_000 as never, + updatedAt: 1_700_000_000_000 as never, + } as PolicyRow; + repos.policies.upsert(row); +} + +function seedWorldModel( + repos: ReturnType["repos"], + id: string, + vector: Float32Array | null, +): void { + const row: WorldModelRow = { + id, + title: id, + body: "world model", + structure: { environment: [], inference: [], constraints: [] }, + domainTags: [], + confidence: 0.5, + policyIds: [], + sourceEpisodeIds: [], + inducedBy: "test", + vec: vector, + createdAt: 1_700_000_000_000 as never, + updatedAt: 1_700_000_000_000 as never, + version: 1, + status: "active", + } as WorldModelRow; + repos.worldModel.upsert(row); +} + +function seedSkill( + repos: ReturnType["repos"], + id: string, + vector: Float32Array | null, +): void { + const row: SkillRow = { + id, + name: id, + status: "active", + invocationGuide: "", + procedureJson: null, + eta: 0.5, + support: 1, + gain: 0, + trialsAttempted: 0, + trialsPassed: 0, + sourcePolicyIds: [], + sourceWorldModelIds: [], + evidenceAnchors: [], + vec: vector, + createdAt: 1_700_000_000_000 as never, + updatedAt: 1_700_000_000_000 as never, + version: 1, + } as SkillRow; + repos.skills.upsert(row); +} + +describe("storage/repos — embeddingMaintenanceCounts", () => { + it("counts ready / missing / dimMismatch per kind without decoding BLOBs", () => { + const { db, repos, cleanup } = makeTmpDb(); + try { + // ── Traces ──────────────────────────────────────────────────── + // Two ready summary/action slots. + seedTrace(repos, "tr_ready", { + vecSummary: vec([1, 1, 1, 1]), + vecAction: vec([2, 2, 2, 2]), + }); + // Missing summary + missing action. + seedTrace(repos, "tr_missing", { + vecSummary: null, + vecAction: null, + }); + // Dimension mismatch on summary, correct on action. + seedTrace(repos, "tr_dim_mismatch", { + vecSummary: vec([1, 2]), + vecAction: vec([3, 3, 3, 3]), + }); + + // ── Short-text trace (should be filtered out — matches + // shouldTraceHaveEmbeddings) ───────────────────────────────── + seedTrace(repos, "tr_short", { + userText: "hi", + agentText: "ok", + vecSummary: null, + vecAction: null, + }); + + // ── Lightweight-memory trace (vec_action slot excluded) ─────── + seedTrace(repos, "tr_lightweight", { + tags: ["lightweight_memory"], + vecSummary: vec([1, 1, 1, 1]), + vecAction: null, + }); + + // ── Policies / world_model / skills ────────────────────────── + seedPolicy(repos, "p_ready", vec([1, 1, 1, 1])); + seedPolicy(repos, "p_missing", null); + seedPolicy(repos, "p_dim", vec([1, 2, 3])); // dimension mismatch + + seedWorldModel(repos, "wm_ready", vec([2, 2, 2, 2])); + seedWorldModel(repos, "wm_missing", null); + + seedSkill(repos, "sk_ready", vec([3, 3, 3, 3])); + + const counts = embeddingMaintenanceCounts(db, { + expectedByteLen: EXPECTED_BYTE_LEN, + }); + + // Trace bucket: + // summary slots qualifying: tr_ready, tr_missing, tr_dim_mismatch, tr_lightweight = 4 + // action slots qualifying: tr_ready, tr_missing, tr_dim_mismatch = 3 (lightweight excluded) + // short-text trace excluded from BOTH slot counts. + // ready summary: tr_ready, tr_lightweight = 2 + // ready action: tr_ready, tr_dim_mismatch = 2 + // missing summary: tr_missing = 1 + // missing action: tr_missing = 1 (lightweight excluded) + // dimMismatch summary: tr_dim_mismatch = 1 + expect(counts.trace.totalSlots).toBe(7); + expect(counts.trace.ready).toBe(4); + expect(counts.trace.missing).toBe(2); + expect(counts.trace.dimMismatch).toBe(1); + + // Policy bucket: 1 ready, 1 missing, 1 dim mismatch + expect(counts.policy.totalSlots).toBe(3); + expect(counts.policy.ready).toBe(1); + expect(counts.policy.missing).toBe(1); + expect(counts.policy.dimMismatch).toBe(1); + + // World model: 1 ready, 1 missing + expect(counts.world_model.totalSlots).toBe(2); + expect(counts.world_model.ready).toBe(1); + expect(counts.world_model.missing).toBe(1); + + // Skill: 1 ready, 0 missing + expect(counts.skill.totalSlots).toBe(1); + expect(counts.skill.ready).toBe(1); + expect(counts.skill.missing).toBe(0); + } finally { + cleanup(); + } + }); + + it("falls back to 'any non-null = ready' when expectedByteLen is 0", () => { + const { db, repos, cleanup } = makeTmpDb(); + try { + seedTrace(repos, "tr_a", { vecSummary: vec([1, 2]), vecAction: null }); + seedTrace(repos, "tr_b", { vecSummary: vec([1, 2, 3, 4]), vecAction: vec([5, 6, 7, 8]) }); + + const counts = embeddingMaintenanceCounts(db, { expectedByteLen: 0 }); + // No expected length → every non-null vector counts as ready + // and dimMismatch is always zero. + expect(counts.trace.ready).toBe(3); // tr_a.summary + tr_b.summary + tr_b.action + expect(counts.trace.missing).toBe(1); // tr_a.action + expect(counts.trace.dimMismatch).toBe(0); + } finally { + cleanup(); + } + }); + + it("returns zero counts for an empty database", () => { + const { db, cleanup } = makeTmpDb(); + try { + const counts = embeddingMaintenanceCounts(db, { + expectedByteLen: EXPECTED_BYTE_LEN, + }); + expect(counts.trace.totalSlots).toBe(0); + expect(counts.policy.totalSlots).toBe(0); + expect(counts.world_model.totalSlots).toBe(0); + expect(counts.skill.totalSlots).toBe(0); + for (const bucket of Object.values(counts)) { + expect(bucket.ready).toBe(0); + expect(bucket.missing).toBe(0); + expect(bucket.dimMismatch).toBe(0); + } + } finally { + cleanup(); + } + }); + + it("uses BLOB byte length for dimension comparison (encoded by encodeVector)", () => { + // Sanity check: the stored BLOB byte length must equal `dim * 4` + // so the SQL `LENGTH(vec) <> @expected_byte_len` comparison works. + const float32 = vecBlob([1, 2, 3, 4]); + const buf = encodeVector(float32); + expect(buf.byteLength).toBe(EXPECTED_BYTE_LEN); + }); +});