From 29f4df113f76d1919ab0c3fab2f63b9a28a4321a Mon Sep 17 00:00:00 2001 From: Dev Chiniwala Date: Thu, 17 Sep 2026 15:36:18 +0530 Subject: [PATCH 1/6] fix: honor memory scope filters during recall --- datamind/capabilities/memory/service.py | 12 ++++++++-- datamind/capabilities/memory/tools.py | 1 + datamind/tests/test_memory.py | 30 +++++++++++++++++++++++++ 3 files changed, 41 insertions(+), 2 deletions(-) diff --git a/datamind/capabilities/memory/service.py b/datamind/capabilities/memory/service.py index 882b858..c79b45b 100644 --- a/datamind/capabilities/memory/service.py +++ b/datamind/capabilities/memory/service.py @@ -92,12 +92,20 @@ async def recall( session_id: str | None = None, top_k: int = 8, kinds: Sequence[str] | None = None, + scope_filter: Sequence[str] | None = None, include_archived: bool = False, ) -> list[dict[str, Any]]: + effective_profile = profile or self._default_profile + effective_session = session_id + if scope_filter: + if 'profile' not in scope_filter: + effective_profile = None + if 'session' not in scope_filter: + effective_session = None hits = await self.long_term.recall( query, - profile=profile or self._default_profile, - session_id=session_id, + profile=effective_profile, + session_id=effective_session, top_k=top_k, kinds=kinds, include_archived=include_archived, diff --git a/datamind/capabilities/memory/tools.py b/datamind/capabilities/memory/tools.py index a328fa3..dc5eb50 100644 --- a/datamind/capabilities/memory/tools.py +++ b/datamind/capabilities/memory/tools.py @@ -82,6 +82,7 @@ async def _recall( session_id=eff_session, top_k=top_k, kinds=kinds, + scope_filter=scope_filter, ) return {"query": query, "count": len(hits), "results": hits} diff --git a/datamind/tests/test_memory.py b/datamind/tests/test_memory.py index 9b9c452..1f47a56 100644 --- a/datamind/tests/test_memory.py +++ b/datamind/tests/test_memory.py @@ -8,6 +8,7 @@ from datamind.capabilities.memory import ( MemoryService, ShortTermMemory, + build_memory_tools, ) from datamind.capabilities.memory.providers.sqlite_store import SQLiteMemoryStore from datamind.core.protocols import MemoryStore @@ -276,6 +277,35 @@ async def test_service_combines_short_and_long(tmp_path): assert await svc.forget(rid) +@pytest.mark.asyncio +async def test_memory_recall_scope_filter_excludes_unselected_scopes(tmp_path): + '''A global-only recall must not reintroduce the default profile.''' + lt = SQLiteMemoryStore(db_path=str(tmp_path / 'm.db'), embedding=_FakeEmbed()) + svc = MemoryService( + short_term=ShortTermMemory(max_turns=5), + long_term=lt, + default_profile='acme', + ) + tools = {tool.name: tool.handler for tool in build_memory_tools(svc)} + + await tools['memory_save']('global fact', scope='global') + await tools['memory_save']('profile fact', scope='profile', profile='acme') + await tools['memory_save']( + 'session fact', scope='session', session_id='chat-1' + ) + + result = await tools['memory_recall']( + 'fact', + profile='acme', + session_id='chat-1', + scope_filter=['global'], + ) + + assert result['count'] == 1 + assert result['results'][0]['scope'] == 'global' + assert result['results'][0]['content'] == 'global fact' + + @pytest.mark.asyncio async def test_service_default_profile_used_when_omitted(tmp_path): """save without explicit profile should land under default_profile.""" From 22661bf4832d24eb83ac80c315cde9a2e6e02108 Mon Sep 17 00:00:00 2001 From: Dev Chiniwala Date: Thu, 17 Sep 2026 16:10:48 +0530 Subject: [PATCH 2/6] fix: reject invalid SQL row limits --- datamind/capabilities/db/base.py | 2 ++ datamind/config.py | 2 +- datamind/tests/test_config.py | 7 ++++++- datamind/tests/test_db_safeguard.py | 16 ++++++++++++++++ 4 files changed, 25 insertions(+), 2 deletions(-) diff --git a/datamind/capabilities/db/base.py b/datamind/capabilities/db/base.py index d0bfeab..0ebfc32 100644 --- a/datamind/capabilities/db/base.py +++ b/datamind/capabilities/db/base.py @@ -106,6 +106,8 @@ async def execute_readonly( ) -> QueryResult: if not sql or not sql.strip(): raise CapabilityError("db", "Empty SQL") + if row_limit < 1: + raise CapabilityError("db", "row_limit must be at least 1") if contains_multiple_statements(sql): raise MultiStatementSQLError( "multiple statements are not allowed (use a single SELECT)" diff --git a/datamind/config.py b/datamind/config.py index ea512e1..b251a26 100644 --- a/datamind/config.py +++ b/datamind/config.py @@ -112,7 +112,7 @@ class DBConfig(BaseModel): dialect: str = "sqlite" # sqlite | mysql | postgres | ... dsn: str | None = None # e.g. mysql+pymysql://user:pw@host/db read_only: bool = True - row_limit: int = 1000 + row_limit: int = Field(default=1000, ge=1) query_timeout_s: float = 10.0 diff --git a/datamind/tests/test_config.py b/datamind/tests/test_config.py index 8dda7ea..11743ed 100644 --- a/datamind/tests/test_config.py +++ b/datamind/tests/test_config.py @@ -14,7 +14,7 @@ import pytest from pydantic import ValidationError -from datamind.config import Settings +from datamind.config import DBConfig, Settings def test_nested_env_hydrates_required_llm(monkeypatch, tmp_path): @@ -101,3 +101,8 @@ def test_ensure_dirs_is_idempotent(monkeypatch, tmp_path): assert (tmp_path / "data" / "profiles" / "tp").is_dir() assert (tmp_path / "storage" / "tp").is_dir() + + +def test_db_config_rejects_nonpositive_row_limit(): + with pytest.raises(ValidationError): + DBConfig(row_limit=0) diff --git a/datamind/tests/test_db_safeguard.py b/datamind/tests/test_db_safeguard.py index bda0bd6..874043f 100644 --- a/datamind/tests/test_db_safeguard.py +++ b/datamind/tests/test_db_safeguard.py @@ -12,6 +12,8 @@ leading_verb, strip_comments, ) +from datamind.capabilities.db.providers.sqlite import SQLiteDialect +from datamind.core.errors import CapabilityError @pytest.mark.parametrize( @@ -67,3 +69,17 @@ def test_ensure_row_limit_strips_trailing_semicolon(): def test_strip_comments(): assert strip_comments("SELECT /* x */ 1") == "SELECT 1" assert strip_comments("-- line\nSELECT 1") == "\nSELECT 1" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("row_limit", [0, -1]) +async def test_execute_readonly_rejects_nonpositive_row_limit(tmp_path, row_limit): + dialect = SQLiteDialect() + engine = dialect.build_engine( + None, default_path=str(tmp_path / "db.sqlite") + ) + + with pytest.raises(CapabilityError, match="row_limit"): + await dialect.execute_readonly( + engine, "SELECT 1", row_limit=row_limit + ) From 8d69e562d9c1851d1d4119e2cfe3842c0a7aeae3 Mon Sep 17 00:00:00 2001 From: Dev Chiniwala Date: Thu, 17 Sep 2026 16:15:14 +0530 Subject: [PATCH 3/6] fix: preserve duplicate CSV columns --- datamind/capabilities/ingest/service.py | 17 +++++++++++++++-- datamind/tests/test_replace_receipts.py | 14 ++++++++++++++ 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/datamind/capabilities/ingest/service.py b/datamind/capabilities/ingest/service.py index b04387e..1510c43 100644 --- a/datamind/capabilities/ingest/service.py +++ b/datamind/capabilities/ingest/service.py @@ -882,12 +882,25 @@ async def db_import_csv( # Sanitise column names: same rule as table names. safe_cols: list[str] = [] - for raw in header: + used_cols: set[str] = set() + for index, raw in enumerate(header, start=1): col = raw.strip() if not re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]{0,63}", col): # Fall back to col_ if header is unusable. - col = f"col_{len(safe_cols) + 1}" + col = f"col_{index}" + if col in used_cols: + # SQL tables cannot contain duplicate column names. Keep the + # first header unchanged and give later occurrences a stable + # fallback name without losing their values in the row dict. + base = f"col_{index}" + col = base + suffix = 2 + while col in used_cols: + suffix_text = f"_{suffix}" + col = f"{base[:64 - len(suffix_text)]}{suffix_text}" + suffix += 1 safe_cols.append(col) + used_cols.add(col) rows: list[dict[str, str]] = [] for raw_row in reader: diff --git a/datamind/tests/test_replace_receipts.py b/datamind/tests/test_replace_receipts.py index ee4b2d6..285199d 100644 --- a/datamind/tests/test_replace_receipts.py +++ b/datamind/tests/test_replace_receipts.py @@ -77,3 +77,17 @@ async def test_append_retry_remains_deduplicated(database, tmp_path): assert second["results"][0]["status"] == "unchanged" assert second["revision"] == first["revision"] assert (await db.query_sql("SELECT amount FROM sales")).rows == [["200"]] + + +@pytest.mark.asyncio +async def test_csv_import_keeps_duplicate_headers_as_distinct_columns(database, tmp_path): + db, raw = database + source = tmp_path / "duplicate-columns.csv" + source.write_text("a,a\nfirst,second\n", encoding="utf-8") + + receipt = await raw.get("db_import_csv").handler( + path=str(source), table="duplicate_columns", if_exists="replace" + ) + + assert receipt["columns"] == ["a", "col_2"] + assert (await db.query_sql("SELECT * FROM duplicate_columns")).rows == [["first", "second"]] From dd9e7f4c33b630234b24ae167b6ae713603beca6 Mon Sep 17 00:00:00 2001 From: Dev Chiniwala Date: Thu, 17 Sep 2026 16:17:16 +0530 Subject: [PATCH 4/6] fix: redact secrets from audit errors --- datamind/capabilities/hooks/audit.py | 21 +++++++++++++++++---- datamind/tests/test_hooks.py | 20 ++++++++++++++++++++ 2 files changed, 37 insertions(+), 4 deletions(-) diff --git a/datamind/capabilities/hooks/audit.py b/datamind/capabilities/hooks/audit.py index b1b69dd..e22b622 100644 --- a/datamind/capabilities/hooks/audit.py +++ b/datamind/capabilities/hooks/audit.py @@ -16,9 +16,9 @@ Secret redaction: arg values whose KEY matches a redaction regex (`api_key`, `password`, `token`, `secret`, ...) are replaced with -`"[REDACTED]"` before logging. Values are not scanned (we trust the -caller to put secrets in well-named fields, which is true across -DataMind's own tools). +`"[REDACTED]"` before logging. Credential-shaped values in exception +text are redacted as well because provider and transport errors commonly +include the rejected request detail. Concurrency: writes are serialised via an asyncio.Lock per AuditLogHook instance. For multi-process deployments use one process per profile @@ -45,6 +45,11 @@ r"(?i)(api[_-]?key|password|passwd|token|secret|authorization|bearer|access[_-]?key|client[_-]?secret)" ) _REDACTED = "[REDACTED]" +_BEARER_TEXT_RE = re.compile(r"(?i)\bBearer\s+([^\s,;]+)") +_SECRET_TEXT_RE = re.compile( + r"(?i)\b(api[_-]?key|password|passwd|token|secret|authorization|access[_-]?key|client[_-]?secret)\b" + r"(\s*[:=]\s*)([^\s,;]+)" +) _HASH_HEX_LEN = 16 # truncated SHA-256 hex; 64 bits of collision resistance @@ -74,6 +79,14 @@ def _redact(value: Any) -> Any: return value +def _redact_text(value: str) -> str: + """Redact common credential formats embedded in diagnostic text.""" + value = _BEARER_TEXT_RE.sub(f"Bearer {_REDACTED}", value) + return _SECRET_TEXT_RE.sub( + lambda match: f"{match.group(1)}{match.group(2)}{_REDACTED}", value + ) + + def _decision_to_record(decision: HookDecision) -> dict[str, Any]: if isinstance(decision, Allow): return {"kind": "allow"} @@ -163,7 +176,7 @@ async def post_tool_use( "error": ( None if error is None - else f"{type(error).__name__}: {error}" + else _redact_text(f"{type(error).__name__}: {error}") ), } await self._append(record) diff --git a/datamind/tests/test_hooks.py b/datamind/tests/test_hooks.py index 55b5567..cf8e2e0 100644 --- a/datamind/tests/test_hooks.py +++ b/datamind/tests/test_hooks.py @@ -322,6 +322,26 @@ async def test_audit_log_redacts_secret_keys(tmp_path: Path): assert rec["args"]["username"] == "ann" +@pytest.mark.asyncio +async def test_audit_log_redacts_secrets_in_errors(tmp_path: Path): + audit = tmp_path / "audit.jsonl" + hook = AuditLogHook(audit_path=audit) + await hook.post_tool_use( + _ctx(), + "fake_tool", + {}, + result=None, + error=RuntimeError( + "request failed: api_key=sk-live-secret Authorization: Bearer bearer-secret" + ), + ) + + rec = json.loads(audit.read_text().splitlines()[-1]) + assert "sk-live-secret" not in rec["error"] + assert "bearer-secret" not in rec["error"] + assert "[REDACTED]" in rec["error"] + + @pytest.mark.asyncio async def test_audit_log_records_denied_decision(tmp_path: Path): audit = tmp_path / "audit.jsonl" From 247df097c563f4f5a899b2cd88a108aa4da2a175 Mon Sep 17 00:00:00 2001 From: Dev Chiniwala Date: Thu, 17 Sep 2026 16:26:56 +0530 Subject: [PATCH 5/6] fix: remove stale chunks for empty reingest --- datamind/capabilities/ingest/service.py | 8 +++++++ datamind/tests/test_ingest_revisions.py | 28 +++++++++++++++++++++++++ 2 files changed, 36 insertions(+) diff --git a/datamind/capabilities/ingest/service.py b/datamind/capabilities/ingest/service.py index b04387e..9044ea7 100644 --- a/datamind/capabilities/ingest/service.py +++ b/datamind/capabilities/ingest/service.py @@ -725,6 +725,9 @@ async def kb_add_file(self, *, path: str, copy_to_profile: bool = True) -> dict[ )) if not chunks: + await self._upsert_chunks( + chunks, replace_sources={source, str(resolved)}, + ) return {"file": str(resolved), "chunks_added": 0, "note": "file was empty"} parsed_chunks = None @@ -830,6 +833,11 @@ async def _upsert_chunks( } if recorded_sources & replace_sources and chunk_id not in new_ids: stale_ids.append(chunk_id) + if not chunks: + if stale_ids: + await store.delete(stale_ids) + await self._kb.record_incremental_ingest() + return await store.add( ids=[c.id for c in chunks], texts=texts, diff --git a/datamind/tests/test_ingest_revisions.py b/datamind/tests/test_ingest_revisions.py index bcf4ed7..31adad5 100644 --- a/datamind/tests/test_ingest_revisions.py +++ b/datamind/tests/test_ingest_revisions.py @@ -43,6 +43,34 @@ async def generate_text(self, prompt, **kwargs): return "[]" +@pytest.mark.asyncio +async def test_reingesting_empty_path_removes_old_kb_chunks(tmp_path: Path): + profile = tmp_path / "profile" + uploads = profile / "uploads" + uploads.mkdir(parents=True) + source = uploads / "release_note.txt" + kb = _KB() + service = IngestService( + kb=kb, + db=None, + graph=None, + llm_client=_Model(), + llm_model="test", + profile_data_dir=profile, + chunk_size=512, + chunk_overlap=64, + ) + + source.write_text("old release note", encoding="utf-8") + await service.kb_add_file(path=str(source)) + source.write_text("\n\n", encoding="utf-8") + + result = await service.kb_add_file(path=str(source)) + + assert result["chunks_added"] == 0 + assert kb.vector_store.items == {} + + @pytest.mark.asyncio async def test_reingesting_same_path_replaces_old_kb_chunks(tmp_path: Path): profile = tmp_path / "profile" From e7c9ca56718c49d01743ba2fc5b1edae4366ce2b Mon Sep 17 00:00:00 2001 From: Hao Liang Date: Sun, 20 Sep 2026 11:35:52 +0800 Subject: [PATCH 6/6] fix: complete pending boundary-condition fixes --- datamind/capabilities/hooks/audit.py | 12 +++-- datamind/capabilities/ingest/service.py | 10 ++-- .../memory/providers/sqlite_store.py | 22 +++++++- datamind/capabilities/memory/service.py | 17 ++++--- datamind/core/protocols.py | 1 + datamind/tests/test_hooks.py | 22 ++++++++ datamind/tests/test_ingest_revisions.py | 2 + datamind/tests/test_memory.py | 50 +++++++++++++------ datamind/tests/test_replace_receipts.py | 16 ++++++ 9 files changed, 118 insertions(+), 34 deletions(-) diff --git a/datamind/capabilities/hooks/audit.py b/datamind/capabilities/hooks/audit.py index e22b622..268bd0e 100644 --- a/datamind/capabilities/hooks/audit.py +++ b/datamind/capabilities/hooks/audit.py @@ -47,9 +47,11 @@ _REDACTED = "[REDACTED]" _BEARER_TEXT_RE = re.compile(r"(?i)\bBearer\s+([^\s,;]+)") _SECRET_TEXT_RE = re.compile( - r"(?i)\b(api[_-]?key|password|passwd|token|secret|authorization|access[_-]?key|client[_-]?secret)\b" - r"(\s*[:=]\s*)([^\s,;]+)" + r"(?i)([\"']?" + r"(?:api[_-]?key|password|passwd|token|secret|authorization|access[_-]?key|client[_-]?secret)" + r"[\"']?\s*[:=]\s*)([\"']?)([^\s,;}\"']+)([\"']?)" ) +_RAW_SECRET_RE = re.compile(r"\bsk-[A-Za-z0-9_-]{8,}\b") _HASH_HEX_LEN = 16 # truncated SHA-256 hex; 64 bits of collision resistance @@ -82,9 +84,11 @@ def _redact(value: Any) -> Any: def _redact_text(value: str) -> str: """Redact common credential formats embedded in diagnostic text.""" value = _BEARER_TEXT_RE.sub(f"Bearer {_REDACTED}", value) - return _SECRET_TEXT_RE.sub( - lambda match: f"{match.group(1)}{match.group(2)}{_REDACTED}", value + value = _SECRET_TEXT_RE.sub( + lambda match: f"{match.group(1)}{match.group(2)}{_REDACTED}{match.group(4)}", + value, ) + return _RAW_SECRET_RE.sub(_REDACTED, value) def _decision_to_record(decision: HookDecision) -> dict[str, Any]: diff --git a/datamind/capabilities/ingest/service.py b/datamind/capabilities/ingest/service.py index 264c8d3..124cd3c 100644 --- a/datamind/capabilities/ingest/service.py +++ b/datamind/capabilities/ingest/service.py @@ -821,8 +821,6 @@ async def _upsert_chunks( raise CapabilityError("ingest", "KB surface is disabled") provider = self._kb.embedding store = self._kb.vector_store - texts = [c.text for c in chunks] - vectors = await provider.embed_texts(texts) stale_ids: list[str] = [] if replace_sources: new_ids = {chunk.id for chunk in chunks} @@ -838,6 +836,8 @@ async def _upsert_chunks( await store.delete(stale_ids) await self._kb.record_incremental_ingest() return + texts = [c.text for c in chunks] + vectors = await provider.embed_texts(texts) await store.add( ids=[c.id for c in chunks], texts=texts, @@ -896,19 +896,19 @@ async def db_import_csv( if not re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]{0,63}", col): # Fall back to col_ if header is unusable. col = f"col_{index}" - if col in used_cols: + if col.casefold() in used_cols: # SQL tables cannot contain duplicate column names. Keep the # first header unchanged and give later occurrences a stable # fallback name without losing their values in the row dict. base = f"col_{index}" col = base suffix = 2 - while col in used_cols: + while col.casefold() in used_cols: suffix_text = f"_{suffix}" col = f"{base[:64 - len(suffix_text)]}{suffix_text}" suffix += 1 safe_cols.append(col) - used_cols.add(col) + used_cols.add(col.casefold()) rows: list[dict[str, str]] = [] for raw_row in reader: diff --git a/datamind/capabilities/memory/providers/sqlite_store.py b/datamind/capabilities/memory/providers/sqlite_store.py index b9cd08e..5dadad8 100644 --- a/datamind/capabilities/memory/providers/sqlite_store.py +++ b/datamind/capabilities/memory/providers/sqlite_store.py @@ -287,14 +287,32 @@ async def recall( session_id: str | None = None, top_k: int = 8, kinds: Sequence[str] | None = None, + scope_filter: Sequence[str] | None = None, include_archived: bool = False, # advanced — let callers tune the per-scope budget for ablation per_scope: dict[str, int] | None = None, ) -> list[MemoryItem]: + valid_scopes = {"session", "profile", "global"} + allowed_scopes = valid_scopes if scope_filter is None else set(scope_filter) + invalid_scopes = allowed_scopes - valid_scopes + if invalid_scopes: + raise CapabilityError( + "memory", f"unsupported scope filter: {sorted(invalid_scopes)!r}" + ) + if not allowed_scopes: + return [] + # Default per-scope budget: 2 (session) + 4 (profile) + 2 (global) = 8. - budgets: dict[str, int] = {"session": 2, "profile": 4, "global": 2} + budgets: dict[str, int] = { + "session": 2 if "session" in allowed_scopes else 0, + "profile": 4 if "profile" in allowed_scopes else 0, + "global": 2 if "global" in allowed_scopes else 0, + } if per_scope: - budgets.update({k: v for k, v in per_scope.items() if k in budgets}) + budgets.update({ + k: v for k, v in per_scope.items() + if k in budgets and k in allowed_scopes + }) # If no embedding, fall back to lexical scoring within the same # scope filters so the contract stays identical. diff --git a/datamind/capabilities/memory/service.py b/datamind/capabilities/memory/service.py index c79b45b..078304d 100644 --- a/datamind/capabilities/memory/service.py +++ b/datamind/capabilities/memory/service.py @@ -95,19 +95,22 @@ async def recall( scope_filter: Sequence[str] | None = None, include_archived: bool = False, ) -> list[dict[str, Any]]: - effective_profile = profile or self._default_profile - effective_session = session_id - if scope_filter: - if 'profile' not in scope_filter: - effective_profile = None - if 'session' not in scope_filter: - effective_session = None + allowed = None if scope_filter is None else set(scope_filter) + effective_profile = ( + profile or self._default_profile + if allowed is None or "profile" in allowed + else None + ) + effective_session = ( + session_id if allowed is None or "session" in allowed else None + ) hits = await self.long_term.recall( query, profile=effective_profile, session_id=effective_session, top_k=top_k, kinds=kinds, + scope_filter=scope_filter, include_archived=include_archived, ) return [h.model_dump() for h in hits] diff --git a/datamind/core/protocols.py b/datamind/core/protocols.py index e680186..c64c747 100644 --- a/datamind/core/protocols.py +++ b/datamind/core/protocols.py @@ -374,6 +374,7 @@ async def recall( session_id: str | None = None, top_k: int = 8, kinds: Sequence[str] | None = None, + scope_filter: Sequence[str] | None = None, include_archived: bool = False, ) -> list[MemoryItem]: ... diff --git a/datamind/tests/test_hooks.py b/datamind/tests/test_hooks.py index cf8e2e0..6a72719 100644 --- a/datamind/tests/test_hooks.py +++ b/datamind/tests/test_hooks.py @@ -342,6 +342,28 @@ async def test_audit_log_redacts_secrets_in_errors(tmp_path: Path): assert "[REDACTED]" in rec["error"] +@pytest.mark.asyncio +async def test_audit_log_redacts_quoted_url_and_raw_secrets_in_errors(tmp_path: Path): + audit = tmp_path / "audit.jsonl" + hook = AuditLogHook(audit_path=audit) + secrets = ["sk-json-secret1", "sk-query-secret2", "sk-raw-secret3"] + await hook.post_tool_use( + _ctx(), + "fake_tool", + {}, + result=None, + error=RuntimeError( + '{"api_key": "sk-json-secret1"} ' + "https://example.test?access_token=sk-query-secret2 " + "provider rejected sk-raw-secret3" + ), + ) + + rec = json.loads(audit.read_text().splitlines()[-1]) + assert all(secret not in rec["error"] for secret in secrets) + assert rec["error"].count("[REDACTED]") >= 3 + + @pytest.mark.asyncio async def test_audit_log_records_denied_decision(tmp_path: Path): audit = tmp_path / "audit.jsonl" diff --git a/datamind/tests/test_ingest_revisions.py b/datamind/tests/test_ingest_revisions.py index 31adad5..926790d 100644 --- a/datamind/tests/test_ingest_revisions.py +++ b/datamind/tests/test_ingest_revisions.py @@ -7,6 +7,8 @@ class _Embedding: async def embed_texts(self, texts): + if not texts: + raise AssertionError("empty revisions must not call the embedding provider") return [[float(len(text))] for text in texts] diff --git a/datamind/tests/test_memory.py b/datamind/tests/test_memory.py index 1f47a56..3a1b495 100644 --- a/datamind/tests/test_memory.py +++ b/datamind/tests/test_memory.py @@ -11,6 +11,7 @@ build_memory_tools, ) from datamind.capabilities.memory.providers.sqlite_store import SQLiteMemoryStore +from datamind.core.errors import CapabilityError from datamind.core.protocols import MemoryStore @@ -278,32 +279,49 @@ async def test_service_combines_short_and_long(tmp_path): @pytest.mark.asyncio -async def test_memory_recall_scope_filter_excludes_unselected_scopes(tmp_path): - '''A global-only recall must not reintroduce the default profile.''' - lt = SQLiteMemoryStore(db_path=str(tmp_path / 'm.db'), embedding=_FakeEmbed()) +@pytest.mark.parametrize( + ("scope_filter", "expected"), + [ + (None, {"global", "profile", "session"}), + (["global"], {"global"}), + (["profile"], {"profile"}), + (["session"], {"session"}), + (["global", "profile"], {"global", "profile"}), + ([], set()), + ], +) +async def test_memory_recall_scope_filter_is_exact(tmp_path, scope_filter, expected): + lt = SQLiteMemoryStore(db_path=str(tmp_path / "m.db"), embedding=_FakeEmbed()) svc = MemoryService( short_term=ShortTermMemory(max_turns=5), long_term=lt, - default_profile='acme', + default_profile="acme", ) tools = {tool.name: tool.handler for tool in build_memory_tools(svc)} - await tools['memory_save']('global fact', scope='global') - await tools['memory_save']('profile fact', scope='profile', profile='acme') - await tools['memory_save']( - 'session fact', scope='session', session_id='chat-1' + await tools["memory_save"]("global fact", scope="global") + await tools["memory_save"]("profile fact", scope="profile", profile="acme") + await tools["memory_save"]( + "session fact", scope="session", session_id="chat-1" ) - result = await tools['memory_recall']( - 'fact', - profile='acme', - session_id='chat-1', - scope_filter=['global'], + result = await tools["memory_recall"]( + "fact", + profile="acme", + session_id="chat-1", + scope_filter=scope_filter, ) + assert {item["scope"] for item in result["results"]} == expected - assert result['count'] == 1 - assert result['results'][0]['scope'] == 'global' - assert result['results'][0]['content'] == 'global fact' + +@pytest.mark.asyncio +async def test_memory_recall_rejects_unknown_scope_filter(tmp_path): + lt = SQLiteMemoryStore(db_path=str(tmp_path / "m.db"), embedding=_FakeEmbed()) + svc = MemoryService( + short_term=ShortTermMemory(max_turns=5), long_term=lt, default_profile="acme" + ) + with pytest.raises(CapabilityError, match="unsupported scope filter"): + await svc.recall("fact", scope_filter=["other"]) @pytest.mark.asyncio diff --git a/datamind/tests/test_replace_receipts.py b/datamind/tests/test_replace_receipts.py index 285199d..34c56de 100644 --- a/datamind/tests/test_replace_receipts.py +++ b/datamind/tests/test_replace_receipts.py @@ -91,3 +91,19 @@ async def test_csv_import_keeps_duplicate_headers_as_distinct_columns(database, assert receipt["columns"] == ["a", "col_2"] assert (await db.query_sql("SELECT * FROM duplicate_columns")).rows == [["first", "second"]] + + +@pytest.mark.asyncio +async def test_csv_import_deduplicates_case_insensitive_headers(database, tmp_path): + db, raw = database + source = tmp_path / "case-columns.csv" + source.write_text("a,A,col_3,col_3\n1,2,3,4\n", encoding="utf-8") + + receipt = await raw.get("db_import_csv").handler( + path=str(source), table="case_columns", if_exists="replace" + ) + + assert receipt["columns"] == ["a", "col_2", "col_3", "col_4"] + assert (await db.query_sql("SELECT * FROM case_columns")).rows == [ + ["1", "2", "3", "4"] + ]