From 5ac50d28caf19e4d3bd29b5213140a0bd9d5a61a Mon Sep 17 00:00:00 2001
From: EQ maintenance for AliZakaee
 <62015010+AliZakaee@users.noreply.github.com>
Date: Sat, 15 Aug 2026 12:07:11 +0330
Subject: [PATCH] fix: skip invalid stored embeddings

---
 services/embedding_service.py                 | 13 +++++++++----
 tests/unit/services/test_embedding_service.py | 12 ++++++++++--
 2 files changed, 19 insertions(+), 6 deletions(-)

diff --git a/services/embedding_service.py b/services/embedding_service.py
index 26be606..3465768 100644
--- a/services/embedding_service.py
+++ b/services/embedding_service.py
@@ -53,15 +53,20 @@ def _load_matrix(chunks):
             continue
         try:
             vec = json.loads(raw)
-        except (json.JSONDecodeError, TypeError):
+            if not isinstance(vec, list) or len(vec) != EMBED_DIM:
+                continue
+            array = np.asarray(vec, dtype=np.float64)
+        except (json.JSONDecodeError, TypeError, ValueError, OverflowError):
             continue
-        if not isinstance(vec, list) or len(vec) != EMBED_DIM:
+        if array.shape != (EMBED_DIM,) or not np.all(np.isfinite(array)):
             continue
-        vectors.append(vec)
+        if np.any(np.abs(array) > np.finfo(np.float32).max):
+            continue
+        vectors.append(array.astype(np.float32))
         kept.append(ch)
     if not vectors:
         return None, []
-    return np.asarray(vectors, dtype=np.float32), kept
+    return np.vstack(vectors), kept
 
 
 def search_chunks(query: str, chunks, top_k: int = KB_TOP_K) -> list[tuple]:
diff --git a/tests/unit/services/test_embedding_service.py b/tests/unit/services/test_embedding_service.py
index 275a63a..f7ef264 100644
--- a/tests/unit/services/test_embedding_service.py
+++ b/tests/unit/services/test_embedding_service.py
@@ -70,14 +70,22 @@ def test_load_matrix_filters_bad_rows(monkeypatch):
         {"content": "missing", "embedding_json": None},
         {"content": "malformed", "embedding_json": "{not json"},
         {"content": "wrong dim", "embedding_json": json.dumps([1.0, 2.0])},
+        {"content": "non-numeric", "embedding_json": json.dumps(["bad", 0.0, 0.0])},
+        {"content": "nested", "embedding_json": json.dumps([[1.0], [0.0], [0.0]])},
+        {"content": "non-finite", "embedding_json": json.dumps([float("nan"), 0.0, 0.0])},
+        {"content": "float32 overflow", "embedding_json": json.dumps([1e40, 0.0, 0.0])},
     ]
     matrix, kept = es._load_matrix(chunks)
     assert matrix.shape == (1, 3)
     assert [c["content"] for c in kept] == ["good"]
 
 
-def test_load_matrix_none_when_no_usable():
-    matrix, kept = es._load_matrix([{"content": "x", "embedding_json": None}])
+def test_load_matrix_none_when_no_usable(monkeypatch):
+    monkeypatch.setattr(es, "EMBED_DIM", 3)
+    matrix, kept = es._load_matrix([
+        {"content": "missing", "embedding_json": None},
+        {"content": "non-numeric", "embedding_json": json.dumps(["bad", 0.0, 0.0])},
+    ])
     assert matrix is None and kept == []