Fix: Pagination bug, atomic JSONL export, resilient JSON parsing

src/analyze.py

@@ -6,7 +6,6 @@
     GEMINI_API_KEY, GEMINI_API_URL,
     LLM_SYSTEM_PROMPT, LLM_USER_PROMPT_TEMPLATE,
     TAXONOMY_PATH, RETRY_ATTEMPTS, RETRY_BACKOFF,
-    RATE_LIMIT_DELAY,
 )
 
 
@@ -42,7 +41,7 @@ def analyze_advisory(advisory: dict, filtered_diff: str) -> Optional[dict]:
 
     raw_response = _call_gemini(user_prompt)
     if not raw_response:
-        print(f"    DEBUG: _call_gemini returned None")
+        print("    DEBUG: _call_gemini returned None")
         return None
 
     parsed = _parse_llm_response(raw_response)
@@ -107,13 +106,13 @@ def _call_gemini(user_prompt: str) -> Optional[str]:
             data = response.json()
             candidates = data.get("candidates", [])
             if not candidates:
-                print(f"    Gemini: no candidates in response")
+                print("    Gemini: no candidates in response")
                 return None
 
             content = candidates[0].get("content", {})
             parts = content.get("parts", [])
             if not parts:
-                print(f"    Gemini: no parts in response")
+                print("    Gemini: no parts in response")
                 return None
 
             return parts[0].get("text", "")

src/backfill_local.py

@@ -1,12 +1,9 @@
 import json
 import sys
-import time
 import requests
 from datetime import datetime, timedelta, timezone
 from src.config import (
-    RATE_LIMIT_DELAY, MAX_DIFF_LINES,
     LLM_SYSTEM_PROMPT, LLM_USER_PROMPT_TEMPLATE,
-    TAXONOMY_PATH, STATE_PATH,
 )
 from src.fetch import fetch_advisories, fetch_commit_diff
 from src.diff_filter import filter_diff
@@ -64,7 +61,7 @@ def analyze_with_ollama(advisory: dict, filtered_diff: str) -> dict | None:
 
     raw_response = call_ollama(user_prompt)
     if not raw_response:
-        print(f"    Ollama returned nothing")
+        print("    Ollama returned nothing")
         return None
 
     parsed = _parse_llm_response(raw_response)
@@ -154,13 +151,13 @@ def backfill_local(days: int):
 
         raw_diff = fetch_commit_diff(advisory["commit_url"])
         if not raw_diff:
-            print(f"    SKIP: no diff")
+            print("    SKIP: no diff")
             errors += 1
             continue
 
         filtered = filter_diff(raw_diff)
         if not filtered:
-            print(f"    SKIP: no relevant files")
+            print("    SKIP: no relevant files")
             continue
 
         result = analyze_with_ollama(advisory, filtered)
@@ -195,12 +192,12 @@ def backfill_local(days: int):
     render_html_index()
 
     stats = get_stats()
-    print(f"\n=== Summary ===")
+    print("\n=== Summary ===")
     print(f"Processed: {processed}")
     print(f"New patterns: {new_patterns}")
     print(f"Errors: {errors}")
     print(f"Total DB: {stats['total_advisories']} advisories, {stats['total_patterns']} patterns")
-    print(f"\nCommit and push:")
+    print("\nCommit and push:")
     print(f"  git add -A && git commit -m 'feat: local backfill — {processed} advisories' && git push")
 
 

src/db.py

@@ -1,6 +1,5 @@
 import json
 import sqlite3
-from pathlib import Path
 from typing import Optional
 from src.config import DB_PATH, JSONL_PATH
 
@@ -63,11 +62,15 @@ def rebuild_from_jsonl():
     conn = sqlite3.connect(str(DB_PATH))
 
     with open(JSONL_PATH, "r") as f:
-        for line in f:
+        for lineno, line in enumerate(f, 1):
             line = line.strip()
             if not line:
                 continue
-            record = json.loads(line)
+            try:
+                record = json.loads(line)
+            except json.JSONDecodeError:
+                print(f"  WARNING: skipping malformed JSONL at line {lineno}")
+                continue
             record_type = record.get("_type")
 
             if record_type == "advisory":
@@ -197,7 +200,9 @@ def export_to_jsonl():
     conn = sqlite3.connect(str(DB_PATH))
     conn.row_factory = sqlite3.Row
 
-    with open(JSONL_PATH, "w") as f:
+    tmp_path = JSONL_PATH.with_suffix(".jsonl.tmp")
+
+    with open(tmp_path, "w") as f:
         for row in conn.execute("SELECT * FROM patterns ORDER BY first_seen"):
             record = dict(row)
             record["_type"] = "pattern"
@@ -208,6 +213,8 @@ def export_to_jsonl():
             record["_type"] = "advisory"
             f.write(json.dumps(record, ensure_ascii=False) + "\n")
 
+    import os
+    os.replace(tmp_path, JSONL_PATH)
     conn.close()
 
 

src/deep_scan.py

@@ -4,7 +4,6 @@
 import os
 import requests
 from datetime import datetime, timezone
-from pathlib import Path
 from src.config import DATA_DIR
 from src.heuristics import score_commit
 from src.fingerprint import match_fingerprints
@@ -62,6 +61,7 @@ def fetch_all_commits(repo: str, since: str = None, until: str = None, per_page:
         params["until"] = until
 
     url = f"/repos/{repo}/commits"
+    use_params = True
 
     page = 0
     while url:
@@ -72,7 +72,7 @@ def fetch_all_commits(repo: str, since: str = None, until: str = None, per_page:
 
         full_url = f"{GITHUB_API}{url}" if url.startswith("/") else url
         try:
-            response = requests.get(full_url, headers=headers, params=params if page == 1 else None, timeout=30)
+            response = requests.get(full_url, headers=headers, params=params if use_params else None, timeout=30)
             if response.status_code == 403:
                 remaining = response.headers.get("X-RateLimit-Remaining", "0")
                 if remaining == "0":
@@ -91,6 +91,7 @@ def fetch_all_commits(repo: str, since: str = None, until: str = None, per_page:
 
             next_url = get_link_next(dict(response.headers))
             url = next_url
+            use_params = False
             time.sleep(REQUEST_DELAY)
         except requests.RequestException as exc:
             print(f"  Error on page {page}: {exc}")
@@ -125,9 +126,13 @@ def deep_scan(repo: str, since: str = None, until: str = None, max_commits: int
         with open(results_path, "r") as f:
             for line in f:
                 line = line.strip()
-                if line:
+                if not line:
+                    continue
+                try:
                     record = json.loads(line)
-                    seen.add(record.get("commit_sha", ""))
+                except json.JSONDecodeError:
+                    continue
+                seen.add(record.get("commit_sha", ""))
         print(f"Already scanned: {len(seen)} commits")
 
     new_commits = [c for c in commits if c.get("sha", "") not in seen]
@@ -146,7 +151,7 @@ def deep_scan(repo: str, since: str = None, until: str = None, max_commits: int
         message = commit.get("commit", {}).get("message", "").split("\n")[0]
 
         if (i + 1) % 50 == 0 or (i + 1) == len(new_commits):
-            print(f"  [{i+1}/{len(new_commits)}] {suspects_count} suspects so far...", flush=True) if 'suspects_count' in dir() else None
+            print(f"  [{i+1}/{len(new_commits)}] {len(suspects)} suspects so far...", flush=True)
 
         detail = github_get(f"/repos/{repo}/commits/{sha}")
         if not detail:
@@ -172,7 +177,6 @@ def deep_scan(repo: str, since: str = None, until: str = None, max_commits: int
         best_fp = fingerprint_matches[0] if fingerprint_matches else None
         fp_score = best_fp["score"] if best_fp else 0.0
 
-        raw_combined = heuristic_result["score"] + (fp_score * 20)
         normalized = heuristic_result["normalized_score"]
         if best_fp:
             normalized = min(normalized + (fp_score * 30), 100)
@@ -206,7 +210,6 @@ def deep_scan(repo: str, since: str = None, until: str = None, max_commits: int
             f.write(json.dumps(result, ensure_ascii=False) + "\n")
 
         suspects.append(result)
-        suspects_count = len(suspects)
 
         severity = "HIGH" if normalized >= 60 else "MEDIUM" if normalized >= 30 else "LOW"
         print(f"  [{i+1}/{len(new_commits)}] {severity} score={normalized} {sha[:8]} {message[:60]}")
@@ -221,7 +224,7 @@ def deep_scan(repo: str, since: str = None, until: str = None, max_commits: int
 
     if suspects:
         print(f"\nResults saved to: {results_path}")
-        print(f"\nTop suspects:")
+        print("\nTop suspects:")
         top = sorted(suspects, key=lambda s: s["normalized_score"], reverse=True)[:10]
         for s in top:
             print(f"  score={s['normalized_score']:5.1f}  {s['commit_sha'][:8]}  {s['message'][:60]}")

src/fetch.py

@@ -28,7 +28,7 @@ def graphql_request(query: str, variables: dict) -> dict:
             if "errors" in data:
                 raise RuntimeError(f"GraphQL errors: {data['errors']}")
             return data
-        except (requests.RequestException, RuntimeError) as exc:
+        except (requests.RequestException, RuntimeError):
             if attempt == RETRY_ATTEMPTS - 1:
                 raise
             time.sleep(RETRY_BACKOFF[attempt])

src/fingerprint.py

@@ -1,6 +1,5 @@
 import json
 import re
-from pathlib import Path
 from src.config import DATA_DIR
 
 

src/fingerprint_builder.py

@@ -2,7 +2,6 @@
 import re
 import sys
 from pathlib import Path
-from collections import defaultdict
 
 
 EXPERT_FINGERPRINTS = {
@@ -246,7 +245,7 @@ def build_fingerprints(output_path: str = "data/fingerprints.json"):
     print(f"  Expert CWE: {len(EXPERT_FINGERPRINTS)}")
     print(f"  OSDC live: {len([k for k in fingerprints if k.startswith('OSDC:')])}")
 
-    print(f"\nTop patterns by token count:")
+    print("\nTop patterns by token count:")
     top = sorted(fingerprints.items(), key=lambda x: len(x[1]["add_tokens"]) + len(x[1]["del_tokens"]), reverse=True)[:10]
     for pid, data in top:
         total = len(data["add_tokens"]) + len(data["del_tokens"])

src/main.py

@@ -1,7 +1,6 @@
 import json
 import time
 import signal
-import sys
 from datetime import datetime, timezone
 from src.config import (
     STATE_PATH, MAX_DAILY_CALLS, RATE_LIMIT_DELAY,
@@ -98,18 +97,18 @@ def run():
 
         raw_diff = fetch_commit_diff(advisory["commit_url"])
         if not raw_diff:
-            print(f"    SKIP: no diff available")
+            print("    SKIP: no diff available")
             errors += 1
             continue
 
         filtered = filter_diff(raw_diff)
         if not filtered:
-            print(f"    SKIP: no relevant files in diff")
+            print("    SKIP: no relevant files in diff")
             continue
 
         result = analyze_advisory(advisory, filtered)
         if not result:
-            print(f"    ERROR: LLM analysis failed")
+            print("    ERROR: LLM analysis failed")
             errors += 1
             continue
 
@@ -174,7 +173,7 @@ def run():
         encoding="utf-8",
     )
 
-    print(f"\n=== Summary ===")
+    print("\n=== Summary ===")
     print(f"Analyzed: {len(analyzed)}")
     print(f"New patterns: {new_patterns}")
     print(f"Errors: {errors}")

src/render.py

@@ -1,15 +1,14 @@
 import json
 import re
 from datetime import date
-from pathlib import Path
 from jinja2 import Environment, FileSystemLoader
 from src.config import (
     TEMPLATES_DIR, PATCHES_DIR, DOCS_DIR, ROOT_DIR,
-    README_DAYS_SHOWN, DATA_DIR,
+    DATA_DIR,
 )
 from src.db import (
-    get_advisories_for_date, get_pattern_info,
-    get_recent_dates, get_all_advisories, get_stats,
+    get_pattern_info,
+    get_all_advisories, get_stats,
 )
 
 
@@ -150,7 +149,10 @@ def render_silent_page():
                 line = line.strip()
                 if not line:
                     continue
-                suspects.append(json.loads(line))
+                try:
+                    suspects.append(json.loads(line))
+                except json.JSONDecodeError:
+                    continue
 
     suspects.sort(key=lambda s: s.get("combined_score", 0), reverse=True)
 

src/render_cli.py

@@ -1,9 +1,7 @@
 import json
-from datetime import date
-from pathlib import Path
 from src.config import DOCS_DIR
 from src.db import (
-    rebuild_from_jsonl, get_all_advisories, get_recent_dates,
+    rebuild_from_jsonl, get_recent_dates,
     get_advisories_for_date, get_stats,
 )
 from src.render import render_readme, render_html_index, render_daily_patch, render_silent_page
@@ -32,7 +30,7 @@ def generate_badge(stats: dict):
     (DOCS_DIR / "badge-patterns.json").write_text(
         json.dumps(badge_patterns), encoding="utf-8"
     )
-    print(f"  Generated badge endpoints")
+    print("  Generated badge endpoints")
 
 
 def run():

src/silent_scan.py

@@ -1,10 +1,8 @@
 import json
 import time
 import sys
-import os
 import requests
 from datetime import datetime, timedelta, timezone
-from pathlib import Path
 from src.config import DATA_DIR, GITHUB_TOKEN
 from src.heuristics import score_commit
 from src.fingerprint import match_fingerprints
@@ -47,7 +45,10 @@ def load_existing_results() -> set:
                 line = line.strip()
                 if not line:
                     continue
-                record = json.loads(line)
+                try:
+                    record = json.loads(line)
+                except json.JSONDecodeError:
+                    continue
                 seen.add(record.get("commit_sha", ""))
     return seen
 
@@ -202,7 +203,7 @@ def run(hours: int = 24):
     }
     save_silent_state(state)
 
-    print(f"\n=== Summary ===")
+    print("\n=== Summary ===")
     print(f"Repos scanned: {len(watchlist) - skipped_repos}/{len(watchlist)}")
     print(f"Commits analyzed: {total_commits}")
     print(f"Layer 1 pass (heuristics >= 8): {layer1_pass}")