Fix: Crash in deep_scan, validate LLM output, ruff cleanup

src/analyze.py

@@ -6,7 +6,6 @@
     GEMINI_API_KEY, GEMINI_API_URL,
     LLM_SYSTEM_PROMPT, LLM_USER_PROMPT_TEMPLATE,
     TAXONOMY_PATH, RETRY_ATTEMPTS, RETRY_BACKOFF,
-    RATE_LIMIT_DELAY,
 )
 
 
@@ -36,13 +35,14 @@ def analyze_advisory(advisory: dict, filtered_diff: str) -> Optional[dict]:
         summary=advisory["summary"],
         package_name=advisory["package_name"],
         ecosystem=advisory["ecosystem"],
-        diff_content=filtered_diff[:8000],
+        diff_content=filtered_diff[:8000]
+        + ("\n... [diff truncated]" if len(filtered_diff) > 8000 else ""),
         taxonomy_list=taxonomy_list,
     )
 
     raw_response = _call_gemini(user_prompt)
     if not raw_response:
-        print(f"    DEBUG: _call_gemini returned None")
+        print("    DEBUG: _call_gemini returned None")
         return None
 
     parsed = _parse_llm_response(raw_response)
@@ -68,7 +68,9 @@ def analyze_advisory(advisory: dict, filtered_diff: str) -> Optional[dict]:
         "impact": parsed.get("impact", ""),
         "fix_summary": parsed.get("fix_summary", ""),
         "key_diff": parsed.get("key_diff", ""),
-        "confidence": parsed.get("confidence", "LOW"),
+        "confidence": parsed.get("confidence", "LOW")
+        if parsed.get("confidence", "LOW") in ("HIGH", "MEDIUM", "LOW")
+        else "LOW",
         "commit_url": advisory["commit_url"],
     }
 
@@ -107,13 +109,13 @@ def _call_gemini(user_prompt: str) -> Optional[str]:
             data = response.json()
             candidates = data.get("candidates", [])
             if not candidates:
-                print(f"    Gemini: no candidates in response")
+                print("    Gemini: no candidates in response")
                 return None
 
             content = candidates[0].get("content", {})
             parts = content.get("parts", [])
             if not parts:
-                print(f"    Gemini: no parts in response")
+                print("    Gemini: no parts in response")
                 return None
 
             return parts[0].get("text", "")

src/backfill_local.py

@@ -1,12 +1,9 @@
 import json
 import sys
-import time
 import requests
 from datetime import datetime, timedelta, timezone
 from src.config import (
-    RATE_LIMIT_DELAY, MAX_DIFF_LINES,
     LLM_SYSTEM_PROMPT, LLM_USER_PROMPT_TEMPLATE,
-    TAXONOMY_PATH, STATE_PATH,
 )
 from src.fetch import fetch_advisories, fetch_commit_diff
 from src.diff_filter import filter_diff
@@ -64,7 +61,7 @@ def analyze_with_ollama(advisory: dict, filtered_diff: str) -> dict | None:
 
     raw_response = call_ollama(user_prompt)
     if not raw_response:
-        print(f"    Ollama returned nothing")
+        print("    Ollama returned nothing")
         return None
 
     parsed = _parse_llm_response(raw_response)
@@ -154,13 +151,13 @@ def backfill_local(days: int):
 
         raw_diff = fetch_commit_diff(advisory["commit_url"])
         if not raw_diff:
-            print(f"    SKIP: no diff")
+            print("    SKIP: no diff")
             errors += 1
             continue
 
         filtered = filter_diff(raw_diff)
         if not filtered:
-            print(f"    SKIP: no relevant files")
+            print("    SKIP: no relevant files")
             continue
 
         result = analyze_with_ollama(advisory, filtered)
@@ -195,12 +192,12 @@ def backfill_local(days: int):
     render_html_index()
 
     stats = get_stats()
-    print(f"\n=== Summary ===")
+    print("\n=== Summary ===")
     print(f"Processed: {processed}")
     print(f"New patterns: {new_patterns}")
     print(f"Errors: {errors}")
     print(f"Total DB: {stats['total_advisories']} advisories, {stats['total_patterns']} patterns")
-    print(f"\nCommit and push:")
+    print("\nCommit and push:")
     print(f"  git add -A && git commit -m 'feat: local backfill — {processed} advisories' && git push")
 
 

src/db.py

@@ -1,6 +1,5 @@
 import json
 import sqlite3
-from pathlib import Path
 from typing import Optional
 from src.config import DB_PATH, JSONL_PATH
 

src/deep_scan.py

@@ -4,7 +4,6 @@
 import os
 import requests
 from datetime import datetime, timezone
-from pathlib import Path
 from src.config import DATA_DIR
 from src.heuristics import score_commit
 from src.fingerprint import match_fingerprints
@@ -146,7 +145,7 @@ def deep_scan(repo: str, since: str = None, until: str = None, max_commits: int
         message = commit.get("commit", {}).get("message", "").split("\n")[0]
 
         if (i + 1) % 50 == 0 or (i + 1) == len(new_commits):
-            print(f"  [{i+1}/{len(new_commits)}] {suspects_count} suspects so far...", flush=True) if 'suspects_count' in dir() else None
+            print(f"  [{i+1}/{len(new_commits)}] {len(suspects)} suspects so far...", flush=True)
 
         detail = github_get(f"/repos/{repo}/commits/{sha}")
         if not detail:
@@ -172,7 +171,6 @@ def deep_scan(repo: str, since: str = None, until: str = None, max_commits: int
         best_fp = fingerprint_matches[0] if fingerprint_matches else None
         fp_score = best_fp["score"] if best_fp else 0.0
 
-        raw_combined = heuristic_result["score"] + (fp_score * 20)
         normalized = heuristic_result["normalized_score"]
         if best_fp:
             normalized = min(normalized + (fp_score * 30), 100)
@@ -206,7 +204,6 @@ def deep_scan(repo: str, since: str = None, until: str = None, max_commits: int
             f.write(json.dumps(result, ensure_ascii=False) + "\n")
 
         suspects.append(result)
-        suspects_count = len(suspects)
 
         severity = "HIGH" if normalized >= 60 else "MEDIUM" if normalized >= 30 else "LOW"
         print(f"  [{i+1}/{len(new_commits)}] {severity} score={normalized} {sha[:8]} {message[:60]}")
@@ -221,7 +218,7 @@ def deep_scan(repo: str, since: str = None, until: str = None, max_commits: int
 
     if suspects:
         print(f"\nResults saved to: {results_path}")
-        print(f"\nTop suspects:")
+        print("\nTop suspects:")
         top = sorted(suspects, key=lambda s: s["normalized_score"], reverse=True)[:10]
         for s in top:
             print(f"  score={s['normalized_score']:5.1f}  {s['commit_sha'][:8]}  {s['message'][:60]}")

src/fetch.py

@@ -28,7 +28,7 @@ def graphql_request(query: str, variables: dict) -> dict:
             if "errors" in data:
                 raise RuntimeError(f"GraphQL errors: {data['errors']}")
             return data
-        except (requests.RequestException, RuntimeError) as exc:
+        except (requests.RequestException, RuntimeError):
             if attempt == RETRY_ATTEMPTS - 1:
                 raise
             time.sleep(RETRY_BACKOFF[attempt])

src/fingerprint.py

@@ -1,6 +1,5 @@
 import json
 import re
-from pathlib import Path
 from src.config import DATA_DIR
 
 

src/fingerprint_builder.py

@@ -2,7 +2,6 @@
 import re
 import sys
 from pathlib import Path
-from collections import defaultdict
 
 
 EXPERT_FINGERPRINTS = {
@@ -246,7 +245,7 @@ def build_fingerprints(output_path: str = "data/fingerprints.json"):
     print(f"  Expert CWE: {len(EXPERT_FINGERPRINTS)}")
     print(f"  OSDC live: {len([k for k in fingerprints if k.startswith('OSDC:')])}")
 
-    print(f"\nTop patterns by token count:")
+    print("\nTop patterns by token count:")
     top = sorted(fingerprints.items(), key=lambda x: len(x[1]["add_tokens"]) + len(x[1]["del_tokens"]), reverse=True)[:10]
     for pid, data in top:
         total = len(data["add_tokens"]) + len(data["del_tokens"])

src/main.py

@@ -1,7 +1,6 @@
 import json
 import time
 import signal
-import sys
 from datetime import datetime, timezone
 from src.config import (
     STATE_PATH, MAX_DAILY_CALLS, RATE_LIMIT_DELAY,
@@ -98,18 +97,18 @@ def run():
 
         raw_diff = fetch_commit_diff(advisory["commit_url"])
         if not raw_diff:
-            print(f"    SKIP: no diff available")
+            print("    SKIP: no diff available")
             errors += 1
             continue
 
         filtered = filter_diff(raw_diff)
         if not filtered:
-            print(f"    SKIP: no relevant files in diff")
+            print("    SKIP: no relevant files in diff")
             continue
 
         result = analyze_advisory(advisory, filtered)
         if not result:
-            print(f"    ERROR: LLM analysis failed")
+            print("    ERROR: LLM analysis failed")
             errors += 1
             continue
 
@@ -174,7 +173,7 @@ def run():
         encoding="utf-8",
     )
 
-    print(f"\n=== Summary ===")
+    print("\n=== Summary ===")
     print(f"Analyzed: {len(analyzed)}")
     print(f"New patterns: {new_patterns}")
     print(f"Errors: {errors}")

src/render.py

@@ -1,15 +1,14 @@
 import json
 import re
 from datetime import date
-from pathlib import Path
 from jinja2 import Environment, FileSystemLoader
 from src.config import (
     TEMPLATES_DIR, PATCHES_DIR, DOCS_DIR, ROOT_DIR,
-    README_DAYS_SHOWN, DATA_DIR,
+    DATA_DIR,
 )
 from src.db import (
-    get_advisories_for_date, get_pattern_info,
-    get_recent_dates, get_all_advisories, get_stats,
+    get_pattern_info,
+    get_all_advisories, get_stats,
 )
 
 

src/render_cli.py

@@ -1,9 +1,7 @@
 import json
-from datetime import date
-from pathlib import Path
 from src.config import DOCS_DIR
 from src.db import (
-    rebuild_from_jsonl, get_all_advisories, get_recent_dates,
+    rebuild_from_jsonl, get_recent_dates,
     get_advisories_for_date, get_stats,
 )
 from src.render import render_readme, render_html_index, render_daily_patch, render_silent_page
@@ -32,7 +30,7 @@ def generate_badge(stats: dict):
     (DOCS_DIR / "badge-patterns.json").write_text(
         json.dumps(badge_patterns), encoding="utf-8"
     )
-    print(f"  Generated badge endpoints")
+    print("  Generated badge endpoints")
 
 
 def run():

src/silent_scan.py

@@ -1,10 +1,8 @@
 import json
 import time
 import sys
-import os
 import requests
 from datetime import datetime, timedelta, timezone
-from pathlib import Path
 from src.config import DATA_DIR, GITHUB_TOKEN
 from src.heuristics import score_commit
 from src.fingerprint import match_fingerprints
@@ -202,7 +200,7 @@ def run(hours: int = 24):
     }
     save_silent_state(state)
 
-    print(f"\n=== Summary ===")
+    print("\n=== Summary ===")
     print(f"Repos scanned: {len(watchlist) - skipped_repos}/{len(watchlist)}")
     print(f"Commits analyzed: {total_commits}")
     print(f"Layer 1 pass (heuristics >= 8): {layer1_pass}")