diff --git a/src/analyze.py b/src/analyze.py index 2c1698210d..1575b9a1d4 100644 --- a/src/analyze.py +++ b/src/analyze.py @@ -6,7 +6,6 @@ GEMINI_API_KEY, GEMINI_API_URL, LLM_SYSTEM_PROMPT, LLM_USER_PROMPT_TEMPLATE, TAXONOMY_PATH, RETRY_ATTEMPTS, RETRY_BACKOFF, - RATE_LIMIT_DELAY, ) @@ -42,7 +41,7 @@ def analyze_advisory(advisory: dict, filtered_diff: str) -> Optional[dict]: raw_response = _call_gemini(user_prompt) if not raw_response: - print(f" DEBUG: _call_gemini returned None") + print(" DEBUG: _call_gemini returned None") return None parsed = _parse_llm_response(raw_response) @@ -107,13 +106,13 @@ def _call_gemini(user_prompt: str) -> Optional[str]: data = response.json() candidates = data.get("candidates", []) if not candidates: - print(f" Gemini: no candidates in response") + print(" Gemini: no candidates in response") return None content = candidates[0].get("content", {}) parts = content.get("parts", []) if not parts: - print(f" Gemini: no parts in response") + print(" Gemini: no parts in response") return None return parts[0].get("text", "") diff --git a/src/backfill_local.py b/src/backfill_local.py index 7d9b82a93d..254272fbd7 100644 --- a/src/backfill_local.py +++ b/src/backfill_local.py @@ -1,12 +1,9 @@ import json import sys -import time import requests from datetime import datetime, timedelta, timezone from src.config import ( - RATE_LIMIT_DELAY, MAX_DIFF_LINES, LLM_SYSTEM_PROMPT, LLM_USER_PROMPT_TEMPLATE, - TAXONOMY_PATH, STATE_PATH, ) from src.fetch import fetch_advisories, fetch_commit_diff from src.diff_filter import filter_diff @@ -64,7 +61,7 @@ def analyze_with_ollama(advisory: dict, filtered_diff: str) -> dict | None: raw_response = call_ollama(user_prompt) if not raw_response: - print(f" Ollama returned nothing") + print(" Ollama returned nothing") return None parsed = _parse_llm_response(raw_response) @@ -154,13 +151,13 @@ def backfill_local(days: int): raw_diff = fetch_commit_diff(advisory["commit_url"]) if not raw_diff: - print(f" SKIP: no diff") + print(" SKIP: no diff") errors += 1 continue filtered = filter_diff(raw_diff) if not filtered: - print(f" SKIP: no relevant files") + print(" SKIP: no relevant files") continue result = analyze_with_ollama(advisory, filtered) @@ -195,12 +192,12 @@ def backfill_local(days: int): render_html_index() stats = get_stats() - print(f"\n=== Summary ===") + print("\n=== Summary ===") print(f"Processed: {processed}") print(f"New patterns: {new_patterns}") print(f"Errors: {errors}") print(f"Total DB: {stats['total_advisories']} advisories, {stats['total_patterns']} patterns") - print(f"\nCommit and push:") + print("\nCommit and push:") print(f" git add -A && git commit -m 'feat: local backfill — {processed} advisories' && git push") diff --git a/src/db.py b/src/db.py index 4c390b8d74..2fbb94f73a 100644 --- a/src/db.py +++ b/src/db.py @@ -1,6 +1,5 @@ import json import sqlite3 -from pathlib import Path from typing import Optional from src.config import DB_PATH, JSONL_PATH @@ -63,11 +62,15 @@ def rebuild_from_jsonl(): conn = sqlite3.connect(str(DB_PATH)) with open(JSONL_PATH, "r") as f: - for line in f: + for lineno, line in enumerate(f, 1): line = line.strip() if not line: continue - record = json.loads(line) + try: + record = json.loads(line) + except json.JSONDecodeError: + print(f" WARNING: skipping malformed JSONL at line {lineno}") + continue record_type = record.get("_type") if record_type == "advisory": @@ -197,7 +200,9 @@ def export_to_jsonl(): conn = sqlite3.connect(str(DB_PATH)) conn.row_factory = sqlite3.Row - with open(JSONL_PATH, "w") as f: + tmp_path = JSONL_PATH.with_suffix(".jsonl.tmp") + + with open(tmp_path, "w") as f: for row in conn.execute("SELECT * FROM patterns ORDER BY first_seen"): record = dict(row) record["_type"] = "pattern" @@ -208,6 +213,8 @@ def export_to_jsonl(): record["_type"] = "advisory" f.write(json.dumps(record, ensure_ascii=False) + "\n") + import os + os.replace(tmp_path, JSONL_PATH) conn.close() diff --git a/src/deep_scan.py b/src/deep_scan.py index 211606d5e0..146178c2f3 100644 --- a/src/deep_scan.py +++ b/src/deep_scan.py @@ -4,7 +4,6 @@ import os import requests from datetime import datetime, timezone -from pathlib import Path from src.config import DATA_DIR from src.heuristics import score_commit from src.fingerprint import match_fingerprints @@ -62,6 +61,7 @@ def fetch_all_commits(repo: str, since: str = None, until: str = None, per_page: params["until"] = until url = f"/repos/{repo}/commits" + use_params = True page = 0 while url: @@ -72,7 +72,7 @@ def fetch_all_commits(repo: str, since: str = None, until: str = None, per_page: full_url = f"{GITHUB_API}{url}" if url.startswith("/") else url try: - response = requests.get(full_url, headers=headers, params=params if page == 1 else None, timeout=30) + response = requests.get(full_url, headers=headers, params=params if use_params else None, timeout=30) if response.status_code == 403: remaining = response.headers.get("X-RateLimit-Remaining", "0") if remaining == "0": @@ -91,6 +91,7 @@ def fetch_all_commits(repo: str, since: str = None, until: str = None, per_page: next_url = get_link_next(dict(response.headers)) url = next_url + use_params = False time.sleep(REQUEST_DELAY) except requests.RequestException as exc: print(f" Error on page {page}: {exc}") @@ -125,9 +126,13 @@ def deep_scan(repo: str, since: str = None, until: str = None, max_commits: int with open(results_path, "r") as f: for line in f: line = line.strip() - if line: + if not line: + continue + try: record = json.loads(line) - seen.add(record.get("commit_sha", "")) + except json.JSONDecodeError: + continue + seen.add(record.get("commit_sha", "")) print(f"Already scanned: {len(seen)} commits") new_commits = [c for c in commits if c.get("sha", "") not in seen] @@ -146,7 +151,7 @@ def deep_scan(repo: str, since: str = None, until: str = None, max_commits: int message = commit.get("commit", {}).get("message", "").split("\n")[0] if (i + 1) % 50 == 0 or (i + 1) == len(new_commits): - print(f" [{i+1}/{len(new_commits)}] {suspects_count} suspects so far...", flush=True) if 'suspects_count' in dir() else None + print(f" [{i+1}/{len(new_commits)}] {len(suspects)} suspects so far...", flush=True) detail = github_get(f"/repos/{repo}/commits/{sha}") if not detail: @@ -172,7 +177,6 @@ def deep_scan(repo: str, since: str = None, until: str = None, max_commits: int best_fp = fingerprint_matches[0] if fingerprint_matches else None fp_score = best_fp["score"] if best_fp else 0.0 - raw_combined = heuristic_result["score"] + (fp_score * 20) normalized = heuristic_result["normalized_score"] if best_fp: normalized = min(normalized + (fp_score * 30), 100) @@ -206,7 +210,6 @@ def deep_scan(repo: str, since: str = None, until: str = None, max_commits: int f.write(json.dumps(result, ensure_ascii=False) + "\n") suspects.append(result) - suspects_count = len(suspects) severity = "HIGH" if normalized >= 60 else "MEDIUM" if normalized >= 30 else "LOW" print(f" [{i+1}/{len(new_commits)}] {severity} score={normalized} {sha[:8]} {message[:60]}") @@ -221,7 +224,7 @@ def deep_scan(repo: str, since: str = None, until: str = None, max_commits: int if suspects: print(f"\nResults saved to: {results_path}") - print(f"\nTop suspects:") + print("\nTop suspects:") top = sorted(suspects, key=lambda s: s["normalized_score"], reverse=True)[:10] for s in top: print(f" score={s['normalized_score']:5.1f} {s['commit_sha'][:8]} {s['message'][:60]}") diff --git a/src/fetch.py b/src/fetch.py index 68a52a9666..f9d2bc258c 100644 --- a/src/fetch.py +++ b/src/fetch.py @@ -28,7 +28,7 @@ def graphql_request(query: str, variables: dict) -> dict: if "errors" in data: raise RuntimeError(f"GraphQL errors: {data['errors']}") return data - except (requests.RequestException, RuntimeError) as exc: + except (requests.RequestException, RuntimeError): if attempt == RETRY_ATTEMPTS - 1: raise time.sleep(RETRY_BACKOFF[attempt]) diff --git a/src/fingerprint.py b/src/fingerprint.py index 3c26ae64be..c8b7ac89f1 100644 --- a/src/fingerprint.py +++ b/src/fingerprint.py @@ -1,6 +1,5 @@ import json import re -from pathlib import Path from src.config import DATA_DIR diff --git a/src/fingerprint_builder.py b/src/fingerprint_builder.py index ee3ab30732..ba8ba9cb30 100644 --- a/src/fingerprint_builder.py +++ b/src/fingerprint_builder.py @@ -2,7 +2,6 @@ import re import sys from pathlib import Path -from collections import defaultdict EXPERT_FINGERPRINTS = { @@ -246,7 +245,7 @@ def build_fingerprints(output_path: str = "data/fingerprints.json"): print(f" Expert CWE: {len(EXPERT_FINGERPRINTS)}") print(f" OSDC live: {len([k for k in fingerprints if k.startswith('OSDC:')])}") - print(f"\nTop patterns by token count:") + print("\nTop patterns by token count:") top = sorted(fingerprints.items(), key=lambda x: len(x[1]["add_tokens"]) + len(x[1]["del_tokens"]), reverse=True)[:10] for pid, data in top: total = len(data["add_tokens"]) + len(data["del_tokens"]) diff --git a/src/main.py b/src/main.py index 2b50f54619..6ddb33566a 100644 --- a/src/main.py +++ b/src/main.py @@ -1,7 +1,6 @@ import json import time import signal -import sys from datetime import datetime, timezone from src.config import ( STATE_PATH, MAX_DAILY_CALLS, RATE_LIMIT_DELAY, @@ -98,18 +97,18 @@ def run(): raw_diff = fetch_commit_diff(advisory["commit_url"]) if not raw_diff: - print(f" SKIP: no diff available") + print(" SKIP: no diff available") errors += 1 continue filtered = filter_diff(raw_diff) if not filtered: - print(f" SKIP: no relevant files in diff") + print(" SKIP: no relevant files in diff") continue result = analyze_advisory(advisory, filtered) if not result: - print(f" ERROR: LLM analysis failed") + print(" ERROR: LLM analysis failed") errors += 1 continue @@ -174,7 +173,7 @@ def run(): encoding="utf-8", ) - print(f"\n=== Summary ===") + print("\n=== Summary ===") print(f"Analyzed: {len(analyzed)}") print(f"New patterns: {new_patterns}") print(f"Errors: {errors}") diff --git a/src/render.py b/src/render.py index 8aba29087c..547fb1c832 100644 --- a/src/render.py +++ b/src/render.py @@ -1,15 +1,14 @@ import json import re from datetime import date -from pathlib import Path from jinja2 import Environment, FileSystemLoader from src.config import ( TEMPLATES_DIR, PATCHES_DIR, DOCS_DIR, ROOT_DIR, - README_DAYS_SHOWN, DATA_DIR, + DATA_DIR, ) from src.db import ( - get_advisories_for_date, get_pattern_info, - get_recent_dates, get_all_advisories, get_stats, + get_pattern_info, + get_all_advisories, get_stats, ) @@ -150,7 +149,10 @@ def render_silent_page(): line = line.strip() if not line: continue - suspects.append(json.loads(line)) + try: + suspects.append(json.loads(line)) + except json.JSONDecodeError: + continue suspects.sort(key=lambda s: s.get("combined_score", 0), reverse=True) diff --git a/src/render_cli.py b/src/render_cli.py index 15a9578d7b..d98d04e7d1 100644 --- a/src/render_cli.py +++ b/src/render_cli.py @@ -1,9 +1,7 @@ import json -from datetime import date -from pathlib import Path from src.config import DOCS_DIR from src.db import ( - rebuild_from_jsonl, get_all_advisories, get_recent_dates, + rebuild_from_jsonl, get_recent_dates, get_advisories_for_date, get_stats, ) from src.render import render_readme, render_html_index, render_daily_patch, render_silent_page @@ -32,7 +30,7 @@ def generate_badge(stats: dict): (DOCS_DIR / "badge-patterns.json").write_text( json.dumps(badge_patterns), encoding="utf-8" ) - print(f" Generated badge endpoints") + print(" Generated badge endpoints") def run(): diff --git a/src/silent_scan.py b/src/silent_scan.py index acaafb4b70..bcace2501e 100644 --- a/src/silent_scan.py +++ b/src/silent_scan.py @@ -1,10 +1,8 @@ import json import time import sys -import os import requests from datetime import datetime, timedelta, timezone -from pathlib import Path from src.config import DATA_DIR, GITHUB_TOKEN from src.heuristics import score_commit from src.fingerprint import match_fingerprints @@ -47,7 +45,10 @@ def load_existing_results() -> set: line = line.strip() if not line: continue - record = json.loads(line) + try: + record = json.loads(line) + except json.JSONDecodeError: + continue seen.add(record.get("commit_sha", "")) return seen @@ -202,7 +203,7 @@ def run(hours: int = 24): } save_silent_state(state) - print(f"\n=== Summary ===") + print("\n=== Summary ===") print(f"Repos scanned: {len(watchlist) - skipped_repos}/{len(watchlist)}") print(f"Commits analyzed: {total_commits}") print(f"Layer 1 pass (heuristics >= 8): {layer1_pass}")