From e32c6102f3abe1fa8c5920e38497335653617d62 Mon Sep 17 00:00:00 2001 From: Chocapikk Date: Thu, 16 Apr 2026 20:30:21 +0200 Subject: [PATCH] Fix: Crash in deep_scan, validate LLM output, ruff cleanup - Fix undefined suspects_count in deep_scan.py (crashes on progress print) - Remove unused raw_combined variable in deep_scan.py - Add confidence validation (HIGH/MEDIUM/LOW) in analyze.py - Add diff truncation marker for LLM context awareness - Fix 32 ruff issues across 11 files (unused imports, f-strings without placeholders) --- src/analyze.py | 14 ++++++++------ src/backfill_local.py | 13 +++++-------- src/db.py | 1 - src/deep_scan.py | 7 ++----- src/fetch.py | 2 +- src/fingerprint.py | 1 - src/fingerprint_builder.py | 3 +-- src/main.py | 9 ++++----- src/render.py | 7 +++---- src/render_cli.py | 6 ++---- src/silent_scan.py | 4 +--- 11 files changed, 27 insertions(+), 40 deletions(-) diff --git a/src/analyze.py b/src/analyze.py index 2c1698210d..b4b3cc395d 100644 --- a/src/analyze.py +++ b/src/analyze.py @@ -6,7 +6,6 @@ GEMINI_API_KEY, GEMINI_API_URL, LLM_SYSTEM_PROMPT, LLM_USER_PROMPT_TEMPLATE, TAXONOMY_PATH, RETRY_ATTEMPTS, RETRY_BACKOFF, - RATE_LIMIT_DELAY, ) @@ -36,13 +35,14 @@ def analyze_advisory(advisory: dict, filtered_diff: str) -> Optional[dict]: summary=advisory["summary"], package_name=advisory["package_name"], ecosystem=advisory["ecosystem"], - diff_content=filtered_diff[:8000], + diff_content=filtered_diff[:8000] + + ("\n... [diff truncated]" if len(filtered_diff) > 8000 else ""), taxonomy_list=taxonomy_list, ) raw_response = _call_gemini(user_prompt) if not raw_response: - print(f" DEBUG: _call_gemini returned None") + print(" DEBUG: _call_gemini returned None") return None parsed = _parse_llm_response(raw_response) @@ -68,7 +68,9 @@ def analyze_advisory(advisory: dict, filtered_diff: str) -> Optional[dict]: "impact": parsed.get("impact", ""), "fix_summary": parsed.get("fix_summary", ""), "key_diff": parsed.get("key_diff", ""), - "confidence": parsed.get("confidence", "LOW"), + "confidence": parsed.get("confidence", "LOW") + if parsed.get("confidence", "LOW") in ("HIGH", "MEDIUM", "LOW") + else "LOW", "commit_url": advisory["commit_url"], } @@ -107,13 +109,13 @@ def _call_gemini(user_prompt: str) -> Optional[str]: data = response.json() candidates = data.get("candidates", []) if not candidates: - print(f" Gemini: no candidates in response") + print(" Gemini: no candidates in response") return None content = candidates[0].get("content", {}) parts = content.get("parts", []) if not parts: - print(f" Gemini: no parts in response") + print(" Gemini: no parts in response") return None return parts[0].get("text", "") diff --git a/src/backfill_local.py b/src/backfill_local.py index 7d9b82a93d..254272fbd7 100644 --- a/src/backfill_local.py +++ b/src/backfill_local.py @@ -1,12 +1,9 @@ import json import sys -import time import requests from datetime import datetime, timedelta, timezone from src.config import ( - RATE_LIMIT_DELAY, MAX_DIFF_LINES, LLM_SYSTEM_PROMPT, LLM_USER_PROMPT_TEMPLATE, - TAXONOMY_PATH, STATE_PATH, ) from src.fetch import fetch_advisories, fetch_commit_diff from src.diff_filter import filter_diff @@ -64,7 +61,7 @@ def analyze_with_ollama(advisory: dict, filtered_diff: str) -> dict | None: raw_response = call_ollama(user_prompt) if not raw_response: - print(f" Ollama returned nothing") + print(" Ollama returned nothing") return None parsed = _parse_llm_response(raw_response) @@ -154,13 +151,13 @@ def backfill_local(days: int): raw_diff = fetch_commit_diff(advisory["commit_url"]) if not raw_diff: - print(f" SKIP: no diff") + print(" SKIP: no diff") errors += 1 continue filtered = filter_diff(raw_diff) if not filtered: - print(f" SKIP: no relevant files") + print(" SKIP: no relevant files") continue result = analyze_with_ollama(advisory, filtered) @@ -195,12 +192,12 @@ def backfill_local(days: int): render_html_index() stats = get_stats() - print(f"\n=== Summary ===") + print("\n=== Summary ===") print(f"Processed: {processed}") print(f"New patterns: {new_patterns}") print(f"Errors: {errors}") print(f"Total DB: {stats['total_advisories']} advisories, {stats['total_patterns']} patterns") - print(f"\nCommit and push:") + print("\nCommit and push:") print(f" git add -A && git commit -m 'feat: local backfill — {processed} advisories' && git push") diff --git a/src/db.py b/src/db.py index 4c390b8d74..1533c9c20f 100644 --- a/src/db.py +++ b/src/db.py @@ -1,6 +1,5 @@ import json import sqlite3 -from pathlib import Path from typing import Optional from src.config import DB_PATH, JSONL_PATH diff --git a/src/deep_scan.py b/src/deep_scan.py index 211606d5e0..cc687749df 100644 --- a/src/deep_scan.py +++ b/src/deep_scan.py @@ -4,7 +4,6 @@ import os import requests from datetime import datetime, timezone -from pathlib import Path from src.config import DATA_DIR from src.heuristics import score_commit from src.fingerprint import match_fingerprints @@ -146,7 +145,7 @@ def deep_scan(repo: str, since: str = None, until: str = None, max_commits: int message = commit.get("commit", {}).get("message", "").split("\n")[0] if (i + 1) % 50 == 0 or (i + 1) == len(new_commits): - print(f" [{i+1}/{len(new_commits)}] {suspects_count} suspects so far...", flush=True) if 'suspects_count' in dir() else None + print(f" [{i+1}/{len(new_commits)}] {len(suspects)} suspects so far...", flush=True) detail = github_get(f"/repos/{repo}/commits/{sha}") if not detail: @@ -172,7 +171,6 @@ def deep_scan(repo: str, since: str = None, until: str = None, max_commits: int best_fp = fingerprint_matches[0] if fingerprint_matches else None fp_score = best_fp["score"] if best_fp else 0.0 - raw_combined = heuristic_result["score"] + (fp_score * 20) normalized = heuristic_result["normalized_score"] if best_fp: normalized = min(normalized + (fp_score * 30), 100) @@ -206,7 +204,6 @@ def deep_scan(repo: str, since: str = None, until: str = None, max_commits: int f.write(json.dumps(result, ensure_ascii=False) + "\n") suspects.append(result) - suspects_count = len(suspects) severity = "HIGH" if normalized >= 60 else "MEDIUM" if normalized >= 30 else "LOW" print(f" [{i+1}/{len(new_commits)}] {severity} score={normalized} {sha[:8]} {message[:60]}") @@ -221,7 +218,7 @@ def deep_scan(repo: str, since: str = None, until: str = None, max_commits: int if suspects: print(f"\nResults saved to: {results_path}") - print(f"\nTop suspects:") + print("\nTop suspects:") top = sorted(suspects, key=lambda s: s["normalized_score"], reverse=True)[:10] for s in top: print(f" score={s['normalized_score']:5.1f} {s['commit_sha'][:8]} {s['message'][:60]}") diff --git a/src/fetch.py b/src/fetch.py index 68a52a9666..f9d2bc258c 100644 --- a/src/fetch.py +++ b/src/fetch.py @@ -28,7 +28,7 @@ def graphql_request(query: str, variables: dict) -> dict: if "errors" in data: raise RuntimeError(f"GraphQL errors: {data['errors']}") return data - except (requests.RequestException, RuntimeError) as exc: + except (requests.RequestException, RuntimeError): if attempt == RETRY_ATTEMPTS - 1: raise time.sleep(RETRY_BACKOFF[attempt]) diff --git a/src/fingerprint.py b/src/fingerprint.py index 3c26ae64be..c8b7ac89f1 100644 --- a/src/fingerprint.py +++ b/src/fingerprint.py @@ -1,6 +1,5 @@ import json import re -from pathlib import Path from src.config import DATA_DIR diff --git a/src/fingerprint_builder.py b/src/fingerprint_builder.py index ee3ab30732..ba8ba9cb30 100644 --- a/src/fingerprint_builder.py +++ b/src/fingerprint_builder.py @@ -2,7 +2,6 @@ import re import sys from pathlib import Path -from collections import defaultdict EXPERT_FINGERPRINTS = { @@ -246,7 +245,7 @@ def build_fingerprints(output_path: str = "data/fingerprints.json"): print(f" Expert CWE: {len(EXPERT_FINGERPRINTS)}") print(f" OSDC live: {len([k for k in fingerprints if k.startswith('OSDC:')])}") - print(f"\nTop patterns by token count:") + print("\nTop patterns by token count:") top = sorted(fingerprints.items(), key=lambda x: len(x[1]["add_tokens"]) + len(x[1]["del_tokens"]), reverse=True)[:10] for pid, data in top: total = len(data["add_tokens"]) + len(data["del_tokens"]) diff --git a/src/main.py b/src/main.py index 2b50f54619..6ddb33566a 100644 --- a/src/main.py +++ b/src/main.py @@ -1,7 +1,6 @@ import json import time import signal -import sys from datetime import datetime, timezone from src.config import ( STATE_PATH, MAX_DAILY_CALLS, RATE_LIMIT_DELAY, @@ -98,18 +97,18 @@ def run(): raw_diff = fetch_commit_diff(advisory["commit_url"]) if not raw_diff: - print(f" SKIP: no diff available") + print(" SKIP: no diff available") errors += 1 continue filtered = filter_diff(raw_diff) if not filtered: - print(f" SKIP: no relevant files in diff") + print(" SKIP: no relevant files in diff") continue result = analyze_advisory(advisory, filtered) if not result: - print(f" ERROR: LLM analysis failed") + print(" ERROR: LLM analysis failed") errors += 1 continue @@ -174,7 +173,7 @@ def run(): encoding="utf-8", ) - print(f"\n=== Summary ===") + print("\n=== Summary ===") print(f"Analyzed: {len(analyzed)}") print(f"New patterns: {new_patterns}") print(f"Errors: {errors}") diff --git a/src/render.py b/src/render.py index 8aba29087c..f272dc7373 100644 --- a/src/render.py +++ b/src/render.py @@ -1,15 +1,14 @@ import json import re from datetime import date -from pathlib import Path from jinja2 import Environment, FileSystemLoader from src.config import ( TEMPLATES_DIR, PATCHES_DIR, DOCS_DIR, ROOT_DIR, - README_DAYS_SHOWN, DATA_DIR, + DATA_DIR, ) from src.db import ( - get_advisories_for_date, get_pattern_info, - get_recent_dates, get_all_advisories, get_stats, + get_pattern_info, + get_all_advisories, get_stats, ) diff --git a/src/render_cli.py b/src/render_cli.py index 15a9578d7b..d98d04e7d1 100644 --- a/src/render_cli.py +++ b/src/render_cli.py @@ -1,9 +1,7 @@ import json -from datetime import date -from pathlib import Path from src.config import DOCS_DIR from src.db import ( - rebuild_from_jsonl, get_all_advisories, get_recent_dates, + rebuild_from_jsonl, get_recent_dates, get_advisories_for_date, get_stats, ) from src.render import render_readme, render_html_index, render_daily_patch, render_silent_page @@ -32,7 +30,7 @@ def generate_badge(stats: dict): (DOCS_DIR / "badge-patterns.json").write_text( json.dumps(badge_patterns), encoding="utf-8" ) - print(f" Generated badge endpoints") + print(" Generated badge endpoints") def run(): diff --git a/src/silent_scan.py b/src/silent_scan.py index acaafb4b70..d674fde986 100644 --- a/src/silent_scan.py +++ b/src/silent_scan.py @@ -1,10 +1,8 @@ import json import time import sys -import os import requests from datetime import datetime, timedelta, timezone -from pathlib import Path from src.config import DATA_DIR, GITHUB_TOKEN from src.heuristics import score_commit from src.fingerprint import match_fingerprints @@ -202,7 +200,7 @@ def run(hours: int = 24): } save_silent_state(state) - print(f"\n=== Summary ===") + print("\n=== Summary ===") print(f"Repos scanned: {len(watchlist) - skipped_repos}/{len(watchlist)}") print(f"Commits analyzed: {total_commits}") print(f"Layer 1 pass (heuristics >= 8): {layer1_pass}")