From 18745f14cd4da6600ce56f03b90da6e5f7964c51 Mon Sep 17 00:00:00 2001 From: Jean Paul Elisa NIYOKWIZERWA <140616733+Ndevu12@users.noreply.github.com> Date: Tue, 11 Aug 2026 08:29:21 +0000 Subject: [PATCH 1/4] fix: correct concurrency, recency, persistence, and API bugs in core pipeline - Run LLM paper extraction concurrently under the configured semaphore instead of sequentially awaiting inside the loop; the circuit breaker still short-circuits remaining papers after repeated failures - Derive recency scores from the current UTC year instead of a hardcoded year that silently goes stale - Persist the expanded query variants generated during a run to session memory instead of always saving an empty list - Return failed provider names structurally from the retrieval fan-out instead of re-parsing warning strings, and aggregate results for providers that share a name - Use the current pydantic-ai structured-output API (output_type / result.output) in LLMProvider.complete; the legacy result_type call raised on pydantic-ai 1.x - Point partial-recovery fallbacks at the real recovery helper; the previous import target never existed and raised ImportError --- src/analysis/synthesis.py | 90 ++++++++++++++-------------- src/core/pipeline.py | 1 + src/models/base.py | 13 ++-- src/research/ranking.py | 78 +++--------------------- src/retrieval/orchestrator.py | 24 +++++++- src/retrieval/providers/registry.py | 18 ++++-- src/retrieval/retrieval_stage.py | 93 +++++++++++++---------------- src/utils/enhanced_validation.py | 4 +- src/utils/fallback_processing.py | 7 +-- tests/test_providers.py | 3 +- tests/test_retrieval_stage.py | 25 +++++--- 11 files changed, 160 insertions(+), 196 deletions(-) diff --git a/src/analysis/synthesis.py b/src/analysis/synthesis.py index 66dcb96..dcba385 100644 --- a/src/analysis/synthesis.py +++ b/src/analysis/synthesis.py @@ -12,8 +12,8 @@ from ..core.context import PipelineContext, StageResult from ..core.paper_adapters import ensure_ranked_papers -from ..models import AgentFactory, AgentRole, ROLE_SYSTEM_PROMPTS -from ..research.query_expansion import extract_core_concepts +from ..models import AgentFactory, AgentRole, create_llm_agent +from ..research.text_utils import extract_core_concepts from ..retrieval.models import ( PaperAnalysis, PaperCluster, @@ -29,13 +29,7 @@ from ..config.settings import LLMConfig, SynthesisConfig -def create_llm_agent(system_prompt: str, llm_config: LLMConfig | None = None) -> Agent: - """Create a pydantic-ai agent from application LLM settings.""" - return AgentFactory(llm_config).create_agent_with_prompt(system_prompt, llm_config) - - -EXTRACTION_SYSTEM_PROMPT = ROLE_SYSTEM_PROMPTS[AgentRole.EXTRACTION] -SYNTHESIS_SYSTEM_PROMPT = ROLE_SYSTEM_PROMPTS[AgentRole.SYNTHESIS] +__all__ = ["create_llm_agent", "SynthesisStage", "extract_papers", "synthesize_collective"] HEURISTIC_DISAGREEMENT_PLACEHOLDER = ( "Cross-paper disagreement analysis limited in heuristic mode." @@ -261,10 +255,6 @@ def _synthesis_handler_config(max_retries: int) -> ResponseHandlerConfig: ) -def _extraction_handler_config(max_retries: int) -> ResponseHandlerConfig: - return _synthesis_handler_config(max_retries) - - def resolve_synthesis_input(data: object, ctx: PipelineContext) -> SynthesisResult: """Resolve a synthesis result from stage output or pipeline artifacts.""" artifact = ctx.get_artifact("synthesis_result") @@ -367,7 +357,7 @@ async def extract_papers( agent = AgentFactory(llm_config).create_agent(AgentRole.EXTRACTION, config=llm_config) response_handler = handler or EnhancedResponseHandler( - _extraction_handler_config(synthesis_config.extraction_max_retries) + _synthesis_handler_config(synthesis_config.extraction_max_retries) ) context = RequestContext( user_query=query, @@ -375,33 +365,30 @@ async def extract_papers( session_id=session_id, ) - semaphore = asyncio.Semaphore(max(concurrency, 1)) - consecutive_failures = 0 - circuit_open = False - llm_extractions: list[PaperExtraction] = [] - - for index, paper in enumerate(llm_targets, start=1): - if circuit_open: - llm_extractions.append(_heuristic_extraction(paper)) - continue - - title_preview = paper.paper.title[:72] - logger.info( - "LLM extracting paper %d/%d: %s", - index, - len(llm_targets), - paper.paper.title[:80], - ) + from ..utils.progress_reporter import get_progress_reporter - from ..utils.progress_reporter import get_progress_reporter + semaphore = asyncio.Semaphore(max(concurrency, 1)) + breaker = {"consecutive_failures": 0, "open": False} - reporter = get_progress_reporter() - if reporter is not None: - reporter.set_activity( - f"Analyzing paper {index}/{len(llm_targets)}: {title_preview}…" + async def _extract_with_circuit(index: int, paper: RankedPaper) -> PaperExtraction: + async with semaphore: + if breaker["open"]: + return _heuristic_extraction(paper) + + title_preview = paper.paper.title[:72] + logger.info( + "LLM extracting paper %d/%d: %s", + index, + len(llm_targets), + paper.paper.title[:80], ) - async with semaphore: + reporter = get_progress_reporter() + if reporter is not None: + reporter.set_activity( + f"Analyzing paper {index}/{len(llm_targets)}: {title_preview}…" + ) + extraction, llm_success = await _extract_single_paper( paper, query, @@ -410,19 +397,32 @@ async def extract_papers( context, ) - if not llm_success: - consecutive_failures += 1 - if consecutive_failures >= synthesis_config.circuit_breaker_failures: - circuit_open = True + if llm_success: + breaker["consecutive_failures"] = 0 + else: + breaker["consecutive_failures"] += 1 + if ( + not breaker["open"] + and breaker["consecutive_failures"] + >= synthesis_config.circuit_breaker_failures + ): + breaker["open"] = True logger.warning( "LLM extraction circuit breaker open after %d failures; " "using heuristics for remaining papers", - consecutive_failures, + breaker["consecutive_failures"], ) - else: - consecutive_failures = 0 - llm_extractions.append(extraction) + return extraction + + llm_extractions = list( + await asyncio.gather( + *( + _extract_with_circuit(index, paper) + for index, paper in enumerate(llm_targets, start=1) + ) + ) + ) heuristic_extractions = [_heuristic_extraction(paper) for paper in heuristic_targets] return llm_extractions + heuristic_extractions diff --git a/src/core/pipeline.py b/src/core/pipeline.py index b5d5734..682be1c 100644 --- a/src/core/pipeline.py +++ b/src/core/pipeline.py @@ -165,6 +165,7 @@ async def execute( } artifact_keys = ( + "expanded_queries", "ranked_papers", "retrieved_papers", "paper_analyses", diff --git a/src/models/base.py b/src/models/base.py index 4a61e3d..c8292d2 100644 --- a/src/models/base.py +++ b/src/models/base.py @@ -107,11 +107,14 @@ async def complete( ) -> str: """Run a single completion and return the model output text.""" model = self.create_model(config) - agent = Agent(model=model, system_prompt=system_prompt or "") if schema is not None: - result = await agent.run(prompt, result_type=schema) - if hasattr(result, "data") and result.data is not None: - return result.data.model_dump_json() - return str(result.output) + structured_agent: Agent[None, BaseModel] = Agent( + model=model, + system_prompt=system_prompt or "", + output_type=schema, + ) + structured_result = await structured_agent.run(prompt) + return structured_result.output.model_dump_json() + agent = Agent(model=model, system_prompt=system_prompt or "") result = await agent.run(prompt) return str(result.output) diff --git a/src/research/ranking.py b/src/research/ranking.py index 1a9b2ae..a47d4bc 100644 --- a/src/research/ranking.py +++ b/src/research/ranking.py @@ -7,6 +7,7 @@ import re import time from dataclasses import dataclass +from datetime import datetime, timezone from typing import TYPE_CHECKING import numpy as np @@ -17,39 +18,11 @@ from .canonical_works import CanonicalWork, load_canonical_works, match_canonical_work from .embedding_context import store_ranking_embedding_result from .metadata_sanity import sanitize_papers_metadata +from .text_utils import GENERIC_QUERY_TERMS, extract_query_terms, term_matches_text if TYPE_CHECKING: from ..config.settings import RankingConfig, RankingWeights -_STOP_WORDS = { - "the", - "a", - "an", - "and", - "or", - "but", - "in", - "on", - "at", - "to", - "for", - "of", - "with", - "by", - "is", - "are", - "was", - "were", - "be", - "been", - "being", - "paper", - "papers", - "research", - "study", - "studies", -} - _KNOWN_VENUES = { "neurips", "nips", @@ -68,31 +41,6 @@ "jmlr", } -_GENERIC_QUERY_TERMS = frozenset( - { - "mechanism", - "mechanisms", - "method", - "methods", - "approach", - "approaches", - "application", - "applications", - "model", - "models", - "system", - "systems", - "based", - "using", - "recent", - } -) - - -def _extract_query_terms(query: str) -> set[str]: - words = re.findall(r"\b\w+\b", query.lower()) - return {word for word in words if word not in _STOP_WORDS and len(word) > 2} - def _paper_text(paper: RetrievedPaper) -> str: parts = [paper.title] @@ -110,28 +58,18 @@ def _paper_text_lower(paper: RetrievedPaper) -> str: def _core_query_terms(query_terms: set[str]) -> list[str]: - return sorted(term for term in query_terms if term not in _GENERIC_QUERY_TERMS) - - -def _term_matches_text(term: str, text: str) -> bool: - if term in text: - return True - if term.endswith("s") and term[:-1] in text: - return True - if f"{term}s" in text: - return True - return False + return sorted(term for term in query_terms if term not in GENERIC_QUERY_TERMS) def applies_domain_penalty(query: str, paper: RetrievedPaper) -> bool: """Return True when a multi-concept query is only partially matched.""" - query_terms = _extract_query_terms(query) + query_terms = extract_query_terms(query) core_terms = _core_query_terms(query_terms) if len(core_terms) < 2: return False text = _paper_text_lower(paper) - matched_core = [term for term in core_terms if _term_matches_text(term, text)] + matched_core = [term for term in core_terms if term_matches_text(term, text)] return len(matched_core) < len(core_terms) @@ -232,9 +170,11 @@ def signal_citation_count(paper: RetrievedPaper, papers: list[RetrievedPaper]) - return min(normalized, 1.0) -def signal_recency(paper: RetrievedPaper, current_year: int = 2026) -> float | None: +def signal_recency(paper: RetrievedPaper, current_year: int | None = None) -> float | None: if paper.year is None: return None + if current_year is None: + current_year = datetime.now(timezone.utc).year age = max(current_year - paper.year, 0) if age >= 20: return 0.1 @@ -311,7 +251,7 @@ def score_paper( canonical_boost: float = 0.0, canonical_works: list[CanonicalWork] | None = None, ) -> RankedPaper: - query_terms = _extract_query_terms(query) + query_terms = extract_query_terms(query) embedding_sim = signal_embedding_similarity( paper, query_embedding, diff --git a/src/retrieval/orchestrator.py b/src/retrieval/orchestrator.py index 21d1ca4..eb2b456 100644 --- a/src/retrieval/orchestrator.py +++ b/src/retrieval/orchestrator.py @@ -22,7 +22,12 @@ from ..research.ranking import RankingStage from ..research.relevance_scoring import RelevanceScoringStage from ..retrieval.deduplication import DeduplicationStage -from ..retrieval.models import EnhancedResearchReport, RankedPaper, RetrievedPaper +from ..retrieval.models import ( + EnhancedResearchReport, + ExpandedQuerySet, + RankedPaper, + RetrievedPaper, +) from ..retrieval.retrieval_stage import RetrievalStage from ..utils.message_formatter import MessageFormatter from ..utils.progress_reporter import ( @@ -52,6 +57,21 @@ def build_pipeline(settings: AppSettings) -> ResearchPipeline: ) +def _resolve_expanded_queries(result: ResearchPipelineResult, query: str) -> list[str]: + """Collect expanded query variants generated during the run.""" + artifact = result.artifacts.get("expanded_queries") + if isinstance(artifact, ExpandedQuerySet): + expanded = [*artifact.variants, *artifact.sub_questions] + elif isinstance(artifact, dict): + expanded = [ + *artifact.get("variants", []), + *artifact.get("sub_questions", []), + ] + else: + expanded = [] + return [item for item in dict.fromkeys(expanded) if item and item != query] + + def _resolve_report(result: ResearchPipelineResult, query: str) -> EnhancedResearchReport: report = result.output if isinstance(report, EnhancedResearchReport): @@ -74,7 +94,7 @@ async def _persist_memory( enabled_providers = [name for name, cfg in settings.retrieval.providers.items() if cfg.enabled] cache_key = build_cache_key(query, enabled_providers, build_config_hash(settings)) - expanded: list[str] = [] + expanded = _resolve_expanded_queries(result, query) search_id = await store.save_search( session.id, query, diff --git a/src/retrieval/providers/registry.py b/src/retrieval/providers/registry.py index 72ed981..00167db 100644 --- a/src/retrieval/providers/registry.py +++ b/src/retrieval/providers/registry.py @@ -95,11 +95,15 @@ async def search_enabled_providers( query: str, settings: AppSettings | None = None, limit: int | None = None, -) -> tuple[dict[str, list[RetrievedPaper]], list[str]]: - """Search all enabled providers with graceful per-provider fallback.""" +) -> tuple[dict[str, list[RetrievedPaper]], list[str], set[str]]: + """Search all enabled providers with graceful per-provider fallback. + + Returns papers grouped by provider name, warning messages, and the names of + providers that raised an error. + """ providers = get_enabled_providers(settings) if not providers: - return {}, ["No retrieval providers enabled"] + return {}, ["No retrieval providers enabled"], set() async def _safe_search( provider: RetrievalProvider, @@ -114,12 +118,14 @@ async def _safe_search( by_provider: dict[str, list[RetrievedPaper]] = {} warnings: list[str] = [] + failed: set[str] = set() for provider_name, papers, error in results: - by_provider[provider_name] = papers + by_provider.setdefault(provider_name, []).extend(papers) if error: warnings.append(f"Provider '{provider_name}' failed for query '{query}': {error}") + failed.add(provider_name) - return by_provider, warnings + return by_provider, warnings, failed async def search_all_enabled( @@ -129,7 +135,7 @@ async def search_all_enabled( limit: int | None = None, ) -> list[RetrievedPaper]: """Search all enabled providers and return a flat combined list.""" - by_provider, _warnings = await search_enabled_providers( + by_provider, _warnings, _failed = await search_enabled_providers( session, query, settings=settings, diff --git a/src/retrieval/retrieval_stage.py b/src/retrieval/retrieval_stage.py index 729fed6..5290d4a 100644 --- a/src/retrieval/retrieval_stage.py +++ b/src/retrieval/retrieval_stage.py @@ -11,7 +11,7 @@ from ..core.context import PipelineContext, StageResult from ..retrieval.models import ExpandedQuerySet, RetrievedPaper -from .providers.registry import get_enabled_providers +from .providers.registry import search_enabled_providers if TYPE_CHECKING: from ..config.settings import AppSettings @@ -21,73 +21,64 @@ async def _search_query( session: aiohttp.ClientSession, query: str, settings: AppSettings, -) -> tuple[str, list[RetrievedPaper], list[str]]: +) -> tuple[str, list[RetrievedPaper], list[str], set[str]]: """Search all enabled providers for a single query string.""" - warnings: list[str] = [] - providers = get_enabled_providers(settings) - if not providers: - warnings.append("No retrieval providers enabled") - return query, [], warnings - - limit = settings.retrieval.per_provider_limit - - async def _safe_search(provider) -> tuple[str, list[RetrievedPaper]]: - try: - papers = await provider.search(session, query, limit=limit) - return provider.name, papers - except Exception as exc: - warnings.append(f"Provider '{provider.name}' failed for query '{query}': {exc}") - return provider.name, [] - - results = await asyncio.gather(*(_safe_search(provider) for provider in providers)) + by_provider, warnings, failed_providers = await search_enabled_providers( + session, + query, + settings=settings, + limit=settings.retrieval.per_provider_limit, + ) + combined: list[RetrievedPaper] = [] - failed_providers: list[str] = [] - - for provider_name, papers in results: - if papers: - updated: list[RetrievedPaper] = [] - for paper in papers: - provenance = list(paper.raw_metadata.get("found_by_queries", [])) - if query not in provenance: - provenance.append(query) - updated.append( - paper.model_copy( - update={ - "raw_metadata": { - **paper.raw_metadata, - "found_by_queries": provenance, - } + for papers in by_provider.values(): + for paper in papers: + provenance = list(paper.raw_metadata.get("found_by_queries", [])) + if query not in provenance: + provenance.append(query) + combined.append( + paper.model_copy( + update={ + "raw_metadata": { + **paper.raw_metadata, + "found_by_queries": provenance, } - ) + } ) - combined.extend(updated) - else: - failed_providers.append(provider_name) + ) - return query, combined, warnings + return query, combined, warnings, failed_providers async def retrieve_papers( expanded: ExpandedQuerySet, settings: AppSettings, session: aiohttp.ClientSession | None = None, -) -> tuple[list[RetrievedPaper], list[str]]: - """Retrieve papers for the original query and expanded variants.""" +) -> tuple[list[RetrievedPaper], list[str], list[str]]: + """Retrieve papers for the original query and expanded variants. + + Returns the combined papers, accumulated warnings, and the sorted names of + providers that raised at least one error. + """ queries = [expanded.original, *expanded.variants] queries = list(dict.fromkeys(query.strip() for query in queries if query.strip())) concurrency = max(1, settings.retrieval.concurrency_limit) semaphore = asyncio.Semaphore(concurrency) all_warnings: list[str] = [] + all_failed: set[str] = set() async def _run_query(query: str) -> list[RetrievedPaper]: async with semaphore: if session is not None: - _, papers, warnings = await _search_query(session, query, settings) + _, papers, warnings, failed = await _search_query(session, query, settings) else: async with aiohttp.ClientSession() as local_session: - _, papers, warnings = await _search_query(local_session, query, settings) + _, papers, warnings, failed = await _search_query( + local_session, query, settings + ) all_warnings.extend(warnings) + all_failed.update(failed) return papers batches = await asyncio.gather(*(_run_query(query) for query in queries)) @@ -95,7 +86,7 @@ async def _run_query(query: str) -> list[RetrievedPaper]: for papers in batches: combined.extend(papers) - return combined, all_warnings + return combined, all_warnings, sorted(all_failed) class RetrievalStage: @@ -128,9 +119,12 @@ async def run( warnings=["Retrieval skipped: using cached papers"], ) + providers_failed: list[str] = [] async with aiohttp.ClientSession() as session: try: - papers, search_warnings = await retrieve_papers(data, ctx.config, session) + papers, search_warnings, providers_failed = await retrieve_papers( + data, ctx.config, session + ) warnings.extend(search_warnings) except Exception as exc: warnings.append(f"Retrieval failed: {exc}") @@ -140,13 +134,6 @@ async def run( if warnings: partial = True - providers_failed = sorted( - { - warning.split("'")[1] - for warning in warnings - if warning.startswith("Provider '") - } - ) ctx.metrics.record_retrieval( papers_found=len(papers), providers_failed=providers_failed, diff --git a/src/utils/enhanced_validation.py b/src/utils/enhanced_validation.py index fd53b33..9936e0b 100644 --- a/src/utils/enhanced_validation.py +++ b/src/utils/enhanced_validation.py @@ -132,5 +132,5 @@ def _enhanced_partial_recovery( ) except ImportError: # Fallback processing not available, use existing recovery - from ..retrieval.orchestrator import _attempt_partial_recovery - return _attempt_partial_recovery(raw_output, clean_json, parsed_data) \ No newline at end of file + from ..retrieval.helpers_modules.recovery import enhanced_partial_recovery + return enhanced_partial_recovery(raw_output, clean_json, parsed_data) \ No newline at end of file diff --git a/src/utils/fallback_processing.py b/src/utils/fallback_processing.py index 98a4f8c..1b530f0 100644 --- a/src/utils/fallback_processing.py +++ b/src/utils/fallback_processing.py @@ -285,7 +285,6 @@ def _extract_research_themes(self, response: str, query: str) -> FallbackResult: Returns: FallbackResult: Extracted themes """ - themes = [] warnings = [] # Look for research-related keywords @@ -517,10 +516,10 @@ def enhance_partial_recovery_with_fallback( RecoveryResult: Enhanced recovery result with fallback data """ # First try existing recovery methods - from ..retrieval.orchestrator import _attempt_partial_recovery - + from ..retrieval.helpers_modules.recovery import enhanced_partial_recovery + try: - existing_result = _attempt_partial_recovery(raw_output, clean_json, parsed_data) + existing_result = enhanced_partial_recovery(raw_output, clean_json, parsed_data) if existing_result.success and existing_result.confidence_score > 0.5: logger.info("Existing recovery method succeeded, using that result") return existing_result diff --git a/tests/test_providers.py b/tests/test_providers.py index bd9607d..fd607ea 100644 --- a/tests/test_providers.py +++ b/tests/test_providers.py @@ -137,7 +137,7 @@ async def test_search_enabled_providers_falls_back_when_one_provider_fails() -> "src.retrieval.providers.registry.get_enabled_providers", return_value=providers, ): - by_provider, warnings = await search_enabled_providers( + by_provider, warnings, failed = await search_enabled_providers( session, "transformers", settings=settings, @@ -146,6 +146,7 @@ async def test_search_enabled_providers_falls_back_when_one_provider_fails() -> assert by_provider["success_registry_provider"][0].title == "Recovered" assert by_provider["failing_registry_provider"] == [] assert any("failing_registry_provider" in warning for warning in warnings) + assert failed == {"failing_registry_provider"} @pytest.mark.asyncio diff --git a/tests/test_retrieval_stage.py b/tests/test_retrieval_stage.py index 12b970e..6b043d0 100644 --- a/tests/test_retrieval_stage.py +++ b/tests/test_retrieval_stage.py @@ -80,15 +80,18 @@ async def test_search_query_continues_when_one_provider_fails() -> None: async with aiohttp.ClientSession() as session: with patch( - "src.retrieval.retrieval_stage.get_enabled_providers", + "src.retrieval.providers.registry.get_enabled_providers", return_value=providers, ): - query, papers, warnings = await _search_query(session, "transformers", settings) + query, papers, warnings, failed = await _search_query( + session, "transformers", settings + ) assert query == "transformers" assert len(papers) == 1 assert papers[0].title == "OpenAlex Paper" assert any("failing_provider" in warning for warning in warnings) + assert failed == {"failing_provider"} @pytest.mark.asyncio @@ -98,13 +101,14 @@ async def test_search_query_returns_empty_when_all_providers_fail() -> None: async with aiohttp.ClientSession() as session: with patch( - "src.retrieval.retrieval_stage.get_enabled_providers", + "src.retrieval.providers.registry.get_enabled_providers", return_value=providers, ): - _, papers, warnings = await _search_query(session, "biology", settings) + _, papers, warnings, failed = await _search_query(session, "biology", settings) assert papers == [] assert len(warnings) == 2 + assert failed == {"failing_provider"} @pytest.mark.asyncio @@ -118,14 +122,17 @@ async def test_retrieve_papers_merges_successful_provider_results() -> None: async with aiohttp.ClientSession() as session: with patch( - "src.retrieval.retrieval_stage.get_enabled_providers", + "src.retrieval.providers.registry.get_enabled_providers", return_value=providers, ): - papers, warnings = await retrieve_papers(expanded, settings, session) + papers, warnings, providers_failed = await retrieve_papers( + expanded, settings, session + ) assert len(papers) == 1 assert papers[0].title == "Paper A" assert warnings + assert providers_failed == ["failing_provider"] @pytest.mark.asyncio @@ -140,7 +147,7 @@ async def test_retrieval_stage_marks_partial_when_provider_fails() -> None: ] with patch( - "src.retrieval.retrieval_stage.get_enabled_providers", + "src.retrieval.providers.registry.get_enabled_providers", return_value=providers, ): result = await stage.run(ctx, expanded) @@ -164,7 +171,7 @@ async def test_retrieval_stage_not_partial_when_all_providers_succeed() -> None: ] with patch( - "src.retrieval.retrieval_stage.get_enabled_providers", + "src.retrieval.providers.registry.get_enabled_providers", return_value=providers, ): result = await stage.run(ctx, expanded) @@ -212,7 +219,7 @@ async def run(self, ctx: PipelineContext, data: object) -> object: return StageResult(output=f"processed:{len(data)}", duration_ms=1.0) with patch( - "src.retrieval.retrieval_stage.get_enabled_providers", + "src.retrieval.providers.registry.get_enabled_providers", return_value=providers, ): pipeline = ResearchPipeline( From 33bcda41177a3623524f1ac26973c42f66ab6fdc Mon Sep 17 00:00:00 2001 From: Jean Paul Elisa NIYOKWIZERWA <140616733+Ndevu12@users.noreply.github.com> Date: Tue, 11 Aug 2026 08:29:21 +0000 Subject: [PATCH 2/4] refactor: consolidate duplicated helpers and remove dead code - Add src/research/text_utils.py as the single home for stop-word sets, generic query terms, query-term extraction, and plural-aware term matching that were copy-pasted across ranking, relevance scoring, clustering, and query expansion - Make cosine similarity a single implementation on the embeddings base module; providers and embedding_context now delegate to it - Vectorize embedding deduplication with one matrix product instead of per-pair Python cosine calls - Route the retrieval stage through the registry's provider fan-out so the gather-and-collect logic exists once - Extract the repeated macro-cluster fallback block in clustering into a helper - Drop the deprecated search_openalex / search_semantic_scholar shims, the unused GAP_ANALYSIS_SYSTEM_PROMPT rebinding, and alias functions that only forwarded to other helpers --- src/analysis/gap_analysis.py | 5 +- src/embeddings/base.py | 11 ++- src/embeddings/sentence_transformers.py | 11 --- src/research/clustering.py | 83 ++++--------------- src/research/embedding_context.py | 21 ++--- src/research/query_expansion.py | 47 +++-------- src/research/relevance_scoring.py | 42 ++-------- src/research/text_utils.py | 101 ++++++++++++++++++++++++ src/retrieval/__init__.py | 2 - src/retrieval/deduplication.py | 23 +++--- src/retrieval/openalex.py | 28 ------- src/retrieval/semanticscholar.py | 28 ------- 12 files changed, 171 insertions(+), 231 deletions(-) create mode 100644 src/research/text_utils.py delete mode 100644 src/retrieval/openalex.py delete mode 100644 src/retrieval/semanticscholar.py diff --git a/src/analysis/gap_analysis.py b/src/analysis/gap_analysis.py index e99659c..db7dccd 100644 --- a/src/analysis/gap_analysis.py +++ b/src/analysis/gap_analysis.py @@ -8,7 +8,7 @@ from typing import TYPE_CHECKING from ..core.context import PipelineContext, StageResult -from ..models import AgentFactory, AgentRole, ROLE_SYSTEM_PROMPTS +from ..models import AgentFactory, AgentRole from ..retrieval.models import GapAnalysisResult, PaperCluster, SynthesisResult from ..utils.enhanced_response_handler import EnhancedResponseHandler from ..utils.response_models import RequestContext, ResponseHandlerConfig @@ -17,9 +17,6 @@ from ..config.settings import LLMConfig -GAP_ANALYSIS_SYSTEM_PROMPT = ROLE_SYSTEM_PROMPTS[AgentRole.GAP_ANALYSIS] - - def resolve_synthesis_input(data: object, ctx: PipelineContext) -> SynthesisResult: """Resolve synthesis output even when an upstream stage passed the wrong type.""" from .synthesis import resolve_synthesis_input as _resolve diff --git a/src/embeddings/base.py b/src/embeddings/base.py index 4bea1e4..04303f2 100644 --- a/src/embeddings/base.py +++ b/src/embeddings/base.py @@ -8,6 +8,15 @@ import numpy as np +def cosine_similarity(a: np.ndarray, b: np.ndarray) -> float: + """Cosine similarity between two vectors.""" + a_norm = np.linalg.norm(a) + b_norm = np.linalg.norm(b) + if a_norm == 0.0 or b_norm == 0.0: + return 0.0 + return float(np.dot(a, b) / (a_norm * b_norm)) + + class EmbeddingProvider(ABC): """Interface for text embedding backends.""" @@ -15,9 +24,9 @@ class EmbeddingProvider(ABC): def embed_texts(self, texts: list[str]) -> np.ndarray: """Return an array of shape ``(len(texts), dim)``.""" - @abstractmethod def similarity(self, a: np.ndarray, b: np.ndarray) -> float: """Return cosine similarity between two vectors.""" + return cosine_similarity(a, b) def embed_text(self, text: str) -> np.ndarray: """Embed a single text and return its vector.""" diff --git a/src/embeddings/sentence_transformers.py b/src/embeddings/sentence_transformers.py index dff7010..33f4505 100644 --- a/src/embeddings/sentence_transformers.py +++ b/src/embeddings/sentence_transformers.py @@ -16,14 +16,6 @@ from ..config.settings import EmbeddingConfig -def _cosine_similarity(a: np.ndarray, b: np.ndarray) -> float: - a_norm = np.linalg.norm(a) - b_norm = np.linalg.norm(b) - if a_norm == 0.0 or b_norm == 0.0: - return 0.0 - return float(np.dot(a, b) / (a_norm * b_norm)) - - class SentenceTransformerEmbeddingProvider(EmbeddingProvider): """Embedding provider backed by ``sentence-transformers``.""" @@ -80,9 +72,6 @@ def embed_texts(self, texts: list[str]) -> np.ndarray: return output - def similarity(self, a: np.ndarray, b: np.ndarray) -> float: - return _cosine_similarity(a, b) - def create_embedding_provider( config: EmbeddingConfig | None = None, diff --git a/src/research/clustering.py b/src/research/clustering.py index 13e868e..8aa258c 100644 --- a/src/research/clustering.py +++ b/src/research/clustering.py @@ -15,31 +15,11 @@ from ..embeddings.base import EmbeddingProvider from ..retrieval.models import PaperCluster, RankedPaper from .embedding_context import get_paper_embeddings +from .text_utils import LABEL_STOP_WORDS if TYPE_CHECKING: from ..config.settings import ClusteringConfig -_STOP_WORDS = { - "the", - "a", - "an", - "and", - "or", - "but", - "in", - "on", - "at", - "to", - "for", - "of", - "with", - "by", - "using", - "based", - "via", - "from", -} - def _paper_text(paper: RankedPaper) -> str: parts = [paper.paper.title] @@ -52,7 +32,7 @@ def _label_cluster(papers: list[RankedPaper]) -> tuple[str, str]: words: list[str] = [] for ranked in papers: tokens = re.findall(r"\b[a-z]{4,}\b", ranked.paper.title.lower()) - words.extend(token for token in tokens if token not in _STOP_WORDS) + words.extend(token for token in tokens if token not in LABEL_STOP_WORDS) if not words: return "General", "Related papers grouped by embedding similarity." @@ -91,7 +71,7 @@ def _fallback_single_cluster(papers: list[RankedPaper]) -> list[PaperCluster]: def _extract_tokens(text: str) -> list[str]: tokens = re.findall(r"\b[a-z]{4,}\b", text.lower()) - return [token for token in tokens if token not in _STOP_WORDS] + return [token for token in tokens if token not in LABEL_STOP_WORDS] def _macro_cluster_count(noise_count: int, config: ClusteringConfig) -> int: @@ -100,6 +80,15 @@ def _macro_cluster_count(noise_count: int, config: ClusteringConfig) -> int: return min(config.max_macro_clusters, noise_count) +def _macro_cluster(papers: list[RankedPaper]) -> PaperCluster: + theme, summary = _label_cluster(papers) + return PaperCluster( + theme=f"Theme: {theme}", + summary=summary, + paper_ids=[paper.paper.paper_id for paper in papers], + ) + + def _merge_noise_into_macro_clusters( noise_papers: list[RankedPaper], config: ClusteringConfig, @@ -109,26 +98,12 @@ def _merge_noise_into_macro_clusters( return [] if len(noise_papers) == 1: - theme, summary = _label_cluster(noise_papers) - return [ - PaperCluster( - theme=f"Theme: {theme}", - summary=summary, - paper_ids=[noise_papers[0].paper.paper_id], - ) - ] + return [_macro_cluster(noise_papers)] doc_tokens = [_extract_tokens(_paper_text(paper)) for paper in noise_papers] vocabulary = sorted({token for tokens in doc_tokens for token in tokens}) if not vocabulary: - theme, summary = _label_cluster(noise_papers) - return [ - PaperCluster( - theme=f"Theme: {theme}", - summary=summary, - paper_ids=[paper.paper.paper_id for paper in noise_papers], - ) - ] + return [_macro_cluster(noise_papers)] term_index = {term: index for index, term in enumerate(vocabulary)} matrix = np.zeros((len(noise_papers), len(vocabulary)), dtype=np.float32) @@ -139,44 +114,20 @@ def _merge_noise_into_macro_clusters( cluster_count = _macro_cluster_count(len(noise_papers), config) if cluster_count <= 1: - theme, summary = _label_cluster(noise_papers) - return [ - PaperCluster( - theme=f"Theme: {theme}", - summary=summary, - paper_ids=[paper.paper.paper_id for paper in noise_papers], - ) - ] + return [_macro_cluster(noise_papers)] try: from sklearn.cluster import KMeans labels = KMeans(n_clusters=cluster_count, random_state=0, n_init=10).fit_predict(matrix) except Exception: - theme, summary = _label_cluster(noise_papers) - return [ - PaperCluster( - theme=f"Theme: {theme}", - summary=summary, - paper_ids=[paper.paper.paper_id for paper in noise_papers], - ) - ] + return [_macro_cluster(noise_papers)] grouped: dict[int, list[RankedPaper]] = {} for index, label in enumerate(labels): grouped.setdefault(int(label), []).append(noise_papers[index]) - output: list[PaperCluster] = [] - for group in grouped.values(): - theme, summary = _label_cluster(group) - output.append( - PaperCluster( - theme=f"Theme: {theme}", - summary=summary, - paper_ids=[paper.paper.paper_id for paper in group], - ) - ) - return output + return [_macro_cluster(group) for group in grouped.values()] def cluster_papers( diff --git a/src/research/embedding_context.py b/src/research/embedding_context.py index 13cedc1..d007a62 100644 --- a/src/research/embedding_context.py +++ b/src/research/embedding_context.py @@ -7,23 +7,26 @@ import numpy as np +from ..embeddings.base import cosine_similarity + if TYPE_CHECKING: from ..core.context import PipelineContext from ..retrieval.models import RankedPaper, RetrievedPaper +__all__ = [ + "cosine_similarity", + "store_ranking_embedding_result", + "store_embedding_artifacts", + "get_query_embedding", + "get_paper_embeddings", + "get_paper_embedding", + "resolve_paper_embedding_matrix", +] + QUERY_EMBEDDING_KEY = "query_embedding" PAPER_EMBEDDINGS_KEY = "paper_embeddings" -def cosine_similarity(a: np.ndarray, b: np.ndarray) -> float: - """Cosine similarity between two vectors.""" - a_norm = np.linalg.norm(a) - b_norm = np.linalg.norm(b) - if a_norm == 0.0 or b_norm == 0.0: - return 0.0 - return float(np.dot(a, b) / (a_norm * b_norm)) - - def store_ranking_embedding_result( ctx: PipelineContext, query_embedding: np.ndarray | None, diff --git a/src/research/query_expansion.py b/src/research/query_expansion.py index 65c56e2..831d74a 100644 --- a/src/research/query_expansion.py +++ b/src/research/query_expansion.py @@ -9,10 +9,19 @@ from ..core.context import PipelineContext, StageResult from ..retrieval.models import ExpandedQuerySet, QueryUnderstandingResult +from .text_utils import QUERY_STOP_WORDS, extract_core_concepts if TYPE_CHECKING: from ..config.settings import QueryExpansionConfig +__all__ = [ + "extract_core_concepts", + "expand_query_heuristic", + "QueryExpansionStage", + "DOMAIN_SYNONYMS", + "ACRONYM_EXPANSIONS", +] + DOMAIN_SYNONYMS: dict[str, list[str]] = { "machine learning": ["artificial intelligence", "deep learning", "neural networks"], "ai": ["artificial intelligence", "machine learning", "automation"], @@ -44,42 +53,6 @@ "dl": "deep learning", } -_STOP_WORDS = { - "the", - "a", - "an", - "and", - "or", - "but", - "in", - "on", - "at", - "to", - "for", - "of", - "with", - "by", - "is", - "are", - "was", - "were", - "be", - "been", - "being", - "paper", - "papers", - "research", - "study", - "studies", -} - - -def extract_core_concepts(query: str) -> list[str]: - """Extract core concept terms from a query.""" - words = re.findall(r"\b\w+\b", query.lower()) - return [word for word in words if word not in _STOP_WORDS and len(word) > 3][:5] - - def _token_set(text: str) -> set[str]: return set(re.findall(r"\b\w+\b", text.lower())) @@ -158,7 +131,7 @@ def _passes_broad_term_guard( if len(key_concepts) < 2: return True - variant_tokens = _token_set(variant) - _STOP_WORDS + variant_tokens = _token_set(variant) - QUERY_STOP_WORDS if len(variant_tokens) == 1 and variant_tokens & _BROAD_SINGLE_CONCEPT_TERMS: return False diff --git a/src/research/relevance_scoring.py b/src/research/relevance_scoring.py index bb88b66..4fad3ff 100644 --- a/src/research/relevance_scoring.py +++ b/src/research/relevance_scoring.py @@ -13,41 +13,15 @@ from ..core.paper_adapters import ensure_ranked_papers from ..retrieval.models import QueryUnderstandingResult, RankedPaper, RetrievedPaper from .embedding_context import cosine_similarity, get_paper_embedding, get_query_embedding -from .query_expansion import extract_core_concepts +from .text_utils import ( + GENERIC_QUERY_TERMS, + extract_core_concepts, + term_matches_text, +) if TYPE_CHECKING: from ..config.settings import RelevanceScoringConfig -_GENERIC_CONCEPT_TERMS = frozenset( - { - "mechanism", - "mechanisms", - "method", - "methods", - "approach", - "approaches", - "application", - "applications", - "model", - "models", - "system", - "systems", - "based", - "using", - "recent", - } -) - - -def _term_matches_text(term: str, text: str) -> bool: - if term in text: - return True - if term.endswith("s") and term[:-1] in text: - return True - if f"{term}s" in text: - return True - return False - def _core_concepts(query: str, ctx: PipelineContext) -> list[str]: understanding = ctx.get_artifact("query_understanding") @@ -59,7 +33,7 @@ def _core_concepts(query: str, ctx: PipelineContext) -> list[str]: return [ concept for concept in concepts - if concept.lower() not in _GENERIC_CONCEPT_TERMS + if concept.lower() not in GENERIC_QUERY_TERMS ] @@ -77,10 +51,10 @@ def _concept_groups_match( if mode != "any_group": text = " ".join(part for part in (title, abstract) if part) - return all(_term_matches_text(concept, text) for concept in concepts) + return all(term_matches_text(concept, text) for concept in concepts) return all( - _term_matches_text(concept, title) or _term_matches_text(concept, abstract) + term_matches_text(concept, title) or term_matches_text(concept, abstract) for concept in concepts ) diff --git a/src/research/text_utils.py b/src/research/text_utils.py new file mode 100644 index 0000000..e839daf --- /dev/null +++ b/src/research/text_utils.py @@ -0,0 +1,101 @@ +# -*- coding: utf-8 -*- +"""Shared text heuristics used across ranking, filtering, and clustering.""" + +from __future__ import annotations + +import re + +QUERY_STOP_WORDS = frozenset( + { + "the", + "a", + "an", + "and", + "or", + "but", + "in", + "on", + "at", + "to", + "for", + "of", + "with", + "by", + "is", + "are", + "was", + "were", + "be", + "been", + "being", + "paper", + "papers", + "research", + "study", + "studies", + } +) + +LABEL_STOP_WORDS = frozenset( + { + "the", + "a", + "an", + "and", + "or", + "but", + "in", + "on", + "at", + "to", + "for", + "of", + "with", + "by", + "using", + "based", + "via", + "from", + } +) + +GENERIC_QUERY_TERMS = frozenset( + { + "mechanism", + "mechanisms", + "method", + "methods", + "approach", + "approaches", + "application", + "applications", + "model", + "models", + "system", + "systems", + "based", + "using", + "recent", + } +) + + +def extract_query_terms(query: str) -> set[str]: + """Return meaningful lowercase terms from a query.""" + words = re.findall(r"\b\w+\b", query.lower()) + return {word for word in words if word not in QUERY_STOP_WORDS and len(word) > 2} + + +def extract_core_concepts(query: str) -> list[str]: + """Extract core concept terms from a query.""" + words = re.findall(r"\b\w+\b", query.lower()) + return [word for word in words if word not in QUERY_STOP_WORDS and len(word) > 3][:5] + + +def term_matches_text(term: str, text: str) -> bool: + """Return True when a term (or its simple plural/singular form) occurs in text.""" + if term in text: + return True + if term.endswith("s") and term[:-1] in text: + return True + return f"{term}s" in text diff --git a/src/retrieval/__init__.py b/src/retrieval/__init__.py index dc4f8c8..a7948a4 100644 --- a/src/retrieval/__init__.py +++ b/src/retrieval/__init__.py @@ -13,7 +13,6 @@ SynthesisResult, ) from .helpers import _normalize_title, _dedupe, _openalex_abstract_from_inverted_index -from .openalex import search_openalex from .providers import ( ArxivProvider, CrossRefProvider, @@ -30,5 +29,4 @@ search_all_enabled, search_enabled_providers, ) -from .semanticscholar import search_semantic_scholar from .rendering import render_markdown diff --git a/src/retrieval/deduplication.py b/src/retrieval/deduplication.py index 55142a2..03b7c2c 100644 --- a/src/retrieval/deduplication.py +++ b/src/retrieval/deduplication.py @@ -93,11 +93,6 @@ def dedupe_by_metadata(papers: list[RetrievedPaper]) -> list[RetrievedPaper]: return output -def _paper_preference_key(paper: RetrievedPaper) -> tuple[int, int, int, int, int]: - """Rank duplicates; higher values are preferred to keep.""" - return metadata_quality_key(paper) - - def _paper_dedup_text(paper: RetrievedPaper) -> str: parts = [paper.title] if paper.abstract: @@ -115,19 +110,25 @@ def dedupe_by_embedding( return papers, 0 texts = [_paper_dedup_text(paper) for paper in papers] - embeddings = embedder.embed_texts(texts) + embeddings = np.asarray(embedder.embed_texts(texts), dtype=np.float32) + + # One matrix product yields all pairwise cosine similarities. + norms = np.linalg.norm(embeddings, axis=1, keepdims=True) + norms[norms == 0.0] = 1.0 + normalized = embeddings / norms + similarity_matrix = normalized @ normalized.T + removed = 0 keep_indices: list[int] = [] for index, paper in enumerate(papers): duplicate = False - for kept_index in keep_indices: - similarity = embedder.similarity(embeddings[index], embeddings[kept_index]) - if similarity >= threshold: + for position, kept_index in enumerate(keep_indices): + if similarity_matrix[index, kept_index] >= threshold: duplicate = True removed += 1 - if _paper_preference_key(paper) > _paper_preference_key(papers[kept_index]): - keep_indices[keep_indices.index(kept_index)] = index + if metadata_quality_key(paper) > metadata_quality_key(papers[kept_index]): + keep_indices[position] = index break if not duplicate: keep_indices.append(index) diff --git a/src/retrieval/openalex.py b/src/retrieval/openalex.py deleted file mode 100644 index 4402212..0000000 --- a/src/retrieval/openalex.py +++ /dev/null @@ -1,28 +0,0 @@ -# -*- coding: utf-8 -*- -"""OpenAlex API client for the retrieval module (deprecated shim).""" - -from __future__ import annotations - -import warnings - -import aiohttp - -from .models import RetrievedPaper -from .providers.openalex import OpenAlexProvider - -_provider = OpenAlexProvider() - - -async def search_openalex( - session: aiohttp.ClientSession, - query: str, - per_page: int = 8, -) -> list[RetrievedPaper]: - """Search OpenAlex for papers matching the query.""" - warnings.warn( - "search_openalex is deprecated; use OpenAlexProvider from " - "src.retrieval.providers instead.", - DeprecationWarning, - stacklevel=2, - ) - return await _provider.search(session, query, limit=per_page) diff --git a/src/retrieval/semanticscholar.py b/src/retrieval/semanticscholar.py deleted file mode 100644 index f736af1..0000000 --- a/src/retrieval/semanticscholar.py +++ /dev/null @@ -1,28 +0,0 @@ -# -*- coding: utf-8 -*- -"""Semantic Scholar API client for the retrieval module (deprecated shim).""" - -from __future__ import annotations - -import warnings - -import aiohttp - -from .models import RetrievedPaper -from .providers.semantic_scholar import SemanticScholarProvider - -_provider = SemanticScholarProvider() - - -async def search_semantic_scholar( - session: aiohttp.ClientSession, - query: str, - limit: int = 8, -) -> list[RetrievedPaper]: - """Search Semantic Scholar for papers matching the query.""" - warnings.warn( - "search_semantic_scholar is deprecated; use SemanticScholarProvider from " - "src.retrieval.providers instead.", - DeprecationWarning, - stacklevel=2, - ) - return await _provider.search(session, query, limit=limit) From f52fdfb1992e5aa58b97e9bfbe7f4d2b4aa0c742 Mon Sep 17 00:00:00 2001 From: Jean Paul Elisa NIYOKWIZERWA <140616733+Ndevu12@users.noreply.github.com> Date: Tue, 11 Aug 2026 08:29:21 +0000 Subject: [PATCH 3/4] refactor: log provider retries, build analysis agent lazily, drop unused JSON stack - Retrieval providers report retry and rate-limit events through the logging system instead of printing to stdout from the data layer - The shared analysis agent is now created on first use via get_analysis_agent(); importing src.analysis no longer resolves settings or constructs a model as a side effect (analysis_agent remains importable through module __getattr__) - Remove src/utils/json_processing.py: production code exclusively uses retrieval/helpers_modules/json_extraction, so the parallel processor and its test class were dead weight - Fix undefined type names, an unused-variable pair, and a placeholder-less f-string surfaced by the new lint gate --- docs/_analysis/test-behavior-index.md | 2 +- src/analysis/__init__.py | 12 +- src/analysis/llm.py | 28 +- src/export/_common.py | 1 - .../helpers_modules/json_extraction.py | 8 +- src/retrieval/providers/arxiv.py | 5 +- src/retrieval/providers/crossref.py | 7 +- src/retrieval/providers/openalex.py | 3 +- src/retrieval/providers/semantic_scholar.py | 7 +- src/utils/content_quality.py | 7 +- src/utils/enhanced_response_handler.py | 2 +- src/utils/json_processing.py | 537 ------------------ tests/test_graceful_response_handling.py | 88 --- 13 files changed, 59 insertions(+), 648 deletions(-) delete mode 100644 src/utils/json_processing.py diff --git a/docs/_analysis/test-behavior-index.md b/docs/_analysis/test-behavior-index.md index 55d1738..da41008 100644 --- a/docs/_analysis/test-behavior-index.md +++ b/docs/_analysis/test-behavior-index.md @@ -309,7 +309,7 @@ Source: all 28 files matching `tests/test_*.py`. Internal reference for `docs/de | | | |---|---| -| **Modules** | `src.utils.{response_models,retry_manager,quality_monitor,enhanced_validation,content_quality,json_processing,model_adaptation,fallback_processing}` | +| **Modules** | `src.utils.{response_models,retry_manager,quality_monitor,enhanced_validation,content_quality,model_adaptation,fallback_processing}` | | Class | Behavior | |-------|----------| diff --git a/src/analysis/__init__.py b/src/analysis/__init__.py index 2925854..c767f21 100644 --- a/src/analysis/__init__.py +++ b/src/analysis/__init__.py @@ -1,6 +1,8 @@ # AI Research Assistant - Analysis Module +from typing import Any + from .gap_analysis import GapAnalysisStage, analyze_gaps -from .llm import analysis_agent +from .llm import get_analysis_agent from .synthesis import SynthesisStage, extract_papers, run_synthesis, synthesize_collective from ..retrieval.models import ( GapAnalysisResult, @@ -21,6 +23,14 @@ "analysis_agent", "analyze_gaps", "extract_papers", + "get_analysis_agent", "run_synthesis", "synthesize_collective", ] + + +def __getattr__(name: str) -> Any: + # "analysis_agent" stays importable but is now built lazily on first access. + if name == "analysis_agent": + return get_analysis_agent() + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/src/analysis/llm.py b/src/analysis/llm.py index 5d350d6..4d94947 100644 --- a/src/analysis/llm.py +++ b/src/analysis/llm.py @@ -1,7 +1,31 @@ # -*- coding: utf-8 -*- """Analysis agent configuration for the analysis module.""" +from __future__ import annotations + +from typing import TYPE_CHECKING, Any + from ..models import AgentFactory, AgentRole -_factory = AgentFactory() -analysis_agent = _factory.create_agent(AgentRole.ANALYSIS) +if TYPE_CHECKING: + from pydantic_ai import Agent + +_cached_agent: Agent | None = None + + +def get_analysis_agent() -> Agent: + """Return the shared analysis agent, creating it on first use. + + Lazily constructed so importing the analysis package has no side effects + (no settings resolution, no model creation). + """ + global _cached_agent + if _cached_agent is None: + _cached_agent = AgentFactory().create_agent(AgentRole.ANALYSIS) + return _cached_agent + + +def __getattr__(name: str) -> Any: + if name == "analysis_agent": + return get_analysis_agent() + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/src/export/_common.py b/src/export/_common.py index b4f8560..023f19d 100644 --- a/src/export/_common.py +++ b/src/export/_common.py @@ -42,7 +42,6 @@ def format_author_list_mla(authors: list[str]) -> str: return f"{parts[-1]}, {' '.join(parts[:-1])}." return f"{authors[0]}." first = format_author_list_mla([authors[0]]).rstrip(".") - others = ", ".join(authors[1:]) return f"{first}, et al." diff --git a/src/retrieval/helpers_modules/json_extraction.py b/src/retrieval/helpers_modules/json_extraction.py index d962c9f..8084101 100644 --- a/src/retrieval/helpers_modules/json_extraction.py +++ b/src/retrieval/helpers_modules/json_extraction.py @@ -141,13 +141,7 @@ def _remove_common_prefixes_suffixes(text: str) -> str: r'^.*?(?=\{)', # Everything before the first opening brace r'^[^{]*', # Non-brace characters at the start ] - - # Common suffixes to remove - suffixes = [ - r'\}.*?$', # Everything after the last closing brace (keep the brace) - r'[^}]*$', # Non-brace characters at the end - ] - + cleaned = text # Apply prefix removal diff --git a/src/retrieval/providers/arxiv.py b/src/retrieval/providers/arxiv.py index 38a06a3..cbd876a 100644 --- a/src/retrieval/providers/arxiv.py +++ b/src/retrieval/providers/arxiv.py @@ -10,6 +10,7 @@ import aiohttp from ..models import RetrievedPaper +from ...utils.logging_system import logger from ...utils.message_formatter import MessageFormatter from .base import RetrievalProvider @@ -59,10 +60,10 @@ async def search( except (aiohttp.ClientError, asyncio.TimeoutError) as exc: if attempt == max_retries - 1: raise - print(MessageFormatter.api_retry_message("arXiv", attempt + 1, str(exc))) + logger.warning(MessageFormatter.api_retry_message("arXiv", attempt + 1, str(exc))) await asyncio.sleep(2 ** attempt) else: - print(MessageFormatter.api_max_retries_message("arXiv")) + logger.warning(MessageFormatter.api_max_retries_message("arXiv")) return [] root = ET.fromstring(body) diff --git a/src/retrieval/providers/crossref.py b/src/retrieval/providers/crossref.py index 8f7ace2..d86da47 100644 --- a/src/retrieval/providers/crossref.py +++ b/src/retrieval/providers/crossref.py @@ -10,6 +10,7 @@ import aiohttp from ..models import RetrievedPaper +from ...utils.logging_system import logger from ...utils.message_formatter import MessageFormatter from .base import RetrievalProvider @@ -46,7 +47,7 @@ async def search( ) as response: if response.status == 429: retry_after = response.headers.get("Retry-After", "60") - print( + logger.warning( MessageFormatter.api_rate_limit_message( "CrossRef", retry_after, @@ -60,10 +61,10 @@ async def search( except (aiohttp.ClientError, asyncio.TimeoutError) as exc: if attempt == max_retries - 1: raise - print(MessageFormatter.api_retry_message("CrossRef", attempt + 1, str(exc))) + logger.warning(MessageFormatter.api_retry_message("CrossRef", attempt + 1, str(exc))) await asyncio.sleep(2 ** attempt) else: - print(MessageFormatter.api_max_retries_message("CrossRef")) + logger.warning(MessageFormatter.api_max_retries_message("CrossRef")) return [] message = data.get("message") or {} diff --git a/src/retrieval/providers/openalex.py b/src/retrieval/providers/openalex.py index 5b6c1d1..5636945 100644 --- a/src/retrieval/providers/openalex.py +++ b/src/retrieval/providers/openalex.py @@ -9,6 +9,7 @@ from ..helpers import _openalex_abstract_from_inverted_index from ..models import RetrievedPaper +from ...utils.logging_system import logger from ...utils.message_formatter import MessageFormatter from .base import RetrievalProvider @@ -43,7 +44,7 @@ async def search( except (aiohttp.ClientError, asyncio.TimeoutError) as exc: if attempt == max_retries - 1: raise - print(MessageFormatter.api_retry_message("OpenAlex", attempt + 1, str(exc))) + logger.warning(MessageFormatter.api_retry_message("OpenAlex", attempt + 1, str(exc))) await asyncio.sleep(2 ** attempt) return [self.normalize(item) for item in data.get("results", [])][:per_page] diff --git a/src/retrieval/providers/semantic_scholar.py b/src/retrieval/providers/semantic_scholar.py index 1637abe..79e77ce 100644 --- a/src/retrieval/providers/semantic_scholar.py +++ b/src/retrieval/providers/semantic_scholar.py @@ -9,6 +9,7 @@ import aiohttp from ..models import RetrievedPaper +from ...utils.logging_system import logger from ...utils.message_formatter import MessageFormatter from .base import RetrievalProvider @@ -51,7 +52,7 @@ async def search( ) as response: if response.status == 429: retry_after = response.headers.get("Retry-After", "60") - print( + logger.warning( MessageFormatter.api_rate_limit_message( "Semantic Scholar", retry_after, @@ -65,7 +66,7 @@ async def search( except (aiohttp.ClientError, asyncio.TimeoutError) as exc: if attempt == max_retries - 1: raise - print( + logger.warning( MessageFormatter.api_retry_message( "Semantic Scholar", attempt + 1, @@ -74,7 +75,7 @@ async def search( ) await asyncio.sleep(2 ** attempt) else: - print(MessageFormatter.api_max_retries_message("Semantic Scholar")) + logger.warning(MessageFormatter.api_max_retries_message("Semantic Scholar")) return [] return [self.normalize(item) for item in data.get("data", []) or []][:resolved_limit] diff --git a/src/utils/content_quality.py b/src/utils/content_quality.py index fcf7eae..76d18fe 100644 --- a/src/utils/content_quality.py +++ b/src/utils/content_quality.py @@ -7,9 +7,14 @@ """ import re -from typing import List, Dict, Optional, Set +from typing import List, Dict, Optional, Set, TYPE_CHECKING from collections import Counter +if TYPE_CHECKING: + from pydantic_ai import Agent + + from ..retrieval.models import PaperAnalysis, ResearchReport + from .response_models import ( ContentQualityConfig, EnhancementConfig, ContentQualityResult, ContentIssue, ContentIssueType, IssueSeverity, EnhancementResult, diff --git a/src/utils/enhanced_response_handler.py b/src/utils/enhanced_response_handler.py index e5fdf14..bd85613 100644 --- a/src/utils/enhanced_response_handler.py +++ b/src/utils/enhanced_response_handler.py @@ -311,7 +311,7 @@ async def process_response_with_retries( else: # No more retries - attempt recovery self.logger.warning( - f"Maximum retries reached, attempting recovery", + "Maximum retries reached, attempting recovery", extra={ 'error_type': enhanced_validation.error_type, 'total_attempts': attempt + 1, diff --git a/src/utils/json_processing.py b/src/utils/json_processing.py deleted file mode 100644 index 66165d5..0000000 --- a/src/utils/json_processing.py +++ /dev/null @@ -1,537 +0,0 @@ -# -*- coding: utf-8 -*- -"""Enhanced JSON processing with improved extraction and parsing. - -This module provides enhanced JSON extraction and parsing capabilities, -including support for incremental parsing, better error reporting, and -recovery from common JSON formatting issues. -""" - -import json -import re -from typing import Dict, List, Optional, Tuple, Any -from dataclasses import dataclass - -from .logging_system import logger - - -@dataclass -class JSONParsingError: - """Detailed JSON parsing error information.""" - error_type: str - message: str - line_number: Optional[int] = None - column_number: Optional[int] = None - context: Optional[str] = None - suggestion: Optional[str] = None - - -@dataclass -class JSONExtractionResult: - """Result of JSON extraction from text.""" - success: bool - json_string: Optional[str] = None - error: Optional[JSONParsingError] = None - extraction_method: Optional[str] = None - confidence_score: float = 0.0 - - -class EnhancedJSONProcessor: - """Enhanced JSON extraction and parsing processor.""" - - def __init__(self): - """Initialize JSON processor.""" - self.logger = logger - - def extract_json(self, text: str) -> JSONExtractionResult: - """Extract JSON from text with multiple strategies. - - Args: - text: Text potentially containing JSON - - Returns: - JSONExtractionResult: Extracted JSON or error details - """ - # Try different extraction methods in order of preference - methods = [ - ("direct_parse", self._try_direct_parse), - ("bracket_matching", self._extract_by_bracket_matching), - ("pattern_matching", self._extract_by_pattern_matching), - ("incremental_parse", self._try_incremental_parse), - ] - - for method_name, method_func in methods: - try: - result = method_func(text) - if result.success: - result.extraction_method = method_name - self.logger.debug( - f"Successfully extracted JSON using {method_name}", - extra={'method': method_name, 'confidence': result.confidence_score} - ) - return result - except Exception as e: - self.logger.debug( - f"JSON extraction method {method_name} failed: {str(e)}", - extra={'method': method_name, 'error': str(e)} - ) - continue - - # If all methods fail, return error - return JSONExtractionResult( - success=False, - error=JSONParsingError( - error_type="extraction_failed", - message="Could not extract valid JSON from text using any method", - suggestion="Ensure the response contains valid JSON format" - ), - confidence_score=0.0 - ) - - def _try_direct_parse(self, text: str) -> JSONExtractionResult: - """Try to parse text directly as JSON. - - Args: - text: Text to parse - - Returns: - JSONExtractionResult: Parse result - """ - text = text.strip() - try: - parsed = json.loads(text) - return JSONExtractionResult( - success=True, - json_string=text, - confidence_score=1.0 - ) - except json.JSONDecodeError as e: - return JSONExtractionResult( - success=False, - error=JSONParsingError( - error_type="json_syntax", - message=str(e), - line_number=e.lineno, - column_number=e.colno - ), - confidence_score=0.0 - ) - - def _extract_by_bracket_matching(self, text: str) -> JSONExtractionResult: - """Extract JSON by matching brackets. - - Args: - text: Text to search - - Returns: - JSONExtractionResult: Extraction result - """ - # Find first { or [ - start_idx = -1 - start_char = None - - for i, char in enumerate(text): - if char == '{': - start_idx = i - start_char = '{' - break - elif char == '[': - start_idx = i - start_char = '[' - break - - if start_idx == -1: - return JSONExtractionResult( - success=False, - error=JSONParsingError( - error_type="no_json_found", - message="No JSON structure found in text" - ), - confidence_score=0.0 - ) - - # Find matching closing bracket - end_char = '}' if start_char == '{' else ']' - bracket_count = 0 - end_idx = -1 - in_string = False - escape_next = False - - for i in range(start_idx, len(text)): - char = text[i] - - # Handle string escaping - if escape_next: - escape_next = False - continue - - if char == '\\': - escape_next = True - continue - - # Track string state - if char == '"' and not escape_next: - in_string = not in_string - continue - - # Count brackets only outside strings - if not in_string: - if char == start_char: - bracket_count += 1 - elif char == end_char: - bracket_count -= 1 - if bracket_count == 0: - end_idx = i + 1 - break - - if end_idx == -1: - return JSONExtractionResult( - success=False, - error=JSONParsingError( - error_type="unmatched_brackets", - message="Could not find matching closing bracket" - ), - confidence_score=0.0 - ) - - # Extract and try to parse - json_string = text[start_idx:end_idx] - try: - parsed = json.loads(json_string) - return JSONExtractionResult( - success=True, - json_string=json_string, - confidence_score=0.9 - ) - except json.JSONDecodeError as e: - return JSONExtractionResult( - success=False, - error=JSONParsingError( - error_type="json_syntax", - message=str(e), - line_number=e.lineno, - column_number=e.colno - ), - confidence_score=0.0 - ) - - def _extract_by_pattern_matching(self, text: str) -> JSONExtractionResult: - """Extract JSON using regex patterns. - - Args: - text: Text to search - - Returns: - JSONExtractionResult: Extraction result - """ - # Try to find JSON-like patterns - patterns = [ - r'\{[^{}]*(?:\{[^{}]*\}[^{}]*)*\}', # Nested objects - r'\[[^\[\]]*(?:\[[^\[\]]*\][^\[\]]*)*\]', # Nested arrays - r'\{.*\}', # Simple object - r'\[.*\]', # Simple array - ] - - for pattern in patterns: - matches = re.finditer(pattern, text, re.DOTALL) - for match in matches: - json_string = match.group(0) - try: - parsed = json.loads(json_string) - return JSONExtractionResult( - success=True, - json_string=json_string, - confidence_score=0.7 - ) - except json.JSONDecodeError: - continue - - return JSONExtractionResult( - success=False, - error=JSONParsingError( - error_type="pattern_match_failed", - message="No valid JSON patterns found" - ), - confidence_score=0.0 - ) - - def _try_incremental_parse(self, text: str) -> JSONExtractionResult: - """Try incremental parsing for large responses. - - Args: - text: Text to parse - - Returns: - JSONExtractionResult: Extraction result - """ - # Try to find and parse multiple JSON objects - json_objects = [] - current_obj = "" - bracket_count = 0 - in_string = False - escape_next = False - - for char in text: - if escape_next: - escape_next = False - current_obj += char - continue - - if char == '\\': - escape_next = True - current_obj += char - continue - - if char == '"': - in_string = not in_string - current_obj += char - continue - - if not in_string: - if char in '{[': - bracket_count += 1 - elif char in '}]': - bracket_count -= 1 - - current_obj += char - - # Try to parse when bracket count returns to 0 - if bracket_count == 0 and current_obj.strip(): - try: - parsed = json.loads(current_obj) - json_objects.append(current_obj) - current_obj = "" - except json.JSONDecodeError: - pass - - if json_objects: - # Return the first valid JSON object - return JSONExtractionResult( - success=True, - json_string=json_objects[0], - confidence_score=0.8 - ) - - return JSONExtractionResult( - success=False, - error=JSONParsingError( - error_type="incremental_parse_failed", - message="Could not parse any complete JSON objects" - ), - confidence_score=0.0 - ) - - def parse_json(self, json_string: str) -> Tuple[bool, Optional[Dict[str, Any]], Optional[JSONParsingError]]: - """Parse JSON string with detailed error reporting. - - Args: - json_string: JSON string to parse - - Returns: - Tuple[bool, Optional[Dict], Optional[JSONParsingError]]: Success, parsed data, error - """ - try: - parsed = json.loads(json_string) - return True, parsed, None - except json.JSONDecodeError as e: - # Get context around error - lines = json_string.split('\n') - context_lines = [] - if e.lineno and e.lineno > 0: - start = max(0, e.lineno - 2) - end = min(len(lines), e.lineno + 1) - context_lines = lines[start:end] - - context = '\n'.join(context_lines) if context_lines else None - - # Generate suggestion - suggestion = self._suggest_fix(json_string, e) - - error = JSONParsingError( - error_type="json_syntax", - message=str(e), - line_number=e.lineno, - column_number=e.colno, - context=context, - suggestion=suggestion - ) - - return False, None, error - - def _suggest_fix(self, json_string: str, error: json.JSONDecodeError) -> Optional[str]: - """Suggest a fix for JSON parsing error. - - Args: - json_string: The JSON string that failed - error: The JSONDecodeError - - Returns: - Optional[str]: Suggested fix or None - """ - error_msg = str(error).lower() - - if "expecting property name" in error_msg: - return "Check for missing quotes around property names" - elif "expecting value" in error_msg: - return "Check for missing or invalid values" - elif "trailing comma" in error_msg or "extra data" in error_msg: - return "Remove trailing commas before closing brackets" - elif "unterminated string" in error_msg: - return "Check for unclosed string literals" - elif "invalid escape" in error_msg: - return "Check for invalid escape sequences in strings" - - return None - - def pretty_print_json(self, json_string: str, indent: int = 2) -> Optional[str]: - """Pretty print JSON string. - - Args: - json_string: JSON string to format - indent: Indentation level - - Returns: - Optional[str]: Formatted JSON or None if invalid - """ - try: - parsed = json.loads(json_string) - return json.dumps(parsed, indent=indent, ensure_ascii=False) - except json.JSONDecodeError: - return None - - def validate_json_structure(self, json_string: str, schema: Optional[Dict[str, Any]] = None) -> Tuple[bool, List[str]]: - """Validate JSON structure against optional schema. - - Args: - json_string: JSON string to validate - schema: Optional schema to validate against - - Returns: - Tuple[bool, List[str]]: Valid status and list of errors - """ - try: - parsed = json.loads(json_string) - except json.JSONDecodeError as e: - return False, [f"Invalid JSON: {str(e)}"] - - errors = [] - - # Basic structure validation - if not isinstance(parsed, (dict, list)): - errors.append("JSON must be an object or array at root level") - - # Schema validation if provided - if schema: - errors.extend(self._validate_against_schema(parsed, schema)) - - return len(errors) == 0, errors - - def _validate_against_schema(self, data: Any, schema: Dict[str, Any]) -> List[str]: - """Validate data against schema. - - Args: - data: Data to validate - schema: Schema to validate against - - Returns: - List[str]: List of validation errors - """ - errors = [] - - # Check required fields - if "required" in schema: - if isinstance(data, dict): - for field in schema["required"]: - if field not in data: - errors.append(f"Missing required field: {field}") - - # Check field types - if "properties" in schema and isinstance(data, dict): - for field, field_schema in schema["properties"].items(): - if field in data: - expected_type = field_schema.get("type") - if expected_type and not self._check_type(data[field], expected_type): - errors.append(f"Field '{field}' has wrong type: expected {expected_type}") - - return errors - - def _check_type(self, value: Any, expected_type: str) -> bool: - """Check if value matches expected type. - - Args: - value: Value to check - expected_type: Expected type name - - Returns: - bool: True if type matches - """ - type_map = { - "string": str, - "number": (int, float), - "integer": int, - "boolean": bool, - "array": list, - "object": dict, - "null": type(None), - } - - expected = type_map.get(expected_type) - if expected is None: - return True # Unknown type, assume valid - - return isinstance(value, expected) - - -# Global JSON processor instance -_json_processor: Optional[EnhancedJSONProcessor] = None - - -def get_json_processor() -> EnhancedJSONProcessor: - """Get the global JSON processor. - - Returns: - EnhancedJSONProcessor: Global processor instance - """ - global _json_processor - if _json_processor is None: - _json_processor = EnhancedJSONProcessor() - return _json_processor - - -def extract_json(text: str) -> JSONExtractionResult: - """Extract JSON from text. - - Args: - text: Text to extract JSON from - - Returns: - JSONExtractionResult: Extraction result - """ - processor = get_json_processor() - return processor.extract_json(text) - - -def parse_json(json_string: str) -> Tuple[bool, Optional[Dict[str, Any]], Optional[JSONParsingError]]: - """Parse JSON string. - - Args: - json_string: JSON string to parse - - Returns: - Tuple[bool, Optional[Dict], Optional[JSONParsingError]]: Parse result - """ - processor = get_json_processor() - return processor.parse_json(json_string) - - -def pretty_print_json(json_string: str, indent: int = 2) -> Optional[str]: - """Pretty print JSON. - - Args: - json_string: JSON string to format - indent: Indentation level - - Returns: - Optional[str]: Formatted JSON - """ - processor = get_json_processor() - return processor.pretty_print_json(json_string, indent) \ No newline at end of file diff --git a/tests/test_graceful_response_handling.py b/tests/test_graceful_response_handling.py index 2d614ca..3670022 100644 --- a/tests/test_graceful_response_handling.py +++ b/tests/test_graceful_response_handling.py @@ -360,94 +360,6 @@ def test_relevance_scoring(self): assert all(0.0 <= score <= 1.0 for score in paper_scores) -class TestJSONProcessing: - """Test enhanced JSON processing functionality.""" - - def test_json_processor_initialization(self): - """Test JSON processor initializes correctly.""" - from src.utils.json_processing import EnhancedJSONProcessor - - processor = EnhancedJSONProcessor() - assert processor is not None - assert processor.logger is not None - - def test_direct_json_extraction(self): - """Test direct JSON extraction.""" - from src.utils.json_processing import extract_json - - json_text = '{"query": "test", "papers": []}' - result = extract_json(json_text) - - assert result.success == True - assert result.json_string == json_text - assert result.confidence_score == 1.0 - - def test_json_extraction_with_surrounding_text(self): - """Test JSON extraction from text with surrounding content.""" - from src.utils.json_processing import extract_json - - text = ''' - Here is the analysis: - { - "query": "machine learning", - "papers": [ - {"title": "Paper 1"} - ] - } - End of analysis. - ''' - - result = extract_json(text) - - assert result.success == True - assert '"query"' in result.json_string - assert result.confidence_score > 0.0 - - def test_json_parsing_with_error_details(self): - """Test JSON parsing with detailed error information.""" - from src.utils.json_processing import parse_json - - invalid_json = '{"query": "test", "papers": [}' - success, parsed, error = parse_json(invalid_json) - - assert success == False - assert parsed is None - assert error is not None - assert error.error_type == "json_syntax" - assert error.line_number is not None - - def test_json_pretty_printing(self): - """Test JSON pretty printing.""" - from src.utils.json_processing import pretty_print_json - - compact_json = '{"query":"test","papers":[]}' - pretty = pretty_print_json(compact_json) - - assert pretty is not None - assert '\n' in pretty - assert '"query"' in pretty - - def test_json_structure_validation(self): - """Test JSON structure validation.""" - from src.utils.json_processing import EnhancedJSONProcessor - - processor = EnhancedJSONProcessor() - - # Valid JSON - valid_json = '{"query": "test", "papers": []}' - is_valid, errors = processor.validate_json_structure(valid_json) - - assert is_valid == True - assert len(errors) == 0 - - # Invalid JSON - invalid_json = '{"query": "test", "papers": [}' - is_valid, errors = processor.validate_json_structure(invalid_json) - - assert is_valid == False - assert len(errors) > 0 - - # Integration test class TestBasicIntegration: """Test basic integration between components.""" From daf5be57b3d00b238dca206efabffcf2d4420413 Mon Sep 17 00:00:00 2001 From: Jean Paul Elisa NIYOKWIZERWA <140616733+Ndevu12@users.noreply.github.com> Date: Tue, 11 Aug 2026 08:29:22 +0000 Subject: [PATCH 4/4] ci: auto-discover tests and add lint gate; make suite hermetic; streamline README - CI now runs pytest against the whole tests/ directory, so a new test file can never be silently excluded by a stale hardcoded matrix - Add a ruff lint job (syntax errors, undefined names, unused code) with configuration in pyproject.toml - Add tests/conftest.py stubbing the Ollama setup path so the suite is fast and network-independent, and RA_SKIP_SETUP_CHECK so subprocess tests and CI containers can skip environment setup explicitly - Rewrite the README at a standard level: condensed configuration reference, single architecture diagram, troubleshooting trimmed to essentials, license section added --- .github/workflows/ci.yml | 78 ++---- README.md | 461 ++++++-------------------------- pyproject.toml | 14 + src/__main__.py | 4 + tests/conftest.py | 21 ++ tests/test_complete_workflow.py | 6 +- 6 files changed, 148 insertions(+), 436 deletions(-) create mode 100644 tests/conftest.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 23ebc05..2880ebe 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -17,59 +17,30 @@ env: PIPENV_DONT_LOAD_ENV: "1" jobs: - test: - name: Tests (${{ matrix.scope }}) + lint: + name: Lint runs-on: ubuntu-latest - timeout-minutes: 15 - - strategy: - fail-fast: false - matrix: - include: - - scope: core - paths: >- - tests/test_config_settings.py - tests/test_pipeline_core.py - tests/test_models.py - tests/test_paper_adapters.py - tests/test_memory.py - tests/test_progress_reporter.py - tests/test_message_formatting.py + timeout-minutes: 5 - - scope: retrieval - paths: >- - tests/test_retrieval_stage.py - tests/test_export.py - tests/test_reporting.py - tests/test_embeddings.py - tests/test_providers.py + steps: + - name: Checkout + uses: actions/checkout@v4 - - scope: research - paths: >- - tests/test_research_stages.py - tests/test_research_quality.py - tests/test_synthesis.py - tests/test_resolve_llm_features.py + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: "3.13" - - scope: cli - paths: >- - tests/test_interactive_mode.py - tests/test_input_handler.py - tests/test_main_mode_detection.py - tests/test_signal_handling.py - tests/test_interactive_filters.py - tests/test_complete_workflow.py + - name: Install ruff + run: pip install ruff - - scope: llm - paths: >- - tests/test_llm_providers.py - tests/test_model_selection.py - tests/test_graceful_response_handling.py - tests/test_json_parsing_bug_exploration.py - tests/test_json_parsing_preservation.py + - name: Ruff check + run: ruff check src tests setups - - scope: api - paths: tests/test_phase3_extensibility.py + test: + name: Tests + runs-on: ubuntu-latest + timeout-minutes: 20 steps: - name: Checkout @@ -78,20 +49,21 @@ jobs: - name: Setup Pipenv uses: ./.github/actions/setup-pipenv + # Auto-discovers every test file so new tests can never be silently skipped. - name: Run tests - run: pipenv run pytest -m "not slow" --tb=short -q ${{ matrix.paths }} + run: pipenv run pytest tests/ -m "not slow" --tb=short -q ci-complete: name: CI complete runs-on: ubuntu-latest - needs: test + needs: [lint, test] if: always() steps: - - name: Verify all scopes passed + - name: Verify all jobs passed run: | - if [ "${{ needs.test.result }}" != "success" ]; then - echo "One or more test scopes failed." + if [ "${{ needs.lint.result }}" != "success" ] || [ "${{ needs.test.result }}" != "success" ]; then + echo "One or more CI jobs failed." exit 1 fi - echo "All test scopes passed." + echo "All CI jobs passed." diff --git a/README.md b/README.md index 95b8991..29fa1ee 100644 --- a/README.md +++ b/README.md @@ -4,36 +4,30 @@ A local-first research pipeline that retrieves academic papers from multiple sch Built with Python 3.13, pydantic-ai, sentence-transformers, and async I/O. -**Documentation:** [https://ndevu12.github.io/Research_Assistant_Model/](https://ndevu12.github.io/Research_Assistant_Model/) — architecture, configuration, API, operations, and known issues. +**Documentation:** [https://ndevu12.github.io/Research_Assistant_Model/](https://ndevu12.github.io/Research_Assistant_Model/) — architecture, configuration, API reference, and operations. ## Features -- **Multi-stage pipeline** — query understanding → expansion → retrieval → deduplication → ranking → clustering → synthesis → gap analysis → citation export → report generation -- **Local-first LLM** — Ollama with resource-aware model auto-selection (`llama3.1:8b` or `llama3.2:3b`) -- **Cloud LLM support** — OpenAI and Anthropic via `src/models/` provider abstraction -- **Multi-source retrieval** — OpenAlex, Semantic Scholar (arXiv, CrossRef, and others configurable) -- **Embedding-backed stages** — sentence-transformers (`bge-small-en-v1.5`) for dedup, ranking, and clustering -- **Structured output** — Pydantic models throughout; JSON, Markdown, HTML, and print-ready PDF (HTML) -- **Citation export** — BibTeX, APA, MLA, Chicago -- **Session memory** — optional SQLite-backed interactive sessions -- **Auto-setup** — installs dependencies, Ollama, and pulls the configured local model on first run +- **Multi-stage pipeline** — query understanding → expansion → retrieval → deduplication → ranking → relevance filtering → clustering → synthesis → gap analysis → citation export → report generation +- **Local-first LLM** — Ollama with resource-aware model auto-selection; OpenAI and Anthropic supported via the same provider abstraction +- **Multi-source retrieval** — OpenAlex, Semantic Scholar, arXiv, and CrossRef with per-provider retry, rate-limit handling, and graceful degradation +- **Embedding-backed analysis** — sentence-transformers (`bge-small-en-v1.5`) for deduplication, ranking, and HDBSCAN clustering +- **Report output** — Markdown, JSON, HTML, and print-ready PDF (HTML), plus BibTeX/APA/MLA/Chicago citation export +- **Session memory** — optional SQLite-backed interactive sessions with retrieval caching ## Requirements -- Python 3.13+ -- [Pipenv](https://pipenv.pypa.io/) for dependency management -- Internet connection (API retrieval; optional for fully offline LLM after model download) +- Python 3.13+ and [Pipenv](https://pipenv.pypa.io/) +- Internet access for paper retrieval (LLM inference can run fully offline after model download) -**Local LLM RAM (approximate):** - -| Model | RAM | Disk | -|-------|-----|------| +| Local model | RAM | Disk | +|-------------|-----|------| | `llama3.2:3b` | 4–6 GB | ~2.5 GB | | `llama3.1:8b` | 8–10 GB | ~5 GB | Cloud providers require only an API key — no Ollama install. -## Quick Start +## Quick start ```bash pip install pipenv @@ -42,292 +36,75 @@ cp .env.example .env # optional; edit as needed pipenv run python -m src "transformer attention mechanisms" ``` -On first run with the default Ollama provider, the assistant will: - -1. Check Python and embedding dependencies -2. Install or start Ollama if needed -3. Resolve your target model (`auto`, env override, or config) -4. Pull the model if it is not already installed -5. Run the research pipeline - -Use **Pipenv** for all commands (`pipenv run python -m src`). Running plain `python -m src` may miss dependencies such as `sentence-transformers`. - -While a query runs, the CLI streams **live progress to stderr**: pipeline stage checkmarks, sub-activities (e.g. “Analyzing paper 2/5”), and AI token previews during LLM calls. Disable with `--no-progress` or `RA_PIPELINE__STREAM_PROGRESS=false`. +On first run with the default Ollama provider, the assistant checks dependencies, installs and starts Ollama if needed, pulls the resolved model, and then runs the pipeline. Always run through Pipenv (`pipenv run python -m src`) so all dependencies are available. ## Usage -### Command line - ```bash -# Interactive mode +# Interactive mode with session follow-ups pipenv run python -m src -# Single query (markdown output) +# Single query (markdown to stdout) pipenv run python -m src "your research query" # HTML report saved to file pipenv run python -m src --format html -o reports/report.html "your query" -# Print-ready PDF (HTML — open in browser → Print → Save as PDF) +# Print-ready PDF (open in browser → Print → Save as PDF) pipenv run python -m src --format pdf -o reports/report.pdf.html "your query" # JSON output with citation exports pipenv run python -m src --format json --export bibtex,apa "your query" - -# Session memory in batch mode -pipenv run python -m src --session "your query" ``` -### Setup & health check +| Flag | Description | +|------|-------------| +| `--format` | `markdown` (default), `json`, `html`, `pdf` | +| `--export` | Comma-separated citation formats: `bibtex`, `apa`, `mla`, `chicago` | +| `--output`, `-o` | Write the report to a file | +| `--session` | Enable SQLite session memory in batch mode | +| `--no-progress` | Disable live progress streaming on stderr | + +Setup and health checks can also be run directly — see [setups/README.md](setups/README.md): ```bash pipenv run python -m setups.health_check -pipenv run python -m setups.manager # auto-select model -pipenv run python -m setups.manager --model llama3.1:8b +pipenv run python -m setups.manager [--model llama3.1:8b] ``` -See [setups/README.md](setups/README.md) for setup details. - ## Configuration -Configuration is layered (highest precedence first): - -1. Environment variables (`RA_*` prefix, nested with `__`) -2. Project `.env` file -3. YAML files in `config/` (`default.yaml`, `models.yaml`, `ranking.yaml`, `providers.yaml`) -4. Code defaults - -Copy `.env.example` to `.env` to get started. - -### Environment variables - -#### Retrieval APIs - -| Variable | Required | Description | -|----------|----------|-------------| -| `S2_API_KEY` | No | Semantic Scholar API key (higher rate limits) | -| `RA_CROSSREF_MAILTO` | If CrossRef enabled | Email for CrossRef polite pool | -| `CROSSREF_MAILTO` | If CrossRef enabled | Alias for CrossRef mailto | - -#### LLM — shared settings +Configuration is layered (highest precedence first): shell environment variables (`RA_*`, nested with `__`) → `.env` file → YAML files in `config/` → code defaults. -All providers use the `RA_LLM__*` namespace. API keys can also be set via provider-specific env vars (see below) or the unified `RA_LLM__API_KEY`. +Common settings: | Variable | Default | Description | |----------|---------|-------------| | `RA_LLM__PROVIDER` | `ollama` | `ollama`, `openai`, or `anthropic` | -| `RA_LLM__MODEL` | `auto` | Model name, or `auto` for resource-based selection (Ollama only) | -| `RA_LLM__BASE_URL` | provider-specific | API base URL | -| `RA_LLM__API_KEY` | — | Unified API key override for any provider | -| `RA_LLM__TEMPERATURE` | `0.2` | Sampling temperature | -| `RA_LLM__TIMEOUT_SECONDS` | `120` | Request timeout | - -#### LLM — Ollama (default) - -| Variable | Default | Description | -|----------|---------|-------------| -| `RA_LLM__PROVIDER` | `ollama` | Use local Ollama server | -| `RA_LLM__MODEL` | `auto` | `auto`, `llama3.1:8b`, `llama3.2:3b`, etc. (see `config/ollama_models.yaml`) | -| `RA_LLM__BASE_URL` | `http://localhost:11434/v1` | Ollama OpenAI-compatible endpoint | -| `RA_LLM__API_KEY` | `ollama` | Placeholder key (Ollama ignores it) | -| `OLLAMA_API_KEY` | — | Alternative to `RA_LLM__API_KEY` | - -**Model selection:** Set `RA_LLM__MODEL=auto` to pick the best model for your RAM/disk. Override with a specific model name in `.env` (e.g. `RA_LLM__MODEL=llama3.1:8b`). Supported models are listed in `config/ollama_models.yaml`. Setup pulls missing models automatically on startup. - -#### LLM — OpenAI - -| Variable | Required | Description | -|----------|----------|-------------| -| `RA_LLM__PROVIDER` | Yes | Set to `openai` | -| `RA_LLM__MODEL` | Yes | e.g. `gpt-4o-mini` | -| `OPENAI_API_KEY` | Yes* | OpenAI API key | -| `RA_LLM__API_KEY` | Yes* | Alternative unified key | -| `RA_LLM__BASE_URL` | No | Custom endpoint (defaults to `https://api.openai.com/v1`; use for LM Studio and other OpenAI-compatible servers) | - -\* One of `OPENAI_API_KEY` or `RA_LLM__API_KEY` is required. - -```bash -RA_LLM__PROVIDER=openai -RA_LLM__MODEL=gpt-4o-mini -OPENAI_API_KEY=sk-... -RA_SYNTHESIS__LLM_ENABLED=true -``` - -#### LLM — Anthropic - -| Variable | Required | Description | -|----------|----------|-------------| -| `RA_LLM__PROVIDER` | Yes | Set to `anthropic` | -| `RA_LLM__MODEL` | Yes | e.g. `claude-3-5-haiku-latest` | -| `ANTHROPIC_API_KEY` | Yes* | Anthropic API key | -| `RA_LLM__API_KEY` | Yes* | Alternative unified key | -| `RA_LLM__BASE_URL` | No | Custom Anthropic-compatible endpoint | +| `RA_LLM__MODEL` | `auto` | Model name, or `auto` for resource-based selection (Ollama) | +| `RA_LLM__API_KEY` | — | Unified API key (or `OPENAI_API_KEY` / `ANTHROPIC_API_KEY`) | +| `RA_LLM__BASE_URL` | provider-specific | Custom endpoint (e.g. LM Studio) | +| `RA_SYNTHESIS__LLM_ENABLED` | `false` | Enable LLM-based synthesis and gap analysis | +| `RA_RANKING__TOP_K` | `25` | Papers kept after ranking | +| `RA_PIPELINE__STREAM_PROGRESS` | `true` | Live stage/LLM progress on stderr | +| `RA_PIPELINE__DEBUG` | `false` | Verbose pipeline logging | +| `RA_SKIP_SETUP_CHECK` | — | Skip the Ollama setup/health check on startup | +| `S2_API_KEY` | — | Semantic Scholar API key (higher rate limits) | +| `RA_CROSSREF_MAILTO` | — | Email for the CrossRef polite pool | -\* One of `ANTHROPIC_API_KEY` or `RA_LLM__API_KEY` is required. +Cloud provider example: ```bash -RA_LLM__PROVIDER=anthropic -RA_LLM__MODEL=claude-3-5-haiku-latest -ANTHROPIC_API_KEY=sk-ant-... +RA_LLM__PROVIDER=openai # or anthropic +RA_LLM__MODEL=gpt-4o-mini # or a claude-* model +OPENAI_API_KEY=sk-... # or ANTHROPIC_API_KEY RA_SYNTHESIS__LLM_ENABLED=true ``` -#### Pipeline & synthesis - -| Variable | Default | Description | -|----------|---------|-------------| -| `RA_SYNTHESIS__LLM_ENABLED` | `false` | Enable LLM-based synthesis (recommended for 8B+ local or cloud models) | -| `RA_RANKING__TOP_K` | `25` | Papers kept after ranking | -| `RA_PIPELINE__DEBUG` | `false` | Verbose pipeline logging | -| `RA_PIPELINE__STREAM_PROGRESS` | `true` | Live stage/LLM progress on stderr (TTY only) | -| `RA_DEBUG` | — | Alias for debug mode (`1`, `true`, `yes`) | -| `RA_CONFIG_DIR` | — | Override path to `config/` directory | - -Provider implementations live in `src/models/` (`ollama.py`, `openai.py`, `anthropic.py`). Each resolves API keys via `RA_LLM__API_KEY` first, then the provider-specific env var. - -## Project Structure - -``` -Research_Assistant_Model/ -├── config/ # YAML configuration (merged at runtime) -│ ├── default.yaml # Base settings -│ ├── models.yaml # LLM provider overrides -│ ├── ollama_models.yaml # Supported local models + RAM/disk requirements -│ ├── ranking.yaml # Ranking weights and top-k -│ └── providers.yaml # Retrieval provider toggles -│ -├── src/ # Application source -│ ├── __main__.py # CLI entry point (`python -m src`) -│ ├── config/ # AppSettings, model auto-selection -│ ├── core/ # Pipeline engine, stage recovery, metrics -│ ├── retrieval/ # API clients, providers, deduplication -│ │ ├── providers/ # OpenAlex, Semantic Scholar, arXiv, … -│ │ ├── orchestrator.py # Pipeline facade -│ │ └── models.py # RetrievedPaper, ResearchReport, … -│ ├── research/ # Query expansion, ranking, clustering -│ ├── analysis/ # Synthesis, gap analysis -│ ├── embeddings/ # sentence-transformers + disk cache -│ ├── models/ # LLM providers -│ │ ├── ollama.py # Local Ollama (default) -│ │ ├── openai.py # OpenAI / compatible endpoints -│ │ ├── anthropic.py # Anthropic Claude -│ │ └── factory.py # AgentFactory + provider registry -│ ├── reporting/ # Markdown, HTML, JSON renderers -│ ├── export/ # BibTeX, APA, MLA, Chicago -│ ├── memory/ # SQLite session store -│ └── utils/ # Logging, retry, response handling -│ -├── setups/ # Install & health-check scripts -│ ├── manager.py # Full setup orchestrator -│ ├── ollama.py # Ollama install + model pull -│ └── health_check.py # Validate deps, Ollama, model -│ -├── tests/ # Unit and integration tests -├── reports/ # Generated reports (gitignored) -├── data/ # Embeddings cache, SQLite DB (gitignored) -├── logs/ # Structured logs (gitignored) -├── .env # Local secrets (gitignored; see .env.example) -├── Pipfile / Pipfile.lock -└── README.md -``` +The full configuration reference (all `RA_*` variables, YAML files, ranking weights, provider toggles) is in the [documentation](https://ndevu12.github.io/Research_Assistant_Model/). ## Architecture -### End-to-end pipeline - -``` -┌─────────────────────────────────────────────────────────────────────────────┐ -│ User Query / CLI │ -└───────────────────────────────────┬─────────────────────────────────────────┘ - │ - ▼ -┌─────────────────────────────────────────────────────────────────────────────┐ -│ QUERY UNDERSTANDING & EXPANSION │ -│ Parse intent · generate search variants · optional LLM query expansion │ -└───────────────────────────────────┬─────────────────────────────────────────┘ - │ - ▼ -┌─────────────────────────────────────────────────────────────────────────────┐ -│ PARALLEL PAPER RETRIEVAL │ -│ ┌────────────┐ ┌──────────────────┐ ┌─────────┐ ┌──────────┐ │ -│ │ OpenAlex │ │ Semantic Scholar │ │ arXiv │ │ CrossRef │ … │ -│ └────────────┘ └──────────────────┘ └─────────┘ └──────────┘ │ -└───────────────────────────────────┬─────────────────────────────────────────┘ - │ - ▼ -┌─────────────────────────────────────────────────────────────────────────────┐ -│ EMBEDDING-BACKED PROCESSING (bge-small-en-v1.5) │ -│ Deduplication → Ranking → Relevance Scoring → Clustering │ -└───────────────────────────────────┬─────────────────────────────────────────┘ - │ - ▼ -┌─────────────────────────────────────────────────────────────────────────────┐ -│ ANALYSIS LAYER │ -│ Synthesis (heuristic or LLM) → Gap Analysis → Citation Export │ -└───────────────────────────────────┬─────────────────────────────────────────┘ - │ - ▼ -┌─────────────────────────────────────────────────────────────────────────────┐ -│ REPORT GENERATION │ -│ Markdown · JSON · HTML · PDF-ready HTML · BibTeX/APA/… │ -└─────────────────────────────────────────────────────────────────────────────┘ -``` - -### LLM provider layer - -The analysis stages call one backend selected via `RA_LLM__PROVIDER`: - -``` - ┌──────────────────────────────────────┐ - │ src/models/ (factory) │ - │ AgentFactory · pydantic-ai agents │ - └───────────────────┬──────────────────┘ - │ - ┌─────────────────────────┼─────────────────────────┐ - │ │ │ - ▼ ▼ ▼ - ┌────────────────────┐ ┌────────────────────┐ ┌────────────────────┐ - │ Ollama │ │ OpenAI │ │ Anthropic │ - │ (local default) │ │ gpt-4o-mini, … │ │ claude-3-5-… │ - │ │ │ │ │ │ - │ RA_LLM__MODEL=auto │ │ OPENAI_API_KEY │ │ ANTHROPIC_API_KEY │ - │ ollama_models.yaml │ │ RA_LLM__API_KEY │ │ RA_LLM__API_KEY │ - └────────────────────┘ └────────────────────┘ └────────────────────┘ -``` - -### First-run setup (Ollama) - -When `RA_LLM__PROVIDER=ollama`, startup runs this automatically if anything is missing: - -``` -pipenv run python -m src - │ - ▼ -┌───────────────────┐ no ┌────────────────────────────┐ -│ Ollama installed? │────────────►│ Install Ollama (setups/) │ -└─────────┬─────────┘ └─────────────┬──────────────┘ - │ yes │ - ▼ ▼ -┌───────────────────┐ no ┌────────────────────────────┐ -│ Ollama running? │────────────►│ Start ollama serve │ -└─────────┬─────────┘ └─────────────┬──────────────┘ - │ yes │ - ▼ ▼ -┌───────────────────┐ no ┌────────────────────────────┐ -│ Model installed? │────────────►│ ollama pull │ -│ (from .env/auto) │ │ e.g. llama3.1:8b / 3b │ -└─────────┬─────────┘ └─────────────┬──────────────┘ - │ yes │ - └─────────────────┬───────────────────┘ - ▼ - Run research pipeline -``` - -Cloud providers (`openai`, `anthropic`) skip Ollama setup and use API keys directly. - -
-Detailed pipeline flow (Mermaid — renders on GitHub) - ```mermaid flowchart TD Q[User Query] --> QU[Query Understanding] @@ -335,20 +112,18 @@ flowchart TD QE --> R[Parallel Retrieval] R --> OA[OpenAlex] R --> SS[Semantic Scholar] - R --> AX[arXiv / CrossRef / …] + R --> AX[arXiv / CrossRef] OA --> DEDUP[Deduplication] SS --> DEDUP AX --> DEDUP DEDUP --> RANK[Ranking] - RANK --> REL[Relevance Scoring] + RANK --> REL[Relevance Filtering] REL --> CLU[Clustering] CLU --> SYN[Synthesis] SYN --> GAP[Gap Analysis] GAP --> CIT[Citation Export] CIT --> REP[Report Generation] - REP --> MD[Markdown] - REP --> JSON[JSON] - REP --> HTML[HTML / PDF-ready] + REP --> MD[Markdown / JSON / HTML] subgraph LLM["LLM backend (RA_LLM__PROVIDER)"] OLL[Ollama] @@ -361,129 +136,53 @@ flowchart TD GAP -.-> LLM ``` -
- -### Configuration precedence - -``` - Highest ───────────────────────────────────────────────► Lowest +Every stage degrades gracefully: retrieval continues when a provider fails, ranking falls back to keyword signals without embeddings, and synthesis/gap analysis use heuristics when the LLM is disabled or unavailable. - ┌─────────────┐ ┌─────────────┐ ┌─────────────┐ ┌─────────────┐ - │ RA_* env │ → │ .env │ → │ config/*.yaml│ → │ defaults │ - │ (shell) │ │ file │ │ (merged) │ │ (in code) │ - └─────────────┘ └─────────────┘ └─────────────┘ └─────────────┘ ``` - -## Output Format - -Reports include query summary, ranked papers, synthesis themes, gap analysis, and citations. - -**Example (markdown excerpt):** - -```markdown -# Research Report: transformer attention mechanisms - -## Executive Summary -Cross-paper synthesis highlights scaled dot-product attention, multi-head variants, -and efficiency techniques for long-context models. - -## Thematic Clusters - -### 1. Core Attention Architectures -2023 | NeurIPS -DOI: https://doi.org/10.xxxx/xxxxx - -Key findings: -- Multi-head attention improves representational capacity -- FlashAttention reduces memory bandwidth bottlenecks - -## Research Gaps -- Limited benchmarks on edge-device deployment -- Under-explored sparse attention for retrieval-augmented pipelines -``` - -**Export formats:** - -| `--format` | Output | -|------------|--------| -| `markdown` | Terminal / stdout (default) | -| `json` | Structured `EnhancedResearchReport` JSON | -| `html` | Styled HTML report | -| `pdf` | Print-ready HTML (open → Print → Save as PDF) | - -Use `--export bibtex,apa,mla,chicago` alongside any format for citation files. - - -### Setup / Ollama - -```bash -pipenv run python -m setups.health_check -pipenv run python -m setups.manager -``` - -- **Model not installed** — startup auto-pulls the resolved model; or run `ollama pull ` manually -- **Wrong model** — set `RA_LLM__MODEL` in `.env` or use `--model` with setup -- **Ollama not running** — `ollama serve` or re-run setup - -### Cloud providers - -- Set `RA_LLM__PROVIDER` to `openai` or `anthropic` and provide the API key -- Ollama setup is skipped automatically for non-Ollama providers -- Enable `RA_SYNTHESIS__LLM_ENABLED=true` for LLM-based synthesis - -### Missing embeddings / import errors - -Always use Pipenv: - -```bash -pipenv install -pipenv run python -m src -``` - -### Logs - -```bash -tail -f logs/combined_*.log +Research_Assistant_Model/ +├── config/ # YAML configuration (merged at runtime) +├── src/ +│ ├── __main__.py # CLI entry point (python -m src) +│ ├── config/ # Settings, model auto-selection +│ ├── core/ # Pipeline engine, stage recovery, metrics +│ ├── retrieval/ # Providers, deduplication, retrieval stage +│ ├── research/ # Query expansion, ranking, relevance, clustering +│ ├── analysis/ # Synthesis, gap analysis +│ ├── embeddings/ # sentence-transformers + disk cache +│ ├── models/ # LLM providers (Ollama, OpenAI, Anthropic) +│ ├── reporting/ # Markdown, HTML, JSON renderers +│ ├── export/ # BibTeX, APA, MLA, Chicago +│ ├── memory/ # SQLite session store +│ └── utils/ # Logging, retry, response handling +├── setups/ # Install and health-check scripts +├── tests/ # Test suite (pytest) +└── docs/ # mkdocs documentation site ``` ## Development -### Running locally - ```bash pipenv install --dev -pipenv run pytest -pipenv shell -python -m src +pipenv run pytest # run the test suite +ruff check src tests setups # lint ``` -### Import paths - -**Within the package (relative imports):** +Within the package use relative imports (`from .models import RetrievedPaper`); from external scripts use absolute imports (`from src.retrieval.orchestrator import run_research_helper`). -```python -# In src/retrieval/openalex.py -from .models import RetrievedPaper - -# In src/analysis/synthesis.py -from ..models import AgentFactory, AgentRole -from ..retrieval.models import RankedPaper -``` - -**From external scripts (absolute imports):** - -```python -from src.retrieval.orchestrator import run_research_helper -from src.models import AgentFactory, create_llm_provider -from setups import run_setup, print_report -``` - -### Core dependencies - -| Package | Role | -|---------|------| +| Core dependency | Role | +|-----------------|------| | `pydantic-ai` | LLM agents with structured outputs | | `aiohttp` | Async HTTP for scholarly APIs | | `sentence-transformers` | Embeddings for dedup, ranking, clustering | | `pydantic` / `pydantic-settings` | Schemas and configuration | | `hdbscan` | Thematic paper clustering | + +## Troubleshooting + +- **Ollama not running / model missing** — startup auto-installs and pulls; or run `pipenv run python -m setups.manager` +- **Import errors** — run through Pipenv: `pipenv install && pipenv run python -m src` +- **Logs** — `tail -f logs/combined_*.log` + +## License + +MIT — see [LICENSE](LICENSE). diff --git a/pyproject.toml b/pyproject.toml index 18d57f4..1647f84 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,3 +7,17 @@ asyncio_mode = "auto" markers = [ "slow: subprocess or end-to-end tests (excluded from default CI)", ] + +[tool.ruff] +line-length = 100 +target-version = "py313" + +[tool.ruff.lint] +# Correctness-focused: syntax errors, undefined names, unused imports/variables. +select = ["E9", "F"] +ignore = [ + "F401", # __init__.py re-exports are intentional +] + +[tool.ruff.lint.per-file-ignores] +"tests/*" = ["F811", "F841"] diff --git a/src/__main__.py b/src/__main__.py index 5f8c5ca..f78f43c 100644 --- a/src/__main__.py +++ b/src/__main__.py @@ -29,6 +29,10 @@ def ensure_setup() -> bool: """ import os + if os.environ.get("RA_SKIP_SETUP_CHECK", "").strip().lower() in {"1", "true", "yes"}: + logger.info("RA_SKIP_SETUP_CHECK is set; skipping environment setup checks.") + return True + if os.environ.get("PIPENV_ACTIVE") != "1": logger.warning( "Not running inside Pipenv. Use: pipenv run python -m src ... " diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..488b8a0 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,21 @@ +# -*- coding: utf-8 -*- +"""Shared test fixtures.""" + +from __future__ import annotations + +from collections.abc import Iterator +from unittest.mock import patch + +import pytest + + +@pytest.fixture(autouse=True) +def _skip_environment_setup() -> Iterator[None]: + """Keep tests hermetic: never run the real Ollama install/health-check path. + + ``src.__main__.main`` calls ``ensure_setup`` which can install Ollama and + pull models. No test exercises that path intentionally, so it is stubbed + out globally to keep the suite fast and network-independent. + """ + with patch("src.__main__.ensure_setup", return_value=True): + yield diff --git a/tests/test_complete_workflow.py b/tests/test_complete_workflow.py index 0030927..e36264f 100644 --- a/tests/test_complete_workflow.py +++ b/tests/test_complete_workflow.py @@ -242,7 +242,8 @@ def test_system_startup_without_arguments(self): stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, - cwd=os.getcwd() + cwd=os.getcwd(), + env={**os.environ, 'RA_SKIP_SETUP_CHECK': '1'} ) # Send exit command immediately @@ -269,7 +270,8 @@ def test_batch_mode_compatibility(self): capture_output=True, text=True, timeout=30, - cwd=os.getcwd() + cwd=os.getcwd(), + env={**os.environ, 'RA_SKIP_SETUP_CHECK': '1'} ) # Verify batch mode behavior (no interactive prompts)