Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion src/tendertracker/cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -136,7 +136,7 @@ def run(
"""Run the daily fetch pipeline. Plan-only by default — pass --apply to write for real."""
from .pipeline.run_daily import run_all

results = run_all(apply=apply)
results = run_all(settings, apply=apply)
if not results:
console.print(
"[yellow]No enabled sources to run. Copy config/portals.example.yaml to config/portals.yaml.[/yellow]"
Expand Down
13 changes: 11 additions & 2 deletions src/tendertracker/pipeline/run_daily.py
Original file line number Diff line number Diff line change
@@ -1,14 +1,18 @@
import importlib
import logging
from dataclasses import dataclass, field
from typing import TYPE_CHECKING

from sqlalchemy.orm import Session

from ..config import PortalConfig, load_portals, settings
from ..config import PortalConfig, load_portals
from ..scrapers.base import Scraper
from ..storage.db import get_engine, get_session_factory
from ..storage.models import SyncLog, Tender, utcnow

if TYPE_CHECKING:
from ..config import Settings

logger = logging.getLogger(__name__)

# Fields a re-scrape must never overwrite once a human may have touched them —
Expand Down Expand Up @@ -90,12 +94,17 @@ def run_source(session: Session, portal: PortalConfig) -> RunResult:
return result


def run_all(apply: bool = False) -> list[RunResult]:
def run_all(settings: "Settings", apply: bool = False) -> list[RunResult]:
"""Run the pipeline across all enabled sources.

Defaults to plan-only (apply=False): computes what would change, records
a SyncLog entry either way, but rolls back the Tender writes instead of
committing them. Pass apply=True to actually write.

Takes `settings` explicitly, matching sync_pipedrive/sync_calendar/
reconcile — this used to read the global config.settings singleton
directly, the only pipeline module that did, which made it harder to
test in isolation and inconsistent with the rest of the codebase.
"""
portals = [p for p in load_portals() if p.enabled]
if not portals:
Expand Down
51 changes: 49 additions & 2 deletions tests/test_run_daily.py
Original file line number Diff line number Diff line change
@@ -1,8 +1,9 @@
from tendertracker.config import PortalConfig
from tendertracker.pipeline.relevance import RelevanceRules
from tendertracker.pipeline.run_daily import run_source
from tendertracker.pipeline.run_daily import run_all, run_source
from tendertracker.scrapers.base import RawTender, Scraper
from tendertracker.storage.models import Tender
from tendertracker.storage.db import get_engine, get_session_factory, init_db
from tendertracker.storage.models import SyncLog, Tender

SANDBOX_PORTAL = PortalConfig(
name="sandbox",
Expand Down Expand Up @@ -78,3 +79,49 @@ def fetch(self):
assert result.new == 2 # the two good records still made it through
assert result.errors == 1
assert "BAD-1" in result.error_messages[0]


# --- run_all(): only testable in isolation because it now takes `settings`
# explicitly instead of reading the global config.settings singleton — the
# fix for the run_daily/settings inconsistency this replaced. ---------------


def test_run_all_dry_run_writes_synclog_but_not_tenders(tmp_settings, monkeypatch):
import tendertracker.pipeline.run_daily as run_daily_module

init_db(get_engine(tmp_settings.db_path))
monkeypatch.setattr(run_daily_module, "load_portals", lambda: [SANDBOX_PORTAL])

results = run_all(tmp_settings, apply=False)
assert len(results) == 1
assert results[0].new == 5

with get_session_factory(get_engine(tmp_settings.db_path))() as s:
assert s.query(Tender).count() == 0
assert s.query(SyncLog).one().status == "dry-run"


def test_run_all_apply_writes_tenders_and_synclog(tmp_settings, monkeypatch):
import tendertracker.pipeline.run_daily as run_daily_module

init_db(get_engine(tmp_settings.db_path))
monkeypatch.setattr(run_daily_module, "load_portals", lambda: [SANDBOX_PORTAL])

results = run_all(tmp_settings, apply=True)
assert results[0].new == 5

with get_session_factory(get_engine(tmp_settings.db_path))() as s:
assert s.query(Tender).count() == 5
assert s.query(SyncLog).one().status == "success"


def test_run_all_skips_disabled_portals(tmp_settings, monkeypatch):
import tendertracker.pipeline.run_daily as run_daily_module

init_db(get_engine(tmp_settings.db_path))
disabled = PortalConfig(
name="sandbox", scraper_class="tendertracker.scrapers.sandbox_feed.SandboxFeedScraper", enabled=False
)
monkeypatch.setattr(run_daily_module, "load_portals", lambda: [disabled])

assert run_all(tmp_settings, apply=True) == []
Loading