diff --git a/.env.dev b/.env.dev index ca0e7415..c257ad91 100644 --- a/.env.dev +++ b/.env.dev @@ -28,6 +28,6 @@ MCP_WRITE_TOKEN=agentique-dev-write MCP_DB_USER=postgres MCP_DB_PASSWORD=agentique-dev -# Jev (pipeline/jev.py), the newsletter link classifier. Empty locally: set a +# Jev (backend/app/core/jev.py), the newsletter link classifier. Empty locally: set a # real key only in .env, never here. OPENROUTER_API_KEY= diff --git a/.github/workflows/claude-code-review.yml b/.github/workflows/claude-code-review.yml index 1a422372..d4a8cb7b 100644 --- a/.github/workflows/claude-code-review.yml +++ b/.github/workflows/claude-code-review.yml @@ -34,7 +34,7 @@ jobs: - name: Run Claude Code Review id: claude-review - uses: anthropics/claude-code-action@f27be6b65a4652f08a2ba3cff79114829b588967 # v1 + uses: anthropics/claude-code-action@f27be6b65a4652f08a2ba3cff79114829b588967 # v1.0.252 with: claude_code_oauth_token: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }} plugin_marketplaces: 'https://github.com/anthropics/claude-code.git' diff --git a/.github/workflows/claude.yml b/.github/workflows/claude.yml index 04a576af..c5b76092 100644 --- a/.github/workflows/claude.yml +++ b/.github/workflows/claude.yml @@ -33,7 +33,7 @@ jobs: - name: Run Claude Code id: claude - uses: anthropics/claude-code-action@f27be6b65a4652f08a2ba3cff79114829b588967 # v1 + uses: anthropics/claude-code-action@f27be6b65a4652f08a2ba3cff79114829b588967 # v1.0.252 with: claude_code_oauth_token: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }} diff --git a/.github/workflows/deploy-production.yml b/.github/workflows/deploy-production.yml index 6183c932..2e46a075 100644 --- a/.github/workflows/deploy-production.yml +++ b/.github/workflows/deploy-production.yml @@ -49,25 +49,21 @@ jobs: uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0 with: version: "0.11.18" + # Straight into frontend/dist, so the one sync below ships it. - name: Download dist uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: frontend-dist - path: frontend-dist + path: frontend/dist # The workspace is a staging area only — actions/checkout resets it every - # run, so the services live in /opt/agentique. .env and .venv belong to - # the box; frontend/dist is replaced separately below. - - name: Sync source to /opt/agentique - run: > - rsync -a --delete - --exclude '.git' - --exclude '.env' - --exclude '.venv' - --exclude 'frontend/dist' - --exclude 'frontend-dist' - ./ /opt/agentique/ - - name: Sync frontend dist - run: rsync -a --delete frontend-dist/ /opt/agentique/frontend/dist/ + # run, so the services live in /opt/agentique. deploy/rsync-filter lists + # what the box runs; --delete-excluded removes everything else there, + # except what the filter protects (.env, .venv, .git). A missing dist + # would delete the live site, so check for it first. + - name: Sync to /opt/agentique + run: | + test -f frontend/dist/index.html + rsync -a --delete --delete-excluded --filter='merge deploy/rsync-filter' ./ /opt/agentique/ # --package app: the workspace root is virtual, a bare sync would not # install the backend (same flag the old Dockerfile used). - name: Sync venv @@ -76,7 +72,7 @@ jobs: # --env-file is load-bearing: prestart.sh's seed guard checks the shell # ENVIRONMENT, and a bare deploy shell doesn't have it. - name: Run migrations + prestart - run: uv run --env-file ../.env bash scripts/prestart.sh + run: uv run --env-file ../.env bash app/entry/cli/prestart.sh working-directory: /opt/agentique/backend - name: Restart backend run: sudo systemctl restart agentique-backend diff --git a/.github/workflows/test-backend.yml b/.github/workflows/test-backend.yml index 287ceddc..48323c5f 100644 --- a/.github/workflows/test-backend.yml +++ b/.github/workflows/test-backend.yml @@ -39,12 +39,15 @@ jobs: POSTGRES_USER=postgres POSTGRES_PASSWORD=ci-test-password EOF + - name: Check import layers + run: uv run lint-imports + working-directory: backend - run: docker compose up -d - name: Migrate DB - run: uv run bash scripts/prestart.sh + run: uv run bash app/entry/cli/prestart.sh working-directory: backend - name: Run tests - run: uv run bash scripts/tests-start.sh "Coverage for ${{ github.sha }}" + run: uv run bash ../scripts/tests-start.sh "Coverage for ${{ github.sha }}" working-directory: backend - run: docker compose down -v - name: Store coverage files diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 27344c55..5d8f43c7 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -53,14 +53,21 @@ repos: - id: local-mypy name: mypy check - entry: uv run mypy backend/app backend/scripts + entry: uv run mypy --config-file backend/pyproject.toml backend/app require_serial: true language: unsupported pass_filenames: false - id: local-ty name: ty check - entry: uv run ty check backend/app backend/scripts + entry: uv run ty check --project backend backend/app + require_serial: true + language: unsupported + pass_filenames: false + + - id: local-lint-imports + name: import-linter + entry: uv run lint-imports --config backend/pyproject.toml require_serial: true language: unsupported pass_filenames: false diff --git a/CLAUDE.md b/CLAUDE.md index 6cafa302..a9bbc7f4 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -36,7 +36,7 @@ - Read and mutate freely. It's a local copy, not prod. ## MCP tools -- `.mcp.json` connects to the `agentique` MCP server (`backend/app/mcp/`), exposing `sql_query` (read-only prod Postgres), `web_fetch`, `web_search`. +- `.mcp.json` connects to the `agentique` MCP server (`backend/app/entry/mcp/`), exposing `sql_query` (read-only prod Postgres), `web_fetch`, `web_search`. - With the write token it also exposes `list_candidates`, `get_content`, `similar`, `stories`, `check_link`, `vocabulary`, `approve`, `reject`, `reject_many` - the curation agent's tools. They publish to prod. Don't call them outside a curation session. - The write token also reaches `week`, `related`, `draft_issue` - the newsletter agent's tools (`.claude/skills/newsletter/SKILL.md`). `draft_issue` creates a Resend draft addressed to every subscriber; sending is done by hand in Resend. - And `story_candidates`, `save_story`, `close_story` - the stories agent's tools (`.claude/skills/stories/SKILL.md`). They write the right rail's stories in prod. diff --git a/backend/app/features/catalog/tests/factories.py b/backend/app/core/db/factories.py similarity index 56% rename from backend/app/features/catalog/tests/factories.py rename to backend/app/core/db/factories.py index b8f2f11e..c5908ba7 100644 --- a/backend/app/features/catalog/tests/factories.py +++ b/backend/app/core/db/factories.py @@ -1,15 +1,61 @@ -"""Factory for creating random Article rows in tests.""" +"""Test factories: random users, publishers, tags and articles. + +They write rows straight to the tables, so a test anywhere can use them without +importing a feature. +""" from __future__ import annotations import random +import string +from datetime import timedelta -from sqlmodel import Session +from sqlmodel import Session, select from app.core.db.article import Article, ArticleKind from app.core.db.publisher import Publisher, PublisherKind, slugify from app.core.db.tag import ArticleTag, Tag -from tests.random_data import random_lower_string +from app.core.db.user import User +from app.core.security import create_access_token +from app.core.settings import settings + + +def random_lower_string() -> str: + """Random enough that two tests never collide on a unique column.""" + return "".join(random.choices(string.ascii_lowercase, k=32)) + + +def random_email() -> str: + return f"{random_lower_string()}@{random_lower_string()}.com" + + +def create_user(db: Session, email: str) -> User: + user = User(email=email) + db.add(user) + db.commit() + db.refresh(user) + return user + + +def create_random_user(db: Session) -> User: + return create_user(db, random_email()) + + +def user_authentication_headers(user: User) -> dict[str, str]: + token = create_access_token(user.id, expires_delta=timedelta(minutes=5)) + return {"Authorization": f"Bearer {token}"} + + +def authentication_token_from_email(*, email: str, db: Session) -> dict[str, str]: + """Headers for the user with this email, created first if it isn't there.""" + user = db.exec(select(User).where(User.email == email)).first() + if not user: + user = create_user(db, email) + return user_authentication_headers(user) + + +def get_superuser_token_headers(db: Session) -> dict[str, str]: + return authentication_token_from_email(email=settings.FIRST_SUPERUSER, db=db) def create_random_publisher(db: Session, **overrides: object) -> Publisher: diff --git a/backend/app/core/db/story.py b/backend/app/core/db/story.py index 85e08b35..5458cc3d 100644 --- a/backend/app/core/db/story.py +++ b/backend/app/core/db/story.py @@ -41,7 +41,7 @@ class Story(SQLModel, table=True): class StoryArticle(SQLModel, table=True): """An article in a story. At most one open story per article, enforced in - `catalog.stories.save_story`: a closed story keeps its articles.""" + `stories.service.save_story`: a closed story keeps its articles.""" __tablename__ = "story_article" # The primary key leads with story_id; "which story holds this article" is diff --git a/backend/tests/core/test_db.py b/backend/app/core/db/test_engine.py similarity index 50% rename from backend/tests/core/test_db.py rename to backend/app/core/db/test_engine.py index 63fb16c3..eb3a5145 100644 --- a/backend/tests/core/test_db.py +++ b/backend/app/core/db/test_engine.py @@ -7,28 +7,11 @@ from sqlalchemy import text from sqlmodel import SQLModel -from app.core import db from app.core.db.engine import engine, get_session from app.core.settings import settings -from app.entry.api.deps import SessionDep -from app.features.collect import service as collect -from app.main import app # noqa: F401 # every module that could declare a table -def test_every_table_lives_in_core_db(): - """The app is fully imported here, so a table declared anywhere else shows - up in the metadata and not in `app.core.db`.""" - declared = set() - for name in db.__all__: - for cls in vars(getattr(db, name)).values(): - if isinstance(cls, type) and issubclass(cls, SQLModel): - table = getattr(cls, "__table__", None) - if table is not None: - declared.add(table.name) - assert declared == set(SQLModel.metadata.tables) - - -def test_tables_match_the_migrations(): +def test_tables_match_the_migrations() -> None: """`alembic check` in the test run: a table edit without a migration, or a migration the tables do not declare, fails here.""" with engine.connect() as conn: @@ -37,23 +20,18 @@ def test_tables_match_the_migrations(): assert diff == [] -def test_engine_connects_with_the_settings_credentials(): +def test_engine_connects_with_the_settings_credentials() -> None: assert engine.url.database == settings.POSTGRES_DB with engine.connect() as conn: user = conn.execute(text("SELECT current_user")).scalar_one() assert user == settings.POSTGRES_USER -def test_the_request_session_is_closed_after_the_request(): +def test_the_request_session_is_closed_after_the_request() -> None: sessions = get_session() session = next(sessions) assert session.bind is engine - session.execute(text("SELECT 1")) + session.connection().execute(text("SELECT 1")) assert session.in_transaction() sessions.close() assert not session.in_transaction() - - -def test_routes_and_the_collect_run_share_the_engine(): - assert collect.engine is engine - assert SessionDep.__metadata__[0].dependency is get_session diff --git a/backend/tests/test_models_enums.py b/backend/app/core/db/test_enums.py similarity index 92% rename from backend/tests/test_models_enums.py rename to backend/app/core/db/test_enums.py index bd6fccba..a0eafa50 100644 --- a/backend/tests/test_models_enums.py +++ b/backend/app/core/db/test_enums.py @@ -19,7 +19,7 @@ # Every table module, so a new one is guarded without a line here. MODEL_MODULES = [getattr(db, name) for name in db.__all__] -VERSIONS = Path(__file__).parent.parent / "migrations" / "versions" +VERSIONS = Path(__file__).resolve().parents[3] / "migrations" / "versions" # SQLModel names a Postgres enum type after the lowercased class name. ENUM_MODELS = { @@ -34,7 +34,7 @@ ) -def test_migration_enum_labels_exist_in_python(): +def test_migration_enum_labels_exist_in_python() -> None: added = [ (m.group(1).lower(), m.group(2)) for path in VERSIONS.glob("*.py") diff --git a/backend/tests/core/test_ledger.py b/backend/app/core/db/test_ledger.py similarity index 70% rename from backend/tests/core/test_ledger.py rename to backend/app/core/db/test_ledger.py index f2a70eb6..abe99206 100644 --- a/backend/tests/core/test_ledger.py +++ b/backend/app/core/db/test_ledger.py @@ -3,10 +3,12 @@ from __future__ import annotations from collections.abc import Generator +from typing import cast import pytest from sqlmodel import Session, col, delete, select +from app.core.db.factories import create_random_publisher, random_lower_string from app.core.db.ledger import ( CANDIDATE_CAP, CONTENT_CAP, @@ -16,21 +18,25 @@ record_reject, ) from app.core.db.publisher import Publisher -from app.features.catalog.tests.factories import create_random_publisher -from tests.random_data import random_lower_string URL = f"https://example.com/{random_lower_string()}" class _FakeSession: def __init__(self) -> None: - self.rows: list = [] + self.rows: list[Reject] = [] - def merge(self, obj) -> None: + def merge(self, obj: Reject) -> None: self.rows.append(obj) -def _candidate(**overrides) -> LedgerItem: +def _session() -> tuple[Session, list[Reject]]: + """A session that keeps what `record_reject` merges, and the rows it kept.""" + fake = _FakeSession() + return cast("Session", fake), fake.rows + + +def _candidate(**overrides: object) -> LedgerItem: a: LedgerItem = { "title": "A repo", "url": "https://github.com/someone/thing", @@ -40,59 +46,62 @@ def _candidate(**overrides) -> LedgerItem: "publisher_id": 7, "traction": "3 points, 0 comments on Hacker News", } - return {**a, **overrides} + return cast("LedgerItem", {**a, **overrides}) -def test_keeps_what_the_step_saw(): - session = _FakeSession() +def test_keeps_what_the_step_saw() -> None: + session, rows = _session() record_reject(session, _candidate(), RejectStage.thin_repo, detail={"stars": 4}) - [row] = session.rows + [row] = rows assert row.stage == RejectStage.thin_repo assert (row.title, row.source, row.publisher_id) == ("A repo", "Hacker News", 7) assert row.traction == "3 points, 0 comments on Hacker News" + assert row.published_at is not None assert row.published_at.year == 2026 assert row.detail == {"stars": 4} -def test_caps_content_and_strips_nul_bytes(): +def test_caps_content_and_strips_nul_bytes() -> None: """Postgres refuses a NUL in a text column, and the failed commit would take the whole source down with it.""" - session = _FakeSession() + session, rows = _session() content = "a\x00b" + "x" * (CONTENT_CAP * 2) record_reject( session, _candidate(title="t\x00", content=content), RejectStage.prefilter ) - [row] = session.rows + [row] = rows assert row.title == "t" + assert row.content is not None assert "\x00" not in row.content assert len(row.content) < CONTENT_CAP -def test_a_pending_candidate_keeps_the_article_for_the_agent_to_read(): - session = _FakeSession() +def test_a_pending_candidate_keeps_the_article_for_the_agent_to_read() -> None: + session, rows = _session() record_reject( session, _candidate(content="x" * (CANDIDATE_CAP * 2)), RejectStage.pending ) - assert len(session.rows[0].content) == CANDIDATE_CAP + assert rows[0].content is not None + assert len(rows[0].content) == CANDIDATE_CAP -def test_empty_content_is_stored_as_null(): - session = _FakeSession() +def test_empty_content_is_stored_as_null() -> None: + session, rows = _session() record_reject(session, _candidate(content=""), RejectStage.prefilter) - assert session.rows[0].content is None + assert rows[0].content is None -def test_keeps_the_links_extraction_found(): - session = _FakeSession() +def test_keeps_the_links_extraction_found() -> None: + session, rows = _session() links = {"repo": ["https://github.com/someone/thing"]} record_reject(session, _candidate(links=links), RejectStage.pending) - assert session.rows[0].links == links + assert rows[0].links == links -def test_no_links_is_stored_as_null(): - session = _FakeSession() +def test_no_links_is_stored_as_null() -> None: + session, rows = _session() record_reject(session, _candidate(links={}), RejectStage.pending) - assert session.rows[0].links is None + assert rows[0].links is None @pytest.fixture @@ -121,14 +130,16 @@ def _stored( return row -def test_no_detail_is_sql_null_not_json_null(db: Session, publisher: Publisher): +def test_no_detail_is_sql_null_not_json_null(db: Session, publisher: Publisher) -> None: """`detail IS NULL` is how the ledger asks for rows without evidence.""" _stored(db, publisher) found = db.exec(select(Reject.url).where(col(Reject.detail).is_(None))).all() assert URL in found -def test_recording_a_url_again_overwrites_its_row(db: Session, publisher: Publisher): +def test_recording_a_url_again_overwrites_its_row( + db: Session, publisher: Publisher +) -> None: """The nightly run can meet a URL it already queued; the row is replaced, not duplicated, so the commit does not fail on the primary key.""" _stored(db, publisher, title="first") @@ -136,7 +147,9 @@ def test_recording_a_url_again_overwrites_its_row(db: Session, publisher: Publis assert (row.title, row.stage) == ("second", RejectStage.prefilter) -def test_nul_bytes_and_a_bad_date_still_commit(db: Session, publisher: Publisher): +def test_nul_bytes_and_a_bad_date_still_commit( + db: Session, publisher: Publisher +) -> None: row = _stored( db, publisher, title="a\x00b", content="c\x00d", published_date="not a date" ) diff --git a/backend/app/core/email.py b/backend/app/core/email.py index d3ea0fac..61d6c4f5 100644 --- a/backend/app/core/email.py +++ b/backend/app/core/email.py @@ -17,7 +17,6 @@ from app.core.logging import log from app.core.settings import settings -logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) @@ -65,7 +64,8 @@ def send_email( def send_mail(subject_line: str, body: str) -> None: """Email through Resend, or log the mail when the environment has no key. - Best-effort: a failed send is logged, never raised.""" + A failed send raises: the daily report's exit code is the only sign that + no mail went out.""" api_key = os.environ.get("RESEND_API_KEY") from_email = os.environ.get("EMAILS_FROM_EMAIL") to_email = os.environ.get("PIPELINE_ALERT_EMAIL") or from_email @@ -73,16 +73,13 @@ def send_mail(subject_line: str, body: str) -> None: if not (api_key and from_email and to_email): log(f" [mail suppressed — email env not set] {subject_line}\n{body}") return - try: - resend.api_key = api_key - resend.Emails.send( - { - "from": f"{project} pipeline <{from_email}>", - "to": to_email, - "subject": f"[{project}] {subject_line}", - "html": f"
{html.escape(body)}
", - } - ) - log(f" Mail sent to {to_email}: {subject_line}") - except Exception as e: - log(f" Mail send failed: {e}") + resend.api_key = api_key + resend.Emails.send( + { + "from": f"{project} pipeline <{from_email}>", + "to": to_email, + "subject": f"[{project}] {subject_line}", + "html": f"
{html.escape(body)}
", + } + ) + log(f" Mail sent to {to_email}: {subject_line}") diff --git a/backend/app/core/jev.py b/backend/app/core/jev.py index 0c4f3b41..1a1cea5d 100644 --- a/backend/app/core/jev.py +++ b/backend/app/core/jev.py @@ -107,6 +107,14 @@ def _answers(body: dict[str, Json], questions: dict[str, Question]) -> dict[str, missing = [qid for qid, a in answers.items() if a is None] if missing: raise JevError(f"unanswered: {', '.join(missing)}") + # "0.93" as a string would read as 0 the same way. + odd = [ + qid + for qid, q in questions.items() + if q["type"] == "noul" and not isinstance(answers[qid], int | float) + ] + if odd: + raise JevError(f"non-numeric: {', '.join(odd)}") return answers diff --git a/backend/app/core/settings.py b/backend/app/core/settings.py index 73f61f46..f7cd4424 100644 --- a/backend/app/core/settings.py +++ b/backend/app/core/settings.py @@ -72,7 +72,7 @@ def SQLALCHEMY_DATABASE_URI(self) -> PostgresDsn: path=self.POSTGRES_DB, ) - # The MCP server (app/mcp/) is gated by this one shared bearer token. Unset + # The MCP server (app/entry/mcp/) is gated by this one shared bearer token. Unset # means every request is rejected, so a deploy that forgets it is closed, # not open. MCP_TOKEN: str | None = None diff --git a/backend/tests/core/test_email.py b/backend/app/core/test_email.py similarity index 85% rename from backend/tests/core/test_email.py rename to backend/app/core/test_email.py index aa15171c..5a9ce8a3 100644 --- a/backend/tests/core/test_email.py +++ b/backend/app/core/test_email.py @@ -1,4 +1,4 @@ -"""The daily mail's send: suppressed without the env, never raising.""" +"""The daily mail's send: suppressed without the env, raising when it fails.""" from __future__ import annotations @@ -47,9 +47,10 @@ def test_the_mail_goes_to_the_alert_address_with_the_body_escaped( @pytest.mark.usefixtures("mail_env") -def test_a_failed_send_does_not_raise(monkeypatch: pytest.MonkeyPatch) -> None: +def test_a_failed_send_raises(monkeypatch: pytest.MonkeyPatch) -> None: def fail(_payload: dict[str, str]) -> None: raise RuntimeError("resend is down") monkeypatch.setattr(resend.Emails, "send", fail) - send_mail("Pipeline needs a look", "body") + with pytest.raises(RuntimeError, match="resend is down"): + send_mail("Pipeline needs a look", "body") diff --git a/backend/tests/core/test_embedding.py b/backend/app/core/test_embedding.py similarity index 100% rename from backend/tests/core/test_embedding.py rename to backend/app/core/test_embedding.py diff --git a/backend/tests/core/test_extract_content.py b/backend/app/core/test_extract.py similarity index 89% rename from backend/tests/core/test_extract_content.py rename to backend/app/core/test_extract.py index a3030b75..27048fb3 100644 --- a/backend/tests/core/test_extract_content.py +++ b/backend/app/core/test_extract.py @@ -17,7 +17,7 @@ ) -def test_a_real_article_body_is_not_a_blocker(): +def test_a_real_article_body_is_not_a_blocker() -> None: assert _is_blocker(_REAL_ARTICLE) is False @@ -28,7 +28,7 @@ def test_a_real_article_body_is_not_a_blocker(): "", ], ) -def test_anything_under_50_chars_is_a_blocker(given): +def test_anything_under_50_chars_is_a_blocker(given: str) -> None: """A teaser must not masquerade as an article regardless of its wording.""" assert _is_blocker(given) is True @@ -45,17 +45,17 @@ def test_anything_under_50_chars_is_a_blocker(given): "Something went wrong loading this page, but don't fret, try again soon.", ], ) -def test_known_blocker_wording_is_flagged(given): +def test_known_blocker_wording_is_flagged(given: str) -> None: assert _is_blocker(given) is True -def test_blocker_wording_is_flagged_regardless_of_casing(): +def test_blocker_wording_is_flagged_regardless_of_casing() -> None: assert ( _is_blocker("PLEASE SIGN IN TO CONTINUE reading this long article now.") is True ) -def test_a_long_article_about_auth_is_not_flagged(): +def test_a_long_article_about_auth_is_not_flagged() -> None: """The patterns target a page THAT IS a wall, not an article about the general topic of authentication — wording matters, not the subject.""" text = ( @@ -68,7 +68,7 @@ def test_a_long_article_about_auth_is_not_flagged(): assert _is_blocker(text) is False -def test_wording_close_to_but_not_matching_a_pattern_is_not_flagged(): +def test_wording_close_to_but_not_matching_a_pattern_is_not_flagged() -> None: """Documents that the gate is substring-driven, not semantic: an article that happens to use the exact phrase "sign in to continue" mid-sentence would be misflagged even outside a real wall. Regression coverage for that diff --git a/backend/tests/core/test_github.py b/backend/app/core/test_github.py similarity index 100% rename from backend/tests/core/test_github.py rename to backend/app/core/test_github.py diff --git a/backend/tests/core/test_jev.py b/backend/app/core/test_jev.py similarity index 58% rename from backend/tests/core/test_jev.py rename to backend/app/core/test_jev.py index d355f898..edd0f3ca 100644 --- a/backend/tests/core/test_jev.py +++ b/backend/app/core/test_jev.py @@ -1,10 +1,12 @@ """The Jev transport's two guarantees: a 429 waits as long as it is told to -and tries again, and an answer set with a hole in it is an error, never a -quiet zero that reads as "not a sponsor" or "off topic". +and tries again, and an answer set with a hole or a non-numeric probability in +it is an error, never a quiet zero that reads as "not a sponsor" or "off topic". """ from __future__ import annotations +from collections.abc import Mapping + import httpx import pytest @@ -16,7 +18,11 @@ } -def _resp(status: int, body: dict | None = None, headers: dict | None = None): +def _resp( + status: int, + body: Mapping[str, object] | None = None, + headers: dict[str, str] | None = None, +) -> httpx.Response: return httpx.Response( status, json=body or {}, @@ -25,11 +31,13 @@ def _resp(status: int, body: dict | None = None, headers: dict | None = None): ) -def _stub(monkeypatch, responses): - sent: list[dict] = [] +def _stub( + monkeypatch: pytest.MonkeyPatch, responses: list[httpx.Response] +) -> tuple[list[dict[str, object]], list[float]]: + sent: list[dict[str, object]] = [] slept: list[float] = [] - def post(payload): + def post(payload: dict[str, object]) -> httpx.Response: sent.append(payload) return responses.pop(0) @@ -38,7 +46,9 @@ def post(payload): return sent, slept -def test_a_429_waits_for_retry_after_then_succeeds(monkeypatch): +def test_a_429_waits_for_retry_after_then_succeeds( + monkeypatch: pytest.MonkeyPatch, +) -> None: ok = { "answers": {"is_sponsor": {"noul": 0.9}, "kind": {"choice": "product"}}, "usage": {"cost": 0.00002}, @@ -55,7 +65,9 @@ def test_a_429_waits_for_retry_after_then_succeeds(monkeypatch): assert reply.cost == pytest.approx(0.00002) -def test_the_request_carries_model_state_and_questions(monkeypatch): +def test_the_request_carries_model_state_and_questions( + monkeypatch: pytest.MonkeyPatch, +) -> None: ok = {"answers": {"is_sponsor": 0.1, "kind": "article"}} sent, _ = _stub(monkeypatch, [_resp(200, ok)]) @@ -68,7 +80,9 @@ def test_the_request_carries_model_state_and_questions(monkeypatch): } -def test_a_429_without_retry_after_backs_off_on_its_own(monkeypatch): +def test_a_429_without_retry_after_backs_off_on_its_own( + monkeypatch: pytest.MonkeyPatch, +) -> None: ok = {"answers": {"is_sponsor": 0.1, "kind": "article"}} _, slept = _stub(monkeypatch, [_resp(429), _resp(429), _resp(200, ok)]) @@ -77,37 +91,51 @@ def test_a_429_without_retry_after_backs_off_on_its_own(monkeypatch): assert slept == [jev.BACKOFF_START_SECS, jev.BACKOFF_START_SECS * 2] -def test_429s_past_the_retry_budget_raise(monkeypatch): +def test_429s_past_the_retry_budget_raise(monkeypatch: pytest.MonkeyPatch) -> None: _stub(monkeypatch, [_resp(429) for _ in range(3)]) with pytest.raises(httpx.HTTPStatusError): jev.ask({}, QUESTIONS, retries=2) -def test_a_missing_answer_raises_instead_of_reading_as_zero(monkeypatch): +def test_a_missing_answer_raises_instead_of_reading_as_zero( + monkeypatch: pytest.MonkeyPatch, +) -> None: _stub(monkeypatch, [_resp(200, {"answers": {"kind": {"choice": "article"}}})]) with pytest.raises(jev.JevError, match="is_sponsor"): jev.ask({}, QUESTIONS) -def test_a_response_with_no_answers_raises(monkeypatch): +def test_a_non_numeric_probability_raises_instead_of_reading_as_zero( + monkeypatch: pytest.MonkeyPatch, +) -> None: + body = {"answers": {"is_sponsor": {"noul": "0.93"}, "kind": "article"}} + _stub(monkeypatch, [_resp(200, body)]) + + with pytest.raises(jev.JevError, match="is_sponsor"): + jev.ask({}, QUESTIONS) + + +def test_a_response_with_no_answers_raises(monkeypatch: pytest.MonkeyPatch) -> None: _stub(monkeypatch, [_resp(200, {"error": "nope"})]) with pytest.raises(jev.JevError): jev.ask({}, QUESTIONS) -def test_a_zero_probability_is_an_answer_not_a_hole(monkeypatch): +def test_a_zero_probability_is_an_answer_not_a_hole( + monkeypatch: pytest.MonkeyPatch, +) -> None: _stub(monkeypatch, [_resp(200, {"answers": {"is_sponsor": 0.0, "kind": "x"}})]) assert jev.ask({}, QUESTIONS).answers["is_sponsor"] == 0.0 -def test_criteria_are_optional_on_a_noul(): +def test_criteria_are_optional_on_a_noul() -> None: assert "criteria" not in jev.noul("sponsor?") -def test_score_needs_two_to_ten_levels(): +def test_score_needs_two_to_ten_levels() -> None: with pytest.raises(ValueError): jev.score("how good?", ["only one"]) diff --git a/backend/tests/core/test_sanitize.py b/backend/app/core/test_llm_text.py similarity index 71% rename from backend/tests/core/test_sanitize.py rename to backend/app/core/test_llm_text.py index 03f98ef3..0433a452 100644 --- a/backend/tests/core/test_sanitize.py +++ b/backend/app/core/test_llm_text.py @@ -1,16 +1,31 @@ -"""Cleanup of model-authored text before it is stored. +"""Normalising an enum-typed BAML field to a plain string, and cleaning +model-authored text before it is stored. Fixtures marked "prod" are real corrupted values pulled from the production -`article` table — the failure modes the pipeline actually produces. +`article` table, the failure modes the pipeline actually produces. """ from __future__ import annotations +from enum import StrEnum + import pytest -from app.core.llm_text import sanitize_llm_text +from app.core.llm_text import enum_value, sanitize_llm_text + + +class _Colour(StrEnum): + red = "red" + + +def test_enum_value_unwraps_an_enum() -> None: + assert enum_value(_Colour.red) == "red" + -# ─── sanitize_llm_text ────────────────────────────────────────────────────── +def test_enum_value_passes_a_plain_string_through() -> None: + """BAML returns plain strings for some fields and enum members for others, + and which one has changed across regenerations.""" + assert enum_value("red") == "red" @pytest.mark.parametrize( diff --git a/backend/tests/core/test_logging.py b/backend/app/core/test_logging.py similarity index 75% rename from backend/tests/core/test_logging.py rename to backend/app/core/test_logging.py index e21fa50f..6e154e0c 100644 --- a/backend/tests/core/test_logging.py +++ b/backend/app/core/test_logging.py @@ -5,11 +5,11 @@ from app.core.logging import short_error -def test_short_error_names_the_type_and_the_message(): +def test_short_error_names_the_type_and_the_message() -> None: assert short_error(ValueError("bad input")) == "ValueError: bad input" -def test_short_error_collapses_a_multi_line_message(): +def test_short_error_collapses_a_multi_line_message() -> None: """A provider error arrives as a wall of prompt and HTML — one line of it is what lands in the run row and the alert email.""" assert short_error(RuntimeError("failed\n attempt 0\n attempt 1")) == ( @@ -17,13 +17,13 @@ def test_short_error_collapses_a_multi_line_message(): ) -def test_short_error_truncates_past_the_limit_and_says_so(): +def test_short_error_truncates_past_the_limit_and_says_so() -> None: out = short_error(RuntimeError("x" * 5000), limit=100) assert out.startswith("RuntimeError: " + "x" * 80) assert out.endswith("[truncated, 5014 chars]") assert len(out) < 200 -def test_short_error_leaves_a_message_at_the_limit_alone(): +def test_short_error_leaves_a_message_at_the_limit_alone() -> None: message = "y" * (100 - len("ValueError: ")) assert short_error(ValueError(message), limit=100) == f"ValueError: {message}" diff --git a/backend/tests/core/test_settings.py b/backend/app/core/test_settings.py similarity index 82% rename from backend/tests/core/test_settings.py rename to backend/app/core/test_settings.py index 899b638f..8b4afbfb 100644 --- a/backend/tests/core/test_settings.py +++ b/backend/app/core/test_settings.py @@ -2,7 +2,7 @@ from __future__ import annotations -import psycopg +import psycopg.conninfo from sqlalchemy.engine import make_url from app.core.settings import settings @@ -10,7 +10,7 @@ PASSWORD = "p@ss/w:rd#% x" -def test_engine_dsn_keeps_reserved_characters_in_the_password(): +def test_engine_dsn_keeps_reserved_characters_in_the_password() -> None: copy = settings.model_copy( update={"POSTGRES_USER": "me@host", "POSTGRES_PASSWORD": PASSWORD} ) @@ -19,14 +19,14 @@ def test_engine_dsn_keeps_reserved_characters_in_the_password(): assert (url.host, url.database) == (settings.POSTGRES_SERVER, settings.POSTGRES_DB) -def test_mcp_dsn_keeps_reserved_characters_in_the_password(): +def test_mcp_dsn_keeps_reserved_characters_in_the_password() -> None: copy = settings.model_copy(update={"MCP_DB_PASSWORD": PASSWORD}) params = psycopg.conninfo.conninfo_to_dict(copy.MCP_DATABASE_URI) assert (params["user"], params["password"]) == (settings.MCP_DB_USER, PASSWORD) assert "default_transaction_read_only=on" in str(params["options"]) -def test_mcp_dsn_without_a_password_has_none(): +def test_mcp_dsn_without_a_password_has_none() -> None: copy = settings.model_copy(update={"MCP_DB_PASSWORD": ""}) params = psycopg.conninfo.conninfo_to_dict(copy.MCP_DATABASE_URI) assert "password" not in params diff --git a/backend/tests/core/test_urls.py b/backend/app/core/test_urls.py similarity index 83% rename from backend/tests/core/test_urls.py rename to backend/app/core/test_urls.py index 375fb285..567464eb 100644 --- a/backend/tests/core/test_urls.py +++ b/backend/app/core/test_urls.py @@ -18,10 +18,10 @@ ("https://www2.example.com", "www2.example.com"), ], ) -def test_hostname_normalises(url, expected): +def test_hostname_normalises(url: str, expected: str) -> None: assert hostname(url) == expected @pytest.mark.parametrize("given", ["", "not a url", "mailto:x@y.com"]) -def test_hostname_returns_empty_when_there_is_no_host(given): +def test_hostname_returns_empty_when_there_is_no_host(given: str) -> None: assert hostname(given) == "" diff --git a/backend/app/entry/api/articles.py b/backend/app/entry/api/articles.py index 268e53aa..1d31d76f 100644 --- a/backend/app/entry/api/articles.py +++ b/backend/app/entry/api/articles.py @@ -7,7 +7,6 @@ from fastapi import APIRouter, HTTPException, Query from app.entry.api.deps import CurrentUserOptional, SessionDep -from app.features.catalog import facets, semantic_search from app.features.catalog.model import ( ArticleFacets, ArticlesPublic, @@ -18,7 +17,16 @@ Voice, VoiceKind, ) -from app.features.catalog.service import list_articles, stats +from app.features.catalog.service import ( + all_facets, + list_articles, + origin_counts, + publisher_facets, + search, + stats, + tag_facets, + voices, +) router = APIRouter(prefix="/articles", tags=["articles"]) @@ -72,7 +80,7 @@ def search_articles( q: str, limit: int = Query(default=20, ge=1, le=50), ) -> ArticlesPublic: - return semantic_search.search( + return search( session, q, limit, viewer_id=current_user.id if current_user else None ) @@ -81,7 +89,7 @@ def search_articles( def article_facets( session: SessionDep, limit: int = Query(default=8, ge=1, le=20) ) -> ArticleFacets: - return facets.all_facets(session, limit) + return all_facets(session, limit) @router.get("/origins", response_model=OriginCounts) @@ -90,7 +98,7 @@ def article_origins(session: SessionDep, since: str) -> OriginCounts: since_dt = datetime.fromisoformat(since) except ValueError: raise HTTPException(status_code=422, detail="Invalid 'since' datetime") - return facets.origin_counts(session, since_dt) + return origin_counts(session, since_dt) @router.get("/voices", response_model=list[Voice]) @@ -104,7 +112,7 @@ def article_voices( since_dt = datetime.fromisoformat(since) except ValueError: raise HTTPException(status_code=422, detail="Invalid 'since' datetime") - return facets.voices(session, kind, since_dt, limit) + return voices(session, kind, since_dt, limit) @router.get("/publishers", response_model=list[PublisherFacet]) @@ -113,7 +121,7 @@ def search_publishers( q: str | None = None, limit: int = Query(default=20, ge=1, le=50), ) -> list[PublisherFacet]: - return facets.publisher_facets(session, q, limit) + return publisher_facets(session, q, limit) @router.get("/tags", response_model=list[TagFacet]) @@ -122,7 +130,7 @@ def search_tags( q: str | None = None, limit: int = Query(default=20, ge=1, le=50), ) -> list[TagFacet]: - return facets.tag_facets(session, q, limit) + return tag_facets(session, q, limit) @router.get("/stats") diff --git a/backend/app/entry/api/login.py b/backend/app/entry/api/login.py index bb7e74a4..952d9312 100644 --- a/backend/app/entry/api/login.py +++ b/backend/app/entry/api/login.py @@ -11,11 +11,6 @@ from app.core.settings import settings from app.entry.api.deps import CurrentUser, SessionDep from app.features.audience import service -from app.features.audience.email import ( - generate_sign_in_email, - generate_welcome_email, - send_link_email, -) from app.features.audience.model import ( MagicLogin, Message, @@ -49,8 +44,12 @@ def request_sign_in_link( elif not user.is_active: return Message(message=LINK_SENT) token = service.issue_login_token(session=session, db_user=user) - email = generate_welcome_email(token) if is_new else generate_sign_in_email(token) - background_tasks.add_task(send_link_email, user.email, email, token) + email = ( + service.generate_welcome_email(token) + if is_new + else service.generate_sign_in_email(token) + ) + background_tasks.add_task(service.send_link_email, user.email, email, token) return Message(message=LINK_SENT) diff --git a/backend/app/features/analytics/tests/test_service.py b/backend/app/entry/api/test_analytics.py similarity index 60% rename from backend/app/features/analytics/tests/test_service.py rename to backend/app/entry/api/test_analytics.py index 4e1f651c..32ef8e64 100644 --- a/backend/app/features/analytics/tests/test_service.py +++ b/backend/app/entry/api/test_analytics.py @@ -1,4 +1,4 @@ -"""Tests for the admin analytics report.""" +"""The analytics endpoints: the ingest beacon and the admin report.""" from __future__ import annotations @@ -7,20 +7,130 @@ import pytest from fastapi.testclient import TestClient -from sqlmodel import Session, delete, select +from sqlmodel import Session, col, delete, select from app.core.dates import get_datetime_utc from app.core.db.event import AnalyticsEvent +from app.core.db.factories import create_random_article, create_random_user from app.core.db.user import User from app.core.settings import settings -from app.features.audience.tests.factories import create_random_user -from app.features.catalog.tests.factories import create_random_article REPORT = f"{settings.API_V1_STR}/analytics/report" BROWSER = "Mozilla/5.0 (Macintosh; Intel Mac OS X 14_0) AppleWebKit/605.1.15" PHONE = "Mozilla/5.0 (iPhone; CPU iPhone OS 18_0 like Mac OS X) Mobile/15E148" +def _latest_event(db: Session, visitor_id: str) -> AnalyticsEvent | None: + return db.exec( + select(AnalyticsEvent) + .where(AnalyticsEvent.visitor_id == visitor_id) + .order_by(col(AnalyticsEvent.id).desc()) + ).first() + + +def test_collect_anonymous_pageview(client: TestClient, db: Session) -> None: + r = client.post( + f"{settings.API_V1_STR}/analytics/collect", + json={"path": "/articles", "visitor_id": "anon-1"}, + ) + assert r.status_code == 204 + + event = _latest_event(db, "anon-1") + assert event is not None + assert event.event == "pageview" + assert event.path == "/articles" + assert event.user_id is None + # user agent captured from the request headers + assert event.user_agent is not None + + +def test_collect_ignores_referer_header(client: TestClient, db: Session) -> None: + r = client.post( + f"{settings.API_V1_STR}/analytics/collect", + headers={"Referer": "https://agentique.ch/feed"}, + json={"path": "/feed", "visitor_id": "anon-5"}, + ) + assert r.status_code == 204 + event = _latest_event(db, "anon-5") + assert event is not None + assert event.referrer is None + + +def test_collect_drops_query_strings(client: TestClient, db: Session) -> None: + r = client.post( + f"{settings.API_V1_STR}/analytics/collect", + json={ + "path": "/auth?token=secret", + "referrer": "https://agentique.ch/reset-password?token=secret#top", + "visitor_id": "anon-6", + }, + ) + assert r.status_code == 204 + event = _latest_event(db, "anon-6") + assert event is not None + assert event.path == "/auth" + assert event.referrer == "https://agentique.ch/reset-password" + + +def test_collect_defaults_event_to_pageview(client: TestClient, db: Session) -> None: + r = client.post( + f"{settings.API_V1_STR}/analytics/collect", + json={"visitor_id": "anon-2"}, + ) + assert r.status_code == 204 + event = _latest_event(db, "anon-2") + assert event is not None + assert event.event == "pageview" + assert event.path is None + + +def test_collect_custom_event_with_props(client: TestClient, db: Session) -> None: + r = client.post( + f"{settings.API_V1_STR}/analytics/collect", + json={ + "event": "article_liked", + "path": "/articles/5", + "visitor_id": "anon-3", + "props": {"article_id": 5}, + }, + ) + assert r.status_code == 204 + event = _latest_event(db, "anon-3") + assert event is not None + assert event.event == "article_liked" + assert event.props == {"article_id": 5} + + +def test_collect_attaches_user_when_authenticated( + client: TestClient, db: Session, normal_user_token_headers: dict[str, str] +) -> None: + r = client.post( + f"{settings.API_V1_STR}/analytics/collect", + headers=normal_user_token_headers, + json={"path": "/profile", "visitor_id": "known-1"}, + ) + assert r.status_code == 204 + + user = db.exec(select(User).where(User.email == settings.EMAIL_TEST_USER)).first() + assert user is not None + event = _latest_event(db, "known-1") + assert event is not None + assert event.user_id == user.id + + +def test_collect_truncates_oversized_strings(client: TestClient, db: Session) -> None: + long_path = "/" + "x" * 5000 + r = client.post( + f"{settings.API_V1_STR}/analytics/collect", + json={"path": long_path, "visitor_id": "anon-4"}, + ) + assert r.status_code == 204 + event = _latest_event(db, "anon-4") + assert event is not None + assert event.path is not None + assert len(event.path) == 2048 + + @pytest.fixture(autouse=True) def empty_events(db: Session) -> Generator[None]: db.exec(delete(AnalyticsEvent)) diff --git a/backend/app/features/catalog/tests/test_routes.py b/backend/app/entry/api/test_articles.py similarity index 96% rename from backend/app/features/catalog/tests/test_routes.py rename to backend/app/entry/api/test_articles.py index 8b2944ed..bf3e11e3 100644 --- a/backend/app/features/catalog/tests/test_routes.py +++ b/backend/app/entry/api/test_articles.py @@ -3,7 +3,6 @@ from __future__ import annotations import uuid -from collections.abc import Generator from dataclasses import dataclass from datetime import UTC, datetime, timedelta @@ -14,21 +13,19 @@ from sqlmodel import Session, col, select from app.core.db.article import Article, ArticleKind -from app.core.db.publisher import PublisherKind -from app.core.security import create_access_token -from app.core.settings import settings -from app.features.audience import service -from app.features.audience.tests.factories import authentication_token_from_email -from app.features.catalog import semantic_search -from app.features.catalog.model import Origin, Voice -from app.features.catalog.tests.factories import ( +from app.core.db.factories import ( + authentication_token_from_email, create_random_article, create_random_publisher, create_random_tag, + random_email, tag_article, ) -from app.main import app -from tests.random_data import random_email +from app.core.db.publisher import PublisherKind +from app.core.security import create_access_token +from app.core.settings import settings +from app.features.catalog import semantic_search +from app.features.catalog.model import Origin, Voice ARTICLES_URL = f"{settings.API_V1_STR}/articles" @@ -36,28 +33,6 @@ RECENT_SINCE = (datetime.now(UTC) - timedelta(days=1)).isoformat() -@pytest.fixture(scope="module") -def reader_token_headers(db: Session) -> dict[str, str]: - """Headers for an ordinary logged-in reader.""" - email = random_email() - headers = authentication_token_from_email(email=email, db=db) - assert service.get_user_by_email(session=db, email=email) is not None - return headers - - -@pytest.fixture(scope="module") -def auth_client( - reader_token_headers: dict[str, str], -) -> Generator[TestClient]: - """A TestClient that sends a logged-in reader's bearer token on every request. - - Reads are public; the token only decides whether `liked_by_me` is filled in. - """ - with TestClient(app) as c: - c.headers.update(reader_token_headers) - yield c - - # --- public reads ------------------------------------------------------------ READ_ENDPOINTS = [ diff --git a/backend/app/features/audience/tests/test_likes.py b/backend/app/entry/api/test_likes.py similarity index 97% rename from backend/app/features/audience/tests/test_likes.py rename to backend/app/entry/api/test_likes.py index 2c4a9a6b..ebc1caf3 100644 --- a/backend/app/features/audience/tests/test_likes.py +++ b/backend/app/entry/api/test_likes.py @@ -9,14 +9,14 @@ from sqlalchemy.exc import IntegrityError from sqlmodel import Session, select -from app.core.db.like import ArticleLike -from app.core.settings import settings -from app.features.audience.tests.factories import ( +from app.core.db.factories import ( authentication_token_from_email, + create_random_article, create_random_user, + random_email, ) -from app.features.catalog.tests.factories import create_random_article -from tests.random_data import random_email +from app.core.db.like import ArticleLike +from app.core.settings import settings def test_article_like_duplicate_pk_raises(db: Session) -> None: diff --git a/backend/app/features/audience/tests/test_login.py b/backend/app/entry/api/test_login.py similarity index 86% rename from backend/app/features/audience/tests/test_login.py rename to backend/app/entry/api/test_login.py index 171a1fcf..e88f856e 100644 --- a/backend/app/features/audience/tests/test_login.py +++ b/backend/app/entry/api/test_login.py @@ -13,16 +13,16 @@ from sqlmodel import Session, select from app.core.dates import get_datetime_utc +from app.core.db.factories import ( + create_random_user, + random_email, + user_authentication_headers, +) from app.core.db.subscriber import NewsletterSubscriber from app.core.db.user import User from app.core.security import ALGORITHM, create_access_token from app.core.settings import settings from app.features.audience import service -from app.features.audience.tests.factories import ( - create_random_user, - user_authentication_headers, -) -from tests.random_data import random_email LINK_SENT = {"message": "Check your inbox for your sign-in link"} @@ -142,6 +142,47 @@ def test_login_with_link_inactive_user(client: TestClient, db: Session) -> None: assert r.json()["detail"] == "Inactive user" +def test_link_request_for_an_inactive_user_sends_nothing( + client: TestClient, db: Session, monkeypatch: pytest.MonkeyPatch +) -> None: + """Same answer as for anyone else, so the endpoint does not tell a + switched-off account apart, but no link and no token.""" + sent = enable_resend_email(monkeypatch) + user = create_random_user(db) + user.is_active = False + db.add(user) + db.commit() + + r = client.post(f"{settings.API_V1_STR}/login/link", json={"email": user.email}) + assert r.json() == LINK_SENT + assert sent == [] + db.refresh(user) + assert user.login_token is None + + +def test_an_access_token_for_an_inactive_user_is_refused( + client: TestClient, db: Session +) -> None: + user = create_random_user(db) + headers = user_authentication_headers(user) + user.is_active = False + db.add(user) + db.commit() + + r = client.post(f"{settings.API_V1_STR}/login/test-token", headers=headers) + assert r.status_code == 400 + assert r.json()["detail"] == "Inactive user" + + +def test_an_access_token_for_a_deleted_user_is_refused(client: TestClient) -> None: + token = create_access_token(uuid.uuid4(), timedelta(minutes=5)) + r = client.post( + f"{settings.API_V1_STR}/login/test-token", + headers={"Authorization": f"Bearer {token}"}, + ) + assert r.status_code == 404 + + def test_use_access_token( client: TestClient, superuser_token_headers: dict[str, str] ) -> None: diff --git a/backend/app/entry/api/test_stories.py b/backend/app/entry/api/test_stories.py new file mode 100644 index 00000000..cd0d5d43 --- /dev/null +++ b/backend/app/entry/api/test_stories.py @@ -0,0 +1,46 @@ +"""The stories rail endpoint.""" + +from __future__ import annotations + +from datetime import UTC, datetime, timedelta + +from fastapi.testclient import TestClient +from sqlmodel import Session + +from app.core.db.factories import ( + create_random_article, + create_random_publisher, + random_lower_string, +) +from app.core.settings import settings +from app.features.stories import service as stories + + +def test_the_route_serves_the_list( + client: TestClient, db: Session, made: dict[str, list[int]] +) -> None: + now = datetime.now(UTC) + publishers = [create_random_publisher(db), create_random_publisher(db)] + urls = [] + for i, publisher in enumerate([*publishers, publishers[0]]): + article = create_random_article( + db, publisher_id=publisher.id, published_at=now - timedelta(days=3 - i) + ) + assert article.id is not None + made["articles"].append(article.id) + urls.append(article.url) + made["publishers"] += [p.id for p in publishers if p.id is not None] + slug = f"served-{random_lower_string()[:6]}" + stories.save_story(db, slug, "Served", "What it is. Why now.", urls) + + r = client.get(f"{settings.API_V1_STR}/stories/", params={"limit": 20}) + assert r.status_code == 200 + story = next(s for s in r.json() if s["slug"] == slug) + assert set(story["articles"][0]) == { + "id", + "title", + "url", + "publisher", + "published_at", + "score", + } diff --git a/backend/app/features/audience/tests/test_users.py b/backend/app/entry/api/test_users.py similarity index 97% rename from backend/app/features/audience/tests/test_users.py rename to backend/app/entry/api/test_users.py index 360e8346..b1e31cf2 100644 --- a/backend/app/features/audience/tests/test_users.py +++ b/backend/app/entry/api/test_users.py @@ -5,13 +5,13 @@ from fastapi.testclient import TestClient from sqlmodel import Session, select -from app.core.db.user import User -from app.core.settings import settings -from app.features.audience.tests.factories import ( +from app.core.db.factories import ( create_random_user, + random_email, user_authentication_headers, ) -from tests.random_data import random_email +from app.core.db.user import User +from app.core.settings import settings def test_get_users_superuser_me( diff --git a/backend/app/features/analytics/tests/__init__.py b/backend/app/entry/cli/__init__.py similarity index 100% rename from backend/app/features/analytics/tests/__init__.py rename to backend/app/entry/cli/__init__.py diff --git a/backend/scripts/backfill_publishers.py b/backend/app/entry/cli/backfill_publishers.py similarity index 96% rename from backend/scripts/backfill_publishers.py rename to backend/app/entry/cli/backfill_publishers.py index 3e139997..9264c5e6 100644 --- a/backend/scripts/backfill_publishers.py +++ b/backend/app/entry/cli/backfill_publishers.py @@ -7,9 +7,9 @@ is no longer an aggregator's, and a publisher already made is found by its link. Run it against a restored dump first and read the report. - python scripts/backfill_publishers.py # re-credit + lab kinds - python scripts/backfill_publishers.py --export unknown.csv - python scripts/backfill_publishers.py --kinds kinds.csv + python -m app.entry.cli.backfill_publishers # re-credit + lab kinds + python -m app.entry.cli.backfill_publishers --export unknown.csv + python -m app.entry.cli.backfill_publishers --kinds kinds.csv `--export` writes the `unknown` publishers that have published articles, with titles and summaries to judge them by. Fill in `kind` (lab, company, @@ -31,7 +31,7 @@ from app.core.db.engine import engine from app.core.db.ledger import Reject from app.core.db.publisher import LinkPlatform, Publisher, PublisherKind -from app.features.collect.publishers import AGGREGATOR_TYPES, PublisherResolver +from app.features.collect.service import AGGREGATOR_TYPES, PublisherResolver from app.features.curation.service import NAMEABLE_KINDS # Publishers that train and release their own models. Picked by hand from the diff --git a/backend/scripts/backup_db.py b/backend/app/entry/cli/backup_db.py similarity index 90% rename from backend/scripts/backup_db.py rename to backend/app/entry/cli/backup_db.py index 05edb8f8..c9fb71fe 100644 --- a/backend/scripts/backup_db.py +++ b/backend/app/entry/cli/backup_db.py @@ -1,4 +1,4 @@ -"""Nightly Postgres backup to kDrive. Run with: python scripts/backup_db.py +"""Nightly Postgres backup to kDrive. Run with: python app/entry/cli/backup_db.py pg_dump runs inside the pgvector container (`docker compose exec`) so the client always matches the server version and nothing Postgres needs installing @@ -23,7 +23,7 @@ from kdrive_client.kdrive_client import KDriveClient # type: ignore[import-untyped] from kdrive_client.kdrive_file import KDriveFile # type: ignore[import-untyped] -COMPOSE_FILE = Path(__file__).resolve().parents[2] / "compose.yml" +COMPOSE_FILE = Path(__file__).resolve().parents[4] / "compose.yml" BACKUP_DIR = os.environ.get("KDRIVE_BACKUP_DIR", "/Private/agentique-db") KEEP = int(os.environ.get("KDRIVE_BACKUP_KEEP", "14")) PREFIX = "agentique-db-" @@ -53,8 +53,10 @@ def dump_db(dest: Path) -> None: proc.stdout.close() if proc.wait() != 0: raise RuntimeError(f"pg_dump exited with {proc.returncode}") - if dest.stat().st_size == 0: - raise RuntimeError("dump is empty") + # The uncompressed size: a gzip of nothing still has a header, so the + # file on disk is never 0 bytes. + if out.tell() == 0: + raise RuntimeError("dump is empty") def prune(client: KDriveClient, directory_id: int) -> None: diff --git a/backend/pipeline/run.py b/backend/app/entry/cli/collect.py similarity index 88% rename from backend/pipeline/run.py rename to backend/app/entry/cli/collect.py index 97a0558a..c240956f 100644 --- a/backend/pipeline/run.py +++ b/backend/app/entry/cli/collect.py @@ -1,4 +1,4 @@ -"""The nightly collect run, as the systemd unit starts it: python -m pipeline.run""" +"""The nightly collect run, as the systemd unit starts it: python -m app.entry.cli.collect""" from __future__ import annotations diff --git a/backend/scripts/create_superuser.py b/backend/app/entry/cli/create_superuser.py similarity index 100% rename from backend/scripts/create_superuser.py rename to backend/app/entry/cli/create_superuser.py diff --git a/backend/scripts/find_missing_tags.py b/backend/app/entry/cli/find_missing_tags.py similarity index 86% rename from backend/scripts/find_missing_tags.py rename to backend/app/entry/cli/find_missing_tags.py index 5744b377..64e11c28 100644 --- a/backend/scripts/find_missing_tags.py +++ b/backend/app/entry/cli/find_missing_tags.py @@ -11,14 +11,14 @@ candidates for a missing tag (e.g. text-to-speech, if no such tag exists). We group those under-covered articles into clusters and print each cluster with a few representative titles and its nearest existing tag, so a human can name the -gap and add it to `scripts/tags.json`. +gap and add it to `app/entry/cli/tags.json`. No LLM calls: embeddings are already in the DB, tag vectors are one local encode of ~30 tag strings, and clustering is greedy cosine math. Run it against prod by pointing the POSTGRES_* env vars at prod (same vars the pipeline reads). - cd backend && python -m scripts.find_missing_tags - cd backend && python -m scripts.find_missing_tags --threshold 0.30 --json + cd backend && python -m app.entry.cli.find_missing_tags + cd backend && python -m app.entry.cli.find_missing_tags --threshold 0.30 --json Tune `--threshold` to what "poorly covered" means for your data: run once, look at the coverage percentiles it prints, then set the threshold near the low tail. @@ -34,19 +34,23 @@ from typing import TypedDict import numpy as np -from sqlmodel import Session, select +import numpy.typing as npt +from sqlmodel import Session, col, select from app.core.db.article import Article from app.core.db.engine import engine from app.core.db.tag import ArticleTag, Tag from app.core.embedding import embed_batch +# Rows of float32 vectors, or one vector. +type Vectors = npt.NDArray[np.float32] + @dataclass class ArticleVec: id: int title: str - vec: np.ndarray # L2-normalized + vec: Vectors # L2-normalized tags: list[str] = field(default_factory=list) @@ -59,7 +63,7 @@ class GapCluster(TypedDict): sample_titles: list[str] -def normalize(m: np.ndarray) -> np.ndarray: +def normalize(m: Vectors) -> Vectors: """L2-normalize rows; leave all-zero rows at zero (cosine -> 0).""" norms = np.linalg.norm(m, axis=1, keepdims=True) norms[norms == 0] = 1.0 @@ -71,7 +75,7 @@ def tag_text(name: str, description: str | None) -> str: return f"{name}: {description}" if description else name -def load_tags(session: Session) -> tuple[list[str], list[str], np.ndarray]: +def load_tags(session: Session) -> tuple[list[str], list[str], Vectors]: """Return (slugs, names, normalized tag vectors), aligned by index.""" rows = session.exec(select(Tag)).all() if not rows: @@ -86,10 +90,10 @@ def load_articles( session: Session, since: datetime | None, limit: int ) -> list[ArticleVec]: """Articles that have an embedding, with their assigned tag slugs attached.""" - stmt = select(Article).where(Article.embedding.is_not(None)) # type: ignore[union-attr] # ty: ignore[unresolved-attribute] + stmt = select(Article).where(col(Article.embedding).is_not(None)) if since is not None: - stmt = stmt.where(Article.published_at >= since) # type: ignore[operator] # ty: ignore[unsupported-operator] - stmt = stmt.order_by(Article.id) # type: ignore[arg-type] # ty: ignore[invalid-argument-type] + stmt = stmt.where(col(Article.published_at) >= since) + stmt = stmt.order_by(col(Article.id)) if limit > 0: stmt = stmt.limit(limit) rows = session.exec(stmt).all() @@ -99,8 +103,8 @@ def load_articles( if ids: pairs = session.exec( select(ArticleTag.article_id, Tag.slug) - .join(Tag, Tag.id == ArticleTag.tag_id) # type: ignore[arg-type] # ty: ignore[invalid-argument-type] - .where(ArticleTag.article_id.in_(ids)) # type: ignore[attr-defined] # ty: ignore[unresolved-attribute] + .join(Tag, col(Tag.id) == col(ArticleTag.tag_id)) + .where(col(ArticleTag.article_id).in_(ids)) ).all() for aid, slug in pairs: tags_by_article.setdefault(aid, []).append(slug) @@ -118,16 +122,17 @@ def load_articles( @dataclass class Cluster: - centroid: np.ndarray # normalized running mean + centroid: Vectors # normalized running mean members: list[int] # indices into the under-covered list - def add(self, idx: int, vec: np.ndarray) -> None: + def add(self, idx: int, vec: Vectors) -> None: n = len(self.members) - self.centroid = normalize(((self.centroid * n + vec) / (n + 1))[None, :])[0] + mean = (self.centroid * np.float32(n) + vec) / np.float32(n + 1) + self.centroid = normalize(mean[None, :])[0] self.members.append(idx) -def greedy_cluster(vecs: np.ndarray, sim_threshold: float) -> list[Cluster]: +def greedy_cluster(vecs: Vectors, sim_threshold: float) -> list[Cluster]: """One-pass greedy cosine clustering. Each vector joins the most similar existing cluster if it clears the threshold, else it seeds a new one. Good enough to surface coherent themes; not trying to be k-means.""" @@ -265,8 +270,8 @@ def main() -> None: print(f" - {t}") print() print( - "Next: name each real gap and add it to scripts/tags.json, then " - "re-run `python scripts/seed_tags.py`." + "Next: name each real gap and add it to app/entry/cli/tags.json, then " + "re-run `python -m app.entry.cli.seed_tags`." ) diff --git a/backend/scripts/prestart.sh b/backend/app/entry/cli/prestart.sh similarity index 67% rename from backend/scripts/prestart.sh rename to backend/app/entry/cli/prestart.sh index b23ac4a5..49c36609 100755 --- a/backend/scripts/prestart.sh +++ b/backend/app/entry/cli/prestart.sh @@ -4,18 +4,18 @@ set -e set -x # Let the DB start -python scripts/wait_for_db.py +python -m app.entry.cli.wait_for_db # Run migrations alembic upgrade head # Create initial data in DB -python scripts/create_superuser.py +python -m app.entry.cli.create_superuser # Load the tag vocabulary (all environments; the pipeline requires it) -python scripts/seed_tags.py +python -m app.entry.cli.seed_tags # Seed sample articles for local/CI (module also self-guards on production) if [ "$ENVIRONMENT" != "production" ]; then - python scripts/seed_articles.py + python -m app.entry.cli.seed_articles fi diff --git a/backend/pipeline/report.py b/backend/app/entry/cli/report.py similarity index 65% rename from backend/pipeline/report.py rename to backend/app/entry/cli/report.py index 554f4a07..5aa1662b 100644 --- a/backend/pipeline/report.py +++ b/backend/app/entry/cli/report.py @@ -1,4 +1,4 @@ -"""The daily mail, as the systemd unit starts it: python -m pipeline.report""" +"""The daily mail, as the systemd unit starts it: python -m app.entry.cli.report""" from __future__ import annotations diff --git a/backend/scripts/seed_articles.py b/backend/app/entry/cli/seed_articles.py similarity index 100% rename from backend/scripts/seed_articles.py rename to backend/app/entry/cli/seed_articles.py diff --git a/backend/scripts/seed_stories.py b/backend/app/entry/cli/seed_stories.py similarity index 97% rename from backend/scripts/seed_stories.py rename to backend/app/entry/cli/seed_stories.py index 4dcbe2db..9fce595c 100644 --- a/backend/scripts/seed_stories.py +++ b/backend/app/entry/cli/seed_stories.py @@ -21,6 +21,7 @@ from app.core.db.engine import engine from app.core.settings import settings from app.features.stories import service as stories +from app.features.stories.model import StoryError logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) @@ -98,7 +99,7 @@ def main() -> None: story["blurb"], story["urls"], ) - except stories.StoryError as exc: + except StoryError as exc: session.rollback() logger.warning("Skipped %s: %s", story["slug"], exc) continue diff --git a/backend/scripts/seed_tags.py b/backend/app/entry/cli/seed_tags.py similarity index 94% rename from backend/scripts/seed_tags.py rename to backend/app/entry/cli/seed_tags.py index 22a5b195..1314f9aa 100644 --- a/backend/scripts/seed_tags.py +++ b/backend/app/entry/cli/seed_tags.py @@ -21,7 +21,8 @@ def load_tags() -> list[dict[str, str]]: with TAGS_FILE.open() as f: - return json.load(f)["tags"] + tags: list[dict[str, str]] = json.load(f)["tags"] + return tags def seed(session: Session) -> None: diff --git a/backend/scripts/tags.json b/backend/app/entry/cli/tags.json similarity index 100% rename from backend/scripts/tags.json rename to backend/app/entry/cli/tags.json diff --git a/backend/app/entry/cli/test_backup_db.py b/backend/app/entry/cli/test_backup_db.py new file mode 100644 index 00000000..59e465b3 --- /dev/null +++ b/backend/app/entry/cli/test_backup_db.py @@ -0,0 +1,54 @@ +"""The nightly database backup: where it finds compose, and what it refuses to keep.""" + +from __future__ import annotations + +import subprocess +from pathlib import Path + +import pytest + +from app.entry.cli import backup_db + +REPO = Path(__file__).resolve().parents[4] + + +def test_the_backup_finds_the_compose_file() -> None: + assert backup_db.COMPOSE_FILE == REPO / "compose.yml" + assert backup_db.COMPOSE_FILE.is_file() + + +def _fake_pg_dump(monkeypatch: pytest.MonkeyPatch, script: str) -> None: + """Stand in for `docker compose exec db pg_dump` with a shell one-liner.""" + monkeypatch.setenv("POSTGRES_USER", "postgres") + monkeypatch.setenv("POSTGRES_DB", "app") + real = subprocess.Popen + + def popen(_cmd: list[str], stdout: int) -> subprocess.Popen[bytes]: + return real(["sh", "-c", script], stdout=stdout) + + monkeypatch.setattr(subprocess, "Popen", popen) + + +def test_a_failed_pg_dump_fails_the_backup( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + _fake_pg_dump(monkeypatch, "echo partial; exit 3") + with pytest.raises(RuntimeError, match="exited with 3"): + backup_db.dump_db(tmp_path / "dump.sql.gz") + + +def test_an_empty_dump_fails_the_backup( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + _fake_pg_dump(monkeypatch, "true") + with pytest.raises(RuntimeError, match="empty"): + backup_db.dump_db(tmp_path / "dump.sql.gz") + + +def test_a_dump_with_output_is_kept( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + _fake_pg_dump(monkeypatch, "echo 'CREATE TABLE x ();'") + dest = tmp_path / "dump.sql.gz" + backup_db.dump_db(dest) + assert dest.stat().st_size > 0 diff --git a/backend/app/entry/cli/test_collect.py b/backend/app/entry/cli/test_collect.py new file mode 100644 index 00000000..5ced33e6 --- /dev/null +++ b/backend/app/entry/cli/test_collect.py @@ -0,0 +1,35 @@ +"""The nightly collect timer's entrypoint.""" + +from __future__ import annotations + +import runpy +import subprocess +import sys +from pathlib import Path + +import pytest + +from app.features.collect import service as collect + +BACKEND = Path(__file__).resolve().parents[3] + + +@pytest.mark.parametrize("code", [0, 1]) +def test_the_entrypoint_exits_with_the_runs_code( + monkeypatch: pytest.MonkeyPatch, code: int +) -> None: + monkeypatch.setattr(collect, "run_nightly", lambda: code) + with pytest.raises(SystemExit) as exited: + runpy.run_module("app.entry.cli.collect", run_name="__main__") + assert exited.value.code == code + + +def test_the_collect_run_leaves_the_root_logger_alone() -> None: + # At INFO, httpx logs a line for every request the night makes and buries + # the run's own progress in the journal. + probe = ( + "import logging, app.entry.cli.collect; " + "root = logging.getLogger(); " + "assert root.level == logging.WARNING and not root.handlers, root" + ) + subprocess.run([sys.executable, "-c", probe], cwd=BACKEND, check=True) diff --git a/backend/app/entry/cli/test_deploy_files.py b/backend/app/entry/cli/test_deploy_files.py new file mode 100644 index 00000000..ba60b62c --- /dev/null +++ b/backend/app/entry/cli/test_deploy_files.py @@ -0,0 +1,155 @@ +"""What the deploy ships: the workflow's rsync run against this checkout's +tracked files, then every path the units and the deploy job use looked up in +the result.""" + +from __future__ import annotations + +import re +import shlex +import shutil +import subprocess +import sys +import tomllib +from pathlib import Path + +import pytest + +BACKEND = Path(__file__).resolve().parents[3] +REPO = BACKEND.parent +BOX = Path("/opt/agentique") +UNITS = sorted((REPO / "deploy").glob("*.service")) +WORKFLOW = REPO / ".github/workflows/deploy-production.yml" +# Never shipped: the box keeps its own copy, and the filter protects it. +BOX_OWNED = (BOX / ".env", BOX / ".venv") + + +def rsync_command() -> list[str]: + lines = [ + line.strip() + for line in WORKFLOW.read_text().splitlines() + if line.strip().startswith("rsync ") + ] + assert len(lines) == 1, lines + return shlex.split(lines[0]) + + +@pytest.fixture(scope="module") +def shipped(tmp_path_factory: pytest.TempPathFactory) -> Path: + version = subprocess.run( + ["rsync", "--version"], capture_output=True, text=True, check=False + ).stdout + if "version 3." not in version: + pytest.skip("the filter needs rsync 3, as on the box") + # The deploy syncs a clean checkout, so untracked files must not count. + source = tmp_path_factory.mktemp("checkout") + tracked = subprocess.run( + ["git", "ls-files", "-z"], cwd=REPO, capture_output=True, check=True + ).stdout.decode() + for name in filter(None, tracked.split("\0")): + if (REPO / name).is_file(): + (source / name).parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(REPO / name, source / name) + (source / "frontend/dist").mkdir(parents=True) + (source / "frontend/dist/index.html").touch() + + dest = tmp_path_factory.mktemp("box") + command = rsync_command() + assert command[-1] == f"{BOX}/", command + subprocess.run([*command[:-1], f"{dest}/"], cwd=source, check=True) + return dest + + +def on_box(shipped: Path, path: Path) -> Path: + return shipped / path.relative_to(BOX) + + +def box_owned(path: Path) -> bool: + return any(path == owned or owned in path.parents for owned in BOX_OWNED) + + +@pytest.mark.parametrize("unit", UNITS, ids=lambda p: p.name) +def test_every_path_a_unit_uses_is_shipped(shipped: Path, unit: Path) -> None: + config = dict( + line.split("=", 1) + for line in unit.read_text().splitlines() + if "=" in line and not line.startswith("#") + ) + workdir = on_box(shipped, Path(config["WorkingDirectory"])) + assert workdir.is_dir(), workdir + assert box_owned(Path(config["EnvironmentFile"])) + # systemd's -, @, + and ! prefixes change how it runs, not what. + command, *args = config["ExecStart"].lstrip("-@+!:").split() + assert box_owned(Path(command)), command + if args[:1] == ["-m"]: + module = workdir / args[1].replace(".", "/") + assert module.with_suffix(".py").is_file() or (module / "__main__.py").is_file() + for arg in args: + if arg.endswith(".py"): + assert (workdir / arg).is_file(), arg + + +def test_every_directory_the_deploy_job_works_in_is_shipped(shipped: Path) -> None: + dirs = re.findall(r"working-directory: (/opt/agentique\S*)", WORKFLOW.read_text()) + assert dirs + for workdir in dirs: + assert on_box(shipped, Path(workdir)).is_dir(), workdir + for script in re.findall(r"bash (\S+\.sh)", WORKFLOW.read_text()): + assert (on_box(shipped, BOX / "backend") / script).is_file(), script + + +def test_prestart_runs_only_shipped_modules(shipped: Path) -> None: + prestart = shipped / "backend/app/entry/cli/prestart.sh" + for module in re.findall(r"python -m ([\w.]+)", prestart.read_text()): + assert (shipped / "backend" / (module.replace(".", "/") + ".py")).is_file() + + +def test_uv_sync_and_alembic_find_their_files(shipped: Path) -> None: + root = tomllib.loads((shipped / "pyproject.toml").read_text()) + for member in root["tool"]["uv"]["workspace"]["members"]: + assert (shipped / member / "pyproject.toml").is_file(), member + assert (shipped / "uv.lock").is_file() + assert (shipped / ".python-version").is_file() + assert (shipped / "backend/migrations/env.py").is_file() + # backup_db.py runs docker compose against the repo root's compose.yml. + assert (shipped / "compose.yml").is_file() + + +def test_every_data_file_in_app_is_shipped(shipped: Path) -> None: + data = [ + p.relative_to(REPO) + for p in (BACKEND / "app").rglob("*") + if p.is_file() + and p.suffix not in {".py", ".pyc"} + and not p.name.startswith("test_") + ] + assert data + for path in data: + assert (shipped / path).is_file(), path + + +def test_no_test_code_is_shipped(shipped: Path) -> None: + names = {p.name for p in (shipped / "backend").rglob("*.py")} + assert not [n for n in names if n.startswith("test_")] + assert "conftest.py" not in names + assert "factories.py" not in names + + +def test_every_shipped_module_imports_from_the_shipped_tree(shipped: Path) -> None: + # Run from the shipped backend/, as the units do. The venv's editable + # install still points at the checkout, so a module the filter drops would + # load from there instead. Fail on any module that did. + env_file = REPO / ".env" + if env_file.is_file(): + shutil.copy(env_file, shipped / ".env") + script = f""" +import importlib, pkgutil, sys +import app +for module in pkgutil.walk_packages(app.__path__, "app."): + importlib.import_module(module.name) +checkout = [ + name for name, module in sys.modules.items() + if (getattr(module, "__file__", None) or "").startswith({f"{BACKEND}/"!r}) +] +assert not checkout, checkout +""" + subprocess.run([sys.executable, "-c", script], cwd=shipped / "backend", check=True) diff --git a/backend/app/entry/cli/test_entrypoints.py b/backend/app/entry/cli/test_entrypoints.py new file mode 100644 index 00000000..0c0f8320 --- /dev/null +++ b/backend/app/entry/cli/test_entrypoints.py @@ -0,0 +1,40 @@ +"""What systemd, the deploy and prestart.sh run: that each target exists.""" + +from __future__ import annotations + +import importlib.util +import re +from pathlib import Path + +import pytest + +BACKEND = Path(__file__).resolve().parents[3] +REPO = BACKEND.parent +SHELL_SCRIPTS = ( + BACKEND / "app/entry/cli/prestart.sh", + REPO / "scripts/tests-start.sh", +) + + +@pytest.mark.parametrize( + "unit", sorted((REPO / "deploy").glob("*.service")), ids=lambda p: p.name +) +def test_every_python_unit_starts_something_that_exists(unit: Path) -> None: + # The units run from /opt/agentique/backend. A renamed module fails there + # only when the timer fires, at night. + for line in unit.read_text().splitlines(): + if not line.startswith("ExecStart=") or "/python " not in line: + continue + args = line.split()[1:] + if args[0] == "-m": + assert importlib.util.find_spec(args[1]) is not None, line + else: + assert (BACKEND / args[0]).is_file(), line + + +@pytest.mark.parametrize("script", SHELL_SCRIPTS, ids=lambda p: p.name) +def test_every_module_a_shell_script_runs_exists(script: Path) -> None: + targets = re.findall(r"python -m ([\w.]+)", script.read_text()) + assert targets + for target in targets: + assert importlib.util.find_spec(target) is not None, target diff --git a/backend/app/entry/cli/test_report.py b/backend/app/entry/cli/test_report.py new file mode 100644 index 00000000..5a0792d9 --- /dev/null +++ b/backend/app/entry/cli/test_report.py @@ -0,0 +1,19 @@ +"""The daily report timer's entrypoint.""" + +from __future__ import annotations + +import runpy + +import pytest + +from app.features.collect import service as collect + + +@pytest.mark.parametrize("code", [0, 1]) +def test_the_entrypoint_exits_with_the_reports_code( + monkeypatch: pytest.MonkeyPatch, code: int +) -> None: + monkeypatch.setattr(collect, "send_report", lambda: code) + with pytest.raises(SystemExit) as exited: + runpy.run_module("app.entry.cli.report", run_name="__main__") + assert exited.value.code == code diff --git a/backend/app/entry/cli/test_seed_articles.py b/backend/app/entry/cli/test_seed_articles.py new file mode 100644 index 00000000..46dcb8dd --- /dev/null +++ b/backend/app/entry/cli/test_seed_articles.py @@ -0,0 +1,14 @@ +"""The sample articles seed, which must never run against production.""" + +from __future__ import annotations + +import pytest + +from app.core.settings import settings +from app.entry.cli import seed_articles + + +def test_the_seed_refuses_production(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(settings, "ENVIRONMENT", "production") + with pytest.raises((RuntimeError, SystemExit), match="Refusing"): + seed_articles.main() diff --git a/backend/app/entry/cli/test_seed_stories.py b/backend/app/entry/cli/test_seed_stories.py new file mode 100644 index 00000000..f9687c37 --- /dev/null +++ b/backend/app/entry/cli/test_seed_stories.py @@ -0,0 +1,14 @@ +"""The sample stories seed, which must never run against production.""" + +from __future__ import annotations + +import pytest + +from app.core.settings import settings +from app.entry.cli import seed_stories + + +def test_the_seed_refuses_production(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(settings, "ENVIRONMENT", "production") + with pytest.raises((RuntimeError, SystemExit), match="Refusing"): + seed_stories.main() diff --git a/backend/tests/scripts/test_wait_for_db.py b/backend/app/entry/cli/test_wait_for_db.py similarity index 80% rename from backend/tests/scripts/test_wait_for_db.py rename to backend/app/entry/cli/test_wait_for_db.py index a9aca870..28c7eb48 100644 --- a/backend/tests/scripts/test_wait_for_db.py +++ b/backend/app/entry/cli/test_wait_for_db.py @@ -6,7 +6,7 @@ from sqlmodel import select -from scripts.wait_for_db import init, logger +from app.entry.cli.wait_for_db import init, logger def test_init_successful_connection() -> None: @@ -18,8 +18,8 @@ def test_init_successful_connection() -> None: select1 = select(1) with ( - patch("scripts.wait_for_db.Session", return_value=session_mock), - patch("scripts.wait_for_db.select", return_value=select1), + patch("app.entry.cli.wait_for_db.Session", return_value=session_mock), + patch("app.entry.cli.wait_for_db.select", return_value=select1), patch.object(logger, "info"), patch.object(logger, "error"), patch.object(logger, "warn"), diff --git a/backend/scripts/wait_for_db.py b/backend/app/entry/cli/wait_for_db.py similarity index 100% rename from backend/scripts/wait_for_db.py rename to backend/app/entry/cli/wait_for_db.py diff --git a/backend/app/features/audience/tests/__init__.py b/backend/app/entry/mcp/__init__.py similarity index 100% rename from backend/app/features/audience/tests/__init__.py rename to backend/app/entry/mcp/__init__.py diff --git a/backend/app/entry/mcp/approve.py b/backend/app/entry/mcp/approve.py new file mode 100644 index 00000000..8481ec88 --- /dev/null +++ b/backend/app/entry/mcp/approve.py @@ -0,0 +1,53 @@ +"""The `approve` tool: publish a candidate.""" + +from __future__ import annotations + +from fastmcp.exceptions import ToolError +from sqlmodel import Session + +from app.core.db.engine import engine +from app.entry.mcp.auth import require_write +from app.features.curation import service as curation +from app.features.curation.model import CandidateError + + +def approve( + url: str, + score: int, + reason: str, + summary: str, + kind: str, + tags: list[str], + publisher_kind: str | None = None, + publisher_name: str | None = None, +) -> str: + """Publish a candidate with the labels you chose, then embed it. + + `score` is 1-100 on the same scale the rubric describes, `reason` one short + sentence naming what decided it, and `summary` the text a reader sees under + the title. `kind` and `tags` (up to 3) come from `vocabulary`; a github, + huggingface or arxiv URL sets its own kind. When the candidate's + `publisher_kind` is `unknown`, pass `publisher_kind` (from + `vocabulary`'s `publisher_kinds`) and, if its host is not a readable name, + `publisher_name`; both are ignored for a publisher that has a kind. A label + that is not in the vocabulary is refused and the candidate stays pending. + The candidate stops being pending in the same transaction that inserts the + article, so nothing is ever published twice. + """ + require_write() + with Session(engine) as session: + try: + article_id = curation.approve( + session, + url, + score, + reason, + summary, + kind, + tags, + publisher_kind, + publisher_name, + ) + except CandidateError as exc: + raise ToolError(str(exc)) + return f"Published article #{article_id}: {url}" diff --git a/backend/app/entry/mcp/auth.py b/backend/app/entry/mcp/auth.py new file mode 100644 index 00000000..0905c9a1 --- /dev/null +++ b/backend/app/entry/mcp/auth.py @@ -0,0 +1,64 @@ +"""The bearer-token gate: two tokens, and the scope only the write token carries. + +`sql_query`, `web_fetch` and `web_search` read and are reachable with +`MCP_TOKEN`. The curation tools publish and reject articles, the newsletter +tools draft emails addressed to every subscriber, and the story tools name what +the right rail shows; all of them call `require_write` first. +""" + +from __future__ import annotations + +import secrets + +from fastmcp.exceptions import ToolError +from fastmcp.server.auth import AccessToken, TokenVerifier +from fastmcp.server.dependencies import get_access_token + +from app.core.settings import settings + +# The scope `MCP_WRITE_TOKEN` carries and the read token does not. Named for +# what it lets a caller do, not for the token that carries it. +WRITE_SCOPE = "curate" + + +def _matches(token: str, expected: str | None) -> bool: + """Constant-time compare against a configured token, if there is one. + + Bytes, not str: compare_digest raises on non-ASCII str operands, and the + token is whatever someone put in the header. An unset value matches + nothing, so a deploy that forgets the variable serves 401s instead of an + open database. + """ + return bool(expected) and secrets.compare_digest( + token.encode(), (expected or "").encode() + ) + + +class SharedSecret(TokenVerifier): + """Two bearer tokens, compared in constant time, one scope between them. + + There are two clients — a read-only agent answering questions about the + feed, and the curation agent that publishes to it — so an OAuth provider + would be ceremony around two values that already live in the same `.env` as + every other credential. The write token is checked first and is the only one + that comes back carrying `WRITE_SCOPE`; the read token reaches the three + reading tools and is refused by the rest. + """ + + async def verify_token(self, token: str) -> AccessToken | None: + if _matches(token, settings.MCP_WRITE_TOKEN): + return AccessToken( + token=token, client_id="agentique-curator", scopes=[WRITE_SCOPE] + ) + if _matches(token, settings.MCP_TOKEN): + return AccessToken(token=token, client_id="agentique-mcp", scopes=[]) + return None + + +def require_write() -> None: + """Refuse a write to a caller holding only the read token.""" + token = get_access_token() + if token is None or WRITE_SCOPE not in token.scopes: + raise ToolError( + "This tool needs the curation token — the read token cannot write." + ) diff --git a/backend/app/entry/mcp/check_link.py b/backend/app/entry/mcp/check_link.py new file mode 100644 index 00000000..bee0e17c --- /dev/null +++ b/backend/app/entry/mcp/check_link.py @@ -0,0 +1,26 @@ +"""The `check_link` tool: the facts behind a repo or model URL.""" + +from __future__ import annotations + +from fastmcp.exceptions import ToolError + +from app.entry.mcp.auth import require_write +from app.features.curation import service as curation +from app.features.curation.model import LinkError + + +def check_link(url: str) -> dict[str, object]: + """Look up a GitHub or GitLab repo, or a Hugging Face model or dataset. + + Repos: `stars`, `forks`, `last_push`, `license`, `archived`, `has_readme`, + `description`. Models and datasets: `downloads`, `likes`, `last_modified`, + `license`, `files` (count), and for a model `weights` (true when weight + files are present). A value the lookup could not get reads `unknown`, never + 0; `lookup: unknown` means the whole lookup failed. Any other URL is + refused — read it with `web_fetch`. + """ + require_write() + try: + return curation.check_link(url) + except LinkError as exc: + raise ToolError(str(exc)) diff --git a/backend/app/entry/mcp/close_story.py b/backend/app/entry/mcp/close_story.py new file mode 100644 index 00000000..2bb73256 --- /dev/null +++ b/backend/app/entry/mcp/close_story.py @@ -0,0 +1,23 @@ +"""The `close_story` tool: take a story off the rail.""" + +from __future__ import annotations + +from fastmcp.exceptions import ToolError +from sqlmodel import Session + +from app.core.db.engine import engine +from app.entry.mcp.auth import require_write +from app.features.stories import service as story_domain +from app.features.stories.model import StoryError + + +def close_story(slug: str) -> str: + """Close an open story: it leaves the rail, and its articles are free to + join another.""" + require_write() + with Session(engine) as session: + try: + story_domain.close_story(session, slug) + except StoryError as exc: + raise ToolError(str(exc)) + return f"Closed {slug}" diff --git a/backend/app/entry/mcp/draft_issue.py b/backend/app/entry/mcp/draft_issue.py new file mode 100644 index 00000000..0c1517ff --- /dev/null +++ b/backend/app/entry/mcp/draft_issue.py @@ -0,0 +1,42 @@ +"""The `draft_issue` tool: one weekly issue as a Resend broadcast draft. + +Behind the write scope because a draft lands in the audience subscribers are +mailed from. Sending is a person's, from the Resend dashboard. +""" + +from __future__ import annotations + +from fastmcp.exceptions import ToolError + +from app.entry.mcp.auth import require_write +from app.features.newsletter import service as newsletter +from app.features.newsletter.model import IssueError + + +def draft_issue(post: str) -> str: + """Draft one weekly issue as a Resend broadcast. + + `post` is the whole blog post file: a frontmatter block, then the body. + + --- + title: What does Ollama's MLX backend actually do? + description: One line that makes someone open it. + topic: local-ai + --- + The essay... + + `title` (1-55 characters) is the subject and the post's headline, + `description` (1-160) the line an inbox shows after it, `topic` a kebab + slug that names the draft in Resend. Quote a value in double quotes if it + holds ": " or starts with a quote or a symbol. The body is plain text, + 150-600 words: a blank line between paragraphs, `- ` to start a list line, + `[text](url)` for a link (at least 3 distinct sources), backticks for code. + The server renders it into the site's template. Returns the broadcast id + and the path to save the post at, exactly as passed. Nothing is sent: a + person reads the drafts and sends one from the Resend dashboard. + """ + require_write() + try: + return newsletter.draft(post) + except IssueError as exc: + raise ToolError(str(exc)) diff --git a/backend/app/entry/mcp/get_content.py b/backend/app/entry/mcp/get_content.py new file mode 100644 index 00000000..14a8b590 --- /dev/null +++ b/backend/app/entry/mcp/get_content.py @@ -0,0 +1,31 @@ +"""The `get_content` tool: one page of a candidate's stored text.""" + +from __future__ import annotations + +from fastmcp.exceptions import ToolError +from sqlmodel import Session + +from app.core.db.engine import engine +from app.entry.mcp.auth import require_write +from app.features.curation import service as curation +from app.features.curation.model import CandidateError, ContentPage + + +def get_content( + url: str, offset: int = 0, limit: int = curation.PAGE_LIMIT +) -> ContentPage: + """Return one page of the article text the pipeline extracted for a candidate. + + The stored text is what a page fetch returns, cut at 12000 characters, so + read here first. Returns `text`, `offset`, `next_offset` (null on the last + page — pass it back as `offset` to read on), `total` characters, and, on + the first page only, `links`: the repo / model / paper / docs URLs the + article body links, grouped. Fetch the page only when this is empty, a + teaser, or cut off before the part you need. + """ + require_write() + with Session(engine) as session: + try: + return curation.get_content(session, url, offset, limit) + except CandidateError as exc: + raise ToolError(str(exc)) diff --git a/backend/app/entry/mcp/list_candidates.py b/backend/app/entry/mcp/list_candidates.py new file mode 100644 index 00000000..1de8afa9 --- /dev/null +++ b/backend/app/entry/mcp/list_candidates.py @@ -0,0 +1,34 @@ +"""The `list_candidates` tool: one page of the pending queue.""" + +from __future__ import annotations + +from sqlmodel import Session + +from app.core.db.engine import engine +from app.entry.mcp.auth import require_write +from app.features.curation import service as curation +from app.features.curation.model import CandidatePage + + +def list_candidates(offset: int = 0, limit: int = curation.LIST_LIMIT) -> CandidatePage: + """List one page of the articles waiting on a curation verdict, freshest first. + + Returns `rows`, `offset`, `next_offset` (null on the last page — pass it + back as `offset` for the next one) and `total` pending. Pages hold at most + 50 rows. Read every page before settling any candidate: a verdict removes + its row and shifts the pages after it. + + One row per candidate: `url`, `title`, `source` (where the link was found), + `publisher`, `publisher_kind` (`unknown` for a site the pipeline has just + created a publisher for: `approve` then needs one), `approved` (the + publisher's approvals / decisions over the + last 90 days, e.g. "3/10", or "new" when it has none), `traction`, + `published_at`, `queued_at`, and a 200-character `snippet`. Deliberately not the full text — call + `get_content` for a candidate worth a closer look. + + A candidate stays listed until `approve` or `reject` is called on it, so a + session that stops halfway leaves the rest for the next one. + """ + require_write() + with Session(engine) as session: + return curation.list_candidates(session, offset, limit) diff --git a/backend/app/entry/mcp/reject.py b/backend/app/entry/mcp/reject.py new file mode 100644 index 00000000..1f03d6cb --- /dev/null +++ b/backend/app/entry/mcp/reject.py @@ -0,0 +1,27 @@ +"""The `reject` tool: turn one candidate down.""" + +from __future__ import annotations + +from fastmcp.exceptions import ToolError +from sqlmodel import Session + +from app.core.db.engine import engine +from app.entry.mcp.auth import require_write +from app.features.curation import service as curation +from app.features.curation.model import CandidateError + + +def reject(url: str, score: int, reason: str) -> str: + """Turn a candidate down, keeping the score and the reason for it. + + The row stays in the ledger so the URL is never judged twice, and so the + verdicts can be read back as labels. Write `reason` for a human reading a + hundred of them later, not for a log line. + """ + require_write() + with Session(engine) as session: + try: + curation.reject(session, url, score, reason) + except CandidateError as exc: + raise ToolError(str(exc)) + return f"Rejected [{score}/100] {url}" diff --git a/backend/app/entry/mcp/reject_many.py b/backend/app/entry/mcp/reject_many.py new file mode 100644 index 00000000..7133347d --- /dev/null +++ b/backend/app/entry/mcp/reject_many.py @@ -0,0 +1,23 @@ +"""The `reject_many` tool: turn many candidates down, each on its own.""" + +from __future__ import annotations + +from sqlmodel import Session + +from app.core.db.engine import engine +from app.entry.mcp.auth import require_write +from app.features.curation import service as curation +from app.features.curation.model import Verdict + + +def reject_many(verdicts: list[Verdict]) -> str: + """Turn down many candidates at once: each of `verdicts` is `{url, score, + reason}`, the same fields `reject` takes. + + Each is settled on its own, so one URL that is not pending costs that line + and not the batch. Returns one line per verdict, in order, saying whether + it was rejected or why not. + """ + require_write() + with Session(engine) as session: + return "\n".join(curation.reject_many(session, verdicts)) diff --git a/backend/app/entry/mcp/related.py b/backend/app/entry/mcp/related.py new file mode 100644 index 00000000..a1008117 --- /dev/null +++ b/backend/app/entry/mcp/related.py @@ -0,0 +1,32 @@ +"""The `related` tool: what else covers one published article's subject.""" + +from __future__ import annotations + +from fastmcp.exceptions import ToolError +from sqlmodel import Session + +from app.core.db.engine import engine +from app.entry.mcp.auth import require_write +from app.features.newsletter import service as newsletter +from app.features.newsletter.model import DigestError, Related + + +def related( + url: str, days: int = newsletter.RELATED_DAYS, limit: int = newsletter.RELATED_LIMIT +) -> Related: + """What else covers one published article's subject: the individual writer + who tried it, the repo built on it, the commentary on it. + + Compares by embedding against every article from the last `days` and every + reject from that window that scored 35 or more. Up to `limit` rows, the + closest first: `url`, `title`, `publisher`, `publisher_kind`, `kind`, + `stage` (`published` or `rejected`), `score`, `distance` (under 0.30 is the + same story, up to 0.45 is the same subject), and `summary` for a published + row or curation's `reason` for a rejected one. + """ + require_write() + with Session(engine) as session: + try: + return newsletter.related(session, url, days, limit) + except DigestError as exc: + raise ToolError(str(exc)) diff --git a/backend/app/entry/mcp/save_story.py b/backend/app/entry/mcp/save_story.py new file mode 100644 index 00000000..c75e3bc8 --- /dev/null +++ b/backend/app/entry/mcp/save_story.py @@ -0,0 +1,38 @@ +"""The `save_story` tool: create a story or extend an open one. + +A saved story is on the public site at once (ADR 16). +""" + +from __future__ import annotations + +from fastmcp.exceptions import ToolError +from sqlmodel import Session + +from app.core.db.engine import engine +from app.entry.mcp.auth import require_write +from app.features.stories import service as story_domain +from app.features.stories.model import StoryError + + +def save_story(slug: str, name: str, blurb: str, urls: list[str]) -> str: + """Create a story, or update an open one: set its `name` and `blurb` and + add `urls` to it (ones already in it are skipped). + + `slug` is lowercase words joined by hyphens and never changes. `name` is + at most 4 words, `blurb` at most 2 sentences and 280 characters. Every URL + must be a published article and in no other open story. Any problem + refuses the whole call, saying which, and nothing is written. A closed + story cannot be saved to. The rail shows a story once it has 3 articles + from 2 publishers. + """ + require_write() + with Session(engine) as session: + try: + saved = story_domain.save_story(session, slug, name, blurb, urls) + except StoryError as exc: + raise ToolError(str(exc)) + verb = "Created" if saved["created"] else "Updated" + return ( + f"{verb} {saved['slug']}: added {saved['added']}, " + f"{saved['article_count']} articles in all" + ) diff --git a/backend/app/entry/mcp/server.py b/backend/app/entry/mcp/server.py new file mode 100644 index 00000000..ef11f5dc --- /dev/null +++ b/backend/app/entry/mcp/server.py @@ -0,0 +1,71 @@ +"""The MCP server: the instance, its instructions, its tool list, and the ASGI app `main` mounts.""" + +from __future__ import annotations + +from fastmcp import FastMCP + +from app.entry.mcp.approve import approve +from app.entry.mcp.auth import SharedSecret +from app.entry.mcp.check_link import check_link +from app.entry.mcp.close_story import close_story +from app.entry.mcp.draft_issue import draft_issue +from app.entry.mcp.get_content import get_content +from app.entry.mcp.list_candidates import list_candidates +from app.entry.mcp.reject import reject +from app.entry.mcp.reject_many import reject_many +from app.entry.mcp.related import related +from app.entry.mcp.save_story import save_story +from app.entry.mcp.similar import similar +from app.entry.mcp.sql_query import sql_query +from app.entry.mcp.stories import stories +from app.entry.mcp.story_candidates import story_candidates +from app.entry.mcp.vocabulary import vocabulary +from app.entry.mcp.web_fetch import web_fetch +from app.entry.mcp.web_search import web_search +from app.entry.mcp.week import week + +mcp = FastMCP( + name="agentique", + instructions=( + "Tools for the agentique article pipeline. Reading: `sql_query` reads " + "the Postgres database (publishers, articles, scores, pipeline runs), " + "`web_fetch` reads one page, `web_search` finds pages. Curation, with " + "the write token only: `list_candidates` lists what is waiting on a " + "verdict, `get_content` pages through the stored text for one of them, " + "`similar` and `stories` compare candidates with the feed, " + "`check_link` looks up a repo or model, `vocabulary` lists the labels, " + "and `approve`, `reject` and `reject_many` settle them. Weekly " + "newsletter, with the write token only: `week` ranks the week's " + "topics, `related` finds what else covers one, and `draft_issue` " + "drafts an issue for a person to send and says where to save it as a " + "blog post. Stories, with the write token only: `story_candidates` " + "lists open stories and unassigned clusters, `save_story` creates or " + "extends one, `close_story` closes one." + ), + auth=SharedSecret(), + tools=[ + sql_query, + web_fetch, + web_search, + list_candidates, + get_content, + similar, + stories, + check_link, + vocabulary, + approve, + reject, + reject_many, + week, + related, + draft_issue, + story_candidates, + save_story, + close_story, + ], +) + +# Mounted at /mcp by `app.main`, so the route inside this app is the bare root. +# Its lifespan starts the streamable-HTTP session manager and has to be handed +# to the parent app -- a mounted ASGI app never gets a lifespan of its own. +mcp_app = mcp.http_app(path="/") diff --git a/backend/app/entry/mcp/similar.py b/backend/app/entry/mcp/similar.py new file mode 100644 index 00000000..ccecb097 --- /dev/null +++ b/backend/app/entry/mcp/similar.py @@ -0,0 +1,29 @@ +"""The `similar` tool: what the feed and the ledger hold close to one candidate.""" + +from __future__ import annotations + +from fastmcp.exceptions import ToolError +from sqlmodel import Session + +from app.core.db.engine import engine +from app.entry.mcp.auth import require_write +from app.features.curation import service as curation +from app.features.curation.model import CandidateError, Similar + + +def similar(url: str, days: int = 7, limit: int = 8) -> Similar: + """Show what the feed and the ledger hold that is close to one candidate. + + Compares by embedding against every published article and every candidate + or reject from the last `days`. `rows` lists up to `limit` neighbours, the + closest first: `url`, `title`, `publisher`, `stage` (`published`, + `pending`, or the reject stage), `score` and `distance` (under 0.30 is the + same story). `coverage` is how many distinct publishers carry this story, + this candidate's own included, with their names in `publishers`. + """ + require_write() + with Session(engine) as session: + try: + return curation.similar(session, url, days, limit) + except CandidateError as exc: + raise ToolError(str(exc)) diff --git a/backend/app/entry/mcp/sql_query.py b/backend/app/entry/mcp/sql_query.py new file mode 100644 index 00000000..dec74d1b --- /dev/null +++ b/backend/app/entry/mcp/sql_query.py @@ -0,0 +1,133 @@ +"""The `sql_query` tool: one read-only statement against the agentique database. + +Reachable with the read token. What keeps it read-only is the role it logs in +as, a read-only transaction, a prepared statement, and a plan check that +refuses the tables no question needs. +""" + +from __future__ import annotations + +import json +from collections.abc import Iterator +from datetime import date, datetime + +import psycopg +from fastmcp.exceptions import ToolError + +from app.core.settings import settings + +# A cap, not a page size: an agent that wants more should narrow the query or +# add its own LIMIT. Anything larger is context an agent cannot read anyway. +MAX_ROWS = 1000 + +# `user` holds emails and sign-in link tokens, and no question worth asking this +# tool needs either. The role's grants are the real boundary (see +# deploy/sql-roles.sql); the plan check below holds even where the tool runs as +# a role that has the grant anyway, which is every local database. +BLOCKED_TABLES = frozenset({"user"}) + +# Its own connection, not `app.core.db` — that one logs in as POSTGRES_USER +# and would happily run a DELETE. One per call rather than a pool: the cost is +# nothing at this call rate, and no session state survives a query. +DSN = settings.MCP_DATABASE_URI + + +def _json_safe(value: object) -> str: + """Fallback for the column types `json.dumps` has no encoding for. + + Timestamps get ISO-8601; Decimal, UUID and pgvector embeddings get `str`, + which is lossless for all three and keeps one unreadable column from + failing the whole query. + """ + if isinstance(value, datetime | date): + return value.isoformat() + return str(value) + + +def _plan_tables(node: object) -> Iterator[str]: + """Every table an EXPLAIN plan reads, however deeply nested. + + Walks the whole JSON rather than the top node: the table can sit under a + join, a subplan or a CTE. Views are expanded during planning, so what comes + back is the tables actually read, not the names the query used. + """ + if isinstance(node, dict): + name = node.get("Relation Name") + if isinstance(name, str): + yield name + for value in node.values(): + yield from _plan_tables(value) + elif isinstance(node, list): + for item in node: + yield from _plan_tables(item) + + +def _reject_blocked_tables( + cursor: psycopg.Cursor[tuple[object, ...]], sql: str +) -> None: + """Plan the statement, and refuse it if the plan reads a blocked table. + + EXPLAIN reports the tables actually read -- views expanded, joins, CTEs and + subplans included -- and refusing to plan is what keeps this a query tool, + since COPY, SET and DO never get that far. + + `prepare` is what makes the check honest. Given no parameters psycopg falls + back to the simple query protocol, which runs every command in the string, + so `SELECT 1; SELECT * FROM "user"` would be two commands and only the + first would be the one planned here. A prepared statement takes exactly + one, so the server rejects the second before anything executes. + + psycopg types a `str` query as LiteralString to stop callers interpolating + input into SQL. Doing precisely that is this tool's job, hence the + suppression; what keeps it safe is the role, this check, and `prepare`. + """ + cursor.execute(f"EXPLAIN (FORMAT JSON) {sql}", prepare=True) # ty: ignore[no-matching-overload] + plan = cursor.fetchone() + blocked = BLOCKED_TABLES.intersection(_plan_tables(plan[0] if plan else None)) + if blocked: + raise ToolError(f"Not readable through this tool: {', '.join(sorted(blocked))}") + + +def sql_query(sql: str) -> str: + """Run a read-only SQL query against the agentique Postgres database. + + Accepts one statement, and only one Postgres can plan as a query. The + connection is a read-only transaction held by a role with SELECT grants + only, so writes and DDL fail rather than being filtered out beforehand, and + statements are cancelled after 10 seconds. The `user` table is not + readable, directly or through a join. A rejected statement comes back as + Postgres's own error message. + + Returns JSON: `columns` (in select order), `rows` (a list of objects), and + `truncated` (true when the result hit the 1000-row cap and rows are + missing). Timestamps are ISO-8601 strings; numerics, UUIDs and embedding + vectors are stringified. + """ + try: + with psycopg.connect(DSN) as conn, conn.cursor() as cursor: + _reject_blocked_tables(cursor, sql) + + cursor.execute(sql, prepare=True) # ty: ignore[no-matching-overload] + if cursor.description is None: + # Unreachable while the plan check stands -- nothing that gets + # past it returns without a result set. Kept as an error rather + # than an empty result, which would read as "no rows found". + raise ToolError("Statement returned no result set") + columns = [column.name for column in cursor.description] + # One past the cap, so a full page can be told from an exact fit. + rows = cursor.fetchmany(MAX_ROWS + 1) + return json.dumps( + { + "columns": columns, + "rows": [ + dict(zip(columns, row, strict=True)) for row in rows[:MAX_ROWS] + ], + "truncated": len(rows) > MAX_ROWS, + }, + default=_json_safe, + ) + except psycopg.Error as exc: + # The agent wrote this SQL and can fix it, but only if it sees what + # Postgres actually said; anything but a ToolError reaches the client + # as a bare "internal error". + raise ToolError(str(exc)) diff --git a/backend/app/entry/mcp/stories.py b/backend/app/entry/mcp/stories.py new file mode 100644 index 00000000..45220a5e --- /dev/null +++ b/backend/app/entry/mcp/stories.py @@ -0,0 +1,29 @@ +"""The `stories` tool: pending candidates grouped by story. + +Curation coverage, not the right rail's stories: the name is the curate +rubric's, and the rail's tools are `story_candidates`, `save_story` and +`close_story`. +""" + +from __future__ import annotations + +from sqlmodel import Session + +from app.core.db.engine import engine +from app.entry.mcp.auth import require_write +from app.features.curation import service as curation +from app.features.curation.model import Coverage + + +def stories(days: int = 7) -> list[Coverage]: + """Group the pending candidates that are one story, with each other or with + an article published in the last `days`. + + Only groups of two or more come back, largest first, each with its + `members` (same row shape as `similar`, without the distance), `coverage` + (distinct publishers) and `publishers`. A member at stage `published` + means the feed already carries the story. Call it once, after triage. + """ + require_write() + with Session(engine) as session: + return curation.stories(session, days) diff --git a/backend/app/entry/mcp/story_candidates.py b/backend/app/entry/mcp/story_candidates.py new file mode 100644 index 00000000..1ceed0af --- /dev/null +++ b/backend/app/entry/mcp/story_candidates.py @@ -0,0 +1,28 @@ +"""The `story_candidates` tool: open stories and the clusters no story holds yet.""" + +from __future__ import annotations + +from sqlmodel import Session + +from app.core.db.engine import engine +from app.entry.mcp.auth import require_write +from app.features.stories import service as story_domain +from app.features.stories.model import Candidates + + +def story_candidates(days: int = 14) -> Candidates: + """What the stories routine decides from. + + `open`: every open story, the most recently active first, with `slug`, + `name`, `blurb`, `article_count`, `quiet_days` (since its newest article + was published) and its newest 5 `articles`. `clusters`: the last `days` + of published articles that sit in no open story, grouped by embedding at + a looser cut than coverage, the most publishers first. Each cluster has + `articles` (`url`, `title`, `publisher`, `published_at`, `score`, + `summary`), its `publishers`, and `near`: the open stories one of its + articles is close to. A single article close to a story comes back as a + cluster of one with that story in `near`. + """ + require_write() + with Session(engine) as session: + return story_domain.story_candidates(session, days) diff --git a/backend/app/entry/mcp/test_approve.py b/backend/app/entry/mcp/test_approve.py new file mode 100644 index 00000000..f6e057bd --- /dev/null +++ b/backend/app/entry/mcp/test_approve.py @@ -0,0 +1,40 @@ +"""The `approve` tool: what it reports, and a refused label as a tool error.""" + +from __future__ import annotations + +import pytest +from fastmcp.exceptions import ToolError +from sqlmodel import Session + +from app.core.db.ledger import Reject, RejectStage +from app.entry.mcp.approve import approve +from app.features.curation import service as curation + + +@pytest.mark.usefixtures("curator") +def test_approve_reports_the_article_it_published( + monkeypatch: pytest.MonkeyPatch, +) -> None: + calls: list[tuple[object, ...]] = [] + + def fake(_session: Session, *args: object) -> int: + calls.append(args) + return 42 + + monkeypatch.setattr(curation, "approve", fake) + url = "https://example.com/post" + assert approve(url, 81, "r", "s", "post", ["agents"], "media", "Example") == ( + f"Published article #42: {url}" + ) + assert calls == [(url, 81, "r", "s", "post", ["agents"], "media", "Example")] + + +@pytest.mark.usefixtures("curator") +def test_a_refused_label_is_a_tool_error_and_the_candidate_stays( + db: Session, candidate: Reject +) -> None: + with pytest.raises(ToolError, match="publisher_kind"): + approve(candidate.url, 81, "r", "s", "post", []) + db.expire_all() + row = db.get(Reject, candidate.url) + assert row is not None and row.stage == RejectStage.pending diff --git a/backend/app/mcp/tests/test_curation_tools.py b/backend/app/entry/mcp/test_auth.py similarity index 56% rename from backend/app/mcp/tests/test_curation_tools.py rename to backend/app/entry/mcp/test_auth.py index 58bc8c0f..289be5d9 100644 --- a/backend/app/mcp/tests/test_curation_tools.py +++ b/backend/app/entry/mcp/test_auth.py @@ -7,13 +7,29 @@ from __future__ import annotations +from collections.abc import Callable + import pytest from fastmcp.exceptions import ToolError from fastmcp.server.auth import AccessToken from app.core.settings import settings -from app.mcp import tools -from app.mcp.server import SharedSecret +from app.entry.mcp import auth +from app.entry.mcp.approve import approve +from app.entry.mcp.check_link import check_link +from app.entry.mcp.close_story import close_story +from app.entry.mcp.draft_issue import draft_issue +from app.entry.mcp.get_content import get_content +from app.entry.mcp.list_candidates import list_candidates +from app.entry.mcp.reject import reject +from app.entry.mcp.reject_many import reject_many +from app.entry.mcp.related import related +from app.entry.mcp.save_story import save_story +from app.entry.mcp.similar import similar +from app.entry.mcp.stories import stories +from app.entry.mcp.story_candidates import story_candidates +from app.entry.mcp.vocabulary import vocabulary +from app.entry.mcp.week import week WRITE_TOOLS = ( "list_candidates", @@ -37,43 +53,44 @@ def _as(monkeypatch: pytest.MonkeyPatch, *scopes: str) -> None: """Answer the tools' scope check as a caller holding ``scopes``.""" monkeypatch.setattr( - tools, + auth, "get_access_token", lambda: AccessToken(token="t", client_id="c", scopes=list(scopes)), ) -def _call(name: str): +def _call(name: str) -> object: """Call one curation tool with throwaway arguments. The gate is the first thing each one does, so the arguments never matter: a caller without the scope is refused before any of them is read. """ - return { - "list_candidates": lambda: tools.list_candidates(), - "get_content": lambda: tools.get_content("https://example.com"), - "similar": lambda: tools.similar("https://example.com"), - "stories": lambda: tools.stories(), - "check_link": lambda: tools.check_link("https://github.com/a/b"), - "vocabulary": lambda: tools.vocabulary(), - "approve": lambda: tools.approve( - "https://example.com", 80, "r", "s", "post", [] - ), - "reject": lambda: tools.reject("https://example.com", 20, "r"), - "reject_many": lambda: tools.reject_many( + calls: dict[str, Callable[[], object]] = { + "list_candidates": lambda: list_candidates(), + "get_content": lambda: get_content("https://example.com"), + "similar": lambda: similar("https://example.com"), + "stories": lambda: stories(), + "check_link": lambda: check_link("https://github.com/a/b"), + "vocabulary": lambda: vocabulary(), + "approve": lambda: approve("https://example.com", 80, "r", "s", "post", []), + "reject": lambda: reject("https://example.com", 20, "r"), + "reject_many": lambda: reject_many( [{"url": "https://example.com", "score": 20, "reason": "r"}] ), - "week": lambda: tools.week(), - "related": lambda: tools.related("https://example.com"), - "draft_issue": lambda: tools.draft_issue("---\ntitle: t\n---\nb"), - "story_candidates": lambda: tools.story_candidates(), - "save_story": lambda: tools.save_story("s", "n", "b", []), - "close_story": lambda: tools.close_story("s"), - }[name]() + "week": lambda: week(), + "related": lambda: related("https://example.com"), + "draft_issue": lambda: draft_issue("---\ntitle: t\n---\nb"), + "story_candidates": lambda: story_candidates(), + "save_story": lambda: save_story("s", "n", "b", []), + "close_story": lambda: close_story("s"), + } + return calls[name]() @pytest.mark.parametrize("name", WRITE_TOOLS) -def test_the_read_token_cannot_curate(monkeypatch: pytest.MonkeyPatch, name: str): +def test_the_read_token_cannot_curate( + monkeypatch: pytest.MonkeyPatch, name: str +) -> None: _as(monkeypatch) with pytest.raises(ToolError, match="curation token"): _call(name) @@ -82,48 +99,52 @@ def test_the_read_token_cannot_curate(monkeypatch: pytest.MonkeyPatch, name: str @pytest.mark.parametrize("name", WRITE_TOOLS) def test_an_unauthenticated_call_cannot_curate( monkeypatch: pytest.MonkeyPatch, name: str -): - monkeypatch.setattr(tools, "get_access_token", lambda: None) +) -> None: + monkeypatch.setattr(auth, "get_access_token", lambda: None) with pytest.raises(ToolError, match="curation token"): _call(name) -def test_the_write_scope_gets_past_the_gate(monkeypatch: pytest.MonkeyPatch): +def test_the_write_scope_gets_past_the_gate(monkeypatch: pytest.MonkeyPatch) -> None: """Past the gate it is an ordinary lookup, and a URL nobody queued is a message rather than a traceback.""" - _as(monkeypatch, tools.WRITE_SCOPE) + _as(monkeypatch, auth.WRITE_SCOPE) with pytest.raises(ToolError, match="No candidate for"): - tools.get_content("https://never-queued.example") + get_content("https://never-queued.example") @pytest.mark.anyio -async def test_only_the_write_token_carries_the_scope(monkeypatch: pytest.MonkeyPatch): +async def test_only_the_write_token_carries_the_scope( + monkeypatch: pytest.MonkeyPatch, +) -> None: monkeypatch.setattr(settings, "MCP_TOKEN", "read-token") monkeypatch.setattr(settings, "MCP_WRITE_TOKEN", "write-token") - verifier = SharedSecret() + verifier = auth.SharedSecret() write = await verifier.verify_token("write-token") read = await verifier.verify_token("read-token") - assert write is not None and write.scopes == [tools.WRITE_SCOPE] + assert write is not None and write.scopes == [auth.WRITE_SCOPE] assert read is not None and read.scopes == [] assert await verifier.verify_token("neither") is None @pytest.mark.anyio -async def test_an_unset_write_token_matches_nothing(monkeypatch: pytest.MonkeyPatch): +async def test_an_unset_write_token_matches_nothing( + monkeypatch: pytest.MonkeyPatch, +) -> None: """A deploy that forgets the variable must curate nothing, not let the read token publish — and an empty header must not match an empty setting.""" monkeypatch.setattr(settings, "MCP_TOKEN", "read-token") monkeypatch.setattr(settings, "MCP_WRITE_TOKEN", None) - verifier = SharedSecret() + verifier = auth.SharedSecret() assert await verifier.verify_token("") is None read = await verifier.verify_token("read-token") assert read is not None and read.scopes == [] -def test_the_two_tokens_cannot_be_the_same_value(): +def test_the_two_tokens_cannot_be_the_same_value() -> None: """The verifier checks the write token first, so a copy-paste that sets both to one value hands publishing rights to every holder of what the operator believes is a read-only credential. Nothing downstream would notice.""" @@ -140,25 +161,3 @@ def test_the_two_tokens_cannot_be_the_same_value(): @pytest.fixture def anyio_backend() -> str: return "asyncio" - - -def test_check_link_refuses_what_it_does_not_read(monkeypatch: pytest.MonkeyPatch): - _as(monkeypatch, tools.WRITE_SCOPE) - with pytest.raises(ToolError, match="web_fetch"): - tools.check_link("https://arxiv.org/abs/2609.01234") - - -def test_reject_many_reports_one_line_per_verdict(monkeypatch: pytest.MonkeyPatch): - _as(monkeypatch, tools.WRITE_SCOPE) - result = tools.reject_many( - [ - {"url": "https://never-queued.example/1", "score": 20, "reason": "r"}, - {"url": "https://never-queued.example/2", "score": 20, "reason": "r"}, - ] - ) - assert result.splitlines() == [ - "Skipped https://never-queued.example/1: " - "No candidate for https://never-queued.example/1", - "Skipped https://never-queued.example/2: " - "No candidate for https://never-queued.example/2", - ] diff --git a/backend/app/entry/mcp/test_check_link.py b/backend/app/entry/mcp/test_check_link.py new file mode 100644 index 00000000..d619e401 --- /dev/null +++ b/backend/app/entry/mcp/test_check_link.py @@ -0,0 +1,14 @@ +"""The `check_link` tool.""" + +from __future__ import annotations + +import pytest +from fastmcp.exceptions import ToolError + +from app.entry.mcp.check_link import check_link + + +@pytest.mark.usefixtures("curator") +def test_check_link_refuses_what_it_does_not_read() -> None: + with pytest.raises(ToolError, match="web_fetch"): + check_link("https://arxiv.org/abs/2609.01234") diff --git a/backend/app/entry/mcp/test_draft_issue.py b/backend/app/entry/mcp/test_draft_issue.py new file mode 100644 index 00000000..45d263a2 --- /dev/null +++ b/backend/app/entry/mcp/test_draft_issue.py @@ -0,0 +1,14 @@ +"""The `draft_issue` tool.""" + +from __future__ import annotations + +import pytest +from fastmcp.exceptions import ToolError + +from app.entry.mcp.draft_issue import draft_issue + + +@pytest.mark.usefixtures("curator") +def test_a_post_without_frontmatter_is_a_tool_error() -> None: + with pytest.raises(ToolError, match="frontmatter"): + draft_issue("Just an essay, no frontmatter.") diff --git a/backend/app/entry/mcp/test_reject.py b/backend/app/entry/mcp/test_reject.py new file mode 100644 index 00000000..2dfd583b --- /dev/null +++ b/backend/app/entry/mcp/test_reject.py @@ -0,0 +1,31 @@ +"""The `reject` tool: the verdict it writes, and an unknown URL as a tool error.""" + +from __future__ import annotations + +import pytest +from fastmcp.exceptions import ToolError +from sqlmodel import Session + +from app.core.db.ledger import Reject, RejectStage +from app.entry.mcp.reject import reject + + +@pytest.mark.usefixtures("curator") +def test_reject_settles_the_candidate(db: Session, candidate: Reject) -> None: + assert reject(candidate.url, 22, "A vendor tutorial.") == ( + f"Rejected [22/100] {candidate.url}" + ) + db.expire_all() + row = db.get(Reject, candidate.url) + assert row is not None + assert (row.stage, row.score, row.reason) == ( + RejectStage.below_threshold, + 22, + "A vendor tutorial.", + ) + + +@pytest.mark.usefixtures("curator") +def test_reject_refuses_a_url_nobody_queued() -> None: + with pytest.raises(ToolError, match="No candidate for"): + reject("https://never-queued.example/post", 22, "r") diff --git a/backend/app/entry/mcp/test_reject_many.py b/backend/app/entry/mcp/test_reject_many.py new file mode 100644 index 00000000..307366ef --- /dev/null +++ b/backend/app/entry/mcp/test_reject_many.py @@ -0,0 +1,23 @@ +"""The `reject_many` tool.""" + +from __future__ import annotations + +import pytest + +from app.entry.mcp.reject_many import reject_many + + +@pytest.mark.usefixtures("curator") +def test_reject_many_reports_one_line_per_verdict() -> None: + result = reject_many( + [ + {"url": "https://never-queued.example/1", "score": 20, "reason": "r"}, + {"url": "https://never-queued.example/2", "score": 20, "reason": "r"}, + ] + ) + assert result.splitlines() == [ + "Skipped https://never-queued.example/1: " + "No candidate for https://never-queued.example/1", + "Skipped https://never-queued.example/2: " + "No candidate for https://never-queued.example/2", + ] diff --git a/backend/app/entry/mcp/test_related.py b/backend/app/entry/mcp/test_related.py new file mode 100644 index 00000000..070ed11b --- /dev/null +++ b/backend/app/entry/mcp/test_related.py @@ -0,0 +1,14 @@ +"""The `related` tool.""" + +from __future__ import annotations + +import pytest +from fastmcp.exceptions import ToolError + +from app.entry.mcp.related import related + + +@pytest.mark.usefixtures("curator") +def test_related_refuses_a_url_that_is_not_published() -> None: + with pytest.raises(ToolError, match="No published article"): + related("https://never-published.example/post") diff --git a/backend/app/entry/mcp/test_save_story.py b/backend/app/entry/mcp/test_save_story.py new file mode 100644 index 00000000..96558852 --- /dev/null +++ b/backend/app/entry/mcp/test_save_story.py @@ -0,0 +1,54 @@ +"""The `save_story` and `close_story` tools: what they report, and a refusal as +a tool error.""" + +from __future__ import annotations + +import pytest +from fastmcp.exceptions import ToolError +from sqlmodel import Session + +from app.core.db.factories import ( + create_random_article, + create_random_publisher, + random_lower_string, +) +from app.entry.mcp.close_story import close_story +from app.entry.mcp.save_story import save_story + + +def _url(db: Session, made: dict[str, list[int]]) -> str: + publisher = create_random_publisher(db) + article = create_random_article(db, publisher_id=publisher.id) + assert publisher.id is not None and article.id is not None + made["publishers"].append(publisher.id) + made["articles"].append(article.id) + return article.url + + +@pytest.mark.usefixtures("curator") +def test_save_story_tool_reports_what_it_did( + db: Session, made: dict[str, list[int]] +) -> None: + slug = f"tool-made-{random_lower_string()[:6]}" + assert save_story(slug, "Tool made", "It works.", [_url(db, made)]) == ( + f"Created {slug}: added 1, 1 articles in all" + ) + assert save_story(slug, "Tool made", "It still works.", [_url(db, made)]) == ( + f"Updated {slug}: added 1, 2 articles in all" + ) + assert close_story(slug) == f"Closed {slug}" + + +@pytest.mark.usefixtures("curator") +def test_save_story_refuses_an_unknown_url_as_a_tool_error() -> None: + slug = f"tool-refused-{random_lower_string()[:6]}" + with pytest.raises(ToolError, match="Not a published article"): + save_story( + slug, "Tool refused", "It fails.", ["https://never-published.example/post"] + ) + + +@pytest.mark.usefixtures("curator") +def test_close_story_refuses_an_unknown_slug_as_a_tool_error() -> None: + with pytest.raises(ToolError, match="No story"): + close_story(f"no-such-story-{random_lower_string()[:6]}") diff --git a/backend/app/entry/mcp/test_server.py b/backend/app/entry/mcp/test_server.py new file mode 100644 index 00000000..d0b36e2a --- /dev/null +++ b/backend/app/entry/mcp/test_server.py @@ -0,0 +1,149 @@ +"""The bearer-token gate on the mounted MCP endpoint, end to end over HTTP.""" + +from __future__ import annotations + +import json + +import pytest +from fastapi.testclient import TestClient + +from app.core.settings import settings + +MCP_URL = "/mcp/" +HEADERS = {"Accept": "application/json, text/event-stream"} +INITIALIZE = { + "jsonrpc": "2.0", + "id": 1, + "method": "initialize", + "params": { + "protocolVersion": "2025-06-18", + "capabilities": {}, + "clientInfo": {"name": "test", "version": "1"}, + }, +} + + +@pytest.fixture +def token(monkeypatch: pytest.MonkeyPatch) -> str: + monkeypatch.setattr(settings, "MCP_TOKEN", "test-mcp-token") + return "test-mcp-token" + + +def test_the_right_token_gets_a_session(client: TestClient, token: str) -> None: + response = client.post( + MCP_URL, + json=INITIALIZE, + headers={**HEADERS, "Authorization": f"Bearer {token}"}, + ) + assert response.status_code == 200 + + +@pytest.mark.usefixtures("token") +def test_a_wrong_token_is_rejected(client: TestClient) -> None: + response = client.post( + MCP_URL, json=INITIALIZE, headers={**HEADERS, "Authorization": "Bearer nope"} + ) + assert response.status_code == 401 + + +@pytest.mark.usefixtures("token") +def test_no_token_is_rejected(client: TestClient) -> None: + response = client.post(MCP_URL, json=INITIALIZE, headers=HEADERS) + assert response.status_code == 401 + + +def test_an_unconfigured_server_rejects_every_token( + client: TestClient, monkeypatch: pytest.MonkeyPatch +) -> None: + # A deploy that forgets MCP_TOKEN must serve 401s, not an open database. + monkeypatch.setattr(settings, "MCP_TOKEN", None) + response = client.post( + MCP_URL, json=INITIALIZE, headers={**HEADERS, "Authorization": "Bearer "} + ) + assert response.status_code == 401 + + +READ_TOKEN = "test-read-token" +WRITE_TOKEN = "test-write-token" + + +@pytest.fixture +def both_tokens(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(settings, "MCP_TOKEN", READ_TOKEN) + monkeypatch.setattr(settings, "MCP_WRITE_TOKEN", WRITE_TOKEN) + + +def _call_tool( + client: TestClient, token: str, name: str, arguments: dict[str, object] +) -> tuple[bool, str]: + """Open a session with ``token`` and call one tool, as an MCP client would. + + Returns the result's ``isError`` and its text. A refused tool call is still + a 200: the gate inside the tool answers, not the HTTP layer. + """ + headers = {**HEADERS, "Authorization": f"Bearer {token}"} + opened = client.post(MCP_URL, json=INITIALIZE, headers=headers) + assert opened.status_code == 200 + headers["mcp-session-id"] = opened.headers["mcp-session-id"] + client.post( + MCP_URL, + json={"jsonrpc": "2.0", "method": "notifications/initialized"}, + headers=headers, + ) + response = client.post( + MCP_URL, + json={ + "jsonrpc": "2.0", + "id": 2, + "method": "tools/call", + "params": {"name": name, "arguments": arguments}, + }, + headers=headers, + ) + assert response.status_code == 200 + data = next( + line.removeprefix("data: ") + for line in response.text.splitlines() + if line.startswith("data: ") + ) + result = json.loads(data)["result"] + return result["isError"], result["content"][0]["text"] + + +@pytest.mark.usefixtures("both_tokens") +def test_the_read_token_is_refused_by_a_write_tool(client: TestClient) -> None: + is_error, text = _call_tool( + client, READ_TOKEN, "get_content", {"url": "https://never-queued.example"} + ) + assert is_error + assert "curation token" in text + + +@pytest.mark.usefixtures("both_tokens") +def test_the_write_token_gets_past_the_gate(client: TestClient) -> None: + # Past the gate it is an ordinary lookup, so the refusal is the candidate's. + is_error, text = _call_tool( + client, WRITE_TOKEN, "get_content", {"url": "https://never-queued.example"} + ) + assert is_error + assert "No candidate for" in text + + +def test_an_unset_write_token_opens_no_write_tool( + client: TestClient, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(settings, "MCP_TOKEN", READ_TOKEN) + monkeypatch.setattr(settings, "MCP_WRITE_TOKEN", None) + + stale = client.post( + MCP_URL, + json=INITIALIZE, + headers={**HEADERS, "Authorization": f"Bearer {WRITE_TOKEN}"}, + ) + assert stale.status_code == 401 + + is_error, text = _call_tool( + client, READ_TOKEN, "get_content", {"url": "https://never-queued.example"} + ) + assert is_error + assert "curation token" in text diff --git a/backend/app/entry/mcp/test_similar.py b/backend/app/entry/mcp/test_similar.py new file mode 100644 index 00000000..a9ca7454 --- /dev/null +++ b/backend/app/entry/mcp/test_similar.py @@ -0,0 +1,14 @@ +"""The `similar` tool.""" + +from __future__ import annotations + +import pytest +from fastmcp.exceptions import ToolError + +from app.entry.mcp.similar import similar + + +@pytest.mark.usefixtures("curator") +def test_similar_refuses_a_url_nobody_queued() -> None: + with pytest.raises(ToolError, match="No candidate or article for"): + similar("https://never-queued.example/post") diff --git a/backend/app/entry/mcp/test_sql_query.py b/backend/app/entry/mcp/test_sql_query.py new file mode 100644 index 00000000..3a6a9070 --- /dev/null +++ b/backend/app/entry/mcp/test_sql_query.py @@ -0,0 +1,106 @@ +"""`sql_query`: the read-only guard, the blocked tables, the row cap, and JSON coercion.""" + +from __future__ import annotations + +import json + +import pytest +from fastmcp.exceptions import ToolError + +from app.core.settings import settings +from app.entry.mcp import sql_query as tool + + +@pytest.fixture(autouse=True) +def _test_dsn(monkeypatch: pytest.MonkeyPatch) -> None: + """Point the tool at the test credentials instead of `agentique_ro`. + + The role does not exist outside the production box, but the DSN is the one + settings build, read-only options and all, so the guard under test is real. + """ + monkeypatch.setattr( + tool, + "DSN", + settings.model_copy( + update={ + "MCP_DB_USER": settings.POSTGRES_USER, + "MCP_DB_PASSWORD": settings.POSTGRES_PASSWORD, + } + ).MCP_DATABASE_URI, + ) + + +def test_select_returns_columns_and_rows() -> None: + result = json.loads(tool.sql_query("SELECT 1 AS n, 'a' AS label")) + assert result == { + "columns": ["n", "label"], + "rows": [{"n": 1, "label": "a"}], + "truncated": False, + } + + +def test_rows_are_capped_and_flagged() -> None: + result = json.loads( + tool.sql_query(f"SELECT i FROM generate_series(1, {tool.MAX_ROWS + 5}) AS i") + ) + assert len(result["rows"]) == tool.MAX_ROWS + assert result["truncated"] is True + + +def test_a_result_that_exactly_fills_the_cap_is_not_truncated() -> None: + result = json.loads( + tool.sql_query(f"SELECT i FROM generate_series(1, {tool.MAX_ROWS}) AS i") + ) + assert len(result["rows"]) == tool.MAX_ROWS + assert result["truncated"] is False + + +def test_timestamps_and_numerics_survive_serialisation() -> None: + result = json.loads( + tool.sql_query( + "SELECT TIMESTAMP '2026-01-02 03:04:05' AS ts, 1.5::numeric AS amount" + ) + ) + assert result["rows"] == [{"ts": "2026-01-02T03:04:05", "amount": "1.5"}] + + +def test_a_write_is_rejected_by_the_read_only_transaction() -> None: + # Not a syntax check on the way in: Postgres refuses it, so anything the + # tool failed to anticipate is refused too. + with pytest.raises(ToolError, match="read-only transaction"): + tool.sql_query("UPDATE article SET title = title WHERE false") + + +def test_anything_postgres_cannot_plan_as_a_query_is_refused() -> None: + # Planning is the filter: COPY, SET, DO and a second statement after a + # semicolon never reach execution. + with pytest.raises(ToolError): + tool.sql_query("SET application_name = 'mcp'") + + +def test_the_user_table_is_not_readable() -> None: + with pytest.raises(ToolError, match="Not readable through this tool: user"): + tool.sql_query('SELECT email FROM "user"') + + +def test_the_user_table_is_not_readable_through_a_subquery() -> None: + # The check walks the whole plan, so hiding it below the top node does not + # help -- nor would a join, a CTE or a view. + with pytest.raises(ToolError, match="Not readable through this tool: user"): + tool.sql_query('SELECT 1 WHERE EXISTS (SELECT 1 FROM "user")') + + +def test_a_second_statement_after_a_semicolon_cannot_smuggle_a_read() -> None: + with pytest.raises(ToolError): + tool.sql_query('SELECT 1; SELECT email FROM "user"') + + +def test_a_statement_crafted_to_close_the_explain_prefix_is_still_one_statement() -> ( + None +): + # The prefix is text, so this would be three valid statements if anything + # but a prepared statement ran it -- and the middle one is the read. + with pytest.raises(ToolError): + tool.sql_query( + 'SELECT 1) AS x; SELECT email FROM "user"; SELECT * FROM (SELECT 1' + ) diff --git a/backend/app/entry/mcp/test_web_fetch.py b/backend/app/entry/mcp/test_web_fetch.py new file mode 100644 index 00000000..9103dcf7 --- /dev/null +++ b/backend/app/entry/mcp/test_web_fetch.py @@ -0,0 +1,31 @@ +"""`web_fetch`: what it forwards, and an empty page as an error.""" + +from __future__ import annotations + +import pytest +from fastmcp.exceptions import ToolError + +from app.entry.mcp import web_fetch as web_fetch_tool + + +def test_web_fetch_returns_extracted_text_and_forwards_the_cap( + monkeypatch: pytest.MonkeyPatch, +) -> None: + seen: list[tuple[str, int]] = [] + + def fake(url: str, max_length: int) -> str: + seen.append((url, max_length)) + return f"text of {url}" + + monkeypatch.setattr(web_fetch_tool, "fetch_and_extract", fake) + text = web_fetch_tool.web_fetch("https://example.com/post", max_length=50) + assert text == "text of https://example.com/post" + assert seen == [("https://example.com/post", 50)] + + +def test_web_fetch_fails_loudly_when_nothing_is_readable( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(web_fetch_tool, "fetch_and_extract", lambda url, max_length: "") + with pytest.raises(ToolError, match="No readable content"): + web_fetch_tool.web_fetch("https://example.com/paywalled") diff --git a/backend/app/entry/mcp/test_web_search.py b/backend/app/entry/mcp/test_web_search.py new file mode 100644 index 00000000..3790c22f --- /dev/null +++ b/backend/app/entry/mcp/test_web_search.py @@ -0,0 +1,24 @@ +"""`web_search`: what it forwards.""" + +from __future__ import annotations + +import pytest + +from app.entry.mcp import web_search as web_search_tool + + +def test_web_search_forwards_every_argument(monkeypatch: pytest.MonkeyPatch) -> None: + calls: list[tuple[str, int, list[str] | None]] = [] + + def fake( + query: str, max_results: int, include_domains: list[str] | None + ) -> list[dict[str, str | None]]: + calls.append((query, max_results, include_domains)) + return [{"title": "t", "url": "u", "description": "d", "published_date": None}] + + monkeypatch.setattr(web_search_tool, "tavily_search", fake) + results = web_search_tool.web_search( + "gpt-5", max_results=3, include_domains=["openai.com"] + ) + assert calls == [("gpt-5", 3, ["openai.com"])] + assert results[0]["url"] == "u" diff --git a/backend/app/entry/mcp/vocabulary.py b/backend/app/entry/mcp/vocabulary.py new file mode 100644 index 00000000..2d014784 --- /dev/null +++ b/backend/app/entry/mcp/vocabulary.py @@ -0,0 +1,17 @@ +"""The `vocabulary` tool: the labels `approve` accepts.""" + +from __future__ import annotations + +from sqlmodel import Session + +from app.core.db.engine import engine +from app.entry.mcp.auth import require_write +from app.features.curation import service as curation + + +def vocabulary() -> dict[str, object]: + """The labels `approve` accepts: `kinds`, `publisher_kinds`, and `tags` as + slug -> description. Call it once per session, before the first approve.""" + require_write() + with Session(engine) as session: + return curation.vocabulary(session) diff --git a/backend/app/entry/mcp/web_fetch.py b/backend/app/entry/mcp/web_fetch.py new file mode 100644 index 00000000..e8b1b970 --- /dev/null +++ b/backend/app/entry/mcp/web_fetch.py @@ -0,0 +1,20 @@ +"""The `web_fetch` tool: one page as readable text.""" + +from __future__ import annotations + +from fastmcp.exceptions import ToolError + +from app.core.extract import fetch_and_extract + + +def web_fetch(url: str, max_length: int = 20000) -> str: + """Fetch a web page and return its readable text, boilerplate stripped. + + Retries through a residential proxy when the direct fetch is blocked by a + 403, a paywall or a login wall. Fails if the page yields nothing readable — + an empty string would look like an empty page rather than a failed fetch. + """ + content = fetch_and_extract(url, max_length) + if not content: + raise ToolError(f"No readable content at {url}") + return content diff --git a/backend/app/entry/mcp/web_search.py b/backend/app/entry/mcp/web_search.py new file mode 100644 index 00000000..4706cd88 --- /dev/null +++ b/backend/app/entry/mcp/web_search.py @@ -0,0 +1,19 @@ +"""The `web_search` tool: Tavily results for a query.""" + +from __future__ import annotations + +from app.core.web_search import tavily_search + + +def web_search( + query: str, max_results: int = 5, include_domains: list[str] | None = None +) -> list[dict[str, str | None]]: + """Search the web (Tavily) and return ranked results. + + Each result has `title`, `url`, `description` and `published_date` + (null unless the source dated the page). `include_domains` restricts hits + to those hosts, e.g. `["openai.com"]` for first-party announcements only. + """ + return tavily_search( + query, max_results=max_results, include_domains=include_domains + ) diff --git a/backend/app/entry/mcp/week.py b/backend/app/entry/mcp/week.py new file mode 100644 index 00000000..3bd7221b --- /dev/null +++ b/backend/app/entry/mcp/week.py @@ -0,0 +1,27 @@ +"""The `week` tool: the week's published articles grouped into ranked topics.""" + +from __future__ import annotations + +from sqlmodel import Session + +from app.core.db.engine import engine +from app.entry.mcp.auth import require_write +from app.features.newsletter import service as newsletter +from app.features.newsletter.model import WeekTopic + + +def week(days: int = 7, limit: int = newsletter.WEEK_LIMIT) -> list[WeekTopic]: + """The weekly newsletter's material: the last `days` of published articles + grouped into topics, the most talked-about first. + + Ranked by `coverage` (distinct publishers carrying it, counting the copies + curation rejected as retellings), then `likes`, then `top_score`; up to + `limit` topics. Each topic's `articles` are the published ones - `url`, + `title`, `publisher`, `publisher_kind` (`lab`, `company`, `individual`, + `media`, `community`, `unknown`), `kind`, `score`, `likes`, and the `summary` curation + wrote after reading it - and `covered_by` the other outlets' copies from + the ledger, with curation's `reason`. + """ + require_write() + with Session(engine) as session: + return newsletter.week(session, days, limit) diff --git a/backend/app/features/analytics/tests/test_routes.py b/backend/app/features/analytics/tests/test_routes.py deleted file mode 100644 index 4a952b0a..00000000 --- a/backend/app/features/analytics/tests/test_routes.py +++ /dev/null @@ -1,121 +0,0 @@ -"""Tests for the analytics ingest endpoint.""" - -from __future__ import annotations - -from fastapi.testclient import TestClient -from sqlmodel import Session, col, select - -from app.core.db.event import AnalyticsEvent -from app.core.db.user import User -from app.core.settings import settings - - -def _latest_event(db: Session, visitor_id: str) -> AnalyticsEvent | None: - return db.exec( - select(AnalyticsEvent) - .where(AnalyticsEvent.visitor_id == visitor_id) - .order_by(col(AnalyticsEvent.id).desc()) - ).first() - - -def test_collect_anonymous_pageview(client: TestClient, db: Session) -> None: - r = client.post( - f"{settings.API_V1_STR}/analytics/collect", - json={"path": "/articles", "visitor_id": "anon-1"}, - ) - assert r.status_code == 204 - - event = _latest_event(db, "anon-1") - assert event is not None - assert event.event == "pageview" - assert event.path == "/articles" - assert event.user_id is None - # user agent captured from the request headers - assert event.user_agent is not None - - -def test_collect_ignores_referer_header(client: TestClient, db: Session) -> None: - r = client.post( - f"{settings.API_V1_STR}/analytics/collect", - headers={"Referer": "https://agentique.ch/feed"}, - json={"path": "/feed", "visitor_id": "anon-5"}, - ) - assert r.status_code == 204 - event = _latest_event(db, "anon-5") - assert event is not None - assert event.referrer is None - - -def test_collect_drops_query_strings(client: TestClient, db: Session) -> None: - r = client.post( - f"{settings.API_V1_STR}/analytics/collect", - json={ - "path": "/auth?token=secret", - "referrer": "https://agentique.ch/reset-password?token=secret#top", - "visitor_id": "anon-6", - }, - ) - assert r.status_code == 204 - event = _latest_event(db, "anon-6") - assert event is not None - assert event.path == "/auth" - assert event.referrer == "https://agentique.ch/reset-password" - - -def test_collect_defaults_event_to_pageview(client: TestClient, db: Session) -> None: - r = client.post( - f"{settings.API_V1_STR}/analytics/collect", - json={"visitor_id": "anon-2"}, - ) - assert r.status_code == 204 - event = _latest_event(db, "anon-2") - assert event is not None - assert event.event == "pageview" - assert event.path is None - - -def test_collect_custom_event_with_props(client: TestClient, db: Session) -> None: - r = client.post( - f"{settings.API_V1_STR}/analytics/collect", - json={ - "event": "article_liked", - "path": "/articles/5", - "visitor_id": "anon-3", - "props": {"article_id": 5}, - }, - ) - assert r.status_code == 204 - event = _latest_event(db, "anon-3") - assert event is not None - assert event.event == "article_liked" - assert event.props == {"article_id": 5} - - -def test_collect_attaches_user_when_authenticated( - client: TestClient, db: Session, normal_user_token_headers: dict[str, str] -) -> None: - r = client.post( - f"{settings.API_V1_STR}/analytics/collect", - headers=normal_user_token_headers, - json={"path": "/profile", "visitor_id": "known-1"}, - ) - assert r.status_code == 204 - - user = db.exec(select(User).where(User.email == settings.EMAIL_TEST_USER)).first() - assert user is not None - event = _latest_event(db, "known-1") - assert event is not None - assert event.user_id == user.id - - -def test_collect_truncates_oversized_strings(client: TestClient, db: Session) -> None: - long_path = "/" + "x" * 5000 - r = client.post( - f"{settings.API_V1_STR}/analytics/collect", - json={"path": long_path, "visitor_id": "anon-4"}, - ) - assert r.status_code == 204 - event = _latest_event(db, "anon-4") - assert event is not None - assert event.path is not None - assert len(event.path) == 2048 diff --git a/backend/app/features/audience/service.py b/backend/app/features/audience/service.py index 260c2298..dcdb35f6 100644 --- a/backend/app/features/audience/service.py +++ b/backend/app/features/audience/service.py @@ -13,6 +13,9 @@ from app.core.db.user import User from app.core.security import generate_login_token from app.core.settings import settings +from app.features.audience.email import generate_sign_in_email as generate_sign_in_email +from app.features.audience.email import generate_welcome_email as generate_welcome_email +from app.features.audience.email import send_link_email as send_link_email from app.features.audience.model import UserCreate, UserUpdateMe LOGIN_LINK_EXPIRE_MINUTES = 60 diff --git a/backend/app/features/audience/tests/test_service.py b/backend/app/features/audience/test_service.py similarity index 93% rename from backend/app/features/audience/tests/test_service.py rename to backend/app/features/audience/test_service.py index 84df59ce..7441975c 100644 --- a/backend/app/features/audience/tests/test_service.py +++ b/backend/app/features/audience/test_service.py @@ -4,8 +4,8 @@ from sqlmodel import Session +from app.core.db.factories import create_random_user from app.features.audience import service -from app.features.audience.tests.factories import create_random_user def test_issue_login_token(db: Session) -> None: diff --git a/backend/app/features/audience/tests/factories.py b/backend/app/features/audience/tests/factories.py deleted file mode 100644 index 383f8fa9..00000000 --- a/backend/app/features/audience/tests/factories.py +++ /dev/null @@ -1,39 +0,0 @@ -"""Factory for creating random User rows in tests.""" - -from __future__ import annotations - -from datetime import timedelta - -from sqlmodel import Session - -from app.core.db.user import User -from app.core.security import create_access_token -from app.core.settings import settings -from app.features.audience import service -from app.features.audience.model import UserCreate -from tests.random_data import random_email - - -def user_authentication_headers(user: User) -> dict[str, str]: - token = create_access_token(user.id, expires_delta=timedelta(minutes=5)) - return {"Authorization": f"Bearer {token}"} - - -def create_random_user(db: Session) -> User: - return service.create_user(session=db, user_create=UserCreate(email=random_email())) - - -def authentication_token_from_email(*, email: str, db: Session) -> dict[str, str]: - """ - Return a valid token for the user with given email. - - If the user doesn't exist it is created first. - """ - user = service.get_user_by_email(session=db, email=email) - if not user: - user = service.create_user(session=db, user_create=UserCreate(email=email)) - return user_authentication_headers(user) - - -def get_superuser_token_headers(db: Session) -> dict[str, str]: - return authentication_token_from_email(email=settings.FIRST_SUPERUSER, db=db) diff --git a/backend/app/features/catalog/facets.py b/backend/app/features/catalog/facets.py index 5c4af37c..02c4c010 100644 --- a/backend/app/features/catalog/facets.py +++ b/backend/app/features/catalog/facets.py @@ -22,7 +22,7 @@ VoiceArticle, VoiceKind, ) -from app.features.catalog.service import origin_condition +from app.features.catalog.rows import origin_condition def publisher_facets( diff --git a/backend/app/features/catalog/model.py b/backend/app/features/catalog/model.py index 4a6de95b..8b95ad2b 100644 --- a/backend/app/features/catalog/model.py +++ b/backend/app/features/catalog/model.py @@ -14,7 +14,7 @@ class Origin(StrEnum): """The feed's "From" filter: what the article is when that is settled (a repo, a paper, a model), otherwise who published it. Not stored; each - article falls under exactly one, see `catalog.articles.origin_condition`.""" + article falls under exactly one, see `catalog.rows.origin_condition`.""" lab = "lab" company = "company" diff --git a/backend/app/features/catalog/rows.py b/backend/app/features/catalog/rows.py new file mode 100644 index 00000000..11ad2a48 --- /dev/null +++ b/backend/app/features/catalog/rows.py @@ -0,0 +1,118 @@ +"""The ArticlePublic assembly and the origin filter that the feed, search and facets share. + +`ArticlePublic` nests the publisher and the article's tags, so every endpoint +that returns articles (feed, search, liked) needs the same joins and the same +assembly. It lives apart from `service` so `semantic_search` and `facets` can +use it while `service` exposes them. +""" + +from __future__ import annotations + +import uuid +from collections import defaultdict + +from sqlalchemy import ColumnElement +from sqlmodel import Session, col, select + +from app.core.db.article import Article, ArticleKind +from app.core.db.like import ArticleLike +from app.core.db.publisher import Publisher, PublisherKind +from app.core.db.tag import ArticleTag, Tag +from app.features.catalog.model import ( + ArticlePublic, + Origin, + PublisherPublic, + TagPublic, +) + + +def liked_article_ids(session: Session, user_id: uuid.UUID) -> set[int]: + return set( + session.exec( + select(ArticleLike.article_id).where(ArticleLike.user_id == user_id) + ).all() + ) + + +def tags_by_article( + session: Session, article_ids: list[int] +) -> dict[int, list[TagPublic]]: + """One query for every article's tags, keyed by article id.""" + if not article_ids: + return {} + rows = session.exec( + select(ArticleTag.article_id, Tag.slug, Tag.name) + .join(Tag, col(Tag.id) == col(ArticleTag.tag_id)) + .where(col(ArticleTag.article_id).in_(article_ids)) + .order_by(col(ArticleTag.article_id), col(Tag.slug)) + ).all() + out: dict[int, list[TagPublic]] = defaultdict(list) + for article_id, slug, name in rows: + out[article_id].append(TagPublic(slug=slug, name=name)) + return out + + +def to_public( + article: Article, + publisher: Publisher, + like_count: int, + liked_by_me: bool, + tags: list[TagPublic], +) -> ArticlePublic: + assert article.id is not None + assert publisher.id is not None + return ArticlePublic( + id=article.id, + title=article.title, + url=article.url, + summary=article.summary, + score=article.score, + kind=article.kind, + found_via=article.found_via, + published_at=article.published_at, + created_at=article.created_at, + publisher=PublisherPublic( + id=publisher.id, + slug=publisher.slug, + name=publisher.name, + kind=publisher.kind, + image=publisher.image, + ), + tags=tags, + like_count=like_count, + liked_by_me=liked_by_me, + ) + + +def build_rows( + session: Session, rows: list[tuple[Article, Publisher, int]], liked_ids: set[int] +) -> list[ArticlePublic]: + """`rows` are (Article, Publisher, like_count) tuples.""" + tags = tags_by_article(session, [a.id for a, _, _ in rows if a.id is not None]) + out = [] + for a, p, like_count in rows: + assert a.id is not None + out.append(to_public(a, p, like_count, a.id in liked_ids, tags.get(a.id, []))) + return out + + +# The article kinds that are their own origin, whoever published them. +ARTIFACT_KINDS = { + Origin.repo: ArticleKind.repo, + Origin.paper: ArticleKind.paper, + Origin.model: ArticleKind.model, +} + + +def origin_condition(origin: Origin) -> ColumnElement[bool]: + """A repo, paper or model is that whoever published it; a post falls under + its publisher's kind. Community aggregators show under media, and an + `unknown` publisher under none until the curation agent names its kind.""" + if origin in ARTIFACT_KINDS: + return col(Article.kind) == ARTIFACT_KINDS[origin] + kinds = [PublisherKind(origin.value)] + if origin == Origin.media: + kinds.append(PublisherKind.community) + return (col(Article.kind) == ArticleKind.post) & col(Article.publisher_id).in_( + select(Publisher.id).where(col(Publisher.kind).in_(kinds)) + ) diff --git a/backend/app/features/catalog/semantic_search.py b/backend/app/features/catalog/semantic_search.py index 64c0b715..18bd658e 100644 --- a/backend/app/features/catalog/semantic_search.py +++ b/backend/app/features/catalog/semantic_search.py @@ -18,7 +18,7 @@ from app.core.db.like import like_counts_subquery from app.core.db.publisher import Publisher from app.features.catalog.model import ArticlesPublic -from app.features.catalog.service import build_rows, liked_article_ids +from app.features.catalog.rows import build_rows, liked_article_ids # model2vec is CPU-only and small (~30 MB), so one process-wide instance loaded # on first use costs less than a model server. diff --git a/backend/app/features/catalog/service.py b/backend/app/features/catalog/service.py index a76bca1e..2f08d98c 100644 --- a/backend/app/features/catalog/service.py +++ b/backend/app/features/catalog/service.py @@ -1,123 +1,29 @@ -"""The article read model: the feed query, the liked list, and the ArticlePublic assembly they share. +"""The public feed's read verbs: the feed query, the liked list, search, facets and stats. -`ArticlePublic` nests the publisher and the article's tags, so every endpoint -that returns articles (feed, search, liked) needs the same joins and the same -assembly. It lives here so route modules never build it themselves, and so -`semantic_search` can reuse it without importing a route module's privates. +Search and the facet counts live in their own modules (`semantic_search`, +`facets`) and are exposed here, so entry imports one module per feature. """ from __future__ import annotations import uuid -from collections import defaultdict from datetime import datetime from sqlalchemy import ColumnElement, func from sqlmodel import Session, col, select -from app.core.db.article import Article, ArticleKind +from app.core.db.article import Article from app.core.db.like import ArticleLike, like_counts_subquery -from app.core.db.publisher import Publisher, PublisherKind +from app.core.db.publisher import Publisher from app.core.db.tag import ArticleTag, Tag -from app.features.catalog.model import ( - ArticlePublic, - ArticlesPublic, - Origin, - PublisherPublic, - TagPublic, -) - - -def liked_article_ids(session: Session, user_id: uuid.UUID) -> set[int]: - return set( - session.exec( - select(ArticleLike.article_id).where(ArticleLike.user_id == user_id) - ).all() - ) - - -def tags_by_article( - session: Session, article_ids: list[int] -) -> dict[int, list[TagPublic]]: - """One query for every article's tags, keyed by article id.""" - if not article_ids: - return {} - rows = session.exec( - select(ArticleTag.article_id, Tag.slug, Tag.name) - .join(Tag, col(Tag.id) == col(ArticleTag.tag_id)) - .where(col(ArticleTag.article_id).in_(article_ids)) - .order_by(col(ArticleTag.article_id), col(Tag.slug)) - ).all() - out: dict[int, list[TagPublic]] = defaultdict(list) - for article_id, slug, name in rows: - out[article_id].append(TagPublic(slug=slug, name=name)) - return out - - -def to_public( - article: Article, - publisher: Publisher, - like_count: int, - liked_by_me: bool, - tags: list[TagPublic], -) -> ArticlePublic: - assert article.id is not None - assert publisher.id is not None - return ArticlePublic( - id=article.id, - title=article.title, - url=article.url, - summary=article.summary, - score=article.score, - kind=article.kind, - found_via=article.found_via, - published_at=article.published_at, - created_at=article.created_at, - publisher=PublisherPublic( - id=publisher.id, - slug=publisher.slug, - name=publisher.name, - kind=publisher.kind, - image=publisher.image, - ), - tags=tags, - like_count=like_count, - liked_by_me=liked_by_me, - ) - - -def build_rows( - session: Session, rows: list[tuple[Article, Publisher, int]], liked_ids: set[int] -) -> list[ArticlePublic]: - """`rows` are (Article, Publisher, like_count) tuples.""" - tags = tags_by_article(session, [a.id for a, _, _ in rows if a.id is not None]) - out = [] - for a, p, like_count in rows: - assert a.id is not None - out.append(to_public(a, p, like_count, a.id in liked_ids, tags.get(a.id, []))) - return out - - -# The article kinds that are their own origin, whoever published them. -ARTIFACT_KINDS = { - Origin.repo: ArticleKind.repo, - Origin.paper: ArticleKind.paper, - Origin.model: ArticleKind.model, -} - - -def origin_condition(origin: Origin) -> ColumnElement[bool]: - """A repo, paper or model is that whoever published it; a post falls under - its publisher's kind. Community aggregators show under media, and an - `unknown` publisher under none until the curation agent names its kind.""" - if origin in ARTIFACT_KINDS: - return col(Article.kind) == ARTIFACT_KINDS[origin] - kinds = [PublisherKind(origin.value)] - if origin == Origin.media: - kinds.append(PublisherKind.community) - return (col(Article.kind) == ArticleKind.post) & col(Article.publisher_id).in_( - select(Publisher.id).where(col(Publisher.kind).in_(kinds)) - ) +from app.features.catalog.facets import all_facets as all_facets +from app.features.catalog.facets import origin_counts as origin_counts +from app.features.catalog.facets import publisher_facets as publisher_facets +from app.features.catalog.facets import tag_facets as tag_facets +from app.features.catalog.facets import voices as voices +from app.features.catalog.model import ArticlesPublic, Origin +from app.features.catalog.rows import build_rows, liked_article_ids, origin_condition +from app.features.catalog.semantic_search import search as search def _filters( diff --git a/backend/app/features/catalog/tests/__init__.py b/backend/app/features/catalog/tests/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/backend/app/features/collect/hn.py b/backend/app/features/collect/hn.py index 4a22c1f3..0a7f5cb5 100644 --- a/backend/app/features/collect/hn.py +++ b/backend/app/features/collect/hn.py @@ -45,7 +45,7 @@ def hn_grace_hours() -> float: return float(os.environ.get("HN_GRACE_HOURS", "6")) -# The title-only AI gate. Moved here from pipeline/topic_gate.py when the +# The title-only AI gate. Moved here from the old topic gate module when the # pipeline stopped gating feeds by topic: HN is the one source whose firehose # is mostly not about AI, so it keeps the gate. AI_TITLE_KEYWORDS = re.compile( diff --git a/backend/app/features/collect/newsletters.py b/backend/app/features/collect/newsletters.py index 55bf8681..17dafed5 100644 --- a/backend/app/features/collect/newsletters.py +++ b/backend/app/features/collect/newsletters.py @@ -374,7 +374,8 @@ def prefilter( def _probability(answers: Answers, key: str) -> float: - """A yes/no answer as a number. A missing or non-numeric answer is 0.""" + """A yes/no answer as a number. `jev.ask` has already failed the link on a + missing or non-numeric one.""" value = answers.get(key) return float(value) if isinstance(value, int | float) else 0.0 diff --git a/backend/app/features/collect/publishers.py b/backend/app/features/collect/publishers.py index c958f345..2e0310a9 100644 --- a/backend/app/features/collect/publishers.py +++ b/backend/app/features/collect/publishers.py @@ -196,7 +196,8 @@ def _quarantine(self, slug: str, name: str) -> Publisher: ) # Shared hosts whose subdomains are one author's site (writer.substack.com). # Any other shared host's subdomain is the platform itself (old.reddit.com, -# m.youtube.com). +# m.youtube.com), unless a publisher's own link names it +# (engineering.linkedin.com, see `hosts_to_publishers`). PERSONAL_SUBDOMAIN_HOSTS = frozenset({"medium.com", "substack.com"}) _HOST_PLATFORMS = ( @@ -313,7 +314,9 @@ def hosts_to_publishers(publishers: Iterable[Publisher]) -> dict[str, Publisher] host = owner_key(link) or hostname(link) else: continue - if host and not is_shared_host(host): + # Only the bare platform: a subdomain a publisher links to + # (engineering.linkedin.com) is that publisher's own site. + if host and host not in SHARED_HOSTS: claims.setdefault(host, {})[p.id] = p return { host: next(iter(pubs.values())) diff --git a/backend/app/features/collect/queue.py b/backend/app/features/collect/queue.py index 46147ce6..21e17c58 100644 --- a/backend/app/features/collect/queue.py +++ b/backend/app/features/collect/queue.py @@ -3,7 +3,7 @@ The last step of the nightly run. Nothing is scored, summarized or inserted here: a candidate becomes a `pending` row in the reject ledger and waits to be read. The agent (see `.claude/skills/curate/SKILL.md`) approves or rejects it -through the MCP tools in `app/mcp/tools.py`, and only an approval writes an +through the MCP tools in `app/entry/mcp/`, and only an approval writes an Article. Why the ledger and not a table of its own: the row already holds everything a diff --git a/backend/app/features/collect/service.py b/backend/app/features/collect/service.py index 853458c4..35ed4505 100644 --- a/backend/app/features/collect/service.py +++ b/backend/app/features/collect/service.py @@ -23,7 +23,8 @@ from app.features.collect.fetch import build_sources, fetch_source, resolve_publishers from app.features.collect.filter import filter_known_urls from app.features.collect.model import Candidate, RunStats -from app.features.collect.publishers import PublisherResolver +from app.features.collect.publishers import AGGREGATOR_TYPES as AGGREGATOR_TYPES +from app.features.collect.publishers import PublisherResolver as PublisherResolver from app.features.collect.queue import queue_candidates from app.features.collect.report import format_night, read_night, subject from app.features.collect.runs import record_new, record_polled, record_run diff --git a/backend/app/features/collect/tests/test_feeds.py b/backend/app/features/collect/test_feeds.py similarity index 100% rename from backend/app/features/collect/tests/test_feeds.py rename to backend/app/features/collect/test_feeds.py diff --git a/backend/app/features/collect/tests/test_fetch.py b/backend/app/features/collect/test_fetch.py similarity index 100% rename from backend/app/features/collect/tests/test_fetch.py rename to backend/app/features/collect/test_fetch.py diff --git a/backend/app/features/collect/test_filter.py b/backend/app/features/collect/test_filter.py new file mode 100644 index 00000000..be357823 --- /dev/null +++ b/backend/app/features/collect/test_filter.py @@ -0,0 +1,62 @@ +"""The known-URL filter against the real schema: a URL that is an article, a +pending candidate or a settled reject is never fetched and judged again.""" + +from __future__ import annotations + +from collections.abc import Generator + +import pytest +from sqlmodel import Session, col, delete + +from app.core.db.factories import create_random_article, random_lower_string +from app.core.db.ledger import Reject, RejectStage +from app.core.db.publisher import Publisher +from app.features.collect.filter import filter_known_urls +from app.features.collect.model import RawItem + +TAG = random_lower_string()[:8] + + +def _item(path: str) -> RawItem: + return { + "title": path, + "url": f"https://{TAG}.example/{path}", + "content": "", + "published_date": None, + "source": "Hacker News", + } + + +@pytest.fixture +def known(db: Session) -> Generator[None]: + article = create_random_article(db, url=_item("published")["url"]) + db.add_all( + [ + Reject(url=_item("pending")["url"], stage=RejectStage.pending), + Reject(url=_item("rejected")["url"], stage=RejectStage.below_threshold), + ] + ) + db.commit() + yield + db.exec(delete(Reject).where(col(Reject.url).contains(TAG))) + publisher_id = article.publisher_id + db.delete(article) + db.exec(delete(Publisher).where(col(Publisher.id) == publisher_id)) + db.commit() + + +@pytest.mark.usefixtures("known") +def test_only_unseen_urls_pass(db: Session) -> None: + items = [_item(p) for p in ("published", "new", "pending", "rejected", "other")] + fresh = filter_known_urls(db, items, "HN") + assert [a["title"] for a in fresh] == ["new", "other"] + + +@pytest.mark.usefixtures("known") +def test_nothing_new_is_an_empty_list(db: Session) -> None: + items = [_item(p) for p in ("published", "pending", "rejected")] + assert filter_known_urls(db, items, "HN") == [] + + +def test_no_items_reads_nothing(db: Session) -> None: + assert filter_known_urls(db, [], "HN") == [] diff --git a/backend/app/features/collect/tests/test_freshness.py b/backend/app/features/collect/test_freshness.py similarity index 100% rename from backend/app/features/collect/tests/test_freshness.py rename to backend/app/features/collect/test_freshness.py diff --git a/backend/app/features/collect/tests/test_hn.py b/backend/app/features/collect/test_hn.py similarity index 100% rename from backend/app/features/collect/tests/test_hn.py rename to backend/app/features/collect/test_hn.py diff --git a/backend/app/features/collect/tests/test_newsletter_links.py b/backend/app/features/collect/test_newsletters.py similarity index 53% rename from backend/app/features/collect/tests/test_newsletter_links.py rename to backend/app/features/collect/test_newsletters.py index 6e7809b3..5a4fc71b 100644 --- a/backend/app/features/collect/tests/test_newsletter_links.py +++ b/backend/app/features/collect/test_newsletters.py @@ -1,9 +1,15 @@ -"""Newsletter link extraction, rule prefilter and the Jev verdict. - -Everything here is a pure function of a string or a dict: what counts as a -link and what text surrounds it, which links are plumbing by rule, and where -the classification thresholds cut. The thresholds were measured on two real -issues; the table below is their specification. +"""The IMAP newsletter source. + +Sender matching and URL unwrapping: which publisher sent an email, and what +URL an item actually points at. Wrong in the first means a real newsletter is +silently skipped or a stranger's mail is attributed to a publisher; wrong in the +second means the pipeline stores a tracking link that dies with the campaign, or +stores one post twice. + +Link extraction, rule prefilter and the Jev verdict: what counts as a link and +what text surrounds it, which links are plumbing by rule, and where the +classification thresholds cut. The thresholds were measured on two real issues; +the verdict table is their specification. """ from __future__ import annotations @@ -11,9 +17,222 @@ import pytest from app.core import jev -from app.features.collect import newsletters as email -from app.features.collect.newsletters import Link, extract, prefilter, verdict -from baml_client.types import NewsletterItemKind +from app.features.collect import newsletters +from app.features.collect.newsletters import ( + Item, + Link, + _canonical_url, + _match_sender, + _unwrap_tracking_url, + extract, + prefilter, + verdict, +) +from baml_client.types import NewsletterItemKind, ProductLinkChoice, SearchCandidate + +SOURCES = [ + ("@substack.com", "Some Substack"), + ("news@bensbites.com", "Ben's Bites"), +] + + +def test_at_prefix_matches_any_local_part_at_that_exact_domain() -> None: + assert _match_sender("writer@substack.com", SOURCES) == "Some Substack" + + +def test_at_prefix_does_not_match_a_deeper_subdomain() -> None: + """ "@substack.com" matches "anyone@substack.com" only — the check is a + string suffix on the whole address, so a subdomain like "foo.substack.com" + changes what precedes "substack.com" and no longer ends in "@substack.com".""" + assert _match_sender("writer@foo.substack.com", SOURCES) is None + + +@pytest.mark.parametrize( + "address", + ["writer@notsubstack.com", "writer@evilsubstack.com"], +) +def test_at_prefix_does_not_match_a_lookalike_domain(address: str) -> None: + """Confirms the check is a suffix match requiring the literal "@" right + before the domain, not a raw substring search for "substack.com".""" + assert _match_sender(address, SOURCES) is None + + +def test_exact_pattern_requires_an_exact_address_match() -> None: + assert _match_sender("news@bensbites.com", SOURCES) == "Ben's Bites" + + +def test_exact_pattern_does_not_match_a_different_local_part_at_the_same_domain() -> ( + None +): + assert _match_sender("other@bensbites.com", SOURCES) is None + + +def test_matching_is_case_insensitive() -> None: + assert _match_sender("Writer@Substack.COM", SOURCES) == "Some Substack" + + +def test_no_match_returns_none() -> None: + assert _match_sender("someone@example.com", SOURCES) is None + + +def test_empty_sources_never_matches() -> None: + assert _match_sender("writer@substack.com", []) is None + + +# ─── tracker unwrapping ────────────────────────────────────────────────────── +# A newsletter links through its own redirector. The destination is what the +# pipeline stores, so recovering it from the wrapper — without a request, when +# the wrapper spells it out — is the whole job of this half of the source. + + +def test_unwraps_a_percent_encoded_destination() -> None: + assert ( + _unwrap_tracking_url( + "https://tracking.tldrnewsletter.com/CL0/" + "https:%2F%2Fexample.com%2Fposts%2Fone/1/0100019a626e0311" + ) + == "https://example.com/posts/one" + ) + + +def test_unwraps_a_literal_destination_with_its_own_path_separators() -> None: + assert ( + _unwrap_tracking_url( + "https://tracking.tldrnewsletter.com/CL0/" + "https://aws.amazon.com/build/ai-series/1/0100019a626e0311" + ) + == "https://aws.amazon.com/build/ai-series" + ) + + +def test_keeps_the_destination_query_while_dropping_the_tracker_tail() -> None: + """The tail is the tracker's message id and click token, appended after the + destination; the destination's own query must survive it.""" + assert ( + _unwrap_tracking_url( + "https://tracking.tldrnewsletter.com/CL0/" + "https:%2F%2Fexample.com%2Fp%3Fid%3D7/1/0100019a626e0311" + ) + == "https://example.com/p?id=7" + ) + + +def test_an_opaque_token_cannot_be_unwrapped() -> None: + """Nothing to recover without a request — the caller follows the redirect.""" + assert _unwrap_tracking_url("https://links.tldrnewsletter.com/zPfy8Y") is None + + +def test_a_direct_link_is_not_a_wrapper() -> None: + assert _unwrap_tracking_url("https://example.com/blog/post") is None + + +# ─── URL canonicalization ──────────────────────────────────────────────────── +# Two links to one post must reduce to one string, or dedup stores it twice. + + +def test_drops_campaign_params_and_keeps_the_rest() -> None: + assert ( + _canonical_url("https://example.com/p?id=7&utm_source=tldr&page=2") + == "https://example.com/p?id=7&page=2" + ) + + +def test_drops_campaign_params_hidden_after_a_fragment() -> None: + """Newsletters append their params after the anchor, where a query parser + never looks.""" + assert ( + _canonical_url("https://example.com/p/#section?utm_source=tldrai") + == "https://example.com/p#section" + ) + + +def test_a_trailing_slash_does_not_make_a_second_url() -> None: + assert _canonical_url("https://example.com/p/") == _canonical_url( + "https://example.com/p" + ) + + +def test_the_host_is_lowercased_but_the_path_is_not() -> None: + """Hosts are case-insensitive; paths are not, and lowercasing one would + 404.""" + assert ( + _canonical_url("https://Example.COM/Posts/One") + == "https://example.com/Posts/One" + ) + + +def test_a_bare_root_url_keeps_its_slash() -> None: + assert _canonical_url("https://example.com/") == "https://example.com/" + + +# ─── resolving a product by search ─────────────────────────────────────────── + + +def _product() -> Item: + return { + "kind": NewsletterItemKind.Product, + "name": "Acme SDK", + "description": "An SDK for agents.", + "url": "https://www.example-writeup.com/acme", + "first_party": False, + "email_date": "2026-09-23T07:00:00+00:00", + "newsletter_name": "TLDR AI", + } + + +class _Picks: + """Stands in for the BAML client: picks ``index`` from what it is shown.""" + + def __init__(self, index: int) -> None: + self.index = index + + def SelectProductLink( + self, name: str, description: str, candidates: list[SearchCandidate] + ) -> ProductLinkChoice: + return ProductLinkChoice(index=self.index, confidence=0.9) + + +def _stub_search(monkeypatch: pytest.MonkeyPatch, index: int) -> None: + results: list[dict[str, str | None]] = [ + { + "title": "Acme SDK", + "url": "https://acme.dev/sdk/", + "description": "Acme's agent SDK.", + "published_date": None, + }, + { + "title": "Acme on X", + "url": "https://x.com/acme/status/1", + "description": "denied", + "published_date": None, + }, + ] + monkeypatch.setattr(newsletters, "tavily_search", lambda query: results) + monkeypatch.setattr(newsletters, "b", _Picks(index)) + + +def test_a_product_resolves_to_the_result_picked( + monkeypatch: pytest.MonkeyPatch, +) -> None: + _stub_search(monkeypatch, 1) + item = newsletters._resolve_product(_product()) + assert item is not None + assert item["url"] == "https://acme.dev/sdk" + assert item["source"] == "TLDR AI" + + +@pytest.mark.parametrize("index", [0, 2, 9]) +def test_no_pick_or_a_pick_past_the_list_drops_the_product( + monkeypatch: pytest.MonkeyPatch, index: int +) -> None: + """The model never sees the denied x.com result, so it picks from a list + of one. Any other index is no pick. Before the bounds check it raised + IndexError, which ended the whole night's newsletter source.""" + _stub_search(monkeypatch, index) + assert newsletters._resolve_product(_product()) is None + + +# --- link extraction, prefilter and verdict --- ZW = "​‌‍­" * 200 @@ -53,7 +272,7 @@ def test_the_context_window_reaches_the_description_after_the_link() -> None: def test_zero_width_padding_never_reaches_the_context() -> None: for link in extract(FIXTURE): - assert not email._ZERO_WIDTH_RE.search(link.context) + assert not newsletters._ZERO_WIDTH_RE.search(link.context) first = extract(FIXTURE)[0] assert first.context.startswith("Today's issue Why agent harnesses fail") @@ -79,7 +298,10 @@ def test_the_window_is_bounded_on_both_sides() -> None: html = f"{'x ' * 400}A{' y' * 400}" html = html.replace("'", '"') [link] = extract(html) - assert len(link.context) <= email.CONTEXT_BEFORE + len("A") + email.CONTEXT_AFTER + assert ( + len(link.context) + <= newsletters.CONTEXT_BEFORE + len("A") + newsletters.CONTEXT_AFTER + ) # ─── prefilter ─────────────────────────────────────────────────────────────── @@ -217,7 +439,9 @@ def ask(state: dict[str, str], _questions: object) -> jev.Reply: for i, anchor in enumerate(answers) ) - items = email._extract_items(html, "TLDR AI", "2026-09-23", "tldrnewsletter.com") + items = newsletters._extract_items( + html, "TLDR AI", "2026-09-23", "tldrnewsletter.com" + ) assert len(seen_states) == 3 assert {s["destination_host"] for s in seen_states} == { @@ -243,7 +467,7 @@ def ask(state: dict[str, str], _questions: object) -> jev.Reply: monkeypatch.setattr(jev, "ask", ask) html = 'Broken Fine' - items = email._extract_items(html, "TLDR AI", "2026-09-23", "tldr.tech") + items = newsletters._extract_items(html, "TLDR AI", "2026-09-23", "tldr.tech") assert [i["name"] for i in items] == ["Fine"] @@ -261,7 +485,7 @@ def test_an_opaque_tracker_is_never_first_party( ) html = 'Acme SDK' - [item] = email._extract_items(html, "Pointer", "2026-09-23", "pointer.io") + [item] = newsletters._extract_items(html, "Pointer", "2026-09-23", "pointer.io") assert item["first_party"] is False @@ -289,4 +513,4 @@ def test_an_article_about_privacy_is_not_a_privacy_policy() -> None: def test_own_host_ignores_shared_sending_platforms(sender: str, own: str) -> None: """A Substack sender's own host would otherwise drop every post on every other Substack.""" - assert email._own_host(sender) == own + assert newsletters._own_host(sender) == own diff --git a/backend/app/features/collect/tests/test_publishers.py b/backend/app/features/collect/test_publishers.py similarity index 61% rename from backend/app/features/collect/tests/test_publishers.py rename to backend/app/features/collect/test_publishers.py index 2584d846..47b8ca1f 100644 --- a/backend/app/features/collect/tests/test_publishers.py +++ b/backend/app/features/collect/test_publishers.py @@ -2,15 +2,18 @@ from __future__ import annotations +import httpx import pytest from app.core.db.publisher import LinkPlatform, Publisher, PublisherKind +from app.features.collect import publishers from app.features.collect.publishers import ( host_kind, hosts_to_publishers, is_shared_host, match_publisher, owner_key, + owner_kind, ) @@ -95,6 +98,7 @@ def test_a_hugging_face_blog_feed_claims_the_blog_not_the_models() -> None: ("https://medium.com/@someone/a-post-123", "medium.com/@someone"), ("https://medium.com/tag/ai", None), ("https://simonwillison.net/2026/", None), + ("https://github.com/", None), ], ) def test_owner_key(url: str, key: str | None) -> None: @@ -126,11 +130,84 @@ def test_a_shared_platform_includes_its_subdomains(host: str, shared: bool) -> N assert is_shared_host(host) is shared -def test_a_publisher_on_a_platform_subdomain_claims_nothing() -> None: - sub = _pub(11, website="https://old.reddit.com/r/LocalLLaMA") - assert _credited("https://old.reddit.com/r/LocalLLaMA/comments/x", sub) is None +@pytest.mark.parametrize( + ("link", "url"), + [ + ("https://engineering.linkedin.com", "https://engineering.linkedin.com/blog/a"), + ("https://docs.github.com/en", "https://docs.github.com/en/copilot/agents"), + ("https://blog.x.com", "https://blog.x.com/engineering/en_us/a-post"), + ], +) +def test_a_publishers_own_link_claims_a_platform_subdomain(link: str, url: str) -> None: + """A platform's first-party site is the publisher's own, unlike the bare + platform host.""" + assert _credited(url, _pub(11, website=link)) == 11 def test_a_substack_writer_still_claims_their_own_subdomain() -> None: writer = _pub(12, substack="https://writer.substack.com") assert _credited("https://writer.substack.com/p/a-post", writer) == 12 + + +def test_only_a_publishers_site_links_claim_a_host() -> None: + """An email sender or an unsaved row names no host a URL could be on.""" + sender = _pub(13, email="@example.com") + unsaved = Publisher( + slug="u", + name="U", + kind=PublisherKind.media, + links={LinkPlatform.website: "https://u.dev"}, + ) + assert hosts_to_publishers([sender, unsaved]) == {} + + +@pytest.mark.parametrize( + "key, kind", + [ + ("medium.com/@someone", PublisherKind.individual), + ("medium.com/airbnb-engineering", PublisherKind.unknown), + ("huggingface.co/blog", PublisherKind.unknown), + ("example.com/someone", PublisherKind.unknown), + ], +) +def test_owner_kind_without_a_lookup(key: str, kind: PublisherKind) -> None: + assert owner_kind(key) == kind + + +@pytest.mark.parametrize( + "account, kind", + [("User", PublisherKind.individual), ("Organization", PublisherKind.unknown)], +) +def test_a_github_owner_is_an_individual_only_when_it_is_a_user( + monkeypatch: pytest.MonkeyPatch, account: str, kind: PublisherKind +) -> None: + monkeypatch.setattr(publishers, "account_type", lambda owner: account) + assert owner_kind("github.com/someone") == kind + + +@pytest.mark.parametrize( + "status, kind", + [(200, PublisherKind.individual), (404, PublisherKind.unknown)], +) +def test_a_hugging_face_owner_is_an_individual_when_it_has_a_user_page( + monkeypatch: pytest.MonkeyPatch, status: int, kind: PublisherKind +) -> None: + seen: list[str] = [] + + def fetch(url: str, **_: object) -> httpx.Response: + seen.append(url) + return httpx.Response(status) + + monkeypatch.setattr(publishers, "fetch_with_timeout", fetch) + assert owner_kind("huggingface.co/someone") == kind + assert seen == ["https://huggingface.co/api/users/someone/overview"] + + +def test_a_failed_hugging_face_lookup_is_unknown( + monkeypatch: pytest.MonkeyPatch, +) -> None: + def fetch(*_: object, **__: object) -> httpx.Response: + raise httpx.ConnectError("offline") + + monkeypatch.setattr(publishers, "fetch_with_timeout", fetch) + assert owner_kind("huggingface.co/someone") == PublisherKind.unknown diff --git a/backend/app/features/collect/tests/test_publishers_db.py b/backend/app/features/collect/test_publishers_db.py similarity index 65% rename from backend/app/features/collect/tests/test_publishers_db.py rename to backend/app/features/collect/test_publishers_db.py index 82e5f60f..c91173cb 100644 --- a/backend/app/features/collect/tests/test_publishers_db.py +++ b/backend/app/features/collect/test_publishers_db.py @@ -8,10 +8,15 @@ import pytest from sqlmodel import Session, col, delete, select +from app.core.db.factories import random_lower_string from app.core.db.publisher import LinkPlatform, Publisher, PublisherKind, PublisherType from app.features.collect import publishers as publishers_module -from app.features.collect.publishers import PublisherResolver -from tests.random_data import random_lower_string +from app.features.collect.publishers import ( + PublisherResolver, + feed_sources_from_db, + lab_watch_targets_from_db, + newsletter_senders_from_db, +) TAG = random_lower_string()[:8] @@ -107,6 +112,7 @@ def test_a_shared_host_that_names_no_author_stays_with_the_aggregator( "https://m.youtube.com/watch?v=abc", "https://youtu.be/abc", "https://mobile.twitter.com/someone/status/1", + "https://gist.github.com/someone/abc", ], ) def test_a_shared_platforms_subdomain_names_no_author_either( @@ -122,6 +128,26 @@ def test_a_shared_platforms_subdomain_names_no_author_either( assert resolver.created == [] +def test_a_platform_subdomain_a_publisher_links_goes_to_that_publisher( + db: Session, sources: dict[str, Publisher] +) -> None: + """The explicit link wins over the rule that keeps a platform's subdomains + with the aggregator.""" + engineering = Publisher( + name=f"{TAG} Engineering", + slug=f"{TAG}-engineering", + kind=PublisherKind.company, + type=PublisherType.rss, + links={LinkPlatform.website: f"https://{TAG}.linkedin.com"}, + ) + db.add(engineering) + db.commit() + resolver = PublisherResolver(db) + url = f"https://{TAG}.linkedin.com/blog/a-post" + assert resolver.publisher_for(url, sources["aggregator"].name).id == engineering.id + assert resolver.created == [] + + def test_a_feeds_own_item_stays_with_the_feed( db: Session, sources: dict[str, Publisher] ) -> None: @@ -174,3 +200,61 @@ def test_a_url_on_a_known_site_beats_the_feed_that_carried_it( assert ( resolver.publisher_for(url, sources["aggregator"].name).id == sources["feed"].id ) + + +@pytest.fixture +def polled(db: Session) -> Generator[dict[str, Publisher]]: + """Active publishers with each kind of pollable link, and one switched off.""" + + def pub(name: str, active: bool = True, **links: str) -> Publisher: + return Publisher( + name=f"{TAG} {name}", + slug=f"{TAG}-{name}", + kind=PublisherKind.media, + is_active=active, + links={LinkPlatform(k): v for k, v in links.items()}, + ) + + made = { + "both": pub( + "both", + rss=f"https://{TAG}-both.com/feed.xml", + substack=f"https://{TAG}both.substack.com", + ), + "substack": pub("substack", substack=f"https://{TAG}stack.substack.com"), + "lab": pub("lab", search=f"{TAG}-lab.ai"), + "letter": pub("letter", email=f"@{TAG}-letter.com"), + "off": pub( + "off", + active=False, + rss=f"https://{TAG}-off.com/feed.xml", + search=f"{TAG}-off.ai", + email=f"@{TAG}-off.com", + ), + } + db.add_all(made.values()) + db.commit() + yield made + db.exec(delete(Publisher).where(col(Publisher.slug).contains(TAG))) + db.commit() + + +@pytest.mark.usefixtures("polled") +def test_feeds_come_from_active_publishers_preferring_rss(db: Session) -> None: + feeds = [f for f in feed_sources_from_db(db) if f["name"].startswith(TAG)] + assert sorted(feeds, key=lambda f: f["name"]) == [ + {"name": f"{TAG} both", "rssUrl": f"https://{TAG}-both.com/feed.xml"}, + {"name": f"{TAG} substack", "rssUrl": f"https://{TAG}stack.substack.com/feed"}, + ] + + +@pytest.mark.usefixtures("polled") +def test_lab_watch_targets_come_from_active_search_links(db: Session) -> None: + targets = [t for t in lab_watch_targets_from_db(db) if t[0].startswith(TAG)] + assert targets == [(f"{TAG} lab", f"{TAG}-lab.ai")] + + +@pytest.mark.usefixtures("polled") +def test_newsletter_senders_come_from_active_email_links(db: Session) -> None: + senders = [s for s in newsletter_senders_from_db(db) if s[1].startswith(TAG)] + assert senders == [(f"@{TAG}-letter.com", f"{TAG} letter")] diff --git a/backend/app/features/collect/tests/test_queue.py b/backend/app/features/collect/test_queue.py similarity index 100% rename from backend/app/features/collect/tests/test_queue.py rename to backend/app/features/collect/test_queue.py diff --git a/backend/app/features/collect/tests/test_report.py b/backend/app/features/collect/test_report.py similarity index 100% rename from backend/app/features/collect/tests/test_report.py rename to backend/app/features/collect/test_report.py diff --git a/backend/app/features/collect/tests/test_runs.py b/backend/app/features/collect/test_runs.py similarity index 98% rename from backend/app/features/collect/tests/test_runs.py rename to backend/app/features/collect/test_runs.py index cf6587d5..71c9cdc4 100644 --- a/backend/app/features/collect/tests/test_runs.py +++ b/backend/app/features/collect/test_runs.py @@ -9,12 +9,12 @@ from sqlmodel import Session, col, delete, select from app.core.db.article import Article +from app.core.db.factories import random_lower_string from app.core.db.publisher import Publisher, PublisherKind, PublisherType from app.core.db.run import PipelineRun from app.features.collect.model import RunStats from app.features.collect.report import read_night from app.features.collect.runs import record_new, record_polled, record_run -from tests.random_data import random_lower_string TAG = random_lower_string()[:8] EARLIER = datetime(2026, 9, 20, 4, 0, tzinfo=UTC) diff --git a/backend/app/features/collect/tests/test_service.py b/backend/app/features/collect/test_service.py similarity index 93% rename from backend/app/features/collect/tests/test_service.py rename to backend/app/features/collect/test_service.py index 0986a29f..8ec1f4dd 100644 --- a/backend/app/features/collect/tests/test_service.py +++ b/backend/app/features/collect/test_service.py @@ -297,3 +297,31 @@ def broken(_session: object) -> None: monkeypatch.setattr(run_module, "read_night", broken) assert run_module.send_report() == 1 + + +def test_a_report_is_read_from_the_db_and_mailed( + monkeypatch: pytest.MonkeyPatch, +) -> None: + sent: list[tuple[str, str]] = [] + monkeypatch.setattr( + run_module, "send_mail", lambda subject, body: sent.append((subject, body)) + ) + + assert run_module.send_report() == 0 + [(subject, body)] = sent + assert subject and body + + +def test_a_report_that_cannot_be_sent_exits_one( + monkeypatch: pytest.MonkeyPatch, +) -> None: + tried: list[str] = [] + + def broken(subject: str, _body: str) -> None: + tried.append(subject) + raise RuntimeError("resend down") + + monkeypatch.setattr(run_module, "send_mail", broken) + + assert run_module.send_report() == 1 + assert len(tried) == 1 diff --git a/backend/app/features/collect/tests/test_titles.py b/backend/app/features/collect/test_titles.py similarity index 100% rename from backend/app/features/collect/tests/test_titles.py rename to backend/app/features/collect/test_titles.py diff --git a/backend/app/features/collect/tests/test_tweets.py b/backend/app/features/collect/test_tweets.py similarity index 100% rename from backend/app/features/collect/tests/test_tweets.py rename to backend/app/features/collect/test_tweets.py diff --git a/backend/app/features/collect/tests/__init__.py b/backend/app/features/collect/tests/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/backend/app/features/collect/tests/test_newsletters.py b/backend/app/features/collect/tests/test_newsletters.py deleted file mode 100644 index 96fe5c38..00000000 --- a/backend/app/features/collect/tests/test_newsletters.py +++ /dev/null @@ -1,220 +0,0 @@ -"""The IMAP newsletter source's two decisions that are pure functions of text: -which publisher sent an email, and what URL an item actually points at. Wrong -in the first means a real newsletter is silently skipped or a stranger's mail -is attributed to a publisher; wrong in the second means the pipeline stores a -tracking link that dies with the campaign, or stores one post twice. -""" - -from __future__ import annotations - -import pytest - -from app.features.collect import newsletters -from app.features.collect.newsletters import ( - Item, - _canonical_url, - _match_sender, - _unwrap_tracking_url, -) -from baml_client.types import NewsletterItemKind, ProductLinkChoice, SearchCandidate - -SOURCES = [ - ("@substack.com", "Some Substack"), - ("news@bensbites.com", "Ben's Bites"), -] - - -def test_at_prefix_matches_any_local_part_at_that_exact_domain() -> None: - assert _match_sender("writer@substack.com", SOURCES) == "Some Substack" - - -def test_at_prefix_does_not_match_a_deeper_subdomain() -> None: - """ "@substack.com" matches "anyone@substack.com" only — the check is a - string suffix on the whole address, so a subdomain like "foo.substack.com" - changes what precedes "substack.com" and no longer ends in "@substack.com".""" - assert _match_sender("writer@foo.substack.com", SOURCES) is None - - -@pytest.mark.parametrize( - "address", - ["writer@notsubstack.com", "writer@evilsubstack.com"], -) -def test_at_prefix_does_not_match_a_lookalike_domain(address: str) -> None: - """Confirms the check is a suffix match requiring the literal "@" right - before the domain, not a raw substring search for "substack.com".""" - assert _match_sender(address, SOURCES) is None - - -def test_exact_pattern_requires_an_exact_address_match() -> None: - assert _match_sender("news@bensbites.com", SOURCES) == "Ben's Bites" - - -def test_exact_pattern_does_not_match_a_different_local_part_at_the_same_domain() -> ( - None -): - assert _match_sender("other@bensbites.com", SOURCES) is None - - -def test_matching_is_case_insensitive() -> None: - assert _match_sender("Writer@Substack.COM", SOURCES) == "Some Substack" - - -def test_no_match_returns_none() -> None: - assert _match_sender("someone@example.com", SOURCES) is None - - -def test_empty_sources_never_matches() -> None: - assert _match_sender("writer@substack.com", []) is None - - -# ─── tracker unwrapping ────────────────────────────────────────────────────── -# A newsletter links through its own redirector. The destination is what the -# pipeline stores, so recovering it from the wrapper — without a request, when -# the wrapper spells it out — is the whole job of this half of the source. - - -def test_unwraps_a_percent_encoded_destination() -> None: - assert ( - _unwrap_tracking_url( - "https://tracking.tldrnewsletter.com/CL0/" - "https:%2F%2Fexample.com%2Fposts%2Fone/1/0100019a626e0311" - ) - == "https://example.com/posts/one" - ) - - -def test_unwraps_a_literal_destination_with_its_own_path_separators() -> None: - assert ( - _unwrap_tracking_url( - "https://tracking.tldrnewsletter.com/CL0/" - "https://aws.amazon.com/build/ai-series/1/0100019a626e0311" - ) - == "https://aws.amazon.com/build/ai-series" - ) - - -def test_keeps_the_destination_query_while_dropping_the_tracker_tail() -> None: - """The tail is the tracker's message id and click token, appended after the - destination; the destination's own query must survive it.""" - assert ( - _unwrap_tracking_url( - "https://tracking.tldrnewsletter.com/CL0/" - "https:%2F%2Fexample.com%2Fp%3Fid%3D7/1/0100019a626e0311" - ) - == "https://example.com/p?id=7" - ) - - -def test_an_opaque_token_cannot_be_unwrapped() -> None: - """Nothing to recover without a request — the caller follows the redirect.""" - assert _unwrap_tracking_url("https://links.tldrnewsletter.com/zPfy8Y") is None - - -def test_a_direct_link_is_not_a_wrapper() -> None: - assert _unwrap_tracking_url("https://example.com/blog/post") is None - - -# ─── URL canonicalization ──────────────────────────────────────────────────── -# Two links to one post must reduce to one string, or dedup stores it twice. - - -def test_drops_campaign_params_and_keeps_the_rest() -> None: - assert ( - _canonical_url("https://example.com/p?id=7&utm_source=tldr&page=2") - == "https://example.com/p?id=7&page=2" - ) - - -def test_drops_campaign_params_hidden_after_a_fragment() -> None: - """Newsletters append their params after the anchor, where a query parser - never looks.""" - assert ( - _canonical_url("https://example.com/p/#section?utm_source=tldrai") - == "https://example.com/p#section" - ) - - -def test_a_trailing_slash_does_not_make_a_second_url() -> None: - assert _canonical_url("https://example.com/p/") == _canonical_url( - "https://example.com/p" - ) - - -def test_the_host_is_lowercased_but_the_path_is_not() -> None: - """Hosts are case-insensitive; paths are not, and lowercasing one would - 404.""" - assert ( - _canonical_url("https://Example.COM/Posts/One") - == "https://example.com/Posts/One" - ) - - -def test_a_bare_root_url_keeps_its_slash() -> None: - assert _canonical_url("https://example.com/") == "https://example.com/" - - -# ─── resolving a product by search ─────────────────────────────────────────── - - -def _product() -> Item: - return { - "kind": NewsletterItemKind.Product, - "name": "Acme SDK", - "description": "An SDK for agents.", - "url": "https://www.example-writeup.com/acme", - "first_party": False, - "email_date": "2026-09-23T07:00:00+00:00", - "newsletter_name": "TLDR AI", - } - - -class _Picks: - """Stands in for the BAML client: picks ``index`` from what it is shown.""" - - def __init__(self, index: int) -> None: - self.index = index - - def SelectProductLink( - self, name: str, description: str, candidates: list[SearchCandidate] - ) -> ProductLinkChoice: - return ProductLinkChoice(index=self.index, confidence=0.9) - - -def _stub_search(monkeypatch: pytest.MonkeyPatch, index: int) -> None: - results: list[dict[str, str | None]] = [ - { - "title": "Acme SDK", - "url": "https://acme.dev/sdk/", - "description": "Acme's agent SDK.", - "published_date": None, - }, - { - "title": "Acme on X", - "url": "https://x.com/acme/status/1", - "description": "denied", - "published_date": None, - }, - ] - monkeypatch.setattr(newsletters, "tavily_search", lambda query: results) - monkeypatch.setattr(newsletters, "b", _Picks(index)) - - -def test_a_product_resolves_to_the_result_picked( - monkeypatch: pytest.MonkeyPatch, -) -> None: - _stub_search(monkeypatch, 1) - item = newsletters._resolve_product(_product()) - assert item is not None - assert item["url"] == "https://acme.dev/sdk" - assert item["source"] == "TLDR AI" - - -@pytest.mark.parametrize("index", [0, 2, 9]) -def test_no_pick_or_a_pick_past_the_list_drops_the_product( - monkeypatch: pytest.MonkeyPatch, index: int -) -> None: - """The model never sees the denied x.com result, so it picks from a list - of one. Any other index is no pick. Before the bounds check it raised - IndexError, which ended the whole night's newsletter source.""" - _stub_search(monkeypatch, index) - assert newsletters._resolve_product(_product()) is None diff --git a/backend/app/features/curation/links.py b/backend/app/features/curation/links.py index 264b2b9d..0a76aa39 100644 --- a/backend/app/features/curation/links.py +++ b/backend/app/features/curation/links.py @@ -15,6 +15,7 @@ from app.core.github import github_repo_from_url, has_readme, repo_for from app.core.http import fetch_with_timeout from app.core.urls import hostname +from app.features.curation.model import LinkError # What any lookup that did not answer reads as. Never 0 or False: a rate limit # must not look like a repo nobody starred or a model with no weights. @@ -37,10 +38,6 @@ type Facts = dict[str, object] -class LinkError(Exception): - """A URL ``check_link`` does not read. The message names the tool that does.""" - - def _or_unknown(value: object) -> object: return UNKNOWN if value is None or value == "" else value diff --git a/backend/app/features/curation/model.py b/backend/app/features/curation/model.py index 8eccc6cc..d024e737 100644 --- a/backend/app/features/curation/model.py +++ b/backend/app/features/curation/model.py @@ -18,6 +18,10 @@ class CandidateError(Exception): or never queued. The agent gets the message, not a traceback.""" +class LinkError(Exception): + """A URL ``check_link`` does not read. The message names the tool that does.""" + + class CandidatePage(TypedDict): """One page of the pending queue. ``next_offset`` is None on the last page.""" diff --git a/backend/app/features/curation/service.py b/backend/app/features/curation/service.py index 406cbeb4..b1b9ca34 100644 --- a/backend/app/features/curation/service.py +++ b/backend/app/features/curation/service.py @@ -2,7 +2,7 @@ reject it. The nightly collect run ends at a `pending` row. This module is the other end: -the verbs the agent drives it with, behind the MCP tools in `app/mcp/tools.py`. +the verbs the agent drives it with, behind the MCP tools in `app/entry/mcp/`. Approving is the only thing in the codebase that writes an Article, so the insert-and-embed tail (`publish.py`) belongs to curation alone. @@ -42,6 +42,7 @@ ) from app.core.logging import log from app.features.curation import coverage +from app.features.curation.links import check_link as check_link from app.features.curation.model import ( CandidateError, CandidatePage, diff --git a/backend/app/features/curation/tests/test_links.py b/backend/app/features/curation/test_links.py similarity index 99% rename from backend/app/features/curation/tests/test_links.py rename to backend/app/features/curation/test_links.py index d687ab0c..7a9d7901 100644 --- a/backend/app/features/curation/tests/test_links.py +++ b/backend/app/features/curation/test_links.py @@ -14,7 +14,8 @@ from app.core import github from app.core.extract import MAX_LINKS_PER_GROUP, group_links, outbound_links from app.features.curation import links as page_facts -from app.features.curation.links import UNKNOWN, LinkError, check_link +from app.features.curation.links import UNKNOWN, check_link +from app.features.curation.model import LinkError PAGE = "https://blog.example.com/2026/09/release" diff --git a/backend/app/features/curation/tests/test_publish.py b/backend/app/features/curation/test_publish.py similarity index 100% rename from backend/app/features/curation/tests/test_publish.py rename to backend/app/features/curation/test_publish.py diff --git a/backend/app/features/curation/tests/test_service.py b/backend/app/features/curation/test_service.py similarity index 100% rename from backend/app/features/curation/tests/test_service.py rename to backend/app/features/curation/test_service.py diff --git a/backend/app/features/curation/tests/test_service_db.py b/backend/app/features/curation/test_service_db.py similarity index 99% rename from backend/app/features/curation/tests/test_service_db.py rename to backend/app/features/curation/test_service_db.py index 8d48cd35..a39dcc14 100644 --- a/backend/app/features/curation/tests/test_service_db.py +++ b/backend/app/features/curation/test_service_db.py @@ -18,13 +18,13 @@ from sqlmodel import Session, col, delete, func, select from app.core.db.article import Article +from app.core.db.factories import random_lower_string from app.core.db.ledger import Reject, RejectStage from app.core.db.publisher import Publisher, PublisherKind, PublisherType from app.core.db.tag import ArticleTag, Tag from app.features.curation import coverage, publish from app.features.curation import service as curation from app.features.curation.model import CandidateError -from tests.random_data import random_lower_string DIM = 256 TAG = random_lower_string()[:8] diff --git a/backend/app/features/curation/tests/test_tags.py b/backend/app/features/curation/test_tags.py similarity index 96% rename from backend/app/features/curation/tests/test_tags.py rename to backend/app/features/curation/test_tags.py index c5f21f72..f5e57404 100644 --- a/backend/app/features/curation/tests/test_tags.py +++ b/backend/app/features/curation/test_tags.py @@ -1,6 +1,6 @@ """validate_tags: the last line of defence between whatever the LLM returns and what gets written as ArticleTag rows. Off-list tags must never be written, and -no tag is ever minted at runtime — see pipeline/tags.py. +no tag is ever minted at runtime. See app/features/curation/tags.py. """ from __future__ import annotations diff --git a/backend/app/features/curation/tests/__init__.py b/backend/app/features/curation/tests/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/backend/app/features/newsletter/service.py b/backend/app/features/newsletter/service.py index f5226c43..21f443e0 100644 --- a/backend/app/features/newsletter/service.py +++ b/backend/app/features/newsletter/service.py @@ -43,6 +43,7 @@ story_groups, to_embedding_text, ) +from app.features.newsletter.broadcast import draft as draft from app.features.newsletter.model import ( CoveredBy, DigestError, diff --git a/backend/app/features/newsletter/tests/test_broadcast.py b/backend/app/features/newsletter/test_broadcast.py similarity index 100% rename from backend/app/features/newsletter/tests/test_broadcast.py rename to backend/app/features/newsletter/test_broadcast.py diff --git a/backend/app/features/newsletter/tests/test_service.py b/backend/app/features/newsletter/test_service.py similarity index 98% rename from backend/app/features/newsletter/tests/test_service.py rename to backend/app/features/newsletter/test_service.py index 15b38d7a..e8527fd2 100644 --- a/backend/app/features/newsletter/tests/test_service.py +++ b/backend/app/features/newsletter/test_service.py @@ -17,13 +17,12 @@ from sqlmodel import Session, col, delete from app.core.db.article import Article, ArticleKind +from app.core.db.factories import create_random_user, random_lower_string from app.core.db.ledger import Reject, RejectStage from app.core.db.like import ArticleLike from app.core.db.publisher import Publisher, PublisherKind, PublisherType -from app.features.audience.tests.factories import create_random_user from app.features.newsletter import service as digest from app.features.newsletter.model import DigestError, WeekTopic -from tests.random_data import random_lower_string DIM = 256 TAG = random_lower_string()[:8] diff --git a/backend/app/features/newsletter/tests/test_subscribe.py b/backend/app/features/newsletter/test_subscribe.py similarity index 97% rename from backend/app/features/newsletter/tests/test_subscribe.py rename to backend/app/features/newsletter/test_subscribe.py index a67e64ca..2882c3ec 100644 --- a/backend/app/features/newsletter/tests/test_subscribe.py +++ b/backend/app/features/newsletter/test_subscribe.py @@ -6,9 +6,9 @@ import resend from sqlmodel import Session +from app.core.db.factories import random_email from app.core.db.subscriber import NewsletterSubscriber from app.features.newsletter.service import subscribe -from tests.random_data import random_email def test_subscribe_defaults_to_all( diff --git a/backend/app/features/newsletter/tests/__init__.py b/backend/app/features/newsletter/tests/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/backend/app/features/stories/service.py b/backend/app/features/stories/service.py index 11c13332..2066815b 100644 --- a/backend/app/features/stories/service.py +++ b/backend/app/features/stories/service.py @@ -3,7 +3,7 @@ A story is wider than coverage. Coverage is one event told by several outlets, within `DEDUP_DIST_THRESHOLD`; a story is a theme a builder follows, which takes judgment, so an agent names and keeps them (ADR 16). This module is what -that agent drives through the MCP tools in `app/mcp/tools.py`, and what the +that agent drives through the MCP tools in `app/entry/mcp/`, and what the public rail reads. An article sits in at most one open story. Postgres cannot say that without a diff --git a/backend/app/features/stories/tests/test_service.py b/backend/app/features/stories/test_service.py similarity index 81% rename from backend/app/features/stories/tests/test_service.py rename to backend/app/features/stories/test_service.py index 0b555269..2ea1c7d0 100644 --- a/backend/app/features/stories/tests/test_service.py +++ b/backend/app/features/stories/test_service.py @@ -1,29 +1,25 @@ -"""Stories: the schema, what the rail lists, what the agent may save, and the -story tools' write gate.""" +"""Stories: the schema, what the rail lists, and what the agent may save.""" from __future__ import annotations -from collections.abc import Generator from datetime import UTC, datetime, timedelta import numpy as np import pytest -from fastapi.testclient import TestClient from sqlalchemy import inspect -from sqlmodel import Session, col, delete, func, select +from sqlmodel import Session, select from app.core.db.article import Article from app.core.db.engine import engine -from app.core.db.publisher import Publisher -from app.core.db.story import Story, StoryArticle -from app.core.settings import settings -from app.features.catalog.tests.factories import ( +from app.core.db.factories import ( create_random_article, create_random_publisher, + random_lower_string, ) +from app.core.db.publisher import Publisher +from app.core.db.story import Story from app.features.stories import service as stories from app.features.stories.model import SavedStory, StoryError, StoryPublic -from tests.random_data import random_lower_string NOW = datetime(2026, 10, 10, 12, tzinfo=UTC) # Slugs are unique per run: the tests share a database with whatever a @@ -38,22 +34,6 @@ def S(slug: str) -> str: DIM = 256 -@pytest.fixture -def made(db: Session) -> Generator[dict[str, list[int]]]: - """Ids of what a test created, deleted afterwards with every story made - meanwhile, so the next test starts from the same rows.""" - created: dict[str, list[int]] = {"articles": [], "publishers": []} - before = db.exec(select(func.max(Story.id))).one() or 0 - yield created - db.rollback() - ours = select(Story.id).where(col(Story.id) > before) - db.exec(delete(StoryArticle).where(col(StoryArticle.story_id).in_(ours))) - db.exec(delete(Story).where(col(Story.id) > before)) - db.exec(delete(Article).where(col(Article.id).in_(created["articles"]))) - db.exec(delete(Publisher).where(col(Publisher.id).in_(created["publishers"]))) - db.commit() - - def _publisher(db: Session, made: dict[str, list[int]]) -> Publisher: p = create_random_publisher(db) assert p.id is not None @@ -217,24 +197,6 @@ def test_the_rail_caps_the_articles_it_sends( assert [a.url for a in shown.articles] == urls[:8] -def test_the_route_serves_the_list( - client: TestClient, db: Session, made: dict[str, list[int]] -) -> None: - _story(db, made, S("served")) - - r = client.get(f"{settings.API_V1_STR}/stories/", params={"limit": 20}) - assert r.status_code == 200 - story = next(s for s in r.json() if s["slug"] == S("served")) - assert set(story["articles"][0]) == { - "id", - "title", - "url", - "publisher", - "published_at", - "score", - } - - # ─── save_story ────────────────────────────────────────────────────────────── @@ -381,26 +343,3 @@ def test_candidates_cluster_free_articles_and_name_the_story_they_reach( assert by_urls[frozenset({a.url, b.url})]["publishers"] == sorted( [p1.name, p2.name] ) - - -# ─── Tool gate ─────────────────────────────────────────────────────────────── - - -def test_save_story_tool_reports_what_it_did( - db: Session, made: dict[str, list[int]], monkeypatch: pytest.MonkeyPatch -) -> None: - from fastmcp.server.auth import AccessToken - - from app.mcp import tools - - monkeypatch.setattr( - tools, - "get_access_token", - lambda: AccessToken(token="t", client_id="c", scopes=[tools.WRITE_SCOPE]), - ) - url = _article(db, made, _publisher(db, made)).url - slug = S("tool-made") - assert tools.save_story(slug, "Tool made", "It works.", [url]) == ( - f"Created {slug}: added 1, 1 articles in all" - ) - assert tools.close_story(slug) == f"Closed {slug}" diff --git a/backend/app/features/stories/tests/__init__.py b/backend/app/features/stories/tests/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/backend/app/main.py b/backend/app/main.py index d6e3be77..c452a631 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -1,22 +1,31 @@ -"""FastAPI application entry point: wires routers, CORS, and the health check.""" +"""FastAPI application entry point: wires the API routers, the MCP app, CORS, and the health check.""" from __future__ import annotations +import logging + import sentry_sdk -from fastapi import FastAPI +from fastapi import APIRouter, FastAPI from fastapi.routing import APIRoute from starlette.middleware.cors import CORSMiddleware from app.core.settings import settings -from app.mcp.server import mcp_app -from app.router import api_router +from app.entry.api import analytics, articles, likes, login, stories, users +from app.entry.mcp.server import mcp_app + +# uvicorn configures only its own loggers, and collect and report print to +# stdout, so the API's INFO lines (the dev sign-in link) need this. +logging.basicConfig(level=logging.INFO) +# They log every request at INFO. +logging.getLogger("httpx").setLevel(logging.WARNING) +logging.getLogger("httpcore").setLevel(logging.WARNING) def custom_generate_unique_id(route: APIRoute) -> str: return f"{route.tags[0]}-{route.name}" -if settings.SENTRY_DSN and settings.ENVIRONMENT != "local": +if settings.SENTRY_DSN and settings.ENVIRONMENT != "development": sentry_sdk.init(dsn=str(settings.SENTRY_DSN), enable_tracing=True) app = FastAPI( @@ -38,6 +47,16 @@ def custom_generate_unique_id(route: APIRoute) -> str: allow_headers=["*"], ) +# Registration order is the order the endpoints appear in the OpenAPI document, +# and the generated client follows that document. Reordering churns the client +# for no behaviour change. +api_router = APIRouter() +api_router.include_router(login.router) +api_router.include_router(users.router) +api_router.include_router(articles.router) +api_router.include_router(likes.router) +api_router.include_router(analytics.router) +api_router.include_router(stories.router) app.include_router(api_router, prefix=settings.API_V1_STR) # Outside the versioned API on purpose: it is an MCP endpoint, not a REST # resource, and it carries its own bearer-token auth rather than a user JWT. diff --git a/backend/app/mcp/__init__.py b/backend/app/mcp/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/backend/app/mcp/server.py b/backend/app/mcp/server.py deleted file mode 100644 index d9c4dac4..00000000 --- a/backend/app/mcp/server.py +++ /dev/null @@ -1,112 +0,0 @@ -"""The MCP server: the tools, the bearer-token gate, and the ASGI app `main` mounts.""" - -from __future__ import annotations - -import secrets - -from fastmcp import FastMCP -from fastmcp.server.auth import AccessToken, TokenVerifier - -from app.core.settings import settings -from app.mcp.tools import ( - WRITE_SCOPE, - approve, - check_link, - close_story, - draft_issue, - get_content, - list_candidates, - reject, - reject_many, - related, - save_story, - similar, - sql_query, - stories, - story_candidates, - vocabulary, - web_fetch, - web_search, - week, -) - - -def _matches(token: str, expected: str | None) -> bool: - """Constant-time compare against a configured token, if there is one. - - Bytes, not str: compare_digest raises on non-ASCII str operands, and the - token is whatever someone put in the header. An unset value matches - nothing, so a deploy that forgets the variable serves 401s instead of an - open database. - """ - return bool(expected) and secrets.compare_digest( - token.encode(), (expected or "").encode() - ) - - -class SharedSecret(TokenVerifier): - """Two bearer tokens, compared in constant time, one scope between them. - - There are two clients — a read-only agent answering questions about the - feed, and the curation agent that publishes to it — so an OAuth provider - would be ceremony around two values that already live in the same `.env` as - every other credential. The write token is checked first and is the only one - that comes back carrying `WRITE_SCOPE`; the read token reaches the three - reading tools and is refused by the rest. - """ - - async def verify_token(self, token: str) -> AccessToken | None: - if _matches(token, settings.MCP_WRITE_TOKEN): - return AccessToken( - token=token, client_id="agentique-curator", scopes=[WRITE_SCOPE] - ) - if _matches(token, settings.MCP_TOKEN): - return AccessToken(token=token, client_id="agentique-mcp", scopes=[]) - return None - - -mcp: FastMCP = FastMCP( - name="agentique", - instructions=( - "Tools for the agentique article pipeline. Reading: `sql_query` reads " - "the Postgres database (publishers, articles, scores, pipeline runs), " - "`web_fetch` reads one page, `web_search` finds pages. Curation, with " - "the write token only: `list_candidates` lists what is waiting on a " - "verdict, `get_content` pages through the stored text for one of them, " - "`similar` and `stories` compare candidates with the feed, " - "`check_link` looks up a repo or model, `vocabulary` lists the labels, " - "and `approve`, `reject` and `reject_many` settle them. Weekly " - "newsletter, with the write token only: `week` ranks the week's " - "topics, `related` finds what else covers one, and `draft_issue` " - "drafts an issue for a person to send and says where to save it as a " - "blog post. Stories, with the write token only: `story_candidates` " - "lists open stories and unassigned clusters, `save_story` creates or " - "extends one, `close_story` closes one." - ), - auth=SharedSecret(), - tools=[ - sql_query, - web_fetch, - web_search, - list_candidates, - get_content, - similar, - stories, - check_link, - vocabulary, - approve, - reject, - reject_many, - week, - related, - draft_issue, - story_candidates, - save_story, - close_story, - ], -) - -# Mounted at /mcp by `app.main`, so the route inside this app is the bare root. -# Its lifespan starts the streamable-HTTP session manager and has to be handed -# to the parent app -- a mounted ASGI app never gets a lifespan of its own. -mcp_app = mcp.http_app(path="/") diff --git a/backend/app/mcp/tests/__init__.py b/backend/app/mcp/tests/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/backend/app/mcp/tests/test_server.py b/backend/app/mcp/tests/test_server.py deleted file mode 100644 index cd08f223..00000000 --- a/backend/app/mcp/tests/test_server.py +++ /dev/null @@ -1,61 +0,0 @@ -"""The bearer-token gate on the mounted MCP endpoint.""" - -from __future__ import annotations - -import pytest -from fastapi.testclient import TestClient - -from app.core.settings import settings - -MCP_URL = "/mcp/" -HEADERS = {"Accept": "application/json, text/event-stream"} -INITIALIZE = { - "jsonrpc": "2.0", - "id": 1, - "method": "initialize", - "params": { - "protocolVersion": "2025-06-18", - "capabilities": {}, - "clientInfo": {"name": "test", "version": "1"}, - }, -} - - -@pytest.fixture -def token(monkeypatch: pytest.MonkeyPatch) -> str: - monkeypatch.setattr(settings, "MCP_TOKEN", "test-mcp-token") - return "test-mcp-token" - - -def test_the_right_token_gets_a_session(client: TestClient, token: str): - response = client.post( - MCP_URL, - json=INITIALIZE, - headers={**HEADERS, "Authorization": f"Bearer {token}"}, - ) - assert response.status_code == 200 - - -@pytest.mark.usefixtures("token") -def test_a_wrong_token_is_rejected(client: TestClient): - response = client.post( - MCP_URL, json=INITIALIZE, headers={**HEADERS, "Authorization": "Bearer nope"} - ) - assert response.status_code == 401 - - -@pytest.mark.usefixtures("token") -def test_no_token_is_rejected(client: TestClient): - response = client.post(MCP_URL, json=INITIALIZE, headers=HEADERS) - assert response.status_code == 401 - - -def test_an_unconfigured_server_rejects_every_token( - client: TestClient, monkeypatch: pytest.MonkeyPatch -): - # A deploy that forgets MCP_TOKEN must serve 401s, not an open database. - monkeypatch.setattr(settings, "MCP_TOKEN", None) - response = client.post( - MCP_URL, json=INITIALIZE, headers={**HEADERS, "Authorization": "Bearer "} - ) - assert response.status_code == 401 diff --git a/backend/app/mcp/tests/test_tools.py b/backend/app/mcp/tests/test_tools.py deleted file mode 100644 index 05b055f9..00000000 --- a/backend/app/mcp/tests/test_tools.py +++ /dev/null @@ -1,142 +0,0 @@ -"""The MCP tools: the read-only guard, the row cap, JSON coercion, and the two wrappers.""" - -from __future__ import annotations - -import json - -import pytest -from fastmcp.exceptions import ToolError - -from app.core.settings import settings -from app.mcp import tools - - -@pytest.fixture(autouse=True) -def _test_dsn(monkeypatch: pytest.MonkeyPatch) -> None: - """Point the tool at the test credentials instead of `agentique_ro`. - - The role does not exist outside the production box, but the DSN is the one - settings build, read-only options and all, so the guard under test is real. - """ - monkeypatch.setattr( - tools, - "DSN", - settings.model_copy( - update={ - "MCP_DB_USER": settings.POSTGRES_USER, - "MCP_DB_PASSWORD": settings.POSTGRES_PASSWORD, - } - ).MCP_DATABASE_URI, - ) - - -def test_select_returns_columns_and_rows(): - result = json.loads(tools.sql_query("SELECT 1 AS n, 'a' AS label")) - assert result == { - "columns": ["n", "label"], - "rows": [{"n": 1, "label": "a"}], - "truncated": False, - } - - -def test_rows_are_capped_and_flagged(): - result = json.loads( - tools.sql_query(f"SELECT i FROM generate_series(1, {tools.MAX_ROWS + 5}) AS i") - ) - assert len(result["rows"]) == tools.MAX_ROWS - assert result["truncated"] is True - - -def test_a_result_that_exactly_fills_the_cap_is_not_truncated(): - result = json.loads( - tools.sql_query(f"SELECT i FROM generate_series(1, {tools.MAX_ROWS}) AS i") - ) - assert len(result["rows"]) == tools.MAX_ROWS - assert result["truncated"] is False - - -def test_timestamps_and_numerics_survive_serialisation(): - result = json.loads( - tools.sql_query( - "SELECT TIMESTAMP '2026-01-02 03:04:05' AS ts, 1.5::numeric AS amount" - ) - ) - assert result["rows"] == [{"ts": "2026-01-02T03:04:05", "amount": "1.5"}] - - -def test_a_write_is_rejected_by_the_read_only_transaction(): - # Not a syntax check on the way in: Postgres refuses it, so anything the - # tool failed to anticipate is refused too. - with pytest.raises(ToolError, match="read-only transaction"): - tools.sql_query("UPDATE article SET title = title WHERE false") - - -def test_anything_postgres_cannot_plan_as_a_query_is_refused(): - # Planning is the filter: COPY, SET, DO and a second statement after a - # semicolon never reach execution. - with pytest.raises(ToolError): - tools.sql_query("SET application_name = 'mcp'") - - -def test_the_user_table_is_not_readable(): - with pytest.raises(ToolError, match="Not readable through this tool: user"): - tools.sql_query('SELECT email FROM "user"') - - -def test_the_user_table_is_not_readable_through_a_subquery(): - # The check walks the whole plan, so hiding it below the top node does not - # help -- nor would a join, a CTE or a view. - with pytest.raises(ToolError, match="Not readable through this tool: user"): - tools.sql_query('SELECT 1 WHERE EXISTS (SELECT 1 FROM "user")') - - -def test_a_second_statement_after_a_semicolon_cannot_smuggle_a_read(): - with pytest.raises(ToolError): - tools.sql_query('SELECT 1; SELECT email FROM "user"') - - -def test_a_statement_crafted_to_close_the_explain_prefix_is_still_one_statement(): - # The prefix is text, so this would be three valid statements if anything - # but a prepared statement ran it -- and the middle one is the read. - with pytest.raises(ToolError): - tools.sql_query( - 'SELECT 1) AS x; SELECT email FROM "user"; SELECT * FROM (SELECT 1' - ) - - -def test_web_fetch_returns_extracted_text_and_forwards_the_cap( - monkeypatch: pytest.MonkeyPatch, -): - seen: list[tuple[str, int]] = [] - - def fake(url: str, max_length: int) -> str: - seen.append((url, max_length)) - return f"text of {url}" - - monkeypatch.setattr(tools, "fetch_and_extract", fake) - text = tools.web_fetch("https://example.com/post", max_length=50) - assert text == "text of https://example.com/post" - assert seen == [("https://example.com/post", 50)] - - -def test_web_fetch_fails_loudly_when_nothing_is_readable( - monkeypatch: pytest.MonkeyPatch, -): - monkeypatch.setattr(tools, "fetch_and_extract", lambda url, max_length: "") - with pytest.raises(ToolError, match="No readable content"): - tools.web_fetch("https://example.com/paywalled") - - -def test_web_search_forwards_every_argument(monkeypatch: pytest.MonkeyPatch): - calls: list[tuple[str, int, list[str] | None]] = [] - - def fake( - query: str, max_results: int, include_domains: list[str] | None - ) -> list[dict[str, str | None]]: - calls.append((query, max_results, include_domains)) - return [{"title": "t", "url": "u", "description": "d", "published_date": None}] - - monkeypatch.setattr(tools, "tavily_search", fake) - results = tools.web_search("gpt-5", max_results=3, include_domains=["openai.com"]) - assert calls == [("gpt-5", 3, ["openai.com"])] - assert results[0]["url"] == "u" diff --git a/backend/app/mcp/tools.py b/backend/app/mcp/tools.py deleted file mode 100644 index 0266c45a..00000000 --- a/backend/app/mcp/tools.py +++ /dev/null @@ -1,511 +0,0 @@ -"""What an agent can do against agentique: read the db and the web, curate, -draft the weekly newsletter, and keep the rail's stories. - -Two groups, two tokens. `sql_query`, `web_fetch` and `web_search` read and are -reachable with `MCP_TOKEN`. The curation tools publish and reject articles, and -the newsletter tools draft emails addressed to every subscriber, and the story -tools at the bottom name what the right rail shows; all three need `MCP_WRITE_TOKEN` — see `WRITE_SCOPE` below. -""" - -from __future__ import annotations - -import json -from collections.abc import Iterator -from datetime import date, datetime - -import psycopg -from fastmcp.exceptions import ToolError -from fastmcp.server.dependencies import get_access_token -from sqlmodel import Session - -from app.core.db.engine import engine -from app.core.extract import fetch_and_extract -from app.core.settings import settings -from app.core.web_search import tavily_search -from app.features.curation import links -from app.features.curation import service as curation -from app.features.curation.model import ( - CandidateError, - CandidatePage, - ContentPage, - Coverage, - Similar, - Verdict, -) -from app.features.newsletter import broadcast -from app.features.newsletter import service as digest -from app.features.newsletter.model import DigestError, IssueError, Related, WeekTopic -from app.features.stories import service as story_domain -from app.features.stories.model import Candidates, StoryError - -# The scope `MCP_WRITE_TOKEN` carries and the read token does not. Named for -# what it lets a caller do, not for the token that carries it: `server.py` mints -# it, the curation tools below demand it. -WRITE_SCOPE = "curate" - -# A cap, not a page size: an agent that wants more should narrow the query or -# add its own LIMIT. Anything larger is context an agent cannot read anyway. -MAX_ROWS = 1000 - -# `user` holds emails and sign-in link tokens, and no question worth asking this -# tool needs either. The role's grants are the real boundary (see -# deploy/sql-roles.sql); the plan check below holds even where the tool runs as -# a role that has the grant anyway, which is every local database. -BLOCKED_TABLES = frozenset({"user"}) - -# Its own connection, not `app.core.db` — that one logs in as POSTGRES_USER -# and would happily run a DELETE. One per call rather than a pool: the cost is -# nothing at this call rate, and no session state survives a query. -DSN = settings.MCP_DATABASE_URI - - -def _json_safe(value: object) -> str: - """Fallback for the column types `json.dumps` has no encoding for. - - Timestamps get ISO-8601; Decimal, UUID and pgvector embeddings get `str`, - which is lossless for all three and keeps one unreadable column from - failing the whole query. - """ - if isinstance(value, datetime | date): - return value.isoformat() - return str(value) - - -def _plan_tables(node: object) -> Iterator[str]: - """Every table an EXPLAIN plan reads, however deeply nested. - - Walks the whole JSON rather than the top node: the table can sit under a - join, a subplan or a CTE. Views are expanded during planning, so what comes - back is the tables actually read, not the names the query used. - """ - if isinstance(node, dict): - name = node.get("Relation Name") - if isinstance(name, str): - yield name - for value in node.values(): - yield from _plan_tables(value) - elif isinstance(node, list): - for item in node: - yield from _plan_tables(item) - - -def _reject_blocked_tables( - cursor: psycopg.Cursor[tuple[object, ...]], sql: str -) -> None: - """Plan the statement, and refuse it if the plan reads a blocked table. - - EXPLAIN reports the tables actually read -- views expanded, joins, CTEs and - subplans included -- and refusing to plan is what keeps this a query tool, - since COPY, SET and DO never get that far. - - `prepare` is what makes the check honest. Given no parameters psycopg falls - back to the simple query protocol, which runs every command in the string, - so `SELECT 1; SELECT * FROM "user"` would be two commands and only the - first would be the one planned here. A prepared statement takes exactly - one, so the server rejects the second before anything executes. - - psycopg types a `str` query as LiteralString to stop callers interpolating - input into SQL. Doing precisely that is this tool's job, hence the - suppression; what keeps it safe is the role, this check, and `prepare`. - """ - cursor.execute(f"EXPLAIN (FORMAT JSON) {sql}", prepare=True) # ty: ignore[no-matching-overload] - plan = cursor.fetchone() - blocked = BLOCKED_TABLES.intersection(_plan_tables(plan[0] if plan else None)) - if blocked: - raise ToolError(f"Not readable through this tool: {', '.join(sorted(blocked))}") - - -def sql_query(sql: str) -> str: - """Run a read-only SQL query against the agentique Postgres database. - - Accepts one statement, and only one Postgres can plan as a query. The - connection is a read-only transaction held by a role with SELECT grants - only, so writes and DDL fail rather than being filtered out beforehand, and - statements are cancelled after 10 seconds. The `user` table is not - readable, directly or through a join. A rejected statement comes back as - Postgres's own error message. - - Returns JSON: `columns` (in select order), `rows` (a list of objects), and - `truncated` (true when the result hit the 1000-row cap and rows are - missing). Timestamps are ISO-8601 strings; numerics, UUIDs and embedding - vectors are stringified. - """ - try: - with psycopg.connect(DSN) as conn, conn.cursor() as cursor: - _reject_blocked_tables(cursor, sql) - - cursor.execute(sql, prepare=True) # ty: ignore[no-matching-overload] - if cursor.description is None: - # Unreachable while the plan check stands -- nothing that gets - # past it returns without a result set. Kept as an error rather - # than an empty result, which would read as "no rows found". - raise ToolError("Statement returned no result set") - columns = [column.name for column in cursor.description] - # One past the cap, so a full page can be told from an exact fit. - rows = cursor.fetchmany(MAX_ROWS + 1) - return json.dumps( - { - "columns": columns, - "rows": [ - dict(zip(columns, row, strict=True)) for row in rows[:MAX_ROWS] - ], - "truncated": len(rows) > MAX_ROWS, - }, - default=_json_safe, - ) - except psycopg.Error as exc: - # The agent wrote this SQL and can fix it, but only if it sees what - # Postgres actually said; anything but a ToolError reaches the client - # as a bare "internal error". - raise ToolError(str(exc)) - - -def web_fetch(url: str, max_length: int = 20000) -> str: - """Fetch a web page and return its readable text, boilerplate stripped. - - Retries through a residential proxy when the direct fetch is blocked by a - 403, a paywall or a login wall. Fails if the page yields nothing readable — - an empty string would look like an empty page rather than a failed fetch. - """ - content = fetch_and_extract(url, max_length) - if not content: - raise ToolError(f"No readable content at {url}") - return content - - -def web_search( - query: str, max_results: int = 5, include_domains: list[str] | None = None -) -> list[dict[str, str | None]]: - """Search the web (Tavily) and return ranked results. - - Each result has `title`, `url`, `description` and `published_date` - (null unless the source dated the page). `include_domains` restricts hits - to those hosts, e.g. `["openai.com"]` for first-party announcements only. - """ - return tavily_search( - query, max_results=max_results, include_domains=include_domains - ) - - -# ─── Curation ──────────────────────────────────────────────────────────────── -# The verbs the curation agent drives the pipeline's tail with. Some write and -# all of them read the queue, so each one checks for the write scope first: the -# read token reaches `sql_query`, `web_fetch` and `web_search` and nothing else. - - -def _require_write() -> None: - """Refuse a write to a caller holding only the read token.""" - token = get_access_token() - if token is None or WRITE_SCOPE not in token.scopes: - raise ToolError( - "This tool needs the curation token — the read token cannot write." - ) - - -def _curation_session() -> Session: - """A read/write session, not the read-only DSN `sql_query` uses.""" - return Session(engine) - - -def list_candidates(offset: int = 0, limit: int = curation.LIST_LIMIT) -> CandidatePage: - """List one page of the articles waiting on a curation verdict, freshest first. - - Returns `rows`, `offset`, `next_offset` (null on the last page — pass it - back as `offset` for the next one) and `total` pending. Pages hold at most - 50 rows. Read every page before settling any candidate: a verdict removes - its row and shifts the pages after it. - - One row per candidate: `url`, `title`, `source` (where the link was found), - `publisher`, `publisher_kind` (`unknown` for a site the pipeline has just - created a publisher for: `approve` then needs one), `approved` (the - publisher's approvals / decisions over the - last 90 days, e.g. "3/10", or "new" when it has none), `traction`, - `published_at`, `queued_at`, and a 200-character `snippet`. Deliberately not the full text — call - `get_content` for a candidate worth a closer look. - - A candidate stays listed until `approve` or `reject` is called on it, so a - session that stops halfway leaves the rest for the next one. - """ - _require_write() - with _curation_session() as session: - return curation.list_candidates(session, offset, limit) - - -def get_content( - url: str, offset: int = 0, limit: int = curation.PAGE_LIMIT -) -> ContentPage: - """Return one page of the article text the pipeline extracted for a candidate. - - The stored text is what a page fetch returns, cut at 12000 characters, so - read here first. Returns `text`, `offset`, `next_offset` (null on the last - page — pass it back as `offset` to read on), `total` characters, and, on - the first page only, `links`: the repo / model / paper / docs URLs the - article body links, grouped. Fetch the page only when this is empty, a - teaser, or cut off before the part you need. - """ - _require_write() - with _curation_session() as session: - try: - return curation.get_content(session, url, offset, limit) - except CandidateError as exc: - raise ToolError(str(exc)) - - -def similar(url: str, days: int = 7, limit: int = 8) -> Similar: - """Show what the feed and the ledger hold that is close to one candidate. - - Compares by embedding against every published article and every candidate - or reject from the last `days`. `rows` lists up to `limit` neighbours, the - closest first: `url`, `title`, `publisher`, `stage` (`published`, - `pending`, or the reject stage), `score` and `distance` (under 0.30 is the - same story). `coverage` is how many distinct publishers carry this story, - this candidate's own included, with their names in `publishers`. - """ - _require_write() - with _curation_session() as session: - try: - return curation.similar(session, url, days, limit) - except CandidateError as exc: - raise ToolError(str(exc)) - - -def stories(days: int = 7) -> list[Coverage]: - """Group the pending candidates that are one story, with each other or with - an article published in the last `days`. - - Only groups of two or more come back, largest first, each with its - `members` (same row shape as `similar`, without the distance), `coverage` - (distinct publishers) and `publishers`. A member at stage `published` - means the feed already carries the story. Call it once, after triage. - """ - _require_write() - with _curation_session() as session: - return curation.stories(session, days) - - -def check_link(url: str) -> dict[str, object]: - """Look up a GitHub or GitLab repo, or a Hugging Face model or dataset. - - Repos: `stars`, `forks`, `last_push`, `license`, `archived`, `has_readme`, - `description`. Models and datasets: `downloads`, `likes`, `last_modified`, - `license`, `files` (count), and for a model `weights` (true when weight - files are present). A value the lookup could not get reads `unknown`, never - 0; `lookup: unknown` means the whole lookup failed. Any other URL is - refused — read it with `web_fetch`. - """ - _require_write() - try: - return links.check_link(url) - except links.LinkError as exc: - raise ToolError(str(exc)) - - -def vocabulary() -> dict[str, object]: - """The labels `approve` accepts: `kinds`, `publisher_kinds`, and `tags` as - slug -> description. Call it once per session, before the first approve.""" - _require_write() - with _curation_session() as session: - return curation.vocabulary(session) - - -def approve( - url: str, - score: int, - reason: str, - summary: str, - kind: str, - tags: list[str], - publisher_kind: str | None = None, - publisher_name: str | None = None, -) -> str: - """Publish a candidate with the labels you chose, then embed it. - - `score` is 1-100 on the same scale the rubric describes, `reason` one short - sentence naming what decided it, and `summary` the text a reader sees under - the title. `kind` and `tags` (up to 3) come from `vocabulary`; a github, - huggingface or arxiv URL sets its own kind. When the candidate's - `publisher_kind` is `unknown`, pass `publisher_kind` (from - `vocabulary`'s `publisher_kinds`) and, if its host is not a readable name, - `publisher_name`; both are ignored for a publisher that has a kind. A label - that is not in the vocabulary is refused and the candidate stays pending. - The candidate stops being pending in the same transaction that inserts the - article, so nothing is ever published twice. - """ - _require_write() - with _curation_session() as session: - try: - article_id = curation.approve( - session, - url, - score, - reason, - summary, - kind, - tags, - publisher_kind, - publisher_name, - ) - except CandidateError as exc: - raise ToolError(str(exc)) - return f"Published article #{article_id}: {url}" - - -def reject_many(verdicts: list[Verdict]) -> str: - """Turn down many candidates at once: each of `verdicts` is `{url, score, - reason}`, the same fields `reject` takes. - - Each is settled on its own, so one URL that is not pending costs that line - and not the batch. Returns one line per verdict, in order, saying whether - it was rejected or why not. - """ - _require_write() - with _curation_session() as session: - return "\n".join(curation.reject_many(session, verdicts)) - - -def reject(url: str, score: int, reason: str) -> str: - """Turn a candidate down, keeping the score and the reason for it. - - The row stays in the ledger so the URL is never judged twice, and so the - verdicts can be read back as labels. Write `reason` for a human reading a - hundred of them later, not for a log line. - """ - _require_write() - with _curation_session() as session: - try: - curation.reject(session, url, score, reason) - except CandidateError as exc: - raise ToolError(str(exc)) - return f"Rejected [{score}/100] {url}" - - -# ─── Newsletter ────────────────────────────────────────────────────────────── -# The weekly issue: find the week's topics, what backs each one, draft it. -# Behind the same write scope as curation, because a draft lands in the audience -# subscribers are mailed from. Sending is a person's, from the Resend dashboard. - - -def week(days: int = 7, limit: int = digest.WEEK_LIMIT) -> list[WeekTopic]: - """The weekly newsletter's material: the last `days` of published articles - grouped into topics, the most talked-about first. - - Ranked by `coverage` (distinct publishers carrying it, counting the copies - curation rejected as retellings), then `likes`, then `top_score`; up to - `limit` topics. Each topic's `articles` are the published ones - `url`, - `title`, `publisher`, `publisher_kind` (`lab`, `company`, `individual`, - `media`, `community`, `unknown`), `kind`, `score`, `likes`, and the `summary` curation - wrote after reading it - and `covered_by` the other outlets' copies from - the ledger, with curation's `reason`. - """ - _require_write() - with _curation_session() as session: - return digest.week(session, days, limit) - - -def related( - url: str, days: int = digest.RELATED_DAYS, limit: int = digest.RELATED_LIMIT -) -> Related: - """What else covers one published article's subject: the individual writer - who tried it, the repo built on it, the commentary on it. - - Compares by embedding against every article from the last `days` and every - reject from that window that scored 35 or more. Up to `limit` rows, the - closest first: `url`, `title`, `publisher`, `publisher_kind`, `kind`, - `stage` (`published` or `rejected`), `score`, `distance` (under 0.30 is the - same story, up to 0.45 is the same subject), and `summary` for a published - row or curation's `reason` for a rejected one. - """ - _require_write() - with _curation_session() as session: - try: - return digest.related(session, url, days, limit) - except DigestError as exc: - raise ToolError(str(exc)) - - -def draft_issue(post: str) -> str: - """Draft one weekly issue as a Resend broadcast. - - `post` is the whole blog post file: a frontmatter block, then the body. - - --- - title: What does Ollama's MLX backend actually do? - description: One line that makes someone open it. - topic: local-ai - --- - The essay... - - `title` (1-55 characters) is the subject and the post's headline, - `description` (1-160) the line an inbox shows after it, `topic` a kebab - slug that names the draft in Resend. Quote a value in double quotes if it - holds ": " or starts with a quote or a symbol. The body is plain text, - 150-600 words: a blank line between paragraphs, `- ` to start a list line, - `[text](url)` for a link (at least 3 distinct sources), backticks for code. - The server renders it into the site's template. Returns the broadcast id - and the path to save the post at, exactly as passed. Nothing is sent: a - person reads the drafts and sends one from the Resend dashboard. - """ - _require_write() - try: - return broadcast.draft(post) - except IssueError as exc: - raise ToolError(str(exc)) - - -# ─── Stories ───────────────────────────────────────────────────────────────── -# The right rail's named threads, kept by their own daily routine (ADR 16). -# Behind the write scope: a saved story is on the public site at once. - - -def story_candidates(days: int = 14) -> Candidates: - """What the stories routine decides from. - - `open`: every open story, the most recently active first, with `slug`, - `name`, `blurb`, `article_count`, `quiet_days` (since its newest article - was published) and its newest 5 `articles`. `clusters`: the last `days` - of published articles that sit in no open story, grouped by embedding at - a looser cut than coverage, the most publishers first. Each cluster has - `articles` (`url`, `title`, `publisher`, `published_at`, `score`, - `summary`), its `publishers`, and `near`: the open stories one of its - articles is close to. A single article close to a story comes back as a - cluster of one with that story in `near`. - """ - _require_write() - with _curation_session() as session: - return story_domain.story_candidates(session, days) - - -def save_story(slug: str, name: str, blurb: str, urls: list[str]) -> str: - """Create a story, or update an open one: set its `name` and `blurb` and - add `urls` to it (ones already in it are skipped). - - `slug` is lowercase words joined by hyphens and never changes. `name` is - at most 4 words, `blurb` at most 2 sentences and 280 characters. Every URL - must be a published article and in no other open story. Any problem - refuses the whole call, saying which, and nothing is written. A closed - story cannot be saved to. The rail shows a story once it has 3 articles - from 2 publishers. - """ - _require_write() - with _curation_session() as session: - try: - saved = story_domain.save_story(session, slug, name, blurb, urls) - except StoryError as exc: - raise ToolError(str(exc)) - verb = "Created" if saved["created"] else "Updated" - return ( - f"{verb} {saved['slug']}: added {saved['added']}, " - f"{saved['article_count']} articles in all" - ) - - -def close_story(slug: str) -> str: - """Close an open story: it leaves the rail, and its articles are free to - join another.""" - _require_write() - with _curation_session() as session: - try: - story_domain.close_story(session, slug) - except StoryError as exc: - raise ToolError(str(exc)) - return f"Closed {slug}" diff --git a/backend/app/router.py b/backend/app/router.py deleted file mode 100644 index 8b4c0166..00000000 --- a/backend/app/router.py +++ /dev/null @@ -1,18 +0,0 @@ -"""Aggregates every domain router into the versioned API router mounted by app.main.""" - -from __future__ import annotations - -from fastapi import APIRouter - -from app.entry.api import analytics, articles, likes, login, stories, users - -# Registration order is the order the endpoints appear in the OpenAPI document, -# and the generated client follows that document. Reordering churns the client -# for no behaviour change. -api_router = APIRouter() -api_router.include_router(login.router) -api_router.include_router(users.router) -api_router.include_router(articles.router) -api_router.include_router(likes.router) -api_router.include_router(analytics.router) -api_router.include_router(stories.router) diff --git a/backend/app/test_main.py b/backend/app/test_main.py new file mode 100644 index 00000000..2bf6cab3 --- /dev/null +++ b/backend/app/test_main.py @@ -0,0 +1,63 @@ +"""What only the whole app can show: every table is declared in `app.core.db`, +the routes and the collect run share one engine, and the API process logs at +INFO.""" + +from __future__ import annotations + +import os +import subprocess +import sys +from pathlib import Path +from typing import get_args + +from sqlmodel import SQLModel + +from app.core import db +from app.core.db.engine import engine, get_session +from app.entry.api.deps import SessionDep +from app.features.collect import service as collect +from app.main import app # noqa: F401 # every module that could declare a table + + +def test_every_table_lives_in_core_db() -> None: + """The app is fully imported here, so a table declared anywhere else shows + up in the metadata and not in `app.core.db`.""" + declared = set() + for name in db.__all__: + for cls in vars(getattr(db, name)).values(): + if isinstance(cls, type) and issubclass(cls, SQLModel): + table = getattr(cls, "__table__", None) + if table is not None: + declared.add(table.name) + assert declared == set(SQLModel.metadata.tables) + + +def test_routes_and_the_collect_run_share_the_engine() -> None: + assert vars(collect)["engine"] is engine + assert get_args(SessionDep)[1].dependency is get_session + + +def test_the_api_process_prints_the_dev_sign_in_link() -> None: + """A fresh interpreter, because pytest installs its own log handlers. Without + a logging config the link is dropped and a developer can't sign in.""" + # Empty env vars don't override `.env`, so the send is stubbed out instead: + # a developer's `.env` may hold a real Resend key. + script = ( + "import logging\n" + "import app.main\n" + "assert logging.getLogger('httpx').getEffectiveLevel() == logging.WARNING\n" + "from app.features.audience import email\n" + "email.send_email = lambda **_: None\n" + "email.send_link_email('dev@example.com', " + "email.generate_sign_in_email('t0ken'), 't0ken')\n" + ) + result = subprocess.run( + [sys.executable, "-c", script], + cwd=Path(__file__).parents[1], + env=os.environ | {"ENVIRONMENT": "development"}, + capture_output=True, + text=True, + ) + assert result.returncode == 0, result.stderr + assert "[dev] Sign-in link for dev@example.com" in result.stderr + assert "token=t0ken" in result.stderr diff --git a/backend/baml_client/inlinedbaml.py b/backend/baml_client/inlinedbaml.py index ee744bec..48cba6d1 100644 --- a/backend/baml_client/inlinedbaml.py +++ b/backend/baml_client/inlinedbaml.py @@ -14,7 +14,7 @@ "clients.baml": "// The provider caps how many requests may be in flight at once, far below its\n// per-minute quota: measured 2026-09-03, 5 parallel calls all answered while 10\n// returned 429 in ~90ms with x-ratelimit-remaining still at 1980/2000, and the\n// cap refills within ~10s. So a 429 here means \"too many at once\", never \"out\n// of budget\", and waiting is the correct response. Without this policy those\n// bursts fell through to the fallback client instead.\nretry_policy Burst {\n max_retries 4\n strategy {\n type exponential_backoff\n delay_ms 500\n multiplier 2\n max_delay_ms 8000\n }\n}\n\n// First in the Nvidia wrapper below since 2026-09-13. Second in line from 2026-09-03, replacing\n// NVIDIA's gpt-oss-120b, which reached end of life that morning and answered\n// every call with 410 Gone - dead weight for hours before anyone noticed,\n// because a fallback only speaks up when the primary fails.\n//\n// Measured 2026-09-03 on a SelectProductLink-shaped prompt: 1.5-6.8s across\n// sequential calls, all answering with clean JSON and the right pick — no sign\n// of the 52-285s latency that made gpt-oss-120b unusable, though that was a\n// property of the endpoint under load, so it is worth re-measuring if runs\n// start crawling. Past 8 parallel calls NIM starts returning 503; the retry\n// policy covers those.\n//\n// Qwen3.5-397B was the other candidate and judged just as well,\n// but it is a reasoning model: it spends its budget in a `reasoning` field and\n// emits an EMPTY content string if it runs out before it stops thinking (427\n// completion tokens for a two-element array). That needs a large explicit\n// max_tokens to be safe.\nclient Nemotron {\n provider openai-generic\n retry_policy Burst\n options {\n base_url env.NVIDIA_NIM_BASE_URL\n api_key env.NVIDIA_NIM_API_KEY\n model \"nvidia/nemotron-3-super-120b-a12b\"\n // The timeout lives in `http`, which BAML applies itself. A bare `timeout`\n // option is sent as a body param, and NIM answers that with 400\n // \"Unsupported parameter(s)\": every call here failed that way from\n // 2026-09-04 until this block replaced it.\n http {\n request_timeout_ms 120000\n }\n }\n}\n\n// Second in the Nvidia fallback. The smaller sibling of the gpt-oss-120b that\n// led scoring until its endpoint reached end of life. Measured on the old\n// scoring prompt (2026-09-13): clean JSON, but it read a rubric floor as a cap.\nclient GptOss20b {\n provider openai-generic\n retry_policy Burst\n options {\n base_url env.NVIDIA_NIM_BASE_URL\n api_key env.NVIDIA_NIM_API_KEY\n model \"openai/gpt-oss-20b\"\n http {\n request_timeout_ms 120000\n }\n }\n}\n\n// Last in the Nvidia fallback since 2026-09-13; primary from 2026-08-06 until\n// then. Fast (2-3s) and reliable, but a lower-quality judge: ±15 run-to-run\n// variance on identical scoring input, and it intermittently returned fewer\n// results than it was given. It stays as\n// the last resort because it is the only client not on NIM: when NIM refuses a\n// burst, the two clients above it fail together.\nclient Ministral {\n provider openai-generic\n retry_policy Burst\n options {\n base_url env.MISTRAL_AI_BASE_URL\n api_key env.MISTRAL_AI_KEY\n model \"mistralai/Ministral-3-14B-Instruct-2512\"\n // A real client-side timeout. The old `timeout 120` went out as a body\n // param, which this endpoint silently ignored: it never timed anything out.\n http {\n request_timeout_ms 120000\n }\n }\n}\n\n// The name every function references. A fallback wrapper: Nemotron first, then\n// gpt-oss-20b, then Ministral. The name stays - with two NIM models in front it\n// is mostly true.\n//\n// A Gemini Flash client sat third here until 2026-09-03, when the run that\n// needed it got 400 FAILED_PRECONDITION \"User location is not supported for\n// the API use\" - the Google AI API does not serve the prod VPS's region, so\n// the last resort had never been able to answer. Removed rather than left in\n// place: a client that always fails is not a fallback, it just hides how thin\n// the chain really is. A replacement has to be reachable from the box.\nclient Nvidia {\n provider fallback\n options {\n strategy [Nemotron, GptOss20b, Ministral]\n }\n}\n", "discover.baml": "// Batched relevance classifier for the Substack discovery crawler.\n// One LLM call judges many candidate profiles at once (array in, array out).\n// Classification runs on cheap\n// metadata (name + description) BEFORE any per-profile post fetch, and gates\n// both DB insertion and whether the crawler recurses into that profile.\n\nclass ProfileInput {\n handle string @description(\"Substack owner handle, echoed back in the verdict\")\n name string @description(\"Publication name\")\n description string? @description(\"Publication hero text and/or author bio, if known\")\n}\n\nclass ProfileVerdict {\n handle string @description(\"Must exactly match the input handle\")\n isAiRelated bool\n confidence int @description(\"0-100 confidence that AI/ML/LLMs is the PRIMARY focus\")\n}\n\nfunction ClassifyProfiles(profiles: ProfileInput[]) -> ProfileVerdict[] {\n client Nvidia\n prompt #\"\n {{ _.role(\"system\") }}\n You curate sources for a developer-focused AI news platform. For each Substack\n publication, decide whether artificial intelligence is its PRIMARY subject:\n AI/ML/LLMs, foundation models, AI engineering, AI products, or AI research.\n\n Mark isAiRelated = true ONLY when AI is clearly the main topic. Mark false for\n general tech, productivity, marketing, business, crypto, politics, lifestyle, or\n publications that merely mention AI in passing. When the signal is thin (sparse\n description, generic name), lean false and assign low confidence - the crawler\n prunes everything you reject, so false positives pollute the whole subtree.\n\n {{ _.role(\"user\") }}\n Classify every profile. Echo each handle back EXACTLY as given.\n\n {% for p in profiles %}\n {{ loop.index }}. handle={{ p.handle }} | {{ p.name }}{% if p.description %} — {{ p.description }}{% endif %}\n {% endfor %}\n\n {{ ctx.output_format }}\n \"#\n}\n", - "extract_items.baml": "// Newsletter extraction.\n//\n// A newsletter reports two kinds of thing worth ingesting, and they need\n// different URL resolution:\n//\n// - Product: a named launch (model, tool, SDK). Its canonical URL is the\n// vendor's own page, which the email rarely links to directly, so the URL is\n// rediscovered by web search (SelectProductLink) rather than trusted from the\n// email. This replaced the per-newsletter tracking-URL decoders.\n// - Article: a blog post, essay, paper, or write-up. Here the link IS the\n// subject — there is nothing to search for, and a search would land on some\n// other page about the same topic. Its href is kept and followed instead\n// (see ``_resolve_article`` in pipeline/sources/email.py), which turns a\n// tracking redirect into the real destination.\n//\n// Hence NewsletterItem.url: the model must copy an article's href verbatim\n// even though it is usually a redirect, because following it is exactly how\n// the real URL is recovered.\n\nenum NewsletterItemKind {\n Product // A named product, tool, model, SDK, dataset, or launch\n Article // A blog post, essay, paper, guide, or write-up\n}\n\nclass NewsletterItem {\n kind NewsletterItemKind\n name string @description(\"Product name, or the article's title\")\n description string @description(\"Concise phrase: what it is and does, usable verbatim as a web search query\")\n url string? @description(\"For an Article, the href of its link, copied verbatim including any tracking redirect. Omit for a Product.\")\n}\n\nfunction ExtractItems(newsletterText: string) -> NewsletterItem[] {\n client Nvidia\n prompt #\"\n {{ _.role(\"system\") }}\n You extract the AI/tech items a newsletter is reporting on: named products, and the articles it links to. Work only from the text; never invent items that are not mentioned.\n\n {{ _.role(\"user\") }}\n Below is the text of a newsletter email. Links appear as [anchor text](url).\n\n Identify every distinct item the newsletter reports as news, and classify each:\n\n - Product — a named product, tool, model, SDK, dataset, or launch a reader could go use. Judge it by the words, not the link: its URL is looked up separately, so do NOT return a url for a Product.\n - Article — a blog post, essay, paper, guide, benchmark write-up, or analysis the newsletter is pointing at. For these the link is the subject, so ALWAYS return its url, copied verbatim from the text — including tracking redirects, which are followed later to recover the real destination. An item with no link is not an Article.\n\n For each item return:\n - kind: Product or Article\n - name: the exact product name, or the article's title\n - description: a concise phrase describing what it is and does, suitable as a web search query\n - url: the article's href (Article only)\n\n A single story often yields one item, not two: if a link is the vendor's own announcement of a named launch, return it as a Product. Return an Article when the piece itself is the point — someone's write-up, teardown, guide, or opinion.\n\n INCLUDE only real, named launches and substantive articles a reader could go use or read.\n\n IGNORE:\n - the author's intro letter and personal commentary\n - sponsor blurbs, advertisements, \"brought to you by\", upgrade / subscribe / manage-preferences prompts\n - social-media embeds (an @handle with like / repost / reply counts) and the newsletter's own social links\n - section headers, navigation, and unsubscribe links\n - job postings\n\n Newsletter text:\n {{ newsletterText }}\n\n If the text names no qualifying items, return an empty list.\n\n {{ ctx.output_format }}\n \"#\n}\n\n// Notability gate: trims the extracted list to the items actually worth\n// resolving, so we don't spend a web search on every incremental SDK bump.\n// Returns the 1-based indices of the items to KEEP.\nfunction SelectNotableItems(items: NewsletterItem[]) -> int[] {\n client Nvidia\n prompt #\"\n {{ _.role(\"user\") }}\n Below is a numbered list of items a newsletter mentioned. Return the numbers of the ones that are NOTABLE enough to feature in an AI-news aggregator.\n\n KEEP an item if it is distinct, real, and individually newsworthy: new models, significant new products or platforms, notable agents/tools, major releases, and articles with real substance — technical write-ups, benchmarks, teardowns, engineering posts, or arguments a developer would want to read.\n\n DROP:\n - minor or incremental items: small SDK/utility bumps, niche helper libraries, design skills, config tweaks\n - vague or marketing-only entries with no concrete product behind them\n - listicles, SEO filler, press-release rewrites, and thin news blurbs that only restate an announcement\n - duplicates or sub-features of another item in the list\n\n Items:\n {% for item in items %}\n {{ loop.index }}. [{{ item.kind }}] {{ item.name }} — {{ item.description }}\n {% endfor %}\n\n Return only the numbers to keep, as a JSON array of integers. If none qualify, return [].\n\n {{ ctx.output_format }}\n \"#\n}\n\nclass SearchCandidate {\n title string\n url string\n snippet string\n}\n\nclass ProductLinkChoice {\n index int @description(\"1-based index of the best first-party result, or 0 if none qualifies\")\n confidence float @description(\"0.0 - 1.0\")\n}\n\n// Picks the best canonical URL for a product among search candidates, biased\n// hard toward the official / first-party source and away from social posts and\n// SEO/aggregator pages. Returns index 0 when nothing is clearly the product.\n// Products only — an article already carries its own link.\nfunction SelectProductLink(\n productName: string,\n productDescription: string,\n candidates: SearchCandidate[]\n) -> ProductLinkChoice {\n client Nvidia\n prompt #\"\n {{ _.role(\"user\") }}\n A newsletter featured this item:\n name: {{ productName }}\n description: {{ productDescription }}\n\n Web search returned these candidate results:\n {% for c in candidates %}\n {{ loop.index }}. {{ c.title }}\n url: {{ c.url }}\n snippet: {{ c.snippet }}\n {% endfor %}\n\n Pick the ONE candidate that is the canonical, first-party source for this exact product — its official site, product page, model card, repo, or the vendor's own announcement.\n\n PREFER the official / first-party domain that matches the product or its maker.\n REJECT (do not pick) when it is the best available:\n - social-media posts (x.com, twitter.com, linkedin.com, reddit.com, threads, mastodon)\n - videos (youtube.com)\n - content-farm / SEO / generic aggregator or listicle pages that merely mention the product\n - a DIFFERENT product that happens to share the name\n\n Return the 1-based index of the best first-party result. If none of the candidates is clearly a first-party source for THIS product, return index 0.\n\n {{ ctx.output_format }}\n \"#\n}\n\ntest extract_items_basic {\n functions [ExtractItems]\n args {\n newsletterText \"Headlines: [OpenAI ships GPT-5](https://sub.stack/redirect/2/abc), a new flagship model. Deep dive: [Why agent harnesses fail](https://stencil.so/blog/harness-playbook?utm_source=tldrai) walks through production agent design. Brought to you by Acme. [Unsubscribe](https://newsletter.com/unsub).\"\n }\n}\n", + "extract_items.baml": "// Newsletter extraction.\n//\n// A newsletter reports two kinds of thing worth ingesting, and they need\n// different URL resolution:\n//\n// - Product: a named launch (model, tool, SDK). Its canonical URL is the\n// vendor's own page, which the email rarely links to directly, so the URL is\n// rediscovered by web search (SelectProductLink) rather than trusted from the\n// email. This replaced the per-newsletter tracking-URL decoders.\n// - Article: a blog post, essay, paper, or write-up. Here the link IS the\n// subject — there is nothing to search for, and a search would land on some\n// other page about the same topic. Its href is kept and followed instead\n// (see ``_resolve_article`` in app/features/collect/newsletters.py), which turns a\n// tracking redirect into the real destination.\n//\n// Hence NewsletterItem.url: the model must copy an article's href verbatim\n// even though it is usually a redirect, because following it is exactly how\n// the real URL is recovered.\n\nenum NewsletterItemKind {\n Product // A named product, tool, model, SDK, dataset, or launch\n Article // A blog post, essay, paper, guide, or write-up\n}\n\nclass NewsletterItem {\n kind NewsletterItemKind\n name string @description(\"Product name, or the article's title\")\n description string @description(\"Concise phrase: what it is and does, usable verbatim as a web search query\")\n url string? @description(\"For an Article, the href of its link, copied verbatim including any tracking redirect. Omit for a Product.\")\n}\n\nfunction ExtractItems(newsletterText: string) -> NewsletterItem[] {\n client Nvidia\n prompt #\"\n {{ _.role(\"system\") }}\n You extract the AI/tech items a newsletter is reporting on: named products, and the articles it links to. Work only from the text; never invent items that are not mentioned.\n\n {{ _.role(\"user\") }}\n Below is the text of a newsletter email. Links appear as [anchor text](url).\n\n Identify every distinct item the newsletter reports as news, and classify each:\n\n - Product — a named product, tool, model, SDK, dataset, or launch a reader could go use. Judge it by the words, not the link: its URL is looked up separately, so do NOT return a url for a Product.\n - Article — a blog post, essay, paper, guide, benchmark write-up, or analysis the newsletter is pointing at. For these the link is the subject, so ALWAYS return its url, copied verbatim from the text — including tracking redirects, which are followed later to recover the real destination. An item with no link is not an Article.\n\n For each item return:\n - kind: Product or Article\n - name: the exact product name, or the article's title\n - description: a concise phrase describing what it is and does, suitable as a web search query\n - url: the article's href (Article only)\n\n A single story often yields one item, not two: if a link is the vendor's own announcement of a named launch, return it as a Product. Return an Article when the piece itself is the point — someone's write-up, teardown, guide, or opinion.\n\n INCLUDE only real, named launches and substantive articles a reader could go use or read.\n\n IGNORE:\n - the author's intro letter and personal commentary\n - sponsor blurbs, advertisements, \"brought to you by\", upgrade / subscribe / manage-preferences prompts\n - social-media embeds (an @handle with like / repost / reply counts) and the newsletter's own social links\n - section headers, navigation, and unsubscribe links\n - job postings\n\n Newsletter text:\n {{ newsletterText }}\n\n If the text names no qualifying items, return an empty list.\n\n {{ ctx.output_format }}\n \"#\n}\n\n// Notability gate: trims the extracted list to the items actually worth\n// resolving, so we don't spend a web search on every incremental SDK bump.\n// Returns the 1-based indices of the items to KEEP.\nfunction SelectNotableItems(items: NewsletterItem[]) -> int[] {\n client Nvidia\n prompt #\"\n {{ _.role(\"user\") }}\n Below is a numbered list of items a newsletter mentioned. Return the numbers of the ones that are NOTABLE enough to feature in an AI-news aggregator.\n\n KEEP an item if it is distinct, real, and individually newsworthy: new models, significant new products or platforms, notable agents/tools, major releases, and articles with real substance — technical write-ups, benchmarks, teardowns, engineering posts, or arguments a developer would want to read.\n\n DROP:\n - minor or incremental items: small SDK/utility bumps, niche helper libraries, design skills, config tweaks\n - vague or marketing-only entries with no concrete product behind them\n - listicles, SEO filler, press-release rewrites, and thin news blurbs that only restate an announcement\n - duplicates or sub-features of another item in the list\n\n Items:\n {% for item in items %}\n {{ loop.index }}. [{{ item.kind }}] {{ item.name }} — {{ item.description }}\n {% endfor %}\n\n Return only the numbers to keep, as a JSON array of integers. If none qualify, return [].\n\n {{ ctx.output_format }}\n \"#\n}\n\nclass SearchCandidate {\n title string\n url string\n snippet string\n}\n\nclass ProductLinkChoice {\n index int @description(\"1-based index of the best first-party result, or 0 if none qualifies\")\n confidence float @description(\"0.0 - 1.0\")\n}\n\n// Picks the best canonical URL for a product among search candidates, biased\n// hard toward the official / first-party source and away from social posts and\n// SEO/aggregator pages. Returns index 0 when nothing is clearly the product.\n// Products only — an article already carries its own link.\nfunction SelectProductLink(\n productName: string,\n productDescription: string,\n candidates: SearchCandidate[]\n) -> ProductLinkChoice {\n client Nvidia\n prompt #\"\n {{ _.role(\"user\") }}\n A newsletter featured this item:\n name: {{ productName }}\n description: {{ productDescription }}\n\n Web search returned these candidate results:\n {% for c in candidates %}\n {{ loop.index }}. {{ c.title }}\n url: {{ c.url }}\n snippet: {{ c.snippet }}\n {% endfor %}\n\n Pick the ONE candidate that is the canonical, first-party source for this exact product — its official site, product page, model card, repo, or the vendor's own announcement.\n\n PREFER the official / first-party domain that matches the product or its maker.\n REJECT (do not pick) when it is the best available:\n - social-media posts (x.com, twitter.com, linkedin.com, reddit.com, threads, mastodon)\n - videos (youtube.com)\n - content-farm / SEO / generic aggregator or listicle pages that merely mention the product\n - a DIFFERENT product that happens to share the name\n\n Return the 1-based index of the best first-party result. If none of the candidates is clearly a first-party source for THIS product, return index 0.\n\n {{ ctx.output_format }}\n \"#\n}\n\ntest extract_items_basic {\n functions [ExtractItems]\n args {\n newsletterText \"Headlines: [OpenAI ships GPT-5](https://sub.stack/redirect/2/abc), a new flagship model. Deep dive: [Why agent harnesses fail](https://stencil.so/blog/harness-playbook?utm_source=tldrai) walks through production agent design. Brought to you by Acme. [Unsubscribe](https://newsletter.com/unsub).\"\n }\n}\n", "generators.baml": "generator python {\n output_type \"python/pydantic\"\n output_dir \"../\"\n version \"0.223.0\"\n}\n", } diff --git a/backend/baml_src/extract_items.baml b/backend/baml_src/extract_items.baml index 0b55e5d9..8af704f2 100644 --- a/backend/baml_src/extract_items.baml +++ b/backend/baml_src/extract_items.baml @@ -10,7 +10,7 @@ // - Article: a blog post, essay, paper, or write-up. Here the link IS the // subject — there is nothing to search for, and a search would land on some // other page about the same topic. Its href is kept and followed instead -// (see ``_resolve_article`` in pipeline/sources/email.py), which turns a +// (see ``_resolve_article`` in app/features/collect/newsletters.py), which turns a // tracking redirect into the real destination. // // Hence NewsletterItem.url: the model must copy an article's href verbatim diff --git a/backend/conftest.py b/backend/conftest.py index 0629ac93..ce735dc1 100644 --- a/backend/conftest.py +++ b/backend/conftest.py @@ -1,4 +1,5 @@ -"""Shared pytest fixtures: test client, db session, and auth token headers.""" +"""Shared pytest fixtures: test clients, db session, auth token headers, and the +MCP write scope.""" from __future__ import annotations @@ -6,18 +7,26 @@ import pytest from fastapi.testclient import TestClient -from sqlmodel import Session, delete +from fastmcp.server.auth import AccessToken +from sqlmodel import Session, col, delete, func, select +from app.core.db.article import Article from app.core.db.engine import engine from app.core.db.event import AnalyticsEvent +from app.core.db.factories import ( + authentication_token_from_email, + get_superuser_token_headers, + random_email, + random_lower_string, +) +from app.core.db.ledger import Reject, RejectStage from app.core.db.like import ArticleLike +from app.core.db.publisher import Publisher, PublisherKind +from app.core.db.story import Story, StoryArticle from app.core.db.user import User from app.core.settings import settings +from app.entry.mcp import auth from app.features.audience.service import init_db -from app.features.audience.tests.factories import ( - authentication_token_from_email, - get_superuser_token_headers, -) from app.main import app @@ -49,3 +58,65 @@ def superuser_token_headers(db: Session) -> dict[str, str]: @pytest.fixture(scope="module") def normal_user_token_headers(db: Session) -> dict[str, str]: return authentication_token_from_email(email=settings.EMAIL_TEST_USER, db=db) + + +@pytest.fixture(scope="module") +def auth_client(db: Session) -> Generator[TestClient]: + """A TestClient that sends a logged-in reader's bearer token on every request. + + Reads are public; the token only decides whether `liked_by_me` is filled in. + """ + headers = authentication_token_from_email(email=random_email(), db=db) + with TestClient(app) as c: + c.headers.update(headers) + yield c + + +@pytest.fixture +def curator(monkeypatch: pytest.MonkeyPatch) -> None: + """Answer the MCP tools' scope check as the curation agent's write token.""" + monkeypatch.setattr( + auth, + "get_access_token", + lambda: AccessToken(token="t", client_id="c", scopes=[auth.WRITE_SCOPE]), + ) + + +@pytest.fixture +def made(db: Session) -> Generator[dict[str, list[int]]]: + """Ids of what a test created, deleted afterwards with every story made + meanwhile, so the next test starts from the same rows.""" + created: dict[str, list[int]] = {"articles": [], "publishers": []} + before = db.exec(select(func.max(Story.id))).one() or 0 + yield created + db.rollback() + ours = select(Story.id).where(col(Story.id) > before) + db.exec(delete(StoryArticle).where(col(StoryArticle.story_id).in_(ours))) + db.exec(delete(Story).where(col(Story.id) > before)) + db.exec(delete(Article).where(col(Article.id).in_(created["articles"]))) + db.exec(delete(Publisher).where(col(Publisher.id).in_(created["publishers"]))) + db.commit() + + +@pytest.fixture +def candidate(db: Session) -> Generator[Reject]: + """One pending candidate from a publisher the run has just created.""" + name = random_lower_string() + publisher = Publisher(name=f"{name}.dev", slug=name, kind=PublisherKind.unknown) + db.add(publisher) + db.commit() + row = Reject( + url=f"https://{name}.dev/post", + stage=RejectStage.pending, + title="An agent harness, measured", + source="Hacker News", + publisher_id=publisher.id, + content="x" * 5000, + ) + db.add(row) + db.commit() + yield row + db.rollback() + db.exec(delete(Reject).where(col(Reject.url) == row.url)) + db.exec(delete(Publisher).where(col(Publisher.id) == publisher.id)) + db.commit() diff --git a/backend/pipeline/__init__.py b/backend/pipeline/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/backend/pyproject.toml b/backend/pyproject.toml index b8f470f3..125bc322 100644 --- a/backend/pyproject.toml +++ b/backend/pyproject.toml @@ -11,7 +11,7 @@ dependencies = [ "pydantic>2.0", "emails>=1.1.2,<2.0", "jinja2<4.0.0,>=3.1.4", - "alembic<2.0.0,>=1.12.1", + "alembic<2.0.0,>=1.18.0", "httpx<1.0.0,>=0.25.1", "psycopg[binary]<4.0.0,>=3.1.13", "sqlmodel>=0.0.39,<1.0.0", @@ -38,6 +38,7 @@ dev = [ "ruff<1.0.0,>=0.2.2", "coverage<8.0.0,>=7.4.3", "types-regex>=2026.9.29.20261017", + "import-linter>=2.15,<3.0.0", ] [build-system] @@ -90,29 +91,25 @@ keep-runtime-typing = true required-imports = ["from __future__ import annotations"] [tool.ruff.lint.per-file-ignores] -# The collect run and ops scripts log to stdout via print() so systemd/journald -# captures it. +# The collect run and the command-line entrypoints log to stdout via print() so +# systemd/journald captures it. "app/features/collect/**" = ["T201"] -"scripts/**" = ["T201"] +"app/entry/cli/**" = ["T201"] [tool.pytest.ini_options] -# Feature tests live inside the feature (ADR 4); `tests/` keeps what belongs to -# no single feature - core, the schema-wide enum guard, ops scripts. -testpaths = ["app", "tests"] +# Each test sits beside the module it tests (ADR 4). +testpaths = ["app"] [tool.coverage.run] source = ["app"] # Colocated tests are inside `source`; measuring the tests themselves would # only inflate the number. -omit = ["*/tests/*"] +omit = ["*/test_*.py", "*/conftest.py", "*/factories.py"] dynamic_context = "test_function" [tool.coverage.report] show_missing = true sort = "-Cover" -# `scripts/` is deliberately outside `source`: those are ops entrypoints run by -# prestart.sh and by hand, not by pytest, so measuring them would only report a -# number nobody can act on. [tool.coverage.html] show_contexts = true @@ -122,3 +119,45 @@ error-on-warning = true [tool.fastapi] entrypoint = "app.main:app" + +[tool.importlinter] +root_package = "app" + +[[tool.importlinter.contracts]] +name = "main > entry > features > core" +type = "layers" +containers = ["app"] +layers = ["main", "entry", "features", "core"] +# A new top-level module under app/ has to join a layer. +exhaustive = true +exhaustive_ignores = ["test_main"] + +[[tool.importlinter.contracts]] +name = "Features never import each other" +type = "independence" +modules = ["app.features.*"] + +[[tool.importlinter.contracts]] +name = "api, mcp and cli never import each other" +type = "independence" +modules = ["app.entry.api", "app.entry.mcp", "app.entry.cli"] + +[[tool.importlinter.contracts]] +name = "Flat core modules never import core.db" +type = "forbidden" +# app.core.* matches app.core.db too; import-linter skips that overlap. +source_modules = ["app.core.*"] +forbidden_modules = ["app.core.db"] + +[[tool.importlinter.contracts]] +name = "Entry reaches a feature only through its service and model" +type = "forbidden" +source_modules = ["app.entry"] +forbidden_modules = ["app.features.*.*"] +allow_indirect_imports = true +ignore_imports = [ + "app.entry.** -> app.features.*.service", + "app.entry.** -> app.features.*.model", + # The search tests stub the query embedding where it is computed. + "app.entry.api.test_articles -> app.features.catalog.semantic_search", +] diff --git a/backend/scripts/__init__.py b/backend/scripts/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/backend/scripts/tests-start.sh b/backend/scripts/tests-start.sh deleted file mode 100755 index 2e643dc4..00000000 --- a/backend/scripts/tests-start.sh +++ /dev/null @@ -1,7 +0,0 @@ -#! /usr/bin/env bash -set -e -set -x - -python scripts/wait_for_db.py - -bash scripts/test.sh "$@" diff --git a/backend/tests/__init__.py b/backend/tests/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/backend/tests/core/__init__.py b/backend/tests/core/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/backend/tests/core/test_llm_text.py b/backend/tests/core/test_llm_text.py deleted file mode 100644 index 8b582d35..00000000 --- a/backend/tests/core/test_llm_text.py +++ /dev/null @@ -1,21 +0,0 @@ -"""Normalising an enum-typed BAML field to a plain string.""" - -from __future__ import annotations - -from enum import StrEnum - -from app.core.llm_text import enum_value - - -class _Colour(StrEnum): - red = "red" - - -def test_enum_value_unwraps_an_enum(): - assert enum_value(_Colour.red) == "red" - - -def test_enum_value_passes_a_plain_string_through(): - """BAML returns plain strings for some fields and enum members for others, - and which one has changed across regenerations.""" - assert enum_value("red") == "red" diff --git a/backend/tests/random_data.py b/backend/tests/random_data.py deleted file mode 100644 index dd00b236..00000000 --- a/backend/tests/random_data.py +++ /dev/null @@ -1,14 +0,0 @@ -"""Random strings and emails, so two tests never collide on a unique column.""" - -from __future__ import annotations - -import random -import string - - -def random_lower_string() -> str: - return "".join(random.choices(string.ascii_lowercase, k=32)) - - -def random_email() -> str: - return f"{random_lower_string()}@{random_lower_string()}.com" diff --git a/backend/tests/scripts/__init__.py b/backend/tests/scripts/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/deploy/README.md b/deploy/README.md index 53dc16cc..7cfcc67f 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -14,13 +14,30 @@ How agentique runs in production. Docker runs only the database | `agentique-backup.service` + `.timer` | nightly DB dump to off-box storage at 03:30 | | `agentique-sql` | run SQL against the prod DB from stdin, inside the `db` container | | `sql-roles.sql` | one-time: creates the read-only and read/write login roles `agentique-sql` uses | +| `rsync-filter` | what the deploy copies to `/opt/agentique`, and what it never deletes there | ## Shape - A push to `master` triggers `.github/workflows/deploy-production.yml`: build the frontend on a GitHub-hosted runner, then a self-hosted runner on the box - rsyncs the source, syncs the venv, runs migrations + prestart, and restarts - the backend. Additive migrations only; rollback is `git revert` + push. + rsyncs what the box runs, syncs the venv, runs migrations + prestart, and + restarts the backend. Additive migrations only; rollback is `git revert` + + push. +- The rsync copies only what `rsync-filter` lists: the backend's `app/` + without tests, `migrations/`, `baml_client/`, the uv workspace files, + `compose.yml`, `deploy/` and the built frontend. It deletes everything + else in `/opt/agentique`, except `.env`, `.venv` and `.git`, which the + deploy never sends. A file the box needs goes in the filter. + `app/entry/cli/test_deploy_files.py` runs the filter and checks every path + the units and the deploy job use. +- The units, the Caddyfile and `agentique-sql` are installed by hand. After + a deploy that changes one, copy it from `/opt/agentique/deploy/` and run + `sudo systemctl daemon-reload`. If an `ExecStart` path changed, the + deploy has already deleted the old file. The backend then fails from the + deploy's own restart until you copy its unit and restart it, and each + timer fails the next time it fires. The deploy job doesn't install units + because the runner can rewrite anything under `/opt/agentique`. Letting + it install units with sudo would give it root. - The pipeline needs no restart — the timer starts a fresh process each run. - The night runs in three steps an hour apart: 04:00 the pipeline queues candidates, 05:00 the agent judges them, 06:00 the report goes out. The box @@ -146,13 +163,17 @@ Finish with a short report: stories created, extended and closed, and anything a ## Newsletter environment -Read by the backend through `app/core/settings.py`. +Read by the backend through `app/core/settings.py`, with two exceptions that +read `os.environ`: `subscribe` (`app/features/newsletter/service.py`) reads +`RESEND_API_KEY` and `RESEND_AUDIENCE_ID`, and `send_mail` +(`app/core/email.py`), which sends the daily mail, reads `RESEND_API_KEY` and +`EMAILS_FROM_EMAIL`. | Variable | Read by | Effect | | --- | --- | --- | -| `RESEND_API_KEY` | `app/newsletter/broadcast.py` | Unset: `draft_issue` refuses, naming it. | -| `RESEND_AUDIENCE_ID` | `app/newsletter/broadcast.py`, `app/newsletter/routes.py` | The audience signups are synced to and the issue is broadcast to. | -| `EMAILS_FROM_EMAIL`, `EMAILS_FROM_NAME` | `app/newsletter/broadcast.py` | The issue's sender. | +| `RESEND_API_KEY` | `app/features/newsletter/broadcast.py`, `app/features/newsletter/service.py`, `app/core/email.py` | Unset: `draft_issue` refuses, naming it, signups are not synced to Resend, and the daily mail is only logged. | +| `RESEND_AUDIENCE_ID` | `app/features/newsletter/broadcast.py`, `app/features/newsletter/service.py` | The audience signups are synced to and the issue is broadcast to. | +| `EMAILS_FROM_EMAIL`, `EMAILS_FROM_NAME` | `app/features/newsletter/broadcast.py`, `app/core/email.py` (email only) | The issue's sender, and the daily mail's. | ## Pipeline environment @@ -163,3 +184,9 @@ Read as raw `os.environ` beside each consumer (ADR 7), so they belong in | --- | --- | --- | | `OPENROUTER_API_KEY` | `app/core/jev.py` | Jev, which classifies each newsletter link (sponsor, plumbing, on-topic, kind). Unset fails every link, so the newsletter channel yields nothing. | | `PIPELINE_ALERT_EMAIL` | `app/core/email.py` | Where the daily mail goes (liveness, failing sources and feeds, what landed); falls back to `EMAILS_FROM_EMAIL`. | +| `IMAP_HOST`, `IMAP_USER`, `IMAP_PASSWORD`, `IMAP_PORT` | `app/features/collect/newsletters.py` | The inbox the newsletter source reads. The first three are required by that source; the port defaults to 993. | +| `HN_MIN_POINTS`, `HN_MIN_COMMENTS`, `HN_GRACE_HOURS` | `app/features/collect/hn.py` | The HN traction gate: 10 points or 5 comments. A story under 6 hours old is held back for the next run. A first-party URL skips the gate, and `HN_MIN_POINTS=0` turns it off. | +| `TAVILY_API_KEY` | `app/core/web_search.py` | Search for the lab-watch source, the newsletter source and the `web_search` tool. | +| `GITHUB_TOKEN` | `app/core/github.py` | Optional. Raises the GitHub API rate limit for repo lookups. | +| `RESIDENTIAL_PROXY_URL` | `app/core/http.py` | Optional. A metered proxy, tried when a feed fetch or a page extraction is blocked direct. | +| `KDRIVE_API_TOKEN`, `KDRIVE_DRIVE_ID`, `KDRIVE_BACKUP_DIR`, `KDRIVE_BACKUP_KEEP` | `app/entry/cli/backup_db.py` | Where the nightly dump goes. The dir defaults to `/Private/agentique-db`, and 14 dumps are kept. | diff --git a/deploy/agentique-backup.service b/deploy/agentique-backup.service index 4e0d81cd..edce5cba 100644 --- a/deploy/agentique-backup.service +++ b/deploy/agentique-backup.service @@ -18,4 +18,4 @@ EnvironmentFile=/opt/agentique/.env # Root running python against the shared venv would otherwise leave root-owned # __pycache__ files behind, breaking `uv sync` on the next deploy. Environment=PYTHONDONTWRITEBYTECODE=1 -ExecStart=/opt/agentique/.venv/bin/python scripts/backup_db.py +ExecStart=/opt/agentique/.venv/bin/python app/entry/cli/backup_db.py diff --git a/deploy/agentique-pipeline.service b/deploy/agentique-pipeline.service index c3f4b244..4c84bc16 100644 --- a/deploy/agentique-pipeline.service +++ b/deploy/agentique-pipeline.service @@ -19,4 +19,4 @@ EnvironmentFile=/opt/agentique/.env # expands ~ for its cache dir — fails with EACCES against /nonexistent. /var/lib/agentique is real and writable (vps-setup.sh) and # lives outside /opt/agentique so the deploy rsync's --delete never touches it. Environment=HOME=/var/lib/agentique -ExecStart=/opt/agentique/.venv/bin/python -m pipeline.run +ExecStart=/opt/agentique/.venv/bin/python -m app.entry.cli.collect diff --git a/deploy/agentique-report.service b/deploy/agentique-report.service index 44703ef6..f4930710 100644 --- a/deploy/agentique-report.service +++ b/deploy/agentique-report.service @@ -15,4 +15,4 @@ WorkingDirectory=/opt/agentique/backend # settings and the raw os.environ reads alike. EnvironmentFile=/opt/agentique/.env Environment=HOME=/var/lib/agentique -ExecStart=/opt/agentique/.venv/bin/python -m pipeline.report +ExecStart=/opt/agentique/.venv/bin/python -m app.entry.cli.report diff --git a/deploy/rsync-filter b/deploy/rsync-filter new file mode 100644 index 00000000..50371b44 --- /dev/null +++ b/deploy/rsync-filter @@ -0,0 +1,35 @@ +# The deploy's rsync filter (.github/workflows/deploy-production.yml): what +# lands in /opt/agentique. The first matching rule wins. +# +# The deploy runs with --delete-excluded, so rsync deletes anything on the box +# that is not listed here, including files from earlier deploys. It never +# deletes the P paths. The deploy doesn't send them either. +P /.env +P /.venv/ +P /.git/ + +# Test-only code. backend/conftest.py falls under the final rule. +- __pycache__/ +- test_*.py +- /backend/app/core/db/factories.py + +# uv sync --frozen --package app reads the workspace root and the member. ++ /pyproject.toml ++ /uv.lock ++ /.python-version +# backup_db.py runs `docker compose -f /compose.yml`. ++ /compose.yml +# The units, agentique-sql, the Caddyfile and sql-roles.sql, for installing +# by hand. ++ /deploy/*** ++ /backend/ ++ /backend/pyproject.toml ++ /backend/.python-version ++ /backend/app/*** ++ /backend/migrations/*** ++ /backend/baml_client/*** +# Caddy serves the built frontend. The deploy downloads it into frontend/dist +# before the sync. ++ /frontend/ ++ /frontend/dist/*** +- * diff --git a/deploy/sql-roles.sql b/deploy/sql-roles.sql index 5c36a0d8..3612e928 100644 --- a/deploy/sql-roles.sql +++ b/deploy/sql-roles.sql @@ -1,5 +1,5 @@ -- Login roles for /usr/local/bin/agentique-sql (deploy/agentique-sql) and for --- the MCP `sql_query` tool (backend/app/mcp/tools.py). +-- the MCP `sql_query` tool (backend/app/entry/mcp/sql_query.py). -- One-time, on the box: -- docker exec -i agentique-db-1 psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" \ -- -v ON_ERROR_STOP=1 < /opt/agentique/deploy/sql-roles.sql diff --git a/docs/adr/0017-layered-features.md b/docs/adr/0017-layered-features.md new file mode 100644 index 00000000..ab7913c8 --- /dev/null +++ b/docs/adr/0017-layered-features.md @@ -0,0 +1,115 @@ +# 17. Three layers, one folder per feature + +## Status + +Accepted. Supersedes 1 and 4. Changes part of 7 and 8. + +## Context + +ADR 1 gave `app/` one folder per domain and left `pipeline/` beside it in its +own shape. The domains still imported each other. Every table sat in some +domain's `models.py`, and the pipeline and every domain read `Article` from +`catalog/models.py`. Routes in `audience/` called into `catalog` and +`newsletter`. `catalog/stories.py` used the embedding maths in +`pipeline/curation.py`, and the newsletter used the curation maths, the +ledger, `url_kind` and `SNIPPET_CAP`. The MCP tools reached into +`pipeline/fetching`. + +Shared code had no single home. Some was in `app/platform/`, some in +`pipeline/`, and there were two engines: `app/platform/db.py` and +`pipeline/db.py`. Tests sat in a `tests/` folder in each domain (ADR 4), plus +`backend/tests/` for the pipeline, the scripts and the enum guard. The deploy +copied the whole checkout to the box. + +## Decision + + app/main.py wires entry.api and entry.mcp into one FastAPI app + entry/ api/ mcp/ cli/ the ways in: HTTP, MCP, command line + features/ catalog/ stories/ curation/ collect/ newsletter/ audience/ analytics/ + core/ db/ plus flat modules tables, engine, shared mechanism + +- **Three layers.** `entry/` composes features, features use `core/`, and + imports only point down. One rule covers every case, so a reader can tell + where a file goes from what it does. +- **Features never import each other.** When two features need the same + thing, it moves to `core/`. When an API call spans features (sign-up + creates a user and subscribes them), `entry/api/` composes them. +- **Every feature folder uses the same names.** `model.py` holds its own + types, never tables. `service.py` holds the verbs entry calls, as + `session` plus plain arguments. A file named after a `core` module + (`audience/email.py`) adapts it. Other files are private helpers named + after what they hold. A template sits beside its one consumer. +- **Tables live in `core/db/`, one module per table.** The table graph is + shared. Foreign keys cross feature lines, and the migrations are already + one history over the whole schema. Tables in a feature would force + sideways imports. `core/db/__init__.py` imports every table module, so + `SQLModel.metadata` is complete for autogenerate. `core/db/` also holds + the one engine and the few table rules two features need + (`like_counts_subquery`, `kind_from_url`). Queries one feature owns stay + in that feature. +- **Shared mechanism is flat in `core/`.** `settings.py`, `http.py`, + `extract.py`, `embedding.py` and the rest sit beside `db/`. A flat module + imports stdlib, third-party packages and other flat modules, never + `core.db`. There is no `shared/` folder, so ADR 3 needs no exception. +- **`entry/` has one file per router, one file per MCP tool, one file per + command.** A route, a tool or a command is an adapter: parse the input, + call a feature's `service`, shape the result. `entry/cli/` replaces + `backend/scripts/`, `pipeline/run.py` and `pipeline/report.py`. +- **Tests sit beside their module as `test_.py`.** No `tests/` + folders. `backend/conftest.py` holds the shared fixtures, and + `core/db/factories.py` the model factories. +- **import-linter enforces it.** `lint-imports` runs in pre-commit and in + the test-backend workflow, with five contracts in `backend/pyproject.toml`: + 1. layers `app.main` > `app.entry` > `app.features` > `app.core`, exhaustive, + so a new top-level module has to join a layer + 2. independence over `app.features.*` + 3. independence over `app.entry.api`, `app.entry.mcp`, `app.entry.cli` + 4. forbidden: the flat `app.core.*` modules never import `app.core.db` + 5. forbidden: entry reaches a feature only through its `service` and `model` +- **The deploy ships only what the box runs.** `deploy/rsync-filter` lists + `backend/app/` without tests or factories, `backend/migrations/`, + `backend/baml_client/`, the uv workspace files, `compose.yml`, `deploy/` + and the built frontend. The sync deletes everything else on the box except + `.env`, `.venv` and `.git`. + +What changes in earlier ADRs: + +- **1** is superseded. Domain folders with `models.py` / `service.py` / + `routes.py` / `tests/` become feature folders with `model.py` / + `service.py` and named helpers. Routes move to `entry/api/`. `platform/` + becomes `core/`. `pipeline/` is no longer an exception. It becomes + `features/collect/`, with its curation half in `features/curation/`. +- **4** is superseded. Tests still sit beside the code they test, but as + `test_.py` files next to the module, not in a `tests/` folder. + `backend/tests/` is gone, and factories moved to `core/db/factories.py`. + The frontend half of 4 is unchanged. +- **7** changes in part. There is one engine, in `core/db/engine.py`, and it + takes the database URL from `Settings`, so the collect run builds Settings + too. The box feeds both processes the same `.env`. The per-source knobs + (IMAP, HN thresholds, the Tavily and GitHub keys, the proxy) still read + `os.environ` in the module that uses them, and `deploy/README.md` still lists them. +- **8** changes in part. The stage-type chain splits at the queue. collect + owns `RawItem` -> `Candidate` (`features/collect/model.py`). curation owns + `Scored` -> `Summarized` -> `Persisted` (`features/curation/model.py`) and + builds `Scored` from the ledger row the agent judged. `Scored` no longer + subclasses `Candidate`, since that would import one feature into another. + +## Consequences + +`ls app/features/` lists what the backend does, and `ls` on one feature +lists its verbs, its types and its helpers. An import that breaks the rules +fails pre-commit and CI rather than review. + +A rule two features share has to move to `core/` before either can use it. +That move gets its own commit. + +The `stories` MCP tool (curation coverage) keeps its name although a +`stories/` feature now exists. Renaming it would change the curate rubric's +tools. + +Contract 5 checks import paths, not names. A feature that re-exports a helper +from its `__init__.py` would let entry reach it without a failure. + +The systemd units now run `app.entry.cli.*`. They are installed by hand, so +the deploy that changed `ExecStart` needed the units copied and +`daemon-reload` run before that night's timers. diff --git a/docs/backend.md b/docs/backend.md index 03fc6958..0b86100b 100644 --- a/docs/backend.md +++ b/docs/backend.md @@ -9,20 +9,20 @@ - Needs a root `.env` (gitignored): `PROJECT_NAME`, `SECRET_KEY`, `FIRST_SUPERUSER`, `POSTGRES_SERVER=localhost`, `POSTGRES_USER`, `POSTGRES_PASSWORD`, `POSTGRES_DB` - Backend loads it via pydantic-settings (`env_file="../.env"`) -- Pipeline reads raw `os.environ` directly, hence `--env-file` below +- The per-source knobs (IMAP, HN thresholds, API keys) are read from raw `os.environ`, hence `--env-file` below - `frontend/.env` is separate (Vite-only, `VITE_API_URL`) - keeps secrets out of the frontend build ```bash docker compose up -d # from repo root: starts the db cd backend -uv run bash scripts/prestart.sh # migrations + initial data +uv run bash app/entry/cli/prestart.sh # migrations + initial data uv run fastapi dev app/main.py ``` -Pipeline, manual run: +The nightly collect, run by hand: ```bash -cd backend && uv run --env-file ../.env python -m pipeline.run +cd backend && uv run --env-file ../.env python -m app.entry.cli.collect ``` The run ends at pending candidates and publishes nothing — the curation agent @@ -30,7 +30,7 @@ does that (ADR 9). Judge them, then send the night's report: ```bash claude "/curate" # from the repo root -cd backend && uv run --env-file ../.env python -m pipeline.report +cd backend && uv run --env-file ../.env python -m app.entry.cli.report ``` `AGENTIQUE_MCP_TOKEN` must be the write token for `/curate` to publish anything; @@ -45,48 +45,58 @@ nothing. Uses [prek](https://prek.j178.dev/). Config: `.pre-commit-config.yaml`. ```bash -uv run prek install -f # install the git hook, from backend/ +uv run prek install -f # install the git hook uv run prek run --all-files # run manually on everything ``` +- Runs ruff (lint and format) on every Python file, `mypy --strict` and `ty` over `backend/app`, typos, biome and the banned-module-names check +- Runs `lint-imports`, which checks the five import contracts in `backend/pyproject.toml` (ADR 17). The test-backend workflow runs it too +- Regenerates the frontend client when anything under `backend/` changes + ## Setup ```bash -cd backend uv sync source .venv/bin/activate ``` -- Point your editor's Python interpreter at `backend/.venv/bin/python` -- Tables live in `app/core/db/`, one module per table. API shapes stay in each domain's `models.py` -- Each domain folder holds its own `routes.py` and service module +- uv workspace: the root `pyproject.toml` lists `backend/` as its one member, so the venv is the root `.venv` +- Point your editor's Python interpreter at `.venv/bin/python` + +## Layout -VS Code: debugger and Python-tests-tab configs are already set up. +- `app/main.py` - the FastAPI app: the routers, then the MCP server at `/mcp` +- `app/entry/` - the ways in: `api/` one file per router, `mcp/` one file per tool, `cli/` one file per command +- `app/features/` - one folder per feature: catalog, stories, curation, collect, newsletter, audience, analytics. Each has `model.py` and `service.py` +- `app/core/` - `db/` (the tables, one module each, and the engine) plus flat shared modules (`settings.py`, `http.py`, `extract.py`, `embedding.py`, ...) +- Imports point down, entry to features to core, and features never import each other (ADR 17) ## Tests With the db up: ```bash -uv run bash scripts/tests-start.sh +cd backend +uv run bash ../scripts/tests-start.sh ``` -- Domain tests live in that domain's `tests/` folder (`app/catalog/tests/`, `app/audience/tests/`, ...) -- `backend/tests/` holds what belongs to no single domain: pipeline, schema-wide enum guard, ops scripts -- Shared fixtures: `backend/conftest.py` -- `tests-start.sh` waits for the db, then runs `scripts/test.sh` (pytest + coverage). Extra args forward to pytest: +- Each test sits beside its module as `test_.py` (`app/features/catalog/service.py` and `test_service.py`). No `tests/` folders +- Shared fixtures: `backend/conftest.py`. Model factories: `app/core/db/factories.py` +- `tests-start.sh` waits for the db, then runs `scripts/test.sh` beside it (pytest + coverage). An argument becomes the coverage report's title, not a pytest flag: ```bash -uv run bash scripts/tests-start.sh -x +uv run bash ../scripts/tests-start.sh "my branch" ``` -- Coverage report: `htmlcov/index.html` +- Coverage report: `htmlcov/index.html`. Test files, `conftest.py` and `factories.py` are left out of the count ## Migrations - After changing a model, create a revision and apply it, or the schema and the app disagree - `app/core/db/__init__.py` imports every table module. A new table module needs a line there or autogenerate can't see it -- Alembic's config is `[tool.alembic]` in `backend/pyproject.toml` +- Alembic's config is `[tool.alembic]` in `backend/pyproject.toml`. Revisions are in `backend/migrations/versions/` + +From `backend/`: ```bash uv run alembic revision --autogenerate -m "Add column last_name to User model" @@ -101,6 +111,6 @@ uv run alembic upgrade head ## Email templates - Two transactional emails: `welcome.html` on signup and `sign_in.html` on a sign-in request, both carrying a fresh sign-in link -- Templates: `app/core/email-templates/`, next to `app/core/email.py`, which renders and sends them -- The weekly newsletter issue: `weekly_issue.html` in the same folder, the site's wordmark over plain prose, rendered (autoescaped) and drafted by `app/newsletter/broadcast.py` +- They sit in `app/features/audience/`, beside `email.py`, which builds them and sends them through `app/core/email.py` +- The weekly newsletter issue: `app/features/newsletter/weekly_issue.html`, the site's wordmark over plain prose, rendered (autoescaped) and drafted by `broadcast.py` beside it - No MJML source - edit the HTML directly diff --git a/docs/glossary.md b/docs/glossary.md index 14de0ff3..57d6a847 100644 --- a/docs/glossary.md +++ b/docs/glossary.md @@ -8,11 +8,11 @@ Glossary for words the code uses. If a term below and the code disagree, the cod - `kind` (`PublisherKind`) - what the publisher is: `lab` (makes models, writing on its own site), `company`, `individual`, `media`, `community` (an aggregator, credited only for its own pages), or `unknown` (a site the pipeline just made a publisher for; the curation agent names it on its first approval, and a kind once set stays). Shown to readers. - `type` (`PublisherType`) - how the pipeline finds its articles: `rss`, `substack`, `search`, `hn`, `reddit`, `email`, `ainews`, `other`. Internal ingestion detail, not on the public API. -- **Source** - one of the pipeline's fetch adapters (`pipeline/sources/*.py`): Hacker News, the RSS/Substack feeds, a lab-watch crawler, email newsletters. A source yields raw items for one or more publishers. Pipeline concept, not a database column. +- **Source** - one of the pipeline's fetch adapters (`hn.py`, `feeds.py`, `lab_watch.py` and `newsletters.py` in `app/features/collect/`): Hacker News, the RSS/Substack feeds, a lab-watch crawler, email newsletters. A source yields raw items for one or more publishers. Pipeline concept, not a database column. - **Trust** (`Publisher.trust`, `TrustLevel`) - `low` / `medium` / `high`, hand-set per publisher. No longer read (ADR 11): the curation agent sees each publisher's approval rate instead (`approved` in `list_candidates`: approvals / decisions over 90 days, or "new"). The column stays. Not on the public API. -- **Topic-gated** (`Publisher.topic_gated`) - a publisher flagged as mostly off-topic (a general engineering blog, not an AI one). No longer read: the fetch step used to drop its off-topic titles, and the curation agent judges topic now (ADR 11). The column stays. Hacker News keeps its own title keyword gate (`is_on_topic` in `pipeline/sources/hn.py`). +- **Topic-gated** (`Publisher.topic_gated`) - a publisher flagged as mostly off-topic (a general engineering blog, not an AI one). No longer read: the fetch step used to drop its off-topic titles, and the curation agent judges topic now (ADR 11). The column stays. Hacker News keeps its own title keyword gate (`is_on_topic` in `app/features/collect/hn.py`). - **Found via** (`Article.found_via`) - the aggregator (Hacker News, a newsletter) that found an article, when that is not its publisher. Null for an article from its publisher's own feed. @@ -21,24 +21,24 @@ Glossary for words the code uses. If a term below and the code disagree, the cod - **Score** (`Article.score`) - a 1-100 rating of how actionable the article is for a developer building with AI right now. Written by the curation agent, which reads the page before it decides (see **Candidate** below and ADR 9); an LLM in the pipeline used to write it from a title and a 200-char snippet. The one-sentence reason is kept in `Article.score_reason` (internal, not on the public API) and, for rejects, in `Reject.reason`. - **Candidate** - overloaded, and the two senses are one step apart. - - The *type* `Candidate` (`pipeline/types.py`) - a raw item resolved to its Publisher, mid-funnel. + - The *type* `Candidate` (`app/features/collect/model.py`) - a raw item resolved to its Publisher, mid-funnel. - A *candidate* in the feed's sense - an article the nightly run has queued and the curation agent has not judged yet: a `Reject` row at stage `pending`. It becomes an `Article` on approval and a `below_threshold` reject otherwise. Until then it is not visible anywhere a reader can see. - **Curation agent** - the Claude Code session that runs at 05:00, an hour after the pipeline, reads the pending candidates through the MCP tools (`list_candidates`, `get_content`, `similar`, `stories`, `check_link`, `approve`, `reject`, `reject_many`) and decides each one. Its rubric is `.claude/skills/curate/SKILL.md`. It is the only judge the feed has and the only thing that writes an `Article`. -- **Coverage** - how many distinct publishers carry one story, counted by `similar` and `stories` (`pipeline/curation.py`): rows within 0.30 cosine distance of each other across the feed and the ledger. Reach evidence for the curation agent, like traction. +- **Coverage** - how many distinct publishers carry one story, counted by `similar` and `stories` (`app/features/curation/service.py`): rows within 0.30 cosine distance of each other across the feed and the ledger. Reach evidence for the curation agent, like traction. -- **Story** (`Story`, `app/catalog/stories.py`) - a named thread that runs over days or weeks, like "Codemode": a name of 1-3 words, a two-sentence blurb, and its articles (`story_article`). Wider than coverage: coverage is one event told by several outlets, a story is a theme a builder follows across several events, which takes judgment. The stories routine (`.claude/skills/stories/SKILL.md`, ADR 16) names and keeps them through `story_candidates`, `save_story` and `close_story`. An article sits in at most one open story. The right rail (`GET /api/v1/stories`) shows open stories with 3+ articles from 2+ publishers. +- **Story** (`Story` in `app/core/db/story.py`, `app/features/stories/`) - a named thread that runs over days or weeks, like "Codemode": a name of 1-3 words, a two-sentence blurb, and its articles (`story_article`). Wider than coverage: coverage is one event told by several outlets, a story is a theme a builder follows across several events, which takes judgment. The stories routine (`.claude/skills/stories/SKILL.md`, ADR 16) names and keeps them through `story_candidates`, `save_story` and `close_story`. An article sits in at most one open story. The right rail (`GET /api/v1/stories`) shows open stories with 3+ articles from 2+ publishers. -- **Issue** - one weekly newsletter email and the blog post with the same text: a 250-450 word explainer of one topic, what it is, how it works and how people are taking it, linking the sources it was written from. The newsletter agent (`.claude/skills/newsletter/SKILL.md`) drafts three a week, one per topic, through `week`, `related` and `draft_issue` (`app/newsletter/`), and a person sends one. Each is a file in `frontend/content/blog/-.md`, the frontmatter giving the title (the subject), the description (the preheader) and the topic; the routine opens one PR with the three, and git stores them. +- **Issue** - one weekly newsletter email and the blog post with the same text: a 250-450 word explainer of one topic, what it is, how it works and how people are taking it, linking the sources it was written from. The newsletter agent (`.claude/skills/newsletter/SKILL.md`) drafts three a week, one per topic, through `week`, `related` and `draft_issue` (`app/features/newsletter/`), and a person sends one. Each is a file in `frontend/content/blog/-.md`, the frontmatter giving the title (the subject), the description (the preheader) and the topic; the routine opens one PR with the three, and git stores them. - **Traction** - outside signal that people found a story worth reading, independent of the agent's score: Hacker News points/comments past a minimum, which hold back low-traction submissions from the "everyone can post" source; a repo's stars or a model's downloads, which the curation agent reads with `check_link`; and coverage (above). A first-party URL (`app/core/urls.py`) skips the gate - a lab's own announcement counts as news at zero votes. - **Reject** (`Reject`, table `scored_url`) - a URL the funnel turned down, one row per URL. `stage` says which step dropped it - `below_threshold` for the agent's rejects; `thin_repo`, `prefilter`, `duplicate` and `unusable_summary` are on old rows only, from gates that are gone - or `pending`, the one stage that is not a rejection at all: a candidate still waiting on the agent. It keeps what that step saw: title, source, publisher, the first 2000 chars of content, traction, the repo / model / paper / docs `links` the article body makes, score, and the judge's `reason`. `filter_known_urls` reads it so a reject is never judged twice. Holds in a source (HN traction, recency) and a failed LLM call are deliberately not rejects: those come back next run. Rows from before 2026-09-13 carry only the URL. -- **Run** (`PipelineRun`) - one row per nightly pipeline execution: start/end time, duration, ok/fail, and per source `{fetched, queued, error}`. The daily mail (`pipeline/report.py`) reads the latest one for liveness. Per-publisher history lives on the Publisher row instead: `last_fetched_at` (its feed last answered), `last_new_at` (it last gave a new candidate), `last_error` (its last fetch failed, cleared by the next good one). +- **Run** (`PipelineRun`) - one row per nightly pipeline execution: start/end time, duration, ok/fail, and per source `{fetched, queued, error}`. The daily mail (`app/features/collect/report.py`) reads the latest one for liveness. Per-publisher history lives on the Publisher row instead: `last_fetched_at` (its feed last answered), `last_new_at` (it last gave a new candidate), `last_error` (its last fetch failed, cleared by the next good one). -- **RawItem / Candidate / Scored / Persisted** (`pipeline/types.py`) - the four shapes an article takes going down the funnel, one per stage. +- **RawItem / Candidate / Scored / Persisted** (`app/features/collect/model.py`, `app/features/curation/model.py`) - the four shapes an article takes going down the funnel, one per stage. - **RawItem** - what a source adapter emitted. Title, url, content, date, source name. Nothing else known yet. - **Candidate** - a raw item resolved to its Publisher: carries `publisher_id`. - **Scored** - a candidate the curation agent approved, with its score. The nightly run never produces one: it ends at a pending candidate, and `curation.approve` builds the shape from the row the agent judged. diff --git a/docs/product.md b/docs/product.md index 5b57b1b6..ad52f541 100644 --- a/docs/product.md +++ b/docs/product.md @@ -102,11 +102,12 @@ aren't wired into the nightly run yet — candidates for whoever picks up sourci ## Under the hood, briefly - **Backend**: FastAPI (Python), auto-generates the OpenAPI spec the frontend client is - built from. + built from. One package in three layers: `entry/` (the API, the MCP server, the + commands) calls `features/`, which call `core/` (docs/adr/0017). - **Database**: Postgres with the pgvector extension — one table holds articles plus their embedding vectors, so relevance search is just a SQL query. -- **Pipeline**: a separate scheduled Python process (once a day, 04:00) that fetches - and filters, plus a Claude Code session an hour later that judges what it queued and +- **Pipeline**: a separate scheduled Python process (once a day, 04:00, + `app.entry.cli.collect`) that fetches and filters, plus a Claude Code session an hour later that judges what it queued and a report email an hour after that. A second Claude Code session drafts the newsletter on Friday mornings. - **Embeddings**: model2vec — a tiny, fast, CPU-only static embedding model, no GPU or @@ -129,4 +130,4 @@ above, and a real load/scale test (current data volume is small — this has bee architecture spike more than a performance one so far). If you want the ground-level detail — endpoints, models, exact commands — see -`backend/README.md` and `frontend/README.md`. +`docs/backend.md` and `docs/frontend.md`. diff --git a/docs/style.md b/docs/style.md index a7d9337b..43843d36 100644 --- a/docs/style.md +++ b/docs/style.md @@ -1,47 +1,64 @@ # Style -Rules a linter can enforce live in `backend/pyproject.toml` (ruff, mypy) and -`biome.json`, and fail CI. This file is the rest — style a linter cannot -express, enforced by review. +Rules a linter can enforce live in `backend/pyproject.toml` (ruff, mypy, +import-linter) and `biome.json`, and fail CI. This file holds the rest, the +style a linter cannot express, enforced by review. ## Backend -- **A route handler is an adapter.** Parse input, call one domain function, - shape the response. No SQL, no model loading, no branching business rules - in a handler body. -- **Domain functions take an explicit `session` plus plain arguments and - return domain types** — the same signature discipline as - `step(session, articles) -> survivors` in the pipeline. No hidden - globals. +- **Three layers, imports point down.** `app/entry/` (api, mcp, cli) calls + `app/features/`, and features call `app/core/`. Features never import each + other, and api, mcp and cli never import each other. `lint-imports` + enforces this (ADR 17). +- **A feature folder uses the same file names.** `model.py` for its own + types (no tables), `service.py` for the verbs entry calls, a file named + after a `core` module for the adapter over it (`audience/email.py`), and + private helpers named after what they hold (`facets.py`, `coverage.py`). + Entry imports a feature's `service` and `model` only. +- **Tables live in `app/core/db/`, one module per table.** A query only one + feature runs stays in that feature. A rule two features need on a table + goes beside the table (`like_counts_subquery` in `like.py`). +- **Shared mechanism is flat in `app/core/`.** A flat `core` module imports + stdlib, third-party packages and other flat `core` modules, never + `core.db`. When a second feature needs something from the first, it + moves to `core/`. +- **A route handler, an MCP tool and a command are adapters.** Parse input, + call feature verbs, shape the response or set the exit code. When one call + spans two features (sign-up creates a user and subscribes them), the + handler composes them; the features don't import each other. No SQL, no + model loading, no branching business rules in their bodies. One file per + router in `entry/api/`, per tool in `entry/mcp/`, per command in + `entry/cli/`. +- **Feature verbs take an explicit `session` plus plain arguments and return + domain types.** No hidden globals. - **Comments explain why, not what.** Well-named identifiers already say what the code does; a comment earns its place only for a hidden constraint, a subtle invariant, or a workaround for a specific bug. -- **Pipeline stage modules are verb-named** (`fetch`, `filter`, `queue`, - `persist`, `enrich`) — they are stages in a funnel. **Domain modules are - noun-named** (`articles`, `likes`, `publishers`) — they are resources. - This is a deliberate, permanent difference: forcing one naming scheme on - both would make each half read wrong. +- **collect's stage modules are verb-named** (`fetch`, `filter`, `queue`), + because they are stages in a funnel. **Other modules are noun-named** + (`facets`, `publishers`, `coverage`), because they are resources. One + naming scheme for both would make each half read wrong. - **Constants live with their single consumer.** A constant read by exactly - one module is an implementation detail of that module — it does not move - to a shared file. A constant read by two or more modules is a real - interface and lives at that seam (see `SNIPPET_CAP` in - `app/core/embedding.py`, read by curation and by the newsletter, for the - two-reader case). See the constants-vs-config discussion in the refactor plan for - the reasoning. -- **`app/` holds the running service; `backend/scripts/` holds ops - entrypoints.** A module that exists to be run by hand or by `prestart.sh` — - waiting for the DB, creating the superuser, seeding — is an entrypoint, not - part of the service, and lives in `backend/scripts/`. They are still - type-checked (mypy and ty run over `backend/scripts` too); they are - deliberately outside coverage's `source`, because pytest never runs them. + one module is an implementation detail of that module and does not move + to a shared file. A constant read by two or more features is an interface + and lives in `core` (`SNIPPET_CAP` in `app/core/embedding.py`, read by + curation and by the newsletter). +- **Command-line entrypoints live in `app/entry/cli/`.** A module run by + systemd, by `prestart.sh` or by hand (the nightly collect, the daily mail, + waiting for the DB, seeding) calls feature verbs and sets the exit code. + Run it with `python -m app.entry.cli.` from `backend/`. It is + type-checked and counted in coverage like the rest of `app/`. - **Config is different from constants.** Deployment-varying, env-provided values (database URL, API keys) live in one settings object (`app/core/settings.py`). Tuning knobs compiled into the code (`CANDIDATE_CAP`, `SNIPPET_CAP`) never move there. -- **The pipeline is the exception, on purpose.** It reads `os.environ` beside - each consumer rather than through that settings object — see ADR 7. The index - of what it reads lives in `deploy/README.md`; add a row there when you add a - var. +- **Per-source knobs are the exception, on purpose.** The IMAP account, the + HN thresholds, the Tavily and GitHub keys and the proxy are read from + `os.environ` in the module that uses them (ADR 7). The index of what is read + lives in `deploy/README.md`; add a row there when you add a var. +- **Tests sit beside their module** as `test_.py`. No `tests/` + folders. Shared fixtures are in `backend/conftest.py`, model factories in + `app/core/db/factories.py` (ADR 17). ## Frontend @@ -58,7 +75,6 @@ express, enforced by review. - No module named `utils`, `helpers`, `common`, `shared`, or `misc` — see ADR 3. One documented exception: `frontend/src/lib/utils.ts`'s `cn()`, which is shadcn convention. -- A domain folder holds only `models.py`, `service.py` (or a domain-named - module), `routes.py`, `tests/` — see ADR 1. - Hide mechanism, never hide capability — see ADR 2. -- Tests colocate with the domain they test — see ADR 4. +- Tests sit beside the code they test. Backend: see ADR 17. Frontend: see + ADR 4. diff --git a/frontend/scripts/blog-posts.ts b/frontend/scripts/blog-posts.ts index 6a5d18fa..2d9d59b6 100644 --- a/frontend/scripts/blog-posts.ts +++ b/frontend/scripts/blog-posts.ts @@ -5,7 +5,7 @@ * * Only structural checks live here: what a page needs to render. The content * rules for an issue (lengths, word count, sources) live in - * backend/app/newsletter/broadcast.py, which refuses a post before it is ever + * backend/app/features/newsletter/broadcast.py, which refuses a post before it is ever * drafted. */ diff --git a/frontend/scripts/prerender-blog.ts b/frontend/scripts/prerender-blog.ts index ce1ba608..5973d57e 100644 --- a/frontend/scripts/prerender-blog.ts +++ b/frontend/scripts/prerender-blog.ts @@ -10,7 +10,7 @@ * The checks here are structural only (see blog-posts.ts): frontmatter parses, * title/description/topic present, filename is YYYY-MM-DD-slug.md, date not in * the future, slugs unique. Any violation exits non-zero and fails the build. - * The content rules for an issue live in backend/app/newsletter/broadcast.py + * The content rules for an issue live in backend/app/features/newsletter/broadcast.py * (draft_issue refuses a post that breaks them); the writing guide is * .claude/skills/newsletter/SKILL.md; the PR is the editorial review. * diff --git a/frontend/src/client/schemas.gen.ts b/frontend/src/client/schemas.gen.ts index 7f6fd0f4..cbee545c 100644 --- a/frontend/src/client/schemas.gen.ts +++ b/frontend/src/client/schemas.gen.ts @@ -322,7 +322,7 @@ export const OriginSchema = { title: 'Origin', description: `The feed's "From" filter: what the article is when that is settled (a repo, a paper, a model), otherwise who published it. Not stored; each -article falls under exactly one, see \`catalog.articles.origin_condition\`.` +article falls under exactly one, see \`catalog.rows.origin_condition\`.` } as const; export const OriginCountSchema = { diff --git a/frontend/src/client/types.gen.ts b/frontend/src/client/types.gen.ts index 0e6de889..9b947054 100644 --- a/frontend/src/client/types.gen.ts +++ b/frontend/src/client/types.gen.ts @@ -69,7 +69,7 @@ export type Message = { /** * The feed's "From" filter: what the article is when that is settled (a * repo, a paper, a model), otherwise who published it. Not stored; each - * article falls under exactly one, see `catalog.articles.origin_condition`. + * article falls under exactly one, see `catalog.rows.origin_condition`. */ export type Origin = 'lab' | 'company' | 'individual' | 'media' | 'repo' | 'paper' | 'model'; diff --git a/scripts/restore-db.sh b/scripts/restore-db.sh index d0633e4b..d0b7ec80 100755 --- a/scripts/restore-db.sh +++ b/scripts/restore-db.sh @@ -18,7 +18,7 @@ set +a dump=$(ls -1 .ignored/dumps/agentique-db-*.sql.gz 2>/dev/null | sort | tail -1) if [ -z "$dump" ]; then - echo "no dump in .ignored/dumps/ — drop one there or run backend/scripts/seed_articles.py" >&2 + echo "no dump in .ignored/dumps/ — drop one there or run python -m app.entry.cli.seed_articles in backend/" >&2 exit 1 fi diff --git a/backend/scripts/test.sh b/scripts/test.sh similarity index 100% rename from backend/scripts/test.sh rename to scripts/test.sh diff --git a/scripts/tests-start.sh b/scripts/tests-start.sh new file mode 100755 index 00000000..02022138 --- /dev/null +++ b/scripts/tests-start.sh @@ -0,0 +1,7 @@ +#! /usr/bin/env bash +set -e +set -x + +python -m app.entry.cli.wait_for_db + +bash "$(dirname "$0")/test.sh" "$@" diff --git a/uv.lock b/uv.lock index 2489d12a..87093778 100644 --- a/uv.lock +++ b/uv.lock @@ -108,6 +108,7 @@ dependencies = [ [package.dev-dependencies] dev = [ { name = "coverage" }, + { name = "import-linter" }, { name = "mypy" }, { name = "pytest" }, { name = "ruff" }, @@ -117,7 +118,7 @@ dev = [ [package.metadata] requires-dist = [ - { name = "alembic", specifier = ">=1.12.1,<2.0.0" }, + { name = "alembic", specifier = ">=1.18.0,<2.0.0" }, { name = "baml-py", specifier = ">=0.223.0" }, { name = "email-validator", specifier = ">=2.1.0.post1,<3.0.0.0" }, { name = "emails", specifier = ">=1.1.2,<2.0" }, @@ -146,6 +147,7 @@ requires-dist = [ [package.metadata.requires-dev] dev = [ { name = "coverage", specifier = ">=7.4.3,<8.0.0" }, + { name = "import-linter", specifier = ">=2.15,<3.0.0" }, { name = "mypy", specifier = ">=1.8.0,<3.0.0" }, { name = "pytest", specifier = ">=7.4.3,<10.0.0" }, { name = "ruff", specifier = ">=0.2.2,<1.0.0" }, @@ -909,6 +911,71 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/41/63/e876e789525063c840ccfa8857febdabd6523bcef9ce7eb979b9305ea895/griffelib-2.3.0-py3-none-any.whl", hash = "sha256:1b8f9cd525681c26b1d6d574faa1371651e8459ca51d209684f50b8096ae06e0", size = 169423, upload-time = "2026-09-04T15:08:12.956Z" }, ] +[[package]] +name = "grimp" +version = "3.17" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ad/2e/95736b7c984705ff5ee376f07c6e71bb80fcbb22c8fa3b8c1cfbfa919d5c/grimp-3.17.tar.gz", hash = "sha256:5161e03c6f518fe4da43b06c97ff49a1e224ecb0e4b9fd6ef48235373d1b7b33", size = 832923, upload-time = "2026-09-04T11:28:54.609Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/10/7a/b1c76982a239347bfccd2ae95b9f79d3390cd14170f62e7eb822a17d786d/grimp-3.17-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:eb8f09da4302f05fc92fe7564f27731c0b43789d8893ae5e6be5df2b01668923", size = 2145452, upload-time = "2026-09-04T11:28:12.121Z" }, + { url = "https://files.pythonhosted.org/packages/cb/ab/2a1a9574b391c0ee62cb33a633c0d377dce1b7d19d04a7ac5d5e6872d395/grimp-3.17-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:0f11566bcf388db4c79f1c7f1051f6ef150f778306350bb7d86d0c9aab32d297", size = 2091654, upload-time = "2026-09-04T11:28:04.635Z" }, + { url = "https://files.pythonhosted.org/packages/e6/12/7dc086132ac10a2409f158f0b0d7cc528a2ebe0a4e95bca895afe7452fa6/grimp-3.17-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bfda3e2ebb4e51beec4a3cb09bb771440f3192163107d9320e6d9ce8fe064e64", size = 2277300, upload-time = "2026-09-04T11:27:01.844Z" }, + { url = "https://files.pythonhosted.org/packages/5b/62/2e16a54580fa76fb7e64e8e8e2ebb899c709a764cb29149a88eb03f2eea7/grimp-3.17-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:b0ab1ab71d807fc6f56d82f1ab24acf2f2e0e5000e37b5540af9449e28c6830c", size = 2220417, upload-time = "2026-09-04T11:27:11.418Z" }, + { url = "https://files.pythonhosted.org/packages/73/c0/e1b0ad0d0b3d9dff996bad306542f3cf000cb86c368bfe3ac6a8bb1e2e2f/grimp-3.17-cp313-cp313-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:187b092664e882d10b9f2c3da99251d0aeb99186d407ed5d20f52005832045f6", size = 2370754, upload-time = "2026-09-04T11:27:39.995Z" }, + { url = "https://files.pythonhosted.org/packages/36/0f/015e48ad8029f788eaa3df71b66859b2b5dc63abb8d7a2014a2b5a581040/grimp-3.17-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:75da770aeee97d26b202d7251fe7f6562e7f2793e348501b0a64ca5c50e881b5", size = 2628257, upload-time = "2026-09-04T11:27:20.491Z" }, + { url = "https://files.pythonhosted.org/packages/eb/d3/66da78504a1731237ed8befd9c66fb91037a7b82816443ca772107acf8fe/grimp-3.17-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f5cc64f891f71f538d3d032d6e7193f69c3a2e0ceac72c55114faee5aa161f9f", size = 2347569, upload-time = "2026-09-04T11:27:30.282Z" }, + { url = "https://files.pythonhosted.org/packages/cb/a2/6474d008fa49cc650d626798cb3b6e3fa7b0daaad24884fd6bb4288f6154/grimp-3.17-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6dbf2d93b0ba7f6c1efba9f5b3e07a3da0a157392c650316a5c1478b0ef7e503", size = 2284227, upload-time = "2026-09-04T11:27:52.462Z" }, + { url = "https://files.pythonhosted.org/packages/c5/b5/91cd378c08494fd0130e5cc2e54d60d12f53b21850e0dbccf668b6e73d91/grimp-3.17-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:7a22b5dcbf55f1c348885e09cbe91e63baedf6ce0f89c60bb01a0475c7d0a90b", size = 2454655, upload-time = "2026-09-04T11:28:19.837Z" }, + { url = "https://files.pythonhosted.org/packages/f1/d0/b582238247ce51857c7ac770c3be10203c992696f3dc9153f1a979c011dc/grimp-3.17-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:73aa9459a604554b1dc1618148312036679f3974bf8b0285588220307147060b", size = 2495852, upload-time = "2026-09-04T11:28:29.606Z" }, + { url = "https://files.pythonhosted.org/packages/8f/99/e0690b02be70f346aadfe2aa8bd7e496c773298ac58c0d0fa6376c1298b2/grimp-3.17-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:c31660d30e79569faac05033637c1eef4595542f44ec2323af54e900030164a6", size = 2527887, upload-time = "2026-09-04T11:28:38.828Z" }, + { url = "https://files.pythonhosted.org/packages/69/7a/33e41d9b08e63cf398d8194a9208b21788c76916b5ac6d90146e8e2ebf95/grimp-3.17-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:8f395c5ab2cf554504f46680b9269b769939ab5d7ea58f93e5c341e55b1affbd", size = 2539000, upload-time = "2026-09-04T11:28:49.037Z" }, + { url = "https://files.pythonhosted.org/packages/69/f5/7af1dd74b9608a85274414fc423a7162844492b198ece3fe31f4e1f88237/grimp-3.17-cp313-cp313-win32.whl", hash = "sha256:34369739ed293accbb674494ae932351516b316e8e85d1d333d4c51796af846a", size = 1874244, upload-time = "2026-09-04T11:29:21.693Z" }, + { url = "https://files.pythonhosted.org/packages/8b/d4/af497adddfcf11f9bd4ab273625a49022bf5ba4f5fc389589f4e6edc67b9/grimp-3.17-cp313-cp313-win_amd64.whl", hash = "sha256:d21b859c50418bdb48b403a84450650fa7aab3102f3212d00d3c21c76a982167", size = 1998225, upload-time = "2026-09-04T11:29:10.5Z" }, + { url = "https://files.pythonhosted.org/packages/e2/2a/07fd537997281049af2390fc8641de5ae063ae4972cb6fcec85873674595/grimp-3.17-cp313-cp313-win_arm64.whl", hash = "sha256:206cd9f7c757b97934f48ba2b8ec6d649725fd6314d6b182dd327b19b4f35707", size = 1922071, upload-time = "2026-09-04T11:28:58.34Z" }, + { url = "https://files.pythonhosted.org/packages/39/e0/e11ea5aeb62534c9706e17952d142aa844ce5659e2d66b4b8ba9377407d6/grimp-3.17-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:0100ff6f08067c8eb8a77dd1433af5e173aea8dbf6a6b76f5372a3b49f2e0a43", size = 2161696, upload-time = "2026-09-11T10:36:49.104Z" }, + { url = "https://files.pythonhosted.org/packages/4b/d6/ddf82f4428d8bdc54ced0db244cbf5d8e5638490c3ed6a8b9f5db55dfa51/grimp-3.17-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:67ada40a5c1cf4321183c0b0cd0393658ccd04b845caed002ac81dc2fdef828e", size = 2107301, upload-time = "2026-09-11T10:36:46.459Z" }, + { url = "https://files.pythonhosted.org/packages/86/1a/1dd0d9d2751e8078547e1cb7131e13d9624a2ea60700d926b3d624c15825/grimp-3.17-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:baa385915a4c17948bfb25f77509cbbfb33795f321d1c5ca7cc602e34ed58e99", size = 2278433, upload-time = "2026-09-04T11:27:03.075Z" }, + { url = "https://files.pythonhosted.org/packages/58/71/3768fd253e7455a83d668193055691c82d3a4d434047116c572a27d2bd39/grimp-3.17-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:8322a435b5ee97ff3db823df36cbefea569faec505cc0ca66223c27b4b44cca6", size = 2221152, upload-time = "2026-09-04T11:27:12.642Z" }, + { url = "https://files.pythonhosted.org/packages/b4/4f/907a130e87554d57be14c2a7cedeb818d0ef519064877297d19636735f6b/grimp-3.17-cp314-cp314-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:cb953c21beb6ac85a59b5b54128be7fdddd89396d34b3c5705cf59829a85b431", size = 2371546, upload-time = "2026-09-04T11:27:41.195Z" }, + { url = "https://files.pythonhosted.org/packages/7e/4f/a3bf534aa7c582a47d8cfefab5fccb50e6f4dcd327ed6a8552d344a04ffe/grimp-3.17-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c34b904c25359f98e3f20771256d62329ed17f1e26a254a4f5c74bacc1f3bd21", size = 2628467, upload-time = "2026-09-04T11:27:21.818Z" }, + { url = "https://files.pythonhosted.org/packages/32/d3/2b2ab7b3a4e2c17bb1e10ec37e7ee8eecce7f72b1bf6c8019dd439943884/grimp-3.17-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f69ce8be301299f15ca99843e89991c240a231836cf45a11c6bfc54fb9fbeaa9", size = 2347520, upload-time = "2026-09-04T11:27:31.73Z" }, + { url = "https://files.pythonhosted.org/packages/72/da/6bc1f783a739afa51ffa2e235791487b5d09202501b25843ecae9833a3ff/grimp-3.17-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:566411d1ac97e1f3a3f00a13ae33aebb9abda9787d3d9914f62f73b589d84fe2", size = 2284007, upload-time = "2026-09-04T11:27:53.889Z" }, + { url = "https://files.pythonhosted.org/packages/63/06/a08f1b16aec11f32bbc86f60e24986659ab4cfab79388c8d4b79f6a6723e/grimp-3.17-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:5f89948029f3c7339a20bab007977194d3dbdddb27edae5c74b948d8f13191c7", size = 2455301, upload-time = "2026-09-04T11:28:21.177Z" }, + { url = "https://files.pythonhosted.org/packages/ff/c1/d55e9686f3797d1a2c6403d2ffec3020c67591c004304a414fe8d7a9755c/grimp-3.17-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:5a895bc27105ada9b0e76a80bf9cc898409e5cc3689e73e64ca1cbd71dc87dce", size = 2497158, upload-time = "2026-09-04T11:28:30.984Z" }, + { url = "https://files.pythonhosted.org/packages/7f/85/a2d2e70c4f272752c395cc35b3732d82ff92c55e229d7bbd6777fd9868ff/grimp-3.17-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:e80b57a7d03f53e96eb0a5b0844340f99f17eede184042fb73e3ef010176ab35", size = 2528862, upload-time = "2026-09-04T11:28:40.251Z" }, + { url = "https://files.pythonhosted.org/packages/cb/03/fc3a58d9b22b011c1c3377c1ecee127a5f642de9b20de32d3ad7213ab71a/grimp-3.17-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5c0cac76283679fe9c1fd761e63deb9bbc7ba46b32988113ecef4b13742f31b9", size = 2538240, upload-time = "2026-09-04T11:28:50.248Z" }, + { url = "https://files.pythonhosted.org/packages/58/13/d7666cf03c79c275020bad40ea1df8597af7877083ef5a84b5d8b566029a/grimp-3.17-cp314-cp314-win32.whl", hash = "sha256:c8982f4fa1276e059665947156740a55a4e8e6b5a4d7f40730a7511afb608762", size = 1874930, upload-time = "2026-09-04T11:29:23.152Z" }, + { url = "https://files.pythonhosted.org/packages/59/ce/abb9ccb50972351461065ca72cd0f528837a2f398621147dfaf1013698f0/grimp-3.17-cp314-cp314-win_amd64.whl", hash = "sha256:0b97dc29bdca523932bbbcb425fc3df3547cf970396b293bfa960b910f3932d1", size = 1999405, upload-time = "2026-09-04T11:29:11.832Z" }, + { url = "https://files.pythonhosted.org/packages/9f/1d/93c12ff4ce9ede9f91aa03df66128d223a4c824a13de6ab702c973fcae0d/grimp-3.17-cp314-cp314-win_arm64.whl", hash = "sha256:80b5b25b8d8ed6cc68dc39ae9e4c2fb64acd1a3e66434d9cf4d68c72e46d505d", size = 1923324, upload-time = "2026-09-04T11:29:00.004Z" }, + { url = "https://files.pythonhosted.org/packages/d2/59/f94dd90757a2195415736f0e2be667ac4cd375fe86d875d4298661751fce/grimp-3.17-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:e15340ee11e34bfaaee496cff53f8287dc9f417a94e5bdf3d6b4204305d004cc", size = 2145199, upload-time = "2026-09-04T11:28:13.287Z" }, + { url = "https://files.pythonhosted.org/packages/ce/83/9d7cb22fc136ad6a61bff3b74cf0b31b13fdf94e224d46ff1edd9b5b8c86/grimp-3.17-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:cbccf2a6cd3f89bfdfd683d9b000bd61f6936e16b6d4d6619da592c54f79f272", size = 2090761, upload-time = "2026-09-04T11:28:05.984Z" }, + { url = "https://files.pythonhosted.org/packages/e2/6b/b70add79e9e315d111bb1643b44e1b9e1ded82888856e1ad68bc484cd635/grimp-3.17-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5332893955688759a9f7868c3c9e56b19bc1694ee251852d3daf54876b35c115", size = 2279005, upload-time = "2026-09-04T11:27:04.543Z" }, + { url = "https://files.pythonhosted.org/packages/d3/4c/f5f001e592f8e8820086604db98373ca129e2831facad81ec0d1820a749f/grimp-3.17-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:848a92d8a63a38d79c89ab1e68fb5429d90079f39c07533ffd45ce756e7f210f", size = 2220430, upload-time = "2026-09-04T11:27:13.814Z" }, + { url = "https://files.pythonhosted.org/packages/83/13/cc57724307c22060e193a1dc9bc99fa14d46584c534a3936319ec579c599/grimp-3.17-cp314-cp314t-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c89cfe49f822a2cc9c42cc7b0f3975ecb1281970735ea31610bfa06a7e04c460", size = 2370318, upload-time = "2026-09-04T11:27:42.535Z" }, + { url = "https://files.pythonhosted.org/packages/50/1b/05d9964d4b47ffe0ef826bfe04b3f4b713fd623c90e424ff002cb0d8a917/grimp-3.17-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:541c9a2b7f1ad5d4f5d1597740e51058fba8ca0c1a225d6ccc6d11a8364fa563", size = 2629127, upload-time = "2026-09-04T11:27:23.18Z" }, + { url = "https://files.pythonhosted.org/packages/ce/ea/186a3087623f1556eb8b9d02baa00300003bb84b7b35677e3fcec34691b6/grimp-3.17-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:66b4a13edb3fe63be6f7dbbc2576786f501af9d386a5a1f60b2938f270aef97c", size = 2348453, upload-time = "2026-09-04T11:27:32.964Z" }, + { url = "https://files.pythonhosted.org/packages/e8/6c/a718328adc50529135510b515c5ff1d9f94ab26ab8df9057bb1ef5ab8993/grimp-3.17-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:160abad142e168545be8d4cc91b2a95b4565f7571fc56f7fced69ce3589e5111", size = 2287338, upload-time = "2026-09-04T11:27:55.16Z" }, + { url = "https://files.pythonhosted.org/packages/ea/ed/bf67bc6749abee0a1e97f8f14ee2cb19ccc435b306e22bcaac6118d402a4/grimp-3.17-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:cd76fb13a534209c1777c807092f51b1570854eb49948d65cd5e1d94d80e981d", size = 2455299, upload-time = "2026-09-04T11:28:22.505Z" }, + { url = "https://files.pythonhosted.org/packages/83/8a/6c954092306e21c93c744432c3d47491c3264d3cc4b465941de5164a1cb3/grimp-3.17-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:1688e2dbe382b6644ecc3f55053e319de70034d9dfe0809d21d441a590c320e6", size = 2496813, upload-time = "2026-09-04T11:28:32.303Z" }, + { url = "https://files.pythonhosted.org/packages/ea/a8/cf0aa245ace55dd2f6bdd3bb646ec50c294bdfe623fd51327bd27efaa585/grimp-3.17-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:0433d177f327c31410f21845322282c832d887f1a42cbdfa176133d8422d84b2", size = 2527052, upload-time = "2026-09-04T11:28:41.869Z" }, + { url = "https://files.pythonhosted.org/packages/75/b8/90cd11e346f12fb5269ff050ae825083410d3bdb54d82a3b8e955e4f9d26/grimp-3.17-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:790d68157e034e7848257ef46b37550040c9e4692cea17372d3d9f87cef1d84b", size = 2541238, upload-time = "2026-09-04T11:28:51.786Z" }, + { url = "https://files.pythonhosted.org/packages/48/92/10cc37db6149030c8cfc80fa5287f739c1e9620c8a05d1ba8dfdfc0a6c4f/grimp-3.17-cp314-cp314t-win32.whl", hash = "sha256:37e6333ca54d4d80b5c4f2eae324774b223dbdc65994dc9bccda2985a9261dd8", size = 1872673, upload-time = "2026-09-04T11:29:24.514Z" }, + { url = "https://files.pythonhosted.org/packages/ac/cc/b257d8da079a87e65d0f1b0e004fd50f31ba25e7136a3b36f896e0e338f8/grimp-3.17-cp314-cp314t-win_amd64.whl", hash = "sha256:91e4bbbed4d934369b83071380e1ad5f37b7f5c80b80ffa924627db8d66fea11", size = 2000197, upload-time = "2026-09-04T11:29:13.138Z" }, + { url = "https://files.pythonhosted.org/packages/29/d0/feebddf766448932f2f1e40324fe75cf50ce6d3a21bdf87fc0eae263218a/grimp-3.17-cp314-cp314t-win_arm64.whl", hash = "sha256:28ae2ff57a9ef535c2e7054dcb18baf0cdc2dee3982367512bff34eaecb85cf0", size = 1920856, upload-time = "2026-09-04T11:29:01.498Z" }, + { url = "https://files.pythonhosted.org/packages/83/39/bb07084f0b97da66b0aa38077bd656c4f831b4aa3f167b1c060f6a65ea5e/grimp-3.17-cp315-cp315-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:fe3e2285a2bd4e62ddb8825de37152d2dc842394d419786d3d5a803b07bb84eb", size = 2372594, upload-time = "2026-09-04T11:27:43.884Z" }, + { url = "https://files.pythonhosted.org/packages/e6/a9/905a484659749f40ed797963147202cb74c3b68d8e90a7a600af55de47ee/grimp-3.17-cp315-cp315-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a377d5935ae9a62ccbb59e492e16624cec97cc7244c4a964b0c6f83940440626", size = 2284577, upload-time = "2026-09-04T11:27:56.428Z" }, + { url = "https://files.pythonhosted.org/packages/30/6a/40fb12b111c95b94db4930f35661a629532d9d12fb484db5b5ef1a6c3da3/grimp-3.17-cp315-cp315-win32.whl", hash = "sha256:65a1dc2741bef8a96ff3ff71252db8e99eac2a32e09d5e2b63597474706186ca", size = 1876437, upload-time = "2026-09-04T11:29:25.725Z" }, + { url = "https://files.pythonhosted.org/packages/71/98/79af8d51961611e13b3dabc7a3b356b27aa696e5549bb439f1d6002be31a/grimp-3.17-cp315-cp315-win_amd64.whl", hash = "sha256:d20fbe5aab8b013b351313f8eaa3a46f4ac57677e357cb1c3d1012c606b565b4", size = 1999947, upload-time = "2026-09-04T11:29:14.623Z" }, + { url = "https://files.pythonhosted.org/packages/96/d6/e1922c4c7e001210f98e0cf89bd76fea8564f590be3e9ccfe99435cfcd59/grimp-3.17-cp315-cp315-win_arm64.whl", hash = "sha256:a0288dc0e8947dead5c02a246e16f3f5e5cb486348171bdf1289885a13de36c1", size = 1923774, upload-time = "2026-09-04T11:29:02.897Z" }, + { url = "https://files.pythonhosted.org/packages/85/f6/f4f84e44a9237350c41e7c77343498cb1ad843491c18f29d6811021c617d/grimp-3.17-cp315-cp315t-macosx_10_12_x86_64.whl", hash = "sha256:1bf67cb05a80796be99d1bf87e12d4421de101950c5246a8220b6c839cbf27c2", size = 2145245, upload-time = "2026-09-04T11:28:14.537Z" }, + { url = "https://files.pythonhosted.org/packages/71/98/822287270c41360f66c4b5638e040afc6bbf11e9f04cf5005bbd5f3e4830/grimp-3.17-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:d058fdb980475eb9e0143792d63757ddff86c5776e792866e8208cfe5e8413f9", size = 2090367, upload-time = "2026-09-04T11:28:07.277Z" }, + { url = "https://files.pythonhosted.org/packages/ac/76/32d303603157a0d98cb8bde8757e6ae8fa02d41f1d62cdf843fb2e7fe727/grimp-3.17-cp315-cp315t-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:92f1989ce9f9ce26b7d45381df164c62aff783add84cd7a4216a6ebba0f9f03a", size = 2371543, upload-time = "2026-09-04T11:27:45.108Z" }, + { url = "https://files.pythonhosted.org/packages/7b/68/4551e23ab9422e2ead806c1fec9f4b3acd21a267e43bfe8e8cd7f4640dce/grimp-3.17-cp315-cp315t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e275cee07826cf36db1914427e68afac87588f759e1dec676870234d4a250ae", size = 2288003, upload-time = "2026-09-04T11:27:57.605Z" }, + { url = "https://files.pythonhosted.org/packages/20/64/79161f05c900f7f7f1b0acd22a730b1be9f96680dd44efeef90aa45e6998/grimp-3.17-cp315-cp315t-win32.whl", hash = "sha256:a7235ccffb0350647ae2b01969f552a9859fea7027032f6cfed110801269a439", size = 1873249, upload-time = "2026-09-04T11:29:27.094Z" }, + { url = "https://files.pythonhosted.org/packages/94/b7/6f9cdeba2c1263fb91b791ddd5e924d6c91c2485f4f16b56698802fdd540/grimp-3.17-cp315-cp315t-win_amd64.whl", hash = "sha256:473d0e0c016ca429b9ec17f34d276db454ba4d467e3c915a46279d86ab004716", size = 2000579, upload-time = "2026-09-04T11:29:15.961Z" }, + { url = "https://files.pythonhosted.org/packages/ac/16/8ae9028f046bf8f48cc13f6de2de40adbcbf38fa79a3a5f265753b248da0/grimp-3.17-cp315-cp315t-win_arm64.whl", hash = "sha256:478aa0b696e5dd79373943e75b998fbb4ef3d146f24fe3569fa9d106ce84991b", size = 1921829, upload-time = "2026-09-04T11:29:04.403Z" }, +] + [[package]] name = "h11" version = "0.16.0" @@ -1099,6 +1166,21 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/19/a8/0f58c13d2660d5fc8f808ce8b46828d2941752ec21e4015bde99c08b37d7/imap_tools-1.13.0-py3-none-any.whl", hash = "sha256:656c37beba22ab2929b73c07d0ca397ae8805b670d390b1127723e3335244e6d", size = 35849, upload-time = "2026-05-12T07:14:52.669Z" }, ] +[[package]] +name = "import-linter" +version = "2.15" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "click" }, + { name = "grimp" }, + { name = "rich" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/3c/81/1bb414f8c8bcd024499e91f112d317c36a86bc264da1fd43f9797fc2b7b0/import_linter-2.15.tar.gz", hash = "sha256:1da912bea5e172a82a3ce617b5543f75cf64dc0d8f4d9b46c5578b68ccb81590", size = 1405973, upload-time = "2026-09-04T14:47:56.097Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/08/31/4571c20652a4002ed9e500046386f3731a77e23d95a608ae8a87f1fbb2da/import_linter-2.15-py3-none-any.whl", hash = "sha256:9aaf16a88ac1e99d5a464cd7f66b6a05f7060bfa761162e0ed441773a267ed3b", size = 640512, upload-time = "2026-09-04T14:47:54.878Z" }, +] + [[package]] name = "iniconfig" version = "2.3.0"