diff --git a/.env.dev b/.env.dev index ca0e7415..c257ad91 100644 --- a/.env.dev +++ b/.env.dev @@ -28,6 +28,6 @@ MCP_WRITE_TOKEN=agentique-dev-write MCP_DB_USER=postgres MCP_DB_PASSWORD=agentique-dev -# Jev (pipeline/jev.py), the newsletter link classifier. Empty locally: set a +# Jev (backend/app/core/jev.py), the newsletter link classifier. Empty locally: set a # real key only in .env, never here. OPENROUTER_API_KEY= diff --git a/.github/workflows/claude-code-review.yml b/.github/workflows/claude-code-review.yml index 1a422372..d4a8cb7b 100644 --- a/.github/workflows/claude-code-review.yml +++ b/.github/workflows/claude-code-review.yml @@ -34,7 +34,7 @@ jobs: - name: Run Claude Code Review id: claude-review - uses: anthropics/claude-code-action@f27be6b65a4652f08a2ba3cff79114829b588967 # v1 + uses: anthropics/claude-code-action@f27be6b65a4652f08a2ba3cff79114829b588967 # v1.0.252 with: claude_code_oauth_token: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }} plugin_marketplaces: 'https://github.com/anthropics/claude-code.git' diff --git a/.github/workflows/claude.yml b/.github/workflows/claude.yml index 04a576af..c5b76092 100644 --- a/.github/workflows/claude.yml +++ b/.github/workflows/claude.yml @@ -33,7 +33,7 @@ jobs: - name: Run Claude Code id: claude - uses: anthropics/claude-code-action@f27be6b65a4652f08a2ba3cff79114829b588967 # v1 + uses: anthropics/claude-code-action@f27be6b65a4652f08a2ba3cff79114829b588967 # v1.0.252 with: claude_code_oauth_token: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }} diff --git a/.github/workflows/deploy-production.yml b/.github/workflows/deploy-production.yml index 6183c932..2e46a075 100644 --- a/.github/workflows/deploy-production.yml +++ b/.github/workflows/deploy-production.yml @@ -49,25 +49,21 @@ jobs: uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0 with: version: "0.11.18" + # Straight into frontend/dist, so the one sync below ships it. - name: Download dist uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: frontend-dist - path: frontend-dist + path: frontend/dist # The workspace is a staging area only — actions/checkout resets it every - # run, so the services live in /opt/agentique. .env and .venv belong to - # the box; frontend/dist is replaced separately below. - - name: Sync source to /opt/agentique - run: > - rsync -a --delete - --exclude '.git' - --exclude '.env' - --exclude '.venv' - --exclude 'frontend/dist' - --exclude 'frontend-dist' - ./ /opt/agentique/ - - name: Sync frontend dist - run: rsync -a --delete frontend-dist/ /opt/agentique/frontend/dist/ + # run, so the services live in /opt/agentique. deploy/rsync-filter lists + # what the box runs; --delete-excluded removes everything else there, + # except what the filter protects (.env, .venv, .git). A missing dist + # would delete the live site, so check for it first. + - name: Sync to /opt/agentique + run: | + test -f frontend/dist/index.html + rsync -a --delete --delete-excluded --filter='merge deploy/rsync-filter' ./ /opt/agentique/ # --package app: the workspace root is virtual, a bare sync would not # install the backend (same flag the old Dockerfile used). - name: Sync venv @@ -76,7 +72,7 @@ jobs: # --env-file is load-bearing: prestart.sh's seed guard checks the shell # ENVIRONMENT, and a bare deploy shell doesn't have it. - name: Run migrations + prestart - run: uv run --env-file ../.env bash scripts/prestart.sh + run: uv run --env-file ../.env bash app/entry/cli/prestart.sh working-directory: /opt/agentique/backend - name: Restart backend run: sudo systemctl restart agentique-backend diff --git a/.github/workflows/test-backend.yml b/.github/workflows/test-backend.yml index 287ceddc..48323c5f 100644 --- a/.github/workflows/test-backend.yml +++ b/.github/workflows/test-backend.yml @@ -39,12 +39,15 @@ jobs: POSTGRES_USER=postgres POSTGRES_PASSWORD=ci-test-password EOF + - name: Check import layers + run: uv run lint-imports + working-directory: backend - run: docker compose up -d - name: Migrate DB - run: uv run bash scripts/prestart.sh + run: uv run bash app/entry/cli/prestart.sh working-directory: backend - name: Run tests - run: uv run bash scripts/tests-start.sh "Coverage for ${{ github.sha }}" + run: uv run bash ../scripts/tests-start.sh "Coverage for ${{ github.sha }}" working-directory: backend - run: docker compose down -v - name: Store coverage files diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index d4591f41..5d8f43c7 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -14,7 +14,7 @@ repos: (?x)^( frontend/src/client/.*| backend/baml_client/.*| - backend/app/core/email-templates/.* + backend/app/features/(audience|newsletter)/.*\.html )$ - id: trailing-whitespace exclude: | @@ -53,14 +53,21 @@ repos: - id: local-mypy name: mypy check - entry: uv run mypy backend/app backend/scripts + entry: uv run mypy --config-file backend/pyproject.toml backend/app require_serial: true language: unsupported pass_filenames: false - id: local-ty name: ty check - entry: uv run ty check backend/app backend/scripts + entry: uv run ty check --project backend backend/app + require_serial: true + language: unsupported + pass_filenames: false + + - id: local-lint-imports + name: import-linter + entry: uv run lint-imports --config backend/pyproject.toml require_serial: true language: unsupported pass_filenames: false diff --git a/CLAUDE.md b/CLAUDE.md index 6cafa302..a9bbc7f4 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -36,7 +36,7 @@ - Read and mutate freely. It's a local copy, not prod. ## MCP tools -- `.mcp.json` connects to the `agentique` MCP server (`backend/app/mcp/`), exposing `sql_query` (read-only prod Postgres), `web_fetch`, `web_search`. +- `.mcp.json` connects to the `agentique` MCP server (`backend/app/entry/mcp/`), exposing `sql_query` (read-only prod Postgres), `web_fetch`, `web_search`. - With the write token it also exposes `list_candidates`, `get_content`, `similar`, `stories`, `check_link`, `vocabulary`, `approve`, `reject`, `reject_many` - the curation agent's tools. They publish to prod. Don't call them outside a curation session. - The write token also reaches `week`, `related`, `draft_issue` - the newsletter agent's tools (`.claude/skills/newsletter/SKILL.md`). `draft_issue` creates a Resend draft addressed to every subscriber; sending is done by hand in Resend. - And `story_candidates`, `save_story`, `close_story` - the stories agent's tools (`.claude/skills/stories/SKILL.md`). They write the right rail's stories in prod. diff --git a/backend/alembic.ini b/backend/alembic.ini deleted file mode 100755 index 24841c2b..00000000 --- a/backend/alembic.ini +++ /dev/null @@ -1,71 +0,0 @@ -# A generic, single database configuration. - -[alembic] -# path to migration scripts -script_location = app/alembic - -# template used to generate migration files -# file_template = %%(rev)s_%%(slug)s - -# timezone to use when rendering the date -# within the migration file as well as the filename. -# string value is passed to dateutil.tz.gettz() -# leave blank for localtime -# timezone = - -# max length of characters to apply to the -# "slug" field -#truncate_slug_length = 40 - -# set to 'true' to run the environment during -# the 'revision' command, regardless of autogenerate -# revision_environment = false - -# set to 'true' to allow .pyc and .pyo files without -# a source .py file to be detected as revisions in the -# versions/ directory -# sourceless = false - -# version location specification; this defaults -# to alembic/versions. When using multiple version -# directories, initial revisions must be specified with --version-path -# version_locations = %(here)s/bar %(here)s/bat alembic/versions - -# the output encoding used when revision files -# are written from script.py.mako -# output_encoding = utf-8 - -# Logging configuration -[loggers] -keys = root,sqlalchemy,alembic - -[handlers] -keys = console - -[formatters] -keys = generic - -[logger_root] -level = WARN -handlers = console -qualname = - -[logger_sqlalchemy] -level = WARN -handlers = -qualname = sqlalchemy.engine - -[logger_alembic] -level = INFO -handlers = -qualname = alembic - -[handler_console] -class = StreamHandler -args = (sys.stderr,) -level = NOTSET -formatter = generic - -[formatter_generic] -format = %(levelname)-5.5s [%(name)s] %(message)s -datefmt = %H:%M:%S diff --git a/backend/app/alembic/README b/backend/app/alembic/README deleted file mode 100755 index 2500aa1b..00000000 --- a/backend/app/alembic/README +++ /dev/null @@ -1 +0,0 @@ -Generic single-database configuration. diff --git a/backend/app/alembic/env.py b/backend/app/alembic/env.py deleted file mode 100755 index 546170b8..00000000 --- a/backend/app/alembic/env.py +++ /dev/null @@ -1,96 +0,0 @@ -import os -from logging.config import fileConfig - -from alembic import context -from sqlalchemy import engine_from_config, pool - -# this is the Alembic Config object, which provides -# access to the values within the .ini file in use. -config = context.config - -# Interpret the config file for Python logging. -# This line sets up loggers basically. -assert config.config_file_name is not None -fileConfig(config.config_file_name) - -# add your model's MetaData object here -# for 'autogenerate' support -# from myapp import mymodel -# target_metadata = mymodel.Base.metadata -# target_metadata = None - -# Every model module is imported for its side effect: SQLModel.metadata is only -# complete once each domain's tables have been declared, and autogenerate -# reads that metadata. A domain added without a line here silently loses its -# migrations. -from sqlmodel import SQLModel # noqa - -from app.analytics import models as analytics_models # noqa -from app.audience import models as audience_models # noqa -from app.catalog import models as catalog_models # noqa -from app.core.settings import settings # noqa -from app.newsletter import models as newsletter_models # noqa -from pipeline import models as pipeline_models # noqa - -target_metadata = SQLModel.metadata - -# other values from the config, defined by the needs of env.py, -# can be acquired: -# my_important_option = config.get_main_option("my_important_option") -# ... etc. - - -def get_url(): - return str(settings.SQLALCHEMY_DATABASE_URI) - - -def run_migrations_offline(): - """Run migrations in 'offline' mode. - - This configures the context with just a URL - and not an Engine, though an Engine is acceptable - here as well. By skipping the Engine creation - we don't even need a DBAPI to be available. - - Calls to context.execute() here emit the given string to the - script output. - - """ - url = get_url() - context.configure( - url=url, target_metadata=target_metadata, literal_binds=True, compare_type=True - ) - - with context.begin_transaction(): - context.run_migrations() - - -def run_migrations_online(): - """Run migrations in 'online' mode. - - In this scenario we need to create an Engine - and associate a connection with the context. - - """ - configuration = config.get_section(config.config_ini_section) - assert configuration is not None - configuration["sqlalchemy.url"] = get_url() - connectable = engine_from_config( - configuration, - prefix="sqlalchemy.", - poolclass=pool.NullPool, - ) - - with connectable.connect() as connection: - context.configure( - connection=connection, target_metadata=target_metadata, compare_type=True - ) - - with context.begin_transaction(): - context.run_migrations() - - -if context.is_offline_mode(): - run_migrations_offline() -else: - run_migrations_online() diff --git a/backend/app/analytics/routes.py b/backend/app/analytics/routes.py deleted file mode 100644 index 3f57896d..00000000 --- a/backend/app/analytics/routes.py +++ /dev/null @@ -1,67 +0,0 @@ -"""First-party analytics: the public ingest endpoint and the admin-only report.""" - -from __future__ import annotations - -import re - -from fastapi import APIRouter, Depends, Query, Request - -from app.analytics.models import AnalyticsEvent, AnalyticsEventCreate, AnalyticsReport -from app.analytics.report import build_report -from app.deps import CurrentUserOptional, SessionDep, get_current_active_superuser - -router = APIRouter(prefix="/analytics", tags=["analytics"]) - -# cap oversized strings so a bad/malicious client can't bloat the table -_MAX_LEN = 2048 - - -def _truncate(value: str | None) -> str | None: - return value[:_MAX_LEN] if value else None - - -def _without_query(value: str | None) -> str | None: - # A sign-in link carries its token in the query string, and nothing reads - # the query back, so none is kept. - return re.split(r"[?#]", value, maxsplit=1)[0] if value else None - - -@router.post("/collect", status_code=204) -def collect_event( - session: SessionDep, - current_user: CurrentUserOptional, - payload: AnalyticsEventCreate, - request: Request, -) -> None: - """Record one analytics event. Public and fire-and-forget (returns 204). - - Attaches the user id when the request carries a valid bearer token; - otherwise the event is anonymous. - """ - event = AnalyticsEvent( - event=_truncate(payload.event) or "pageview", - path=_truncate(_without_query(payload.path)), - # Only the client's `document.referrer` is meaningful. The request's own - # Referer header is always our site, so a fallback to it would file every - # direct visit under agentique.ch. - referrer=_truncate(_without_query(payload.referrer)), - visitor_id=_truncate(payload.visitor_id), - user_id=current_user.id if current_user else None, - user_agent=_truncate(request.headers.get("user-agent")), - props=payload.props, - ) - session.add(event) - session.commit() - - -@router.get( - "/report", - response_model=AnalyticsReport, - dependencies=[Depends(get_current_active_superuser)], -) -def read_report( - session: SessionDep, - days: int | None = Query(default=None, ge=1, le=3650), -) -> AnalyticsReport: - """Superusers only. Outside readers over the last `days` days, all time when omitted.""" - return build_report(session, days) diff --git a/backend/app/analytics/tests/test_routes.py b/backend/app/analytics/tests/test_routes.py deleted file mode 100644 index ad2ea46a..00000000 --- a/backend/app/analytics/tests/test_routes.py +++ /dev/null @@ -1,121 +0,0 @@ -"""Tests for the analytics ingest endpoint.""" - -from __future__ import annotations - -from fastapi.testclient import TestClient -from sqlmodel import Session, col, select - -from app.analytics.models import AnalyticsEvent -from app.audience.models import User -from app.core.settings import settings - - -def _latest_event(db: Session, visitor_id: str) -> AnalyticsEvent | None: - return db.exec( - select(AnalyticsEvent) - .where(AnalyticsEvent.visitor_id == visitor_id) - .order_by(col(AnalyticsEvent.id).desc()) - ).first() - - -def test_collect_anonymous_pageview(client: TestClient, db: Session) -> None: - r = client.post( - f"{settings.API_V1_STR}/analytics/collect", - json={"path": "/articles", "visitor_id": "anon-1"}, - ) - assert r.status_code == 204 - - event = _latest_event(db, "anon-1") - assert event is not None - assert event.event == "pageview" - assert event.path == "/articles" - assert event.user_id is None - # user agent captured from the request headers - assert event.user_agent is not None - - -def test_collect_ignores_referer_header(client: TestClient, db: Session) -> None: - r = client.post( - f"{settings.API_V1_STR}/analytics/collect", - headers={"Referer": "https://agentique.ch/feed"}, - json={"path": "/feed", "visitor_id": "anon-5"}, - ) - assert r.status_code == 204 - event = _latest_event(db, "anon-5") - assert event is not None - assert event.referrer is None - - -def test_collect_drops_query_strings(client: TestClient, db: Session) -> None: - r = client.post( - f"{settings.API_V1_STR}/analytics/collect", - json={ - "path": "/auth?token=secret", - "referrer": "https://agentique.ch/reset-password?token=secret#top", - "visitor_id": "anon-6", - }, - ) - assert r.status_code == 204 - event = _latest_event(db, "anon-6") - assert event is not None - assert event.path == "/auth" - assert event.referrer == "https://agentique.ch/reset-password" - - -def test_collect_defaults_event_to_pageview(client: TestClient, db: Session) -> None: - r = client.post( - f"{settings.API_V1_STR}/analytics/collect", - json={"visitor_id": "anon-2"}, - ) - assert r.status_code == 204 - event = _latest_event(db, "anon-2") - assert event is not None - assert event.event == "pageview" - assert event.path is None - - -def test_collect_custom_event_with_props(client: TestClient, db: Session) -> None: - r = client.post( - f"{settings.API_V1_STR}/analytics/collect", - json={ - "event": "article_liked", - "path": "/articles/5", - "visitor_id": "anon-3", - "props": {"article_id": 5}, - }, - ) - assert r.status_code == 204 - event = _latest_event(db, "anon-3") - assert event is not None - assert event.event == "article_liked" - assert event.props == {"article_id": 5} - - -def test_collect_attaches_user_when_authenticated( - client: TestClient, db: Session, normal_user_token_headers: dict[str, str] -) -> None: - r = client.post( - f"{settings.API_V1_STR}/analytics/collect", - headers=normal_user_token_headers, - json={"path": "/profile", "visitor_id": "known-1"}, - ) - assert r.status_code == 204 - - user = db.exec(select(User).where(User.email == settings.EMAIL_TEST_USER)).first() - assert user is not None - event = _latest_event(db, "known-1") - assert event is not None - assert event.user_id == user.id - - -def test_collect_truncates_oversized_strings(client: TestClient, db: Session) -> None: - long_path = "/" + "x" * 5000 - r = client.post( - f"{settings.API_V1_STR}/analytics/collect", - json={"path": long_path, "visitor_id": "anon-4"}, - ) - assert r.status_code == 204 - event = _latest_event(db, "anon-4") - assert event is not None - assert event.path is not None - assert len(event.path) == 2048 diff --git a/backend/app/audience/models.py b/backend/app/audience/models.py deleted file mode 100644 index 5e6b7689..00000000 --- a/backend/app/audience/models.py +++ /dev/null @@ -1,103 +0,0 @@ -"""Audience schema: the reader account, its auth payloads, and the likes it leaves.""" - -from __future__ import annotations - -import uuid -from datetime import datetime - -from pydantic import EmailStr -from sqlalchemy import DateTime -from sqlmodel import Field, SQLModel - -from app.core.dates import get_datetime_utc - -# ─── User ──────────────────────────────────────────────────────────────────── - - -# Shared properties -class UserBase(SQLModel): - email: EmailStr = Field(unique=True, index=True, max_length=255) - is_active: bool = True - is_superuser: bool = False - full_name: str | None = Field(default=None, max_length=255) - - -# Properties to receive via API on creation -class UserCreate(UserBase): - pass - - -class UserUpdateMe(SQLModel): - full_name: str | None = Field(default=None, max_length=255) - email: EmailStr | None = Field(default=None, max_length=255) - - -# Database model, database table inferred from class name -class User(UserBase, table=True): - id: uuid.UUID = Field(default_factory=uuid.uuid4, primary_key=True) - # The secret in the sign-in link, replaced with every email sent. - login_token: str | None = Field( - default=None, unique=True, index=True, max_length=64 - ) - login_token_expires_at: datetime | None = Field( - default=None, - sa_type=DateTime(timezone=True), # type: ignore - ) - created_at: datetime | None = Field( - default_factory=get_datetime_utc, - sa_type=DateTime(timezone=True), # type: ignore - ) - - -# Properties to return via API, id is always required -class UserPublic(UserBase): - id: uuid.UUID - created_at: datetime | None = None - - -class UsersPublic(SQLModel): - data: list[UserPublic] - count: int - - -# Generic message -class Message(SQLModel): - message: str - - -# ─── Auth payloads ─────────────────────────────────────────────────────────── - - -# JSON payload containing access token -class Token(SQLModel): - access_token: str - token_type: str = "bearer" - - -# Contents of JWT token -class TokenPayload(SQLModel): - sub: str | None = None - - -class SignInRequest(SQLModel): - email: EmailStr = Field(max_length=255) - utm_source: str | None = Field(default=None, max_length=255) - - -class MagicLogin(SQLModel): - token: str = Field(max_length=64) - - -# ─── ArticleLike ───────────────────────────────────────────────────────────── -# A like is a reader action: its lifecycle follows the user, not the article, -# so it lives here rather than in the catalog. - - -class ArticleLike(SQLModel, table=True): - __tablename__ = "article_like" - user_id: uuid.UUID = Field(foreign_key="user.id", primary_key=True) - article_id: int = Field(foreign_key="article.id", primary_key=True) - created_at: datetime = Field( - default_factory=get_datetime_utc, - sa_type=DateTime(timezone=True), # type: ignore - ) diff --git a/backend/app/audience/routes.py b/backend/app/audience/routes.py deleted file mode 100644 index 8320c84f..00000000 --- a/backend/app/audience/routes.py +++ /dev/null @@ -1,202 +0,0 @@ -"""Reader-facing endpoints: signing up and in by email link, the signed-in account, and liking an article. - -Three routers rather than one, because each carries its own OpenAPI tag and the -generated client names its functions after that tag. -""" - -from __future__ import annotations - -import logging -from datetime import timedelta - -from fastapi import APIRouter, BackgroundTasks, HTTPException - -from app.audience import service -from app.audience.models import ( - MagicLogin, - Message, - SignInRequest, - Token, - User, - UserCreate, - UserPublic, - UserUpdateMe, -) -from app.catalog.articles import list_liked_articles -from app.catalog.models import ArticlesPublic -from app.core import security -from app.core.email import ( - EmailData, - generate_sign_in_email, - generate_welcome_email, - send_email, - sign_in_link, -) -from app.core.settings import settings -from app.deps import CurrentUser, SessionDep -from app.newsletter.subscribers import subscribe - -logger = logging.getLogger(__name__) - -login_router = APIRouter(tags=["login"]) -users_router = APIRouter(prefix="/users", tags=["users"]) -likes_router = APIRouter(tags=["likes"]) - - -# ─── Sign-in links ────────────────────────────────────────────────────────── - -LINK_SENT = "Check your inbox for your sign-in link" - - -def _send_link_email(email_to: str, email_data: EmailData, login_token: str) -> None: - # Runs after the response: a failed send must not fail the request, and the - # caller's answer must not depend on whether the account exists. - if settings.ENVIRONMENT == "development": - logger.info( - "[dev] Sign-in link for %s: %s", email_to, sign_in_link(login_token) - ) - if not settings.emails_enabled: - return - try: - send_email( - email_to=email_to, - subject=email_data.subject, - html_content=email_data.html_content, - ) - except Exception as e: - logger.error(f"Sign-in email failed for {email_to}: {e}") - - -@login_router.post("/login/link") -def request_sign_in_link( - session: SessionDep, body: SignInRequest, background_tasks: BackgroundTasks -) -> Message: - """ - Email a sign-in link. An unknown email first gets an account and a - newsletter subscription, and the link comes in a welcome email. - """ - user = service.get_user_by_email(session=session, email=body.email) - is_new = user is None - if user is None: - user = service.create_user( - session=session, user_create=UserCreate(email=body.email) - ) - subscribe(session=session, email=user.email, utm_source=body.utm_source) - elif not user.is_active: - return Message(message=LINK_SENT) - token = service.issue_login_token(session=session, db_user=user) - email = generate_welcome_email(token) if is_new else generate_sign_in_email(token) - background_tasks.add_task(_send_link_email, user.email, email, token) - return Message(message=LINK_SENT) - - -@login_router.post("/login/token") -def login_with_link(session: SessionDep, body: MagicLogin) -> Token: - """ - Swap the token from a sign-in link for an access token - """ - user = service.get_user_by_login_token(session=session, token=body.token) - if not user: - raise HTTPException(status_code=400, detail="Invalid sign-in link") - elif not user.is_active: - raise HTTPException(status_code=400, detail="Inactive user") - return _access_token(user) - - -@login_router.post("/login/refresh") -def refresh_token(current_user: CurrentUser) -> Token: - """ - Renew the access token - """ - return _access_token(current_user) - - -def _access_token(user: User) -> Token: - return Token( - access_token=security.create_access_token( - user.id, - expires_delta=timedelta(minutes=settings.ACCESS_TOKEN_EXPIRE_MINUTES), - ) - ) - - -@login_router.post("/login/test-token", response_model=UserPublic) -def test_token(current_user: CurrentUser) -> User: - """ - Test access token - """ - return current_user - - -# ─── The signed-in account ────────────────────────────────────────────────── - - -@users_router.patch("/me", response_model=UserPublic) -def update_user_me( - *, session: SessionDep, user_in: UserUpdateMe, current_user: CurrentUser -) -> User: - """ - Update own user. - """ - if user_in.email: - existing_user = service.get_user_by_email(session=session, email=user_in.email) - if existing_user and existing_user.id != current_user.id: - raise HTTPException( - status_code=409, detail="User with this email already exists" - ) - return service.update_profile( - session=session, db_user=current_user, changes=user_in - ) - - -@users_router.get("/me", response_model=UserPublic) -def read_user_me(current_user: CurrentUser) -> User: - """ - Get current user. - """ - return current_user - - -@users_router.delete("/me", response_model=Message) -def delete_user_me(session: SessionDep, current_user: CurrentUser) -> Message: - """ - Delete own user. - """ - if current_user.is_superuser: - raise HTTPException( - status_code=403, detail="Super users are not allowed to delete themselves" - ) - service.delete_account(session=session, db_user=current_user) - return Message(message="User deleted successfully") - - -# ─── Likes ────────────────────────────────────────────────────────────────── - - -@likes_router.put("/articles/{article_id}/like") -def like_article( - session: SessionDep, current_user: CurrentUser, article_id: int -) -> dict[str, bool]: - liked = service.like_article( - session=session, user_id=current_user.id, article_id=article_id - ) - if not liked: - raise HTTPException(status_code=404, detail="Article not found") - return {"ok": True} - - -@likes_router.delete("/articles/{article_id}/like") -def unlike_article( - session: SessionDep, current_user: CurrentUser, article_id: int -) -> dict[str, bool]: - service.unlike_article( - session=session, user_id=current_user.id, article_id=article_id - ) - return {"ok": True} - - -@likes_router.get("/me/liked-articles", response_model=ArticlesPublic) -def read_liked_articles( - session: SessionDep, current_user: CurrentUser -) -> ArticlesPublic: - return list_liked_articles(session, current_user.id) diff --git a/backend/app/audience/tests/factories.py b/backend/app/audience/tests/factories.py deleted file mode 100644 index 3640babd..00000000 --- a/backend/app/audience/tests/factories.py +++ /dev/null @@ -1,38 +0,0 @@ -"""Factory for creating random User rows in tests.""" - -from __future__ import annotations - -from datetime import timedelta - -from sqlmodel import Session - -from app.audience import service -from app.audience.models import User, UserCreate -from app.core.security import create_access_token -from app.core.settings import settings -from tests.random_data import random_email - - -def user_authentication_headers(user: User) -> dict[str, str]: - token = create_access_token(user.id, expires_delta=timedelta(minutes=5)) - return {"Authorization": f"Bearer {token}"} - - -def create_random_user(db: Session) -> User: - return service.create_user(session=db, user_create=UserCreate(email=random_email())) - - -def authentication_token_from_email(*, email: str, db: Session) -> dict[str, str]: - """ - Return a valid token for the user with given email. - - If the user doesn't exist it is created first. - """ - user = service.get_user_by_email(session=db, email=email) - if not user: - user = service.create_user(session=db, user_create=UserCreate(email=email)) - return user_authentication_headers(user) - - -def get_superuser_token_headers(db: Session) -> dict[str, str]: - return authentication_token_from_email(email=settings.FIRST_SUPERUSER, db=db) diff --git a/backend/app/catalog/models.py b/backend/app/catalog/models.py deleted file mode 100644 index c7674627..00000000 --- a/backend/app/catalog/models.py +++ /dev/null @@ -1,406 +0,0 @@ -"""Catalog schema: publishers, articles, tags, and the read shapes the feed serves.""" - -from __future__ import annotations - -import re -import unicodedata -from datetime import datetime -from enum import StrEnum - -from pgvector.sqlalchemy import Vector # type: ignore[import-untyped] -from sqlalchemy import JSON, Column, DateTime, Index -from sqlmodel import Field, SQLModel - -from app.core.dates import get_datetime_utc - - -def slugify(name: str) -> str: - """Publisher/tag slug: ASCII-fold, lowercase, non-alnum -> single hyphen. - - Used to map a fetched item's source name -> a Publisher.slug so the pipeline - can look the publisher up (or auto-create it). Deterministic and stable. - """ - ascii_name = ( - unicodedata.normalize("NFKD", name).encode("ascii", "ignore").decode("ascii") - ) - slug = re.sub(r"[^a-z0-9]+", "-", ascii_name.lower()).strip("-") - return slug - - -# ─── Enums (this module is the schema source of truth) ───────────────────── - - -class PublisherKind(StrEnum): - # Makes models and posts about them on its own domain. - lab = "lab" - company = "company" - individual = "individual" - media = "media" - # Aggregators (Hacker News, Reddit). Credited only for posts on their own - # site; anything they link to is credited to its author. - community = "community" - # A publisher the pipeline created from a URL it had not seen. The curation - # agent sets the real kind the first time it approves one of its articles. - unknown = "unknown" - - -class PublisherType(StrEnum): - """How the pipeline discovers a publisher's articles (its ingestion source).""" - - rss = "rss" - substack = "substack" - search = "search" - hn = "hn" - reddit = "reddit" - email = "email" - ainews = "ainews" - other = "other" - - -class TrustLevel(StrEnum): - low = "low" - medium = "medium" - high = "high" - - -class ArticleKind(StrEnum): - # A blog post, launch post or announcement: what kind of post it is follows - # from who published it (PublisherKind), not from a label on the article. - post = "post" - repo = "repo" - paper = "paper" - model = "model" - - -class Origin(StrEnum): - """The feed's "From" filter: what the article is when that is settled (a - repo, a paper, a model), otherwise who published it. Not stored; each - article falls under exactly one, see `catalog.articles.origin_condition`.""" - - lab = "lab" - company = "company" - individual = "individual" - media = "media" - repo = "repo" - paper = "paper" - model = "model" - - -# No longer written: `dev` covered most of the feed, and publisher kind plus -# article kind split it better. The column stays because deploy runs additive -# migrations only. -class Category(StrEnum): - dev = "dev" - models = "models" - research = "research" - - -class LinkPlatform(StrEnum): - website = "website" - rss = "rss" - # first-party domain to discover articles from via search, for labs that - # publish no RSS feed (see pipeline.sources.lab_watch). Value is the bare - # host, e.g. "anthropic.com". - search = "search" - twitter = "twitter" - github = "github" - substack = "substack" - youtube = "youtube" - linkedin = "linkedin" - mastodon = "mastodon" - discord = "discord" - email = "email" - - -# ─── Publisher ───────────────────────────────────────────────────────────── - - -class PublisherBase(SQLModel): - slug: str - name: str - kind: PublisherKind - type: PublisherType = PublisherType.other - description: str | None = None - image: str | None = None - links: dict[LinkPlatform, str] = Field( - default_factory=dict, sa_column=Column(JSON, nullable=False) - ) - trust: TrustLevel = TrustLevel.medium - is_active: bool = True - - -class Publisher(PublisherBase, table=True): - id: int | None = Field(default=None, primary_key=True) - # Broad publishers (a general engineering blog, not an AI one) that post - # mostly off-topic. No longer read: the fetch step used to drop their - # off-topic titles, and the curation agent judges topic now (ADR 11). The - # column stays because deploy runs additive migrations only. - topic_gated: bool = Field(default=False, nullable=False) - # What the nightly run last saw of this publisher, written by - # pipeline.runs: when its feed last answered, when it last gave us a new - # candidate, and the error it last failed with (cleared by the next good - # fetch). Ingestion bookkeeping, not on the public API. - last_fetched_at: datetime | None = Field( - default=None, - sa_type=DateTime(timezone=True), # type: ignore - ) - last_new_at: datetime | None = Field( - default=None, - sa_type=DateTime(timezone=True), # type: ignore - ) - last_error: str | None = None - created_at: datetime = Field( - default_factory=get_datetime_utc, - sa_type=DateTime(timezone=True), # type: ignore - ) - - -# ─── Article ─────────────────────────────────────────────────────────────── - - -class ArticleBase(SQLModel): - title: str - url: str - publisher_id: int = Field(foreign_key="publisher.id") - published_at: datetime | None = Field( - default=None, - sa_type=DateTime(timezone=True), # type: ignore - ) - score: int - kind: ArticleKind = ArticleKind.post - categories: list[Category] = Field( - default_factory=list, sa_column=Column(JSON, nullable=False) - ) - summary: str | None = None - content: str | None = None - - -class Article(ArticleBase, table=True): - id: int | None = Field(default=None, primary_key=True) - # 256-dim model2vec vectors; nullable until the import script runs - embedding: list[float] | None = Field( - default=None, sa_column=Column(Vector(256), nullable=True) - ) - # The judge's one-sentence account of `score`. Internal, not on the public - # API; null for articles scored before the scorer gave one. - score_reason: str | None = None - # The aggregator that found the link (Hacker News, a newsletter), when that - # is not the publisher itself. Null for an article from its publisher's feed. - found_via: str | None = None - # Set by the retired rescore script (removed with the LLM scorer): when this - # row was last rescored, and whether that rescore put it under the old - # threshold. Marking only; nothing deletes a marked row automatically. - rescored_at: datetime | None = Field( - default=None, - sa_type=DateTime(timezone=True), # type: ignore - ) - marked_for_deletion_at: datetime | None = Field( - default=None, - sa_type=DateTime(timezone=True), # type: ignore - ) - created_at: datetime = Field( - default_factory=get_datetime_utc, - sa_type=DateTime(timezone=True), # type: ignore - ) - - -# ─── Tag ─────────────────────────────────────────────────────────────────── - - -class Tag(SQLModel, table=True): - id: int | None = Field(default=None, primary_key=True) - slug: str = Field(unique=True) - name: str - # "when to apply this tag" — fed to the AssignTags prompt as the vocabulary - description: str | None = None - - -class ArticleTag(SQLModel, table=True): - __tablename__ = "article_tag" - article_id: int = Field(foreign_key="article.id", primary_key=True) - tag_id: int = Field(foreign_key="tag.id", primary_key=True) - - -# ─── Story ───────────────────────────────────────────────────────────────── -# A named thread that runs over days or weeks. Wider than coverage (one event -# told by several outlets): an agent names and keeps stories, see ADR 16. - - -class Story(SQLModel, table=True): - id: int | None = Field(default=None, primary_key=True) - slug: str = Field(unique=True) - name: str - blurb: str - created_at: datetime = Field( - default_factory=get_datetime_utc, - sa_type=DateTime(timezone=True), # type: ignore - ) - updated_at: datetime = Field( - default_factory=get_datetime_utc, - sa_type=DateTime(timezone=True), # type: ignore - ) - # When its newest article was published. What the rail orders by and what - # "quiet" is measured from. - last_article_at: datetime | None = Field( - default=None, - sa_type=DateTime(timezone=True), # type: ignore - ) - closed_at: datetime | None = Field( - default=None, - sa_type=DateTime(timezone=True), # type: ignore - ) - - -class StoryArticle(SQLModel, table=True): - """An article in a story. At most one open story per article, enforced in - `catalog.stories.save_story`: a closed story keeps its articles.""" - - __tablename__ = "story_article" - # The primary key leads with story_id; "which story holds this article" is - # asked on every save. - __table_args__ = (Index("ix_story_article_article_id", "article_id"),) - story_id: int = Field(foreign_key="story.id", primary_key=True, ondelete="CASCADE") - article_id: int = Field( - foreign_key="article.id", primary_key=True, ondelete="CASCADE" - ) - added_at: datetime = Field( - default_factory=get_datetime_utc, - sa_type=DateTime(timezone=True), # type: ignore - ) - - -# ─── Read API shapes ─────────────────────────────────────────────────────── -# Consumer-facing views. `publisher` is nested (replaces the old `source` string) -# and `tags` come from the article_tag join. Pipeline provenance and bulk fields -# (content, embedding, links, trust) stay internal. - - -class PublisherPublic(SQLModel): - id: int - slug: str - name: str - kind: PublisherKind - image: str | None = None - - -class TagPublic(SQLModel): - slug: str - name: str - - -class ArticlePublic(SQLModel): - id: int - title: str - url: str - summary: str | None = None - score: int - kind: ArticleKind - found_via: str | None = None - published_at: datetime | None = None - created_at: datetime | None = None - publisher: PublisherPublic - tags: list[TagPublic] = Field(default_factory=list) - like_count: int = 0 - liked_by_me: bool = False - - -class ArticlesPublic(SQLModel): - data: list[ArticlePublic] - count: int - - -# ─── Facets (sidebar filter options) ─────────────────────────────────────── - - -class PublisherFacet(SQLModel): - slug: str - name: str - count: int - - -class TagFacet(SQLModel): - slug: str - name: str - count: int - - -class ArticleFacets(SQLModel): - publishers: list[PublisherFacet] - tags: list[TagFacet] - - -# ─── Origin counts (the left rail's "Who wrote this week") ───────────────── - - -class OriginCount(SQLModel): - origin: Origin - count: int - - -class FoundViaCount(SQLModel): - # Null: the publisher's own feed. - name: str | None - count: int - - -class OriginCounts(SQLModel): - since: datetime - total: int - origins: list[OriginCount] - # Posts by a publisher still `unknown`: under no origin until curation - # names its kind. - unlabelled: int - found_via: list[FoundViaCount] - - -# ─── Voices (the left rail's "Voices") ───────────────────────────────────── - - -class VoiceKind(StrEnum): - """The Voices panel's tabs: Writers, Labs, Companies.""" - - individual = "individual" - lab = "lab" - company = "company" - - -class VoiceArticle(SQLModel): - title: str - url: str - score: int - published_at: datetime | None = None - - -class Voice(SQLModel): - slug: str - name: str - count: int - # Oldest first, the last `facets.VOICE_SCORES` of `count`. - scores: list[int] - best: VoiceArticle - - -# ─── Stories (right rail) ────────────────────────────────────────────────── - - -class StoryArticlePublic(SQLModel): - id: int - title: str - url: str - publisher: str - published_at: datetime | None = None - score: int - - -class StoryPublic(SQLModel): - slug: str - name: str - blurb: str - article_count: int - publisher_count: int - first_at: datetime | None = None - last_at: datetime | None = None - # An article joined it in the last 24 hours. - grew_today: bool - # Newest first, at most `stories.STORY_ARTICLES` of `article_count`. - articles: list[StoryArticlePublic] diff --git a/backend/app/catalog/tests/factories.py b/backend/app/catalog/tests/factories.py deleted file mode 100644 index 96bc7760..00000000 --- a/backend/app/catalog/tests/factories.py +++ /dev/null @@ -1,75 +0,0 @@ -"""Factory for creating random Article rows in tests.""" - -from __future__ import annotations - -import random - -from sqlmodel import Session - -from app.catalog.models import ( - Article, - ArticleKind, - ArticleTag, - Publisher, - PublisherKind, - Tag, - slugify, -) -from tests.random_data import random_lower_string - - -def create_random_publisher(db: Session, **overrides: object) -> Publisher: - name = random_lower_string() - defaults: dict[str, object] = { - "slug": slugify(name), - "name": name, - "kind": PublisherKind.community, - } - defaults.update(overrides) - publisher = Publisher(**defaults) # type: ignore[arg-type] # ty: ignore[invalid-argument-type] - db.add(publisher) - db.commit() - db.refresh(publisher) - return publisher - - -def create_random_tag(db: Session, **overrides: object) -> Tag: - name = random_lower_string() - defaults: dict[str, object] = { - "slug": slugify(name), - "name": name, - "description": random_lower_string(), - } - defaults.update(overrides) - tag = Tag(**defaults) # type: ignore[arg-type] # ty: ignore[invalid-argument-type] - db.add(tag) - db.commit() - db.refresh(tag) - return tag - - -def create_random_article(db: Session, **overrides: object) -> Article: - # every article needs a publisher; make one unless the caller supplied it - if "publisher_id" not in overrides: - overrides["publisher_id"] = create_random_publisher(db).id - - defaults: dict[str, object] = { - "title": random_lower_string(), - "url": f"https://example.com/{random_lower_string()}", - "score": random.randint(1, 10), - "summary": random_lower_string(), - "kind": ArticleKind.post, - "content": random_lower_string(), - } - defaults.update(overrides) - article = Article(**defaults) # type: ignore[arg-type] # ty: ignore[invalid-argument-type] - db.add(article) - db.commit() - db.refresh(article) - return article - - -def tag_article(db: Session, article: Article, tag: Tag) -> None: - assert article.id is not None and tag.id is not None - db.add(ArticleTag(article_id=article.id, tag_id=tag.id)) - db.commit() diff --git a/backend/app/core/db.py b/backend/app/core/db.py deleted file mode 100644 index 2ded8f55..00000000 --- a/backend/app/core/db.py +++ /dev/null @@ -1,9 +0,0 @@ -"""The SQLAlchemy engine every session is opened against.""" - -from __future__ import annotations - -from sqlmodel import create_engine - -from app.core.settings import settings - -engine = create_engine(str(settings.SQLALCHEMY_DATABASE_URI)) diff --git a/backend/app/core/db/__init__.py b/backend/app/core/db/__init__.py new file mode 100644 index 00000000..2e71c225 --- /dev/null +++ b/backend/app/core/db/__init__.py @@ -0,0 +1,34 @@ +"""The tables, one module each. The engine is in `engine`, imported on its own. + +This package imports every table module, so `SQLModel.metadata` is complete as +soon as anything in `app.core.db` is imported. Alembic's autogenerate reads that +metadata, and a table it cannot see gets no migration. +""" + +from __future__ import annotations + +from app.core.db import ( + article, + event, + ledger, + like, + publisher, + run, + story, + subscriber, + tag, + user, +) + +__all__ = [ + "article", + "event", + "ledger", + "like", + "publisher", + "run", + "story", + "subscriber", + "tag", + "user", +] diff --git a/backend/app/core/db/article.py b/backend/app/core/db/article.py new file mode 100644 index 00000000..b3de90d1 --- /dev/null +++ b/backend/app/core/db/article.py @@ -0,0 +1,103 @@ +"""The article table, its kind, and the kind a URL's host settles on its own.""" + +from __future__ import annotations + +from datetime import datetime +from enum import StrEnum + +from pgvector.sqlalchemy import Vector # type: ignore[import-untyped] +from sqlalchemy import JSON, Column, DateTime, Index +from sqlmodel import Field, SQLModel + +from app.core.dates import get_datetime_utc +from app.core.urls import hostname + + +class ArticleKind(StrEnum): + # A blog post, launch post or announcement: what kind of post it is follows + # from who published it (PublisherKind), not from a label on the article. + post = "post" + repo = "repo" + paper = "paper" + model = "model" + + +# No longer written: `dev` covered most of the feed, and publisher kind plus +# article kind split it better. The column stays because deploy runs additive +# migrations only. +class Category(StrEnum): + dev = "dev" + models = "models" + research = "research" + + +class ArticleBase(SQLModel): + title: str + url: str + publisher_id: int = Field(foreign_key="publisher.id") + published_at: datetime | None = Field( + default=None, + sa_type=DateTime(timezone=True), # type: ignore + ) + score: int + kind: ArticleKind = ArticleKind.post + categories: list[Category] = Field( + default_factory=list, sa_column=Column(JSON, nullable=False) + ) + summary: str | None = None + content: str | None = None + + +class Article(ArticleBase, table=True): + __table_args__ = ( + Index("ix_article_score", "score"), + Index("ix_article_published_at", "published_at"), + Index( + "ix_article_embedding_hnsw", + "embedding", + postgresql_using="hnsw", + postgresql_ops={"embedding": "vector_cosine_ops"}, + ), + ) + id: int | None = Field(default=None, primary_key=True) + # 256-dim model2vec vectors; nullable until the import script runs + embedding: list[float] | None = Field( + default=None, sa_column=Column(Vector(256), nullable=True) + ) + # The judge's one-sentence account of `score`. Internal, not on the public + # API; null for articles scored before the scorer gave one. + score_reason: str | None = None + # The aggregator that found the link (Hacker News, a newsletter), when that + # is not the publisher itself. Null for an article from its publisher's feed. + found_via: str | None = None + # Set by the retired rescore script (removed with the LLM scorer): when this + # row was last rescored, and whether that rescore put it under the old + # threshold. Marking only; nothing deletes a marked row automatically. + rescored_at: datetime | None = Field( + default=None, + sa_type=DateTime(timezone=True), # type: ignore + ) + marked_for_deletion_at: datetime | None = Field( + default=None, + sa_type=DateTime(timezone=True), # type: ignore + ) + created_at: datetime = Field( + default_factory=get_datetime_utc, + sa_type=DateTime(timezone=True), # type: ignore + ) + + +def kind_from_url(url: str) -> ArticleKind | None: + """ArticleKind from a URL's host, or None if the host does not settle it. + + A github.com link is a repo and an arxiv.org link is a paper whatever the + text says, so the host is asked first. + """ + host = hostname(url) + if host in ("github.com", "gitlab.com"): + return ArticleKind.repo + if host in ("huggingface.co", "hf.co"): + return ArticleKind.model + if host in ("arxiv.org", "ar5iv.labs.arxiv.org"): + return ArticleKind.paper + return None diff --git a/backend/app/core/db/engine.py b/backend/app/core/db/engine.py new file mode 100644 index 00000000..d1b9c137 --- /dev/null +++ b/backend/app/core/db/engine.py @@ -0,0 +1,16 @@ +"""The one engine for every session, and the session a request uses.""" + +from __future__ import annotations + +from collections.abc import Generator + +from sqlmodel import Session, create_engine + +from app.core.settings import settings + +engine = create_engine(str(settings.SQLALCHEMY_DATABASE_URI)) + + +def get_session() -> Generator[Session]: + with Session(engine) as session: + yield session diff --git a/backend/app/core/db/event.py b/backend/app/core/db/event.py new file mode 100644 index 00000000..2ec7446b --- /dev/null +++ b/backend/app/core/db/event.py @@ -0,0 +1,35 @@ +"""First-party pageview and custom-event rows, keyed on an anonymous visitor.""" + +from __future__ import annotations + +import uuid +from datetime import datetime + +from sqlalchemy import JSON, Column, DateTime +from sqlmodel import Field, SQLModel + +from app.core.dates import get_datetime_utc + + +class AnalyticsEvent(SQLModel, table=True): + """First-party analytics event — one row per pageview or custom event.""" + + __tablename__ = "analytics_event" + id: int | None = Field(default=None, primary_key=True) + event: str = Field(default="pageview", index=True) + path: str | None = Field(default=None, index=True) + referrer: str | None = None + # anonymous client id persisted in the browser (localStorage), not a cookie + visitor_id: str | None = Field(default=None, index=True) + # set only when the request carries a valid bearer token + user_id: uuid.UUID | None = Field(default=None, foreign_key="user.id") + user_agent: str | None = None + # arbitrary metadata for custom events + props: dict[str, object] = Field( + default_factory=dict, sa_column=Column(JSON, nullable=False) + ) + created_at: datetime = Field( + default_factory=get_datetime_utc, + index=True, + sa_type=DateTime(timezone=True), # type: ignore + ) diff --git a/backend/app/core/db/factories.py b/backend/app/core/db/factories.py new file mode 100644 index 00000000..c5908ba7 --- /dev/null +++ b/backend/app/core/db/factories.py @@ -0,0 +1,115 @@ +"""Test factories: random users, publishers, tags and articles. + +They write rows straight to the tables, so a test anywhere can use them without +importing a feature. +""" + +from __future__ import annotations + +import random +import string +from datetime import timedelta + +from sqlmodel import Session, select + +from app.core.db.article import Article, ArticleKind +from app.core.db.publisher import Publisher, PublisherKind, slugify +from app.core.db.tag import ArticleTag, Tag +from app.core.db.user import User +from app.core.security import create_access_token +from app.core.settings import settings + + +def random_lower_string() -> str: + """Random enough that two tests never collide on a unique column.""" + return "".join(random.choices(string.ascii_lowercase, k=32)) + + +def random_email() -> str: + return f"{random_lower_string()}@{random_lower_string()}.com" + + +def create_user(db: Session, email: str) -> User: + user = User(email=email) + db.add(user) + db.commit() + db.refresh(user) + return user + + +def create_random_user(db: Session) -> User: + return create_user(db, random_email()) + + +def user_authentication_headers(user: User) -> dict[str, str]: + token = create_access_token(user.id, expires_delta=timedelta(minutes=5)) + return {"Authorization": f"Bearer {token}"} + + +def authentication_token_from_email(*, email: str, db: Session) -> dict[str, str]: + """Headers for the user with this email, created first if it isn't there.""" + user = db.exec(select(User).where(User.email == email)).first() + if not user: + user = create_user(db, email) + return user_authentication_headers(user) + + +def get_superuser_token_headers(db: Session) -> dict[str, str]: + return authentication_token_from_email(email=settings.FIRST_SUPERUSER, db=db) + + +def create_random_publisher(db: Session, **overrides: object) -> Publisher: + name = random_lower_string() + defaults: dict[str, object] = { + "slug": slugify(name), + "name": name, + "kind": PublisherKind.community, + } + defaults.update(overrides) + publisher = Publisher(**defaults) # ty: ignore[invalid-argument-type] + db.add(publisher) + db.commit() + db.refresh(publisher) + return publisher + + +def create_random_tag(db: Session, **overrides: object) -> Tag: + name = random_lower_string() + defaults: dict[str, object] = { + "slug": slugify(name), + "name": name, + "description": random_lower_string(), + } + defaults.update(overrides) + tag = Tag(**defaults) # ty: ignore[invalid-argument-type] + db.add(tag) + db.commit() + db.refresh(tag) + return tag + + +def create_random_article(db: Session, **overrides: object) -> Article: + # every article needs a publisher; make one unless the caller supplied it + if "publisher_id" not in overrides: + overrides["publisher_id"] = create_random_publisher(db).id + + defaults: dict[str, object] = { + "title": random_lower_string(), + "url": f"https://example.com/{random_lower_string()}", + "score": random.randint(1, 10), + "summary": random_lower_string(), + "kind": ArticleKind.post, + "content": random_lower_string(), + } + defaults.update(overrides) + article = Article(**defaults) # ty: ignore[invalid-argument-type] + db.add(article) + db.commit() + db.refresh(article) + return article + + +def tag_article(db: Session, article: Article, tag: Tag) -> None: + assert article.id is not None and tag.id is not None + db.add(ArticleTag(article_id=article.id, tag_id=tag.id)) + db.commit() diff --git a/backend/app/core/db/ledger.py b/backend/app/core/db/ledger.py new file mode 100644 index 00000000..651c4d43 --- /dev/null +++ b/backend/app/core/db/ledger.py @@ -0,0 +1,146 @@ +"""The reject ledger: URLs the funnel turned down, and candidates waiting on the curation agent. + +Two jobs. ``filter_known_urls`` reads it so a rejected URL is never fetched and +judged again. And it keeps the evidence - the text the judge saw, the links, the +score, the judge's reason - so a reject can be audited, re-judged under a new +rubric, or used as a training negative instead of vanishing. + +The embedding is deliberately not stored: it is a pure function of the stored +title and ``content[:SNIPPET_CAP]``, so it can be rebuilt offline in seconds. +""" + +from __future__ import annotations + +from datetime import datetime +from enum import StrEnum +from typing import NotRequired, TypedDict + +from sqlalchemy import JSON, Column, DateTime, String, Text +from sqlmodel import Field, Session, SQLModel + +from app.core.dates import get_datetime_utc, parse_date + +# What a settled reject keeps: enough to re-score with a longer input or +# re-embed with a better model later; the full body would make the ledger most +# of the database. +CONTENT_CAP = 2000 +# What a pending candidate keeps, so the curation agent reads the article from +# the db instead of fetching the page again. Trimmed to `CONTENT_CAP` once the +# agent rejects it (`curation.reject`), so only the night's queue is ever this +# large. +CANDIDATE_CAP = 12000 + + +class RejectStage(StrEnum): + """The step that turned a URL down, or ``pending`` for one still waiting. + + ``pending`` is the one stage that is not a rejection. The nightly run ends + by writing it, and the curation agent turns it into an Article or into + ``below_threshold``. It lives in this table because the row already holds + everything a judge needs -- title, source, publisher, content, traction -- + and because ``filter_known_urls`` reads the table, so a candidate waiting + for the agent is not fetched again the next night. + """ + + pending = "pending" + thin_repo = "thin_repo" + prefilter = "prefilter" + duplicate = "duplicate" + below_threshold = "below_threshold" + unusable_summary = "unusable_summary" + + +class Reject(SQLModel, table=True): + """A URL the funnel turned down, or one waiting on the curation agent. + + The name is the common case. A ``pending`` row is the exception: the same + columns, holding a candidate the agent has not read yet. + + Every column but ``url`` and ``created_at`` is nullable: rows written before + the ledger kept evidence carry only the URL. + """ + + # Still named scored_url: deploy runs additive migrations only, and a + # rename is not one. + __tablename__ = "scored_url" + url: str = Field(primary_key=True, sa_type=Text) + created_at: datetime = Field( + default_factory=get_datetime_utc, + sa_type=DateTime(timezone=True), # type: ignore + ) + # A plain string, not a Postgres enum, so a new stage needs no migration. + stage: RejectStage | None = Field(default=None, sa_type=String) + title: str | None = None + source: str | None = None + publisher_id: int | None = Field(default=None, foreign_key="publisher.id") + published_at: datetime | None = Field( + default=None, + sa_type=DateTime(timezone=True), # type: ignore + ) + content: str | None = None + traction: str | None = None + score: int | None = None + # The judge's one-sentence account of the score; below_threshold only. + reason: str | None = None + # Stage-specific evidence: {"stars": 12}, {"keep_proba": 0.08}, + # {"dup_of": ""}. + # none_as_null: without it an absent detail is stored as JSON 'null', which + # `detail IS NULL` does not match. + detail: dict[str, object] | None = Field( + default=None, sa_column=Column(JSON(none_as_null=True), nullable=True) + ) + # The repo / model / paper / docs links the article body makes, + # {"repo": [...], "paper": [...]}; see ``app.core.extract``. + links: dict[str, list[str]] | None = Field( + default=None, sa_column=Column(JSON(none_as_null=True), nullable=True) + ) + + +class LedgerItem(TypedDict): + """What `record_reject` reads from an item. The pipeline's ``Candidate`` + has these keys and more.""" + + title: str + url: str + content: str + published_date: str | None + source: str + publisher_id: int + traction: NotRequired[str] + links: NotRequired[dict[str, list[str]]] + + +def _pg_text(text: str) -> str: + # Postgres text columns reject NUL bytes, and fetched pages occasionally + # carry one. The failed commit would take the whole source down with it. + return text.replace("\x00", "") + + +def record_reject( + session: Session, + a: LedgerItem, + stage: RejectStage, + *, + score: int | None = None, + reason: str | None = None, + detail: dict[str, object] | None = None, +) -> None: + """Stage a reject row. The caller commits, as it did for the bare URL.""" + cap = CANDIDATE_CAP if stage == RejectStage.pending else CONTENT_CAP + content = _pg_text(a["content"][:cap]) + session.merge( + Reject( + url=a["url"], + stage=stage, + title=_pg_text(a["title"]), + source=a["source"], + publisher_id=a["publisher_id"], + published_at=parse_date(a["published_date"]), + content=content or None, + traction=a.get("traction"), + links=a.get("links") or None, + score=score, + reason=reason, + detail=detail, + ) + ) diff --git a/backend/app/core/db/like.py b/backend/app/core/db/like.py new file mode 100644 index 00000000..fe94b60e --- /dev/null +++ b/backend/app/core/db/like.py @@ -0,0 +1,32 @@ +"""A reader's like on an article, and the per-article count the feed and newsletter read. + +A like is a reader action: its lifecycle follows the user, not the article. +""" + +from __future__ import annotations + +import uuid +from datetime import datetime + +from sqlalchemy import DateTime, Subquery, func +from sqlmodel import Field, SQLModel, select + +from app.core.dates import get_datetime_utc + + +class ArticleLike(SQLModel, table=True): + __tablename__ = "article_like" + user_id: uuid.UUID = Field(foreign_key="user.id", primary_key=True) + article_id: int = Field(foreign_key="article.id", primary_key=True) + created_at: datetime = Field( + default_factory=get_datetime_utc, + sa_type=DateTime(timezone=True), # type: ignore + ) + + +def like_counts_subquery() -> Subquery: + return ( + select(ArticleLike.article_id, func.count().label("like_count")) + .group_by(ArticleLike.article_id) # type: ignore[arg-type] # ty: ignore[invalid-argument-type] + .subquery() + ) diff --git a/backend/app/core/db/publisher.py b/backend/app/core/db/publisher.py new file mode 100644 index 00000000..60638ff7 --- /dev/null +++ b/backend/app/core/db/publisher.py @@ -0,0 +1,116 @@ +"""The publisher table: who an article is credited to, and how its articles are found.""" + +from __future__ import annotations + +import re +import unicodedata +from datetime import datetime +from enum import StrEnum + +from sqlalchemy import JSON, Column, DateTime +from sqlmodel import Field, SQLModel + +from app.core.dates import get_datetime_utc + + +def slugify(name: str) -> str: + """Publisher/tag slug: ASCII-fold, lowercase, non-alnum -> single hyphen. + + Used to map a fetched item's source name -> a Publisher.slug so the pipeline + can look the publisher up (or auto-create it). Deterministic and stable. + """ + ascii_name = ( + unicodedata.normalize("NFKD", name).encode("ascii", "ignore").decode("ascii") + ) + slug = re.sub(r"[^a-z0-9]+", "-", ascii_name.lower()).strip("-") + return slug + + +class PublisherKind(StrEnum): + # Makes models and posts about them on its own domain. + lab = "lab" + company = "company" + individual = "individual" + media = "media" + # Aggregators (Hacker News, Reddit). Credited only for posts on their own + # site; anything they link to is credited to its author. + community = "community" + # A publisher the pipeline created from a URL it had not seen. The curation + # agent sets the real kind the first time it approves one of its articles. + unknown = "unknown" + + +class PublisherType(StrEnum): + """How the pipeline discovers a publisher's articles (its ingestion source).""" + + rss = "rss" + substack = "substack" + search = "search" + hn = "hn" + reddit = "reddit" + email = "email" + ainews = "ainews" + other = "other" + + +class TrustLevel(StrEnum): + low = "low" + medium = "medium" + high = "high" + + +class LinkPlatform(StrEnum): + website = "website" + rss = "rss" + # first-party domain to discover articles from via search, for labs that + # publish no RSS feed (see features/collect/lab_watch.py). Value is the bare + # host, e.g. "anthropic.com". + search = "search" + twitter = "twitter" + github = "github" + substack = "substack" + youtube = "youtube" + linkedin = "linkedin" + mastodon = "mastodon" + discord = "discord" + email = "email" + + +class PublisherBase(SQLModel): + slug: str + name: str + kind: PublisherKind + type: PublisherType = PublisherType.other + description: str | None = None + image: str | None = None + links: dict[LinkPlatform, str] = Field( + default_factory=dict, sa_column=Column(JSON, nullable=False) + ) + trust: TrustLevel = TrustLevel.medium + is_active: bool = True + + +class Publisher(PublisherBase, table=True): + id: int | None = Field(default=None, primary_key=True) + # Broad publishers (a general engineering blog, not an AI one) that post + # mostly off-topic. No longer read: the fetch step used to drop their + # off-topic titles, and the curation agent judges topic now (ADR 11). The + # column stays because deploy runs additive migrations only. + topic_gated: bool = Field(default=False, nullable=False) + # What the nightly run last saw of this publisher, written by + # features/collect/runs.py: when its feed last answered, when it last gave + # us a new candidate, and the error it last failed with (cleared by the + # next good fetch). Ingestion bookkeeping, not on the public API. + last_fetched_at: datetime | None = Field( + default=None, + sa_type=DateTime(timezone=True), # type: ignore + ) + last_new_at: datetime | None = Field( + default=None, + sa_type=DateTime(timezone=True), # type: ignore + ) + last_error: str | None = None + created_at: datetime = Field( + default_factory=get_datetime_utc, + sa_type=DateTime(timezone=True), # type: ignore + ) diff --git a/backend/app/core/db/run.py b/backend/app/core/db/run.py new file mode 100644 index 00000000..ceb812a4 --- /dev/null +++ b/backend/app/core/db/run.py @@ -0,0 +1,41 @@ +"""One row per nightly pipeline run.""" + +from __future__ import annotations + +from datetime import datetime + +from sqlalchemy import Column, DateTime, text +from sqlalchemy.dialects.postgresql import JSONB +from sqlmodel import Field, SQLModel + +from app.core.dates import get_datetime_utc + + +class PipelineRun(SQLModel, table=True): + """One row per nightly pipeline run: did it finish, how long it took, and + what each source fetched and queued.""" + + __tablename__ = "pipeline_run" + id: int | None = Field(default=None, primary_key=True) + started_at: datetime = Field( + default_factory=get_datetime_utc, + sa_type=DateTime(timezone=True), # type: ignore + ) + finished_at: datetime | None = Field( + default=None, + sa_type=DateTime(timezone=True), # type: ignore + ) + duration_ms: int | None = None + ok: bool = Field(default=False) + # [{source, fetched, queued, error}]. Rows from before 2026-09-23 carry + # the older, longer funnel shape. + sources: list[dict[str, object]] = Field( + default_factory=list, sa_column=Column(JSONB, nullable=False) + ) + # Per-publisher counts from before the Publisher row kept its own + # (last_fetched_at, last_new_at, last_error). No longer read; new rows get + # an empty list. + publishers: list[dict[str, object]] = Field( + default_factory=list, + sa_column=Column(JSONB, nullable=False, server_default=text("'[]'::jsonb")), + ) diff --git a/backend/app/core/db/story.py b/backend/app/core/db/story.py new file mode 100644 index 00000000..5458cc3d --- /dev/null +++ b/backend/app/core/db/story.py @@ -0,0 +1,57 @@ +"""Stories: named threads that run over days or weeks, and the articles in them. + +Wider than coverage (one event told by several outlets): an agent names and +keeps stories, see ADR 16. +""" + +from __future__ import annotations + +from datetime import datetime + +from sqlalchemy import DateTime, Index +from sqlmodel import Field, SQLModel + +from app.core.dates import get_datetime_utc + + +class Story(SQLModel, table=True): + id: int | None = Field(default=None, primary_key=True) + slug: str = Field(unique=True) + name: str + blurb: str + created_at: datetime = Field( + default_factory=get_datetime_utc, + sa_type=DateTime(timezone=True), # type: ignore + ) + updated_at: datetime = Field( + default_factory=get_datetime_utc, + sa_type=DateTime(timezone=True), # type: ignore + ) + # When its newest article was published. What the rail orders by and what + # "quiet" is measured from. + last_article_at: datetime | None = Field( + default=None, + sa_type=DateTime(timezone=True), # type: ignore + ) + closed_at: datetime | None = Field( + default=None, + sa_type=DateTime(timezone=True), # type: ignore + ) + + +class StoryArticle(SQLModel, table=True): + """An article in a story. At most one open story per article, enforced in + `stories.service.save_story`: a closed story keeps its articles.""" + + __tablename__ = "story_article" + # The primary key leads with story_id; "which story holds this article" is + # asked on every save. + __table_args__ = (Index("ix_story_article_article_id", "article_id"),) + story_id: int = Field(foreign_key="story.id", primary_key=True, ondelete="CASCADE") + article_id: int = Field( + foreign_key="article.id", primary_key=True, ondelete="CASCADE" + ) + added_at: datetime = Field( + default_factory=get_datetime_utc, + sa_type=DateTime(timezone=True), # type: ignore + ) diff --git a/backend/app/core/db/subscriber.py b/backend/app/core/db/subscriber.py new file mode 100644 index 00000000..1b4bd543 --- /dev/null +++ b/backend/app/core/db/subscriber.py @@ -0,0 +1,29 @@ +"""The newsletter subscriber row.""" + +from __future__ import annotations + +from datetime import datetime + +from sqlalchemy import Column, DateTime, Text +from sqlalchemy.dialects.postgresql import JSONB +from sqlmodel import Field, SQLModel + +from app.core.dates import get_datetime_utc + + +class NewsletterSubscriber(SQLModel, table=True): + __tablename__ = "newsletter_subscriber" + email: str = Field(primary_key=True, sa_type=Text) + categories: list[str] = Field( + default_factory=lambda: ["all"], sa_column=Column(JSONB, nullable=False) + ) + custom_category: str = Field(default="", sa_type=Text) + utm_source: str | None = Field(default=None, sa_type=Text) + created_at: datetime = Field( + default_factory=get_datetime_utc, + sa_type=DateTime(timezone=True), # type: ignore + ) + updated_at: datetime = Field( + default_factory=get_datetime_utc, + sa_type=DateTime(timezone=True), # type: ignore + ) diff --git a/backend/app/core/db/tag.py b/backend/app/core/db/tag.py new file mode 100644 index 00000000..d930a98f --- /dev/null +++ b/backend/app/core/db/tag.py @@ -0,0 +1,19 @@ +"""The label vocabulary and the join that puts labels on articles.""" + +from __future__ import annotations + +from sqlmodel import Field, SQLModel + + +class Tag(SQLModel, table=True): + id: int | None = Field(default=None, primary_key=True) + slug: str = Field(unique=True) + name: str + # "when to apply this tag" — fed to the AssignTags prompt as the vocabulary + description: str | None = None + + +class ArticleTag(SQLModel, table=True): + __tablename__ = "article_tag" + article_id: int = Field(foreign_key="article.id", primary_key=True) + tag_id: int = Field(foreign_key="tag.id", primary_key=True) diff --git a/backend/app/core/db/test_engine.py b/backend/app/core/db/test_engine.py new file mode 100644 index 00000000..eb3a5145 --- /dev/null +++ b/backend/app/core/db/test_engine.py @@ -0,0 +1,37 @@ +"""The engine, the request session, and the tables against the migrated schema.""" + +from __future__ import annotations + +from alembic.autogenerate import compare_metadata +from alembic.migration import MigrationContext +from sqlalchemy import text +from sqlmodel import SQLModel + +from app.core.db.engine import engine, get_session +from app.core.settings import settings + + +def test_tables_match_the_migrations() -> None: + """`alembic check` in the test run: a table edit without a migration, or a + migration the tables do not declare, fails here.""" + with engine.connect() as conn: + context = MigrationContext.configure(conn, opts={"compare_type": True}) + diff = compare_metadata(context, SQLModel.metadata) + assert diff == [] + + +def test_engine_connects_with_the_settings_credentials() -> None: + assert engine.url.database == settings.POSTGRES_DB + with engine.connect() as conn: + user = conn.execute(text("SELECT current_user")).scalar_one() + assert user == settings.POSTGRES_USER + + +def test_the_request_session_is_closed_after_the_request() -> None: + sessions = get_session() + session = next(sessions) + assert session.bind is engine + session.connection().execute(text("SELECT 1")) + assert session.in_transaction() + sessions.close() + assert not session.in_transaction() diff --git a/backend/tests/test_models_enums.py b/backend/app/core/db/test_enums.py similarity index 71% rename from backend/tests/test_models_enums.py rename to backend/app/core/db/test_enums.py index c1977894..a0eafa50 100644 --- a/backend/tests/test_models_enums.py +++ b/backend/app/core/db/test_enums.py @@ -14,23 +14,12 @@ from enum import StrEnum from pathlib import Path -from app.analytics import models as analytics_models -from app.audience import models as audience_models -from app.catalog import models as catalog_models -from app.newsletter import models as newsletter_models -from pipeline import models as pipeline_models - -# Every module that declares schema. A domain missing here is a domain whose -# enums nobody guards. -MODEL_MODULES = ( - analytics_models, - audience_models, - catalog_models, - newsletter_models, - pipeline_models, -) +from app.core import db + +# Every table module, so a new one is guarded without a line here. +MODEL_MODULES = [getattr(db, name) for name in db.__all__] -VERSIONS = Path(__file__).parent.parent / "app" / "alembic" / "versions" +VERSIONS = Path(__file__).resolve().parents[3] / "migrations" / "versions" # SQLModel names a Postgres enum type after the lowercased class name. ENUM_MODELS = { @@ -45,7 +34,7 @@ ) -def test_migration_enum_labels_exist_in_python(): +def test_migration_enum_labels_exist_in_python() -> None: added = [ (m.group(1).lower(), m.group(2)) for path in VERSIONS.glob("*.py") diff --git a/backend/app/core/db/test_ledger.py b/backend/app/core/db/test_ledger.py new file mode 100644 index 00000000..abe99206 --- /dev/null +++ b/backend/app/core/db/test_ledger.py @@ -0,0 +1,156 @@ +"""What a ledger row keeps, and what it must strip to be insertable at all.""" + +from __future__ import annotations + +from collections.abc import Generator +from typing import cast + +import pytest +from sqlmodel import Session, col, delete, select + +from app.core.db.factories import create_random_publisher, random_lower_string +from app.core.db.ledger import ( + CANDIDATE_CAP, + CONTENT_CAP, + LedgerItem, + Reject, + RejectStage, + record_reject, +) +from app.core.db.publisher import Publisher + +URL = f"https://example.com/{random_lower_string()}" + + +class _FakeSession: + def __init__(self) -> None: + self.rows: list[Reject] = [] + + def merge(self, obj: Reject) -> None: + self.rows.append(obj) + + +def _session() -> tuple[Session, list[Reject]]: + """A session that keeps what `record_reject` merges, and the rows it kept.""" + fake = _FakeSession() + return cast("Session", fake), fake.rows + + +def _candidate(**overrides: object) -> LedgerItem: + a: LedgerItem = { + "title": "A repo", + "url": "https://github.com/someone/thing", + "content": "body", + "published_date": "2026-09-12T10:00:00+00:00", + "source": "Hacker News", + "publisher_id": 7, + "traction": "3 points, 0 comments on Hacker News", + } + return cast("LedgerItem", {**a, **overrides}) + + +def test_keeps_what_the_step_saw() -> None: + session, rows = _session() + record_reject(session, _candidate(), RejectStage.thin_repo, detail={"stars": 4}) + [row] = rows + assert row.stage == RejectStage.thin_repo + assert (row.title, row.source, row.publisher_id) == ("A repo", "Hacker News", 7) + assert row.traction == "3 points, 0 comments on Hacker News" + assert row.published_at is not None + assert row.published_at.year == 2026 + assert row.detail == {"stars": 4} + + +def test_caps_content_and_strips_nul_bytes() -> None: + """Postgres refuses a NUL in a text column, and the failed commit would take + the whole source down with it.""" + session, rows = _session() + content = "a\x00b" + "x" * (CONTENT_CAP * 2) + record_reject( + session, _candidate(title="t\x00", content=content), RejectStage.prefilter + ) + [row] = rows + assert row.title == "t" + assert row.content is not None + assert "\x00" not in row.content + assert len(row.content) < CONTENT_CAP + + +def test_a_pending_candidate_keeps_the_article_for_the_agent_to_read() -> None: + session, rows = _session() + record_reject( + session, _candidate(content="x" * (CANDIDATE_CAP * 2)), RejectStage.pending + ) + assert rows[0].content is not None + assert len(rows[0].content) == CANDIDATE_CAP + + +def test_empty_content_is_stored_as_null() -> None: + session, rows = _session() + record_reject(session, _candidate(content=""), RejectStage.prefilter) + assert rows[0].content is None + + +def test_keeps_the_links_extraction_found() -> None: + session, rows = _session() + links = {"repo": ["https://github.com/someone/thing"]} + record_reject(session, _candidate(links=links), RejectStage.pending) + assert rows[0].links == links + + +def test_no_links_is_stored_as_null() -> None: + session, rows = _session() + record_reject(session, _candidate(links={}), RejectStage.pending) + assert rows[0].links is None + + +@pytest.fixture +def publisher(db: Session) -> Generator[Publisher]: + p = create_random_publisher(db) + yield p + db.rollback() + db.exec(delete(Reject).where(col(Reject.publisher_id) == p.id)) + db.exec(delete(Publisher).where(col(Publisher.id) == p.id)) + db.commit() + + +def _stored( + db: Session, + publisher: Publisher, + stage: RejectStage = RejectStage.pending, + **overrides: object, +) -> Reject: + record_reject( + db, _candidate(publisher_id=publisher.id, url=URL, **overrides), stage + ) + db.commit() + row = db.get(Reject, URL) + assert row is not None + db.refresh(row) + return row + + +def test_no_detail_is_sql_null_not_json_null(db: Session, publisher: Publisher) -> None: + """`detail IS NULL` is how the ledger asks for rows without evidence.""" + _stored(db, publisher) + found = db.exec(select(Reject.url).where(col(Reject.detail).is_(None))).all() + assert URL in found + + +def test_recording_a_url_again_overwrites_its_row( + db: Session, publisher: Publisher +) -> None: + """The nightly run can meet a URL it already queued; the row is replaced, + not duplicated, so the commit does not fail on the primary key.""" + _stored(db, publisher, title="first") + row = _stored(db, publisher, title="second", stage=RejectStage.prefilter) + assert (row.title, row.stage) == ("second", RejectStage.prefilter) + + +def test_nul_bytes_and_a_bad_date_still_commit( + db: Session, publisher: Publisher +) -> None: + row = _stored( + db, publisher, title="a\x00b", content="c\x00d", published_date="not a date" + ) + assert (row.title, row.content, row.published_at) == ("ab", "cd", None) diff --git a/backend/app/core/db/user.py b/backend/app/core/db/user.py new file mode 100644 index 00000000..7c2e5948 --- /dev/null +++ b/backend/app/core/db/user.py @@ -0,0 +1,37 @@ +"""The reader account.""" + +from __future__ import annotations + +import uuid +from datetime import datetime + +from pydantic import EmailStr +from sqlalchemy import DateTime +from sqlmodel import Field, SQLModel + +from app.core.dates import get_datetime_utc + + +# Shared properties +class UserBase(SQLModel): + email: EmailStr = Field(unique=True, index=True, max_length=255) + is_active: bool = True + is_superuser: bool = False + full_name: str | None = Field(default=None, max_length=255) + + +# Database model, database table inferred from class name +class User(UserBase, table=True): + id: uuid.UUID = Field(default_factory=uuid.uuid4, primary_key=True) + # The secret in the sign-in link, replaced with every email sent. + login_token: str | None = Field( + default=None, unique=True, index=True, max_length=64 + ) + login_token_expires_at: datetime | None = Field( + default=None, + sa_type=DateTime(timezone=True), # type: ignore + ) + created_at: datetime | None = Field( + default_factory=get_datetime_utc, + sa_type=DateTime(timezone=True), # type: ignore + ) diff --git a/backend/app/core/email.py b/backend/app/core/email.py index 884e8934..61d6c4f5 100644 --- a/backend/app/core/email.py +++ b/backend/app/core/email.py @@ -1,11 +1,13 @@ -"""Email: rendering the templates and handing them to Resend or SMTP.""" +"""Email: rendering a template and handing the mail to Resend or SMTP. + +The templates sit beside the feature that sends them. +""" from __future__ import annotations import html import logging import os -from dataclasses import dataclass from pathlib import Path import emails @@ -15,22 +17,11 @@ from app.core.logging import log from app.core.settings import settings -logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) -TEMPLATES = Path(__file__).parent / "email-templates" - - -@dataclass -class EmailData: - html_content: str - subject: str - -def render_email_template(*, template_name: str, context: dict[str, str | int]) -> str: - template_str = (TEMPLATES / template_name).read_text() - html_content = Template(template_str).render(context) - return html_content +def render_email_template(*, template: Path, context: dict[str, str | int]) -> str: + return Template(template.read_text()).render(context) def send_email( @@ -73,7 +64,8 @@ def send_email( def send_mail(subject_line: str, body: str) -> None: """Email through Resend, or log the mail when the environment has no key. - Best-effort: a failed send is logged, never raised.""" + A failed send raises: the daily report's exit code is the only sign that + no mail went out.""" api_key = os.environ.get("RESEND_API_KEY") from_email = os.environ.get("EMAILS_FROM_EMAIL") to_email = os.environ.get("PIPELINE_ALERT_EMAIL") or from_email @@ -81,36 +73,13 @@ def send_mail(subject_line: str, body: str) -> None: if not (api_key and from_email and to_email): log(f" [mail suppressed — email env not set] {subject_line}\n{body}") return - try: - resend.api_key = api_key - resend.Emails.send( - { - "from": f"{project} pipeline <{from_email}>", - "to": to_email, - "subject": f"[{project}] {subject_line}", - "html": f"
{html.escape(body)}
", - } - ) - log(f" Mail sent to {to_email}: {subject_line}") - except Exception as e: - log(f" Mail send failed: {e}") - - -def sign_in_link(login_token: str) -> str: - return f"{settings.FRONTEND_HOST}/auth?token={login_token}" - - -def generate_welcome_email(login_token: str) -> EmailData: - html_content = render_email_template( - template_name="welcome.html", - context={"link": sign_in_link(login_token)}, - ) - return EmailData(html_content=html_content, subject="Welcome to Agentique") - - -def generate_sign_in_email(login_token: str) -> EmailData: - html_content = render_email_template( - template_name="sign_in.html", - context={"link": sign_in_link(login_token)}, + resend.api_key = api_key + resend.Emails.send( + { + "from": f"{project} pipeline <{from_email}>", + "to": to_email, + "subject": f"[{project}] {subject_line}", + "html": f"
{html.escape(body)}
", + } ) - return EmailData(html_content=html_content, subject="Sign in to Agentique") + log(f" Mail sent to {to_email}: {subject_line}") diff --git a/backend/app/core/embedding.py b/backend/app/core/embedding.py index e66e69a6..dcaefdc0 100644 --- a/backend/app/core/embedding.py +++ b/backend/app/core/embedding.py @@ -17,6 +17,13 @@ # How much of an article goes into its embedding. The title carries most of # the signal; the opening sentence or two sharpens it. SNIPPET_CAP = 200 +# Two texts this close are the same story. Same-story pairs sit at 0.29-0.35 +# with the potion-base-8M embeddings; tight on purpose, because coverage counts +# what falls under it and a loose cut inflates every story. +DEDUP_DIST_THRESHOLD = 0.30 +# Related out to here: close enough to be about the same subject, not close +# enough to count as the same story. +RELATED_DIST = 0.45 _model: StaticModel | None = None diff --git a/backend/app/core/jev.py b/backend/app/core/jev.py index 0c4f3b41..1a1cea5d 100644 --- a/backend/app/core/jev.py +++ b/backend/app/core/jev.py @@ -107,6 +107,14 @@ def _answers(body: dict[str, Json], questions: dict[str, Question]) -> dict[str, missing = [qid for qid, a in answers.items() if a is None] if missing: raise JevError(f"unanswered: {', '.join(missing)}") + # "0.93" as a string would read as 0 the same way. + odd = [ + qid + for qid, q in questions.items() + if q["type"] == "noul" and not isinstance(answers[qid], int | float) + ] + if odd: + raise JevError(f"non-numeric: {', '.join(odd)}") return answers diff --git a/backend/app/core/settings.py b/backend/app/core/settings.py index ace8dbb4..f7cd4424 100644 --- a/backend/app/core/settings.py +++ b/backend/app/core/settings.py @@ -65,14 +65,14 @@ def all_cors_origins(self) -> list[str]: def SQLALCHEMY_DATABASE_URI(self) -> PostgresDsn: return PostgresDsn.build( scheme="postgresql+psycopg", - username=self.POSTGRES_USER, - password=self.POSTGRES_PASSWORD, + username=quote(self.POSTGRES_USER, safe=""), + password=quote(self.POSTGRES_PASSWORD, safe=""), host=self.POSTGRES_SERVER, port=self.POSTGRES_PORT, path=self.POSTGRES_DB, ) - # The MCP server (app/mcp/) is gated by this one shared bearer token. Unset + # The MCP server (app/entry/mcp/) is gated by this one shared bearer token. Unset # means every request is rejected, so a deploy that forgets it is closed, # not open. MCP_TOKEN: str | None = None @@ -99,8 +99,8 @@ def MCP_DATABASE_URI(self) -> str: """ dsn = PostgresDsn.build( scheme="postgresql", - username=self.MCP_DB_USER, - password=self.MCP_DB_PASSWORD or None, + username=quote(self.MCP_DB_USER, safe=""), + password=quote(self.MCP_DB_PASSWORD, safe="") or None, host=self.POSTGRES_SERVER, port=self.POSTGRES_PORT, path=self.POSTGRES_DB, diff --git a/backend/tests/core/test_email.py b/backend/app/core/test_email.py similarity index 85% rename from backend/tests/core/test_email.py rename to backend/app/core/test_email.py index aa15171c..5a9ce8a3 100644 --- a/backend/tests/core/test_email.py +++ b/backend/app/core/test_email.py @@ -1,4 +1,4 @@ -"""The daily mail's send: suppressed without the env, never raising.""" +"""The daily mail's send: suppressed without the env, raising when it fails.""" from __future__ import annotations @@ -47,9 +47,10 @@ def test_the_mail_goes_to_the_alert_address_with_the_body_escaped( @pytest.mark.usefixtures("mail_env") -def test_a_failed_send_does_not_raise(monkeypatch: pytest.MonkeyPatch) -> None: +def test_a_failed_send_raises(monkeypatch: pytest.MonkeyPatch) -> None: def fail(_payload: dict[str, str]) -> None: raise RuntimeError("resend is down") monkeypatch.setattr(resend.Emails, "send", fail) - send_mail("Pipeline needs a look", "body") + with pytest.raises(RuntimeError, match="resend is down"): + send_mail("Pipeline needs a look", "body") diff --git a/backend/app/core/test_embedding.py b/backend/app/core/test_embedding.py new file mode 100644 index 00000000..8858441e --- /dev/null +++ b/backend/app/core/test_embedding.py @@ -0,0 +1,61 @@ +"""The distance maths `similar`, `stories`, the newsletter and the stories rail +all group by. Pure: synthetic vectors, no model load.""" + +from __future__ import annotations + +import numpy as np +import numpy.typing as npt + +from app.core import embedding + + +def _vecs(*rows: list[float]) -> npt.NDArray[np.float32]: + return np.array(rows, dtype=np.float32) + + +def test_cosine_distance_is_zero_for_the_same_direction() -> None: + d = embedding.cosine_distances(_vecs([1, 0]), _vecs([2, 0], [0, 1], [-1, 0])) + assert np.allclose(d, [[0, 1, 2]]) + + +def test_cosine_distance_survives_a_zero_vector() -> None: + d = embedding.cosine_distances(_vecs([0, 0]), _vecs([1, 0])) + assert np.isfinite(d).all() + + +def test_nearest_is_closest_first_within_the_cutoff() -> None: + distances = np.array([0.40, 0.10, 0.50, 0.25], dtype=np.float32) + assert embedding.nearest(distances, cutoff=0.45, limit=8) == [1, 3, 0] + assert embedding.nearest(distances, cutoff=0.45, limit=2) == [1, 3] + + +def test_story_groups_join_pending_rows_with_each_other_and_the_feed() -> None: + # 0, 1 pending and the same story; 2 a feed article on it; 3 a pending + # candidate on its own; 4 a feed article on nothing queued. + vecs = _vecs([1, 0, 0], [0.98, 0.1, 0], [0.97, 0.12, 0], [0, 1, 0], [0, 0, 1]) + pending = [True, True, False, True, False] + + groups = embedding.story_groups( + embedding.cosine_distances(vecs, vecs), pending, 0.30 + ) + + assert [sorted(g) for g in groups] == [[0, 1, 2]] + + +def test_feed_articles_do_not_chain_a_group_through_each_other() -> None: + # Pending 0 is close to article 1; article 2 is close to article 1 but not + # to 0. Without a pending end on every edge, 2 would ride in through 1. + distances = np.array( + [ + [0.0, 0.2, 0.6], + [0.2, 0.0, 0.2], + [0.6, 0.2, 0.0], + ] + ) + groups = embedding.story_groups(distances, [True, False, False], 0.30) + assert [sorted(g) for g in groups] == [[0, 1]] + + +def test_a_group_of_feed_articles_alone_is_not_a_story() -> None: + distances = np.array([[0.0, 0.1], [0.1, 0.0]], dtype=np.float32) + assert embedding.story_groups(distances, [False, False], 0.30) == [] diff --git a/backend/tests/pipeline/test_extract_content.py b/backend/app/core/test_extract.py similarity index 89% rename from backend/tests/pipeline/test_extract_content.py rename to backend/app/core/test_extract.py index a3030b75..27048fb3 100644 --- a/backend/tests/pipeline/test_extract_content.py +++ b/backend/app/core/test_extract.py @@ -17,7 +17,7 @@ ) -def test_a_real_article_body_is_not_a_blocker(): +def test_a_real_article_body_is_not_a_blocker() -> None: assert _is_blocker(_REAL_ARTICLE) is False @@ -28,7 +28,7 @@ def test_a_real_article_body_is_not_a_blocker(): "", ], ) -def test_anything_under_50_chars_is_a_blocker(given): +def test_anything_under_50_chars_is_a_blocker(given: str) -> None: """A teaser must not masquerade as an article regardless of its wording.""" assert _is_blocker(given) is True @@ -45,17 +45,17 @@ def test_anything_under_50_chars_is_a_blocker(given): "Something went wrong loading this page, but don't fret, try again soon.", ], ) -def test_known_blocker_wording_is_flagged(given): +def test_known_blocker_wording_is_flagged(given: str) -> None: assert _is_blocker(given) is True -def test_blocker_wording_is_flagged_regardless_of_casing(): +def test_blocker_wording_is_flagged_regardless_of_casing() -> None: assert ( _is_blocker("PLEASE SIGN IN TO CONTINUE reading this long article now.") is True ) -def test_a_long_article_about_auth_is_not_flagged(): +def test_a_long_article_about_auth_is_not_flagged() -> None: """The patterns target a page THAT IS a wall, not an article about the general topic of authentication — wording matters, not the subject.""" text = ( @@ -68,7 +68,7 @@ def test_a_long_article_about_auth_is_not_flagged(): assert _is_blocker(text) is False -def test_wording_close_to_but_not_matching_a_pattern_is_not_flagged(): +def test_wording_close_to_but_not_matching_a_pattern_is_not_flagged() -> None: """Documents that the gate is substring-driven, not semantic: an article that happens to use the exact phrase "sign in to continue" mid-sentence would be misflagged even outside a real wall. Regression coverage for that diff --git a/backend/tests/core/test_github.py b/backend/app/core/test_github.py similarity index 100% rename from backend/tests/core/test_github.py rename to backend/app/core/test_github.py diff --git a/backend/tests/pipeline/test_jev.py b/backend/app/core/test_jev.py similarity index 58% rename from backend/tests/pipeline/test_jev.py rename to backend/app/core/test_jev.py index d355f898..edd0f3ca 100644 --- a/backend/tests/pipeline/test_jev.py +++ b/backend/app/core/test_jev.py @@ -1,10 +1,12 @@ """The Jev transport's two guarantees: a 429 waits as long as it is told to -and tries again, and an answer set with a hole in it is an error, never a -quiet zero that reads as "not a sponsor" or "off topic". +and tries again, and an answer set with a hole or a non-numeric probability in +it is an error, never a quiet zero that reads as "not a sponsor" or "off topic". """ from __future__ import annotations +from collections.abc import Mapping + import httpx import pytest @@ -16,7 +18,11 @@ } -def _resp(status: int, body: dict | None = None, headers: dict | None = None): +def _resp( + status: int, + body: Mapping[str, object] | None = None, + headers: dict[str, str] | None = None, +) -> httpx.Response: return httpx.Response( status, json=body or {}, @@ -25,11 +31,13 @@ def _resp(status: int, body: dict | None = None, headers: dict | None = None): ) -def _stub(monkeypatch, responses): - sent: list[dict] = [] +def _stub( + monkeypatch: pytest.MonkeyPatch, responses: list[httpx.Response] +) -> tuple[list[dict[str, object]], list[float]]: + sent: list[dict[str, object]] = [] slept: list[float] = [] - def post(payload): + def post(payload: dict[str, object]) -> httpx.Response: sent.append(payload) return responses.pop(0) @@ -38,7 +46,9 @@ def post(payload): return sent, slept -def test_a_429_waits_for_retry_after_then_succeeds(monkeypatch): +def test_a_429_waits_for_retry_after_then_succeeds( + monkeypatch: pytest.MonkeyPatch, +) -> None: ok = { "answers": {"is_sponsor": {"noul": 0.9}, "kind": {"choice": "product"}}, "usage": {"cost": 0.00002}, @@ -55,7 +65,9 @@ def test_a_429_waits_for_retry_after_then_succeeds(monkeypatch): assert reply.cost == pytest.approx(0.00002) -def test_the_request_carries_model_state_and_questions(monkeypatch): +def test_the_request_carries_model_state_and_questions( + monkeypatch: pytest.MonkeyPatch, +) -> None: ok = {"answers": {"is_sponsor": 0.1, "kind": "article"}} sent, _ = _stub(monkeypatch, [_resp(200, ok)]) @@ -68,7 +80,9 @@ def test_the_request_carries_model_state_and_questions(monkeypatch): } -def test_a_429_without_retry_after_backs_off_on_its_own(monkeypatch): +def test_a_429_without_retry_after_backs_off_on_its_own( + monkeypatch: pytest.MonkeyPatch, +) -> None: ok = {"answers": {"is_sponsor": 0.1, "kind": "article"}} _, slept = _stub(monkeypatch, [_resp(429), _resp(429), _resp(200, ok)]) @@ -77,37 +91,51 @@ def test_a_429_without_retry_after_backs_off_on_its_own(monkeypatch): assert slept == [jev.BACKOFF_START_SECS, jev.BACKOFF_START_SECS * 2] -def test_429s_past_the_retry_budget_raise(monkeypatch): +def test_429s_past_the_retry_budget_raise(monkeypatch: pytest.MonkeyPatch) -> None: _stub(monkeypatch, [_resp(429) for _ in range(3)]) with pytest.raises(httpx.HTTPStatusError): jev.ask({}, QUESTIONS, retries=2) -def test_a_missing_answer_raises_instead_of_reading_as_zero(monkeypatch): +def test_a_missing_answer_raises_instead_of_reading_as_zero( + monkeypatch: pytest.MonkeyPatch, +) -> None: _stub(monkeypatch, [_resp(200, {"answers": {"kind": {"choice": "article"}}})]) with pytest.raises(jev.JevError, match="is_sponsor"): jev.ask({}, QUESTIONS) -def test_a_response_with_no_answers_raises(monkeypatch): +def test_a_non_numeric_probability_raises_instead_of_reading_as_zero( + monkeypatch: pytest.MonkeyPatch, +) -> None: + body = {"answers": {"is_sponsor": {"noul": "0.93"}, "kind": "article"}} + _stub(monkeypatch, [_resp(200, body)]) + + with pytest.raises(jev.JevError, match="is_sponsor"): + jev.ask({}, QUESTIONS) + + +def test_a_response_with_no_answers_raises(monkeypatch: pytest.MonkeyPatch) -> None: _stub(monkeypatch, [_resp(200, {"error": "nope"})]) with pytest.raises(jev.JevError): jev.ask({}, QUESTIONS) -def test_a_zero_probability_is_an_answer_not_a_hole(monkeypatch): +def test_a_zero_probability_is_an_answer_not_a_hole( + monkeypatch: pytest.MonkeyPatch, +) -> None: _stub(monkeypatch, [_resp(200, {"answers": {"is_sponsor": 0.0, "kind": "x"}})]) assert jev.ask({}, QUESTIONS).answers["is_sponsor"] == 0.0 -def test_criteria_are_optional_on_a_noul(): +def test_criteria_are_optional_on_a_noul() -> None: assert "criteria" not in jev.noul("sponsor?") -def test_score_needs_two_to_ten_levels(): +def test_score_needs_two_to_ten_levels() -> None: with pytest.raises(ValueError): jev.score("how good?", ["only one"]) diff --git a/backend/tests/pipeline/test_sanitize.py b/backend/app/core/test_llm_text.py similarity index 71% rename from backend/tests/pipeline/test_sanitize.py rename to backend/app/core/test_llm_text.py index 03f98ef3..0433a452 100644 --- a/backend/tests/pipeline/test_sanitize.py +++ b/backend/app/core/test_llm_text.py @@ -1,16 +1,31 @@ -"""Cleanup of model-authored text before it is stored. +"""Normalising an enum-typed BAML field to a plain string, and cleaning +model-authored text before it is stored. Fixtures marked "prod" are real corrupted values pulled from the production -`article` table — the failure modes the pipeline actually produces. +`article` table, the failure modes the pipeline actually produces. """ from __future__ import annotations +from enum import StrEnum + import pytest -from app.core.llm_text import sanitize_llm_text +from app.core.llm_text import enum_value, sanitize_llm_text + + +class _Colour(StrEnum): + red = "red" + + +def test_enum_value_unwraps_an_enum() -> None: + assert enum_value(_Colour.red) == "red" + -# ─── sanitize_llm_text ────────────────────────────────────────────────────── +def test_enum_value_passes_a_plain_string_through() -> None: + """BAML returns plain strings for some fields and enum members for others, + and which one has changed across regenerations.""" + assert enum_value("red") == "red" @pytest.mark.parametrize( diff --git a/backend/tests/core/test_logging.py b/backend/app/core/test_logging.py similarity index 75% rename from backend/tests/core/test_logging.py rename to backend/app/core/test_logging.py index e21fa50f..6e154e0c 100644 --- a/backend/tests/core/test_logging.py +++ b/backend/app/core/test_logging.py @@ -5,11 +5,11 @@ from app.core.logging import short_error -def test_short_error_names_the_type_and_the_message(): +def test_short_error_names_the_type_and_the_message() -> None: assert short_error(ValueError("bad input")) == "ValueError: bad input" -def test_short_error_collapses_a_multi_line_message(): +def test_short_error_collapses_a_multi_line_message() -> None: """A provider error arrives as a wall of prompt and HTML — one line of it is what lands in the run row and the alert email.""" assert short_error(RuntimeError("failed\n attempt 0\n attempt 1")) == ( @@ -17,13 +17,13 @@ def test_short_error_collapses_a_multi_line_message(): ) -def test_short_error_truncates_past_the_limit_and_says_so(): +def test_short_error_truncates_past_the_limit_and_says_so() -> None: out = short_error(RuntimeError("x" * 5000), limit=100) assert out.startswith("RuntimeError: " + "x" * 80) assert out.endswith("[truncated, 5014 chars]") assert len(out) < 200 -def test_short_error_leaves_a_message_at_the_limit_alone(): +def test_short_error_leaves_a_message_at_the_limit_alone() -> None: message = "y" * (100 - len("ValueError: ")) assert short_error(ValueError(message), limit=100) == f"ValueError: {message}" diff --git a/backend/app/core/test_settings.py b/backend/app/core/test_settings.py new file mode 100644 index 00000000..8b4afbfb --- /dev/null +++ b/backend/app/core/test_settings.py @@ -0,0 +1,32 @@ +"""The database DSNs settings build, for credentials with URL-reserved characters.""" + +from __future__ import annotations + +import psycopg.conninfo +from sqlalchemy.engine import make_url + +from app.core.settings import settings + +PASSWORD = "p@ss/w:rd#% x" + + +def test_engine_dsn_keeps_reserved_characters_in_the_password() -> None: + copy = settings.model_copy( + update={"POSTGRES_USER": "me@host", "POSTGRES_PASSWORD": PASSWORD} + ) + url = make_url(str(copy.SQLALCHEMY_DATABASE_URI)) + assert (url.username, url.password) == ("me@host", PASSWORD) + assert (url.host, url.database) == (settings.POSTGRES_SERVER, settings.POSTGRES_DB) + + +def test_mcp_dsn_keeps_reserved_characters_in_the_password() -> None: + copy = settings.model_copy(update={"MCP_DB_PASSWORD": PASSWORD}) + params = psycopg.conninfo.conninfo_to_dict(copy.MCP_DATABASE_URI) + assert (params["user"], params["password"]) == (settings.MCP_DB_USER, PASSWORD) + assert "default_transaction_read_only=on" in str(params["options"]) + + +def test_mcp_dsn_without_a_password_has_none() -> None: + copy = settings.model_copy(update={"MCP_DB_PASSWORD": ""}) + params = psycopg.conninfo.conninfo_to_dict(copy.MCP_DATABASE_URI) + assert "password" not in params diff --git a/backend/tests/pipeline/test_urls.py b/backend/app/core/test_urls.py similarity index 83% rename from backend/tests/pipeline/test_urls.py rename to backend/app/core/test_urls.py index 375fb285..567464eb 100644 --- a/backend/tests/pipeline/test_urls.py +++ b/backend/app/core/test_urls.py @@ -18,10 +18,10 @@ ("https://www2.example.com", "www2.example.com"), ], ) -def test_hostname_normalises(url, expected): +def test_hostname_normalises(url: str, expected: str) -> None: assert hostname(url) == expected @pytest.mark.parametrize("given", ["", "not a url", "mailto:x@y.com"]) -def test_hostname_returns_empty_when_there_is_no_host(given): +def test_hostname_returns_empty_when_there_is_no_host(given: str) -> None: assert hostname(given) == "" diff --git a/backend/app/analytics/__init__.py b/backend/app/entry/__init__.py similarity index 100% rename from backend/app/analytics/__init__.py rename to backend/app/entry/__init__.py diff --git a/backend/app/analytics/tests/__init__.py b/backend/app/entry/api/__init__.py similarity index 100% rename from backend/app/analytics/tests/__init__.py rename to backend/app/entry/api/__init__.py diff --git a/backend/app/entry/api/analytics.py b/backend/app/entry/api/analytics.py new file mode 100644 index 00000000..530068f0 --- /dev/null +++ b/backend/app/entry/api/analytics.py @@ -0,0 +1,48 @@ +"""First-party analytics: the public ingest endpoint and the admin-only report.""" + +from __future__ import annotations + +from fastapi import APIRouter, Depends, Query, Request + +from app.entry.api.deps import ( + CurrentUserOptional, + SessionDep, + get_current_active_superuser, +) +from app.features.analytics.model import AnalyticsEventCreate, AnalyticsReport +from app.features.analytics.service import build_report, record_event + +router = APIRouter(prefix="/analytics", tags=["analytics"]) + + +@router.post("/collect", status_code=204) +def collect_event( + session: SessionDep, + current_user: CurrentUserOptional, + payload: AnalyticsEventCreate, + request: Request, +) -> None: + """Record one analytics event. Public and fire-and-forget (returns 204). + + Attaches the user id when the request carries a valid bearer token; + otherwise the event is anonymous. + """ + record_event( + session, + payload, + user_id=current_user.id if current_user else None, + user_agent=request.headers.get("user-agent"), + ) + + +@router.get( + "/report", + response_model=AnalyticsReport, + dependencies=[Depends(get_current_active_superuser)], +) +def read_report( + session: SessionDep, + days: int | None = Query(default=None, ge=1, le=3650), +) -> AnalyticsReport: + """Superusers only. Outside readers over the last `days` days, all time when omitted.""" + return build_report(session, days) diff --git a/backend/app/catalog/routes.py b/backend/app/entry/api/articles.py similarity index 79% rename from backend/app/catalog/routes.py rename to backend/app/entry/api/articles.py index 101f5899..1d31d76f 100644 --- a/backend/app/catalog/routes.py +++ b/backend/app/entry/api/articles.py @@ -1,4 +1,4 @@ -"""Article listing, semantic search, facet and story endpoints for the public feed.""" +"""Article listing, semantic search and facet endpoints for the public feed.""" from __future__ import annotations @@ -6,23 +6,29 @@ from fastapi import APIRouter, HTTPException, Query -from app.catalog import facets, semantic_search, stories -from app.catalog.articles import list_articles, stats -from app.catalog.models import ( +from app.entry.api.deps import CurrentUserOptional, SessionDep +from app.features.catalog.model import ( ArticleFacets, ArticlesPublic, Origin, OriginCounts, PublisherFacet, - StoryPublic, TagFacet, Voice, VoiceKind, ) -from app.deps import CurrentUserOptional, SessionDep +from app.features.catalog.service import ( + all_facets, + list_articles, + origin_counts, + publisher_facets, + search, + stats, + tag_facets, + voices, +) router = APIRouter(prefix="/articles", tags=["articles"]) -stories_router = APIRouter(prefix="/stories", tags=["stories"]) # Reads are public and unbounded. A caller may still send a token — it only # decides whether `liked_by_me` is filled in. @@ -74,7 +80,7 @@ def search_articles( q: str, limit: int = Query(default=20, ge=1, le=50), ) -> ArticlesPublic: - return semantic_search.search( + return search( session, q, limit, viewer_id=current_user.id if current_user else None ) @@ -83,7 +89,7 @@ def search_articles( def article_facets( session: SessionDep, limit: int = Query(default=8, ge=1, le=20) ) -> ArticleFacets: - return facets.all_facets(session, limit) + return all_facets(session, limit) @router.get("/origins", response_model=OriginCounts) @@ -92,7 +98,7 @@ def article_origins(session: SessionDep, since: str) -> OriginCounts: since_dt = datetime.fromisoformat(since) except ValueError: raise HTTPException(status_code=422, detail="Invalid 'since' datetime") - return facets.origin_counts(session, since_dt) + return origin_counts(session, since_dt) @router.get("/voices", response_model=list[Voice]) @@ -106,7 +112,7 @@ def article_voices( since_dt = datetime.fromisoformat(since) except ValueError: raise HTTPException(status_code=422, detail="Invalid 'since' datetime") - return facets.voices(session, kind, since_dt, limit) + return voices(session, kind, since_dt, limit) @router.get("/publishers", response_model=list[PublisherFacet]) @@ -115,7 +121,7 @@ def search_publishers( q: str | None = None, limit: int = Query(default=20, ge=1, le=50), ) -> list[PublisherFacet]: - return facets.publisher_facets(session, q, limit) + return publisher_facets(session, q, limit) @router.get("/tags", response_model=list[TagFacet]) @@ -124,16 +130,9 @@ def search_tags( q: str | None = None, limit: int = Query(default=20, ge=1, le=50), ) -> list[TagFacet]: - return facets.tag_facets(session, q, limit) + return tag_facets(session, q, limit) @router.get("/stats") def article_stats(session: SessionDep) -> dict[str, int | str | None]: return stats(session) - - -@stories_router.get("/", response_model=list[StoryPublic]) -def read_stories( - session: SessionDep, limit: int = Query(default=5, ge=1, le=20) -) -> list[StoryPublic]: - return stories.list_stories(session, limit) diff --git a/backend/app/deps.py b/backend/app/entry/api/deps.py similarity index 89% rename from backend/app/deps.py rename to backend/app/entry/api/deps.py index 1ad9d69e..bb207104 100644 --- a/backend/app/deps.py +++ b/backend/app/entry/api/deps.py @@ -2,7 +2,6 @@ from __future__ import annotations -from collections.abc import Generator from typing import Annotated import jwt @@ -12,20 +11,16 @@ from pydantic import ValidationError from sqlmodel import Session -from app.audience.models import TokenPayload, User from app.core import security -from app.core.db import engine +from app.core.db.engine import get_session +from app.core.db.user import User from app.core.settings import settings +from app.features.audience.model import TokenPayload reusable_bearer = HTTPBearer() -def get_db() -> Generator[Session]: - with Session(engine) as session: - yield session - - -SessionDep = Annotated[Session, Depends(get_db)] +SessionDep = Annotated[Session, Depends(get_session)] TokenDep = Annotated[HTTPAuthorizationCredentials, Depends(reusable_bearer)] diff --git a/backend/app/entry/api/likes.py b/backend/app/entry/api/likes.py new file mode 100644 index 00000000..00d2b7a7 --- /dev/null +++ b/backend/app/entry/api/likes.py @@ -0,0 +1,41 @@ +"""Liking an article, and the signed-in reader's liked list.""" + +from __future__ import annotations + +from fastapi import APIRouter, HTTPException + +from app.entry.api.deps import CurrentUser, SessionDep +from app.features.audience import service +from app.features.catalog.model import ArticlesPublic +from app.features.catalog.service import list_liked_articles + +router = APIRouter(tags=["likes"]) + + +@router.put("/articles/{article_id}/like") +def like_article( + session: SessionDep, current_user: CurrentUser, article_id: int +) -> dict[str, bool]: + liked = service.like_article( + session=session, user_id=current_user.id, article_id=article_id + ) + if not liked: + raise HTTPException(status_code=404, detail="Article not found") + return {"ok": True} + + +@router.delete("/articles/{article_id}/like") +def unlike_article( + session: SessionDep, current_user: CurrentUser, article_id: int +) -> dict[str, bool]: + service.unlike_article( + session=session, user_id=current_user.id, article_id=article_id + ) + return {"ok": True} + + +@router.get("/me/liked-articles", response_model=ArticlesPublic) +def read_liked_articles( + session: SessionDep, current_user: CurrentUser +) -> ArticlesPublic: + return list_liked_articles(session, current_user.id) diff --git a/backend/app/entry/api/login.py b/backend/app/entry/api/login.py new file mode 100644 index 00000000..952d9312 --- /dev/null +++ b/backend/app/entry/api/login.py @@ -0,0 +1,91 @@ +"""Signing up and in by email link, and swapping the link for an access token.""" + +from __future__ import annotations + +from datetime import timedelta + +from fastapi import APIRouter, BackgroundTasks, HTTPException + +from app.core import security +from app.core.db.user import User +from app.core.settings import settings +from app.entry.api.deps import CurrentUser, SessionDep +from app.features.audience import service +from app.features.audience.model import ( + MagicLogin, + Message, + SignInRequest, + Token, + UserCreate, + UserPublic, +) +from app.features.newsletter.service import subscribe + +router = APIRouter(tags=["login"]) + +LINK_SENT = "Check your inbox for your sign-in link" + + +@router.post("/login/link") +def request_sign_in_link( + session: SessionDep, body: SignInRequest, background_tasks: BackgroundTasks +) -> Message: + """ + Email a sign-in link. An unknown email first gets an account and a + newsletter subscription, and the link comes in a welcome email. + """ + user = service.get_user_by_email(session=session, email=body.email) + is_new = user is None + if user is None: + user = service.create_user( + session=session, user_create=UserCreate(email=body.email) + ) + subscribe(session=session, email=user.email, utm_source=body.utm_source) + elif not user.is_active: + return Message(message=LINK_SENT) + token = service.issue_login_token(session=session, db_user=user) + email = ( + service.generate_welcome_email(token) + if is_new + else service.generate_sign_in_email(token) + ) + background_tasks.add_task(service.send_link_email, user.email, email, token) + return Message(message=LINK_SENT) + + +@router.post("/login/token") +def login_with_link(session: SessionDep, body: MagicLogin) -> Token: + """ + Swap the token from a sign-in link for an access token + """ + user = service.get_user_by_login_token(session=session, token=body.token) + if not user: + raise HTTPException(status_code=400, detail="Invalid sign-in link") + elif not user.is_active: + raise HTTPException(status_code=400, detail="Inactive user") + return _access_token(user) + + +@router.post("/login/refresh") +def refresh_token(current_user: CurrentUser) -> Token: + """ + Renew the access token + """ + return _access_token(current_user) + + +def _access_token(user: User) -> Token: + return Token( + access_token=security.create_access_token( + user.id, + expires_delta=timedelta(minutes=settings.ACCESS_TOKEN_EXPIRE_MINUTES), + ) + ) + + +@router.post("/login/test-token", response_model=UserPublic) +def test_token(current_user: CurrentUser) -> User: + """ + Test access token + """ + return current_user diff --git a/backend/app/entry/api/stories.py b/backend/app/entry/api/stories.py new file mode 100644 index 00000000..32aa67a5 --- /dev/null +++ b/backend/app/entry/api/stories.py @@ -0,0 +1,18 @@ +"""The right rail's stories, for the public feed.""" + +from __future__ import annotations + +from fastapi import APIRouter, Query + +from app.entry.api.deps import SessionDep +from app.features.stories import service +from app.features.stories.model import StoryPublic + +router = APIRouter(prefix="/stories", tags=["stories"]) + + +@router.get("/", response_model=list[StoryPublic]) +def read_stories( + session: SessionDep, limit: int = Query(default=5, ge=1, le=20) +) -> list[StoryPublic]: + return service.list_stories(session, limit) diff --git a/backend/app/analytics/tests/test_report.py b/backend/app/entry/api/test_analytics.py similarity index 57% rename from backend/app/analytics/tests/test_report.py rename to backend/app/entry/api/test_analytics.py index d7ec19e9..32ef8e64 100644 --- a/backend/app/analytics/tests/test_report.py +++ b/backend/app/entry/api/test_analytics.py @@ -1,4 +1,4 @@ -"""Tests for the admin analytics report.""" +"""The analytics endpoints: the ingest beacon and the admin report.""" from __future__ import annotations @@ -7,13 +7,12 @@ import pytest from fastapi.testclient import TestClient -from sqlmodel import Session, delete, select +from sqlmodel import Session, col, delete, select -from app.analytics.models import AnalyticsEvent -from app.audience.models import User -from app.audience.tests.factories import create_random_user -from app.catalog.tests.factories import create_random_article from app.core.dates import get_datetime_utc +from app.core.db.event import AnalyticsEvent +from app.core.db.factories import create_random_article, create_random_user +from app.core.db.user import User from app.core.settings import settings REPORT = f"{settings.API_V1_STR}/analytics/report" @@ -21,6 +20,117 @@ PHONE = "Mozilla/5.0 (iPhone; CPU iPhone OS 18_0 like Mac OS X) Mobile/15E148" +def _latest_event(db: Session, visitor_id: str) -> AnalyticsEvent | None: + return db.exec( + select(AnalyticsEvent) + .where(AnalyticsEvent.visitor_id == visitor_id) + .order_by(col(AnalyticsEvent.id).desc()) + ).first() + + +def test_collect_anonymous_pageview(client: TestClient, db: Session) -> None: + r = client.post( + f"{settings.API_V1_STR}/analytics/collect", + json={"path": "/articles", "visitor_id": "anon-1"}, + ) + assert r.status_code == 204 + + event = _latest_event(db, "anon-1") + assert event is not None + assert event.event == "pageview" + assert event.path == "/articles" + assert event.user_id is None + # user agent captured from the request headers + assert event.user_agent is not None + + +def test_collect_ignores_referer_header(client: TestClient, db: Session) -> None: + r = client.post( + f"{settings.API_V1_STR}/analytics/collect", + headers={"Referer": "https://agentique.ch/feed"}, + json={"path": "/feed", "visitor_id": "anon-5"}, + ) + assert r.status_code == 204 + event = _latest_event(db, "anon-5") + assert event is not None + assert event.referrer is None + + +def test_collect_drops_query_strings(client: TestClient, db: Session) -> None: + r = client.post( + f"{settings.API_V1_STR}/analytics/collect", + json={ + "path": "/auth?token=secret", + "referrer": "https://agentique.ch/reset-password?token=secret#top", + "visitor_id": "anon-6", + }, + ) + assert r.status_code == 204 + event = _latest_event(db, "anon-6") + assert event is not None + assert event.path == "/auth" + assert event.referrer == "https://agentique.ch/reset-password" + + +def test_collect_defaults_event_to_pageview(client: TestClient, db: Session) -> None: + r = client.post( + f"{settings.API_V1_STR}/analytics/collect", + json={"visitor_id": "anon-2"}, + ) + assert r.status_code == 204 + event = _latest_event(db, "anon-2") + assert event is not None + assert event.event == "pageview" + assert event.path is None + + +def test_collect_custom_event_with_props(client: TestClient, db: Session) -> None: + r = client.post( + f"{settings.API_V1_STR}/analytics/collect", + json={ + "event": "article_liked", + "path": "/articles/5", + "visitor_id": "anon-3", + "props": {"article_id": 5}, + }, + ) + assert r.status_code == 204 + event = _latest_event(db, "anon-3") + assert event is not None + assert event.event == "article_liked" + assert event.props == {"article_id": 5} + + +def test_collect_attaches_user_when_authenticated( + client: TestClient, db: Session, normal_user_token_headers: dict[str, str] +) -> None: + r = client.post( + f"{settings.API_V1_STR}/analytics/collect", + headers=normal_user_token_headers, + json={"path": "/profile", "visitor_id": "known-1"}, + ) + assert r.status_code == 204 + + user = db.exec(select(User).where(User.email == settings.EMAIL_TEST_USER)).first() + assert user is not None + event = _latest_event(db, "known-1") + assert event is not None + assert event.user_id == user.id + + +def test_collect_truncates_oversized_strings(client: TestClient, db: Session) -> None: + long_path = "/" + "x" * 5000 + r = client.post( + f"{settings.API_V1_STR}/analytics/collect", + json={"path": long_path, "visitor_id": "anon-4"}, + ) + assert r.status_code == 204 + event = _latest_event(db, "anon-4") + assert event is not None + assert event.path is not None + assert len(event.path) == 2048 + + @pytest.fixture(autouse=True) def empty_events(db: Session) -> Generator[None]: db.exec(delete(AnalyticsEvent)) @@ -30,7 +140,7 @@ def empty_events(db: Session) -> Generator[None]: def _event(db: Session, **fields: object) -> None: fields.setdefault("user_agent", BROWSER) - db.add(AnalyticsEvent(**fields)) # type: ignore[arg-type] # ty: ignore[invalid-argument-type] + db.add(AnalyticsEvent(**fields)) # ty: ignore[invalid-argument-type] db.commit() @@ -145,9 +255,8 @@ def test_report_counts_registered_readers( ) -> None: def totals(days: int | None) -> dict[str, int]: params = {"days": days} if days else {} - return client.get( - REPORT, headers=superuser_token_headers, params=params - ).json()["totals"] + body = client.get(REPORT, headers=superuser_token_headers, params=params) + return {k: int(v) for k, v in body.json()["totals"].items()} before, before_all = totals(7), totals(None) create_random_user(db) diff --git a/backend/app/catalog/tests/test_routes.py b/backend/app/entry/api/test_articles.py similarity index 93% rename from backend/app/catalog/tests/test_routes.py rename to backend/app/entry/api/test_articles.py index 917fee86..bf3e11e3 100644 --- a/backend/app/catalog/tests/test_routes.py +++ b/backend/app/entry/api/test_articles.py @@ -3,10 +3,8 @@ from __future__ import annotations import uuid -from collections.abc import Generator from dataclasses import dataclass from datetime import UTC, datetime, timedelta -from typing import Any import pytest from fastapi.testclient import TestClient @@ -14,20 +12,20 @@ from sqlalchemy import cast from sqlmodel import Session, col, select -from app.audience import service -from app.audience.tests.factories import authentication_token_from_email -from app.catalog import semantic_search -from app.catalog.models import Article, ArticleKind, Origin, PublisherKind -from app.catalog.tests.factories import ( +from app.core.db.article import Article, ArticleKind +from app.core.db.factories import ( + authentication_token_from_email, create_random_article, create_random_publisher, create_random_tag, + random_email, tag_article, ) +from app.core.db.publisher import PublisherKind from app.core.security import create_access_token from app.core.settings import settings -from app.main import app -from tests.random_data import random_email +from app.features.catalog import semantic_search +from app.features.catalog.model import Origin, Voice ARTICLES_URL = f"{settings.API_V1_STR}/articles" @@ -35,28 +33,6 @@ RECENT_SINCE = (datetime.now(UTC) - timedelta(days=1)).isoformat() -@pytest.fixture(scope="module") -def reader_token_headers(db: Session) -> dict[str, str]: - """Headers for an ordinary logged-in reader.""" - email = random_email() - headers = authentication_token_from_email(email=email, db=db) - assert service.get_user_by_email(session=db, email=email) is not None - return headers - - -@pytest.fixture(scope="module") -def auth_client( - reader_token_headers: dict[str, str], -) -> Generator[TestClient]: - """A TestClient that sends a logged-in reader's bearer token on every request. - - Reads are public; the token only decides whether `liked_by_me` is filled in. - """ - with TestClient(app) as c: - c.headers.update(reader_token_headers) - yield c - - # --- public reads ------------------------------------------------------------ READ_ENDPOINTS = [ @@ -628,12 +604,13 @@ def test_origin_counts_malformed_since_returns_422(client: TestClient) -> None: def _voices( client: TestClient, kind: str, slugs: set[str], since: str = RECENT_SINCE -) -> list[dict[str, Any]]: +) -> list[Voice]: """The voices of `kind`, narrowed to the publishers a test made: other tests' articles share the window.""" r = client.get(VOICES_URL, params={"kind": kind, "since": since, "limit": 20}) assert r.status_code == 200 - return [v for v in r.json() if v["slug"] in slugs] + voices = [Voice.model_validate(v) for v in r.json()] + return [v for v in voices if v.slug in slugs] def _ago(hours: float) -> datetime: @@ -647,8 +624,8 @@ def test_voices_filter_by_kind(client: TestClient, db: Session) -> None: create_random_article(db, publisher_id=p.id, published_at=_ago(1)) slugs = {lab.slug, company.slug} - assert [v["slug"] for v in _voices(client, "lab", slugs)] == [lab.slug] - assert [v["slug"] for v in _voices(client, "company", slugs)] == [company.slug] + assert [v.slug for v in _voices(client, "lab", slugs)] == [lab.slug] + assert [v.slug for v in _voices(client, "company", slugs)] == [company.slug] assert _voices(client, "individual", slugs) == [] @@ -664,21 +641,21 @@ def test_voices_writers_count_posts_only(client: TestClient, db: Session) -> Non ) voices = _voices(client, "individual", {writer.slug, owner.slug}) - assert [(v["slug"], v["count"]) for v in voices] == [(writer.slug, 1)] + assert [(v.slug, v.count) for v in voices] == [(writer.slug, 1)] def test_voices_labs_count_every_kind(client: TestClient, db: Session) -> None: lab = create_random_publisher(db, kind=PublisherKind.lab) for kind in (ArticleKind.post, ArticleKind.model, ArticleKind.paper): create_random_article(db, publisher_id=lab.id, kind=kind, published_at=_ago(1)) - assert _voices(client, "lab", {lab.slug})[0]["count"] == 3 + assert _voices(client, "lab", {lab.slug})[0].count == 3 def test_voices_are_bounded_by_since(client: TestClient, db: Session) -> None: lab = create_random_publisher(db, kind=PublisherKind.lab) create_random_article(db, publisher_id=lab.id, published_at=_ago(1)) create_random_article(db, publisher_id=lab.id, published_at=_ago(24 * 10)) - assert _voices(client, "lab", {lab.slug})[0]["count"] == 1 + assert _voices(client, "lab", {lab.slug})[0].count == 1 def test_voices_scores_oldest_first_capped(client: TestClient, db: Session) -> None: @@ -690,8 +667,8 @@ def test_voices_scores_oldest_first_capped(client: TestClient, db: Session) -> N db, publisher_id=lab.id, score=score, published_at=_ago(20 - i) ) voice = _voices(client, "lab", {lab.slug})[0] - assert voice["count"] == 17 - assert voice["scores"] == scores[-15:] + assert voice.count == 17 + assert voice.scores == scores[-15:] def test_voices_best_is_highest_score(client: TestClient, db: Session) -> None: @@ -701,9 +678,9 @@ def test_voices_best_is_highest_score(client: TestClient, db: Session) -> None: create_random_article(db, publisher_id=lab.id, score=6, published_at=_ago(1)) voice = _voices(client, "lab", {lab.slug})[0] - assert voice["best"]["title"] == best.title - assert voice["best"]["url"] == best.url - assert voice["best"]["score"] == 9 + assert voice.best.title == best.title + assert voice.best.url == best.url + assert voice.best.score == 9 def test_voices_order_by_count_then_best(client: TestClient, db: Session) -> None: @@ -724,7 +701,7 @@ def test_voices_order_by_count_then_best(client: TestClient, db: Session) -> Non ) voices = _voices(client, "company", {busy.slug, strong.slug, weak.slug}) - assert [v["slug"] for v in voices] == [busy.slug, strong.slug, weak.slug] + assert [v.slug for v in voices] == [busy.slug, strong.slug, weak.slug] def test_voices_unknown_kind_returns_422(client: TestClient) -> None: diff --git a/backend/app/audience/tests/test_likes.py b/backend/app/entry/api/test_likes.py similarity index 97% rename from backend/app/audience/tests/test_likes.py rename to backend/app/entry/api/test_likes.py index 88669536..ebc1caf3 100644 --- a/backend/app/audience/tests/test_likes.py +++ b/backend/app/entry/api/test_likes.py @@ -9,14 +9,14 @@ from sqlalchemy.exc import IntegrityError from sqlmodel import Session, select -from app.audience.models import ArticleLike -from app.audience.tests.factories import ( +from app.core.db.factories import ( authentication_token_from_email, + create_random_article, create_random_user, + random_email, ) -from app.catalog.tests.factories import create_random_article +from app.core.db.like import ArticleLike from app.core.settings import settings -from tests.random_data import random_email def test_article_like_duplicate_pk_raises(db: Session) -> None: diff --git a/backend/app/audience/tests/test_login.py b/backend/app/entry/api/test_login.py similarity index 77% rename from backend/app/audience/tests/test_login.py rename to backend/app/entry/api/test_login.py index 6237f6cf..e88f856e 100644 --- a/backend/app/audience/tests/test_login.py +++ b/backend/app/entry/api/test_login.py @@ -3,6 +3,7 @@ from __future__ import annotations import uuid +from collections.abc import Callable from datetime import timedelta import jwt @@ -11,22 +12,22 @@ from fastapi.testclient import TestClient from sqlmodel import Session, select -from app.audience import service -from app.audience.models import User -from app.audience.tests.factories import ( +from app.core.dates import get_datetime_utc +from app.core.db.factories import ( create_random_user, + random_email, user_authentication_headers, ) -from app.core.dates import get_datetime_utc +from app.core.db.subscriber import NewsletterSubscriber +from app.core.db.user import User from app.core.security import ALGORITHM, create_access_token from app.core.settings import settings -from app.newsletter.models import NewsletterSubscriber -from tests.random_data import random_email +from app.features.audience import service LINK_SENT = {"message": "Check your inbox for your sign-in link"} -def enable_resend_email(monkeypatch) -> list[dict]: +def enable_resend_email(monkeypatch: pytest.MonkeyPatch) -> list[dict[str, str]]: monkeypatch.setattr( "app.core.settings.settings.RESEND_API_KEY", "test-key", raising=False ) @@ -34,13 +35,13 @@ def enable_resend_email(monkeypatch) -> list[dict]: "app.core.settings.settings.EMAILS_FROM_EMAIL", "hello@example.com" ) monkeypatch.setattr(resend.Contacts, "create", lambda *args, **kwargs: None) - sent: list[dict] = [] + sent: list[dict[str, str]] = [] monkeypatch.setattr(resend.Emails, "send", lambda payload: sent.append(payload)) return sent def test_link_request_emails_existing_user( - client: TestClient, db: Session, monkeypatch + client: TestClient, db: Session, monkeypatch: pytest.MonkeyPatch ) -> None: sent = enable_resend_email(monkeypatch) user = create_random_user(db) @@ -68,11 +69,11 @@ def test_new_link_replaces_the_old_one(client: TestClient, db: Session) -> None: def test_link_request_succeeds_when_send_fails( - client: TestClient, db: Session, monkeypatch + client: TestClient, db: Session, monkeypatch: pytest.MonkeyPatch ) -> None: enable_resend_email(monkeypatch) - def fail(_payload: dict) -> None: + def fail(_payload: dict[str, object]) -> None: raise RuntimeError("resend down") monkeypatch.setattr(resend.Emails, "send", fail) @@ -141,6 +142,47 @@ def test_login_with_link_inactive_user(client: TestClient, db: Session) -> None: assert r.json()["detail"] == "Inactive user" +def test_link_request_for_an_inactive_user_sends_nothing( + client: TestClient, db: Session, monkeypatch: pytest.MonkeyPatch +) -> None: + """Same answer as for anyone else, so the endpoint does not tell a + switched-off account apart, but no link and no token.""" + sent = enable_resend_email(monkeypatch) + user = create_random_user(db) + user.is_active = False + db.add(user) + db.commit() + + r = client.post(f"{settings.API_V1_STR}/login/link", json={"email": user.email}) + assert r.json() == LINK_SENT + assert sent == [] + db.refresh(user) + assert user.login_token is None + + +def test_an_access_token_for_an_inactive_user_is_refused( + client: TestClient, db: Session +) -> None: + user = create_random_user(db) + headers = user_authentication_headers(user) + user.is_active = False + db.add(user) + db.commit() + + r = client.post(f"{settings.API_V1_STR}/login/test-token", headers=headers) + assert r.status_code == 400 + assert r.json()["detail"] == "Inactive user" + + +def test_an_access_token_for_a_deleted_user_is_refused(client: TestClient) -> None: + token = create_access_token(uuid.uuid4(), timedelta(minutes=5)) + r = client.post( + f"{settings.API_V1_STR}/login/test-token", + headers={"Authorization": f"Bearer {token}"}, + ) + assert r.status_code == 404 + + def test_use_access_token( client: TestClient, superuser_token_headers: dict[str, str] ) -> None: @@ -168,7 +210,9 @@ def _signed(user_id: uuid.UUID, key: str, expires_in: timedelta) -> str: ], ids=["expired", "wrong-key", "tampered"], ) -def test_access_token_is_rejected(client: TestClient, db: Session, make_token) -> None: +def test_access_token_is_rejected( + client: TestClient, db: Session, make_token: Callable[[uuid.UUID], str] +) -> None: user = create_random_user(db) r = client.post( f"{settings.API_V1_STR}/login/test-token", @@ -178,7 +222,7 @@ def test_access_token_is_rejected(client: TestClient, db: Session, make_token) - def test_link_request_unknown_email_creates_account( - client: TestClient, db: Session, monkeypatch + client: TestClient, db: Session, monkeypatch: pytest.MonkeyPatch ) -> None: sent = enable_resend_email(monkeypatch) email = random_email() @@ -201,11 +245,11 @@ def test_link_request_unknown_email_creates_account( def test_new_account_syncs_resend_contact( - client: TestClient, db: Session, monkeypatch + client: TestClient, db: Session, monkeypatch: pytest.MonkeyPatch ) -> None: monkeypatch.setenv("RESEND_API_KEY", "test-key") monkeypatch.setenv("RESEND_AUDIENCE_ID", "test-audience") - calls: list[dict] = [] + calls: list[dict[str, object]] = [] monkeypatch.setattr( resend.Contacts, "create", lambda payload: calls.append(payload) ) @@ -226,7 +270,7 @@ def test_new_account_syncs_resend_contact( def test_link_request_never_duplicates_an_account( - client: TestClient, db: Session, monkeypatch + client: TestClient, db: Session, monkeypatch: pytest.MonkeyPatch ) -> None: sent = enable_resend_email(monkeypatch) user = create_random_user(db) diff --git a/backend/app/entry/api/test_stories.py b/backend/app/entry/api/test_stories.py new file mode 100644 index 00000000..cd0d5d43 --- /dev/null +++ b/backend/app/entry/api/test_stories.py @@ -0,0 +1,46 @@ +"""The stories rail endpoint.""" + +from __future__ import annotations + +from datetime import UTC, datetime, timedelta + +from fastapi.testclient import TestClient +from sqlmodel import Session + +from app.core.db.factories import ( + create_random_article, + create_random_publisher, + random_lower_string, +) +from app.core.settings import settings +from app.features.stories import service as stories + + +def test_the_route_serves_the_list( + client: TestClient, db: Session, made: dict[str, list[int]] +) -> None: + now = datetime.now(UTC) + publishers = [create_random_publisher(db), create_random_publisher(db)] + urls = [] + for i, publisher in enumerate([*publishers, publishers[0]]): + article = create_random_article( + db, publisher_id=publisher.id, published_at=now - timedelta(days=3 - i) + ) + assert article.id is not None + made["articles"].append(article.id) + urls.append(article.url) + made["publishers"] += [p.id for p in publishers if p.id is not None] + slug = f"served-{random_lower_string()[:6]}" + stories.save_story(db, slug, "Served", "What it is. Why now.", urls) + + r = client.get(f"{settings.API_V1_STR}/stories/", params={"limit": 20}) + assert r.status_code == 200 + story = next(s for s in r.json() if s["slug"] == slug) + assert set(story["articles"][0]) == { + "id", + "title", + "url", + "publisher", + "published_at", + "score", + } diff --git a/backend/app/audience/tests/test_users.py b/backend/app/entry/api/test_users.py similarity index 94% rename from backend/app/audience/tests/test_users.py rename to backend/app/entry/api/test_users.py index ad1215a9..b1e31cf2 100644 --- a/backend/app/audience/tests/test_users.py +++ b/backend/app/entry/api/test_users.py @@ -5,10 +5,13 @@ from fastapi.testclient import TestClient from sqlmodel import Session, select -from app.audience.models import User -from app.audience.tests.factories import create_random_user, user_authentication_headers +from app.core.db.factories import ( + create_random_user, + random_email, + user_authentication_headers, +) +from app.core.db.user import User from app.core.settings import settings -from tests.random_data import random_email def test_get_users_superuser_me( diff --git a/backend/app/entry/api/users.py b/backend/app/entry/api/users.py new file mode 100644 index 00000000..fb8ad6cf --- /dev/null +++ b/backend/app/entry/api/users.py @@ -0,0 +1,51 @@ +"""The signed-in reader's own account.""" + +from __future__ import annotations + +from fastapi import APIRouter, HTTPException + +from app.core.db.user import User +from app.entry.api.deps import CurrentUser, SessionDep +from app.features.audience import service +from app.features.audience.model import Message, UserPublic, UserUpdateMe + +router = APIRouter(prefix="/users", tags=["users"]) + + +@router.patch("/me", response_model=UserPublic) +def update_user_me( + *, session: SessionDep, user_in: UserUpdateMe, current_user: CurrentUser +) -> User: + """ + Update own user. + """ + if user_in.email: + existing_user = service.get_user_by_email(session=session, email=user_in.email) + if existing_user and existing_user.id != current_user.id: + raise HTTPException( + status_code=409, detail="User with this email already exists" + ) + return service.update_profile( + session=session, db_user=current_user, changes=user_in + ) + + +@router.get("/me", response_model=UserPublic) +def read_user_me(current_user: CurrentUser) -> User: + """ + Get current user. + """ + return current_user + + +@router.delete("/me", response_model=Message) +def delete_user_me(session: SessionDep, current_user: CurrentUser) -> Message: + """ + Delete own user. + """ + if current_user.is_superuser: + raise HTTPException( + status_code=403, detail="Super users are not allowed to delete themselves" + ) + service.delete_account(session=session, db_user=current_user) + return Message(message="User deleted successfully") diff --git a/backend/app/audience/__init__.py b/backend/app/entry/cli/__init__.py similarity index 100% rename from backend/app/audience/__init__.py rename to backend/app/entry/cli/__init__.py diff --git a/backend/scripts/backfill_publishers.py b/backend/app/entry/cli/backfill_publishers.py similarity index 93% rename from backend/scripts/backfill_publishers.py rename to backend/app/entry/cli/backfill_publishers.py index 1b528757..9264c5e6 100644 --- a/backend/scripts/backfill_publishers.py +++ b/backend/app/entry/cli/backfill_publishers.py @@ -7,9 +7,9 @@ is no longer an aggregator's, and a publisher already made is found by its link. Run it against a restored dump first and read the report. - python scripts/backfill_publishers.py # re-credit + lab kinds - python scripts/backfill_publishers.py --export unknown.csv - python scripts/backfill_publishers.py --kinds kinds.csv + python -m app.entry.cli.backfill_publishers # re-credit + lab kinds + python -m app.entry.cli.backfill_publishers --export unknown.csv + python -m app.entry.cli.backfill_publishers --kinds kinds.csv `--export` writes the `unknown` publishers that have published articles, with titles and summaries to judge them by. Fill in `kind` (lab, company, @@ -27,17 +27,12 @@ from sqlmodel import Session, col, func, select -from app.catalog.models import ( - Article, - ArticleKind, - LinkPlatform, - Publisher, - PublisherKind, -) -from app.core.db import engine -from pipeline.curation import NAMEABLE_KINDS -from pipeline.models import Reject -from pipeline.publishers import AGGREGATOR_TYPES, PublisherResolver +from app.core.db.article import Article, ArticleKind +from app.core.db.engine import engine +from app.core.db.ledger import Reject +from app.core.db.publisher import LinkPlatform, Publisher, PublisherKind +from app.features.collect.service import AGGREGATOR_TYPES, PublisherResolver +from app.features.curation.service import NAMEABLE_KINDS # Publishers that train and release their own models. Picked by hand from the # 2026-10-03 publisher list. diff --git a/backend/scripts/backup_db.py b/backend/app/entry/cli/backup_db.py similarity index 90% rename from backend/scripts/backup_db.py rename to backend/app/entry/cli/backup_db.py index 05edb8f8..c9fb71fe 100644 --- a/backend/scripts/backup_db.py +++ b/backend/app/entry/cli/backup_db.py @@ -1,4 +1,4 @@ -"""Nightly Postgres backup to kDrive. Run with: python scripts/backup_db.py +"""Nightly Postgres backup to kDrive. Run with: python app/entry/cli/backup_db.py pg_dump runs inside the pgvector container (`docker compose exec`) so the client always matches the server version and nothing Postgres needs installing @@ -23,7 +23,7 @@ from kdrive_client.kdrive_client import KDriveClient # type: ignore[import-untyped] from kdrive_client.kdrive_file import KDriveFile # type: ignore[import-untyped] -COMPOSE_FILE = Path(__file__).resolve().parents[2] / "compose.yml" +COMPOSE_FILE = Path(__file__).resolve().parents[4] / "compose.yml" BACKUP_DIR = os.environ.get("KDRIVE_BACKUP_DIR", "/Private/agentique-db") KEEP = int(os.environ.get("KDRIVE_BACKUP_KEEP", "14")) PREFIX = "agentique-db-" @@ -53,8 +53,10 @@ def dump_db(dest: Path) -> None: proc.stdout.close() if proc.wait() != 0: raise RuntimeError(f"pg_dump exited with {proc.returncode}") - if dest.stat().st_size == 0: - raise RuntimeError("dump is empty") + # The uncompressed size: a gzip of nothing still has a header, so the + # file on disk is never 0 bytes. + if out.tell() == 0: + raise RuntimeError("dump is empty") def prune(client: KDriveClient, directory_id: int) -> None: diff --git a/backend/app/entry/cli/collect.py b/backend/app/entry/cli/collect.py new file mode 100644 index 00000000..c240956f --- /dev/null +++ b/backend/app/entry/cli/collect.py @@ -0,0 +1,10 @@ +"""The nightly collect run, as the systemd unit starts it: python -m app.entry.cli.collect""" + +from __future__ import annotations + +import sys + +from app.features.collect.service import run_nightly + +if __name__ == "__main__": + sys.exit(run_nightly()) diff --git a/backend/scripts/create_superuser.py b/backend/app/entry/cli/create_superuser.py similarity index 83% rename from backend/scripts/create_superuser.py rename to backend/app/entry/cli/create_superuser.py index 14aa5ed2..639378d2 100644 --- a/backend/scripts/create_superuser.py +++ b/backend/app/entry/cli/create_superuser.py @@ -6,8 +6,8 @@ from sqlmodel import Session -from app.audience.service import init_db -from app.core.db import engine +from app.core.db.engine import engine +from app.features.audience.service import init_db logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) diff --git a/backend/scripts/find_missing_tags.py b/backend/app/entry/cli/find_missing_tags.py similarity index 85% rename from backend/scripts/find_missing_tags.py rename to backend/app/entry/cli/find_missing_tags.py index be15b7b9..64e11c28 100644 --- a/backend/scripts/find_missing_tags.py +++ b/backend/app/entry/cli/find_missing_tags.py @@ -11,14 +11,14 @@ candidates for a missing tag (e.g. text-to-speech, if no such tag exists). We group those under-covered articles into clusters and print each cluster with a few representative titles and its nearest existing tag, so a human can name the -gap and add it to `scripts/tags.json`. +gap and add it to `app/entry/cli/tags.json`. No LLM calls: embeddings are already in the DB, tag vectors are one local encode of ~30 tag strings, and clustering is greedy cosine math. Run it against prod by pointing the POSTGRES_* env vars at prod (same vars the pipeline reads). - cd backend && python -m scripts.find_missing_tags - cd backend && python -m scripts.find_missing_tags --threshold 0.30 --json + cd backend && python -m app.entry.cli.find_missing_tags + cd backend && python -m app.entry.cli.find_missing_tags --threshold 0.30 --json Tune `--threshold` to what "poorly covered" means for your data: run once, look at the coverage percentiles it prints, then set the threshold near the low tail. @@ -34,18 +34,23 @@ from typing import TypedDict import numpy as np -from sqlmodel import Session, select +import numpy.typing as npt +from sqlmodel import Session, col, select -from app.catalog.models import Article, ArticleTag, Tag +from app.core.db.article import Article +from app.core.db.engine import engine +from app.core.db.tag import ArticleTag, Tag from app.core.embedding import embed_batch -from pipeline.db import get_engine + +# Rows of float32 vectors, or one vector. +type Vectors = npt.NDArray[np.float32] @dataclass class ArticleVec: id: int title: str - vec: np.ndarray # L2-normalized + vec: Vectors # L2-normalized tags: list[str] = field(default_factory=list) @@ -58,7 +63,7 @@ class GapCluster(TypedDict): sample_titles: list[str] -def normalize(m: np.ndarray) -> np.ndarray: +def normalize(m: Vectors) -> Vectors: """L2-normalize rows; leave all-zero rows at zero (cosine -> 0).""" norms = np.linalg.norm(m, axis=1, keepdims=True) norms[norms == 0] = 1.0 @@ -70,7 +75,7 @@ def tag_text(name: str, description: str | None) -> str: return f"{name}: {description}" if description else name -def load_tags(session: Session) -> tuple[list[str], list[str], np.ndarray]: +def load_tags(session: Session) -> tuple[list[str], list[str], Vectors]: """Return (slugs, names, normalized tag vectors), aligned by index.""" rows = session.exec(select(Tag)).all() if not rows: @@ -85,10 +90,10 @@ def load_articles( session: Session, since: datetime | None, limit: int ) -> list[ArticleVec]: """Articles that have an embedding, with their assigned tag slugs attached.""" - stmt = select(Article).where(Article.embedding.is_not(None)) # type: ignore[union-attr] # ty: ignore[unresolved-attribute] + stmt = select(Article).where(col(Article.embedding).is_not(None)) if since is not None: - stmt = stmt.where(Article.published_at >= since) # type: ignore[operator] # ty: ignore[unsupported-operator] - stmt = stmt.order_by(Article.id) # type: ignore[arg-type] # ty: ignore[invalid-argument-type] + stmt = stmt.where(col(Article.published_at) >= since) + stmt = stmt.order_by(col(Article.id)) if limit > 0: stmt = stmt.limit(limit) rows = session.exec(stmt).all() @@ -98,8 +103,8 @@ def load_articles( if ids: pairs = session.exec( select(ArticleTag.article_id, Tag.slug) - .join(Tag, Tag.id == ArticleTag.tag_id) # type: ignore[arg-type] # ty: ignore[invalid-argument-type] - .where(ArticleTag.article_id.in_(ids)) # type: ignore[attr-defined] # ty: ignore[unresolved-attribute] + .join(Tag, col(Tag.id) == col(ArticleTag.tag_id)) + .where(col(ArticleTag.article_id).in_(ids)) ).all() for aid, slug in pairs: tags_by_article.setdefault(aid, []).append(slug) @@ -117,16 +122,17 @@ def load_articles( @dataclass class Cluster: - centroid: np.ndarray # normalized running mean + centroid: Vectors # normalized running mean members: list[int] # indices into the under-covered list - def add(self, idx: int, vec: np.ndarray) -> None: + def add(self, idx: int, vec: Vectors) -> None: n = len(self.members) - self.centroid = normalize(((self.centroid * n + vec) / (n + 1))[None, :])[0] + mean = (self.centroid * np.float32(n) + vec) / np.float32(n + 1) + self.centroid = normalize(mean[None, :])[0] self.members.append(idx) -def greedy_cluster(vecs: np.ndarray, sim_threshold: float) -> list[Cluster]: +def greedy_cluster(vecs: Vectors, sim_threshold: float) -> list[Cluster]: """One-pass greedy cosine clustering. Each vector joins the most similar existing cluster if it clears the threshold, else it seeds a new one. Good enough to surface coherent themes; not trying to be k-means.""" @@ -177,7 +183,7 @@ def main() -> None: since = datetime.fromisoformat(args.since) if args.since else None - with Session(get_engine()) as session: + with Session(engine) as session: slugs, names, tag_vecs = load_tags(session) articles = load_articles(session, since, args.limit) @@ -264,8 +270,8 @@ def main() -> None: print(f" - {t}") print() print( - "Next: name each real gap and add it to scripts/tags.json, then " - "re-run `python scripts/seed_tags.py`." + "Next: name each real gap and add it to app/entry/cli/tags.json, then " + "re-run `python -m app.entry.cli.seed_tags`." ) diff --git a/backend/scripts/prestart.sh b/backend/app/entry/cli/prestart.sh similarity index 67% rename from backend/scripts/prestart.sh rename to backend/app/entry/cli/prestart.sh index b23ac4a5..49c36609 100755 --- a/backend/scripts/prestart.sh +++ b/backend/app/entry/cli/prestart.sh @@ -4,18 +4,18 @@ set -e set -x # Let the DB start -python scripts/wait_for_db.py +python -m app.entry.cli.wait_for_db # Run migrations alembic upgrade head # Create initial data in DB -python scripts/create_superuser.py +python -m app.entry.cli.create_superuser # Load the tag vocabulary (all environments; the pipeline requires it) -python scripts/seed_tags.py +python -m app.entry.cli.seed_tags # Seed sample articles for local/CI (module also self-guards on production) if [ "$ENVIRONMENT" != "production" ]; then - python scripts/seed_articles.py + python -m app.entry.cli.seed_articles fi diff --git a/backend/app/entry/cli/report.py b/backend/app/entry/cli/report.py new file mode 100644 index 00000000..5aa1662b --- /dev/null +++ b/backend/app/entry/cli/report.py @@ -0,0 +1,10 @@ +"""The daily mail, as the systemd unit starts it: python -m app.entry.cli.report""" + +from __future__ import annotations + +import sys + +from app.features.collect.service import send_report + +if __name__ == "__main__": + sys.exit(send_report()) diff --git a/backend/scripts/seed_articles.py b/backend/app/entry/cli/seed_articles.py similarity index 95% rename from backend/scripts/seed_articles.py rename to backend/app/entry/cli/seed_articles.py index d1454001..5827d27e 100644 --- a/backend/scripts/seed_articles.py +++ b/backend/app/entry/cli/seed_articles.py @@ -10,17 +10,10 @@ from sqlalchemy import func from sqlmodel import Session, select -from app.catalog.models import ( - Article, - ArticleKind, - ArticleTag, - Category, - Publisher, - PublisherKind, - Tag, - TrustLevel, -) -from app.core.db import engine +from app.core.db.article import Article, ArticleKind, Category +from app.core.db.engine import engine +from app.core.db.publisher import Publisher, PublisherKind, TrustLevel +from app.core.db.tag import ArticleTag, Tag from app.core.settings import settings logging.basicConfig(level=logging.INFO) diff --git a/backend/scripts/seed_stories.py b/backend/app/entry/cli/seed_stories.py similarity index 93% rename from backend/scripts/seed_stories.py rename to backend/app/entry/cli/seed_stories.py index a3c9605b..9fce595c 100644 --- a/backend/scripts/seed_stories.py +++ b/backend/app/entry/cli/seed_stories.py @@ -17,10 +17,11 @@ from sqlmodel import Session, col, select -from app.catalog import stories -from app.catalog.models import Article -from app.core.db import engine +from app.core.db.article import Article +from app.core.db.engine import engine from app.core.settings import settings +from app.features.stories import service as stories +from app.features.stories.model import StoryError logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) @@ -98,7 +99,7 @@ def main() -> None: story["blurb"], story["urls"], ) - except stories.StoryError as exc: + except StoryError as exc: session.rollback() logger.warning("Skipped %s: %s", story["slug"], exc) continue diff --git a/backend/scripts/seed_tags.py b/backend/app/entry/cli/seed_tags.py similarity index 84% rename from backend/scripts/seed_tags.py rename to backend/app/entry/cli/seed_tags.py index f72cc9f4..1314f9aa 100644 --- a/backend/scripts/seed_tags.py +++ b/backend/app/entry/cli/seed_tags.py @@ -1,6 +1,6 @@ """Load the controlled tag vocabulary into the `tag` table. -Unlike `seed_articles`, this is real data, not sample data: `pipeline.tags. +Unlike `seed_articles`, this is real data, not sample data: `app.features.curation.tags. load_vocabulary()` raises on an empty `tag` table, so every environment — production included — needs it. Idempotent, keyed on `slug`; tags no longer in the file are left alone, because `article_tag` rows may still point at them. @@ -13,15 +13,16 @@ from sqlmodel import Session, select -from app.catalog.models import Tag -from app.core.db import engine +from app.core.db.engine import engine +from app.core.db.tag import Tag TAGS_FILE = Path(__file__).resolve().parent / "tags.json" def load_tags() -> list[dict[str, str]]: with TAGS_FILE.open() as f: - return json.load(f)["tags"] + tags: list[dict[str, str]] = json.load(f)["tags"] + return tags def seed(session: Session) -> None: diff --git a/backend/scripts/tags.json b/backend/app/entry/cli/tags.json similarity index 100% rename from backend/scripts/tags.json rename to backend/app/entry/cli/tags.json diff --git a/backend/app/entry/cli/test_backup_db.py b/backend/app/entry/cli/test_backup_db.py new file mode 100644 index 00000000..59e465b3 --- /dev/null +++ b/backend/app/entry/cli/test_backup_db.py @@ -0,0 +1,54 @@ +"""The nightly database backup: where it finds compose, and what it refuses to keep.""" + +from __future__ import annotations + +import subprocess +from pathlib import Path + +import pytest + +from app.entry.cli import backup_db + +REPO = Path(__file__).resolve().parents[4] + + +def test_the_backup_finds_the_compose_file() -> None: + assert backup_db.COMPOSE_FILE == REPO / "compose.yml" + assert backup_db.COMPOSE_FILE.is_file() + + +def _fake_pg_dump(monkeypatch: pytest.MonkeyPatch, script: str) -> None: + """Stand in for `docker compose exec db pg_dump` with a shell one-liner.""" + monkeypatch.setenv("POSTGRES_USER", "postgres") + monkeypatch.setenv("POSTGRES_DB", "app") + real = subprocess.Popen + + def popen(_cmd: list[str], stdout: int) -> subprocess.Popen[bytes]: + return real(["sh", "-c", script], stdout=stdout) + + monkeypatch.setattr(subprocess, "Popen", popen) + + +def test_a_failed_pg_dump_fails_the_backup( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + _fake_pg_dump(monkeypatch, "echo partial; exit 3") + with pytest.raises(RuntimeError, match="exited with 3"): + backup_db.dump_db(tmp_path / "dump.sql.gz") + + +def test_an_empty_dump_fails_the_backup( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + _fake_pg_dump(monkeypatch, "true") + with pytest.raises(RuntimeError, match="empty"): + backup_db.dump_db(tmp_path / "dump.sql.gz") + + +def test_a_dump_with_output_is_kept( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + _fake_pg_dump(monkeypatch, "echo 'CREATE TABLE x ();'") + dest = tmp_path / "dump.sql.gz" + backup_db.dump_db(dest) + assert dest.stat().st_size > 0 diff --git a/backend/app/entry/cli/test_collect.py b/backend/app/entry/cli/test_collect.py new file mode 100644 index 00000000..5ced33e6 --- /dev/null +++ b/backend/app/entry/cli/test_collect.py @@ -0,0 +1,35 @@ +"""The nightly collect timer's entrypoint.""" + +from __future__ import annotations + +import runpy +import subprocess +import sys +from pathlib import Path + +import pytest + +from app.features.collect import service as collect + +BACKEND = Path(__file__).resolve().parents[3] + + +@pytest.mark.parametrize("code", [0, 1]) +def test_the_entrypoint_exits_with_the_runs_code( + monkeypatch: pytest.MonkeyPatch, code: int +) -> None: + monkeypatch.setattr(collect, "run_nightly", lambda: code) + with pytest.raises(SystemExit) as exited: + runpy.run_module("app.entry.cli.collect", run_name="__main__") + assert exited.value.code == code + + +def test_the_collect_run_leaves_the_root_logger_alone() -> None: + # At INFO, httpx logs a line for every request the night makes and buries + # the run's own progress in the journal. + probe = ( + "import logging, app.entry.cli.collect; " + "root = logging.getLogger(); " + "assert root.level == logging.WARNING and not root.handlers, root" + ) + subprocess.run([sys.executable, "-c", probe], cwd=BACKEND, check=True) diff --git a/backend/app/entry/cli/test_deploy_files.py b/backend/app/entry/cli/test_deploy_files.py new file mode 100644 index 00000000..ba60b62c --- /dev/null +++ b/backend/app/entry/cli/test_deploy_files.py @@ -0,0 +1,155 @@ +"""What the deploy ships: the workflow's rsync run against this checkout's +tracked files, then every path the units and the deploy job use looked up in +the result.""" + +from __future__ import annotations + +import re +import shlex +import shutil +import subprocess +import sys +import tomllib +from pathlib import Path + +import pytest + +BACKEND = Path(__file__).resolve().parents[3] +REPO = BACKEND.parent +BOX = Path("/opt/agentique") +UNITS = sorted((REPO / "deploy").glob("*.service")) +WORKFLOW = REPO / ".github/workflows/deploy-production.yml" +# Never shipped: the box keeps its own copy, and the filter protects it. +BOX_OWNED = (BOX / ".env", BOX / ".venv") + + +def rsync_command() -> list[str]: + lines = [ + line.strip() + for line in WORKFLOW.read_text().splitlines() + if line.strip().startswith("rsync ") + ] + assert len(lines) == 1, lines + return shlex.split(lines[0]) + + +@pytest.fixture(scope="module") +def shipped(tmp_path_factory: pytest.TempPathFactory) -> Path: + version = subprocess.run( + ["rsync", "--version"], capture_output=True, text=True, check=False + ).stdout + if "version 3." not in version: + pytest.skip("the filter needs rsync 3, as on the box") + # The deploy syncs a clean checkout, so untracked files must not count. + source = tmp_path_factory.mktemp("checkout") + tracked = subprocess.run( + ["git", "ls-files", "-z"], cwd=REPO, capture_output=True, check=True + ).stdout.decode() + for name in filter(None, tracked.split("\0")): + if (REPO / name).is_file(): + (source / name).parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(REPO / name, source / name) + (source / "frontend/dist").mkdir(parents=True) + (source / "frontend/dist/index.html").touch() + + dest = tmp_path_factory.mktemp("box") + command = rsync_command() + assert command[-1] == f"{BOX}/", command + subprocess.run([*command[:-1], f"{dest}/"], cwd=source, check=True) + return dest + + +def on_box(shipped: Path, path: Path) -> Path: + return shipped / path.relative_to(BOX) + + +def box_owned(path: Path) -> bool: + return any(path == owned or owned in path.parents for owned in BOX_OWNED) + + +@pytest.mark.parametrize("unit", UNITS, ids=lambda p: p.name) +def test_every_path_a_unit_uses_is_shipped(shipped: Path, unit: Path) -> None: + config = dict( + line.split("=", 1) + for line in unit.read_text().splitlines() + if "=" in line and not line.startswith("#") + ) + workdir = on_box(shipped, Path(config["WorkingDirectory"])) + assert workdir.is_dir(), workdir + assert box_owned(Path(config["EnvironmentFile"])) + # systemd's -, @, + and ! prefixes change how it runs, not what. + command, *args = config["ExecStart"].lstrip("-@+!:").split() + assert box_owned(Path(command)), command + if args[:1] == ["-m"]: + module = workdir / args[1].replace(".", "/") + assert module.with_suffix(".py").is_file() or (module / "__main__.py").is_file() + for arg in args: + if arg.endswith(".py"): + assert (workdir / arg).is_file(), arg + + +def test_every_directory_the_deploy_job_works_in_is_shipped(shipped: Path) -> None: + dirs = re.findall(r"working-directory: (/opt/agentique\S*)", WORKFLOW.read_text()) + assert dirs + for workdir in dirs: + assert on_box(shipped, Path(workdir)).is_dir(), workdir + for script in re.findall(r"bash (\S+\.sh)", WORKFLOW.read_text()): + assert (on_box(shipped, BOX / "backend") / script).is_file(), script + + +def test_prestart_runs_only_shipped_modules(shipped: Path) -> None: + prestart = shipped / "backend/app/entry/cli/prestart.sh" + for module in re.findall(r"python -m ([\w.]+)", prestart.read_text()): + assert (shipped / "backend" / (module.replace(".", "/") + ".py")).is_file() + + +def test_uv_sync_and_alembic_find_their_files(shipped: Path) -> None: + root = tomllib.loads((shipped / "pyproject.toml").read_text()) + for member in root["tool"]["uv"]["workspace"]["members"]: + assert (shipped / member / "pyproject.toml").is_file(), member + assert (shipped / "uv.lock").is_file() + assert (shipped / ".python-version").is_file() + assert (shipped / "backend/migrations/env.py").is_file() + # backup_db.py runs docker compose against the repo root's compose.yml. + assert (shipped / "compose.yml").is_file() + + +def test_every_data_file_in_app_is_shipped(shipped: Path) -> None: + data = [ + p.relative_to(REPO) + for p in (BACKEND / "app").rglob("*") + if p.is_file() + and p.suffix not in {".py", ".pyc"} + and not p.name.startswith("test_") + ] + assert data + for path in data: + assert (shipped / path).is_file(), path + + +def test_no_test_code_is_shipped(shipped: Path) -> None: + names = {p.name for p in (shipped / "backend").rglob("*.py")} + assert not [n for n in names if n.startswith("test_")] + assert "conftest.py" not in names + assert "factories.py" not in names + + +def test_every_shipped_module_imports_from_the_shipped_tree(shipped: Path) -> None: + # Run from the shipped backend/, as the units do. The venv's editable + # install still points at the checkout, so a module the filter drops would + # load from there instead. Fail on any module that did. + env_file = REPO / ".env" + if env_file.is_file(): + shutil.copy(env_file, shipped / ".env") + script = f""" +import importlib, pkgutil, sys +import app +for module in pkgutil.walk_packages(app.__path__, "app."): + importlib.import_module(module.name) +checkout = [ + name for name, module in sys.modules.items() + if (getattr(module, "__file__", None) or "").startswith({f"{BACKEND}/"!r}) +] +assert not checkout, checkout +""" + subprocess.run([sys.executable, "-c", script], cwd=shipped / "backend", check=True) diff --git a/backend/app/entry/cli/test_entrypoints.py b/backend/app/entry/cli/test_entrypoints.py new file mode 100644 index 00000000..0c0f8320 --- /dev/null +++ b/backend/app/entry/cli/test_entrypoints.py @@ -0,0 +1,40 @@ +"""What systemd, the deploy and prestart.sh run: that each target exists.""" + +from __future__ import annotations + +import importlib.util +import re +from pathlib import Path + +import pytest + +BACKEND = Path(__file__).resolve().parents[3] +REPO = BACKEND.parent +SHELL_SCRIPTS = ( + BACKEND / "app/entry/cli/prestart.sh", + REPO / "scripts/tests-start.sh", +) + + +@pytest.mark.parametrize( + "unit", sorted((REPO / "deploy").glob("*.service")), ids=lambda p: p.name +) +def test_every_python_unit_starts_something_that_exists(unit: Path) -> None: + # The units run from /opt/agentique/backend. A renamed module fails there + # only when the timer fires, at night. + for line in unit.read_text().splitlines(): + if not line.startswith("ExecStart=") or "/python " not in line: + continue + args = line.split()[1:] + if args[0] == "-m": + assert importlib.util.find_spec(args[1]) is not None, line + else: + assert (BACKEND / args[0]).is_file(), line + + +@pytest.mark.parametrize("script", SHELL_SCRIPTS, ids=lambda p: p.name) +def test_every_module_a_shell_script_runs_exists(script: Path) -> None: + targets = re.findall(r"python -m ([\w.]+)", script.read_text()) + assert targets + for target in targets: + assert importlib.util.find_spec(target) is not None, target diff --git a/backend/app/entry/cli/test_report.py b/backend/app/entry/cli/test_report.py new file mode 100644 index 00000000..5a0792d9 --- /dev/null +++ b/backend/app/entry/cli/test_report.py @@ -0,0 +1,19 @@ +"""The daily report timer's entrypoint.""" + +from __future__ import annotations + +import runpy + +import pytest + +from app.features.collect import service as collect + + +@pytest.mark.parametrize("code", [0, 1]) +def test_the_entrypoint_exits_with_the_reports_code( + monkeypatch: pytest.MonkeyPatch, code: int +) -> None: + monkeypatch.setattr(collect, "send_report", lambda: code) + with pytest.raises(SystemExit) as exited: + runpy.run_module("app.entry.cli.report", run_name="__main__") + assert exited.value.code == code diff --git a/backend/app/entry/cli/test_seed_articles.py b/backend/app/entry/cli/test_seed_articles.py new file mode 100644 index 00000000..46dcb8dd --- /dev/null +++ b/backend/app/entry/cli/test_seed_articles.py @@ -0,0 +1,14 @@ +"""The sample articles seed, which must never run against production.""" + +from __future__ import annotations + +import pytest + +from app.core.settings import settings +from app.entry.cli import seed_articles + + +def test_the_seed_refuses_production(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(settings, "ENVIRONMENT", "production") + with pytest.raises((RuntimeError, SystemExit), match="Refusing"): + seed_articles.main() diff --git a/backend/app/entry/cli/test_seed_stories.py b/backend/app/entry/cli/test_seed_stories.py new file mode 100644 index 00000000..f9687c37 --- /dev/null +++ b/backend/app/entry/cli/test_seed_stories.py @@ -0,0 +1,14 @@ +"""The sample stories seed, which must never run against production.""" + +from __future__ import annotations + +import pytest + +from app.core.settings import settings +from app.entry.cli import seed_stories + + +def test_the_seed_refuses_production(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(settings, "ENVIRONMENT", "production") + with pytest.raises((RuntimeError, SystemExit), match="Refusing"): + seed_stories.main() diff --git a/backend/tests/scripts/test_wait_for_db.py b/backend/app/entry/cli/test_wait_for_db.py similarity index 80% rename from backend/tests/scripts/test_wait_for_db.py rename to backend/app/entry/cli/test_wait_for_db.py index a9aca870..28c7eb48 100644 --- a/backend/tests/scripts/test_wait_for_db.py +++ b/backend/app/entry/cli/test_wait_for_db.py @@ -6,7 +6,7 @@ from sqlmodel import select -from scripts.wait_for_db import init, logger +from app.entry.cli.wait_for_db import init, logger def test_init_successful_connection() -> None: @@ -18,8 +18,8 @@ def test_init_successful_connection() -> None: select1 = select(1) with ( - patch("scripts.wait_for_db.Session", return_value=session_mock), - patch("scripts.wait_for_db.select", return_value=select1), + patch("app.entry.cli.wait_for_db.Session", return_value=session_mock), + patch("app.entry.cli.wait_for_db.select", return_value=select1), patch.object(logger, "info"), patch.object(logger, "error"), patch.object(logger, "warn"), diff --git a/backend/scripts/wait_for_db.py b/backend/app/entry/cli/wait_for_db.py similarity index 96% rename from backend/scripts/wait_for_db.py rename to backend/app/entry/cli/wait_for_db.py index dd42130a..c6b8e4af 100644 --- a/backend/scripts/wait_for_db.py +++ b/backend/app/entry/cli/wait_for_db.py @@ -12,7 +12,7 @@ from sqlmodel import Session, select from tenacity import after_log, before_log, retry, stop_after_attempt, wait_fixed -from app.core.db import engine +from app.core.db.engine import engine logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) diff --git a/backend/app/audience/tests/__init__.py b/backend/app/entry/mcp/__init__.py similarity index 100% rename from backend/app/audience/tests/__init__.py rename to backend/app/entry/mcp/__init__.py diff --git a/backend/app/entry/mcp/approve.py b/backend/app/entry/mcp/approve.py new file mode 100644 index 00000000..8481ec88 --- /dev/null +++ b/backend/app/entry/mcp/approve.py @@ -0,0 +1,53 @@ +"""The `approve` tool: publish a candidate.""" + +from __future__ import annotations + +from fastmcp.exceptions import ToolError +from sqlmodel import Session + +from app.core.db.engine import engine +from app.entry.mcp.auth import require_write +from app.features.curation import service as curation +from app.features.curation.model import CandidateError + + +def approve( + url: str, + score: int, + reason: str, + summary: str, + kind: str, + tags: list[str], + publisher_kind: str | None = None, + publisher_name: str | None = None, +) -> str: + """Publish a candidate with the labels you chose, then embed it. + + `score` is 1-100 on the same scale the rubric describes, `reason` one short + sentence naming what decided it, and `summary` the text a reader sees under + the title. `kind` and `tags` (up to 3) come from `vocabulary`; a github, + huggingface or arxiv URL sets its own kind. When the candidate's + `publisher_kind` is `unknown`, pass `publisher_kind` (from + `vocabulary`'s `publisher_kinds`) and, if its host is not a readable name, + `publisher_name`; both are ignored for a publisher that has a kind. A label + that is not in the vocabulary is refused and the candidate stays pending. + The candidate stops being pending in the same transaction that inserts the + article, so nothing is ever published twice. + """ + require_write() + with Session(engine) as session: + try: + article_id = curation.approve( + session, + url, + score, + reason, + summary, + kind, + tags, + publisher_kind, + publisher_name, + ) + except CandidateError as exc: + raise ToolError(str(exc)) + return f"Published article #{article_id}: {url}" diff --git a/backend/app/entry/mcp/auth.py b/backend/app/entry/mcp/auth.py new file mode 100644 index 00000000..0905c9a1 --- /dev/null +++ b/backend/app/entry/mcp/auth.py @@ -0,0 +1,64 @@ +"""The bearer-token gate: two tokens, and the scope only the write token carries. + +`sql_query`, `web_fetch` and `web_search` read and are reachable with +`MCP_TOKEN`. The curation tools publish and reject articles, the newsletter +tools draft emails addressed to every subscriber, and the story tools name what +the right rail shows; all of them call `require_write` first. +""" + +from __future__ import annotations + +import secrets + +from fastmcp.exceptions import ToolError +from fastmcp.server.auth import AccessToken, TokenVerifier +from fastmcp.server.dependencies import get_access_token + +from app.core.settings import settings + +# The scope `MCP_WRITE_TOKEN` carries and the read token does not. Named for +# what it lets a caller do, not for the token that carries it. +WRITE_SCOPE = "curate" + + +def _matches(token: str, expected: str | None) -> bool: + """Constant-time compare against a configured token, if there is one. + + Bytes, not str: compare_digest raises on non-ASCII str operands, and the + token is whatever someone put in the header. An unset value matches + nothing, so a deploy that forgets the variable serves 401s instead of an + open database. + """ + return bool(expected) and secrets.compare_digest( + token.encode(), (expected or "").encode() + ) + + +class SharedSecret(TokenVerifier): + """Two bearer tokens, compared in constant time, one scope between them. + + There are two clients — a read-only agent answering questions about the + feed, and the curation agent that publishes to it — so an OAuth provider + would be ceremony around two values that already live in the same `.env` as + every other credential. The write token is checked first and is the only one + that comes back carrying `WRITE_SCOPE`; the read token reaches the three + reading tools and is refused by the rest. + """ + + async def verify_token(self, token: str) -> AccessToken | None: + if _matches(token, settings.MCP_WRITE_TOKEN): + return AccessToken( + token=token, client_id="agentique-curator", scopes=[WRITE_SCOPE] + ) + if _matches(token, settings.MCP_TOKEN): + return AccessToken(token=token, client_id="agentique-mcp", scopes=[]) + return None + + +def require_write() -> None: + """Refuse a write to a caller holding only the read token.""" + token = get_access_token() + if token is None or WRITE_SCOPE not in token.scopes: + raise ToolError( + "This tool needs the curation token — the read token cannot write." + ) diff --git a/backend/app/entry/mcp/check_link.py b/backend/app/entry/mcp/check_link.py new file mode 100644 index 00000000..bee0e17c --- /dev/null +++ b/backend/app/entry/mcp/check_link.py @@ -0,0 +1,26 @@ +"""The `check_link` tool: the facts behind a repo or model URL.""" + +from __future__ import annotations + +from fastmcp.exceptions import ToolError + +from app.entry.mcp.auth import require_write +from app.features.curation import service as curation +from app.features.curation.model import LinkError + + +def check_link(url: str) -> dict[str, object]: + """Look up a GitHub or GitLab repo, or a Hugging Face model or dataset. + + Repos: `stars`, `forks`, `last_push`, `license`, `archived`, `has_readme`, + `description`. Models and datasets: `downloads`, `likes`, `last_modified`, + `license`, `files` (count), and for a model `weights` (true when weight + files are present). A value the lookup could not get reads `unknown`, never + 0; `lookup: unknown` means the whole lookup failed. Any other URL is + refused — read it with `web_fetch`. + """ + require_write() + try: + return curation.check_link(url) + except LinkError as exc: + raise ToolError(str(exc)) diff --git a/backend/app/entry/mcp/close_story.py b/backend/app/entry/mcp/close_story.py new file mode 100644 index 00000000..2bb73256 --- /dev/null +++ b/backend/app/entry/mcp/close_story.py @@ -0,0 +1,23 @@ +"""The `close_story` tool: take a story off the rail.""" + +from __future__ import annotations + +from fastmcp.exceptions import ToolError +from sqlmodel import Session + +from app.core.db.engine import engine +from app.entry.mcp.auth import require_write +from app.features.stories import service as story_domain +from app.features.stories.model import StoryError + + +def close_story(slug: str) -> str: + """Close an open story: it leaves the rail, and its articles are free to + join another.""" + require_write() + with Session(engine) as session: + try: + story_domain.close_story(session, slug) + except StoryError as exc: + raise ToolError(str(exc)) + return f"Closed {slug}" diff --git a/backend/app/entry/mcp/draft_issue.py b/backend/app/entry/mcp/draft_issue.py new file mode 100644 index 00000000..0c1517ff --- /dev/null +++ b/backend/app/entry/mcp/draft_issue.py @@ -0,0 +1,42 @@ +"""The `draft_issue` tool: one weekly issue as a Resend broadcast draft. + +Behind the write scope because a draft lands in the audience subscribers are +mailed from. Sending is a person's, from the Resend dashboard. +""" + +from __future__ import annotations + +from fastmcp.exceptions import ToolError + +from app.entry.mcp.auth import require_write +from app.features.newsletter import service as newsletter +from app.features.newsletter.model import IssueError + + +def draft_issue(post: str) -> str: + """Draft one weekly issue as a Resend broadcast. + + `post` is the whole blog post file: a frontmatter block, then the body. + + --- + title: What does Ollama's MLX backend actually do? + description: One line that makes someone open it. + topic: local-ai + --- + The essay... + + `title` (1-55 characters) is the subject and the post's headline, + `description` (1-160) the line an inbox shows after it, `topic` a kebab + slug that names the draft in Resend. Quote a value in double quotes if it + holds ": " or starts with a quote or a symbol. The body is plain text, + 150-600 words: a blank line between paragraphs, `- ` to start a list line, + `[text](url)` for a link (at least 3 distinct sources), backticks for code. + The server renders it into the site's template. Returns the broadcast id + and the path to save the post at, exactly as passed. Nothing is sent: a + person reads the drafts and sends one from the Resend dashboard. + """ + require_write() + try: + return newsletter.draft(post) + except IssueError as exc: + raise ToolError(str(exc)) diff --git a/backend/app/entry/mcp/get_content.py b/backend/app/entry/mcp/get_content.py new file mode 100644 index 00000000..14a8b590 --- /dev/null +++ b/backend/app/entry/mcp/get_content.py @@ -0,0 +1,31 @@ +"""The `get_content` tool: one page of a candidate's stored text.""" + +from __future__ import annotations + +from fastmcp.exceptions import ToolError +from sqlmodel import Session + +from app.core.db.engine import engine +from app.entry.mcp.auth import require_write +from app.features.curation import service as curation +from app.features.curation.model import CandidateError, ContentPage + + +def get_content( + url: str, offset: int = 0, limit: int = curation.PAGE_LIMIT +) -> ContentPage: + """Return one page of the article text the pipeline extracted for a candidate. + + The stored text is what a page fetch returns, cut at 12000 characters, so + read here first. Returns `text`, `offset`, `next_offset` (null on the last + page — pass it back as `offset` to read on), `total` characters, and, on + the first page only, `links`: the repo / model / paper / docs URLs the + article body links, grouped. Fetch the page only when this is empty, a + teaser, or cut off before the part you need. + """ + require_write() + with Session(engine) as session: + try: + return curation.get_content(session, url, offset, limit) + except CandidateError as exc: + raise ToolError(str(exc)) diff --git a/backend/app/entry/mcp/list_candidates.py b/backend/app/entry/mcp/list_candidates.py new file mode 100644 index 00000000..1de8afa9 --- /dev/null +++ b/backend/app/entry/mcp/list_candidates.py @@ -0,0 +1,34 @@ +"""The `list_candidates` tool: one page of the pending queue.""" + +from __future__ import annotations + +from sqlmodel import Session + +from app.core.db.engine import engine +from app.entry.mcp.auth import require_write +from app.features.curation import service as curation +from app.features.curation.model import CandidatePage + + +def list_candidates(offset: int = 0, limit: int = curation.LIST_LIMIT) -> CandidatePage: + """List one page of the articles waiting on a curation verdict, freshest first. + + Returns `rows`, `offset`, `next_offset` (null on the last page — pass it + back as `offset` for the next one) and `total` pending. Pages hold at most + 50 rows. Read every page before settling any candidate: a verdict removes + its row and shifts the pages after it. + + One row per candidate: `url`, `title`, `source` (where the link was found), + `publisher`, `publisher_kind` (`unknown` for a site the pipeline has just + created a publisher for: `approve` then needs one), `approved` (the + publisher's approvals / decisions over the + last 90 days, e.g. "3/10", or "new" when it has none), `traction`, + `published_at`, `queued_at`, and a 200-character `snippet`. Deliberately not the full text — call + `get_content` for a candidate worth a closer look. + + A candidate stays listed until `approve` or `reject` is called on it, so a + session that stops halfway leaves the rest for the next one. + """ + require_write() + with Session(engine) as session: + return curation.list_candidates(session, offset, limit) diff --git a/backend/app/entry/mcp/reject.py b/backend/app/entry/mcp/reject.py new file mode 100644 index 00000000..1f03d6cb --- /dev/null +++ b/backend/app/entry/mcp/reject.py @@ -0,0 +1,27 @@ +"""The `reject` tool: turn one candidate down.""" + +from __future__ import annotations + +from fastmcp.exceptions import ToolError +from sqlmodel import Session + +from app.core.db.engine import engine +from app.entry.mcp.auth import require_write +from app.features.curation import service as curation +from app.features.curation.model import CandidateError + + +def reject(url: str, score: int, reason: str) -> str: + """Turn a candidate down, keeping the score and the reason for it. + + The row stays in the ledger so the URL is never judged twice, and so the + verdicts can be read back as labels. Write `reason` for a human reading a + hundred of them later, not for a log line. + """ + require_write() + with Session(engine) as session: + try: + curation.reject(session, url, score, reason) + except CandidateError as exc: + raise ToolError(str(exc)) + return f"Rejected [{score}/100] {url}" diff --git a/backend/app/entry/mcp/reject_many.py b/backend/app/entry/mcp/reject_many.py new file mode 100644 index 00000000..7133347d --- /dev/null +++ b/backend/app/entry/mcp/reject_many.py @@ -0,0 +1,23 @@ +"""The `reject_many` tool: turn many candidates down, each on its own.""" + +from __future__ import annotations + +from sqlmodel import Session + +from app.core.db.engine import engine +from app.entry.mcp.auth import require_write +from app.features.curation import service as curation +from app.features.curation.model import Verdict + + +def reject_many(verdicts: list[Verdict]) -> str: + """Turn down many candidates at once: each of `verdicts` is `{url, score, + reason}`, the same fields `reject` takes. + + Each is settled on its own, so one URL that is not pending costs that line + and not the batch. Returns one line per verdict, in order, saying whether + it was rejected or why not. + """ + require_write() + with Session(engine) as session: + return "\n".join(curation.reject_many(session, verdicts)) diff --git a/backend/app/entry/mcp/related.py b/backend/app/entry/mcp/related.py new file mode 100644 index 00000000..a1008117 --- /dev/null +++ b/backend/app/entry/mcp/related.py @@ -0,0 +1,32 @@ +"""The `related` tool: what else covers one published article's subject.""" + +from __future__ import annotations + +from fastmcp.exceptions import ToolError +from sqlmodel import Session + +from app.core.db.engine import engine +from app.entry.mcp.auth import require_write +from app.features.newsletter import service as newsletter +from app.features.newsletter.model import DigestError, Related + + +def related( + url: str, days: int = newsletter.RELATED_DAYS, limit: int = newsletter.RELATED_LIMIT +) -> Related: + """What else covers one published article's subject: the individual writer + who tried it, the repo built on it, the commentary on it. + + Compares by embedding against every article from the last `days` and every + reject from that window that scored 35 or more. Up to `limit` rows, the + closest first: `url`, `title`, `publisher`, `publisher_kind`, `kind`, + `stage` (`published` or `rejected`), `score`, `distance` (under 0.30 is the + same story, up to 0.45 is the same subject), and `summary` for a published + row or curation's `reason` for a rejected one. + """ + require_write() + with Session(engine) as session: + try: + return newsletter.related(session, url, days, limit) + except DigestError as exc: + raise ToolError(str(exc)) diff --git a/backend/app/entry/mcp/save_story.py b/backend/app/entry/mcp/save_story.py new file mode 100644 index 00000000..c75e3bc8 --- /dev/null +++ b/backend/app/entry/mcp/save_story.py @@ -0,0 +1,38 @@ +"""The `save_story` tool: create a story or extend an open one. + +A saved story is on the public site at once (ADR 16). +""" + +from __future__ import annotations + +from fastmcp.exceptions import ToolError +from sqlmodel import Session + +from app.core.db.engine import engine +from app.entry.mcp.auth import require_write +from app.features.stories import service as story_domain +from app.features.stories.model import StoryError + + +def save_story(slug: str, name: str, blurb: str, urls: list[str]) -> str: + """Create a story, or update an open one: set its `name` and `blurb` and + add `urls` to it (ones already in it are skipped). + + `slug` is lowercase words joined by hyphens and never changes. `name` is + at most 4 words, `blurb` at most 2 sentences and 280 characters. Every URL + must be a published article and in no other open story. Any problem + refuses the whole call, saying which, and nothing is written. A closed + story cannot be saved to. The rail shows a story once it has 3 articles + from 2 publishers. + """ + require_write() + with Session(engine) as session: + try: + saved = story_domain.save_story(session, slug, name, blurb, urls) + except StoryError as exc: + raise ToolError(str(exc)) + verb = "Created" if saved["created"] else "Updated" + return ( + f"{verb} {saved['slug']}: added {saved['added']}, " + f"{saved['article_count']} articles in all" + ) diff --git a/backend/app/entry/mcp/server.py b/backend/app/entry/mcp/server.py new file mode 100644 index 00000000..ef11f5dc --- /dev/null +++ b/backend/app/entry/mcp/server.py @@ -0,0 +1,71 @@ +"""The MCP server: the instance, its instructions, its tool list, and the ASGI app `main` mounts.""" + +from __future__ import annotations + +from fastmcp import FastMCP + +from app.entry.mcp.approve import approve +from app.entry.mcp.auth import SharedSecret +from app.entry.mcp.check_link import check_link +from app.entry.mcp.close_story import close_story +from app.entry.mcp.draft_issue import draft_issue +from app.entry.mcp.get_content import get_content +from app.entry.mcp.list_candidates import list_candidates +from app.entry.mcp.reject import reject +from app.entry.mcp.reject_many import reject_many +from app.entry.mcp.related import related +from app.entry.mcp.save_story import save_story +from app.entry.mcp.similar import similar +from app.entry.mcp.sql_query import sql_query +from app.entry.mcp.stories import stories +from app.entry.mcp.story_candidates import story_candidates +from app.entry.mcp.vocabulary import vocabulary +from app.entry.mcp.web_fetch import web_fetch +from app.entry.mcp.web_search import web_search +from app.entry.mcp.week import week + +mcp = FastMCP( + name="agentique", + instructions=( + "Tools for the agentique article pipeline. Reading: `sql_query` reads " + "the Postgres database (publishers, articles, scores, pipeline runs), " + "`web_fetch` reads one page, `web_search` finds pages. Curation, with " + "the write token only: `list_candidates` lists what is waiting on a " + "verdict, `get_content` pages through the stored text for one of them, " + "`similar` and `stories` compare candidates with the feed, " + "`check_link` looks up a repo or model, `vocabulary` lists the labels, " + "and `approve`, `reject` and `reject_many` settle them. Weekly " + "newsletter, with the write token only: `week` ranks the week's " + "topics, `related` finds what else covers one, and `draft_issue` " + "drafts an issue for a person to send and says where to save it as a " + "blog post. Stories, with the write token only: `story_candidates` " + "lists open stories and unassigned clusters, `save_story` creates or " + "extends one, `close_story` closes one." + ), + auth=SharedSecret(), + tools=[ + sql_query, + web_fetch, + web_search, + list_candidates, + get_content, + similar, + stories, + check_link, + vocabulary, + approve, + reject, + reject_many, + week, + related, + draft_issue, + story_candidates, + save_story, + close_story, + ], +) + +# Mounted at /mcp by `app.main`, so the route inside this app is the bare root. +# Its lifespan starts the streamable-HTTP session manager and has to be handed +# to the parent app -- a mounted ASGI app never gets a lifespan of its own. +mcp_app = mcp.http_app(path="/") diff --git a/backend/app/entry/mcp/similar.py b/backend/app/entry/mcp/similar.py new file mode 100644 index 00000000..ccecb097 --- /dev/null +++ b/backend/app/entry/mcp/similar.py @@ -0,0 +1,29 @@ +"""The `similar` tool: what the feed and the ledger hold close to one candidate.""" + +from __future__ import annotations + +from fastmcp.exceptions import ToolError +from sqlmodel import Session + +from app.core.db.engine import engine +from app.entry.mcp.auth import require_write +from app.features.curation import service as curation +from app.features.curation.model import CandidateError, Similar + + +def similar(url: str, days: int = 7, limit: int = 8) -> Similar: + """Show what the feed and the ledger hold that is close to one candidate. + + Compares by embedding against every published article and every candidate + or reject from the last `days`. `rows` lists up to `limit` neighbours, the + closest first: `url`, `title`, `publisher`, `stage` (`published`, + `pending`, or the reject stage), `score` and `distance` (under 0.30 is the + same story). `coverage` is how many distinct publishers carry this story, + this candidate's own included, with their names in `publishers`. + """ + require_write() + with Session(engine) as session: + try: + return curation.similar(session, url, days, limit) + except CandidateError as exc: + raise ToolError(str(exc)) diff --git a/backend/app/entry/mcp/sql_query.py b/backend/app/entry/mcp/sql_query.py new file mode 100644 index 00000000..dec74d1b --- /dev/null +++ b/backend/app/entry/mcp/sql_query.py @@ -0,0 +1,133 @@ +"""The `sql_query` tool: one read-only statement against the agentique database. + +Reachable with the read token. What keeps it read-only is the role it logs in +as, a read-only transaction, a prepared statement, and a plan check that +refuses the tables no question needs. +""" + +from __future__ import annotations + +import json +from collections.abc import Iterator +from datetime import date, datetime + +import psycopg +from fastmcp.exceptions import ToolError + +from app.core.settings import settings + +# A cap, not a page size: an agent that wants more should narrow the query or +# add its own LIMIT. Anything larger is context an agent cannot read anyway. +MAX_ROWS = 1000 + +# `user` holds emails and sign-in link tokens, and no question worth asking this +# tool needs either. The role's grants are the real boundary (see +# deploy/sql-roles.sql); the plan check below holds even where the tool runs as +# a role that has the grant anyway, which is every local database. +BLOCKED_TABLES = frozenset({"user"}) + +# Its own connection, not `app.core.db` — that one logs in as POSTGRES_USER +# and would happily run a DELETE. One per call rather than a pool: the cost is +# nothing at this call rate, and no session state survives a query. +DSN = settings.MCP_DATABASE_URI + + +def _json_safe(value: object) -> str: + """Fallback for the column types `json.dumps` has no encoding for. + + Timestamps get ISO-8601; Decimal, UUID and pgvector embeddings get `str`, + which is lossless for all three and keeps one unreadable column from + failing the whole query. + """ + if isinstance(value, datetime | date): + return value.isoformat() + return str(value) + + +def _plan_tables(node: object) -> Iterator[str]: + """Every table an EXPLAIN plan reads, however deeply nested. + + Walks the whole JSON rather than the top node: the table can sit under a + join, a subplan or a CTE. Views are expanded during planning, so what comes + back is the tables actually read, not the names the query used. + """ + if isinstance(node, dict): + name = node.get("Relation Name") + if isinstance(name, str): + yield name + for value in node.values(): + yield from _plan_tables(value) + elif isinstance(node, list): + for item in node: + yield from _plan_tables(item) + + +def _reject_blocked_tables( + cursor: psycopg.Cursor[tuple[object, ...]], sql: str +) -> None: + """Plan the statement, and refuse it if the plan reads a blocked table. + + EXPLAIN reports the tables actually read -- views expanded, joins, CTEs and + subplans included -- and refusing to plan is what keeps this a query tool, + since COPY, SET and DO never get that far. + + `prepare` is what makes the check honest. Given no parameters psycopg falls + back to the simple query protocol, which runs every command in the string, + so `SELECT 1; SELECT * FROM "user"` would be two commands and only the + first would be the one planned here. A prepared statement takes exactly + one, so the server rejects the second before anything executes. + + psycopg types a `str` query as LiteralString to stop callers interpolating + input into SQL. Doing precisely that is this tool's job, hence the + suppression; what keeps it safe is the role, this check, and `prepare`. + """ + cursor.execute(f"EXPLAIN (FORMAT JSON) {sql}", prepare=True) # ty: ignore[no-matching-overload] + plan = cursor.fetchone() + blocked = BLOCKED_TABLES.intersection(_plan_tables(plan[0] if plan else None)) + if blocked: + raise ToolError(f"Not readable through this tool: {', '.join(sorted(blocked))}") + + +def sql_query(sql: str) -> str: + """Run a read-only SQL query against the agentique Postgres database. + + Accepts one statement, and only one Postgres can plan as a query. The + connection is a read-only transaction held by a role with SELECT grants + only, so writes and DDL fail rather than being filtered out beforehand, and + statements are cancelled after 10 seconds. The `user` table is not + readable, directly or through a join. A rejected statement comes back as + Postgres's own error message. + + Returns JSON: `columns` (in select order), `rows` (a list of objects), and + `truncated` (true when the result hit the 1000-row cap and rows are + missing). Timestamps are ISO-8601 strings; numerics, UUIDs and embedding + vectors are stringified. + """ + try: + with psycopg.connect(DSN) as conn, conn.cursor() as cursor: + _reject_blocked_tables(cursor, sql) + + cursor.execute(sql, prepare=True) # ty: ignore[no-matching-overload] + if cursor.description is None: + # Unreachable while the plan check stands -- nothing that gets + # past it returns without a result set. Kept as an error rather + # than an empty result, which would read as "no rows found". + raise ToolError("Statement returned no result set") + columns = [column.name for column in cursor.description] + # One past the cap, so a full page can be told from an exact fit. + rows = cursor.fetchmany(MAX_ROWS + 1) + return json.dumps( + { + "columns": columns, + "rows": [ + dict(zip(columns, row, strict=True)) for row in rows[:MAX_ROWS] + ], + "truncated": len(rows) > MAX_ROWS, + }, + default=_json_safe, + ) + except psycopg.Error as exc: + # The agent wrote this SQL and can fix it, but only if it sees what + # Postgres actually said; anything but a ToolError reaches the client + # as a bare "internal error". + raise ToolError(str(exc)) diff --git a/backend/app/entry/mcp/stories.py b/backend/app/entry/mcp/stories.py new file mode 100644 index 00000000..45220a5e --- /dev/null +++ b/backend/app/entry/mcp/stories.py @@ -0,0 +1,29 @@ +"""The `stories` tool: pending candidates grouped by story. + +Curation coverage, not the right rail's stories: the name is the curate +rubric's, and the rail's tools are `story_candidates`, `save_story` and +`close_story`. +""" + +from __future__ import annotations + +from sqlmodel import Session + +from app.core.db.engine import engine +from app.entry.mcp.auth import require_write +from app.features.curation import service as curation +from app.features.curation.model import Coverage + + +def stories(days: int = 7) -> list[Coverage]: + """Group the pending candidates that are one story, with each other or with + an article published in the last `days`. + + Only groups of two or more come back, largest first, each with its + `members` (same row shape as `similar`, without the distance), `coverage` + (distinct publishers) and `publishers`. A member at stage `published` + means the feed already carries the story. Call it once, after triage. + """ + require_write() + with Session(engine) as session: + return curation.stories(session, days) diff --git a/backend/app/entry/mcp/story_candidates.py b/backend/app/entry/mcp/story_candidates.py new file mode 100644 index 00000000..1ceed0af --- /dev/null +++ b/backend/app/entry/mcp/story_candidates.py @@ -0,0 +1,28 @@ +"""The `story_candidates` tool: open stories and the clusters no story holds yet.""" + +from __future__ import annotations + +from sqlmodel import Session + +from app.core.db.engine import engine +from app.entry.mcp.auth import require_write +from app.features.stories import service as story_domain +from app.features.stories.model import Candidates + + +def story_candidates(days: int = 14) -> Candidates: + """What the stories routine decides from. + + `open`: every open story, the most recently active first, with `slug`, + `name`, `blurb`, `article_count`, `quiet_days` (since its newest article + was published) and its newest 5 `articles`. `clusters`: the last `days` + of published articles that sit in no open story, grouped by embedding at + a looser cut than coverage, the most publishers first. Each cluster has + `articles` (`url`, `title`, `publisher`, `published_at`, `score`, + `summary`), its `publishers`, and `near`: the open stories one of its + articles is close to. A single article close to a story comes back as a + cluster of one with that story in `near`. + """ + require_write() + with Session(engine) as session: + return story_domain.story_candidates(session, days) diff --git a/backend/app/entry/mcp/test_approve.py b/backend/app/entry/mcp/test_approve.py new file mode 100644 index 00000000..f6e057bd --- /dev/null +++ b/backend/app/entry/mcp/test_approve.py @@ -0,0 +1,40 @@ +"""The `approve` tool: what it reports, and a refused label as a tool error.""" + +from __future__ import annotations + +import pytest +from fastmcp.exceptions import ToolError +from sqlmodel import Session + +from app.core.db.ledger import Reject, RejectStage +from app.entry.mcp.approve import approve +from app.features.curation import service as curation + + +@pytest.mark.usefixtures("curator") +def test_approve_reports_the_article_it_published( + monkeypatch: pytest.MonkeyPatch, +) -> None: + calls: list[tuple[object, ...]] = [] + + def fake(_session: Session, *args: object) -> int: + calls.append(args) + return 42 + + monkeypatch.setattr(curation, "approve", fake) + url = "https://example.com/post" + assert approve(url, 81, "r", "s", "post", ["agents"], "media", "Example") == ( + f"Published article #42: {url}" + ) + assert calls == [(url, 81, "r", "s", "post", ["agents"], "media", "Example")] + + +@pytest.mark.usefixtures("curator") +def test_a_refused_label_is_a_tool_error_and_the_candidate_stays( + db: Session, candidate: Reject +) -> None: + with pytest.raises(ToolError, match="publisher_kind"): + approve(candidate.url, 81, "r", "s", "post", []) + db.expire_all() + row = db.get(Reject, candidate.url) + assert row is not None and row.stage == RejectStage.pending diff --git a/backend/app/mcp/tests/test_curation_tools.py b/backend/app/entry/mcp/test_auth.py similarity index 56% rename from backend/app/mcp/tests/test_curation_tools.py rename to backend/app/entry/mcp/test_auth.py index 58bc8c0f..289be5d9 100644 --- a/backend/app/mcp/tests/test_curation_tools.py +++ b/backend/app/entry/mcp/test_auth.py @@ -7,13 +7,29 @@ from __future__ import annotations +from collections.abc import Callable + import pytest from fastmcp.exceptions import ToolError from fastmcp.server.auth import AccessToken from app.core.settings import settings -from app.mcp import tools -from app.mcp.server import SharedSecret +from app.entry.mcp import auth +from app.entry.mcp.approve import approve +from app.entry.mcp.check_link import check_link +from app.entry.mcp.close_story import close_story +from app.entry.mcp.draft_issue import draft_issue +from app.entry.mcp.get_content import get_content +from app.entry.mcp.list_candidates import list_candidates +from app.entry.mcp.reject import reject +from app.entry.mcp.reject_many import reject_many +from app.entry.mcp.related import related +from app.entry.mcp.save_story import save_story +from app.entry.mcp.similar import similar +from app.entry.mcp.stories import stories +from app.entry.mcp.story_candidates import story_candidates +from app.entry.mcp.vocabulary import vocabulary +from app.entry.mcp.week import week WRITE_TOOLS = ( "list_candidates", @@ -37,43 +53,44 @@ def _as(monkeypatch: pytest.MonkeyPatch, *scopes: str) -> None: """Answer the tools' scope check as a caller holding ``scopes``.""" monkeypatch.setattr( - tools, + auth, "get_access_token", lambda: AccessToken(token="t", client_id="c", scopes=list(scopes)), ) -def _call(name: str): +def _call(name: str) -> object: """Call one curation tool with throwaway arguments. The gate is the first thing each one does, so the arguments never matter: a caller without the scope is refused before any of them is read. """ - return { - "list_candidates": lambda: tools.list_candidates(), - "get_content": lambda: tools.get_content("https://example.com"), - "similar": lambda: tools.similar("https://example.com"), - "stories": lambda: tools.stories(), - "check_link": lambda: tools.check_link("https://github.com/a/b"), - "vocabulary": lambda: tools.vocabulary(), - "approve": lambda: tools.approve( - "https://example.com", 80, "r", "s", "post", [] - ), - "reject": lambda: tools.reject("https://example.com", 20, "r"), - "reject_many": lambda: tools.reject_many( + calls: dict[str, Callable[[], object]] = { + "list_candidates": lambda: list_candidates(), + "get_content": lambda: get_content("https://example.com"), + "similar": lambda: similar("https://example.com"), + "stories": lambda: stories(), + "check_link": lambda: check_link("https://github.com/a/b"), + "vocabulary": lambda: vocabulary(), + "approve": lambda: approve("https://example.com", 80, "r", "s", "post", []), + "reject": lambda: reject("https://example.com", 20, "r"), + "reject_many": lambda: reject_many( [{"url": "https://example.com", "score": 20, "reason": "r"}] ), - "week": lambda: tools.week(), - "related": lambda: tools.related("https://example.com"), - "draft_issue": lambda: tools.draft_issue("---\ntitle: t\n---\nb"), - "story_candidates": lambda: tools.story_candidates(), - "save_story": lambda: tools.save_story("s", "n", "b", []), - "close_story": lambda: tools.close_story("s"), - }[name]() + "week": lambda: week(), + "related": lambda: related("https://example.com"), + "draft_issue": lambda: draft_issue("---\ntitle: t\n---\nb"), + "story_candidates": lambda: story_candidates(), + "save_story": lambda: save_story("s", "n", "b", []), + "close_story": lambda: close_story("s"), + } + return calls[name]() @pytest.mark.parametrize("name", WRITE_TOOLS) -def test_the_read_token_cannot_curate(monkeypatch: pytest.MonkeyPatch, name: str): +def test_the_read_token_cannot_curate( + monkeypatch: pytest.MonkeyPatch, name: str +) -> None: _as(monkeypatch) with pytest.raises(ToolError, match="curation token"): _call(name) @@ -82,48 +99,52 @@ def test_the_read_token_cannot_curate(monkeypatch: pytest.MonkeyPatch, name: str @pytest.mark.parametrize("name", WRITE_TOOLS) def test_an_unauthenticated_call_cannot_curate( monkeypatch: pytest.MonkeyPatch, name: str -): - monkeypatch.setattr(tools, "get_access_token", lambda: None) +) -> None: + monkeypatch.setattr(auth, "get_access_token", lambda: None) with pytest.raises(ToolError, match="curation token"): _call(name) -def test_the_write_scope_gets_past_the_gate(monkeypatch: pytest.MonkeyPatch): +def test_the_write_scope_gets_past_the_gate(monkeypatch: pytest.MonkeyPatch) -> None: """Past the gate it is an ordinary lookup, and a URL nobody queued is a message rather than a traceback.""" - _as(monkeypatch, tools.WRITE_SCOPE) + _as(monkeypatch, auth.WRITE_SCOPE) with pytest.raises(ToolError, match="No candidate for"): - tools.get_content("https://never-queued.example") + get_content("https://never-queued.example") @pytest.mark.anyio -async def test_only_the_write_token_carries_the_scope(monkeypatch: pytest.MonkeyPatch): +async def test_only_the_write_token_carries_the_scope( + monkeypatch: pytest.MonkeyPatch, +) -> None: monkeypatch.setattr(settings, "MCP_TOKEN", "read-token") monkeypatch.setattr(settings, "MCP_WRITE_TOKEN", "write-token") - verifier = SharedSecret() + verifier = auth.SharedSecret() write = await verifier.verify_token("write-token") read = await verifier.verify_token("read-token") - assert write is not None and write.scopes == [tools.WRITE_SCOPE] + assert write is not None and write.scopes == [auth.WRITE_SCOPE] assert read is not None and read.scopes == [] assert await verifier.verify_token("neither") is None @pytest.mark.anyio -async def test_an_unset_write_token_matches_nothing(monkeypatch: pytest.MonkeyPatch): +async def test_an_unset_write_token_matches_nothing( + monkeypatch: pytest.MonkeyPatch, +) -> None: """A deploy that forgets the variable must curate nothing, not let the read token publish — and an empty header must not match an empty setting.""" monkeypatch.setattr(settings, "MCP_TOKEN", "read-token") monkeypatch.setattr(settings, "MCP_WRITE_TOKEN", None) - verifier = SharedSecret() + verifier = auth.SharedSecret() assert await verifier.verify_token("") is None read = await verifier.verify_token("read-token") assert read is not None and read.scopes == [] -def test_the_two_tokens_cannot_be_the_same_value(): +def test_the_two_tokens_cannot_be_the_same_value() -> None: """The verifier checks the write token first, so a copy-paste that sets both to one value hands publishing rights to every holder of what the operator believes is a read-only credential. Nothing downstream would notice.""" @@ -140,25 +161,3 @@ def test_the_two_tokens_cannot_be_the_same_value(): @pytest.fixture def anyio_backend() -> str: return "asyncio" - - -def test_check_link_refuses_what_it_does_not_read(monkeypatch: pytest.MonkeyPatch): - _as(monkeypatch, tools.WRITE_SCOPE) - with pytest.raises(ToolError, match="web_fetch"): - tools.check_link("https://arxiv.org/abs/2609.01234") - - -def test_reject_many_reports_one_line_per_verdict(monkeypatch: pytest.MonkeyPatch): - _as(monkeypatch, tools.WRITE_SCOPE) - result = tools.reject_many( - [ - {"url": "https://never-queued.example/1", "score": 20, "reason": "r"}, - {"url": "https://never-queued.example/2", "score": 20, "reason": "r"}, - ] - ) - assert result.splitlines() == [ - "Skipped https://never-queued.example/1: " - "No candidate for https://never-queued.example/1", - "Skipped https://never-queued.example/2: " - "No candidate for https://never-queued.example/2", - ] diff --git a/backend/app/entry/mcp/test_check_link.py b/backend/app/entry/mcp/test_check_link.py new file mode 100644 index 00000000..d619e401 --- /dev/null +++ b/backend/app/entry/mcp/test_check_link.py @@ -0,0 +1,14 @@ +"""The `check_link` tool.""" + +from __future__ import annotations + +import pytest +from fastmcp.exceptions import ToolError + +from app.entry.mcp.check_link import check_link + + +@pytest.mark.usefixtures("curator") +def test_check_link_refuses_what_it_does_not_read() -> None: + with pytest.raises(ToolError, match="web_fetch"): + check_link("https://arxiv.org/abs/2609.01234") diff --git a/backend/app/entry/mcp/test_draft_issue.py b/backend/app/entry/mcp/test_draft_issue.py new file mode 100644 index 00000000..45d263a2 --- /dev/null +++ b/backend/app/entry/mcp/test_draft_issue.py @@ -0,0 +1,14 @@ +"""The `draft_issue` tool.""" + +from __future__ import annotations + +import pytest +from fastmcp.exceptions import ToolError + +from app.entry.mcp.draft_issue import draft_issue + + +@pytest.mark.usefixtures("curator") +def test_a_post_without_frontmatter_is_a_tool_error() -> None: + with pytest.raises(ToolError, match="frontmatter"): + draft_issue("Just an essay, no frontmatter.") diff --git a/backend/app/entry/mcp/test_reject.py b/backend/app/entry/mcp/test_reject.py new file mode 100644 index 00000000..2dfd583b --- /dev/null +++ b/backend/app/entry/mcp/test_reject.py @@ -0,0 +1,31 @@ +"""The `reject` tool: the verdict it writes, and an unknown URL as a tool error.""" + +from __future__ import annotations + +import pytest +from fastmcp.exceptions import ToolError +from sqlmodel import Session + +from app.core.db.ledger import Reject, RejectStage +from app.entry.mcp.reject import reject + + +@pytest.mark.usefixtures("curator") +def test_reject_settles_the_candidate(db: Session, candidate: Reject) -> None: + assert reject(candidate.url, 22, "A vendor tutorial.") == ( + f"Rejected [22/100] {candidate.url}" + ) + db.expire_all() + row = db.get(Reject, candidate.url) + assert row is not None + assert (row.stage, row.score, row.reason) == ( + RejectStage.below_threshold, + 22, + "A vendor tutorial.", + ) + + +@pytest.mark.usefixtures("curator") +def test_reject_refuses_a_url_nobody_queued() -> None: + with pytest.raises(ToolError, match="No candidate for"): + reject("https://never-queued.example/post", 22, "r") diff --git a/backend/app/entry/mcp/test_reject_many.py b/backend/app/entry/mcp/test_reject_many.py new file mode 100644 index 00000000..307366ef --- /dev/null +++ b/backend/app/entry/mcp/test_reject_many.py @@ -0,0 +1,23 @@ +"""The `reject_many` tool.""" + +from __future__ import annotations + +import pytest + +from app.entry.mcp.reject_many import reject_many + + +@pytest.mark.usefixtures("curator") +def test_reject_many_reports_one_line_per_verdict() -> None: + result = reject_many( + [ + {"url": "https://never-queued.example/1", "score": 20, "reason": "r"}, + {"url": "https://never-queued.example/2", "score": 20, "reason": "r"}, + ] + ) + assert result.splitlines() == [ + "Skipped https://never-queued.example/1: " + "No candidate for https://never-queued.example/1", + "Skipped https://never-queued.example/2: " + "No candidate for https://never-queued.example/2", + ] diff --git a/backend/app/entry/mcp/test_related.py b/backend/app/entry/mcp/test_related.py new file mode 100644 index 00000000..070ed11b --- /dev/null +++ b/backend/app/entry/mcp/test_related.py @@ -0,0 +1,14 @@ +"""The `related` tool.""" + +from __future__ import annotations + +import pytest +from fastmcp.exceptions import ToolError + +from app.entry.mcp.related import related + + +@pytest.mark.usefixtures("curator") +def test_related_refuses_a_url_that_is_not_published() -> None: + with pytest.raises(ToolError, match="No published article"): + related("https://never-published.example/post") diff --git a/backend/app/entry/mcp/test_save_story.py b/backend/app/entry/mcp/test_save_story.py new file mode 100644 index 00000000..96558852 --- /dev/null +++ b/backend/app/entry/mcp/test_save_story.py @@ -0,0 +1,54 @@ +"""The `save_story` and `close_story` tools: what they report, and a refusal as +a tool error.""" + +from __future__ import annotations + +import pytest +from fastmcp.exceptions import ToolError +from sqlmodel import Session + +from app.core.db.factories import ( + create_random_article, + create_random_publisher, + random_lower_string, +) +from app.entry.mcp.close_story import close_story +from app.entry.mcp.save_story import save_story + + +def _url(db: Session, made: dict[str, list[int]]) -> str: + publisher = create_random_publisher(db) + article = create_random_article(db, publisher_id=publisher.id) + assert publisher.id is not None and article.id is not None + made["publishers"].append(publisher.id) + made["articles"].append(article.id) + return article.url + + +@pytest.mark.usefixtures("curator") +def test_save_story_tool_reports_what_it_did( + db: Session, made: dict[str, list[int]] +) -> None: + slug = f"tool-made-{random_lower_string()[:6]}" + assert save_story(slug, "Tool made", "It works.", [_url(db, made)]) == ( + f"Created {slug}: added 1, 1 articles in all" + ) + assert save_story(slug, "Tool made", "It still works.", [_url(db, made)]) == ( + f"Updated {slug}: added 1, 2 articles in all" + ) + assert close_story(slug) == f"Closed {slug}" + + +@pytest.mark.usefixtures("curator") +def test_save_story_refuses_an_unknown_url_as_a_tool_error() -> None: + slug = f"tool-refused-{random_lower_string()[:6]}" + with pytest.raises(ToolError, match="Not a published article"): + save_story( + slug, "Tool refused", "It fails.", ["https://never-published.example/post"] + ) + + +@pytest.mark.usefixtures("curator") +def test_close_story_refuses_an_unknown_slug_as_a_tool_error() -> None: + with pytest.raises(ToolError, match="No story"): + close_story(f"no-such-story-{random_lower_string()[:6]}") diff --git a/backend/app/entry/mcp/test_server.py b/backend/app/entry/mcp/test_server.py new file mode 100644 index 00000000..d0b36e2a --- /dev/null +++ b/backend/app/entry/mcp/test_server.py @@ -0,0 +1,149 @@ +"""The bearer-token gate on the mounted MCP endpoint, end to end over HTTP.""" + +from __future__ import annotations + +import json + +import pytest +from fastapi.testclient import TestClient + +from app.core.settings import settings + +MCP_URL = "/mcp/" +HEADERS = {"Accept": "application/json, text/event-stream"} +INITIALIZE = { + "jsonrpc": "2.0", + "id": 1, + "method": "initialize", + "params": { + "protocolVersion": "2025-06-18", + "capabilities": {}, + "clientInfo": {"name": "test", "version": "1"}, + }, +} + + +@pytest.fixture +def token(monkeypatch: pytest.MonkeyPatch) -> str: + monkeypatch.setattr(settings, "MCP_TOKEN", "test-mcp-token") + return "test-mcp-token" + + +def test_the_right_token_gets_a_session(client: TestClient, token: str) -> None: + response = client.post( + MCP_URL, + json=INITIALIZE, + headers={**HEADERS, "Authorization": f"Bearer {token}"}, + ) + assert response.status_code == 200 + + +@pytest.mark.usefixtures("token") +def test_a_wrong_token_is_rejected(client: TestClient) -> None: + response = client.post( + MCP_URL, json=INITIALIZE, headers={**HEADERS, "Authorization": "Bearer nope"} + ) + assert response.status_code == 401 + + +@pytest.mark.usefixtures("token") +def test_no_token_is_rejected(client: TestClient) -> None: + response = client.post(MCP_URL, json=INITIALIZE, headers=HEADERS) + assert response.status_code == 401 + + +def test_an_unconfigured_server_rejects_every_token( + client: TestClient, monkeypatch: pytest.MonkeyPatch +) -> None: + # A deploy that forgets MCP_TOKEN must serve 401s, not an open database. + monkeypatch.setattr(settings, "MCP_TOKEN", None) + response = client.post( + MCP_URL, json=INITIALIZE, headers={**HEADERS, "Authorization": "Bearer "} + ) + assert response.status_code == 401 + + +READ_TOKEN = "test-read-token" +WRITE_TOKEN = "test-write-token" + + +@pytest.fixture +def both_tokens(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(settings, "MCP_TOKEN", READ_TOKEN) + monkeypatch.setattr(settings, "MCP_WRITE_TOKEN", WRITE_TOKEN) + + +def _call_tool( + client: TestClient, token: str, name: str, arguments: dict[str, object] +) -> tuple[bool, str]: + """Open a session with ``token`` and call one tool, as an MCP client would. + + Returns the result's ``isError`` and its text. A refused tool call is still + a 200: the gate inside the tool answers, not the HTTP layer. + """ + headers = {**HEADERS, "Authorization": f"Bearer {token}"} + opened = client.post(MCP_URL, json=INITIALIZE, headers=headers) + assert opened.status_code == 200 + headers["mcp-session-id"] = opened.headers["mcp-session-id"] + client.post( + MCP_URL, + json={"jsonrpc": "2.0", "method": "notifications/initialized"}, + headers=headers, + ) + response = client.post( + MCP_URL, + json={ + "jsonrpc": "2.0", + "id": 2, + "method": "tools/call", + "params": {"name": name, "arguments": arguments}, + }, + headers=headers, + ) + assert response.status_code == 200 + data = next( + line.removeprefix("data: ") + for line in response.text.splitlines() + if line.startswith("data: ") + ) + result = json.loads(data)["result"] + return result["isError"], result["content"][0]["text"] + + +@pytest.mark.usefixtures("both_tokens") +def test_the_read_token_is_refused_by_a_write_tool(client: TestClient) -> None: + is_error, text = _call_tool( + client, READ_TOKEN, "get_content", {"url": "https://never-queued.example"} + ) + assert is_error + assert "curation token" in text + + +@pytest.mark.usefixtures("both_tokens") +def test_the_write_token_gets_past_the_gate(client: TestClient) -> None: + # Past the gate it is an ordinary lookup, so the refusal is the candidate's. + is_error, text = _call_tool( + client, WRITE_TOKEN, "get_content", {"url": "https://never-queued.example"} + ) + assert is_error + assert "No candidate for" in text + + +def test_an_unset_write_token_opens_no_write_tool( + client: TestClient, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(settings, "MCP_TOKEN", READ_TOKEN) + monkeypatch.setattr(settings, "MCP_WRITE_TOKEN", None) + + stale = client.post( + MCP_URL, + json=INITIALIZE, + headers={**HEADERS, "Authorization": f"Bearer {WRITE_TOKEN}"}, + ) + assert stale.status_code == 401 + + is_error, text = _call_tool( + client, READ_TOKEN, "get_content", {"url": "https://never-queued.example"} + ) + assert is_error + assert "curation token" in text diff --git a/backend/app/entry/mcp/test_similar.py b/backend/app/entry/mcp/test_similar.py new file mode 100644 index 00000000..a9ca7454 --- /dev/null +++ b/backend/app/entry/mcp/test_similar.py @@ -0,0 +1,14 @@ +"""The `similar` tool.""" + +from __future__ import annotations + +import pytest +from fastmcp.exceptions import ToolError + +from app.entry.mcp.similar import similar + + +@pytest.mark.usefixtures("curator") +def test_similar_refuses_a_url_nobody_queued() -> None: + with pytest.raises(ToolError, match="No candidate or article for"): + similar("https://never-queued.example/post") diff --git a/backend/app/entry/mcp/test_sql_query.py b/backend/app/entry/mcp/test_sql_query.py new file mode 100644 index 00000000..3a6a9070 --- /dev/null +++ b/backend/app/entry/mcp/test_sql_query.py @@ -0,0 +1,106 @@ +"""`sql_query`: the read-only guard, the blocked tables, the row cap, and JSON coercion.""" + +from __future__ import annotations + +import json + +import pytest +from fastmcp.exceptions import ToolError + +from app.core.settings import settings +from app.entry.mcp import sql_query as tool + + +@pytest.fixture(autouse=True) +def _test_dsn(monkeypatch: pytest.MonkeyPatch) -> None: + """Point the tool at the test credentials instead of `agentique_ro`. + + The role does not exist outside the production box, but the DSN is the one + settings build, read-only options and all, so the guard under test is real. + """ + monkeypatch.setattr( + tool, + "DSN", + settings.model_copy( + update={ + "MCP_DB_USER": settings.POSTGRES_USER, + "MCP_DB_PASSWORD": settings.POSTGRES_PASSWORD, + } + ).MCP_DATABASE_URI, + ) + + +def test_select_returns_columns_and_rows() -> None: + result = json.loads(tool.sql_query("SELECT 1 AS n, 'a' AS label")) + assert result == { + "columns": ["n", "label"], + "rows": [{"n": 1, "label": "a"}], + "truncated": False, + } + + +def test_rows_are_capped_and_flagged() -> None: + result = json.loads( + tool.sql_query(f"SELECT i FROM generate_series(1, {tool.MAX_ROWS + 5}) AS i") + ) + assert len(result["rows"]) == tool.MAX_ROWS + assert result["truncated"] is True + + +def test_a_result_that_exactly_fills_the_cap_is_not_truncated() -> None: + result = json.loads( + tool.sql_query(f"SELECT i FROM generate_series(1, {tool.MAX_ROWS}) AS i") + ) + assert len(result["rows"]) == tool.MAX_ROWS + assert result["truncated"] is False + + +def test_timestamps_and_numerics_survive_serialisation() -> None: + result = json.loads( + tool.sql_query( + "SELECT TIMESTAMP '2026-01-02 03:04:05' AS ts, 1.5::numeric AS amount" + ) + ) + assert result["rows"] == [{"ts": "2026-01-02T03:04:05", "amount": "1.5"}] + + +def test_a_write_is_rejected_by_the_read_only_transaction() -> None: + # Not a syntax check on the way in: Postgres refuses it, so anything the + # tool failed to anticipate is refused too. + with pytest.raises(ToolError, match="read-only transaction"): + tool.sql_query("UPDATE article SET title = title WHERE false") + + +def test_anything_postgres_cannot_plan_as_a_query_is_refused() -> None: + # Planning is the filter: COPY, SET, DO and a second statement after a + # semicolon never reach execution. + with pytest.raises(ToolError): + tool.sql_query("SET application_name = 'mcp'") + + +def test_the_user_table_is_not_readable() -> None: + with pytest.raises(ToolError, match="Not readable through this tool: user"): + tool.sql_query('SELECT email FROM "user"') + + +def test_the_user_table_is_not_readable_through_a_subquery() -> None: + # The check walks the whole plan, so hiding it below the top node does not + # help -- nor would a join, a CTE or a view. + with pytest.raises(ToolError, match="Not readable through this tool: user"): + tool.sql_query('SELECT 1 WHERE EXISTS (SELECT 1 FROM "user")') + + +def test_a_second_statement_after_a_semicolon_cannot_smuggle_a_read() -> None: + with pytest.raises(ToolError): + tool.sql_query('SELECT 1; SELECT email FROM "user"') + + +def test_a_statement_crafted_to_close_the_explain_prefix_is_still_one_statement() -> ( + None +): + # The prefix is text, so this would be three valid statements if anything + # but a prepared statement ran it -- and the middle one is the read. + with pytest.raises(ToolError): + tool.sql_query( + 'SELECT 1) AS x; SELECT email FROM "user"; SELECT * FROM (SELECT 1' + ) diff --git a/backend/app/entry/mcp/test_web_fetch.py b/backend/app/entry/mcp/test_web_fetch.py new file mode 100644 index 00000000..9103dcf7 --- /dev/null +++ b/backend/app/entry/mcp/test_web_fetch.py @@ -0,0 +1,31 @@ +"""`web_fetch`: what it forwards, and an empty page as an error.""" + +from __future__ import annotations + +import pytest +from fastmcp.exceptions import ToolError + +from app.entry.mcp import web_fetch as web_fetch_tool + + +def test_web_fetch_returns_extracted_text_and_forwards_the_cap( + monkeypatch: pytest.MonkeyPatch, +) -> None: + seen: list[tuple[str, int]] = [] + + def fake(url: str, max_length: int) -> str: + seen.append((url, max_length)) + return f"text of {url}" + + monkeypatch.setattr(web_fetch_tool, "fetch_and_extract", fake) + text = web_fetch_tool.web_fetch("https://example.com/post", max_length=50) + assert text == "text of https://example.com/post" + assert seen == [("https://example.com/post", 50)] + + +def test_web_fetch_fails_loudly_when_nothing_is_readable( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(web_fetch_tool, "fetch_and_extract", lambda url, max_length: "") + with pytest.raises(ToolError, match="No readable content"): + web_fetch_tool.web_fetch("https://example.com/paywalled") diff --git a/backend/app/entry/mcp/test_web_search.py b/backend/app/entry/mcp/test_web_search.py new file mode 100644 index 00000000..3790c22f --- /dev/null +++ b/backend/app/entry/mcp/test_web_search.py @@ -0,0 +1,24 @@ +"""`web_search`: what it forwards.""" + +from __future__ import annotations + +import pytest + +from app.entry.mcp import web_search as web_search_tool + + +def test_web_search_forwards_every_argument(monkeypatch: pytest.MonkeyPatch) -> None: + calls: list[tuple[str, int, list[str] | None]] = [] + + def fake( + query: str, max_results: int, include_domains: list[str] | None + ) -> list[dict[str, str | None]]: + calls.append((query, max_results, include_domains)) + return [{"title": "t", "url": "u", "description": "d", "published_date": None}] + + monkeypatch.setattr(web_search_tool, "tavily_search", fake) + results = web_search_tool.web_search( + "gpt-5", max_results=3, include_domains=["openai.com"] + ) + assert calls == [("gpt-5", 3, ["openai.com"])] + assert results[0]["url"] == "u" diff --git a/backend/app/entry/mcp/vocabulary.py b/backend/app/entry/mcp/vocabulary.py new file mode 100644 index 00000000..2d014784 --- /dev/null +++ b/backend/app/entry/mcp/vocabulary.py @@ -0,0 +1,17 @@ +"""The `vocabulary` tool: the labels `approve` accepts.""" + +from __future__ import annotations + +from sqlmodel import Session + +from app.core.db.engine import engine +from app.entry.mcp.auth import require_write +from app.features.curation import service as curation + + +def vocabulary() -> dict[str, object]: + """The labels `approve` accepts: `kinds`, `publisher_kinds`, and `tags` as + slug -> description. Call it once per session, before the first approve.""" + require_write() + with Session(engine) as session: + return curation.vocabulary(session) diff --git a/backend/app/entry/mcp/web_fetch.py b/backend/app/entry/mcp/web_fetch.py new file mode 100644 index 00000000..e8b1b970 --- /dev/null +++ b/backend/app/entry/mcp/web_fetch.py @@ -0,0 +1,20 @@ +"""The `web_fetch` tool: one page as readable text.""" + +from __future__ import annotations + +from fastmcp.exceptions import ToolError + +from app.core.extract import fetch_and_extract + + +def web_fetch(url: str, max_length: int = 20000) -> str: + """Fetch a web page and return its readable text, boilerplate stripped. + + Retries through a residential proxy when the direct fetch is blocked by a + 403, a paywall or a login wall. Fails if the page yields nothing readable — + an empty string would look like an empty page rather than a failed fetch. + """ + content = fetch_and_extract(url, max_length) + if not content: + raise ToolError(f"No readable content at {url}") + return content diff --git a/backend/app/entry/mcp/web_search.py b/backend/app/entry/mcp/web_search.py new file mode 100644 index 00000000..4706cd88 --- /dev/null +++ b/backend/app/entry/mcp/web_search.py @@ -0,0 +1,19 @@ +"""The `web_search` tool: Tavily results for a query.""" + +from __future__ import annotations + +from app.core.web_search import tavily_search + + +def web_search( + query: str, max_results: int = 5, include_domains: list[str] | None = None +) -> list[dict[str, str | None]]: + """Search the web (Tavily) and return ranked results. + + Each result has `title`, `url`, `description` and `published_date` + (null unless the source dated the page). `include_domains` restricts hits + to those hosts, e.g. `["openai.com"]` for first-party announcements only. + """ + return tavily_search( + query, max_results=max_results, include_domains=include_domains + ) diff --git a/backend/app/entry/mcp/week.py b/backend/app/entry/mcp/week.py new file mode 100644 index 00000000..3bd7221b --- /dev/null +++ b/backend/app/entry/mcp/week.py @@ -0,0 +1,27 @@ +"""The `week` tool: the week's published articles grouped into ranked topics.""" + +from __future__ import annotations + +from sqlmodel import Session + +from app.core.db.engine import engine +from app.entry.mcp.auth import require_write +from app.features.newsletter import service as newsletter +from app.features.newsletter.model import WeekTopic + + +def week(days: int = 7, limit: int = newsletter.WEEK_LIMIT) -> list[WeekTopic]: + """The weekly newsletter's material: the last `days` of published articles + grouped into topics, the most talked-about first. + + Ranked by `coverage` (distinct publishers carrying it, counting the copies + curation rejected as retellings), then `likes`, then `top_score`; up to + `limit` topics. Each topic's `articles` are the published ones - `url`, + `title`, `publisher`, `publisher_kind` (`lab`, `company`, `individual`, + `media`, `community`, `unknown`), `kind`, `score`, `likes`, and the `summary` curation + wrote after reading it - and `covered_by` the other outlets' copies from + the ledger, with curation's `reason`. + """ + require_write() + with Session(engine) as session: + return newsletter.week(session, days, limit) diff --git a/backend/app/catalog/__init__.py b/backend/app/features/__init__.py similarity index 100% rename from backend/app/catalog/__init__.py rename to backend/app/features/__init__.py diff --git a/backend/app/catalog/tests/__init__.py b/backend/app/features/analytics/__init__.py similarity index 100% rename from backend/app/catalog/tests/__init__.py rename to backend/app/features/analytics/__init__.py diff --git a/backend/app/analytics/models.py b/backend/app/features/analytics/model.py similarity index 52% rename from backend/app/analytics/models.py rename to backend/app/features/analytics/model.py index d8870fde..0b57e974 100644 --- a/backend/app/analytics/models.py +++ b/backend/app/features/analytics/model.py @@ -1,44 +1,18 @@ -"""Analytics schema: first-party pageview and custom-event rows, keyed on an anonymous visitor, and the admin report built from them.""" +"""Analytics shapes: the event a page posts, and the admin report.""" from __future__ import annotations -import uuid -from datetime import date, datetime +from datetime import date -from sqlalchemy import JSON, Column, DateTime from sqlmodel import Field, SQLModel -from app.core.dates import get_datetime_utc - - -class AnalyticsEvent(SQLModel, table=True): - """First-party analytics event — one row per pageview or custom event.""" - - __tablename__ = "analytics_event" - id: int | None = Field(default=None, primary_key=True) - event: str = Field(default="pageview", index=True) - path: str | None = Field(default=None, index=True) - referrer: str | None = None - # anonymous client id persisted in the browser (localStorage), not a cookie - visitor_id: str | None = Field(default=None, index=True) - # set only when the request carries a valid bearer token - user_id: uuid.UUID | None = Field(default=None, foreign_key="user.id") - user_agent: str | None = None - # arbitrary metadata for custom events - props: dict = Field(default_factory=dict, sa_column=Column(JSON, nullable=False)) - created_at: datetime = Field( - default_factory=get_datetime_utc, - index=True, - sa_type=DateTime(timezone=True), # type: ignore - ) - class AnalyticsEventCreate(SQLModel): event: str = "pageview" path: str | None = None referrer: str | None = None visitor_id: str | None = None - props: dict = Field(default_factory=dict) + props: dict[str, object] = Field(default_factory=dict) # ─── Admin report ──────────────────────────────────────────────────────────── diff --git a/backend/app/analytics/report.py b/backend/app/features/analytics/service.py similarity index 79% rename from backend/app/analytics/report.py rename to backend/app/features/analytics/service.py index 0edc765d..83eb64a8 100644 --- a/backend/app/analytics/report.py +++ b/backend/app/features/analytics/service.py @@ -1,6 +1,7 @@ -"""The admin analytics report: who visited, what they opened, where they came from. +"""First-party analytics: recording an event, and the admin report on who +visited, what they opened and where they came from. -Counts outside readers only. A device a superuser has ever signed in on is the +The report counts outside readers only. A device a superuser has ever signed in on is the admin's own traffic and is left out everywhere, signed in or not, and so is any user agent that reads as a crawler or a script. Both are reported as one number each so the exclusion stays visible. @@ -8,22 +9,25 @@ from __future__ import annotations +import re +import uuid from datetime import UTC, date, datetime, time, timedelta from sqlalchemy import CTE, ColumnElement, String, case, cast, distinct, func, or_, true from sqlmodel import Session, col, select -from app.analytics.models import ( - AnalyticsEvent, +from app.core.dates import get_datetime_utc +from app.core.db.article import Article +from app.core.db.event import AnalyticsEvent +from app.core.db.user import User +from app.features.analytics.model import ( + AnalyticsEventCreate, AnalyticsReport, ReportArticle, ReportDay, ReportRow, ReportTotals, ) -from app.audience.models import User -from app.catalog.models import Article -from app.core.dates import get_datetime_utc # Loose on purpose: a human caught here costs one visitor, a bot let through # inflates every number. @@ -32,6 +36,44 @@ TOP = 10 +# Caps oversized strings so a bad or malicious client cannot bloat the table. +_MAX_LEN = 2048 + + +def _truncate(value: str | None) -> str | None: + return value[:_MAX_LEN] if value else None + + +def _without_query(value: str | None) -> str | None: + # A sign-in link carries its token in the query string, and nothing reads + # the query back, so none is kept. + return re.split(r"[?#]", value, maxsplit=1)[0] if value else None + + +def record_event( + session: Session, + payload: AnalyticsEventCreate, + *, + user_id: uuid.UUID | None, + user_agent: str | None, +) -> None: + session.add( + AnalyticsEvent( + event=_truncate(payload.event) or "pageview", + path=_truncate(_without_query(payload.path)), + # Only the client's `document.referrer` is meaningful. The request's + # own Referer header is always our site, so a fallback to it would + # file every direct visit under agentique.ch. + referrer=_truncate(_without_query(payload.referrer)), + visitor_id=_truncate(payload.visitor_id), + user_id=user_id, + user_agent=_truncate(user_agent), + props=payload.props, + ) + ) + session.commit() + + def build_report(session: Session, days: int | None) -> AnalyticsReport: """The report for the last `days` calendar days in UTC, today included; all time when None.""" today = get_datetime_utc().date() diff --git a/backend/app/mcp/__init__.py b/backend/app/features/audience/__init__.py similarity index 100% rename from backend/app/mcp/__init__.py rename to backend/app/features/audience/__init__.py diff --git a/backend/app/features/audience/email.py b/backend/app/features/audience/email.py new file mode 100644 index 00000000..0f8e7e17 --- /dev/null +++ b/backend/app/features/audience/email.py @@ -0,0 +1,60 @@ +"""The sign-in and welcome mails: built from the templates beside this module, +sent through `app.core.email`.""" + +from __future__ import annotations + +import logging +from dataclasses import dataclass +from pathlib import Path + +from app.core.email import render_email_template, send_email +from app.core.settings import settings + +logger = logging.getLogger(__name__) + +TEMPLATES = Path(__file__).parent + + +@dataclass +class EmailData: + html_content: str + subject: str + + +def sign_in_link(login_token: str) -> str: + return f"{settings.FRONTEND_HOST}/auth?token={login_token}" + + +def generate_welcome_email(login_token: str) -> EmailData: + html_content = render_email_template( + template=TEMPLATES / "welcome.html", + context={"link": sign_in_link(login_token)}, + ) + return EmailData(html_content=html_content, subject="Welcome to Agentique") + + +def generate_sign_in_email(login_token: str) -> EmailData: + html_content = render_email_template( + template=TEMPLATES / "sign_in.html", + context={"link": sign_in_link(login_token)}, + ) + return EmailData(html_content=html_content, subject="Sign in to Agentique") + + +def send_link_email(email_to: str, email_data: EmailData, login_token: str) -> None: + """Runs after the response: a failed send must not fail the request, and the + caller's answer must not depend on whether the account exists.""" + if settings.ENVIRONMENT == "development": + logger.info( + "[dev] Sign-in link for %s: %s", email_to, sign_in_link(login_token) + ) + if not settings.emails_enabled: + return + try: + send_email( + email_to=email_to, + subject=email_data.subject, + html_content=email_data.html_content, + ) + except Exception as e: + logger.error(f"Sign-in email failed for {email_to}: {e}") diff --git a/backend/app/features/audience/model.py b/backend/app/features/audience/model.py new file mode 100644 index 00000000..18f63b1a --- /dev/null +++ b/backend/app/features/audience/model.py @@ -0,0 +1,62 @@ +"""Audience shapes: the reader account as the API reads and writes it, and the auth payloads.""" + +from __future__ import annotations + +import uuid +from datetime import datetime + +from pydantic import EmailStr +from sqlmodel import Field, SQLModel + +from app.core.db.user import UserBase + +# ─── User ──────────────────────────────────────────────────────────────────── + + +# Properties to receive via API on creation +class UserCreate(UserBase): + pass + + +class UserUpdateMe(SQLModel): + full_name: str | None = Field(default=None, max_length=255) + email: EmailStr | None = Field(default=None, max_length=255) + + +# Properties to return via API, id is always required +class UserPublic(UserBase): + id: uuid.UUID + created_at: datetime | None = None + + +class UsersPublic(SQLModel): + data: list[UserPublic] + count: int + + +# Generic message +class Message(SQLModel): + message: str + + +# ─── Auth payloads ─────────────────────────────────────────────────────────── + + +# JSON payload containing access token +class Token(SQLModel): + access_token: str + token_type: str = "bearer" + + +# Contents of JWT token +class TokenPayload(SQLModel): + sub: str | None = None + + +class SignInRequest(SQLModel): + email: EmailStr = Field(max_length=255) + utm_source: str | None = Field(default=None, max_length=255) + + +class MagicLogin(SQLModel): + token: str = Field(max_length=64) diff --git a/backend/app/audience/service.py b/backend/app/features/audience/service.py similarity index 87% rename from backend/app/audience/service.py rename to backend/app/features/audience/service.py index b9b4fb7b..dcdb35f6 100644 --- a/backend/app/audience/service.py +++ b/backend/app/features/audience/service.py @@ -7,11 +7,16 @@ from sqlmodel import Session, col, select -from app.audience.models import ArticleLike, User, UserCreate, UserUpdateMe -from app.catalog.models import Article from app.core.dates import get_datetime_utc +from app.core.db.article import Article +from app.core.db.like import ArticleLike +from app.core.db.user import User from app.core.security import generate_login_token from app.core.settings import settings +from app.features.audience.email import generate_sign_in_email as generate_sign_in_email +from app.features.audience.email import generate_welcome_email as generate_welcome_email +from app.features.audience.email import send_link_email as send_link_email +from app.features.audience.model import UserCreate, UserUpdateMe LOGIN_LINK_EXPIRE_MINUTES = 60 diff --git a/backend/app/core/email-templates/sign_in.html b/backend/app/features/audience/sign_in.html similarity index 100% rename from backend/app/core/email-templates/sign_in.html rename to backend/app/features/audience/sign_in.html diff --git a/backend/app/audience/tests/test_service.py b/backend/app/features/audience/test_service.py similarity index 90% rename from backend/app/audience/tests/test_service.py rename to backend/app/features/audience/test_service.py index 7cfcae2a..7441975c 100644 --- a/backend/app/audience/tests/test_service.py +++ b/backend/app/features/audience/test_service.py @@ -4,8 +4,8 @@ from sqlmodel import Session -from app.audience import service -from app.audience.tests.factories import create_random_user +from app.core.db.factories import create_random_user +from app.features.audience import service def test_issue_login_token(db: Session) -> None: diff --git a/backend/app/core/email-templates/welcome.html b/backend/app/features/audience/welcome.html similarity index 100% rename from backend/app/core/email-templates/welcome.html rename to backend/app/features/audience/welcome.html diff --git a/backend/app/mcp/tests/__init__.py b/backend/app/features/catalog/__init__.py similarity index 100% rename from backend/app/mcp/tests/__init__.py rename to backend/app/features/catalog/__init__.py diff --git a/backend/app/catalog/facets.py b/backend/app/features/catalog/facets.py similarity index 95% rename from backend/app/catalog/facets.py rename to backend/app/features/catalog/facets.py index f370cb5e..02c4c010 100644 --- a/backend/app/catalog/facets.py +++ b/backend/app/features/catalog/facets.py @@ -7,25 +7,22 @@ from sqlalchemy import ColumnElement, func from sqlmodel import Session, col, select -from app.catalog.articles import origin_condition -from app.catalog.models import ( - Article, +from app.core.db.article import Article, ArticleKind +from app.core.db.publisher import Publisher, PublisherKind +from app.core.db.tag import ArticleTag, Tag +from app.features.catalog.model import ( ArticleFacets, - ArticleKind, - ArticleTag, FoundViaCount, Origin, OriginCount, OriginCounts, - Publisher, PublisherFacet, - PublisherKind, - Tag, TagFacet, Voice, VoiceArticle, VoiceKind, ) +from app.features.catalog.rows import origin_condition def publisher_facets( diff --git a/backend/app/features/catalog/model.py b/backend/app/features/catalog/model.py new file mode 100644 index 00000000..8b95ad2b --- /dev/null +++ b/backend/app/features/catalog/model.py @@ -0,0 +1,136 @@ +"""The read shapes the feed serves: articles, facets, origins and voices.""" + +from __future__ import annotations + +from datetime import datetime +from enum import StrEnum + +from sqlmodel import Field, SQLModel + +from app.core.db.article import ArticleKind +from app.core.db.publisher import PublisherKind + + +class Origin(StrEnum): + """The feed's "From" filter: what the article is when that is settled (a + repo, a paper, a model), otherwise who published it. Not stored; each + article falls under exactly one, see `catalog.rows.origin_condition`.""" + + lab = "lab" + company = "company" + individual = "individual" + media = "media" + repo = "repo" + paper = "paper" + model = "model" + + +# ─── Read API shapes ─────────────────────────────────────────────────────── +# Consumer-facing views. `publisher` is nested (replaces the old `source` string) +# and `tags` come from the article_tag join. Pipeline provenance and bulk fields +# (content, embedding, links, trust) stay internal. + + +class PublisherPublic(SQLModel): + id: int + slug: str + name: str + kind: PublisherKind + image: str | None = None + + +class TagPublic(SQLModel): + slug: str + name: str + + +class ArticlePublic(SQLModel): + id: int + title: str + url: str + summary: str | None = None + score: int + kind: ArticleKind + found_via: str | None = None + published_at: datetime | None = None + created_at: datetime | None = None + publisher: PublisherPublic + tags: list[TagPublic] = Field(default_factory=list) + like_count: int = 0 + liked_by_me: bool = False + + +class ArticlesPublic(SQLModel): + data: list[ArticlePublic] + count: int + + +# ─── Facets (sidebar filter options) ─────────────────────────────────────── + + +class PublisherFacet(SQLModel): + slug: str + name: str + count: int + + +class TagFacet(SQLModel): + slug: str + name: str + count: int + + +class ArticleFacets(SQLModel): + publishers: list[PublisherFacet] + tags: list[TagFacet] + + +# ─── Origin counts (the left rail's "Who wrote this week") ───────────────── + + +class OriginCount(SQLModel): + origin: Origin + count: int + + +class FoundViaCount(SQLModel): + # Null: the publisher's own feed. + name: str | None + count: int + + +class OriginCounts(SQLModel): + since: datetime + total: int + origins: list[OriginCount] + # Posts by a publisher still `unknown`: under no origin until curation + # names its kind. + unlabelled: int + found_via: list[FoundViaCount] + + +# ─── Voices (the left rail's "Voices") ───────────────────────────────────── + + +class VoiceKind(StrEnum): + """The Voices panel's tabs: Writers, Labs, Companies.""" + + individual = "individual" + lab = "lab" + company = "company" + + +class VoiceArticle(SQLModel): + title: str + url: str + score: int + published_at: datetime | None = None + + +class Voice(SQLModel): + slug: str + name: str + count: int + # Oldest first, the last `facets.VOICE_SCORES` of `count`. + scores: list[int] + best: VoiceArticle diff --git a/backend/app/features/catalog/rows.py b/backend/app/features/catalog/rows.py new file mode 100644 index 00000000..11ad2a48 --- /dev/null +++ b/backend/app/features/catalog/rows.py @@ -0,0 +1,118 @@ +"""The ArticlePublic assembly and the origin filter that the feed, search and facets share. + +`ArticlePublic` nests the publisher and the article's tags, so every endpoint +that returns articles (feed, search, liked) needs the same joins and the same +assembly. It lives apart from `service` so `semantic_search` and `facets` can +use it while `service` exposes them. +""" + +from __future__ import annotations + +import uuid +from collections import defaultdict + +from sqlalchemy import ColumnElement +from sqlmodel import Session, col, select + +from app.core.db.article import Article, ArticleKind +from app.core.db.like import ArticleLike +from app.core.db.publisher import Publisher, PublisherKind +from app.core.db.tag import ArticleTag, Tag +from app.features.catalog.model import ( + ArticlePublic, + Origin, + PublisherPublic, + TagPublic, +) + + +def liked_article_ids(session: Session, user_id: uuid.UUID) -> set[int]: + return set( + session.exec( + select(ArticleLike.article_id).where(ArticleLike.user_id == user_id) + ).all() + ) + + +def tags_by_article( + session: Session, article_ids: list[int] +) -> dict[int, list[TagPublic]]: + """One query for every article's tags, keyed by article id.""" + if not article_ids: + return {} + rows = session.exec( + select(ArticleTag.article_id, Tag.slug, Tag.name) + .join(Tag, col(Tag.id) == col(ArticleTag.tag_id)) + .where(col(ArticleTag.article_id).in_(article_ids)) + .order_by(col(ArticleTag.article_id), col(Tag.slug)) + ).all() + out: dict[int, list[TagPublic]] = defaultdict(list) + for article_id, slug, name in rows: + out[article_id].append(TagPublic(slug=slug, name=name)) + return out + + +def to_public( + article: Article, + publisher: Publisher, + like_count: int, + liked_by_me: bool, + tags: list[TagPublic], +) -> ArticlePublic: + assert article.id is not None + assert publisher.id is not None + return ArticlePublic( + id=article.id, + title=article.title, + url=article.url, + summary=article.summary, + score=article.score, + kind=article.kind, + found_via=article.found_via, + published_at=article.published_at, + created_at=article.created_at, + publisher=PublisherPublic( + id=publisher.id, + slug=publisher.slug, + name=publisher.name, + kind=publisher.kind, + image=publisher.image, + ), + tags=tags, + like_count=like_count, + liked_by_me=liked_by_me, + ) + + +def build_rows( + session: Session, rows: list[tuple[Article, Publisher, int]], liked_ids: set[int] +) -> list[ArticlePublic]: + """`rows` are (Article, Publisher, like_count) tuples.""" + tags = tags_by_article(session, [a.id for a, _, _ in rows if a.id is not None]) + out = [] + for a, p, like_count in rows: + assert a.id is not None + out.append(to_public(a, p, like_count, a.id in liked_ids, tags.get(a.id, []))) + return out + + +# The article kinds that are their own origin, whoever published them. +ARTIFACT_KINDS = { + Origin.repo: ArticleKind.repo, + Origin.paper: ArticleKind.paper, + Origin.model: ArticleKind.model, +} + + +def origin_condition(origin: Origin) -> ColumnElement[bool]: + """A repo, paper or model is that whoever published it; a post falls under + its publisher's kind. Community aggregators show under media, and an + `unknown` publisher under none until the curation agent names its kind.""" + if origin in ARTIFACT_KINDS: + return col(Article.kind) == ARTIFACT_KINDS[origin] + kinds = [PublisherKind(origin.value)] + if origin == Origin.media: + kinds.append(PublisherKind.community) + return (col(Article.kind) == ArticleKind.post) & col(Article.publisher_id).in_( + select(Publisher.id).where(col(Publisher.kind).in_(kinds)) + ) diff --git a/backend/app/catalog/semantic_search.py b/backend/app/features/catalog/semantic_search.py similarity index 89% rename from backend/app/catalog/semantic_search.py rename to backend/app/features/catalog/semantic_search.py index f171915f..18bd658e 100644 --- a/backend/app/catalog/semantic_search.py +++ b/backend/app/features/catalog/semantic_search.py @@ -14,8 +14,11 @@ from sqlalchemy import cast, func from sqlmodel import Session, col, select -from app.catalog.articles import build_rows, like_counts_subquery, liked_article_ids -from app.catalog.models import Article, ArticlesPublic, Publisher +from app.core.db.article import Article +from app.core.db.like import like_counts_subquery +from app.core.db.publisher import Publisher +from app.features.catalog.model import ArticlesPublic +from app.features.catalog.rows import build_rows, liked_article_ids # model2vec is CPU-only and small (~30 MB), so one process-wide instance loaded # on first use costs less than a model server. @@ -39,7 +42,7 @@ def embed(text: str) -> list[float]: # pragma: no cover norm = np.linalg.norm(vec) if norm > 0: vec = vec / norm - return vec.tolist() + return [float(x) for x in vec] def search( diff --git a/backend/app/catalog/articles.py b/backend/app/features/catalog/service.py similarity index 53% rename from backend/app/catalog/articles.py rename to backend/app/features/catalog/service.py index 68444fda..2f08d98c 100644 --- a/backend/app/catalog/articles.py +++ b/backend/app/features/catalog/service.py @@ -1,134 +1,29 @@ -"""The article read model: the feed query, the liked list, and the ArticlePublic assembly they share. +"""The public feed's read verbs: the feed query, the liked list, search, facets and stats. -`ArticlePublic` nests the publisher and the article's tags, so every endpoint -that returns articles (feed, search, liked) needs the same joins and the same -assembly. It lives here so route modules never build it themselves, and so -`semantic_search` can reuse it without importing a route module's privates. +Search and the facet counts live in their own modules (`semantic_search`, +`facets`) and are exposed here, so entry imports one module per feature. """ from __future__ import annotations import uuid -from collections import defaultdict from datetime import datetime -from sqlalchemy import ColumnElement, Subquery, func +from sqlalchemy import ColumnElement, func from sqlmodel import Session, col, select -from app.audience.models import ArticleLike -from app.catalog.models import ( - Article, - ArticleKind, - ArticlePublic, - ArticlesPublic, - ArticleTag, - Origin, - Publisher, - PublisherKind, - PublisherPublic, - Tag, - TagPublic, -) - - -def like_counts_subquery() -> Subquery: - return ( - select(ArticleLike.article_id, func.count().label("like_count")) - .group_by(ArticleLike.article_id) # type: ignore[arg-type] # ty: ignore[invalid-argument-type] - .subquery() - ) - - -def liked_article_ids(session: Session, user_id: uuid.UUID) -> set[int]: - return set( - session.exec( - select(ArticleLike.article_id).where(ArticleLike.user_id == user_id) - ).all() - ) - - -def tags_by_article( - session: Session, article_ids: list[int] -) -> dict[int, list[TagPublic]]: - """One query for every article's tags, keyed by article id.""" - if not article_ids: - return {} - rows = session.exec( - select(ArticleTag.article_id, Tag.slug, Tag.name) - .join(Tag, col(Tag.id) == col(ArticleTag.tag_id)) - .where(col(ArticleTag.article_id).in_(article_ids)) - .order_by(col(ArticleTag.article_id), col(Tag.slug)) - ).all() - out: dict[int, list[TagPublic]] = defaultdict(list) - for article_id, slug, name in rows: - out[article_id].append(TagPublic(slug=slug, name=name)) - return out - - -def to_public( - article: Article, - publisher: Publisher, - like_count: int, - liked_by_me: bool, - tags: list[TagPublic], -) -> ArticlePublic: - assert article.id is not None - assert publisher.id is not None - return ArticlePublic( - id=article.id, - title=article.title, - url=article.url, - summary=article.summary, - score=article.score, - kind=article.kind, - found_via=article.found_via, - published_at=article.published_at, - created_at=article.created_at, - publisher=PublisherPublic( - id=publisher.id, - slug=publisher.slug, - name=publisher.name, - kind=publisher.kind, - image=publisher.image, - ), - tags=tags, - like_count=like_count, - liked_by_me=liked_by_me, - ) - - -def build_rows( - session: Session, rows: list[tuple[Article, Publisher, int]], liked_ids: set[int] -) -> list[ArticlePublic]: - """`rows` are (Article, Publisher, like_count) tuples.""" - tags = tags_by_article(session, [a.id for a, _, _ in rows if a.id is not None]) - out = [] - for a, p, like_count in rows: - assert a.id is not None - out.append(to_public(a, p, like_count, a.id in liked_ids, tags.get(a.id, []))) - return out - - -# The article kinds that are their own origin, whoever published them. -ARTIFACT_KINDS = { - Origin.repo: ArticleKind.repo, - Origin.paper: ArticleKind.paper, - Origin.model: ArticleKind.model, -} - - -def origin_condition(origin: Origin) -> ColumnElement[bool]: - """A repo, paper or model is that whoever published it; a post falls under - its publisher's kind. Community aggregators show under media, and an - `unknown` publisher under none until the curation agent names its kind.""" - if origin in ARTIFACT_KINDS: - return col(Article.kind) == ARTIFACT_KINDS[origin] - kinds = [PublisherKind(origin.value)] - if origin == Origin.media: - kinds.append(PublisherKind.community) - return (col(Article.kind) == ArticleKind.post) & col(Article.publisher_id).in_( - select(Publisher.id).where(col(Publisher.kind).in_(kinds)) - ) +from app.core.db.article import Article +from app.core.db.like import ArticleLike, like_counts_subquery +from app.core.db.publisher import Publisher +from app.core.db.tag import ArticleTag, Tag +from app.features.catalog.facets import all_facets as all_facets +from app.features.catalog.facets import origin_counts as origin_counts +from app.features.catalog.facets import publisher_facets as publisher_facets +from app.features.catalog.facets import tag_facets as tag_facets +from app.features.catalog.facets import voices as voices +from app.features.catalog.model import ArticlesPublic, Origin +from app.features.catalog.rows import build_rows, liked_article_ids, origin_condition +from app.features.catalog.semantic_search import search as search def _filters( @@ -252,7 +147,5 @@ def list_liked_articles(session: Session, viewer_id: uuid.UUID) -> ArticlesPubli def stats(session: Session) -> dict[str, int | str | None]: """How many articles the catalog holds and when it last grew.""" total = session.exec(select(func.count()).select_from(Article)).one() - last = session.exec( - select(func.max(Article.created_at)) # type: ignore[arg-type] - ).one() + last = session.exec(select(func.max(Article.created_at))).one() return {"total": total, "lastUpdated": last.isoformat() if last else None} diff --git a/backend/app/newsletter/__init__.py b/backend/app/features/collect/__init__.py similarity index 100% rename from backend/app/newsletter/__init__.py rename to backend/app/features/collect/__init__.py diff --git a/backend/pipeline/sources/substack.py b/backend/app/features/collect/feeds.py similarity index 73% rename from backend/pipeline/sources/substack.py rename to backend/app/features/collect/feeds.py index be67d5c4..80a08f9f 100644 --- a/backend/pipeline/sources/substack.py +++ b/backend/app/features/collect/feeds.py @@ -3,10 +3,11 @@ from __future__ import annotations import time +from collections.abc import Mapping from concurrent.futures import ThreadPoolExecutor, as_completed from urllib.parse import urlparse -import feedparser +import feedparser # type: ignore[import-untyped] from app.core.extract import extract_text, outbound_links from app.core.http import ( @@ -16,9 +17,9 @@ ) from app.core.logging import log from app.core.urls import feed_url -from pipeline.freshness import is_within_window -from pipeline.titles import clean_title -from pipeline.types import RawItem +from app.features.collect.freshness import is_within_window +from app.features.collect.model import FeedSource, RawItem +from app.features.collect.titles import clean_title _PROXY_URL = RESIDENTIAL_PROXY_URL _proxy_status_logged = False @@ -45,18 +46,33 @@ def _log_proxy_status_once() -> None: log(f"Substack proxy: set but unparsable (len {len(_PROXY_URL)})") -def _entry_html(entry) -> str: +# One feedparser entry. Every field a feed may leave out, or fill with +# something other than what the spec says. +type Entry = Mapping[str, object] + + +def _text(entry: Entry, *keys: str) -> str: + """The first of ``keys`` that holds a non-empty string, else "".""" + for key in keys: + value = entry.get(key) + if isinstance(value, str) and value: + return value + return "" + + +def _entry_html(entry: Entry) -> str: """A feed entry's HTML, preferring ``content:encoded`` over the ``summary`` teaser.""" - encoded = max( - ((c.get("value") or "") for c in (entry.get("content") or [])), - key=len, - default="", - ) - return encoded or entry.get("summary") or "" + content = entry.get("content") + encoded = "" + for part in content if isinstance(content, list) else []: + value = part.get("value") if isinstance(part, Mapping) else None + if isinstance(value, str) and len(value) > len(encoded): + encoded = value + return encoded or _text(entry, "summary") -def _entry_item(entry, name: str) -> RawItem: +def _entry_item(entry: Entry, name: str) -> RawItem: """One feed entry as an item: plain text and the links its body makes. The HTML goes through the same trafilatura pass the network extractor uses @@ -64,14 +80,14 @@ def _entry_item(entry, name: str) -> RawItem: (anything under 50 chars) as empty rather than passing a blurb off as an article. """ - url = entry.get("link") or "" + url = _text(entry, "link") html = _entry_html(entry) content = extract_text(html) item: RawItem = { - "title": clean_title(entry.get("title") or "(no title)"), + "title": clean_title(_text(entry, "title") or "(no title)"), "url": url, "content": content, - "published_date": entry.get("published") or entry.get("updated") or "", + "published_date": _text(entry, "published", "updated"), "source": name, } if content: @@ -104,20 +120,20 @@ def _fetch_feed_xml(url: str, retries: int = 2, backoff: float = 2.0) -> str: raise RuntimeError("Exhausted retries") -def _fetch_source(source: dict) -> tuple[list[RawItem], str | None]: +def _fetch_source(source: FeedSource) -> tuple[list[RawItem], str | None]: name = source["name"] rss_url = source["rssUrl"] log(f"Fetching {name}...") try: xml = _fetch_feed_xml(feed_url(rss_url)) - feed = feedparser.parse(xml) - items = feed.get("entries", []) + entries = feedparser.parse(xml).get("entries") + items: list[Entry] = [e for e in entries or [] if isinstance(e, Mapping)] if not items: log(f" {name}: feed parsed but has no entries — check the feed URL") within = [ it for it in items - if is_within_window(it.get("published") or it.get("updated")) + if is_within_window(_text(it, "published", "updated") or None) ] log(f" {name}: {len(within)} in window") return [_entry_item(it, name) for it in within], None @@ -126,12 +142,12 @@ def _fetch_source(source: dict) -> tuple[list[RawItem], str | None]: return [], f"{type(e).__name__}: {e}" -def fetch_feeds(sources: list[dict]) -> tuple[list[RawItem], dict[str, str]]: +def fetch_feeds(sources: list[FeedSource]) -> tuple[list[RawItem], dict[str, str]]: """Fetch a list of RSS/substack feeds in parallel. ``sources`` is ``[{"name": ..., "rssUrl": ...}]`` — the runtime builds it from the DB (active publishers with rss/substack links) via - ``pipeline.publishers.feed_sources_from_db``. The caller resolves each + ``publishers.feed_sources_from_db``. The caller resolves each ``source`` name to a Publisher row. Returns ``(articles, errors)`` where ``errors`` maps a feed's ``name`` to diff --git a/backend/pipeline/steps/fetch.py b/backend/app/features/collect/fetch.py similarity index 95% rename from backend/pipeline/steps/fetch.py rename to backend/app/features/collect/fetch.py index d3d5984a..75a4efe7 100644 --- a/backend/pipeline/steps/fetch.py +++ b/backend/app/features/collect/fetch.py @@ -11,18 +11,18 @@ from app.core.extract import fetch_full_content from app.core.logging import log from app.core.urls import hostname -from pipeline.fetching.tweets import unwrap_tweets -from pipeline.publishers import ( +from app.features.collect.feeds import fetch_feeds +from app.features.collect.hn import fetch_hn +from app.features.collect.lab_watch import fetch_lab_watch +from app.features.collect.model import Candidate, RawItem +from app.features.collect.newsletters import fetch_newsletter +from app.features.collect.publishers import ( PublisherResolver, feed_sources_from_db, lab_watch_targets_from_db, newsletter_senders_from_db, ) -from pipeline.sources.email import fetch_newsletter -from pipeline.sources.hn import fetch_hn -from pipeline.sources.lab_watch import fetch_lab_watch -from pipeline.sources.substack import fetch_feeds -from pipeline.types import Candidate, RawItem +from app.features.collect.tweets import unwrap_tweets # Below this, content is a teaser/blurb rather than an article, and the # categorizer fails on it ~30-40% of the time (vs ~2% above it). Used to decide diff --git a/backend/pipeline/steps/filter.py b/backend/app/features/collect/filter.py similarity index 76% rename from backend/pipeline/steps/filter.py rename to backend/app/features/collect/filter.py index d19f28cb..39339c80 100644 --- a/backend/pipeline/steps/filter.py +++ b/backend/app/features/collect/filter.py @@ -8,12 +8,12 @@ from __future__ import annotations -from sqlmodel import Session, select +from sqlmodel import Session, col, select -from app.catalog.models import Article +from app.core.db.article import Article +from app.core.db.ledger import Reject from app.core.logging import log -from pipeline.models import Reject -from pipeline.types import RawItem +from app.features.collect.model import RawItem def filter_known_urls[T: RawItem]( @@ -24,10 +24,10 @@ def filter_known_urls[T: RawItem]( all_urls = [a["url"] for a in articles] existing_urls = set( - session.exec(select(Article.url).where(Article.url.in_(all_urls))).all() + session.exec(select(Article.url).where(col(Article.url).in_(all_urls))).all() ) scored_urls = set( - session.exec(select(Reject.url).where(Reject.url.in_(all_urls))).all() + session.exec(select(Reject.url).where(col(Reject.url).in_(all_urls))).all() ) fresh = [ diff --git a/backend/pipeline/freshness.py b/backend/app/features/collect/freshness.py similarity index 100% rename from backend/pipeline/freshness.py rename to backend/app/features/collect/freshness.py diff --git a/backend/pipeline/sources/hn.py b/backend/app/features/collect/hn.py similarity index 93% rename from backend/pipeline/sources/hn.py rename to backend/app/features/collect/hn.py index 80d90050..0a7f5cb5 100644 --- a/backend/pipeline/sources/hn.py +++ b/backend/app/features/collect/hn.py @@ -6,13 +6,14 @@ import re from concurrent.futures import ThreadPoolExecutor from datetime import UTC, datetime +from typing import TypedDict from app.core.http import fetch_with_timeout from app.core.logging import log from app.core.urls import is_first_party -from pipeline.freshness import is_within_window -from pipeline.titles import clean_title -from pipeline.types import RawItem +from app.features.collect.freshness import is_within_window +from app.features.collect.model import RawItem +from app.features.collect.titles import clean_title HN_ITEM = "https://hacker-news.firebaseio.com/v0/item" @@ -44,7 +45,7 @@ def hn_grace_hours() -> float: return float(os.environ.get("HN_GRACE_HOURS", "6")) -# The title-only AI gate. Moved here from pipeline/topic_gate.py when the +# The title-only AI gate. Moved here from the old topic gate module when the # pipeline stopped gating feeds by topic: HN is the one source whose firehose # is mostly not about AI, so it keeps the gate. AI_TITLE_KEYWORDS = re.compile( @@ -160,12 +161,26 @@ def is_on_topic(title: str) -> bool: HN_FETCH_LIMIT = 200 -def _fetch_item(item_id: int) -> dict | None: +class HNItem(TypedDict, total=False): + """The slice of an HN item this module reads. Every key can be missing: a + deleted or dead item comes back with little more than its id.""" + + id: int + type: str + title: str + url: str + time: int + score: int + descendants: int + + +def _fetch_item(item_id: int) -> HNItem | None: try: resp = fetch_with_timeout(f"{HN_ITEM}/{item_id}.json", timeout=10.0) - return resp.json() + item: HNItem | None = resp.json() except Exception: return None + return item def _story_ids(label: str, url: str) -> list[int]: @@ -178,7 +193,7 @@ def _story_ids(label: str, url: str) -> list[int]: return ids[:HN_FETCH_LIMIT] -def _age_hours(item: dict) -> float | None: +def _age_hours(item: HNItem) -> float | None: """Hours since the story was posted, or None if it carries no timestamp.""" posted = item.get("time") if not posted: @@ -186,7 +201,7 @@ def _age_hours(item: dict) -> float | None: return (datetime.now(UTC).timestamp() - float(posted)) / 3600 -def _passes_topic_and_window(item: dict) -> bool: +def _passes_topic_and_window(item: HNItem) -> bool: """Every ``_to_article`` gate except traction — the denominator the run log reports the traction gate against.""" if not item or item.get("type") != "story" or not item.get("title"): @@ -198,7 +213,7 @@ def _passes_topic_and_window(item: dict) -> bool: return is_within_window(pub_date) -def has_traction(item: dict) -> bool: +def has_traction(item: HNItem) -> bool: """Has HN itself said this story is worth reading? The firehose is not an editorial feed — ``newstories`` is every submission, @@ -240,13 +255,13 @@ def has_traction(item: dict) -> bool: return points >= hn_min_points() or comments >= hn_min_comments() -def _to_article(item: dict) -> RawItem | None: +def _to_article(item: HNItem) -> RawItem | None: """One HN item -> a fetched article, or None if it fails a gate. Gates, cheapest first: it must be a titled story, its title must look AI-related, it must fall inside the pipeline's recency window, and HN's own readers must have given it some traction. Content stays empty here: the - fetch step (``steps.fetch._with_content``) fetches the full text, as it + fetch step (``fetch._with_content``) fetches the full text, as it does for every thin source. """ if not item or item.get("type") != "story" or not item.get("title"): diff --git a/backend/pipeline/sources/lab_watch.py b/backend/app/features/collect/lab_watch.py similarity index 95% rename from backend/pipeline/sources/lab_watch.py rename to backend/app/features/collect/lab_watch.py index f0b00e65..17ed9dfe 100644 --- a/backend/pipeline/sources/lab_watch.py +++ b/backend/app/features/collect/lab_watch.py @@ -22,7 +22,7 @@ from app.core.logging import log from app.core.urls import hostname from app.core.web_search import tavily_search -from pipeline.types import RawItem +from app.features.collect.model import RawItem WATCH_DAYS = 2 # nightly cadence + one missed run of slack WATCH_CONCURRENCY = 5 @@ -157,17 +157,18 @@ def _watch_one(name: str, domain: str, cutoff: datetime) -> list[RawItem]: articles: list[RawItem] = [] for r in results: - if not _host_is_first_party(r["url"], domain): + url = r["url"] or "" + if not _host_is_first_party(url, domain): continue - if not _is_article_url(r["url"]): + if not _is_article_url(url): continue if not _is_recent(r.get("published_date"), cutoff): continue articles.append( RawItem( - title=r["title"], - url=r["url"], - content=r["description"], + title=r["title"] or "", + url=url, + content=r["description"] or "", published_date=r.get("published_date"), source=name, ) diff --git a/backend/app/features/collect/model.py b/backend/app/features/collect/model.py new file mode 100644 index 00000000..57c52266 --- /dev/null +++ b/backend/app/features/collect/model.py @@ -0,0 +1,94 @@ +"""The item as it changes shape on its way down the collect funnel, and the +run's per-source counts. + + RawItem what a source emits -- title, url, content, date, source + Candidate + the publisher it belongs to (fetch.resolve_publishers) + +``Candidate`` subclasses ``RawItem``, so a step that only needs a ``RawItem`` +also accepts a ``Candidate``, while a step that needs the publisher cannot be +handed an item that has not been resolved. The funnel ends at the queue: a +candidate becomes a `pending` ledger row, and curation reads it from there with +types of its own. + +This replaced a single ``FetchedArticle`` where every key was optional at every +stage. That shape could not say which step filled what, so each step's contract +lived in a docstring and a misordered call failed at runtime with a KeyError +instead of at the call site. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from datetime import UTC, datetime +from typing import NotRequired, TypedDict + + +class RawItem(TypedDict): + """One item as a source adapter emits it, before anything is known about it. + + ``content`` may be empty at this point: the thin sources (Hacker News, + Lab Watch, Newsletter) arrive with a blurb or nothing at all, and + ``fetch._with_content`` fills it before the item goes any further. + """ + + title: str + url: str + content: str + published_date: str | None + source: str + # Only the aggregator sources that carry a public reception signal (Hacker + # News points/comments) have one. Shown to the agent as evidence, not + # merely used as a gate -- see hn.py. + traction: NotRequired[str] + # The repo / model / paper / docs links the article body makes, grouped by + # ``app.core.extract``. Absent when the text came from nowhere with HTML. + links: NotRequired[dict[str, list[str]]] + + +class FeedSource(TypedDict): + """One rss or substack feed to poll, built from an active publisher.""" + + name: str + rssUrl: str + + +class Candidate(RawItem): + """An item resolved to the Publisher it came from.""" + + publisher_id: int + + +@dataclass +class SourceStats: + """One source in one run: how many items it fetched, how many became + candidates, and the error that stopped it, if one did. + + ``source`` is the run-level fetcher label (Feeds, Hacker News, Newsletter, + Lab Watch), not a publisher. + """ + + source: str + fetched: int = 0 + queued: int = 0 + error: str | None = None + + +@dataclass +class RunStats: + started_at: datetime = field(default_factory=lambda: datetime.now(UTC)) + finished_at: datetime | None = None + duration_ms: int | None = None + ok: bool = False + sources: list[SourceStats] = field(default_factory=list) + + def source(self, label: str) -> SourceStats: + s = SourceStats(source=label) + self.sources.append(s) + return s + + def finish(self, ok: bool) -> None: + self.finished_at = datetime.now(UTC) + self.duration_ms = int( + (self.finished_at - self.started_at).total_seconds() * 1000 + ) + self.ok = ok diff --git a/backend/pipeline/sources/email.py b/backend/app/features/collect/newsletters.py similarity index 92% rename from backend/pipeline/sources/email.py rename to backend/app/features/collect/newsletters.py index e1137a62..17dafed5 100644 --- a/backend/pipeline/sources/email.py +++ b/backend/app/features/collect/newsletters.py @@ -19,10 +19,12 @@ import os import re +from collections.abc import Mapping from concurrent.futures import ThreadPoolExecutor, as_completed from dataclasses import dataclass, field from datetime import UTC, date, datetime, timedelta from html import unescape +from typing import TypedDict from urllib.parse import parse_qsl, unquote, urlencode, urlsplit, urlunsplit from imap_tools import AND, MailBox @@ -32,9 +34,9 @@ from app.core.logging import log, short_error from app.core.urls import hostname from app.core.web_search import tavily_search +from app.features.collect.model import RawItem from baml_client.sync_client import b from baml_client.types import NewsletterItem, NewsletterItemKind, SearchCandidate -from pipeline.types import RawItem IMAP_FOLDER = "sub" @@ -368,15 +370,25 @@ def prefilter( CLASSIFY_CONCURRENCY = 10 -def verdict(answers: dict) -> str: +type Answers = Mapping[str, jev.Json] + + +def _probability(answers: Answers, key: str) -> float: + """A yes/no answer as a number. `jev.ask` has already failed the link on a + missing or non-numeric one.""" + value = answers.get(key) + return float(value) if isinstance(value, int | float) else 0.0 + + +def verdict(answers: Answers) -> str: """ "keep", or the reason a classified link is dropped.""" - if answers["is_admin"] > ADMIN_THRESHOLD: + if _probability(answers, "is_admin") > ADMIN_THRESHOLD: return "admin" - if answers["is_sponsor"] > SPONSOR_THRESHOLD: + if _probability(answers, "is_sponsor") > SPONSOR_THRESHOLD: return "sponsor" - if answers["kind"] == "neither": + if answers.get("kind") == "neither": return "neither" - if answers["is_ai_topic"] < AI_TOPIC_THRESHOLD: + if _probability(answers, "is_ai_topic") < AI_TOPIC_THRESHOLD: return "off_topic" return "keep" @@ -391,8 +403,20 @@ def _classify(link: Link) -> jev.Reply: return jev.ask(state, QUESTIONS) -def _item(link: Link, answers: dict, newsletter_name: str, email_date: str) -> dict: - is_product = answers["kind"] == "product" +class Item(TypedDict): + """One link an issue makes, kept by Jev, before it is resolved to a URL.""" + + kind: NewsletterItemKind + name: str + description: str + url: str + first_party: bool + email_date: str + newsletter_name: str + + +def _item(link: Link, answers: Answers, newsletter_name: str, email_date: str) -> Item: + is_product = answers.get("kind") == "product" return { "kind": NewsletterItemKind.Product if is_product @@ -402,7 +426,7 @@ def _item(link: Link, answers: dict, newsletter_name: str, email_date: str) -> d "url": link.href, # Behind an opaque tracker Jev saw no host, so its answer is a guess. "first_party": _known_host(link.href) is not None - and answers["is_first_party"] > FIRST_PARTY_THRESHOLD, + and _probability(answers, "is_first_party") > FIRST_PARTY_THRESHOLD, "email_date": email_date, "newsletter_name": newsletter_name, } @@ -410,12 +434,12 @@ def _item(link: Link, answers: dict, newsletter_name: str, email_date: str) -> d def _extract_items( html: str, newsletter_name: str, email_date: str, own_host: str -) -> list[dict]: +) -> list[Item]: kept, dropped = prefilter(extract(html), own_host) reasons = {r: len(links) for r, links in dropped.items()} log(f" {len(kept)} link(s) to classify, prefilter dropped {reasons}") - items: list[dict] = [] + items: list[Item] = [] verdicts: dict[str, int] = {} cost = 0.0 with ThreadPoolExecutor(max_workers=CLASSIFY_CONCURRENCY) as ex: @@ -490,7 +514,7 @@ def _unwrap_tracking_url(url: str) -> str | None: return inner if hostname(inner) else None -def _resolve_article(item: dict) -> RawItem | None: +def _resolve_article(item: Item) -> RawItem | None: """One article -> its real URL, recovered from the newsletter's redirect. Unwrap first, follow only if that fails: most trackers spell the @@ -539,7 +563,7 @@ def _resolve_article(item: dict) -> RawItem | None: } -def _resolve_product(product: dict) -> RawItem | None: +def _resolve_product(product: Item) -> RawItem | None: """One product -> one canonical URL, or None if nothing is a clean first-party source.""" name = product["name"] description = product["description"] @@ -551,42 +575,42 @@ def _resolve_product(product: dict) -> RawItem | None: log(f' Search failed for "{name}": {e}') return None - results = [r for r in results if not _is_denied(r["url"])] - if not results: + candidates = [ + SearchCandidate( + title=r["title"] or "", url=r["url"] or "", snippet=r["description"] or "" + ) + for r in results + ] + candidates = [c for c in candidates if not _is_denied(c.url)] + if not candidates: log(f' No usable search result for "{name}"') return None try: - choice = b.SelectProductLink( - name, - description, - [ - SearchCandidate( - title=r["title"], url=r["url"], snippet=r["description"] - ) - for r in results - ], - ) + choice = b.SelectProductLink(name, description, candidates) except Exception as e: log(f' Link selection failed for "{name}": {e}') return None - # index is 1-based; 0 means "no candidate is a clean first-party source". - picked = results[choice.index - 1] if choice.index >= 1 else None + # index is 1-based; 0 means "no candidate is a clean first-party source", + # and one past the list is a model mistake read the same way. + picked = ( + candidates[choice.index - 1] if 1 <= choice.index <= len(candidates) else None + ) if not picked: log(f' Dropped "{name}": no first-party result among candidates') return None return { "title": name, - "url": _canonical_url(picked["url"]), + "url": _canonical_url(picked.url), "content": "", "published_date": product["email_date"], "source": product["newsletter_name"], } -def _resolve_one(item: dict) -> RawItem | None: +def _resolve_one(item: Item) -> RawItem | None: """Route an item to the resolution its kind needs. A product already linked at its maker's own page needs no search — its href is the answer.""" if item["kind"] == NewsletterItemKind.Article or item["first_party"]: @@ -594,7 +618,7 @@ def _resolve_one(item: dict) -> RawItem | None: return _resolve_product(item) -def _resolve_items(raw: list[dict]) -> list[RawItem]: +def _resolve_items(raw: list[Item]) -> list[RawItem]: """Dedupe items by name across all issues in this run, drop the un-notable ones, then resolve each survivor to a URL. @@ -606,7 +630,7 @@ def _resolve_items(raw: list[dict]) -> list[RawItem]: if not raw: return [] - by_name: dict[str, dict] = {} + by_name: dict[str, Item] = {} for item in raw: by_name.setdefault(item["name"].lower(), item) unique = list(by_name.values()) @@ -658,7 +682,7 @@ def _resolve_items(raw: list[dict]) -> list[RawItem]: def _run_imap_fetch( config: ImapConfig, sources: list[tuple[str, str]], lookback_days: int -) -> list[dict]: +) -> list[Item]: """One connect -> fetch -> logout cycle. Two passes: headers only to find matches in the window (avoids downloading full bodies of unrelated mail), then full source for matched UIDs only. @@ -666,7 +690,7 @@ def _run_imap_fetch( Nothing is flagged: the window is the selection, so leaving read state alone keeps the mailbox usable by a human and a failed run repeatable. """ - raw: list[dict] = [] + raw: list[Item] = [] since = date.today() - timedelta(days=lookback_days) with MailBox(config.host, port=config.port).login( @@ -675,7 +699,7 @@ def _run_imap_fetch( matches: dict[str, str] = {} for msg in mb.fetch(AND(date_gte=since), mark_seen=False, headers_only=True): name = _match_sender(msg.from_ or "", sources) - if name: + if name and msg.uid: matches[msg.uid] = name log(f" Found {len(matches)} newsletter emails since {since.isoformat()}") @@ -685,7 +709,7 @@ def _run_imap_fetch( for msg in mb.fetch( AND(uid=",".join(matches.keys())), mark_seen=False, headers_only=False ): - name = matches.get(msg.uid) + name = matches.get(msg.uid or "") if not name: continue diff --git a/backend/pipeline/publishers.py b/backend/app/features/collect/publishers.py similarity index 92% rename from backend/pipeline/publishers.py rename to backend/app/features/collect/publishers.py index cb859cbe..2e0310a9 100644 --- a/backend/pipeline/publishers.py +++ b/backend/app/features/collect/publishers.py @@ -26,7 +26,7 @@ from sqlmodel import Session, select -from app.catalog.models import ( +from app.core.db.publisher import ( LinkPlatform, Publisher, PublisherKind, @@ -39,6 +39,7 @@ from app.core.llm_text import enum_value from app.core.logging import log from app.core.urls import feed_url, hostname +from app.features.collect.model import FeedSource # Publishers whose items link to other people's posts. An item one of them # found is credited to the site it links to, never to them. @@ -96,7 +97,7 @@ def author(self, url: str) -> Publisher | None: """ key = owner_key(url) host = hostname(url) - if key is None and (not host or host in SHARED_HOSTS): + if key is None and (not host or is_shared_host(host)): return None site = key or host name = site.partition("/")[2].removeprefix("@") if key else host @@ -189,9 +190,15 @@ def _quarantine(self, slug: str, name: str) -> Publisher: "substack.com", "twitter.com", "x.com", + "youtu.be", "youtube.com", } ) +# Shared hosts whose subdomains are one author's site (writer.substack.com). +# Any other shared host's subdomain is the platform itself (old.reddit.com, +# m.youtube.com), unless a publisher's own link names it +# (engineering.linkedin.com, see `hosts_to_publishers`). +PERSONAL_SUBDOMAIN_HOSTS = frozenset({"medium.com", "substack.com"}) _HOST_PLATFORMS = ( LinkPlatform.website.value, @@ -211,6 +218,15 @@ def _quarantine(self, slug: str, name: str) -> Publisher: PERSONAL_HOST_SUFFIXES = (".github.io", ".substack.com", ".bearblog.dev") +def is_shared_host(host: str) -> bool: + """A host many unrelated authors publish under. Pure.""" + return any( + host == shared + or (host.endswith(f".{shared}") and shared not in PERSONAL_SUBDOMAIN_HOSTS) + for shared in SHARED_HOSTS + ) + + def owner_key(url: str) -> str | None: """``host/owner`` for a URL on an OWNER_HOSTS platform that names one, else None. Pure. @@ -298,6 +314,8 @@ def hosts_to_publishers(publishers: Iterable[Publisher]) -> dict[str, Publisher] host = owner_key(link) or hostname(link) else: continue + # Only the bare platform: a subdomain a publisher links to + # (engineering.linkedin.com) is that publisher's own site. if host and host not in SHARED_HOSTS: claims.setdefault(host, {})[p.id] = p return { @@ -344,14 +362,14 @@ def _active_publisher_links(session: Session) -> list[tuple[Publisher, dict[str, ] -def feed_sources_from_db(session: Session) -> list[dict]: +def feed_sources_from_db(session: Session) -> list[FeedSource]: """Active publishers with an rss/substack link -> feed configs. Replaces ``sources/substack-sources.json``. Shape mirrors the old JSON (``name`` / ``rssUrl``) so the substack fetcher needs no interface change. Prefers an explicit ``rss`` link, else falls back to ``substack``. """ - sources: list[dict] = [] + sources: list[FeedSource] = [] for pub, links in _active_publisher_links(session): for platform in _FEED_PLATFORMS: url = links.get(platform) @@ -377,8 +395,7 @@ def lab_watch_targets_from_db(session: Session) -> list[tuple[str, str]]: """Active publishers with a ``search`` link -> (name, first-party domain). Fallback for labs that publish no RSS feed and no newsletter: their own site - is searched for recent first-party articles (see ``pipeline.sources. - lab_watch``). The ``search`` value is the bare host to restrict results to. + is searched for recent first-party articles (see ``lab_watch``). The ``search`` value is the bare host to restrict results to. """ targets = [ (pub.name, domain) diff --git a/backend/pipeline/steps/queue.py b/backend/app/features/collect/queue.py similarity index 87% rename from backend/pipeline/steps/queue.py rename to backend/app/features/collect/queue.py index ba328465..21e17c58 100644 --- a/backend/pipeline/steps/queue.py +++ b/backend/app/features/collect/queue.py @@ -3,7 +3,7 @@ The last step of the nightly run. Nothing is scored, summarized or inserted here: a candidate becomes a `pending` row in the reject ledger and waits to be read. The agent (see `.claude/skills/curate/SKILL.md`) approves or rejects it -through the MCP tools in `app/mcp/tools.py`, and only an approval writes an +through the MCP tools in `app/entry/mcp/`, and only an approval writes an Article. Why the ledger and not a table of its own: the row already holds everything a @@ -17,10 +17,9 @@ from sqlmodel import Session +from app.core.db.ledger import RejectStage, record_reject from app.core.logging import log -from pipeline.models import RejectStage -from pipeline.rejects import record_reject -from pipeline.types import Candidate +from app.features.collect.model import Candidate def queue_candidates(session: Session, articles: list[Candidate]) -> list[Candidate]: diff --git a/backend/pipeline/report.py b/backend/app/features/collect/report.py similarity index 90% rename from backend/pipeline/report.py rename to backend/app/features/collect/report.py index 57d36149..a9ba87b9 100644 --- a/backend/pipeline/report.py +++ b/backend/app/features/collect/report.py @@ -1,4 +1,4 @@ -"""The daily mail. Run with: python -m pipeline.report +"""The daily mail's content: the night read out of the database, as text. Sent after the curation session, not after the fetch: the 04:00 run queues candidates and lands nothing, so a mail at that hour would be a page of counts @@ -20,16 +20,15 @@ from __future__ import annotations -import sys from dataclasses import dataclass from datetime import UTC, datetime, timedelta from sqlmodel import Session, col, func, select -from app.catalog.models import Article, Publisher -from app.core.email import send_mail -from pipeline.db import get_engine -from pipeline.models import PipelineRun, Reject, RejectStage +from app.core.db.article import Article +from app.core.db.ledger import Reject, RejectStage +from app.core.db.publisher import Publisher +from app.core.db.run import PipelineRun # Older than this with no successful run: "pipeline may be down". STALE_RUN_HOURS = 26 @@ -75,7 +74,7 @@ def liveness(last: PipelineRun | None, now: datetime) -> str | None: return None -def source_lines(sources: list[dict]) -> tuple[list[str], list[str]]: +def source_lines(sources: list[dict[str, object]]) -> tuple[list[str], list[str]]: """``(silent, errors)`` from a run's stored per-source counts. Pure. Tolerates the older, longer shape and the ``errors`` list it carried, so a @@ -88,7 +87,8 @@ def source_lines(sources: list[dict]) -> tuple[list[str], list[str]]: if not s.get("fetched"): silent.append(name) error = s.get("error") - for e in [error] if error else s.get("errors") or []: + older = s.get("errors") + for e in [error] if error else older if isinstance(older, list) else []: errors.append(f"{name}: {e}") return silent, errors @@ -188,13 +188,3 @@ def format_night(night: Night) -> str: else "Queue empty — every candidate got a verdict." ) return "\n".join(lines) - - -if __name__ == "__main__": - try: - with Session(get_engine()) as session: - night = read_night(session) - send_mail(subject(night), format_night(night)) - except Exception as e: - print(f"Daily report failed: {e}", file=sys.stderr) - sys.exit(1) diff --git a/backend/pipeline/runs.py b/backend/app/features/collect/runs.py similarity index 69% rename from backend/pipeline/runs.py rename to backend/app/features/collect/runs.py index fd5be20e..9540cbb9 100644 --- a/backend/pipeline/runs.py +++ b/backend/app/features/collect/runs.py @@ -8,57 +8,22 @@ with. That replaced a per-publisher JSON list stored on every run, which had to be unpacked across runs to answer "which feeds are broken". -Everything here is best-effort from the run's point of view: `run.py` catches +Everything here is best-effort from the run's point of view: `service.py` catches around it, so a bug in the bookkeeping never changes whether the run succeeded. """ from __future__ import annotations from collections.abc import Iterable -from dataclasses import asdict, dataclass, field +from dataclasses import asdict from datetime import UTC, datetime from sqlmodel import Session, col, select -from app.catalog.models import Publisher +from app.core.db.publisher import Publisher +from app.core.db.run import PipelineRun from app.core.logging import log -from pipeline.models import PipelineRun - - -@dataclass -class SourceStats: - """One source in one run: how many items it fetched, how many became - candidates, and the error that stopped it, if one did. - - ``source`` is the run-level fetcher label (Feeds, Hacker News, Newsletter, - Lab Watch), not a publisher. - """ - - source: str - fetched: int = 0 - queued: int = 0 - error: str | None = None - - -@dataclass -class RunStats: - started_at: datetime = field(default_factory=lambda: datetime.now(UTC)) - finished_at: datetime | None = None - duration_ms: int | None = None - ok: bool = False - sources: list[SourceStats] = field(default_factory=list) - - def source(self, label: str) -> SourceStats: - s = SourceStats(source=label) - self.sources.append(s) - return s - - def finish(self, ok: bool) -> None: - self.finished_at = datetime.now(UTC) - self.duration_ms = int( - (self.finished_at - self.started_at).total_seconds() * 1000 - ) - self.ok = ok +from app.features.collect.model import RunStats def record_run(session: Session, stats: RunStats) -> PipelineRun: diff --git a/backend/pipeline/run.py b/backend/app/features/collect/service.py similarity index 66% rename from backend/pipeline/run.py rename to backend/app/features/collect/service.py index ecf73918..35ed4505 100644 --- a/backend/pipeline/run.py +++ b/backend/app/features/collect/service.py @@ -1,9 +1,9 @@ -"""Pipeline entry point. Run with: python -m pipeline.run +"""The nightly collect run and the daily mail, the two verbs the timers call. -Orchestration only — each step lives in pipeline.steps.* and owns its own -logging and commits. This file's job is the funnel: for every source, run the -steps in order, record the counts, and make sure one bad source cannot take the -others down with it. +Orchestration only: each step (`fetch`, `filter`, `queue`) owns its own logging +and commits. This file's job is the funnel: for every source, run the steps in +order, record the counts, and make sure one bad source cannot take the others +down with it. The funnel ends at `queue_candidates`: every survivor is written as a pending candidate and nothing is scored, summarized or inserted. The curation agent @@ -17,20 +17,24 @@ from sqlmodel import Session +from app.core.db.engine import engine +from app.core.email import send_mail from app.core.logging import log, short_error -from pipeline.db import get_engine -from pipeline.publishers import PublisherResolver -from pipeline.runs import RunStats, record_new, record_polled, record_run -from pipeline.steps.fetch import build_sources, fetch_source, resolve_publishers -from pipeline.steps.filter import filter_known_urls -from pipeline.steps.queue import queue_candidates -from pipeline.types import Candidate - - -def run_pipeline(stats: RunStats) -> None: +from app.features.collect.fetch import build_sources, fetch_source, resolve_publishers +from app.features.collect.filter import filter_known_urls +from app.features.collect.model import Candidate, RunStats +from app.features.collect.publishers import AGGREGATOR_TYPES as AGGREGATOR_TYPES +from app.features.collect.publishers import PublisherResolver as PublisherResolver +from app.features.collect.queue import queue_candidates +from app.features.collect.report import format_night, read_night, subject +from app.features.collect.runs import record_new, record_polled, record_run + + +def run(stats: RunStats) -> None: + """One pass over every source, counted into ``stats``.""" log("=== Pipeline start ===") - with Session(get_engine()) as session: + with Session(engine) as session: resolver = PublisherResolver(session=session) for source in build_sources(session): @@ -98,26 +102,39 @@ def run_pipeline(stats: RunStats) -> None: log("=== Pipeline complete ===") -# ─── CLI ──────────────────────────────────────────────────────────────────── +def run_nightly() -> int: + """The whole nightly run: collect, then record it. Returns the exit code. -if __name__ == "__main__": + No email from here. The daily mail (`send_report`) reads the recorded row + after the curation session: a crashed run, or no run at all, is the first + thing it says. Recording is best-effort and never flips the exit code. + """ stats = RunStats() crashed: Exception | None = None try: - run_pipeline(stats) + run(stats) except Exception as e: crashed = e print(f"Pipeline failed: {e}", file=sys.stderr) stats.finish(ok=crashed is None) - # No email from here. The daily mail (`python -m pipeline.report`) reads - # this row after the curation session: a crashed run, or no run at all, is - # the first thing it says. Best-effort: never flips the exit code. try: - with Session(get_engine()) as session: + with Session(engine) as session: record_run(session, stats) except Exception as e: print(f"Recording the run failed: {e}", file=sys.stderr) - sys.exit(1 if crashed else 0) + return 1 if crashed else 0 + + +def send_report() -> int: + """Read the night out of the database and mail it. Returns the exit code.""" + try: + with Session(engine) as session: + night = read_night(session) + send_mail(subject(night), format_night(night)) + except Exception as e: + print(f"Daily report failed: {e}", file=sys.stderr) + return 1 + return 0 diff --git a/backend/tests/pipeline/test_feed_content.py b/backend/app/features/collect/test_feeds.py similarity index 81% rename from backend/tests/pipeline/test_feed_content.py rename to backend/app/features/collect/test_feeds.py index c72867f0..41f15a37 100644 --- a/backend/tests/pipeline/test_feed_content.py +++ b/backend/app/features/collect/test_feeds.py @@ -15,9 +15,10 @@ from app.core.extract import Page from app.core.urls import feed_url -from pipeline.sources.substack import _entry_item -from pipeline.steps import fetch as fetch_step -from pipeline.steps.fetch import MIN_CONTENT_CHARS, MIN_SUMMARIZABLE_CHARS +from app.features.collect import fetch as fetch_step +from app.features.collect.feeds import Entry, _entry_item +from app.features.collect.fetch import MIN_CONTENT_CHARS, MIN_SUMMARIZABLE_CHARS +from app.features.collect.model import RawItem # Varied prose on purpose: trafilatura deduplicates repeated segments, so a # fixture built from one repeated sentence extracts to nothing. @@ -54,7 +55,7 @@ ("https://console.dev/rss.xml", "https://console.dev/rss.xml"), ], ) -def test_feed_url_normalises_bare_substack_links(given, expected): +def test_feed_url_normalises_bare_substack_links(given: str, expected: str) -> None: assert feed_url(given) == expected @@ -68,21 +69,23 @@ def test_feed_url_normalises_bare_substack_links(given, expected): ("https://www.latent.space/feed", "https://www.latent.space/feed"), ], ) -def test_feed_url_appends_feed_for_substack_platform_links(given, expected): +def test_feed_url_appends_feed_for_substack_platform_links( + given: str, expected: str +) -> None: assert feed_url(given, is_substack=True) == expected -def test_feed_url_is_idempotent(): +def test_feed_url_is_idempotent() -> None: """Applied by publisher config and again by the fetcher — must not double up.""" once = feed_url("https://linas.substack.com") assert feed_url(once) == once -def _entry_content(entry: dict) -> str: +def _entry_content(entry: Entry) -> str: return _entry_item(entry, "Feed A")["content"] -def test_content_encoded_preferred_over_summary_teaser(): +def test_content_encoded_preferred_over_summary_teaser() -> None: entry = { "summary": "

A short teaser.

", "content": [{"value": _ARTICLE_HTML}], @@ -93,29 +96,29 @@ def test_content_encoded_preferred_over_summary_teaser(): assert "A short teaser" not in text -def test_falls_back_to_summary_when_no_content_encoded(): +def test_falls_back_to_summary_when_no_content_encoded() -> None: entry = {"summary": _ARTICLE_HTML, "content": []} text = _entry_content(entry) assert len(text) >= MIN_CONTENT_CHARS -def test_html_is_stripped_not_stored_raw(): +def test_html_is_stripped_not_stored_raw() -> None: entry = {"summary": "", "content": [{"value": _ARTICLE_HTML}]} text = _entry_content(entry) assert "

" not in text and "

" not in text -def test_short_teaser_yields_empty_rather_than_pretending_to_be_content(): +def test_short_teaser_yields_empty_rather_than_pretending_to_be_content() -> None: """A teaser must not masquerade as an article — the gate reads its length.""" entry = {"summary": "

Read more at the link.

", "content": []} assert _entry_content(entry) == "" -def test_missing_fields_do_not_raise(): +def test_missing_fields_do_not_raise() -> None: assert _entry_content({}) == "" -def test_a_feed_item_carries_the_links_its_body_makes(): +def test_a_feed_item_carries_the_links_its_body_makes() -> None: html = _ARTICLE_HTML.replace( "
", '

Code is on ' @@ -126,7 +129,7 @@ def test_a_feed_item_carries_the_links_its_body_makes(): assert item["links"] == {"repo": ["https://github.com/anthropics/sdk"]} -def test_a_teaser_carries_no_links(): +def test_a_teaser_carries_no_links() -> None: entry = {"summary": '

x

'} assert "links" not in _entry_item(entry, "Feed A") @@ -140,11 +143,19 @@ def test_a_teaser_carries_no_links(): _FULL_TEXT = "x" * MIN_CONTENT_CHARS -def _fetched(url: str, content: str = "") -> dict: - return {"title": "A title", "url": url, "content": content, "source": "Lab Watch"} +def _fetched(url: str, content: str = "") -> RawItem: + return { + "title": "A title", + "url": url, + "content": content, + "published_date": None, + "source": "Lab Watch", + } -def _stub_refetch(monkeypatch, content_map: dict[str, str]) -> list[list[str]]: +def _stub_refetch( + monkeypatch: pytest.MonkeyPatch, content_map: dict[str, str] +) -> list[list[str]]: """Replace the network re-fetch, recording which URLs it was asked for.""" asked: list[list[str]] = [] @@ -160,7 +171,9 @@ def fetch_full_content(urls: list[str]) -> dict[str, Page]: return asked -def test_thin_item_is_refetched_and_kept_when_the_refetch_lands(monkeypatch): +def test_thin_item_is_refetched_and_kept_when_the_refetch_lands( + monkeypatch: pytest.MonkeyPatch, +) -> None: asked = _stub_refetch(monkeypatch, {"u1": _FULL_TEXT}) kept = fetch_step._with_content([_fetched("u1", "a snippet")], "Lab Watch") assert asked == [["u1"]] @@ -168,21 +181,27 @@ def test_thin_item_is_refetched_and_kept_when_the_refetch_lands(monkeypatch): assert kept[0]["links"] == {"paper": ["https://arxiv.org/abs/1"]} -def test_thin_item_the_refetch_could_not_fill_is_dropped(monkeypatch): +def test_thin_item_the_refetch_could_not_fill_is_dropped( + monkeypatch: pytest.MonkeyPatch, +) -> None: """The cause of the Lab Watch LLM bill: a snippet that never became an article still reached the scorer. It must not survive this step.""" _stub_refetch(monkeypatch, {}) assert fetch_step._with_content([_fetched("u1", "")], "Lab Watch") == [] -def test_a_snippet_that_stays_thin_is_dropped_even_though_it_is_non_empty(monkeypatch): +def test_a_snippet_that_stays_thin_is_dropped_even_though_it_is_non_empty( + monkeypatch: pytest.MonkeyPatch, +) -> None: """A failed re-fetch leaves the original snippet in place; length, not presence, is what decides.""" _stub_refetch(monkeypatch, {}) assert fetch_step._with_content([_fetched("u1", "a snippet")], "Lab Watch") == [] -def test_content_just_long_enough_to_summarize_is_kept(monkeypatch): +def test_content_just_long_enough_to_summarize_is_kept( + monkeypatch: pytest.MonkeyPatch, +) -> None: _stub_refetch(monkeypatch, {}) text = "x" * MIN_SUMMARIZABLE_CHARS assert ( @@ -191,7 +210,9 @@ def test_content_just_long_enough_to_summarize_is_kept(monkeypatch): ) -def test_an_item_that_already_has_full_content_is_not_refetched(monkeypatch): +def test_an_item_that_already_has_full_content_is_not_refetched( + monkeypatch: pytest.MonkeyPatch, +) -> None: asked = _stub_refetch(monkeypatch, {}) kept = fetch_step._with_content([_fetched("u1", _FULL_TEXT)], "Feeds") assert asked == [] diff --git a/backend/tests/pipeline/test_steps.py b/backend/app/features/collect/test_fetch.py similarity index 56% rename from backend/tests/pipeline/test_steps.py rename to backend/app/features/collect/test_fetch.py index 81f386bf..9353a54b 100644 --- a/backend/tests/pipeline/test_steps.py +++ b/backend/app/features/collect/test_fetch.py @@ -1,56 +1,30 @@ -"""The pure cores of the fetch and persist steps. +"""The pure cores of the fetch step. -These carry the decisions that silently change what gets published — which -article wins a duplicate URL, whose article it is — so they are pulled out of -the I/O steps and pinned here. +These decide what gets queued: whose article it is, which URLs are never +fetched, which channels are polled. A wrong one changes the queue without an +error, so they are pulled out of the I/O and pinned here. """ from __future__ import annotations -import pytest - -from pipeline.steps import fetch as fetch_step -from pipeline.steps.persist import best_per_url - - -def _article(url: str, source: str = "Hacker News", **extra) -> dict: - return {"title": "A title", "url": url, "source": source, **extra} - - -# ─── best_per_url ──────────────────────────────────────────────────────────── - - -def test_same_url_from_two_sources_keeps_the_higher_score(): - """Two sources can surface one URL in a run (an HN post and a feed item for - the same post); the URL is the identity, so the better score wins.""" - items = [ - _article("dupe", source="Hacker News", score=40), - _article("dupe", source="Feeds", score=90), - ] - best = best_per_url(items) - assert len(best) == 1 - assert best[0]["score"] == 90 - assert best[0]["source"] == "Feeds" - - -def test_higher_score_wins_regardless_of_order(): - items = [_article("dupe", score=90), _article("dupe", score=40)] - assert best_per_url(items)[0]["score"] == 90 +from typing import cast +import pytest +from sqlmodel import Session -def test_distinct_urls_all_survive(): - items = [_article("u1", score=80), _article("u2", score=70)] - assert len(best_per_url(items)) == 2 - +from app.core.db.publisher import Publisher, PublisherKind +from app.features.collect import fetch as fetch_step +from app.features.collect.model import FeedSource, RawItem +from app.features.collect.publishers import PublisherResolver -def test_empty_input(): - assert best_per_url([]) == [] +# `build_sources` hands the session to the db reads, which these tests stub. +NO_SESSION = cast(Session, None) # ─── resolve_publishers ────────────────────────────────────────────────────── -def _raw(url: str, source: str) -> dict: +def _raw(url: str, source: str) -> RawItem: return { "title": "A title", "url": url, @@ -60,38 +34,39 @@ def _raw(url: str, source: str) -> dict: } -class _FakePublisher: - def __init__(self, id_: int = 1) -> None: - self.id = id_ - - class _FakeResolver: - def __init__(self, credited: dict | None = None) -> None: - self._publisher = _FakePublisher() + def __init__(self, credited: dict[str, Publisher] | None = None) -> None: + self._publisher = Publisher( + id=1, slug="feed", name="Feed", kind=PublisherKind.media + ) self._credited = credited or {} - def publisher_for(self, url: str, source: str) -> _FakePublisher: + def publisher_for(self, url: str, source: str) -> Publisher: return self._credited.get(url) or self._publisher + @property + def resolver(self) -> PublisherResolver: + return cast(PublisherResolver, self) + -def test_every_item_is_stamped_with_its_publisher(): +def test_every_item_is_stamped_with_its_publisher() -> None: """resolve_publishers is the only producer of a Candidate, so pin that it stamps every item it returns.""" resolver = _FakeResolver() candidates = fetch_step.resolve_publishers( - [_raw("u1", "Feed A"), _raw("u2", "Feed B")], resolver + [_raw("u1", "Feed A"), _raw("u2", "Feed B")], resolver.resolver ) assert [c["publisher_id"] for c in candidates] == [1, 1] -def test_credits_the_author_over_the_aggregator_that_found_it(): +def test_credits_the_author_over_the_aggregator_that_found_it() -> None: """A post found through Hacker News is the author's when we know their site; the source stays Hacker News, which is where we found it.""" - author = _FakePublisher(id_=7) + author = Publisher(id=7, slug="simon", name="Simon", kind=PublisherKind.individual) url = "https://simonwillison.net/2026/Aug/8/auto-mode/" resolver = _FakeResolver(credited={url: author}) - [c] = fetch_step.resolve_publishers([_raw(url, "Hacker News")], resolver) + [c] = fetch_step.resolve_publishers([_raw(url, "Hacker News")], resolver.resolver) assert c["publisher_id"] == 7 assert c["source"] == "Hacker News" @@ -107,7 +82,7 @@ def _urls(urls: list[str]) -> list[str]: return [a["url"] for a in kept] -def test_a_glossary_entry_is_dropped_before_it_is_fetched(): +def test_a_glossary_entry_is_dropped_before_it_is_fetched() -> None: """AI Hero's dictionary emits one entry per term; the agent rejected 71 of them with the same sentence before this gate existed.""" assert _urls(["https://www.aihero.dev/ai-coding-dictionary/sycophancy"]) == [] @@ -123,16 +98,16 @@ def test_a_glossary_entry_is_dropped_before_it_is_fetched(): "https://www.eventbrite.com/e/123", ], ) -def test_an_event_page_is_dropped(url): +def test_an_event_page_is_dropped(url: str) -> None: assert _urls([url]) == [] -def test_an_article_that_only_mentions_a_glossary_survives(): +def test_an_article_that_only_mentions_a_glossary_survives() -> None: kept = "https://example.com/blog/a-glossary-of-agent-terms-explained" assert _urls([kept]) == [kept] -def test_ordinary_articles_are_untouched(): +def test_ordinary_articles_are_untouched() -> None: urls = ["https://example.com/a", "https://example.com/b"] assert _urls(urls) == urls @@ -143,11 +118,11 @@ def test_ordinary_articles_are_untouched(): def _stub_db_sources( - monkeypatch, - feeds: list[dict], + monkeypatch: pytest.MonkeyPatch, + feeds: list[FeedSource], lab_watch: list[tuple[str, str]], newsletter: list[tuple[str, str]] | None = None, -): +) -> None: monkeypatch.setattr(fetch_step, "feed_sources_from_db", lambda session: feeds) monkeypatch.setattr( fetch_step, "lab_watch_targets_from_db", lambda session: lab_watch @@ -157,9 +132,9 @@ def _stub_db_sources( ) -def test_every_channel_is_polled(monkeypatch): +def test_every_channel_is_polled(monkeypatch: pytest.MonkeyPatch) -> None: _stub_db_sources(monkeypatch, [], []) - labels = [s.label for s in fetch_step.build_sources(session=None)] + labels = [s.label for s in fetch_step.build_sources(NO_SESSION)] assert labels == [ "Feeds", "Hacker News", @@ -168,25 +143,25 @@ def test_every_channel_is_polled(monkeypatch): ] -def test_lab_watch_reports_its_targets_as_publishers(monkeypatch): +def test_lab_watch_reports_its_targets_as_publishers( + monkeypatch: pytest.MonkeyPatch, +) -> None: """Per-publisher health needs the full expected set, so a lab that returned nothing still gets a row instead of vanishing from the stats.""" _stub_db_sources(monkeypatch, [], [("Anthropic", "anthropic.com")]) lab_watch = next( - s for s in fetch_step.build_sources(session=None) if s.label == "Lab Watch" + s for s in fetch_step.build_sources(NO_SESSION) if s.label == "Lab Watch" ) assert lab_watch.publisher_names == ("Anthropic",) -def test_feeds_reports_its_publishers(monkeypatch): +def test_feeds_reports_its_publishers(monkeypatch: pytest.MonkeyPatch) -> None: _stub_db_sources(monkeypatch, [{"name": "Cloudflare", "rssUrl": "x"}], []) - feeds = next( - s for s in fetch_step.build_sources(session=None) if s.label == "Feeds" - ) + feeds = next(s for s in fetch_step.build_sources(NO_SESSION) if s.label == "Feeds") assert feeds.publisher_names == ("Cloudflare",) -def test_each_source_binds_its_own_db_config(monkeypatch): +def test_each_source_binds_its_own_db_config(monkeypatch: pytest.MonkeyPatch) -> None: """The fetchers are lambdas closing over lists read once per run — a shared or late-bound one would poll the wrong set.""" _stub_db_sources( @@ -195,8 +170,12 @@ def test_each_source_binds_its_own_db_config(monkeypatch): [("Anthropic", "anthropic.com")], newsletter=[("@tldr.tech", "TLDR")], ) - monkeypatch.setattr(fetch_step, "fetch_lab_watch", lambda targets: targets) - monkeypatch.setattr(fetch_step, "fetch_newsletter", lambda senders: senders) - sources = {s.label: s for s in fetch_step.build_sources(session=None)} - assert sources["Lab Watch"].fetcher() == ([("Anthropic", "anthropic.com")], {}) - assert sources["Newsletter"].fetcher() == ([("@tldr.tech", "TLDR")], {}) + monkeypatch.setattr( + fetch_step, "fetch_lab_watch", lambda targets: [_raw(d, n) for n, d in targets] + ) + monkeypatch.setattr( + fetch_step, "fetch_newsletter", lambda senders: [_raw(p, n) for p, n in senders] + ) + sources = {s.label: s for s in fetch_step.build_sources(NO_SESSION)} + assert sources["Lab Watch"].fetcher() == ([_raw("anthropic.com", "Anthropic")], {}) + assert sources["Newsletter"].fetcher() == ([_raw("@tldr.tech", "TLDR")], {}) diff --git a/backend/app/features/collect/test_filter.py b/backend/app/features/collect/test_filter.py new file mode 100644 index 00000000..be357823 --- /dev/null +++ b/backend/app/features/collect/test_filter.py @@ -0,0 +1,62 @@ +"""The known-URL filter against the real schema: a URL that is an article, a +pending candidate or a settled reject is never fetched and judged again.""" + +from __future__ import annotations + +from collections.abc import Generator + +import pytest +from sqlmodel import Session, col, delete + +from app.core.db.factories import create_random_article, random_lower_string +from app.core.db.ledger import Reject, RejectStage +from app.core.db.publisher import Publisher +from app.features.collect.filter import filter_known_urls +from app.features.collect.model import RawItem + +TAG = random_lower_string()[:8] + + +def _item(path: str) -> RawItem: + return { + "title": path, + "url": f"https://{TAG}.example/{path}", + "content": "", + "published_date": None, + "source": "Hacker News", + } + + +@pytest.fixture +def known(db: Session) -> Generator[None]: + article = create_random_article(db, url=_item("published")["url"]) + db.add_all( + [ + Reject(url=_item("pending")["url"], stage=RejectStage.pending), + Reject(url=_item("rejected")["url"], stage=RejectStage.below_threshold), + ] + ) + db.commit() + yield + db.exec(delete(Reject).where(col(Reject.url).contains(TAG))) + publisher_id = article.publisher_id + db.delete(article) + db.exec(delete(Publisher).where(col(Publisher.id) == publisher_id)) + db.commit() + + +@pytest.mark.usefixtures("known") +def test_only_unseen_urls_pass(db: Session) -> None: + items = [_item(p) for p in ("published", "new", "pending", "rejected", "other")] + fresh = filter_known_urls(db, items, "HN") + assert [a["title"] for a in fresh] == ["new", "other"] + + +@pytest.mark.usefixtures("known") +def test_nothing_new_is_an_empty_list(db: Session) -> None: + items = [_item(p) for p in ("published", "pending", "rejected")] + assert filter_known_urls(db, items, "HN") == [] + + +def test_no_items_reads_nothing(db: Session) -> None: + assert filter_known_urls(db, [], "HN") == [] diff --git a/backend/tests/pipeline/test_freshness.py b/backend/app/features/collect/test_freshness.py similarity index 78% rename from backend/tests/pipeline/test_freshness.py rename to backend/app/features/collect/test_freshness.py index 45308e85..a19435be 100644 --- a/backend/tests/pipeline/test_freshness.py +++ b/backend/app/features/collect/test_freshness.py @@ -7,24 +7,24 @@ import pytest from app.core.dates import parse_date -from pipeline.freshness import is_within_window +from app.features.collect.freshness import is_within_window # ─── parse_date ────────────────────────────────────────────────────────────── -def test_parses_rfc_2822_as_rss_pubdate_emits_it(): +def test_parses_rfc_2822_as_rss_pubdate_emits_it() -> None: parsed = parse_date("Wed, 02 Oct 2002 13:00:00 GMT") assert parsed is not None assert (parsed.year, parsed.month, parsed.day) == (2002, 10, 2) -def test_parses_iso_8601(): +def test_parses_iso_8601() -> None: parsed = parse_date("2026-07-17T12:30:00+00:00") assert parsed is not None assert (parsed.year, parsed.month, parsed.day) == (2026, 7, 17) -def test_parses_iso_with_z_suffix(): +def test_parses_iso_with_z_suffix() -> None: """`Z` is what feeds actually emit; fromisoformat rejected it before 3.11.""" parsed = parse_date("2026-07-17T12:30:00Z") assert parsed is not None @@ -34,37 +34,37 @@ def test_parses_iso_with_z_suffix(): @pytest.mark.parametrize( "given", ["2026-07-17T12:30:00", "Fri, 17 Jul 2026 12:30:00 -0000"] ) -def test_a_date_with_no_offset_is_read_as_utc(given): +def test_a_date_with_no_offset_is_read_as_utc(given: str) -> None: """A naive datetime would be compared and stored in the server's local time.""" assert parse_date(given) == datetime(2026, 7, 17, 12, 30, tzinfo=UTC) @pytest.mark.parametrize("given", [None, "", "not a date", "2026-13-45"]) -def test_unparsable_dates_return_none_rather_than_raising(given): +def test_unparsable_dates_return_none_rather_than_raising(given: str | None) -> None: assert parse_date(given) is None # ─── is_within_window ──────────────────────────────────────────────────────── -def test_recent_date_is_within_window(): +def test_recent_date_is_within_window() -> None: recent = (datetime.now(UTC) - timedelta(hours=2)).isoformat() assert is_within_window(recent) is True -def test_old_date_is_outside_window(): +def test_old_date_is_outside_window() -> None: old = (datetime.now(UTC) - timedelta(days=30)).isoformat() assert is_within_window(old) is False -def test_window_boundary_is_inclusive(): +def test_window_boundary_is_inclusive() -> None: edge = (datetime.now(UTC) - timedelta(hours=10)).isoformat() assert is_within_window(edge, window_hours=24) is True assert is_within_window(edge, window_hours=1) is False @pytest.mark.parametrize("given", [None, "", "garbage"]) -def test_missing_or_unparsable_date_is_kept_not_dropped(given): +def test_missing_or_unparsable_date_is_kept_not_dropped(given: str | None) -> None: """Deliberate: dropping an article because its feed emitted a date we cannot read loses real content, while keeping it only risks a stale item that scoring can still reject.""" diff --git a/backend/tests/pipeline/test_hn.py b/backend/app/features/collect/test_hn.py similarity index 68% rename from backend/tests/pipeline/test_hn.py rename to backend/app/features/collect/test_hn.py index a82fc4f4..a60785ba 100644 --- a/backend/tests/pipeline/test_hn.py +++ b/backend/app/features/collect/test_hn.py @@ -10,8 +10,8 @@ import pytest -from pipeline.sources import hn -from pipeline.sources.hn import _to_article, has_traction, is_on_topic +from app.features.collect import hn +from app.features.collect.hn import HNItem, _to_article, has_traction, is_on_topic HOUR = 3600 @@ -74,26 +74,37 @@ def test_skips_non_ai_titles(title: str) -> None: # ─── _to_article ───────────────────────────────────────────────────────────── -def _item(**overrides) -> dict: +def _item( + *, + type: str = "story", + title: str = "Qwen3-Max is out", + url: str | None = "https://example.com/qwen3", + posted: int | None = int(time.time()) - HOUR * 12, + score: int = 120, + descendants: int = 40, +) -> HNItem: """A firebase item as HN serves it, defaulted to one that passes. Aged past the grace window and carrying real points, because that is the shape of a story worth keeping — the traction gate is exercised by the - tests that deliberately vary those two fields. + tests that deliberately vary those two fields. ``url`` or ``posted`` set to + None leaves the key out, as firebase does. """ - return { + item: HNItem = { "id": 42, - "type": "story", - "title": "Qwen3-Max is out", - "url": "https://example.com/qwen3", - "time": int(time.time()) - HOUR * 12, - "score": 120, - "descendants": 40, - **overrides, + "type": type, + "title": title, + "score": score, + "descendants": descendants, } + if url is not None: + item["url"] = url + if posted is not None: + item["time"] = posted + return item -def test_converts_a_passing_story(): +def test_converts_a_passing_story() -> None: article = _to_article(_item()) assert article is not None assert article["url"] == "https://example.com/qwen3" @@ -101,13 +112,13 @@ def test_converts_a_passing_story(): assert article["content"] == "" -def test_strips_the_show_hn_prefix(): +def test_strips_the_show_hn_prefix() -> None: article = _to_article(_item(title="Show HN: My LLM inference server")) assert article is not None assert article["title"] == "My LLM inference server" -def test_text_post_falls_back_to_its_hn_permalink(): +def test_text_post_falls_back_to_its_hn_permalink() -> None: """An Ask HN post carries no url; the discussion itself is the article.""" article = _to_article(_item(url=None, title="Ask HN: best local LLM setup?")) assert article is not None @@ -115,25 +126,24 @@ def test_text_post_falls_back_to_its_hn_permalink(): @pytest.mark.parametrize( - "override, why", + "item, why", [ ({}, "not an item at all"), - ({"type": "comment"}, "a comment, not a story"), - ({"type": "job"}, "a job ad, not a story"), - ({"title": ""}, "no title"), - ({"title": "Postgres 18 released"}, "title is not AI-related"), - ({"time": int(time.time()) - 60 * 60 * 24 * 30}, "outside the window"), + (_item(type="comment"), "a comment, not a story"), + (_item(type="job"), "a job ad, not a story"), + (_item(title=""), "no title"), + (_item(title="Postgres 18 released"), "title is not AI-related"), + (_item(posted=int(time.time()) - 60 * 60 * 24 * 30), "outside the window"), ], ) -def test_rejects(override, why): - item = {} if not override else _item(**override) +def test_rejects(item: HNItem, why: str) -> None: assert _to_article(item) is None, why -def test_an_item_with_no_time_is_kept_and_dated_now(): +def test_an_item_with_no_time_is_kept_and_dated_now() -> None: """is_within_window treats a missing date as recent on purpose — an HN item without a timestamp is a firebase oddity, not a reason to lose the story.""" - article = _to_article(_item(time=None)) + article = _to_article(_item(posted=None)) assert article is not None assert article["published_date"] @@ -141,7 +151,9 @@ def test_an_item_with_no_time_is_kept_and_dated_now(): # ─── fetch_hn: top + new ───────────────────────────────────────────────────── -def _stub_hn(monkeypatch, ids_by_feed: dict[str, list[int]]) -> list[int]: +def _stub_hn( + monkeypatch: pytest.MonkeyPatch, ids_by_feed: dict[str, list[int]] +) -> list[int]: """Run fetch_hn against canned id lists, recording which ids it looked up. Everything that touches the network is replaced: the two listing calls, the @@ -152,16 +164,18 @@ def _stub_hn(monkeypatch, ids_by_feed: dict[str, list[int]]) -> list[int]: def story_ids(label: str, _url: str) -> list[int]: return ids_by_feed.get(label, []) - def fetch_item(item_id: int) -> dict: + def fetch_item(item_id: int) -> HNItem: looked_up.append(item_id) - return _item(id=item_id, url=f"https://example.com/{item_id}") + item = _item(url=f"https://example.com/{item_id}") + item["id"] = item_id + return item monkeypatch.setattr(hn, "_story_ids", story_ids) monkeypatch.setattr(hn, "_fetch_item", fetch_item) return looked_up -def test_polls_both_the_top_and_new_listings(monkeypatch): +def test_polls_both_the_top_and_new_listings(monkeypatch: pytest.MonkeyPatch) -> None: """newstories is where a release lands before it trends — dropping it back to topstories-only silently halves the source.""" looked_up = _stub_hn(monkeypatch, {"top": [1, 2], "new": [3, 4]}) @@ -170,7 +184,9 @@ def test_polls_both_the_top_and_new_listings(monkeypatch): assert len(articles) == 4 -def test_an_id_on_both_listings_is_fetched_once(monkeypatch): +def test_an_id_on_both_listings_is_fetched_once( + monkeypatch: pytest.MonkeyPatch, +) -> None: """The lists overlap heavily — a new story that trends is on both.""" looked_up = _stub_hn(monkeypatch, {"top": [1, 2], "new": [2, 3]}) articles = hn.fetch_hn() @@ -178,12 +194,14 @@ def test_an_id_on_both_listings_is_fetched_once(monkeypatch): assert len({a["url"] for a in articles}) == 3 -def test_both_listings_failing_yields_nothing(monkeypatch): +def test_both_listings_failing_yields_nothing(monkeypatch: pytest.MonkeyPatch) -> None: _stub_hn(monkeypatch, {}) assert hn.fetch_hn() == [] -def test_items_leave_the_source_with_no_content(monkeypatch): +def test_items_leave_the_source_with_no_content( + monkeypatch: pytest.MonkeyPatch, +) -> None: """The fetch step re-fetches anything under 500 characters in full. A 500-character snippet filled here slipped past that check, so every HN candidate was stored as a teaser and the agent fetched the page again.""" @@ -198,38 +216,40 @@ def test_items_leave_the_source_with_no_content(monkeypatch): # cover the three cases in ``has_traction``. -def test_an_aged_story_nobody_read_is_held_back(): +def test_an_aged_story_nobody_read_is_held_back() -> None: assert not has_traction( - _item(time=int(time.time()) - HOUR * 20, score=3, descendants=0) + _item(posted=int(time.time()) - HOUR * 20, score=3, descendants=0) ) -def test_points_alone_clear_the_gate(): +def test_points_alone_clear_the_gate() -> None: assert has_traction( - _item(time=int(time.time()) - HOUR * 20, score=40, descendants=0) + _item(posted=int(time.time()) - HOUR * 20, score=40, descendants=0) ) -def test_discussion_alone_clears_the_gate(): +def test_discussion_alone_clears_the_gate() -> None: """A story that got argued about is real even when the votes stayed flat.""" assert has_traction( - _item(time=int(time.time()) - HOUR * 20, score=2, descendants=25) + _item(posted=int(time.time()) - HOUR * 20, score=2, descendants=25) ) -def test_a_young_story_is_held_back_not_judged(): +def test_a_young_story_is_held_back_not_judged() -> None: """Every story starts at 1 point, so nothing can be concluded yet. It is held, not rejected — the source records nothing, so the next run sees the same id again with settled numbers, still inside the 48h window.""" - assert not has_traction(_item(time=int(time.time()) - HOUR, score=1, descendants=0)) + assert not has_traction( + _item(posted=int(time.time()) - HOUR, score=1, descendants=0) + ) -def test_a_young_first_party_release_goes_straight_through(): +def test_a_young_first_party_release_goes_straight_through() -> None: """The reason newstories is polled at all: a lab's own post is news at zero points, and holding it for a day would publish it late.""" assert has_traction( _item( - time=int(time.time()) - 60, + posted=int(time.time()) - 60, score=1, descendants=0, url="https://openai.com/index/gpt-5-6/", @@ -237,31 +257,31 @@ def test_a_young_first_party_release_goes_straight_through(): ) -def test_first_party_matching_covers_subdomains(): +def test_first_party_matching_covers_subdomains() -> None: assert has_traction( _item( - time=int(time.time()) - 60, + posted=int(time.time()) - 60, score=1, url="https://platform.claude.com/docs/en/about-claude/models", ) ) -def test_the_gate_can_be_switched_off(monkeypatch): +def test_the_gate_can_be_switched_off(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr(hn, "hn_min_points", lambda: 0) assert has_traction( - _item(time=int(time.time()) - HOUR * 20, score=0, descendants=0) + _item(posted=int(time.time()) - HOUR * 20, score=0, descendants=0) ) -def test_a_story_without_traction_never_becomes_an_article(): +def test_a_story_without_traction_never_becomes_an_article() -> None: assert ( - _to_article(_item(time=int(time.time()) - HOUR * 20, score=1, descendants=0)) + _to_article(_item(posted=int(time.time()) - HOUR * 20, score=1, descendants=0)) is None ) -def test_the_article_carries_its_traction_to_the_scorer(): +def test_the_article_carries_its_traction_to_the_scorer() -> None: """Passed on as evidence, not only used as a gate: the scorer put unread Show HN repos in the 90s while a title was all it could see. diff --git a/backend/app/features/collect/test_newsletters.py b/backend/app/features/collect/test_newsletters.py new file mode 100644 index 00000000..5a4fc71b --- /dev/null +++ b/backend/app/features/collect/test_newsletters.py @@ -0,0 +1,516 @@ +"""The IMAP newsletter source. + +Sender matching and URL unwrapping: which publisher sent an email, and what +URL an item actually points at. Wrong in the first means a real newsletter is +silently skipped or a stranger's mail is attributed to a publisher; wrong in the +second means the pipeline stores a tracking link that dies with the campaign, or +stores one post twice. + +Link extraction, rule prefilter and the Jev verdict: what counts as a link and +what text surrounds it, which links are plumbing by rule, and where the +classification thresholds cut. The thresholds were measured on two real issues; +the verdict table is their specification. +""" + +from __future__ import annotations + +import pytest + +from app.core import jev +from app.features.collect import newsletters +from app.features.collect.newsletters import ( + Item, + Link, + _canonical_url, + _match_sender, + _unwrap_tracking_url, + extract, + prefilter, + verdict, +) +from baml_client.types import NewsletterItemKind, ProductLinkChoice, SearchCandidate + +SOURCES = [ + ("@substack.com", "Some Substack"), + ("news@bensbites.com", "Ben's Bites"), +] + + +def test_at_prefix_matches_any_local_part_at_that_exact_domain() -> None: + assert _match_sender("writer@substack.com", SOURCES) == "Some Substack" + + +def test_at_prefix_does_not_match_a_deeper_subdomain() -> None: + """ "@substack.com" matches "anyone@substack.com" only — the check is a + string suffix on the whole address, so a subdomain like "foo.substack.com" + changes what precedes "substack.com" and no longer ends in "@substack.com".""" + assert _match_sender("writer@foo.substack.com", SOURCES) is None + + +@pytest.mark.parametrize( + "address", + ["writer@notsubstack.com", "writer@evilsubstack.com"], +) +def test_at_prefix_does_not_match_a_lookalike_domain(address: str) -> None: + """Confirms the check is a suffix match requiring the literal "@" right + before the domain, not a raw substring search for "substack.com".""" + assert _match_sender(address, SOURCES) is None + + +def test_exact_pattern_requires_an_exact_address_match() -> None: + assert _match_sender("news@bensbites.com", SOURCES) == "Ben's Bites" + + +def test_exact_pattern_does_not_match_a_different_local_part_at_the_same_domain() -> ( + None +): + assert _match_sender("other@bensbites.com", SOURCES) is None + + +def test_matching_is_case_insensitive() -> None: + assert _match_sender("Writer@Substack.COM", SOURCES) == "Some Substack" + + +def test_no_match_returns_none() -> None: + assert _match_sender("someone@example.com", SOURCES) is None + + +def test_empty_sources_never_matches() -> None: + assert _match_sender("writer@substack.com", []) is None + + +# ─── tracker unwrapping ────────────────────────────────────────────────────── +# A newsletter links through its own redirector. The destination is what the +# pipeline stores, so recovering it from the wrapper — without a request, when +# the wrapper spells it out — is the whole job of this half of the source. + + +def test_unwraps_a_percent_encoded_destination() -> None: + assert ( + _unwrap_tracking_url( + "https://tracking.tldrnewsletter.com/CL0/" + "https:%2F%2Fexample.com%2Fposts%2Fone/1/0100019a626e0311" + ) + == "https://example.com/posts/one" + ) + + +def test_unwraps_a_literal_destination_with_its_own_path_separators() -> None: + assert ( + _unwrap_tracking_url( + "https://tracking.tldrnewsletter.com/CL0/" + "https://aws.amazon.com/build/ai-series/1/0100019a626e0311" + ) + == "https://aws.amazon.com/build/ai-series" + ) + + +def test_keeps_the_destination_query_while_dropping_the_tracker_tail() -> None: + """The tail is the tracker's message id and click token, appended after the + destination; the destination's own query must survive it.""" + assert ( + _unwrap_tracking_url( + "https://tracking.tldrnewsletter.com/CL0/" + "https:%2F%2Fexample.com%2Fp%3Fid%3D7/1/0100019a626e0311" + ) + == "https://example.com/p?id=7" + ) + + +def test_an_opaque_token_cannot_be_unwrapped() -> None: + """Nothing to recover without a request — the caller follows the redirect.""" + assert _unwrap_tracking_url("https://links.tldrnewsletter.com/zPfy8Y") is None + + +def test_a_direct_link_is_not_a_wrapper() -> None: + assert _unwrap_tracking_url("https://example.com/blog/post") is None + + +# ─── URL canonicalization ──────────────────────────────────────────────────── +# Two links to one post must reduce to one string, or dedup stores it twice. + + +def test_drops_campaign_params_and_keeps_the_rest() -> None: + assert ( + _canonical_url("https://example.com/p?id=7&utm_source=tldr&page=2") + == "https://example.com/p?id=7&page=2" + ) + + +def test_drops_campaign_params_hidden_after_a_fragment() -> None: + """Newsletters append their params after the anchor, where a query parser + never looks.""" + assert ( + _canonical_url("https://example.com/p/#section?utm_source=tldrai") + == "https://example.com/p#section" + ) + + +def test_a_trailing_slash_does_not_make_a_second_url() -> None: + assert _canonical_url("https://example.com/p/") == _canonical_url( + "https://example.com/p" + ) + + +def test_the_host_is_lowercased_but_the_path_is_not() -> None: + """Hosts are case-insensitive; paths are not, and lowercasing one would + 404.""" + assert ( + _canonical_url("https://Example.COM/Posts/One") + == "https://example.com/Posts/One" + ) + + +def test_a_bare_root_url_keeps_its_slash() -> None: + assert _canonical_url("https://example.com/") == "https://example.com/" + + +# ─── resolving a product by search ─────────────────────────────────────────── + + +def _product() -> Item: + return { + "kind": NewsletterItemKind.Product, + "name": "Acme SDK", + "description": "An SDK for agents.", + "url": "https://www.example-writeup.com/acme", + "first_party": False, + "email_date": "2026-09-23T07:00:00+00:00", + "newsletter_name": "TLDR AI", + } + + +class _Picks: + """Stands in for the BAML client: picks ``index`` from what it is shown.""" + + def __init__(self, index: int) -> None: + self.index = index + + def SelectProductLink( + self, name: str, description: str, candidates: list[SearchCandidate] + ) -> ProductLinkChoice: + return ProductLinkChoice(index=self.index, confidence=0.9) + + +def _stub_search(monkeypatch: pytest.MonkeyPatch, index: int) -> None: + results: list[dict[str, str | None]] = [ + { + "title": "Acme SDK", + "url": "https://acme.dev/sdk/", + "description": "Acme's agent SDK.", + "published_date": None, + }, + { + "title": "Acme on X", + "url": "https://x.com/acme/status/1", + "description": "denied", + "published_date": None, + }, + ] + monkeypatch.setattr(newsletters, "tavily_search", lambda query: results) + monkeypatch.setattr(newsletters, "b", _Picks(index)) + + +def test_a_product_resolves_to_the_result_picked( + monkeypatch: pytest.MonkeyPatch, +) -> None: + _stub_search(monkeypatch, 1) + item = newsletters._resolve_product(_product()) + assert item is not None + assert item["url"] == "https://acme.dev/sdk" + assert item["source"] == "TLDR AI" + + +@pytest.mark.parametrize("index", [0, 2, 9]) +def test_no_pick_or_a_pick_past_the_list_drops_the_product( + monkeypatch: pytest.MonkeyPatch, index: int +) -> None: + """The model never sees the denied x.com result, so it picks from a list + of one. Any other index is no pick. Before the bounds check it raised + IndexError, which ended the whole night's newsletter source.""" + _stub_search(monkeypatch, index) + assert newsletters._resolve_product(_product()) is None + + +# --- link extraction, prefilter and verdict --- + +ZW = "​‌‍­" * 200 + +FIXTURE = f""" + +
Today's issue‌​{ZW}
+ + + +
+ + Why agent harnesses fail (6 minute read) + A walk through production agent design and where retries go wrong. +
Acme ships Agent SDK + lets you build agents in ten lines.
+""" + + +def _by_anchor(links: list[Link]) -> dict[str, Link]: + return {link.anchor: link for link in links} + + +def test_extract_finds_every_anchor_in_order() -> None: + links = extract(FIXTURE) + assert [link.anchor for link in links] == [ + "Why agent harnesses fail (6 minute read)", + "", + "Acme ships Agent SDK", + ] + + +def test_the_context_window_reaches_the_description_after_the_link() -> None: + link = _by_anchor(extract(FIXTURE))["Why agent harnesses fail (6 minute read)"] + assert "where retries go wrong" in link.context + assert "where retries go wrong" in link.blurb + + +def test_zero_width_padding_never_reaches_the_context() -> None: + for link in extract(FIXTURE): + assert not newsletters._ZERO_WIDTH_RE.search(link.context) + first = extract(FIXTURE)[0] + assert first.context.startswith("Today's issue Why agent harnesses fail") + + +def test_nested_tags_collapse_to_their_text() -> None: + link = _by_anchor(extract(FIXTURE))["Acme ships Agent SDK"] + assert link.href == "https://acme.dev/launch" + assert link.blurb.startswith("lets you build agents in ten lines.") + + +def test_an_empty_anchor_is_still_extracted_with_its_surroundings() -> None: + link = _by_anchor(extract(FIXTURE))[""] + assert link.href == "https://example.com/post" + assert "where retries go wrong" in link.context + + +def test_entities_in_hrefs_are_decoded() -> None: + [link] = extract('Post') + assert link.href == "https://example.com/p?a=1&b=2" + + +def test_the_window_is_bounded_on_both_sides() -> None: + html = f"{'x ' * 400}A{' y' * 400}" + html = html.replace("'", '"') + [link] = extract(html) + assert ( + len(link.context) + <= newsletters.CONTEXT_BEFORE + len("A") + newsletters.CONTEXT_AFTER + ) + + +# ─── prefilter ─────────────────────────────────────────────────────────────── + + +def _link(href: str, anchor: str = "A story") -> Link: + return Link(href=href, anchor=anchor, context="") + + +def test_prefilter_drops_plumbing_and_keeps_stories() -> None: + links = [ + _link("mailto:editor@tldrnewsletter.com", "Email us"), + _link("#top", "Back to top"), + _link("https://tldrnewsletter.com/unsubscribe?ep=1", "Unsubscribe"), + _link("https://example.org/manage", "Manage your preferences"), + _link("https://x.com/tldrnewsletter", "Follow us"), + _link("https://tldrnewsletter.com/ai", "Our AI edition"), + _link("https://example.com/post?utm_source=tldr", "Post"), + _link("https://example.com/post?utm_medium=email&utm_campaign=x", "Post"), + _link( + "https://tracking.tldrnewsletter.com/CL0/" + "https:%2F%2Fwww.youtube.com%2Fwatch%3Fv%3Dabc/1/0100019a626e0311", + "Watch the demo", + ), + _link("https://acme.dev/launch", ""), + _link("https://acme.dev/launch", "Acme ships Agent SDK"), + ] + + kept, dropped = prefilter(links, "tldrnewsletter.com") + + assert [link.anchor for link in kept] == ["Post", "Acme ships Agent SDK"] + assert kept[0].href == "https://example.com/post?utm_source=tldr" + reasons = {r: [link.anchor for link in ls] for r, ls in dropped.items()} + assert reasons == { + "non_http": ["Email us", "Back to top"], + "admin": ["Unsubscribe", "Manage your preferences"], + "denied": ["Follow us", "Watch the demo"], + "own_host": ["Our AI edition"], + "duplicate": ["Post"], + "no_text": [""], + } + + +def test_the_tracker_is_unwrapped_before_the_host_check() -> None: + """The tracker lives on the newsletter's own domain; its destination does + not, and the destination is what decides.""" + tracked = _link( + "https://tracking.tldrnewsletter.com/CL0/" + "https:%2F%2Fexample.com%2Fpost/1/0100019a626e0311" + ) + kept, dropped = prefilter([tracked], "tldrnewsletter.com") + assert kept == [tracked] + assert dropped == {} + + +def test_a_tracked_link_and_a_direct_one_to_the_same_page_are_one() -> None: + kept, dropped = prefilter( + [ + _link( + "https://tracking.tldrnewsletter.com/CL0/" + "https:%2F%2Fexample.com%2Fpost%3Futm_source%3Dtldr/1/0100019a626e0311" + ), + _link("https://example.com/post/"), + ], + "tldrnewsletter.com", + ) + assert len(kept) == 1 + assert len(dropped["duplicate"]) == 1 + + +def test_an_opaque_tracker_is_kept_for_jev_to_judge() -> None: + """Beehiiv hides the destination behind a token: no host to check, and the + tracker's own host must not read as the sender's.""" + opaque = _link("https://link.mail.beehiiv.com/ss/c/u001.abcdef") + kept, _ = prefilter([opaque], "mail.beehiiv.com") + assert kept == [opaque] + + +# ─── verdict ───────────────────────────────────────────────────────────────── + +_EDITORIAL: dict[str, jev.Json] = { + "is_admin": 0.02, + "is_sponsor": 0.05, + "is_ai_topic": 0.95, + "kind": "article", + "is_first_party": 0.3, +} + + +@pytest.mark.parametrize( + ("overrides", "expected"), + [ + ({}, "keep"), + ({"is_sponsor": 0.74}, "keep"), + ({"is_sponsor": 0.76}, "sponsor"), + ({"is_admin": 0.54}, "keep"), + ({"is_admin": 0.56}, "admin"), + ({"is_ai_topic": 0.51}, "keep"), + ({"is_ai_topic": 0.49}, "off_topic"), + ({"kind": "product"}, "keep"), + ({"kind": "neither"}, "neither"), + # The measured false positive at 0.5: a launch post that reads promotional. + ({"is_sponsor": 0.64}, "keep"), + ({"is_admin": 0.9, "is_sponsor": 0.9}, "admin"), + ], +) +def test_verdict_thresholds(overrides: dict[str, jev.Json], expected: str) -> None: + assert verdict({**_EDITORIAL, **overrides}) == expected + + +# ─── classification wiring ─────────────────────────────────────────────────── + + +def test_extract_items_keeps_only_what_the_verdict_keeps( + monkeypatch: pytest.MonkeyPatch, +) -> None: + answers: dict[str, dict[str, jev.Json]] = { + "Sponsored: Try Acme free": {**_EDITORIAL, "is_sponsor": 0.96}, + "Acme ships Agent SDK": { + **_EDITORIAL, + "kind": "product", + "is_first_party": 0.9, + }, + "The Senior Engineer Death Spiral": {**_EDITORIAL, "is_ai_topic": 0.05}, + } + seen_states: list[dict[str, str]] = [] + + def ask(state: dict[str, str], _questions: object) -> jev.Reply: + seen_states.append(state) + return jev.Reply(answers=answers[state["link_text"]], cost=0.00002) + + monkeypatch.setattr(jev, "ask", ask) + html = "".join( + f'

{anchor} blurb {i}.

' + for i, anchor in enumerate(answers) + ) + + items = newsletters._extract_items( + html, "TLDR AI", "2026-09-23", "tldrnewsletter.com" + ) + + assert len(seen_states) == 3 + assert {s["destination_host"] for s in seen_states} == { + "site0.com", + "site1.com", + "site2.com", + } + [item] = items + assert item["name"] == "Acme ships Agent SDK" + assert item["kind"] == NewsletterItemKind.Product + assert item["first_party"] is True + assert item["description"].startswith("blurb 1.") + + +def test_a_failed_classification_drops_only_that_link( + monkeypatch: pytest.MonkeyPatch, +) -> None: + def ask(state: dict[str, str], _questions: object) -> jev.Reply: + if state["link_text"] == "Broken": + raise jev.JevError("unanswered: is_sponsor") + return jev.Reply(answers=_EDITORIAL, cost=0.0) + + monkeypatch.setattr(jev, "ask", ask) + html = 'Broken Fine' + + items = newsletters._extract_items(html, "TLDR AI", "2026-09-23", "tldr.tech") + + assert [i["name"] for i in items] == ["Fine"] + + +def test_an_opaque_tracker_is_never_first_party( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Jev saw "unknown" for the host, so its first-party answer is a guess.""" + monkeypatch.setattr( + jev, + "ask", + lambda state, questions: jev.Reply( + answers={**_EDITORIAL, "kind": "product", "is_first_party": 0.99}, cost=0 + ), + ) + html = 'Acme SDK' + + [item] = newsletters._extract_items(html, "Pointer", "2026-09-23", "pointer.io") + + assert item["first_party"] is False + + +def test_an_article_about_privacy_is_not_a_privacy_policy() -> None: + kept, _ = prefilter( + [ + _link("https://example.com/blog/privacy-in-llms", "Privacy in LLMs"), + _link("https://example.com/privacy", "Read more"), + ], + "tldrnewsletter.com", + ) + assert [link.anchor for link in kept] == ["Privacy in LLMs"] + + +@pytest.mark.parametrize( + ("sender", "own"), + [ + ("dan@tldrnewsletter.com", "tldrnewsletter.com"), + ("News@WWW.Example.com", "example.com"), + ("writer@substack.com", ""), + ("pointer@mail.beehiiv.com", ""), + ], +) +def test_own_host_ignores_shared_sending_platforms(sender: str, own: str) -> None: + """A Substack sender's own host would otherwise drop every post on every + other Substack.""" + assert newsletters._own_host(sender) == own diff --git a/backend/app/features/collect/test_publishers.py b/backend/app/features/collect/test_publishers.py new file mode 100644 index 00000000..47b8ca1f --- /dev/null +++ b/backend/app/features/collect/test_publishers.py @@ -0,0 +1,213 @@ +"""Crediting a URL to the publisher whose site it is on.""" + +from __future__ import annotations + +import httpx +import pytest + +from app.core.db.publisher import LinkPlatform, Publisher, PublisherKind +from app.features.collect import publishers +from app.features.collect.publishers import ( + host_kind, + hosts_to_publishers, + is_shared_host, + match_publisher, + owner_key, + owner_kind, +) + + +def _pub(id_: int, **links: str) -> Publisher: + return Publisher( + id=id_, + slug=f"p{id_}", + name=f"P{id_}", + kind=PublisherKind.media, + links={LinkPlatform(k): v for k, v in links.items()}, + ) + + +def _credited(url: str, *publishers: Publisher) -> int | None: + match = match_publisher(url, hosts_to_publishers(publishers)) + return match.id if match else None + + +def test_credits_a_url_to_the_publisher_whose_site_it_is_on() -> None: + simon = _pub(1, website="https://simonwillison.net") + assert _credited("https://simonwillison.net/2026/Aug/8/auto-mode/", simon) == 1 + + +def test_a_feed_link_names_the_host_too() -> None: + hamel = _pub(2, rss="https://hamelhusain.substack.com/feed") + assert _credited("https://hamelhusain.substack.com/p/evals", hamel) == 2 + + +def test_climbs_to_the_parent_domain() -> None: + cloudflare = _pub(3, website="https://www.cloudflare.com") + assert _credited("https://blog.cloudflare.com/ai-code-review/", cloudflare) == 3 + + +def test_a_search_link_is_a_bare_host() -> None: + anthropic = _pub(4, search="anthropic.com") + assert _credited("https://www.anthropic.com/news/claude-opus-5", anthropic) == 4 + + +def test_a_shared_platform_host_credits_nobody() -> None: + """A publisher whose site is a GitHub or Hugging Face page must not claim + every repo or model card on the platform.""" + hf_blog = _pub(5, website="https://huggingface.co/blog") + assert _credited("https://huggingface.co/Qwen/Qwen3.8-27B-FP8", hf_blog) is None + + +def test_a_host_two_publishers_claim_credits_neither() -> None: + a = _pub(6, website="https://example.com") + b = _pub(7, rss="https://example.com/feed") + assert _credited("https://example.com/post", a, b) is None + + +def test_an_unknown_host_credits_nobody() -> None: + assert ( + _credited("https://nobody-we-know.dev/post", _pub(8, website="https://a.dev")) + is None + ) + + +def test_a_platform_owner_is_credited_by_path() -> None: + anthropic = _pub( + 9, website="https://anthropic.com", github="https://github.com/anthropics" + ) + assert _credited("https://github.com/anthropics/claude-code/pull/1", anthropic) == 9 + assert _credited("https://github.com/someone-else/repo", anthropic) is None + + +def test_a_hugging_face_blog_feed_claims_the_blog_not_the_models() -> None: + hf = _pub(10, rss="https://huggingface.co/blog/feed.xml") + assert _credited("https://huggingface.co/blog/smolagents", hf) == 10 + assert _credited("https://huggingface.co/Qwen/Qwen3", hf) is None + + +@pytest.mark.parametrize( + "url, key", + [ + ("https://github.com/MattPocock/skills", "github.com/mattpocock"), + ("https://github.com/features/copilot", None), + ("https://huggingface.co/datasets/allenai/tulu", "huggingface.co/allenai"), + ("https://hf.co/Qwen/Qwen3", "huggingface.co/qwen"), + ("https://huggingface.co/papers/2609.1", None), + ("https://medium.com/feed/airbnb-engineering", "medium.com/airbnb-engineering"), + ("https://medium.com/@someone/a-post-123", "medium.com/@someone"), + ("https://medium.com/tag/ai", None), + ("https://simonwillison.net/2026/", None), + ("https://github.com/", None), + ], +) +def test_owner_key(url: str, key: str | None) -> None: + assert owner_key(url) == key + + +def test_a_personal_host_is_an_individual() -> None: + assert host_kind("someone.github.io") == PublisherKind.individual + assert host_kind("goodfire.ai") == PublisherKind.unknown + + +@pytest.mark.parametrize( + "host, shared", + [ + ("reddit.com", True), + ("old.reddit.com", True), + ("m.youtube.com", True), + ("youtu.be", True), + ("mobile.twitter.com", True), + ("gist.github.com", True), + # A subdomain of these is one writer's site. + ("writer.substack.com", False), + ("someone.medium.com", False), + ("notreddit.com", False), + ("simonwillison.net", False), + ], +) +def test_a_shared_platform_includes_its_subdomains(host: str, shared: bool) -> None: + assert is_shared_host(host) is shared + + +@pytest.mark.parametrize( + ("link", "url"), + [ + ("https://engineering.linkedin.com", "https://engineering.linkedin.com/blog/a"), + ("https://docs.github.com/en", "https://docs.github.com/en/copilot/agents"), + ("https://blog.x.com", "https://blog.x.com/engineering/en_us/a-post"), + ], +) +def test_a_publishers_own_link_claims_a_platform_subdomain(link: str, url: str) -> None: + """A platform's first-party site is the publisher's own, unlike the bare + platform host.""" + assert _credited(url, _pub(11, website=link)) == 11 + + +def test_a_substack_writer_still_claims_their_own_subdomain() -> None: + writer = _pub(12, substack="https://writer.substack.com") + assert _credited("https://writer.substack.com/p/a-post", writer) == 12 + + +def test_only_a_publishers_site_links_claim_a_host() -> None: + """An email sender or an unsaved row names no host a URL could be on.""" + sender = _pub(13, email="@example.com") + unsaved = Publisher( + slug="u", + name="U", + kind=PublisherKind.media, + links={LinkPlatform.website: "https://u.dev"}, + ) + assert hosts_to_publishers([sender, unsaved]) == {} + + +@pytest.mark.parametrize( + "key, kind", + [ + ("medium.com/@someone", PublisherKind.individual), + ("medium.com/airbnb-engineering", PublisherKind.unknown), + ("huggingface.co/blog", PublisherKind.unknown), + ("example.com/someone", PublisherKind.unknown), + ], +) +def test_owner_kind_without_a_lookup(key: str, kind: PublisherKind) -> None: + assert owner_kind(key) == kind + + +@pytest.mark.parametrize( + "account, kind", + [("User", PublisherKind.individual), ("Organization", PublisherKind.unknown)], +) +def test_a_github_owner_is_an_individual_only_when_it_is_a_user( + monkeypatch: pytest.MonkeyPatch, account: str, kind: PublisherKind +) -> None: + monkeypatch.setattr(publishers, "account_type", lambda owner: account) + assert owner_kind("github.com/someone") == kind + + +@pytest.mark.parametrize( + "status, kind", + [(200, PublisherKind.individual), (404, PublisherKind.unknown)], +) +def test_a_hugging_face_owner_is_an_individual_when_it_has_a_user_page( + monkeypatch: pytest.MonkeyPatch, status: int, kind: PublisherKind +) -> None: + seen: list[str] = [] + + def fetch(url: str, **_: object) -> httpx.Response: + seen.append(url) + return httpx.Response(status) + + monkeypatch.setattr(publishers, "fetch_with_timeout", fetch) + assert owner_kind("huggingface.co/someone") == kind + assert seen == ["https://huggingface.co/api/users/someone/overview"] + + +def test_a_failed_hugging_face_lookup_is_unknown( + monkeypatch: pytest.MonkeyPatch, +) -> None: + def fetch(*_: object, **__: object) -> httpx.Response: + raise httpx.ConnectError("offline") + + monkeypatch.setattr(publishers, "fetch_with_timeout", fetch) + assert owner_kind("huggingface.co/someone") == PublisherKind.unknown diff --git a/backend/app/features/collect/test_publishers_db.py b/backend/app/features/collect/test_publishers_db.py new file mode 100644 index 00000000..c91173cb --- /dev/null +++ b/backend/app/features/collect/test_publishers_db.py @@ -0,0 +1,260 @@ +"""Who an item is credited to, against the real schema: an aggregator's links +go to their authors, made on the spot when we have not seen the site.""" + +from __future__ import annotations + +from collections.abc import Generator + +import pytest +from sqlmodel import Session, col, delete, select + +from app.core.db.factories import random_lower_string +from app.core.db.publisher import LinkPlatform, Publisher, PublisherKind, PublisherType +from app.features.collect import publishers as publishers_module +from app.features.collect.publishers import ( + PublisherResolver, + feed_sources_from_db, + lab_watch_targets_from_db, + newsletter_senders_from_db, +) + +TAG = random_lower_string()[:8] + + +@pytest.fixture +def sources( + db: Session, monkeypatch: pytest.MonkeyPatch +) -> Generator[dict[str, Publisher]]: + monkeypatch.setattr(publishers_module, "account_type", lambda owner: "User") + made = { + "aggregator": Publisher( + name=f"{TAG} News", + slug=f"{TAG}-news", + kind=PublisherKind.community, + type=PublisherType.hn, + links={LinkPlatform.website: f"https://news.{TAG}.com"}, + ), + "feed": Publisher( + name=f"{TAG} Feed", + slug=f"{TAG}-feed", + kind=PublisherKind.company, + type=PublisherType.rss, + links={LinkPlatform.website: f"https://{TAG}-feed.com"}, + ), + } + db.add_all(made.values()) + db.commit() + yield made + db.exec(delete(Publisher).where(col(Publisher.slug).contains(TAG))) + db.commit() + + +def test_an_aggregators_link_to_a_new_site_gets_a_publisher_of_its_own( + db: Session, sources: dict[str, Publisher] +) -> None: + resolver = PublisherResolver(db) + url = f"https://blog.{TAG}.dev/a-post" + + publisher = resolver.publisher_for(url, sources["aggregator"].name) + + assert publisher.id not in (sources["aggregator"].id, sources["feed"].id) + assert publisher.name == f"blog.{TAG}.dev" + assert publisher.kind == PublisherKind.unknown + # Never polled: inactive, and only a website link. + assert publisher.is_active is False + assert publisher.links == {"website": f"https://blog.{TAG}.dev"} + # The next post on the site is credited to the same one, even in a new run. + again = PublisherResolver(db).publisher_for(f"https://blog.{TAG}.dev/b", "x") + assert again.id == publisher.id + + +def test_a_github_user_is_an_individual( + db: Session, sources: dict[str, Publisher] +) -> None: + resolver = PublisherResolver(db) + url = f"https://github.com/{TAG}owner/repo" + + publisher = resolver.publisher_for(url, sources["aggregator"].name) + + assert (publisher.name, publisher.kind) == (f"{TAG}owner", PublisherKind.individual) + assert ( + resolver.publisher_for(f"https://github.com/{TAG}owner/other", "x").id + == publisher.id + ) + + +def test_the_aggregators_own_page_stays_with_it( + db: Session, sources: dict[str, Publisher] +) -> None: + resolver = PublisherResolver(db) + url = f"https://news.{TAG}.com/item?id=1" + assert ( + resolver.publisher_for(url, sources["aggregator"].name).id + == sources["aggregator"].id + ) + + +def test_a_shared_host_that_names_no_author_stays_with_the_aggregator( + db: Session, sources: dict[str, Publisher] +) -> None: + resolver = PublisherResolver(db) + url = "https://x.com/someone/status/1" + assert ( + resolver.publisher_for(url, sources["aggregator"].name).id + == sources["aggregator"].id + ) + + +@pytest.mark.parametrize( + "url", + [ + "https://old.reddit.com/r/LocalLLaMA/comments/abc/a_thread/", + "https://m.youtube.com/watch?v=abc", + "https://youtu.be/abc", + "https://mobile.twitter.com/someone/status/1", + "https://gist.github.com/someone/abc", + ], +) +def test_a_shared_platforms_subdomain_names_no_author_either( + db: Session, sources: dict[str, Publisher], url: str +) -> None: + """A platform's subdomain is the platform, not a site of its own, so no + publisher is made for it.""" + resolver = PublisherResolver(db) + assert ( + resolver.publisher_for(url, sources["aggregator"].name).id + == sources["aggregator"].id + ) + assert resolver.created == [] + + +def test_a_platform_subdomain_a_publisher_links_goes_to_that_publisher( + db: Session, sources: dict[str, Publisher] +) -> None: + """The explicit link wins over the rule that keeps a platform's subdomains + with the aggregator.""" + engineering = Publisher( + name=f"{TAG} Engineering", + slug=f"{TAG}-engineering", + kind=PublisherKind.company, + type=PublisherType.rss, + links={LinkPlatform.website: f"https://{TAG}.linkedin.com"}, + ) + db.add(engineering) + db.commit() + resolver = PublisherResolver(db) + url = f"https://{TAG}.linkedin.com/blog/a-post" + assert resolver.publisher_for(url, sources["aggregator"].name).id == engineering.id + assert resolver.created == [] + + +def test_a_feeds_own_item_stays_with_the_feed( + db: Session, sources: dict[str, Publisher] +) -> None: + """A feed's posts may live on a host its links do not name; that is not a + new author.""" + resolver = PublisherResolver(db) + url = f"https://cdn.{TAG}-elsewhere.com/post" + assert resolver.publisher_for(url, sources["feed"].name).id == sources["feed"].id + + +@pytest.mark.usefixtures("sources") +def test_an_unknown_source_is_quarantined_once(db: Session) -> None: + """A source name the db has never seen gets one inactive publisher, and a + second item from it in the same run reuses it.""" + resolver = PublisherResolver(db) + name = f"{TAG} Brand New Feed" + + first = resolver.resolve(name) + second = resolver.resolve(name) + + assert first is second + assert (first.slug, first.is_active) == (f"{TAG}-brand-new-feed", False) + assert resolver.quarantined == [first.slug] + rows = db.exec(select(Publisher).where(Publisher.slug == first.slug)).all() + assert len(rows) == 1 + + +def test_a_new_author_never_takes_a_slug_already_in_use( + db: Session, sources: dict[str, Publisher] +) -> None: + """A GitHub owner can share a name with a publisher we already carry.""" + db.add(Publisher(name="taken", slug=f"{TAG}owner", kind=PublisherKind.media)) + db.commit() + resolver = PublisherResolver(db) + + publisher = resolver.publisher_for( + f"https://github.com/{TAG}owner/repo", sources["aggregator"].name + ) + + assert publisher.slug == f"{TAG}owner-2" + assert resolver.created == [f"github.com/{TAG}owner"] + + +def test_a_url_on_a_known_site_beats_the_feed_that_carried_it( + db: Session, sources: dict[str, Publisher] +) -> None: + """Credit by URL comes first, whichever source found the item.""" + resolver = PublisherResolver(db) + url = f"https://blog.{TAG}-feed.com/a-guest-post" + assert ( + resolver.publisher_for(url, sources["aggregator"].name).id == sources["feed"].id + ) + + +@pytest.fixture +def polled(db: Session) -> Generator[dict[str, Publisher]]: + """Active publishers with each kind of pollable link, and one switched off.""" + + def pub(name: str, active: bool = True, **links: str) -> Publisher: + return Publisher( + name=f"{TAG} {name}", + slug=f"{TAG}-{name}", + kind=PublisherKind.media, + is_active=active, + links={LinkPlatform(k): v for k, v in links.items()}, + ) + + made = { + "both": pub( + "both", + rss=f"https://{TAG}-both.com/feed.xml", + substack=f"https://{TAG}both.substack.com", + ), + "substack": pub("substack", substack=f"https://{TAG}stack.substack.com"), + "lab": pub("lab", search=f"{TAG}-lab.ai"), + "letter": pub("letter", email=f"@{TAG}-letter.com"), + "off": pub( + "off", + active=False, + rss=f"https://{TAG}-off.com/feed.xml", + search=f"{TAG}-off.ai", + email=f"@{TAG}-off.com", + ), + } + db.add_all(made.values()) + db.commit() + yield made + db.exec(delete(Publisher).where(col(Publisher.slug).contains(TAG))) + db.commit() + + +@pytest.mark.usefixtures("polled") +def test_feeds_come_from_active_publishers_preferring_rss(db: Session) -> None: + feeds = [f for f in feed_sources_from_db(db) if f["name"].startswith(TAG)] + assert sorted(feeds, key=lambda f: f["name"]) == [ + {"name": f"{TAG} both", "rssUrl": f"https://{TAG}-both.com/feed.xml"}, + {"name": f"{TAG} substack", "rssUrl": f"https://{TAG}stack.substack.com/feed"}, + ] + + +@pytest.mark.usefixtures("polled") +def test_lab_watch_targets_come_from_active_search_links(db: Session) -> None: + targets = [t for t in lab_watch_targets_from_db(db) if t[0].startswith(TAG)] + assert targets == [(f"{TAG} lab", f"{TAG}-lab.ai")] + + +@pytest.mark.usefixtures("polled") +def test_newsletter_senders_come_from_active_email_links(db: Session) -> None: + senders = [s for s in newsletter_senders_from_db(db) if s[1].startswith(TAG)] + assert senders == [(f"@{TAG}-letter.com", f"{TAG} letter")] diff --git a/backend/tests/pipeline/test_queue.py b/backend/app/features/collect/test_queue.py similarity index 64% rename from backend/tests/pipeline/test_queue.py rename to backend/app/features/collect/test_queue.py index ba0bccf3..58e0d0f4 100644 --- a/backend/tests/pipeline/test_queue.py +++ b/backend/app/features/collect/test_queue.py @@ -8,26 +8,35 @@ from __future__ import annotations -from pipeline.models import RejectStage -from pipeline.steps.queue import queue_candidates -from pipeline.types import Candidate +from typing import cast + +from sqlmodel import Session + +from app.core.db.ledger import Reject, RejectStage +from app.features.collect.model import Candidate +from app.features.collect.queue import queue_candidates class _FakeSession: """Just enough Session for the step: it records what it would record.""" def __init__(self) -> None: - self.rows: list = [] + self.rows: list[Reject] = [] self.commits = 0 - def merge(self, obj) -> None: + def merge(self, obj: Reject) -> Reject: self.rows.append(obj) + return obj def commit(self) -> None: self.commits += 1 + @property + def session(self) -> Session: + return cast(Session, self) + -def _candidate(url: str, **overrides) -> Candidate: +def _candidate(url: str, traction: str | None = None) -> Candidate: a: Candidate = { "title": "Something about agents", "url": url, @@ -36,13 +45,16 @@ def _candidate(url: str, **overrides) -> Candidate: "source": "Hacker News", "publisher_id": 7, } - return {**a, **overrides} + if traction: + a["traction"] = traction + return a -def test_every_survivor_is_queued_as_pending(): +def test_every_survivor_is_queued_as_pending() -> None: session = _FakeSession() queued = queue_candidates( - session, [_candidate("https://a.example"), _candidate("https://b.example")] + session.session, + [_candidate("https://a.example"), _candidate("https://b.example")], ) assert [row.url for row in session.rows] == [ @@ -53,10 +65,10 @@ def test_every_survivor_is_queued_as_pending(): assert len(queued) == 2 -def test_the_row_carries_what_the_agent_will_judge_by(): +def test_the_row_carries_what_the_agent_will_judge_by() -> None: session = _FakeSession() queue_candidates( - session, + session.session, [_candidate("https://a.example", traction="120 points, 40 comments")], ) @@ -68,19 +80,20 @@ def test_the_row_carries_what_the_agent_will_judge_by(): ) assert row.traction == "120 points, 40 comments" assert row.content == "body" + assert row.published_at is not None assert row.published_at.year == 2026 # No verdict yet: the agent writes both. assert (row.score, row.reason) == (None, None) -def test_the_candidates_are_committed(): +def test_the_candidates_are_committed() -> None: """A later source crashing must not take this source's candidates with it.""" session = _FakeSession() - queue_candidates(session, [_candidate("https://a.example")]) + queue_candidates(session.session, [_candidate("https://a.example")]) assert session.commits == 1 -def test_nothing_to_queue_touches_nothing(): +def test_nothing_to_queue_touches_nothing() -> None: session = _FakeSession() - assert queue_candidates(session, []) == [] + assert queue_candidates(session.session, []) == [] assert (session.rows, session.commits) == ([], 0) diff --git a/backend/tests/pipeline/test_report.py b/backend/app/features/collect/test_report.py similarity index 79% rename from backend/tests/pipeline/test_report.py rename to backend/app/features/collect/test_report.py index 5629871d..71df6821 100644 --- a/backend/tests/pipeline/test_report.py +++ b/backend/app/features/collect/test_report.py @@ -7,10 +7,11 @@ from __future__ import annotations +from dataclasses import replace from datetime import UTC, datetime, timedelta -from pipeline.models import PipelineRun -from pipeline.report import ( +from app.core.db.run import PipelineRun +from app.features.collect.report import ( STALE_RUN_HOURS, Landed, Night, @@ -27,8 +28,8 @@ def _run(hours_ago: float, ok: bool = True) -> PipelineRun: return PipelineRun(ok=ok, finished_at=NOW - timedelta(hours=hours_ago)) -def _night(**overrides) -> Night: - night = Night( +def _night() -> Night: + return Night( liveness=None, silent_sources=[], source_errors=[], @@ -44,33 +45,32 @@ def _night(**overrides) -> Night: rejected=17, still_pending=2, ) - return Night(**{**night.__dict__, **overrides}) # ─── liveness ──────────────────────────────────────────────────────────────── -def test_a_fresh_good_run_is_fine(): +def test_a_fresh_good_run_is_fine() -> None: assert liveness(_run(2), NOW) is None -def test_a_crashed_run_is_said(): +def test_a_crashed_run_is_said() -> None: assert "crashed" in (liveness(_run(2, ok=False), NOW) or "") -def test_a_stale_run_is_said(): +def test_a_stale_run_is_said() -> None: message = liveness(_run(STALE_RUN_HOURS + 4), NOW) or "" assert f"No pipeline run in {STALE_RUN_HOURS + 4}h" in message -def test_no_run_at_all_is_said(): +def test_no_run_at_all_is_said() -> None: assert liveness(None, NOW) == "No pipeline run is recorded." # ─── sources ───────────────────────────────────────────────────────────────── -def test_silent_sources_and_errors_come_from_the_stored_counts(): +def test_silent_sources_and_errors_come_from_the_stored_counts() -> None: silent, errors = source_lines( [ {"source": "Feeds", "fetched": 120, "queued": 19, "error": None}, @@ -82,7 +82,7 @@ def test_silent_sources_and_errors_come_from_the_stored_counts(): assert errors == ["Lab Watch: boom"] -def test_a_run_recorded_in_the_old_shape_still_reports(): +def test_a_run_recorded_in_the_old_shape_still_reports() -> None: """Old rows carry an ``errors`` list and fields that are gone. The mail is how a reader finds out the pipeline is unwell, so it cannot be the thing that breaks on them.""" @@ -96,7 +96,7 @@ def test_a_run_recorded_in_the_old_shape_still_reports(): # ─── the mail ──────────────────────────────────────────────────────────────── -def test_what_landed_is_named_one_by_one(): +def test_what_landed_is_named_one_by_one() -> None: body = format_night(_night()) assert "LANDED (1)" in body assert "[86/100] Gemma 4 open model family — Google DeepMind Blog" in body @@ -106,14 +106,15 @@ def test_what_landed_is_named_one_by_one(): assert subject(_night()) == "1 article(s) landed" -def test_a_quiet_night_has_no_problem_section(): +def test_a_quiet_night_has_no_problem_section() -> None: assert "NEEDS A LOOK" not in format_night(_night()) assert "PIPELINE" not in format_night(_night()) -def test_problems_come_before_the_articles(): +def test_problems_come_before_the_articles() -> None: body = format_night( - _night( + replace( + _night(), liveness="The last run crashed (2026-09-23 04:10 UTC).", silent_sources=["Newsletter"], source_errors=["Lab Watch: boom"], @@ -124,16 +125,16 @@ def test_problems_come_before_the_articles(): assert "Newsletter fetched nothing." in body assert "Lab Watch: boom" in body assert "Cloudflare: RuntimeError: Status code 403" in body - assert subject(_night(liveness="x")) == "Pipeline needs a look" + assert subject(replace(_night(), liveness="x")) == "Pipeline needs a look" -def test_an_empty_night_says_where_to_look(): - body = format_night(_night(landed=[], rejected=0, still_pending=40)) +def test_an_empty_night_says_where_to_look() -> None: + body = format_night(replace(_night(), landed=[], rejected=0, still_pending=40)) assert "LANDED (0)" in body assert "curation session" in body assert "40 candidate(s) still waiting" in body - assert subject(_night(landed=[])) == "Nothing landed" + assert subject(replace(_night(), landed=[])) == "Nothing landed" -def test_an_empty_queue_is_said_plainly(): - assert "Queue empty" in format_night(_night(still_pending=0)) +def test_an_empty_queue_is_said_plainly() -> None: + assert "Queue empty" in format_night(replace(_night(), still_pending=0)) diff --git a/backend/tests/pipeline/test_runs_db.py b/backend/app/features/collect/test_runs.py similarity index 88% rename from backend/tests/pipeline/test_runs_db.py rename to backend/app/features/collect/test_runs.py index 872d3312..71c9cdc4 100644 --- a/backend/tests/pipeline/test_runs_db.py +++ b/backend/app/features/collect/test_runs.py @@ -8,11 +8,13 @@ import pytest from sqlmodel import Session, col, delete, select -from app.catalog.models import Article, Publisher, PublisherKind, PublisherType -from pipeline.models import PipelineRun -from pipeline.report import read_night -from pipeline.runs import RunStats, record_new, record_polled, record_run -from tests.random_data import random_lower_string +from app.core.db.article import Article +from app.core.db.factories import random_lower_string +from app.core.db.publisher import Publisher, PublisherKind, PublisherType +from app.core.db.run import PipelineRun +from app.features.collect.model import RunStats +from app.features.collect.report import read_night +from app.features.collect.runs import record_new, record_polled, record_run TAG = random_lower_string()[:8] EARLIER = datetime(2026, 9, 20, 4, 0, tzinfo=UTC) @@ -47,7 +49,7 @@ def _fresh(db: Session, p: Publisher) -> Publisher: def test_a_good_fetch_stamps_and_clears_a_failed_one_keeps_the_last_good( db: Session, feeds: dict[str, Publisher] -): +) -> None: names = [p.name for p in feeds.values()] record_polled(db, names, {f"{TAG} Broken": "RuntimeError: 403"}, now=NOW) @@ -62,7 +64,7 @@ def test_a_good_fetch_stamps_and_clears_a_failed_one_keeps_the_last_good( def test_a_queued_candidate_stamps_its_publisher_new( db: Session, feeds: dict[str, Publisher] -): +) -> None: record_new(db, [feeds["Healed"].id or 0], now=NOW) assert _fresh(db, feeds["Healed"]).last_new_at == NOW assert _fresh(db, feeds["Quiet"]).last_new_at is None @@ -70,7 +72,7 @@ def test_a_queued_candidate_stamps_its_publisher_new( def test_the_night_reads_the_last_run_and_the_failing_feeds( db: Session, feeds: dict[str, Publisher] -): +) -> None: names = [p.name for p in feeds.values()] record_polled(db, names, {feeds["Broken"].name: "boom"}) db.add( diff --git a/backend/app/features/collect/test_service.py b/backend/app/features/collect/test_service.py new file mode 100644 index 00000000..8ec1f4dd --- /dev/null +++ b/backend/app/features/collect/test_service.py @@ -0,0 +1,327 @@ +"""What the run records: per-source counts, and the publishers it polled. + +On 2026-09-03 the Feeds source died after the fetch and took the per-publisher +record with it, so every feed was invisible on exactly the night something was +wrong. These pin that the publisher bookkeeping survives a failure after the +fetch, that a failed fetch stamps nothing, that one source failing leaves the +others running, and that the stored error stays a readable length. +""" + +from __future__ import annotations + +from collections.abc import Callable, Iterable +from dataclasses import dataclass, field + +import pytest + +from app.features.collect import service as run_module +from app.features.collect.fetch import Source +from app.features.collect.model import Candidate, RawItem, RunStats + + +class _FakeSession: + def __init__(self) -> None: + self.rollbacks = 0 + + def __enter__(self) -> _FakeSession: + return self + + def __exit__(self, *exc: object) -> None: + return None + + def rollback(self) -> None: + self.rollbacks += 1 + + +class _FakeResolver: + quarantined: list[str] = [] + created: list[str] = [] + + def __init__(self, **kwargs: object) -> None: + pass + + +def _article(url: str, source: str, publisher_id: int = 1) -> Candidate: + return { + "title": "t", + "url": url, + "content": "body", + "published_date": None, + "source": source, + "publisher_id": publisher_id, + } + + +def _source( + label: str, *articles: Candidate, publisher_names: tuple[str, ...] = () +) -> Source: + return Source(label, lambda: (list(articles), {}), publisher_names) + + +@dataclass +class _Seen: + """What the publisher bookkeeping was handed, and the session the run used.""" + + polled: list[tuple[Iterable[str], dict[str, str]]] = field(default_factory=list) + new: list[list[int]] = field(default_factory=list) + session: _FakeSession = field(default_factory=_FakeSession) + + +def _stub_run( + monkeypatch: pytest.MonkeyPatch, + sources: list[Source], + **overrides: object, +) -> tuple[RunStats, _Seen]: + """Run collect over ``sources`` with every step a no-op passthrough.""" + seen = _Seen() + + def record_polled( + _session: object, names: Iterable[str], errors: dict[str, str] + ) -> None: + seen.polled.append((names, errors)) + + def record_new(_session: object, ids: Iterable[int]) -> None: + seen.new.append(sorted(ids)) + + def resolve(articles: list[RawItem], _resolver: object) -> list[RawItem]: + return articles + + def passthrough(_session: object, articles: list[Candidate]) -> list[Candidate]: + return articles + + def keep(_session: object, articles: list[RawItem], _label: str) -> list[RawItem]: + return articles + + monkeypatch.setattr(run_module, "engine", None) + monkeypatch.setattr(run_module, "Session", lambda engine: seen.session) + monkeypatch.setattr(run_module, "PublisherResolver", _FakeResolver) + monkeypatch.setattr(run_module, "build_sources", lambda session: sources) + monkeypatch.setattr(run_module, "resolve_publishers", resolve) + monkeypatch.setattr(run_module, "filter_known_urls", keep) + monkeypatch.setattr(run_module, "queue_candidates", passthrough) + monkeypatch.setattr(run_module, "fetch_source", lambda source: source.fetcher()) + monkeypatch.setattr(run_module, "record_polled", record_polled) + monkeypatch.setattr(run_module, "record_new", record_new) + for name, value in overrides.items(): + monkeypatch.setattr(run_module, name, value) + + stats = RunStats() + run_module.run(stats) + return stats, seen + + +def _feeds_source() -> Source: + return _source( + "Feeds", + _article("u1", "Feed A", publisher_id=7), + publisher_names=("Feed A", "Feed B"), + ) + + +def test_a_clean_run_counts_and_stamps(monkeypatch: pytest.MonkeyPatch) -> None: + stats, seen = _stub_run(monkeypatch, [_feeds_source()]) + + [source] = stats.sources + assert (source.fetched, source.queued, source.error) == (1, 1, None) + assert seen.polled == [(("Feed A", "Feed B"), {})] + assert seen.new == [[7]] + + +def test_polled_publishers_are_stamped_after_a_failure_past_the_fetch( + monkeypatch: pytest.MonkeyPatch, +) -> None: + def boom(_session: object, _articles: object) -> list[Candidate]: + raise RuntimeError("queue down") + + stats, seen = _stub_run(monkeypatch, [_feeds_source()], queue_candidates=boom) + + assert seen.polled == [(("Feed A", "Feed B"), {})] + # Nothing was queued, so nothing is new. + assert seen.new == [[]] + assert stats.sources[0].error == "RuntimeError: queue down" + + +def test_a_fetch_that_failed_stamps_no_publisher( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Nothing was polled, so there was no answer to record — stamping every + publisher as fetched would hide a source that is down.""" + + def boom(_source: object) -> tuple[list[RawItem], dict[str, str]]: + raise RuntimeError("feeds unreachable") + + stats, seen = _stub_run(monkeypatch, [_feeds_source()], fetch_source=boom) + + assert seen.polled == [] + assert stats.sources[0].error == "RuntimeError: feeds unreachable" + + +def test_a_failed_source_rolls_its_session_back( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """The next source shares the session; a failed flush left in it would + fail every source after it.""" + + def boom(_session: object, _articles: object) -> list[Candidate]: + raise RuntimeError("queue down") + + _, seen = _stub_run(monkeypatch, [_feeds_source()], queue_candidates=boom) + + assert seen.session.rollbacks == 1 + + +def test_the_errors_a_fetch_reports_reach_the_bookkeeping( + monkeypatch: pytest.MonkeyPatch, +) -> None: + def fetch(_source: object) -> tuple[list[RawItem], dict[str, str]]: + return [], {"Feed B": "RuntimeError: Status code 403"} + + _, seen = _stub_run(monkeypatch, [_feeds_source()], fetch_source=fetch) + + assert seen.polled == [ + (("Feed A", "Feed B"), {"Feed B": "RuntimeError: Status code 403"}) + ] + + +def test_bookkeeping_that_fails_does_not_fail_the_source( + monkeypatch: pytest.MonkeyPatch, +) -> None: + def boom(*_args: object) -> None: + raise RuntimeError("db gone") + + stats, _ = _stub_run(monkeypatch, [_feeds_source()], record_polled=boom) + + assert (stats.sources[0].queued, stats.sources[0].error) == (1, None) + + +def test_one_source_failing_does_not_stop_the_next( + monkeypatch: pytest.MonkeyPatch, +) -> None: + def boom(_session: object, articles: list[Candidate]) -> list[Candidate]: + if articles[0]["source"] == "Feed A": + raise RuntimeError("queue down") + return articles + + stats, seen = _stub_run( + monkeypatch, + [_feeds_source(), _source("Hacker News", _article("u2", "HN"))], + queue_candidates=boom, + ) + + assert [(s.source, s.queued, s.error) for s in stats.sources] == [ + ("Feeds", 0, "RuntimeError: queue down"), + ("Hacker News", 1, None), + ] + assert seen.new == [[], [1]] + + +def test_the_stored_error_is_truncated(monkeypatch: pytest.MonkeyPatch) -> None: + def boom(_session: object, _articles: object) -> list[Candidate]: + raise RuntimeError("x" * 50_000) + + stats, _ = _stub_run(monkeypatch, [_feeds_source()], queue_candidates=boom) + + error = stats.sources[0].error or "" + assert len(error) < 1_000 + assert error.endswith("chars]") + + +# ─── run_nightly ───────────────────────────────────────────────────────────── + + +def _stub_nightly( + monkeypatch: pytest.MonkeyPatch, + run: Callable[[RunStats], None], + record: Callable[[object, RunStats], object] | None = None, +) -> list[RunStats]: + recorded: list[RunStats] = [] + + def record_run(_session: object, stats: RunStats) -> None: + recorded.append(stats) + + monkeypatch.setattr(run_module, "engine", None) + monkeypatch.setattr(run_module, "Session", lambda engine: _FakeSession()) + monkeypatch.setattr(run_module, "run", run) + monkeypatch.setattr(run_module, "record_run", record or record_run) + return recorded + + +def test_a_clean_night_exits_zero_and_records_an_ok_run( + monkeypatch: pytest.MonkeyPatch, +) -> None: + recorded = _stub_nightly(monkeypatch, lambda stats: None) + + assert run_module.run_nightly() == 0 + [stats] = recorded + assert stats.ok and stats.finished_at is not None + + +def test_a_crash_exits_one_and_is_still_recorded( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """The daily mail reads the recorded row: a crash that recorded nothing + would read as a run that never happened.""" + + def crash(_stats: RunStats) -> None: + raise RuntimeError("db unreachable") + + recorded = _stub_nightly(monkeypatch, crash) + + assert run_module.run_nightly() == 1 + [stats] = recorded + assert not stats.ok + + +def test_recording_that_fails_does_not_flip_the_exit_code( + monkeypatch: pytest.MonkeyPatch, +) -> None: + def broken(_session: object, _stats: RunStats) -> None: + raise RuntimeError("db gone") + + _stub_nightly(monkeypatch, lambda stats: None, record=broken) + + assert run_module.run_nightly() == 0 + + +# ─── send_report ───────────────────────────────────────────────────────────── + + +def test_a_report_that_cannot_be_read_exits_one( + monkeypatch: pytest.MonkeyPatch, +) -> None: + def broken(_session: object) -> None: + raise RuntimeError("db gone") + + monkeypatch.setattr(run_module, "engine", None) + monkeypatch.setattr(run_module, "Session", lambda engine: _FakeSession()) + monkeypatch.setattr(run_module, "read_night", broken) + + assert run_module.send_report() == 1 + + +def test_a_report_is_read_from_the_db_and_mailed( + monkeypatch: pytest.MonkeyPatch, +) -> None: + sent: list[tuple[str, str]] = [] + monkeypatch.setattr( + run_module, "send_mail", lambda subject, body: sent.append((subject, body)) + ) + + assert run_module.send_report() == 0 + [(subject, body)] = sent + assert subject and body + + +def test_a_report_that_cannot_be_sent_exits_one( + monkeypatch: pytest.MonkeyPatch, +) -> None: + tried: list[str] = [] + + def broken(subject: str, _body: str) -> None: + tried.append(subject) + raise RuntimeError("resend down") + + monkeypatch.setattr(run_module, "send_mail", broken) + + assert run_module.send_report() == 1 + assert len(tried) == 1 diff --git a/backend/tests/pipeline/test_titles.py b/backend/app/features/collect/test_titles.py similarity index 82% rename from backend/tests/pipeline/test_titles.py rename to backend/app/features/collect/test_titles.py index 44da9023..722c43a9 100644 --- a/backend/tests/pipeline/test_titles.py +++ b/backend/app/features/collect/test_titles.py @@ -4,7 +4,7 @@ import pytest -from pipeline.titles import clean_title +from app.features.collect.titles import clean_title @pytest.mark.parametrize( @@ -21,5 +21,5 @@ ("Why Show HN posts do well", "Why Show HN posts do well"), ], ) -def test_clean_title_strips_only_a_leading_hn_prefix(given, expected): +def test_clean_title_strips_only_a_leading_hn_prefix(given: str, expected: str) -> None: assert clean_title(given) == expected diff --git a/backend/tests/pipeline/test_tweets.py b/backend/app/features/collect/test_tweets.py similarity index 58% rename from backend/tests/pipeline/test_tweets.py rename to backend/app/features/collect/test_tweets.py index e1213f37..debad554 100644 --- a/backend/tests/pipeline/test_tweets.py +++ b/backend/app/features/collect/test_tweets.py @@ -9,24 +9,36 @@ import pytest -from pipeline.fetching import tweets +from app.features.collect import tweets +from app.features.collect.model import RawItem +from app.features.collect.tweets import Tweet _TWEET_URL = "https://x.com/cantinasecurity/status/2078158487759290614" -def _item(url: str, content: str = "a blurb") -> dict: - return {"title": "A title", "url": url, "content": content, "source": "Feeds"} +def _item(url: str, content: str = "a blurb") -> RawItem: + return { + "title": "A title", + "url": url, + "content": content, + "published_date": None, + "source": "Feeds", + } -def _tweet(text: str, *links: str) -> dict: - facets = [{"type": "url", "replacement": link} for link in links] - return {"text": text, "raw_text": {"text": text, "facets": facets}} +def _tweet(text: str, *links: str) -> Tweet: + return { + "text": text, + "raw_text": { + "facets": [{"type": "url", "replacement": link} for link in links] + }, + } -def _stub(monkeypatch, tweet: dict | None) -> list[str]: +def _stub(monkeypatch: pytest.MonkeyPatch, tweet: Tweet | None) -> list[str]: asked: list[str] = [] - def fetch_tweet(status_id: str) -> dict | None: + def fetch_tweet(status_id: str) -> Tweet | None: asked.append(status_id) return tweet @@ -44,11 +56,13 @@ def fetch_tweet(status_id: str) -> dict | None: ("https://example.com/a/status/1", None), ], ) -def test_tweet_id(url, expected): +def test_tweet_id(url: str, expected: str | None) -> None: assert tweets.tweet_id(url) == expected -def test_a_tweet_that_links_out_becomes_that_link(monkeypatch): +def test_a_tweet_that_links_out_becomes_that_link( + monkeypatch: pytest.MonkeyPatch, +) -> None: link = "https://cantina.review/cantina-agentic-5a0a34" _stub(monkeypatch, _tweet("Full story:", f"{_TWEET_URL}/photo/1", link)) [item] = tweets.unwrap_tweets([_item(_TWEET_URL)]) @@ -56,7 +70,9 @@ def test_a_tweet_that_links_out_becomes_that_link(monkeypatch): assert item["content"] == "" -def test_a_tweet_with_no_link_keeps_its_full_text(monkeypatch): +def test_a_tweet_with_no_link_keeps_its_full_text( + monkeypatch: pytest.MonkeyPatch, +) -> None: text = "A long thread opener. " * 20 _stub(monkeypatch, _tweet(text)) [item] = tweets.unwrap_tweets([_item(_TWEET_URL)]) @@ -64,13 +80,17 @@ def test_a_tweet_with_no_link_keeps_its_full_text(monkeypatch): assert item["content"] == text -def test_a_tweet_fxtwitter_cannot_return_is_left_as_it_came(monkeypatch): +def test_a_tweet_fxtwitter_cannot_return_is_left_as_it_came( + monkeypatch: pytest.MonkeyPatch, +) -> None: _stub(monkeypatch, None) [item] = tweets.unwrap_tweets([_item(_TWEET_URL)]) assert item == _item(_TWEET_URL) -def test_items_that_are_not_tweets_are_never_looked_up(monkeypatch): +def test_items_that_are_not_tweets_are_never_looked_up( + monkeypatch: pytest.MonkeyPatch, +) -> None: asked = _stub(monkeypatch, None) tweets.unwrap_tweets([_item("https://example.com/post")]) assert asked == [] diff --git a/backend/pipeline/titles.py b/backend/app/features/collect/titles.py similarity index 100% rename from backend/pipeline/titles.py rename to backend/app/features/collect/titles.py diff --git a/backend/pipeline/fetching/tweets.py b/backend/app/features/collect/tweets.py similarity index 98% rename from backend/pipeline/fetching/tweets.py rename to backend/app/features/collect/tweets.py index d2f111da..79d01a59 100644 --- a/backend/pipeline/fetching/tweets.py +++ b/backend/app/features/collect/tweets.py @@ -22,7 +22,7 @@ from app.core.http import fetch_with_timeout from app.core.logging import log from app.core.urls import hostname -from pipeline.types import RawItem +from app.features.collect.model import RawItem FXTWITTER_URL = "https://api.fxtwitter.com/status/{id}" TIMEOUT_SECS = 5.0 diff --git a/backend/app/newsletter/tests/__init__.py b/backend/app/features/curation/__init__.py similarity index 100% rename from backend/app/newsletter/tests/__init__.py rename to backend/app/features/curation/__init__.py diff --git a/backend/app/features/curation/coverage.py b/backend/app/features/curation/coverage.py new file mode 100644 index 00000000..aeac2b9f --- /dev/null +++ b/backend/app/features/curation/coverage.py @@ -0,0 +1,138 @@ +"""The rows `similar` and `stories` compare a candidate against: the feed's +recent articles and the ledger, each with the vector or the text it embeds to. + +The distance maths is in `app.core.embedding`; this module loads the rows and +counts who covers a story. +""" + +from __future__ import annotations + +from datetime import datetime + +import numpy as np +import numpy.typing as npt +from sqlmodel import Session, col, select + +from app.core.db.article import Article +from app.core.db.ledger import Reject, RejectStage +from app.core.db.publisher import Publisher +from app.core.embedding import SNIPPET_CAP, embed_batch, to_embedding_text +from app.features.curation.model import CandidateError, Neighbour + +# Stage `similar` and `stories` report for a row that is an Article. +PUBLISHED = "published" + + +def covering( + publishers: list[str], + distances: npt.NDArray[np.float32], + own: str | None, + cutoff: float, +) -> list[str]: + """The distinct publishers within ``cutoff``, plus the candidate's own. Pure.""" + names = {p for p, d in zip(publishers, distances, strict=True) if d <= cutoff} + if own: + names.add(own) + return sorted(n for n in names if n) + + +def publisher_names(session: Session) -> dict[int, str]: + rows = session.exec(select(Publisher.id, Publisher.name)).all() + return {pid: name for pid, name in rows if pid is not None} + + +def feed_rows( + session: Session, since: datetime, names: dict[int, str] +) -> tuple[list[Neighbour], list[list[float]]]: + """Articles created since ``since`` that carry an embedding, and those + embeddings.""" + articles = session.exec( + select(Article).where( + col(Article.created_at) >= since, + col(Article.embedding).is_not(None), + ) + ).all() + out: list[Neighbour] = [] + vecs: list[list[float]] = [] + for a in articles: + # pgvector hands back a numpy array, and bool(array) raises. + if a.embedding is None: + continue + out.append( + { + "url": a.url, + "title": a.title, + "publisher": names.get(a.publisher_id, ""), + "stage": PUBLISHED, + "score": a.score, + } + ) + vecs.append(list(a.embedding)) + return out, vecs + + +def ledger_rows( + session: Session, + names: dict[int, str], + *, + since: datetime | None = None, + pending_only: bool = False, +) -> tuple[list[Neighbour], list[str]]: + """Ledger rows and the text each one embeds to: the same title and + ``SNIPPET_CAP`` characters an Article's stored embedding is built from.""" + query = select(Reject) + if since is not None: + query = query.where(col(Reject.created_at) >= since) + if pending_only: + query = query.where(col(Reject.stage) == RejectStage.pending) + out: list[Neighbour] = [] + texts: list[str] = [] + for row in session.exec(query).all(): + out.append( + { + "url": row.url, + "title": row.title or "", + "publisher": names.get(row.publisher_id, "") + if row.publisher_id + else "", + "stage": str(row.stage or ""), + "score": row.score, + } + ) + texts.append( + to_embedding_text(row.title or "", (row.content or "")[:SNIPPET_CAP]) + ) + return out, texts + + +def target( + session: Session, url: str, names: dict[int, str] +) -> tuple[Neighbour, npt.NDArray[np.float32]]: + """The row ``similar`` compares from, and its vector.""" + row = session.get(Reject, url) + if row is not None: + text = to_embedding_text(row.title or "", (row.content or "")[:SNIPPET_CAP]) + publisher = names.get(row.publisher_id, "") if row.publisher_id else "" + target: Neighbour = { + "url": url, + "title": row.title or "", + "publisher": publisher, + "stage": str(row.stage or ""), + "score": row.score, + } + return target, np.array(embed_batch([text])[0], dtype=np.float32) + + article = session.exec(select(Article).where(col(Article.url) == url)).first() + if article is None: + raise CandidateError(f"No candidate or article for {url}") + published: Neighbour = { + "url": url, + "title": article.title, + "publisher": names.get(article.publisher_id, ""), + "stage": PUBLISHED, + "score": article.score, + } + if article.embedding is not None: + return published, np.array(article.embedding, dtype=np.float32) + text = to_embedding_text(article.title, (article.content or "")[:SNIPPET_CAP]) + return published, np.array(embed_batch([text])[0], dtype=np.float32) diff --git a/backend/pipeline/fetching/page_facts.py b/backend/app/features/curation/links.py similarity index 98% rename from backend/pipeline/fetching/page_facts.py rename to backend/app/features/curation/links.py index 264b2b9d..0a76aa39 100644 --- a/backend/pipeline/fetching/page_facts.py +++ b/backend/app/features/curation/links.py @@ -15,6 +15,7 @@ from app.core.github import github_repo_from_url, has_readme, repo_for from app.core.http import fetch_with_timeout from app.core.urls import hostname +from app.features.curation.model import LinkError # What any lookup that did not answer reads as. Never 0 or False: a rate limit # must not look like a repo nobody starred or a model with no weights. @@ -37,10 +38,6 @@ type Facts = dict[str, object] -class LinkError(Exception): - """A URL ``check_link`` does not read. The message names the tool that does.""" - - def _or_unknown(value: object) -> object: return UNKNOWN if value is None or value == "" else value diff --git a/backend/app/features/curation/model.py b/backend/app/features/curation/model.py new file mode 100644 index 00000000..d024e737 --- /dev/null +++ b/backend/app/features/curation/model.py @@ -0,0 +1,117 @@ +"""Curation's shapes: what the agent reads about a candidate, the verdicts it +hands back, and the article as approving builds it. + +The stage types pick up where collect's ``Candidate`` ends, at the ledger row: +approving reads the row and builds a ``Summarized`` from it, which +``publish.insert_articles`` turns into a ``Persisted``. They repeat the fields +rather than subclass collect's types, so the two features share nothing but +the table between them. +""" + +from __future__ import annotations + +from typing import NotRequired, TypedDict + + +class CandidateError(Exception): + """No pending candidate at that URL — approved already, rejected already, + or never queued. The agent gets the message, not a traceback.""" + + +class LinkError(Exception): + """A URL ``check_link`` does not read. The message names the tool that does.""" + + +class CandidatePage(TypedDict): + """One page of the pending queue. ``next_offset`` is None on the last page.""" + + rows: list[dict[str, str]] + offset: int + next_offset: int | None + total: int + + +class ContentPage(TypedDict): + """One slice of a candidate's stored text. ``next_offset`` is None on the + last page; ``links`` rides on the first page only.""" + + text: str + offset: int + next_offset: int | None + total: int + links: NotRequired[dict[str, list[str]]] + + +class Neighbour(TypedDict): + """A row beside a candidate: an Article (stage ``published``) or a ledger row.""" + + url: str + title: str + publisher: str + stage: str + score: int | None + distance: NotRequired[float] + + +class Similar(TypedDict): + url: str + # Distinct publishers carrying this story, the candidate's own included. + coverage: int + publishers: list[str] + rows: list[Neighbour] + + +class Coverage(TypedDict): + coverage: int + publishers: list[str] + members: list[Neighbour] + + +class Verdict(TypedDict): + """One rejection in a `reject_many` batch.""" + + url: str + score: int + reason: str + + +class Scored(TypedDict): + """A candidate the curation agent approved, with the score it gave.""" + + title: str + url: str + content: str + published_date: str | None + source: str + publisher_id: int + score: int + # The agent's one-sentence account of the score. + score_reason: str | None + + +class Summarized(Scored): + """A scored article with the summary the agent wrote. Nothing is inserted + without one.""" + + summary: str + + +class Persisted(Summarized): + """A scored article that is now a row: ``id`` is the Article's primary key. + + ``title`` and ``content`` are the sanitized values actually stored, not the + raw ones the source emitted. + """ + + id: int + + +class ProcessedArticle(TypedDict): + """What the embedder needs: a stored article reduced to the fields the + embedding text is built from.""" + + id: int + url: str + title: str + score: int + snippet: str diff --git a/backend/pipeline/steps/persist.py b/backend/app/features/curation/publish.py similarity index 61% rename from backend/pipeline/steps/persist.py rename to backend/app/features/curation/publish.py index cc0a338f..a6d21cb8 100644 --- a/backend/pipeline/steps/persist.py +++ b/backend/app/features/curation/publish.py @@ -1,14 +1,25 @@ -"""Step 5: insert the articles that passed scoring and were summarized.""" +"""Writing an approved article: the Article row, then its embedding. + +The embedding is the one piece of enrichment left after the curation agent +writes the labels. Local and best-effort: a failed embedding costs the field, +never the article. +""" from __future__ import annotations from sqlmodel import Session -from app.catalog.models import Article from app.core.dates import parse_date +from app.core.db.article import Article +from app.core.embedding import embed_batch, to_embedding_text from app.core.llm_text import sanitize_llm_text from app.core.logging import log -from pipeline.types import Persisted, Scored, Summarized +from app.features.curation.model import ( + Persisted, + ProcessedArticle, + Scored, + Summarized, +) def best_per_url[T: Scored](scored: list[T]) -> list[T]: @@ -51,3 +62,25 @@ def insert_articles(session: Session, summarized: list[Summarized]) -> list[Pers session.commit() return inserted + + +def embed_articles(session: Session, items: list[ProcessedArticle]) -> None: + if not items: + return + + log(f" Embedding {len(items)} articles...") + + texts = [to_embedding_text(item["title"], item.get("snippet")) for item in items] + try: + vecs = embed_batch(texts) + except Exception as e: + log(f" Embed batch failed, continuing: {e}") + return + + for item, vec in zip(items, vecs, strict=True): + article = session.get(Article, item["id"]) + if article: + article.embedding = vec + session.add(article) + + session.commit() diff --git a/backend/pipeline/curation.py b/backend/app/features/curation/service.py similarity index 68% rename from backend/pipeline/curation.py rename to backend/app/features/curation/service.py index e51a1af1..b1b9ca34 100644 --- a/backend/pipeline/curation.py +++ b/backend/app/features/curation/service.py @@ -1,11 +1,10 @@ """What the curation agent does to a candidate: read it, compare it, approve it, reject it. -The nightly run ends at a `pending` row (`steps/queue.py`). This module is the -other end: the verbs the agent drives it with, behind the MCP tools in -`app/mcp/tools.py`. Approving is the only thing in the codebase that writes an -Article now, so the whole insert-and-enrich tail lives here, reusing the -pipeline's own steps rather than a second copy of them. +The nightly collect run ends at a `pending` row. This module is the other end: +the verbs the agent drives it with, behind the MCP tools in `app/entry/mcp/`. +Approving is the only thing in the codebase that writes an Article, so the +insert-and-embed tail (`publish.py`) belongs to curation alone. The agent read the page, so it writes everything a reader sees: the summary, the kind and the tags, and the kind of a publisher the pipeline has only just @@ -18,35 +17,48 @@ Comparing is by embedding: `similar` and `stories` put a candidate beside what the feed already carries and what else is queued, so the agent can tell a story that three outlets covered from one only its publisher did. The distance maths -is pure numpy in `app.core.embedding`, kept apart from the queries, so it is -tested on synthetic vectors with no model load. +is pure numpy in `app.core.embedding`, and the row loading is in `coverage.py`, +so the maths is tested on synthetic vectors with no model load. """ from __future__ import annotations from datetime import UTC, datetime, timedelta -from typing import NotRequired, TypedDict import numpy as np from sqlmodel import Session, col, func, select -from app.catalog.models import Article, ArticleKind, Publisher, PublisherKind +from app.core.db.article import Article, ArticleKind, kind_from_url +from app.core.db.ledger import CANDIDATE_CAP, CONTENT_CAP, Reject, RejectStage +from app.core.db.publisher import Publisher, PublisherKind from app.core.embedding import ( + DEDUP_DIST_THRESHOLD, + RELATED_DIST, SNIPPET_CAP, cosine_distances, embed_batch, nearest, story_groups, - to_embedding_text, ) from app.core.logging import log -from pipeline.models import Reject, RejectStage -from pipeline.rejects import CANDIDATE_CAP, CONTENT_CAP -from pipeline.steps.enrich import embed_articles -from pipeline.steps.persist import insert_articles -from pipeline.tags import load_vocabulary, validate_tags, write_article_tags -from pipeline.types import Summarized -from pipeline.url_kind import kind_from_url +from app.features.curation import coverage +from app.features.curation.links import check_link as check_link +from app.features.curation.model import ( + CandidateError, + CandidatePage, + ContentPage, + Coverage, + Neighbour, + Similar, + Summarized, + Verdict, +) +from app.features.curation.publish import embed_articles, insert_articles +from app.features.curation.tags import ( + load_vocabulary, + validate_tags, + write_article_tags, +) # What the agent triages on. Deliberately not the stored text: a list of 80 # candidates at full content is more than a reading pass can hold, and the @@ -61,20 +73,6 @@ APPROVAL_WINDOW_DAYS = 90 # One page of `get_content`. Enough to triage on; an approval reads on to the end. PAGE_LIMIT = 4000 -# Two texts this close are the same story. Same-story pairs sit at 0.29-0.35 -# with the potion-base-8M embeddings; tight on purpose, because coverage counts -# what falls under it and a loose cut inflates every story. -DEDUP_DIST_THRESHOLD = 0.30 -# `similar` lists neighbours out to here: related enough to read the title, not -# close enough to count as the same story. -RELATED_DIST = 0.45 -# Stage `similar` and `stories` report for a row that is an Article. -PUBLISHED = "published" - - -class CandidateError(Exception): - """No pending candidate at that URL — approved already, rejected already, - or never queued. The agent gets the message, not a traceback.""" def _pending(session: Session, url: str) -> Reject: @@ -123,21 +121,12 @@ def approval_counts( counts: dict[int, tuple[int, int]] = {} for pid, n in approved: counts[pid] = (n, 0) - for pid, n in rejected: - if pid is not None: - counts[pid] = (counts.get(pid, (0, 0))[0], n) + for rejected_pid, n in rejected: + if rejected_pid is not None: + counts[rejected_pid] = (counts.get(rejected_pid, (0, 0))[0], n) return counts -class CandidatePage(TypedDict): - """One page of the pending queue. ``next_offset`` is None on the last page.""" - - rows: list[dict[str, str]] - offset: int - next_offset: int | None - total: int - - def list_candidates( session: Session, offset: int = 0, limit: int = LIST_LIMIT ) -> CandidatePage: @@ -197,17 +186,6 @@ def list_candidates( } -class ContentPage(TypedDict): - """One slice of a candidate's stored text. ``next_offset`` is None on the - last page; ``links`` rides on the first page only.""" - - text: str - offset: int - next_offset: int | None - total: int - links: NotRequired[dict[str, list[str]]] - - def page_bounds(total: int, offset: int, limit: int) -> tuple[int, int]: """``(start, end)`` of one page of a ``total``-char text. Pure. @@ -225,7 +203,7 @@ def get_content( ) -> ContentPage: """One page of the article text the pipeline extracted for a candidate. - The stored text is up to `rejects.CANDIDATE_CAP` characters, the same text + The stored text is up to `ledger.CANDIDATE_CAP` characters, the same text a page fetch would return, so the agent reads from here and fetches only when it is missing or cut short. """ @@ -246,143 +224,6 @@ def get_content( # ─── Similar and stories ───────────────────────────────────────────────────── -class Neighbour(TypedDict): - """A row beside a candidate: an Article (stage ``published``) or a ledger row.""" - - url: str - title: str - publisher: str - stage: str - score: int | None - distance: NotRequired[float] - - -class Similar(TypedDict): - url: str - # Distinct publishers carrying this story, the candidate's own included. - coverage: int - publishers: list[str] - rows: list[Neighbour] - - -class Story(TypedDict): - coverage: int - publishers: list[str] - members: list[Neighbour] - - -def covering( - publishers: list[str], distances: np.ndarray, own: str | None, cutoff: float -) -> list[str]: - """The distinct publishers within ``cutoff``, plus the candidate's own. Pure.""" - names = {p for p, d in zip(publishers, distances, strict=True) if d <= cutoff} - if own: - names.add(own) - return sorted(n for n in names if n) - - -def _publisher_names(session: Session) -> dict[int, str]: - rows = session.exec(select(Publisher.id, Publisher.name)).all() - return {pid: name for pid, name in rows if pid is not None} - - -def _feed_rows( - session: Session, since: datetime, names: dict[int, str] -) -> tuple[list[Neighbour], list[list[float]]]: - """Articles created since ``since`` that carry an embedding, and those - embeddings.""" - articles = session.exec( - select(Article).where( - col(Article.created_at) >= since, - col(Article.embedding).is_not(None), - ) - ).all() - out: list[Neighbour] = [] - vecs: list[list[float]] = [] - for a in articles: - # pgvector hands back a numpy array, and bool(array) raises. - if a.embedding is None: - continue - out.append( - { - "url": a.url, - "title": a.title, - "publisher": names.get(a.publisher_id, ""), - "stage": PUBLISHED, - "score": a.score, - } - ) - vecs.append(list(a.embedding)) - return out, vecs - - -def _ledger_rows( - session: Session, - names: dict[int, str], - *, - since: datetime | None = None, - pending_only: bool = False, -) -> tuple[list[Neighbour], list[str]]: - """Ledger rows and the text each one embeds to: the same title and - ``SNIPPET_CAP`` characters an Article's stored embedding is built from.""" - query = select(Reject) - if since is not None: - query = query.where(col(Reject.created_at) >= since) - if pending_only: - query = query.where(col(Reject.stage) == RejectStage.pending) - out: list[Neighbour] = [] - texts: list[str] = [] - for row in session.exec(query).all(): - out.append( - { - "url": row.url, - "title": row.title or "", - "publisher": names.get(row.publisher_id, "") - if row.publisher_id - else "", - "stage": str(row.stage or ""), - "score": row.score, - } - ) - texts.append( - to_embedding_text(row.title or "", (row.content or "")[:SNIPPET_CAP]) - ) - return out, texts - - -def _target( - session: Session, url: str, names: dict[int, str] -) -> tuple[Neighbour, np.ndarray]: - """The row ``similar`` compares from, and its vector.""" - row = session.get(Reject, url) - if row is not None: - text = to_embedding_text(row.title or "", (row.content or "")[:SNIPPET_CAP]) - publisher = names.get(row.publisher_id, "") if row.publisher_id else "" - target: Neighbour = { - "url": url, - "title": row.title or "", - "publisher": publisher, - "stage": str(row.stage or ""), - "score": row.score, - } - return target, np.array(embed_batch([text])[0], dtype=np.float32) - - article = session.exec(select(Article).where(col(Article.url) == url)).first() - if article is None: - raise CandidateError(f"No candidate or article for {url}") - published: Neighbour = { - "url": url, - "title": article.title, - "publisher": names.get(article.publisher_id, ""), - "stage": PUBLISHED, - "score": article.score, - } - if article.embedding is not None: - return published, np.array(article.embedding, dtype=np.float32) - text = to_embedding_text(article.title, (article.content or "")[:SNIPPET_CAP]) - return published, np.array(embed_batch([text])[0], dtype=np.float32) - - def similar(session: Session, url: str, days: int = 7, limit: int = 8) -> Similar: """What the feed and the ledger hold that is close to one candidate. @@ -392,12 +233,12 @@ def similar(session: Session, url: str, days: int = 7, limit: int = 8) -> Simila ``DEDUP_DIST_THRESHOLD``, the candidate's own included, over every row in the window rather than just the listed ones. """ - names = _publisher_names(session) - target, target_vec = _target(session, url, names) + names = coverage.publisher_names(session) + target, target_vec = coverage.target(session, url, names) since = datetime.now(UTC) - timedelta(days=days) - feed, feed_vecs = _feed_rows(session, since, names) - ledger, ledger_texts = _ledger_rows(session, names, since=since) + feed, feed_vecs = coverage.feed_rows(session, since, names) + ledger, ledger_texts = coverage.ledger_rows(session, names, since=since) ledger_vecs = embed_batch(ledger_texts) rows = [r for r in feed + ledger if r["url"] != url] @@ -417,7 +258,7 @@ def similar(session: Session, url: str, days: int = 7, limit: int = 8) -> Simila distances = cosine_distances(target_vec[None, :], np.array(vecs, dtype=np.float32))[ 0 ] - publishers = covering( + publishers = coverage.covering( [r["publisher"] for r in rows], distances, target["publisher"], @@ -436,7 +277,7 @@ def similar(session: Session, url: str, days: int = 7, limit: int = 8) -> Simila } -def stories(session: Session, days: int = 7) -> list[Story]: +def stories(session: Session, days: int = 7) -> list[Coverage]: """The pending candidates that are one story, with each other or with something the feed published in the last ``days``. @@ -444,20 +285,20 @@ def stories(session: Session, days: int = 7) -> list[Story]: Article in it is a story several outlets are carrying tonight; a group with one is a story the feed already has. """ - names = _publisher_names(session) + names = coverage.publisher_names(session) since = datetime.now(UTC) - timedelta(days=days) - queued, queued_texts = _ledger_rows(session, names, pending_only=True) + queued, queued_texts = coverage.ledger_rows(session, names, pending_only=True) if not queued: return [] - feed, feed_vecs = _feed_rows(session, since, names) + feed, feed_vecs = coverage.feed_rows(session, since, names) rows = queued + feed vecs = np.array(embed_batch(queued_texts) + feed_vecs, dtype=np.float32) pending = [True] * len(queued) + [False] * len(feed) groups = story_groups(cosine_distances(vecs, vecs), pending, DEDUP_DIST_THRESHOLD) - out: list[Story] = [] + out: list[Coverage] = [] for group in groups: members = [rows[i] for i in group] publishers = sorted({m["publisher"] for m in members if m["publisher"]}) @@ -603,14 +444,6 @@ def approve( return persisted["id"] -class Verdict(TypedDict): - """One rejection in a `reject_many` batch.""" - - url: str - score: int - reason: str - - def reject(session: Session, url: str, score: int, reason: str) -> None: """Turn a candidate down. The row stays, carrying the agent's verdict. diff --git a/backend/pipeline/tags.py b/backend/app/features/curation/tags.py similarity index 77% rename from backend/pipeline/tags.py rename to backend/app/features/curation/tags.py index 5e84954a..5f528ba4 100644 --- a/backend/pipeline/tags.py +++ b/backend/app/features/curation/tags.py @@ -1,19 +1,20 @@ -"""Controlled tag vocabulary (loaded from the DB) + validation for AssignTags. - -The vocabulary is the `tag` table — slug, display name, and the "when to apply -this" description that gets fed to the LLM. Nothing is hardcoded here: adding or -renaming a tag is a DB change. We still post-validate the LLM output against the -loaded slug set and drop anything off-list, and we never mint tags at runtime. +"""The controlled tag vocabulary, loaded from the db, and the check on the tags +the curation agent passes to `approve`. + +The vocabulary is the `tag` table: slug, display name, and the "when to apply +this" description `vocabulary` shows the agent. Nothing is hardcoded here: +adding or renaming a tag is a db change. Tags the agent passes are validated +against the loaded slug set and anything off-list is dropped; tags are never +minted at runtime. """ from __future__ import annotations from dataclasses import dataclass -from typing import Any from sqlmodel import Session, select -from app.catalog.models import ArticleTag, Tag +from app.core.db.tag import ArticleTag, Tag from app.core.llm_text import enum_value from app.core.logging import log @@ -42,7 +43,7 @@ @dataclass(frozen=True) class Vocabulary: - """The controlled tag set for one pipeline run.""" + """The controlled tag set, read once per call.""" slug_to_id: dict[str, int] slug_to_description: dict[str, str] @@ -53,7 +54,7 @@ def slugs(self) -> frozenset[str]: def load_vocabulary(session: Session) -> Vocabulary: - """Read the whole `tag` table once per run. + """Read the whole `tag` table. Raises if it's empty — an unseeded vocabulary would silently tag nothing, which is worse than failing loudly. @@ -69,11 +70,11 @@ def load_vocabulary(session: Session) -> Vocabulary: ) -def normalize_tag(raw: Any, valid: frozenset[str]) -> str | None: +def normalize_tag(raw: object, valid: frozenset[str]) -> str | None: """Coerce a returned tag to a canonical slug, or None if it's off-list. - The model returns plain strings now, but tolerate stray casing/underscores - before validating against the vocabulary. + Tolerates stray casing and underscores before validating against the + vocabulary. """ if raw is None: return None @@ -81,7 +82,7 @@ def normalize_tag(raw: Any, valid: frozenset[str]) -> str | None: return slug if slug in valid else None -def validate_tags(raw_tags: list[Any], valid: frozenset[str]) -> list[str]: +def validate_tags(raw_tags: list[str], valid: frozenset[str]) -> list[str]: """Normalize, drop off-list, dedupe (order-preserving), cap subject tags at 3. Lab tags are kept uncapped.""" seen: dict[str, None] = {} diff --git a/backend/tests/pipeline/test_page_facts.py b/backend/app/features/curation/test_links.py similarity index 85% rename from backend/tests/pipeline/test_page_facts.py rename to backend/app/features/curation/test_links.py index ef7f9e16..7a9d7901 100644 --- a/backend/tests/pipeline/test_page_facts.py +++ b/backend/app/features/curation/test_links.py @@ -13,8 +13,9 @@ from app.core import github from app.core.extract import MAX_LINKS_PER_GROUP, group_links, outbound_links -from pipeline.fetching import page_facts -from pipeline.fetching.page_facts import UNKNOWN, LinkError, check_link +from app.features.curation import links as page_facts +from app.features.curation.links import UNKNOWN, check_link +from app.features.curation.model import LinkError PAGE = "https://blog.example.com/2026/09/release" @@ -47,7 +48,7 @@ """ -def test_groups_the_links_an_article_body_makes(): +def test_groups_the_links_an_article_body_makes() -> None: links = outbound_links(_RELEASE_HTML, PAGE) assert links == { "model": [ @@ -63,17 +64,17 @@ def test_groups_the_links_an_article_body_makes(): } -def test_nav_and_footer_links_are_not_the_articles(): +def test_nav_and_footer_links_are_not_the_articles() -> None: links = outbound_links(_RELEASE_HTML, PAGE) assert "https://github.com/example/website" not in links.get("repo", []) assert "https://blog.example.com/about" not in links.get("docs", []) -def test_empty_html_has_no_links(): +def test_empty_html_has_no_links() -> None: assert outbound_links("", PAGE) == {} -def test_every_blob_of_one_repo_is_the_repo(): +def test_every_blob_of_one_repo_is_the_repo() -> None: urls = [ "https://github.com/a/b", "https://github.com/a/b/pull/12", @@ -85,7 +86,7 @@ def test_every_blob_of_one_repo_is_the_repo(): } -def test_papers_come_from_every_paper_host(): +def test_papers_come_from_every_paper_host() -> None: urls = [ "https://arxiv.org/pdf/2609.01234", "https://openreview.net/forum?id=abc", @@ -95,7 +96,7 @@ def test_papers_come_from_every_paper_host(): assert group_links(urls, PAGE) == {"paper": urls} -def test_hugging_face_site_pages_are_not_models(): +def test_hugging_face_site_pages_are_not_models() -> None: urls = [ "https://huggingface.co/docs/transformers", "https://huggingface.co/blog/example", @@ -106,16 +107,16 @@ def test_hugging_face_site_pages_are_not_models(): assert group_links(urls, PAGE) == {"docs": ["https://huggingface.co/blog/example"]} -def test_links_to_nowhere_in_particular_are_dropped(): +def test_links_to_nowhere_in_particular_are_dropped() -> None: urls = ["https://x.com/a", "https://news.ycombinator.com/item?id=1", "mailto:a@b"] assert group_links(urls, PAGE) == {} -def test_a_link_back_to_the_page_is_not_a_link(): +def test_a_link_back_to_the_page_is_not_a_link() -> None: assert group_links([PAGE, f"{PAGE}#comments"], PAGE) == {} -def test_each_group_is_capped(): +def test_each_group_is_capped() -> None: urls = [f"https://arxiv.org/abs/2609.{i:05d}" for i in range(25)] assert len(group_links(urls, PAGE)["paper"]) == MAX_LINKS_PER_GROUP @@ -135,11 +136,13 @@ def json(self) -> object: return self._body -def _stub_http(monkeypatch, answers: dict[str, _Response]) -> list[str]: +def _stub_http( + monkeypatch: pytest.MonkeyPatch, answers: dict[str, _Response] +) -> list[str]: """Answer each API URL from ``answers`` (by prefix); 404 otherwise.""" asked: list[str] = [] - def fetch(url: str, **_options: object): + def fetch(url: str, **_options: object) -> _Response: asked.append(url) for prefix, response in answers.items(): if url.startswith(prefix): @@ -161,7 +164,7 @@ def fetch(url: str, **_options: object): } -def test_a_github_repo_maps_to_its_facts(monkeypatch): +def test_a_github_repo_maps_to_its_facts(monkeypatch: pytest.MonkeyPatch) -> None: _stub_http( monkeypatch, { @@ -185,7 +188,7 @@ def test_a_github_repo_maps_to_its_facts(monkeypatch): } -def test_a_repo_with_no_readme_says_so(monkeypatch): +def test_a_repo_with_no_readme_says_so(monkeypatch: pytest.MonkeyPatch) -> None: _stub_http( monkeypatch, { @@ -196,7 +199,9 @@ def test_a_repo_with_no_readme_says_so(monkeypatch): assert check_link("https://github.com/a/b")["has_readme"] is False -def test_a_rate_limited_github_lookup_is_unknown_not_zero(monkeypatch): +def test_a_rate_limited_github_lookup_is_unknown_not_zero( + monkeypatch: pytest.MonkeyPatch, +) -> None: _stub_http( monkeypatch, {"https://api.github.com/": _Response(403, text="API rate limit exceeded")}, @@ -207,7 +212,7 @@ def test_a_rate_limited_github_lookup_is_unknown_not_zero(monkeypatch): assert facts == {"url": "https://github.com/a/b", "kind": "repo", "lookup": UNKNOWN} -def test_a_missing_field_is_unknown_not_zero(monkeypatch): +def test_a_missing_field_is_unknown_not_zero(monkeypatch: pytest.MonkeyPatch) -> None: repo = {**GITHUB_REPO, "license": None, "stargazers_count": None} _stub_http( monkeypatch, @@ -222,7 +227,7 @@ def test_a_missing_field_is_unknown_not_zero(monkeypatch): assert (facts["license"], facts["stars"], facts["has_readme"]) == (UNKNOWN,) * 3 -def test_a_gitlab_project_maps_to_its_facts(monkeypatch): +def test_a_gitlab_project_maps_to_its_facts(monkeypatch: pytest.MonkeyPatch) -> None: asked = _stub_http( monkeypatch, { @@ -259,7 +264,9 @@ def test_a_gitlab_project_maps_to_its_facts(monkeypatch): } -def test_a_hugging_face_model_maps_to_its_facts(monkeypatch): +def test_a_hugging_face_model_maps_to_its_facts( + monkeypatch: pytest.MonkeyPatch, +) -> None: asked = _stub_http( monkeypatch, { @@ -295,7 +302,9 @@ def test_a_hugging_face_model_maps_to_its_facts(monkeypatch): } -def test_a_model_repo_with_only_a_readme_has_no_weights(monkeypatch): +def test_a_model_repo_with_only_a_readme_has_no_weights( + monkeypatch: pytest.MonkeyPatch, +) -> None: _stub_http( monkeypatch, { @@ -318,7 +327,9 @@ def test_a_model_repo_with_only_a_readme_has_no_weights(monkeypatch): ) -def test_a_hugging_face_dataset_has_no_weights_field(monkeypatch): +def test_a_hugging_face_dataset_has_no_weights_field( + monkeypatch: pytest.MonkeyPatch, +) -> None: _stub_http( monkeypatch, { @@ -335,7 +346,9 @@ def test_a_hugging_face_dataset_has_no_weights_field(monkeypatch): assert "weights" not in facts -def test_a_failed_hugging_face_lookup_is_unknown(monkeypatch): +def test_a_failed_hugging_face_lookup_is_unknown( + monkeypatch: pytest.MonkeyPatch, +) -> None: _stub_http(monkeypatch, {}) facts = check_link("https://huggingface.co/org/Model-7B") assert facts["lookup"] == UNKNOWN @@ -352,7 +365,9 @@ def test_a_failed_hugging_face_lookup_is_unknown(monkeypatch): "https://huggingface.co/blog/release", ], ) -def test_anything_else_is_refused_toward_web_fetch(monkeypatch, url): +def test_anything_else_is_refused_toward_web_fetch( + monkeypatch: pytest.MonkeyPatch, url: str +) -> None: asked = _stub_http(monkeypatch, {}) with pytest.raises(LinkError, match="web_fetch"): check_link(url) diff --git a/backend/app/features/curation/test_publish.py b/backend/app/features/curation/test_publish.py new file mode 100644 index 00000000..e4fac289 --- /dev/null +++ b/backend/app/features/curation/test_publish.py @@ -0,0 +1,46 @@ +"""Which article wins when two candidates share a URL.""" + +from __future__ import annotations + +from app.features.curation.model import Scored +from app.features.curation.publish import best_per_url + + +def _article(url: str, source: str = "Hacker News", score: int = 50) -> Scored: + return { + "title": "A title", + "url": url, + "content": "body", + "published_date": None, + "source": source, + "publisher_id": 1, + "score": score, + "score_reason": None, + } + + +def test_same_url_from_two_sources_keeps_the_higher_score() -> None: + """Two sources can surface one URL in a run (an HN post and a feed item for + the same post); the URL is the identity, so the better score wins.""" + items = [ + _article("dupe", source="Hacker News", score=40), + _article("dupe", source="Feeds", score=90), + ] + best = best_per_url(items) + assert len(best) == 1 + assert best[0]["score"] == 90 + assert best[0]["source"] == "Feeds" + + +def test_higher_score_wins_regardless_of_order() -> None: + items = [_article("dupe", score=90), _article("dupe", score=40)] + assert best_per_url(items)[0]["score"] == 90 + + +def test_distinct_urls_all_survive() -> None: + items = [_article("u1", score=80), _article("u2", score=70)] + assert len(best_per_url(items)) == 2 + + +def test_empty_input() -> None: + assert best_per_url([]) == [] diff --git a/backend/tests/pipeline/test_curation.py b/backend/app/features/curation/test_service.py similarity index 62% rename from backend/tests/pipeline/test_curation.py rename to backend/app/features/curation/test_service.py index 407218a9..36bdf246 100644 --- a/backend/tests/pipeline/test_curation.py +++ b/backend/app/features/curation/test_service.py @@ -9,56 +9,67 @@ from __future__ import annotations +from dataclasses import dataclass, field from datetime import UTC, datetime +from typing import cast import numpy as np import pytest - -from app.catalog.models import ( - Article, - ArticleKind, - Publisher, - PublisherKind, - PublisherType, - TrustLevel, +from sqlmodel import Session + +from app.core.db.article import Article, ArticleKind +from app.core.db.ledger import CANDIDATE_CAP, CONTENT_CAP, Reject, RejectStage +from app.core.db.publisher import Publisher, PublisherKind, PublisherType, TrustLevel +from app.features.curation import coverage +from app.features.curation import service as curation +from app.features.curation.model import ( + CandidateError, + Persisted, + ProcessedArticle, + Summarized, ) -from app.core import embedding -from pipeline import curation -from pipeline.models import Reject, RejectStage -from pipeline.rejects import CANDIDATE_CAP, CONTENT_CAP -from pipeline.tags import Vocabulary +from app.features.curation.tags import Vocabulary class _FakeSession: """Just enough Session for the module: `get` by primary key, and a record of what was written.""" - def __init__(self, *rows) -> None: + def __init__(self, *rows: Reject | Publisher) -> None: self.rejects = {r.url: r for r in rows if isinstance(r, Reject)} self.publishers = {p.id: p for p in rows if isinstance(p, Publisher)} - self.articles: dict = {} - self.deleted: list = [] - self.added: list = [] + self.articles: dict[int, Article] = {} + self.deleted: list[object] = [] + self.added: list[object] = [] self.commits = 0 + self.rollbacks = 0 + self.fail_commits = 0 - def get(self, model, key): + def get(self, model: type[object], key: str | int) -> object: if model is Reject: - return self.rejects.get(key) + return self.rejects.get(str(key)) if model is Article: - return self.articles.get(key) - return self.publishers.get(key) + return self.articles.get(int(key)) + return self.publishers.get(int(key)) - def delete(self, obj) -> None: + def delete(self, obj: object) -> None: self.deleted.append(obj) - def add(self, obj) -> None: + def add(self, obj: object) -> None: self.added.append(obj) def commit(self) -> None: + if self.fail_commits: + self.fail_commits -= 1 + raise RuntimeError("connection reset") self.commits += 1 def rollback(self) -> None: - self.rollbacks = getattr(self, "rollbacks", 0) + 1 + self.rollbacks += 1 + + @property + def session(self) -> Session: + return cast(Session, self) def _publisher(kind: PublisherKind = PublisherKind.individual) -> Publisher: @@ -73,7 +84,7 @@ def _publisher(kind: PublisherKind = PublisherKind.individual) -> Publisher: ) -def _pending_row(**overrides) -> Reject: +def _pending_row(**overrides: object) -> Reject: row = Reject( url="https://simonwillison.net/post", stage=RejectStage.pending, @@ -95,38 +106,68 @@ def _pending_row(**overrides) -> Reject: ) +@dataclass +class _Published: + """What the insert, the tags and the embedding were handed.""" + + inserted: list[Summarized] = field(default_factory=list) + tags: list[str] = field(default_factory=list) + embedded: list[ProcessedArticle] = field(default_factory=list) + + @pytest.fixture -def stub_publish(monkeypatch: pytest.MonkeyPatch) -> dict: +def stub_publish(monkeypatch: pytest.MonkeyPatch) -> _Published: """Stand in for the insert and the embedding, and record what they were handed.""" - seen: dict = {} + seen = _Published() - def insert(_session, items): - seen["inserted"] = items + def insert(_session: object, items: list[Summarized]) -> list[Persisted]: + seen.inserted = items return [{**items[0], "id": 42}] + def write_tags( + _session: object, _article_id: int, slugs: list[str], _vocab: object + ) -> None: + seen.tags = slugs + + def embed(_session: object, items: list[ProcessedArticle]) -> None: + seen.embedded = items + monkeypatch.setattr(curation, "insert_articles", insert) monkeypatch.setattr(curation, "load_vocabulary", lambda session: VOCAB) - monkeypatch.setattr( - curation, - "write_article_tags", - lambda _s, article_id, slugs, vocab: seen.setdefault("tags", slugs), - ) - monkeypatch.setattr( - curation, "embed_articles", lambda _s, items: seen.setdefault("embedded", items) - ) + monkeypatch.setattr(curation, "write_article_tags", write_tags) + monkeypatch.setattr(curation, "embed_articles", embed) return seen -def _approve(session, url, score=84, reason="r", summary="s", **labels): - labels = { - "kind": "post", - "tags": ["agents"], - } | labels - return curation.approve(session, url, score, reason, summary, **labels) +def _approve( + session: _FakeSession, + url: str, + score: int = 84, + reason: str = "r", + summary: str = "s", + *, + kind: str = "post", + tags: tuple[str, ...] = ("agents",), + publisher_kind: str | None = None, + publisher_name: str | None = None, +) -> int: + return curation.approve( + session.session, + url, + score, + reason, + summary, + kind, + list(tags), + publisher_kind, + publisher_name, + ) -def test_approving_publishes_and_clears_the_candidate(stub_publish: dict): +def test_approving_publishes_and_clears_the_candidate( + stub_publish: _Published, +) -> None: row = _pending_row() session = _FakeSession(row, _publisher()) @@ -138,7 +179,7 @@ def test_approving_publishes_and_clears_the_candidate(stub_publish: dict): # Deleted, not left pending: the URL is an Article now, and # `filter_known_urls` reads either one. assert session.deleted == [row] - [item] = stub_publish["inserted"] + [item] = stub_publish.inserted assert (item["score"], item["score_reason"]) == ( 84, "Open weights, downloadable today.", @@ -147,26 +188,26 @@ def test_approving_publishes_and_clears_the_candidate(stub_publish: dict): assert item["publisher_id"] == 7 # The stored text is the article: approving never fetches the page again. assert item["content"] == "the stored two thousand characters" - assert stub_publish["embedded"][0]["id"] == 42 + assert stub_publish.embedded[0]["id"] == 42 -def test_the_agents_labels_are_written(stub_publish: dict): +def test_the_agents_labels_are_written(stub_publish: _Published) -> None: row = _pending_row() - article = Article(id=42, title="t", url=row.url, publisher_id=7) + article = Article(id=42, title="t", url=row.url, publisher_id=7, score=84) session = _FakeSession(row, _publisher()) session.articles = {42: article} - _approve(session, row.url, kind="Post", tags=["agents", "not-a-tag"]) + _approve(session, row.url, kind="Post", tags=("agents", "not-a-tag")) assert article.kind == ArticleKind.post # Off-list tags are dropped, never minted. - assert stub_publish["tags"] == ["agents"] + assert stub_publish.tags == ["agents"] @pytest.mark.usefixtures("stub_publish") -def test_the_aggregator_that_found_it_is_kept(): +def test_the_aggregator_that_found_it_is_kept() -> None: row = _pending_row(source="Hacker News") - article = Article(id=42, title="t", url=row.url, publisher_id=7) + article = Article(id=42, title="t", url=row.url, publisher_id=7, score=84) session = _FakeSession(row, _publisher()) session.articles = {42: article} @@ -176,9 +217,9 @@ def test_the_aggregator_that_found_it_is_kept(): @pytest.mark.usefixtures("stub_publish") -def test_a_feeds_own_item_was_found_via_nothing(): +def test_a_feeds_own_item_was_found_via_nothing() -> None: row = _pending_row(source="Simon Willison") - article = Article(id=42, title="t", url=row.url, publisher_id=7) + article = Article(id=42, title="t", url=row.url, publisher_id=7, score=84) session = _FakeSession(row, _publisher()) session.articles = {42: article} @@ -188,7 +229,7 @@ def test_a_feeds_own_item_was_found_via_nothing(): @pytest.mark.usefixtures("stub_publish") -def test_approving_names_a_new_publisher(): +def test_approving_names_a_new_publisher() -> None: publisher = _publisher(PublisherKind.unknown) row = _pending_row() session = _FakeSession(row, publisher) @@ -200,7 +241,7 @@ def test_approving_names_a_new_publisher(): @pytest.mark.usefixtures("stub_publish") -def test_a_publishers_kind_once_set_stays(): +def test_a_publishers_kind_once_set_stays() -> None: """The agent cannot relabel a publisher from one night to the next.""" publisher = _publisher(PublisherKind.individual) row = _pending_row() @@ -213,10 +254,10 @@ def test_a_publishers_kind_once_set_stays(): @pytest.mark.usefixtures("stub_publish") -def test_the_url_host_overrules_the_agents_kind(): +def test_the_url_host_overrules_the_agents_kind() -> None: url = "https://github.com/someone/thing" row = _pending_row(url=url) - article = Article(id=42, title="t", url=url, publisher_id=7) + article = Article(id=42, title="t", url=url, publisher_id=7, score=84) session = _FakeSession(row, _publisher()) session.articles = {42: article} @@ -235,25 +276,32 @@ def test_the_url_host_overrules_the_agents_kind(): ], ) def test_a_bad_label_leaves_the_candidate_pending( - stub_publish: dict, labels: dict, message: str -): + stub_publish: _Published, labels: dict[str, str], message: str +) -> None: row = _pending_row() unknown = "publisher_kind" in labels or not labels session = _FakeSession( row, _publisher(PublisherKind.unknown if unknown else PublisherKind.individual) ) - with pytest.raises(curation.CandidateError, match=message): - _approve(session, row.url, **labels) + with pytest.raises(CandidateError, match=message): + _approve( + session, + row.url, + kind=labels.get("kind", "post"), + publisher_kind=labels.get("publisher_kind"), + ) assert session.deleted == [] - assert "inserted" not in stub_publish + assert stub_publish.inserted == [] -def test_rejecting_keeps_the_row_with_the_verdict_on_it(): +def test_rejecting_keeps_the_row_with_the_verdict_on_it() -> None: row = _pending_row() session = _FakeSession(row) - curation.reject(session, row.url, 32, "Console steps for one vendor's service.") + curation.reject( + session.session, row.url, 32, "Console steps for one vendor's service." + ) assert row.stage == RejectStage.below_threshold assert (row.score, row.reason) == (32, "Console steps for one vendor's service.") @@ -263,40 +311,42 @@ def test_rejecting_keeps_the_row_with_the_verdict_on_it(): assert session.deleted == [] -def test_rejecting_trims_the_full_text_back_to_the_ledger_cap(): +def test_rejecting_trims_the_full_text_back_to_the_ledger_cap() -> None: """A candidate carries the whole article so the agent can read it; a settled reject keeps only what the ledger keeps for every other reject.""" row = _pending_row(content="x" * (CONTENT_CAP * 5)) session = _FakeSession(row) - curation.reject(session, row.url, 20, "reason") + curation.reject(session.session, row.url, 20, "reason") - assert len(row.content) == CONTENT_CAP + assert len(row.content or "") == CONTENT_CAP -def test_a_url_that_was_never_queued_is_a_message_not_a_traceback(): +def test_a_url_that_was_never_queued_is_a_message_not_a_traceback() -> None: session = _FakeSession() - with pytest.raises(curation.CandidateError, match="No candidate for"): - curation.reject(session, "https://unknown.example", 10, "reason") + with pytest.raises(CandidateError, match="No candidate for"): + curation.reject(session.session, "https://unknown.example", 10, "reason") -def test_a_candidate_cannot_be_judged_twice(): +def test_a_candidate_cannot_be_judged_twice() -> None: row = _pending_row(stage=RejectStage.below_threshold) session = _FakeSession(row, _publisher()) - with pytest.raises(curation.CandidateError, match="already"): + with pytest.raises(CandidateError, match="already"): _approve(session, row.url, 90) -def test_a_candidate_with_no_publisher_is_not_published(stub_publish: dict): +def test_a_candidate_with_no_publisher_is_not_published( + stub_publish: _Published, +) -> None: """Every Article has a publisher; a candidate that somehow lost one is a bug to look at, not a row to force in.""" row = _pending_row(publisher_id=None) session = _FakeSession(row) - with pytest.raises(curation.CandidateError, match="no publisher"): + with pytest.raises(CandidateError, match="no publisher"): _approve(session, row.url, 90) - assert "inserted" not in stub_publish + assert stub_publish.inserted == [] # ─── approval rate ─────────────────────────────────────────────────────────── @@ -306,19 +356,19 @@ def test_a_candidate_with_no_publisher_is_not_published(stub_publish: dict): "approved, rejected, expected", [(3, 7, "3/10"), (0, 4, "0/4"), (2, 0, "2/2"), (0, 0, "new")], ) -def test_approval_rate(approved, rejected, expected): +def test_approval_rate(approved: int, rejected: int, expected: str) -> None: assert curation.approval_rate(approved, rejected) == expected # ─── get_content paging ────────────────────────────────────────────────────── -def test_the_first_page_carries_the_text_and_the_links(): +def test_the_first_page_carries_the_text_and_the_links() -> None: links = {"repo": ["https://github.com/a/b"]} row = _pending_row(content="abcdefghij", links=links) session = _FakeSession(row) - page = curation.get_content(session, row.url, limit=4) + page = curation.get_content(session.session, row.url, limit=4) assert page == { "text": "abcd", @@ -329,18 +379,18 @@ def test_the_first_page_carries_the_text_and_the_links(): } -def test_a_later_page_reads_on_without_the_links(): +def test_a_later_page_reads_on_without_the_links() -> None: row = _pending_row(content="abcdefghij", links={"repo": ["x"]}) session = _FakeSession(row) - page = curation.get_content(session, row.url, offset=8, limit=4) + page = curation.get_content(session.session, row.url, offset=8, limit=4) assert page == {"text": "ij", "offset": 8, "next_offset": None, "total": 10} -def test_a_short_article_is_one_page(): +def test_a_short_article_is_one_page() -> None: row = _pending_row() - page = curation.get_content(_FakeSession(row), row.url) + page = curation.get_content(_FakeSession(row).session, row.url) assert page["text"] == "the stored two thousand characters" assert page["next_offset"] is None assert page["links"] == {} @@ -358,26 +408,28 @@ def test_a_short_article_is_one_page(): (CANDIDATE_CAP * 2, 0, CANDIDATE_CAP * 2, (0, CANDIDATE_CAP)), ], ) -def test_page_bounds(total, offset, limit, expected): +def test_page_bounds( + total: int, offset: int, limit: int, expected: tuple[int, int] +) -> None: assert curation.page_bounds(total, offset, limit) == expected -def test_get_content_needs_a_pending_candidate(): +def test_get_content_needs_a_pending_candidate() -> None: row = _pending_row(stage=RejectStage.below_threshold) - with pytest.raises(curation.CandidateError, match="already"): - curation.get_content(_FakeSession(row), row.url) + with pytest.raises(CandidateError, match="already"): + curation.get_content(_FakeSession(row).session, row.url) # ─── reject_many ───────────────────────────────────────────────────────────── -def test_reject_many_settles_each_verdict_on_its_own(): +def test_reject_many_settles_each_verdict_on_its_own() -> None: first = _pending_row(url="https://a.example/1") third = _pending_row(url="https://a.example/3") session = _FakeSession(first, third) lines = curation.reject_many( - session, + session.session, [ {"url": first.url, "score": 20, "reason": "Vendor tutorial."}, {"url": "https://a.example/2", "score": 30, "reason": "r"}, @@ -394,21 +446,14 @@ def test_reject_many_settles_each_verdict_on_its_own(): assert third.reason == "Opinion, no evidence." -def test_reject_many_survives_a_failed_write(): +def test_reject_many_survives_a_failed_write() -> None: first = _pending_row(url="https://a.example/1") second = _pending_row(url="https://a.example/2") session = _FakeSession(first, second) - calls = {"n": 0} - - def commit_fails_once() -> None: - calls["n"] += 1 - if calls["n"] == 1: - raise RuntimeError("connection reset") - - session.commit = commit_fails_once # type: ignore[method-assign] + session.fail_commits = 1 lines = curation.reject_many( - session, + session.session, [ {"url": first.url, "score": 20, "reason": "r"}, {"url": second.url, "score": 20, "reason": "r"}, @@ -420,66 +465,14 @@ def commit_fails_once() -> None: assert session.rollbacks == 1 -# ─── similar and stories: the pure parts ───────────────────────────────────── - - -def _vecs(*rows: list[float]) -> np.ndarray: - return np.array(rows, dtype=np.float32) +# ─── coverage ──────────────────────────────────────────────────────────────── -def test_cosine_distance_is_zero_for_the_same_direction(): - d = embedding.cosine_distances(_vecs([1, 0]), _vecs([2, 0], [0, 1], [-1, 0])) - assert np.allclose(d, [[0, 1, 2]]) - - -def test_cosine_distance_survives_a_zero_vector(): - d = embedding.cosine_distances(_vecs([0, 0]), _vecs([1, 0])) - assert np.isfinite(d).all() - - -def test_nearest_is_closest_first_within_the_cutoff(): - distances = np.array([0.40, 0.10, 0.50, 0.25]) - assert embedding.nearest(distances, cutoff=0.45, limit=8) == [1, 3, 0] - assert embedding.nearest(distances, cutoff=0.45, limit=2) == [1, 3] - - -def test_coverage_counts_distinct_publishers_under_the_cutoff_and_the_own(): +def test_coverage_counts_distinct_publishers_under_the_cutoff_and_the_own() -> None: publishers = ["The Verge", "The Verge", "Wired", "Simon Willison", ""] - distances = np.array([0.10, 0.20, 0.29, 0.40, 0.05]) - assert curation.covering(publishers, distances, "OpenAI", 0.30) == [ + distances = np.array([0.10, 0.20, 0.29, 0.40, 0.05], dtype=np.float32) + assert coverage.covering(publishers, distances, "OpenAI", 0.30) == [ "OpenAI", "The Verge", "Wired", ] - - -def test_story_groups_join_pending_rows_with_each_other_and_the_feed(): - # 0, 1 pending and the same story; 2 a feed article on it; 3 a pending - # candidate on its own; 4 a feed article on nothing queued. - vecs = _vecs([1, 0, 0], [0.98, 0.1, 0], [0.97, 0.12, 0], [0, 1, 0], [0, 0, 1]) - pending = [True, True, False, True, False] - - groups = embedding.story_groups( - embedding.cosine_distances(vecs, vecs), pending, 0.30 - ) - - assert [sorted(g) for g in groups] == [[0, 1, 2]] - - -def test_feed_articles_do_not_chain_a_group_through_each_other(): - # Pending 0 is close to article 1; article 2 is close to article 1 but not - # to 0. Without a pending end on every edge, 2 would ride in through 1. - distances = np.array( - [ - [0.0, 0.2, 0.6], - [0.2, 0.0, 0.2], - [0.6, 0.2, 0.0], - ] - ) - groups = embedding.story_groups(distances, [True, False, False], 0.30) - assert [sorted(g) for g in groups] == [[0, 1]] - - -def test_a_group_of_feed_articles_alone_is_not_a_story(): - distances = np.array([[0.0, 0.1], [0.1, 0.0]]) - assert embedding.story_groups(distances, [False, False], 0.30) == [] diff --git a/backend/app/features/curation/test_service_db.py b/backend/app/features/curation/test_service_db.py new file mode 100644 index 00000000..a39dcc14 --- /dev/null +++ b/backend/app/features/curation/test_service_db.py @@ -0,0 +1,398 @@ +"""The curation queries against the real schema, with the embedding stubbed. + +The pure parts are pinned in test_service.py. This pins what those cannot: +that `similar` and `stories` read the right rows (the feed's stored vectors, +every ledger stage, the window), that `list_candidates` counts a publisher's +record from the right ones, and that approving and rejecting write exactly the +rows they say they do. +""" + +from __future__ import annotations + +import hashlib +from collections.abc import Generator +from datetime import UTC, datetime, timedelta + +import numpy as np +import pytest +from sqlmodel import Session, col, delete, func, select + +from app.core.db.article import Article +from app.core.db.factories import random_lower_string +from app.core.db.ledger import Reject, RejectStage +from app.core.db.publisher import Publisher, PublisherKind, PublisherType +from app.core.db.tag import ArticleTag, Tag +from app.features.curation import coverage, publish +from app.features.curation import service as curation +from app.features.curation.model import CandidateError + +DIM = 256 +TAG = random_lower_string()[:8] + + +def _axis(i: int, wobble: float = 0.0) -> list[float]: + """A unit-ish vector along one of the high axes, far from anything real.""" + v = np.zeros(DIM, dtype=np.float32) + v[200 + i] = 1.0 + v[250] = wobble + return v.tolist() + + +def _noise(text: str) -> list[float]: + seed = int(hashlib.sha256(text.encode()).hexdigest()[:8], 16) + return [float(x) for x in np.random.default_rng(seed).standard_normal(DIM)] + + +# Title -> vector. Everything else embeds to deterministic noise. +VECTORS = { + f"{TAG} GPT-6 Astra ships": _axis(0), + f"{TAG} OpenAI releases GPT-6 Astra": _axis(0, 0.05), + f"{TAG} Astra is here, a retelling": _axis(0, 0.10), + f"{TAG} An old Astra rumour": _axis(0, 0.02), + f"{TAG} A lone post about kernels": _axis(1), +} + + +def _fake_embed(texts: list[str]) -> list[list[float]]: + return [VECTORS.get(t.split("\n\n")[0], _noise(t)) for t in texts] + + +@pytest.fixture +def world(db: Session, monkeypatch: pytest.MonkeyPatch) -> Generator[dict[str, str]]: + monkeypatch.setattr(curation, "embed_batch", _fake_embed) + monkeypatch.setattr(coverage, "embed_batch", _fake_embed) + + publishers = [ + Publisher( + name=f"{TAG} {name}", + slug=f"{TAG}-{name.lower()}", + kind=PublisherKind.media, + type=PublisherType.rss, + ) + for name in ("OpenAI", "Verge", "Wired", "Blog") + ] + db.add_all(publishers) + db.commit() + openai, verge, wired, blog = publishers + now = datetime.now(UTC) + + feed = Article( + title=f"{TAG} OpenAI releases GPT-6 Astra", + url=f"https://openai.example/{TAG}/astra", + publisher_id=openai.id or 0, + score=90, + embedding=_axis(0, 0.05), + ) + old = Article( + title=f"{TAG} An old Astra rumour", + url=f"https://old.example/{TAG}/rumour", + publisher_id=blog.id or 0, + score=60, + embedding=_axis(0, 0.02), + created_at=now - timedelta(days=30), + ) + pending = [ + Reject( + url=f"https://verge.example/{TAG}/astra", + stage=RejectStage.pending, + title=f"{TAG} GPT-6 Astra ships", + publisher_id=verge.id, + content="body", + ), + Reject( + url=f"https://blog.example/{TAG}/kernels", + stage=RejectStage.pending, + title=f"{TAG} A lone post about kernels", + publisher_id=blog.id, + content="body", + ), + ] + rejected = Reject( + url=f"https://wired.example/{TAG}/astra", + stage=RejectStage.below_threshold, + title=f"{TAG} Astra is here, a retelling", + publisher_id=wired.id, + content="body", + score=30, + ) + db.add_all([feed, old, *pending, rejected]) + db.commit() + + yield {"verge": pending[0].url, "lone": pending[1].url, "feed": feed.url} + + db.exec(delete(Reject).where(col(Reject.url).contains(TAG))) + db.exec(delete(Article).where(col(Article.url).contains(TAG))) + db.exec(delete(Publisher).where(col(Publisher.slug).startswith(TAG))) + db.commit() + + +def test_similar_lists_the_feed_and_every_ledger_stage( + db: Session, world: dict[str, str] +) -> None: + result = curation.similar(db, world["verge"]) + + urls = [r["url"] for r in result["rows"]] + assert urls[:2] == [world["feed"], f"https://wired.example/{TAG}/astra"] + stages = {r["url"]: r["stage"] for r in result["rows"]} + assert stages[world["feed"]] == coverage.PUBLISHED + assert stages[f"https://wired.example/{TAG}/astra"] == "below_threshold" + # Outside the window: the 30-day-old article is not a neighbour. + assert f"https://old.example/{TAG}/rumour" not in urls + # Never its own neighbour. + assert world["verge"] not in urls + + +def test_similar_counts_coverage_by_publisher( + db: Session, world: dict[str, str] +) -> None: + result = curation.similar(db, world["verge"]) + assert result["publishers"] == [f"{TAG} OpenAI", f"{TAG} Verge", f"{TAG} Wired"] + assert result["coverage"] == 3 + + +def test_a_story_nobody_else_carries_has_coverage_one( + db: Session, world: dict[str, str] +) -> None: + result = curation.similar(db, world["lone"]) + assert result["coverage"] == 1 + assert result["publishers"] == [f"{TAG} Blog"] + + +def test_similar_on_an_unknown_url_is_a_message( + db: Session, world: dict[str, str] +) -> None: + with pytest.raises(CandidateError, match="No candidate or article"): + curation.similar(db, f"https://nowhere.example/{TAG}") + assert world + + +def test_stories_groups_the_queue_with_the_feed( + db: Session, world: dict[str, str] +) -> None: + groups = [ + g for g in curation.stories(db) if any(TAG in m["url"] for m in g["members"]) + ] + + [story] = groups + assert {m["url"] for m in story["members"]} == {world["verge"], world["feed"]} + assert story["coverage"] == 2 + + +def _all_candidates( + db: Session, limit: int = curation.LIST_LIMIT +) -> list[dict[str, str]]: + """Every pending row, read page by page as the agent does.""" + page = curation.list_candidates(db, limit=limit) + rows = page["rows"] + while page["next_offset"] is not None: + page = curation.list_candidates(db, offset=page["next_offset"], limit=limit) + rows += page["rows"] + return rows + + +def test_list_candidates_shows_each_publishers_approval_rate( + db: Session, world: dict[str, str] +) -> None: + rows = {r["url"]: r for r in _all_candidates(db)} + # Verge: nothing published, nothing turned down yet. + assert rows[world["verge"]]["approved"] == "new" + assert "trust" not in rows[world["verge"]] + # Blog's one article is 30 days old: outside `similar`'s week, inside the + # 90 days a record reaches. + assert rows[world["lone"]]["approved"] == "1/1" + + +def test_list_candidates_pages_through_the_whole_queue( + db: Session, world: dict[str, str] +) -> None: + first = curation.list_candidates(db, limit=1) + assert first["offset"] == 0 + assert len(first["rows"]) == 1 + + urls = [r["url"] for r in _all_candidates(db, limit=1)] + + # Every pending row once, and only pending rows. + assert len(urls) == len(set(urls)) == first["total"] + assert {world["verge"], world["lone"]} <= set(urls) + assert f"https://wired.example/{TAG}/astra" not in urls + + +def test_list_candidates_past_the_end_is_an_empty_last_page( + db: Session, world: dict[str, str] +) -> None: + page = curation.list_candidates(db, offset=10**6) + assert page["rows"] == [] + assert page["next_offset"] is None + assert page["offset"] == page["total"] + assert world + + +def test_approval_counts_reads_articles_and_agent_rejects( + db: Session, world: dict[str, str] +) -> None: + ids = { + p.name: p.id + for p in db.exec(select(Publisher).where(col(Publisher.slug).startswith(TAG))) + if p.id is not None + } + since = datetime.now(UTC) - timedelta(days=90) + counts = curation.approval_counts(db, list(ids.values()), since) + + assert counts[ids[f"{TAG} OpenAI"]] == (1, 0) + assert counts[ids[f"{TAG} Wired"]] == (0, 1) + # A pending row is not a decision. + assert ids[f"{TAG} Verge"] not in counts + assert world + + +# ─── approve and reject ────────────────────────────────────────────────────── + + +@pytest.fixture +def queued(db: Session, monkeypatch: pytest.MonkeyPatch) -> Generator[Reject]: + """One pending candidate from a publisher the run has just created.""" + monkeypatch.setattr(publish, "embed_batch", _fake_embed) + publisher = Publisher( + name=f"{TAG}.dev", + slug=f"{TAG}-dev", + kind=PublisherKind.unknown, + is_active=False, + ) + db.add(publisher) + db.commit() + row = Reject( + url=f"https://{TAG}.dev/post", + stage=RejectStage.pending, + title=f"{TAG} — An agent harness, measured", + source="Hacker News", + publisher_id=publisher.id, + content="x" * 5000, + ) + db.add(row) + db.commit() + yield row + + ids = select(Article.id).where(col(Article.url).contains(TAG)) + db.exec(delete(ArticleTag).where(col(ArticleTag.article_id).in_(ids))) + db.exec(delete(Reject).where(col(Reject.url).contains(TAG))) + db.exec(delete(Article).where(col(Article.url).contains(TAG))) + db.exec(delete(Publisher).where(col(Publisher.slug).startswith(TAG))) + db.commit() + + +def _a_tag(db: Session) -> str: + tag = db.exec(select(Tag).order_by(col(Tag.slug))).first() + assert tag is not None, "the tag vocabulary is seeded by prestart" + return tag.slug + + +def test_approving_writes_one_article_and_settles_the_candidate( + db: Session, queued: Reject +) -> None: + slug = _a_tag(db) + article_id = curation.approve( + db, + queued.url, + 81, + "Numbers from a real run.", + "What the harness got right.", + "Post", + [slug, "not-a-tag"], + publisher_kind="individual", + publisher_name=f" {TAG} Writer ", + ) + + db.expire_all() + article = db.get(Article, article_id) + assert article is not None + assert (article.url, article.score, article.kind) == (queued.url, 81, "post") + # Sanitized on the way in, as every stored title is. + assert article.title == f"{TAG} - An agent harness, measured" + assert article.found_via == "Hacker News" + assert article.embedding is not None + assert db.get(Reject, queued.url) is None + tags = db.exec( + select(Tag.slug) + .join(ArticleTag, col(ArticleTag.tag_id) == col(Tag.id)) + .where(col(ArticleTag.article_id) == article_id) + ).all() + assert list(tags) == [slug] + publisher = db.get(Publisher, article.publisher_id) + assert publisher is not None + assert (publisher.kind, publisher.name) == ( + PublisherKind.individual, + f"{TAG} Writer", + ) + + +def test_a_candidate_is_published_once(db: Session, queued: Reject) -> None: + def approve() -> int: + return curation.approve(db, queued.url, 81, "r", "s", "post", [], "media") + + approve() + with pytest.raises(CandidateError, match="No candidate"): + approve() + count = db.exec( + select(func.count()).select_from(Article).where(Article.url == queued.url) + ).one() + assert count == 1 + + +def test_a_refused_approval_writes_nothing(db: Session, queued: Reject) -> None: + """approve checks the labels before it touches the candidate.""" + with pytest.raises(CandidateError, match="publisher_kind"): + curation.approve(db, queued.url, 81, "r", "s", "post", []) + + db.expire_all() + row = db.get(Reject, queued.url) + assert row is not None and row.stage == RejectStage.pending + assert db.exec(select(Article).where(Article.url == queued.url)).first() is None + + +def test_a_failed_embedding_still_publishes( + db: Session, queued: Reject, monkeypatch: pytest.MonkeyPatch +) -> None: + def broken(_texts: list[str]) -> list[list[float]]: + raise RuntimeError("model download blocked") + + monkeypatch.setattr(publish, "embed_batch", broken) + + article_id = curation.approve(db, queued.url, 81, "r", "s", "post", [], "media") + + db.expire_all() + article = db.get(Article, article_id) + assert article is not None and article.embedding is None + + +def test_a_rejected_candidate_cannot_then_be_approved( + db: Session, queued: Reject +) -> None: + curation.reject(db, queued.url, 22, "A vendor tutorial.") + + with pytest.raises(CandidateError, match="already below_threshold"): + curation.approve(db, queued.url, 81, "r", "s", "post", [], "media") + db.expire_all() + row = db.get(Reject, queued.url) + assert row is not None + assert (row.stage, row.score, row.reason) == ( + RejectStage.below_threshold, + 22, + "A vendor tutorial.", + ) + assert row.content is not None and len(row.content) < 5000 + + +def test_a_url_twice_in_one_batch_is_rejected_once(db: Session, queued: Reject) -> None: + lines = curation.reject_many( + db, + [ + {"url": queued.url, "score": 20, "reason": "first"}, + {"url": queued.url, "score": 90, "reason": "second"}, + ], + ) + + assert lines[0] == f"Rejected [20/100] {queued.url}" + assert lines[1].startswith(f"Skipped {queued.url}: ") + db.expire_all() + row = db.get(Reject, queued.url) + assert row is not None and (row.score, row.reason) == (20, "first") diff --git a/backend/tests/pipeline/test_tags.py b/backend/app/features/curation/test_tags.py similarity index 62% rename from backend/tests/pipeline/test_tags.py rename to backend/app/features/curation/test_tags.py index f0dedfbd..f5e57404 100644 --- a/backend/tests/pipeline/test_tags.py +++ b/backend/app/features/curation/test_tags.py @@ -1,33 +1,33 @@ """validate_tags: the last line of defence between whatever the LLM returns and what gets written as ArticleTag rows. Off-list tags must never be written, and -no tag is ever minted at runtime — see pipeline/tags.py. +no tag is ever minted at runtime. See app/features/curation/tags.py. """ from __future__ import annotations -from pipeline.tags import normalize_tag, validate_tags +from app.features.curation.tags import normalize_tag, validate_tags VALID = frozenset({"llm", "agents", "open-source"}) -def test_keeps_tags_in_the_vocabulary(): +def test_keeps_tags_in_the_vocabulary() -> None: assert validate_tags(["llm", "agents"], VALID) == ["llm", "agents"] -def test_drops_tags_outside_the_vocabulary(): +def test_drops_tags_outside_the_vocabulary() -> None: assert validate_tags(["llm", "not-a-real-tag"], VALID) == ["llm"] -def test_dedupes_preserving_first_occurrence_order(): +def test_dedupes_preserving_first_occurrence_order() -> None: assert validate_tags(["agents", "llm", "agents"], VALID) == ["agents", "llm"] -def test_caps_at_three_even_if_more_are_valid(): +def test_caps_at_three_even_if_more_are_valid() -> None: valid = frozenset({"a", "b", "c", "d"}) assert len(validate_tags(["a", "b", "c", "d"], valid)) == 3 -def test_lab_tags_do_not_count_toward_the_cap(): +def test_lab_tags_do_not_count_toward_the_cap() -> None: valid = frozenset({"a", "b", "c", "d", "openai", "google"}) assert validate_tags(["openai", "a", "b", "google", "c", "d"], valid) == [ "openai", @@ -38,23 +38,23 @@ def test_lab_tags_do_not_count_toward_the_cap(): ] -def test_normalizes_casing_and_underscores_before_validating(): +def test_normalizes_casing_and_underscores_before_validating() -> None: assert validate_tags(["LLM", "open_source"], VALID) == ["llm", "open-source"] -def test_empty_input_yields_no_tags(): +def test_empty_input_yields_no_tags() -> None: assert validate_tags([], VALID) == [] -def test_all_off_list_yields_no_tags(): +def test_all_off_list_yields_no_tags() -> None: assert validate_tags(["nope", "also-nope"], VALID) == [] -def test_normalize_tag_rejects_none(): +def test_normalize_tag_rejects_none() -> None: assert normalize_tag(None, VALID) is None -def test_normalize_tag_tolerates_an_enum_like_value(): +def test_normalize_tag_tolerates_an_enum_like_value() -> None: class _Tag: value = "llm" diff --git a/backend/pipeline/__init__.py b/backend/app/features/newsletter/__init__.py similarity index 100% rename from backend/pipeline/__init__.py rename to backend/app/features/newsletter/__init__.py diff --git a/backend/app/newsletter/broadcast.py b/backend/app/features/newsletter/broadcast.py similarity index 98% rename from backend/app/newsletter/broadcast.py rename to backend/app/features/newsletter/broadcast.py index 984716cc..7b8a4062 100644 --- a/backend/app/newsletter/broadcast.py +++ b/backend/app/features/newsletter/broadcast.py @@ -22,15 +22,16 @@ import re import unicodedata from datetime import date +from pathlib import Path from typing import TypedDict import resend from jinja2 import Environment, FileSystemLoader, select_autoescape from markupsafe import Markup, escape -from app.core.email import TEMPLATES from app.core.logging import log from app.core.settings import settings +from app.features.newsletter.model import IssueError # Resend swaps this for each recipient's own unsubscribe link. UNSUBSCRIBE_PLACEHOLDER = "{{{RESEND_UNSUBSCRIBE_URL}}}" @@ -65,14 +66,11 @@ FIELDS = ("title", "description", "topic") _templates = Environment( - loader=FileSystemLoader(TEMPLATES), autoescape=select_autoescape(["html"]) + loader=FileSystemLoader(Path(__file__).parent), + autoescape=select_autoescape(["html"]), ) -class IssueError(ValueError): - """An issue that cannot be drafted as given, returned as a message.""" - - class Issue(TypedDict): # The email's subject and the post's headline. title: str diff --git a/backend/app/features/newsletter/model.py b/backend/app/features/newsletter/model.py new file mode 100644 index 00000000..1fecf66d --- /dev/null +++ b/backend/app/features/newsletter/model.py @@ -0,0 +1,68 @@ +"""The newsletter's shapes: the week's topics, what relates to one, and the +errors the agent gets back as a message.""" + +from __future__ import annotations + +from typing import NotRequired, TypedDict + + +class DigestError(ValueError): + """A request the agent can correct, returned as a message.""" + + +class WeekArticle(TypedDict): + url: str + title: str + publisher: str + publisher_kind: str + kind: str + score: int + likes: int + summary: str + + +class CoveredBy(TypedDict): + """A ledger row on the same story: another outlet's copy, usually rejected + as a retelling.""" + + url: str + title: str + publisher: str + publisher_kind: str + score: int | None + reason: str + + +class WeekTopic(TypedDict): + # Distinct publishers carrying it this week, the ledger's included. + coverage: int + publishers: list[str] + likes: int + top_score: int + articles: list[WeekArticle] + covered_by: list[CoveredBy] + + +class RelatedRow(TypedDict): + url: str + title: str + publisher: str + publisher_kind: str + kind: str + stage: str + score: int | None + distance: float + # Published rows: what a reader sees under the title. + summary: NotRequired[str] + # Rejected rows: why curation turned it down, e.g. as a retelling. + reason: NotRequired[str] + + +class Related(TypedDict): + url: str + title: str + rows: list[RelatedRow] + + +class IssueError(ValueError): + """An issue that cannot be drafted as given, returned as a message.""" diff --git a/backend/app/newsletter/digest.py b/backend/app/features/newsletter/service.py similarity index 83% rename from backend/app/newsletter/digest.py rename to backend/app/features/newsletter/service.py index bcdc4f74..21f443e0 100644 --- a/backend/app/newsletter/digest.py +++ b/backend/app/features/newsletter/service.py @@ -1,5 +1,6 @@ -"""What the weekly newsletter draws on: the week's most talked-about topics, -and what else the feed and the ledger hold on each one. +"""The weekly newsletter: who it goes to, and what it draws on - the week's +most talked-about topics, and what else the feed and the ledger hold on each +one. The curation agent already read and scored every article, so nothing here judges. `week` groups the week into topics and ranks them by how many @@ -11,7 +12,7 @@ rejects the rest as retellings, so the feed alone says every story was carried once; the rejects are where the other outlets are. -The maths is curation's (`pipeline.curation`): one topic is rows within +The maths is curation's (`app.core.embedding`): one topic is rows within `DEDUP_DIST_THRESHOLD` of a published article, and related is out to `RELATED_DIST`. A review of a release sits in between: close enough to be about it, far enough not to be a retelling. @@ -19,15 +20,22 @@ from __future__ import annotations +import logging +import os from datetime import UTC, datetime, timedelta -from typing import NotRequired, TypedDict import numpy as np +import resend from sqlmodel import Session, col, func, select -from app.catalog.articles import like_counts_subquery -from app.catalog.models import Article, Publisher +from app.core.db.article import Article, kind_from_url +from app.core.db.ledger import Reject, RejectStage +from app.core.db.like import like_counts_subquery +from app.core.db.publisher import Publisher +from app.core.db.subscriber import NewsletterSubscriber from app.core.embedding import ( + DEDUP_DIST_THRESHOLD, + RELATED_DIST, SNIPPET_CAP, cosine_distances, embed_batch, @@ -35,9 +43,15 @@ story_groups, to_embedding_text, ) -from pipeline.curation import DEDUP_DIST_THRESHOLD, RELATED_DIST -from pipeline.models import Reject, RejectStage -from pipeline.url_kind import kind_from_url +from app.features.newsletter.broadcast import draft as draft +from app.features.newsletter.model import ( + CoveredBy, + DigestError, + Related, + RelatedRow, + WeekArticle, + WeekTopic, +) # The agent drafts three issues a week; the rest are there so it can pass over a # topic it finds nothing to say about. @@ -49,68 +63,12 @@ # how people took a thing is part of what the issue explains. RELATED_MIN_SCORE = 35 +logger = logging.getLogger(__name__) + PUBLISHED = "published" REJECTED = "rejected" -class DigestError(ValueError): - """A request the agent can correct, returned as a message.""" - - -class WeekArticle(TypedDict): - url: str - title: str - publisher: str - publisher_kind: str - kind: str - score: int - likes: int - summary: str - - -class CoveredBy(TypedDict): - """A ledger row on the same story: another outlet's copy, usually rejected - as a retelling.""" - - url: str - title: str - publisher: str - publisher_kind: str - score: int | None - reason: str - - -class WeekTopic(TypedDict): - # Distinct publishers carrying it this week, the ledger's included. - coverage: int - publishers: list[str] - likes: int - top_score: int - articles: list[WeekArticle] - covered_by: list[CoveredBy] - - -class RelatedRow(TypedDict): - url: str - title: str - publisher: str - publisher_kind: str - kind: str - stage: str - score: int | None - distance: float - # Published rows: what a reader sees under the title. - summary: NotRequired[str] - # Rejected rows: why curation turned it down, e.g. as a retelling. - reason: NotRequired[str] - - -class Related(TypedDict): - url: str - title: str - rows: list[RelatedRow] - - def _vectors(articles: list[Article]) -> list[list[float]]: """Each article's stored embedding, or one made now from the same text.""" missing = [a for a in articles if a.embedding is None] @@ -309,3 +267,20 @@ def related( row["distance"] = round(float(distances[i]), 3) listed.append(row) return {"url": url, "title": target.title, "rows": listed} + + +def subscribe(*, session: Session, email: str, utm_source: str | None = None) -> None: + """Add an email to the newsletter. An address already on it is left as is.""" + if session.get(NewsletterSubscriber, email) is None: + session.add(NewsletterSubscriber(email=email, utm_source=utm_source)) + session.commit() + + api_key = os.getenv("RESEND_API_KEY") + audience_id = os.getenv("RESEND_AUDIENCE_ID") + if api_key and audience_id: + resend.api_key = api_key + try: + resend.Contacts.create({"email": email, "audience_id": audience_id}) + except Exception as e: + if "already exists" not in str(e): + logger.error(f"Resend contact create failed for {email}: {e}") diff --git a/backend/app/newsletter/tests/test_broadcast.py b/backend/app/features/newsletter/test_broadcast.py similarity index 81% rename from backend/app/newsletter/tests/test_broadcast.py rename to backend/app/features/newsletter/test_broadcast.py index 9201b44c..b0ffe0e1 100644 --- a/backend/app/newsletter/tests/test_broadcast.py +++ b/backend/app/features/newsletter/test_broadcast.py @@ -14,7 +14,8 @@ import resend from app.core.settings import settings -from app.newsletter import broadcast +from app.features.newsletter import broadcast +from app.features.newsletter.model import IssueError FILLER = " ".join(["word"] * 70) BODY = ( @@ -53,15 +54,15 @@ def _post( @pytest.fixture -def configured(monkeypatch: pytest.MonkeyPatch) -> dict[str, list]: +def configured(monkeypatch: pytest.MonkeyPatch) -> dict[str, list[dict[str, str]]]: monkeypatch.setattr(settings, "RESEND_API_KEY", "key") monkeypatch.setattr(settings, "RESEND_AUDIENCE_ID", "audience") monkeypatch.setattr(settings, "EMAILS_FROM_EMAIL", "news@agentique.example") monkeypatch.setattr(settings, "EMAILS_FROM_NAME", "agentique") monkeypatch.setattr(settings, "FRONTEND_HOST", "https://agentique.example") - calls: dict[str, list] = {"create": []} + calls: dict[str, list[dict[str, str]]] = {"create": []} - def create(params): + def create(params: dict[str, str]) -> dict[str, str]: calls["create"].append(params) return {"id": "b-1"} @@ -69,11 +70,11 @@ def create(params): return calls -def test_a_post_parses_into_its_fields_and_body(): +def test_a_post_parses_into_its_fields_and_body() -> None: assert broadcast.parse(_post()) == _issue() -def test_quoted_values_read_as_yaml_reads_them(): +def test_quoted_values_read_as_yaml_reads_them() -> None: post = ( '---\ntitle: "Agents: \\"cheap\\" now"\n' "description: 'It''s a line'\ntopic: agents\n---\nBody" @@ -96,8 +97,8 @@ def test_quoted_values_read_as_yaml_reads_them(): ('---\ntitle: "open\n---\nb', "double-quoted"), ], ) -def test_a_malformed_post_is_refused(post: str, message: str): - with pytest.raises(broadcast.IssueError, match=message): +def test_a_malformed_post_is_refused(post: str, message: str) -> None: + with pytest.raises(IssueError, match=message): broadcast.parse(post) @@ -115,12 +116,12 @@ def test_a_malformed_post_is_refused(post: str, message: str): ), ], ) -def test_the_slug_comes_from_the_title(title: str, slug: str): +def test_the_slug_comes_from_the_title(title: str, slug: str) -> None: assert broadcast.slugify(title) == slug assert len(slug) <= broadcast.MAX_SLUG -def test_blocks_are_paragraphs_and_lists(): +def test_blocks_are_paragraphs_and_lists() -> None: assert broadcast.blocks("One\nline.\n\n- a\n- b\n\n- a\nnot a list\n\n\n") == [ {"kind": "p", "lines": ["One line."]}, {"kind": "ul", "lines": ["a", "b"]}, @@ -128,19 +129,19 @@ def test_blocks_are_paragraphs_and_lists(): ] -def test_inline_html_links_code_and_escapes_the_rest(): +def test_inline_html_links_code_and_escapes_the_rest() -> None: html = broadcast.inline_html("[see](https://a.example/?x=1&y=2) `asee</b>" in html assert "a<b" in html -def test_inline_text_spells_the_link_out(): +def test_inline_text_spells_the_link_out() -> None: line = "Read [this](https://a.example) and run `ls`." assert broadcast.inline_text(line) == "Read this (https://a.example) and run ls." -def test_word_count_skips_the_urls(): +def test_word_count_skips_the_urls() -> None: assert broadcast.word_count("[two words](https://a.example/long/path) `x`") == 3 @@ -159,13 +160,15 @@ def test_word_count_skips_the_urls(): ({"body": BODY.replace(REPO, "https://ollama.example/blog/mlx")}, "at least 3"), ], ) -def test_an_issue_a_reader_should_not_get_is_refused(overrides: dict, message: str): - with pytest.raises(broadcast.IssueError, match=message): +def test_an_issue_a_reader_should_not_get_is_refused( + overrides: dict[str, str], message: str +) -> None: + with pytest.raises(IssueError, match=message): broadcast.validate(_issue(**overrides)) @pytest.mark.usefixtures("configured") -def test_the_html_is_the_wordmark_the_prose_and_two_links(): +def test_the_html_is_the_wordmark_the_prose_and_two_links() -> None: html = broadcast.render_html(_issue()) assert ">agentique" in html for url in ( @@ -181,7 +184,7 @@ def test_the_html_is_the_wordmark_the_prose_and_two_links(): @pytest.mark.usefixtures("configured") -def test_the_text_twin_ends_with_the_feed_and_unsubscribe(): +def test_the_text_twin_ends_with_the_feed_and_unsubscribe() -> None: text = broadcast.render_text(_issue()) assert "I kept seeing the MLX backend (https://ollama.example/blog/mlx)" in text assert "- a benchmark (https://writer.example/mlx) on three Macs" in text @@ -191,7 +194,9 @@ def test_the_text_twin_ends_with_the_feed_and_unsubscribe(): ) -def test_draft_creates_a_named_draft_broadcast(configured: dict[str, list]): +def test_draft_creates_a_named_draft_broadcast( + configured: dict[str, list[dict[str, str]]], +) -> None: assert broadcast.draft(_post(), date(2026, 10, 9)) == ( "Drafted broadcast b-1 (local-ai). Save the post, exactly as passed, at " "frontend/content/blog/2026-10-09-what-ollama-s-mlx-backend-actually-does.md" @@ -207,16 +212,18 @@ def test_draft_creates_a_named_draft_broadcast(configured: dict[str, list]): assert broadcast.UNSUBSCRIBE_PLACEHOLDER in created["text"] -def test_draft_refuses_before_reaching_resend(configured: dict[str, list]): - with pytest.raises(broadcast.IssueError, match="title is 56"): +def test_draft_refuses_before_reaching_resend( + configured: dict[str, list[dict[str, str]]], +) -> None: + with pytest.raises(IssueError, match="title is 56"): broadcast.draft(_post(title="x" * 56)) assert configured["create"] == [] def test_draft_without_resend_says_what_is_missing( - configured: dict[str, list], monkeypatch: pytest.MonkeyPatch -): + configured: dict[str, list[dict[str, str]]], monkeypatch: pytest.MonkeyPatch +) -> None: monkeypatch.setattr(settings, "RESEND_AUDIENCE_ID", None) - with pytest.raises(broadcast.IssueError, match="RESEND_AUDIENCE_ID unset"): + with pytest.raises(IssueError, match="RESEND_AUDIENCE_ID unset"): broadcast.draft(_post()) assert configured["create"] == [] diff --git a/backend/app/newsletter/tests/test_digest.py b/backend/app/features/newsletter/test_service.py similarity index 82% rename from backend/app/newsletter/tests/test_digest.py rename to backend/app/features/newsletter/test_service.py index 7d6b23b6..e8527fd2 100644 --- a/backend/app/newsletter/tests/test_digest.py +++ b/backend/app/features/newsletter/test_service.py @@ -16,18 +16,13 @@ import pytest from sqlmodel import Session, col, delete -from app.audience.models import ArticleLike -from app.audience.tests.factories import create_random_user -from app.catalog.models import ( - Article, - ArticleKind, - Publisher, - PublisherKind, - PublisherType, -) -from app.newsletter import digest -from pipeline.models import Reject, RejectStage -from tests.random_data import random_lower_string +from app.core.db.article import Article, ArticleKind +from app.core.db.factories import create_random_user, random_lower_string +from app.core.db.ledger import Reject, RejectStage +from app.core.db.like import ArticleLike +from app.core.db.publisher import Publisher, PublisherKind, PublisherType +from app.features.newsletter import service as digest +from app.features.newsletter.model import DigestError, WeekTopic DIM = 256 TAG = random_lower_string()[:8] @@ -49,7 +44,7 @@ def _axis(i: int, wobble: float = 0.0) -> list[float]: def _noise(text: str) -> list[float]: seed = int(hashlib.sha256(text.encode()).hexdigest()[:8], 16) - return np.random.default_rng(seed).standard_normal(DIM).tolist() + return [float(x) for x in np.random.default_rng(seed).standard_normal(DIM)] # Title -> vector, for the ledger rows (embedded on the fly). @@ -72,7 +67,7 @@ def _url(host: str, path: str) -> str: @pytest.fixture -def world(db: Session, monkeypatch: pytest.MonkeyPatch) -> Generator[dict]: +def world(db: Session, monkeypatch: pytest.MonkeyPatch) -> Generator[dict[str, str]]: monkeypatch.setattr(digest, "embed_batch", _fake_embed) publishers = { @@ -96,7 +91,15 @@ def world(db: Session, monkeypatch: pytest.MonkeyPatch) -> Generator[dict]: db.commit() now = datetime.now(UTC) - def article(pub: str, url: str, score: int, vec: list[float], **kw): + def article( + pub: str, + url: str, + score: int, + vec: list[float], + kind: ArticleKind = ArticleKind.post, + created_at: datetime = now, + marked_for_deletion_at: datetime | None = None, + ) -> Article: return Article( title=f"{TAG} {url}", url=url, @@ -104,17 +107,26 @@ def article(pub: str, url: str, score: int, vec: list[float], **kw): score=score, embedding=vec, summary=f"summary of {url}", - **kw, + kind=kind, + created_at=created_at, + marked_for_deletion_at=marked_for_deletion_at, ) - def reject(url: str, title: str, score: int, pub: str | None = None, **kw): + def reject( + url: str, + title: str, + score: int, + pub: str | None = None, + stage: RejectStage = RejectStage.below_threshold, + reason: str | None = None, + ) -> Reject: return Reject( url=url, - stage=kw.pop("stage", RejectStage.below_threshold), + stage=stage, title=f"{TAG} {title}", publisher_id=publishers[pub].id if pub else None, score=score, - **kw, + reason=reason, ) release = article("Ollama", _url("ollama.example", "mlx"), 92, _axis(0)) @@ -205,7 +217,7 @@ def reject(url: str, title: str, score: int, pub: str | None = None, **kw): db.commit() -def _ours(db: Session) -> list[digest.WeekTopic]: +def _ours(db: Session) -> list[WeekTopic]: # A high limit: the local db carries other articles from this week. return [ t @@ -214,14 +226,14 @@ def _ours(db: Session) -> list[digest.WeekTopic]: ] -def _topic(db: Session, url: str) -> digest.WeekTopic: +def _topic(db: Session, url: str) -> WeekTopic: [topic] = [t for t in _ours(db) if t["articles"][0]["url"] == url] return topic def test_week_ranks_topics_by_how_many_publishers_carried_them( - db: Session, world: dict -): + db: Session, world: dict[str, str] +) -> None: """The kernel post scores lowest of the three, and two outlets whose copies curation rejected carried it too: it is the week's most talked-about.""" assert [t["articles"][0]["url"] for t in _ours(db)] == [ @@ -232,7 +244,9 @@ def test_week_ranks_topics_by_how_many_publishers_carried_them( ] -def test_week_counts_the_rejected_copies_as_coverage(db: Session, world: dict): +def test_week_counts_the_rejected_copies_as_coverage( + db: Session, world: dict[str, str] +) -> None: topic = _topic(db, world["kernels"]) assert topic["coverage"] == 3 assert topic["publishers"] == [f"{TAG} Blog", f"{TAG} Register", f"{TAG} Wired"] @@ -245,8 +259,8 @@ def test_week_counts_the_rejected_copies_as_coverage(db: Session, world: dict): def test_a_week_topic_carries_its_articles_likes_and_summaries( - db: Session, world: dict -): + db: Session, world: dict[str, str] +) -> None: topic = _topic(db, world["release"]) assert [a["url"] for a in topic["articles"]] == [ world["release"], @@ -264,20 +278,22 @@ def test_a_week_topic_carries_its_articles_likes_and_summaries( assert lead["summary"] == f"summary of {world['release']}" -def test_week_leaves_out_what_is_not_this_weeks(db: Session, world: dict): +def test_week_leaves_out_what_is_not_this_weeks( + db: Session, world: dict[str, str] +) -> None: urls = {a["url"] for t in _ours(db) for a in t["articles"]} for name in ("repo", "ancient", "marked"): assert world[name] not in urls -def test_week_returns_at_most_limit_topics(db: Session, world: dict): +def test_week_returns_at_most_limit_topics(db: Session, world: dict[str, str]) -> None: assert world assert len(digest.week(db, limit=1)) == 1 def test_related_reaches_the_review_the_repo_and_the_commentary( - db: Session, world: dict -): + db: Session, world: dict[str, str] +) -> None: result = digest.related(db, world["release"]) rows = {r["url"]: r for r in result["rows"]} @@ -295,12 +311,16 @@ def test_related_reaches_the_review_the_repo_and_the_commentary( assert distances == sorted(distances) -def test_related_leaves_out_slop_the_queue_and_the_window(db: Session, world: dict): +def test_related_leaves_out_slop_the_queue_and_the_window( + db: Session, world: dict[str, str] +) -> None: urls = {r["url"] for r in digest.related(db, world["release"])["rows"]} for name in ("release", "hot_take", "pending", "ancient", "marked", "kernels"): assert world[name] not in urls -def test_related_on_an_unknown_url_is_a_message(db: Session, world: dict): - with pytest.raises(digest.DigestError, match="No published article"): +def test_related_on_an_unknown_url_is_a_message( + db: Session, world: dict[str, str] +) -> None: + with pytest.raises(DigestError, match="No published article"): digest.related(db, world["pending"]) diff --git a/backend/app/newsletter/tests/test_subscribers.py b/backend/app/features/newsletter/test_subscribe.py similarity index 71% rename from backend/app/newsletter/tests/test_subscribers.py rename to backend/app/features/newsletter/test_subscribe.py index b5108c5a..2882c3ec 100644 --- a/backend/app/newsletter/tests/test_subscribers.py +++ b/backend/app/features/newsletter/test_subscribe.py @@ -2,15 +2,18 @@ from __future__ import annotations +import pytest import resend from sqlmodel import Session -from app.newsletter.models import NewsletterSubscriber -from app.newsletter.subscribers import subscribe -from tests.random_data import random_email +from app.core.db.factories import random_email +from app.core.db.subscriber import NewsletterSubscriber +from app.features.newsletter.service import subscribe -def test_subscribe_defaults_to_all(db: Session, monkeypatch) -> None: +def test_subscribe_defaults_to_all( + db: Session, monkeypatch: pytest.MonkeyPatch +) -> None: monkeypatch.delenv("RESEND_API_KEY", raising=False) email = random_email() @@ -23,7 +26,9 @@ def test_subscribe_defaults_to_all(db: Session, monkeypatch) -> None: db.commit() -def test_subscribe_keeps_an_existing_row(db: Session, monkeypatch) -> None: +def test_subscribe_keeps_an_existing_row( + db: Session, monkeypatch: pytest.MonkeyPatch +) -> None: monkeypatch.delenv("RESEND_API_KEY", raising=False) email = random_email() db.add(NewsletterSubscriber(email=email, categories=["dev"], utm_source="x")) @@ -40,11 +45,13 @@ def test_subscribe_keeps_an_existing_row(db: Session, monkeypatch) -> None: db.commit() -def test_subscribe_ignores_existing_resend_contact(db: Session, monkeypatch) -> None: +def test_subscribe_ignores_existing_resend_contact( + db: Session, monkeypatch: pytest.MonkeyPatch +) -> None: monkeypatch.setenv("RESEND_API_KEY", "test-key") monkeypatch.setenv("RESEND_AUDIENCE_ID", "test-audience") - def exists(_payload: dict) -> None: + def exists(_payload: dict[str, object]) -> None: raise RuntimeError("Contact already exists") monkeypatch.setattr(resend.Contacts, "create", exists) diff --git a/backend/app/core/email-templates/weekly_issue.html b/backend/app/features/newsletter/weekly_issue.html similarity index 100% rename from backend/app/core/email-templates/weekly_issue.html rename to backend/app/features/newsletter/weekly_issue.html diff --git a/backend/pipeline/sources/__init__.py b/backend/app/features/stories/__init__.py similarity index 100% rename from backend/pipeline/sources/__init__.py rename to backend/app/features/stories/__init__.py diff --git a/backend/app/features/stories/model.py b/backend/app/features/stories/model.py new file mode 100644 index 00000000..71975eb5 --- /dev/null +++ b/backend/app/features/stories/model.py @@ -0,0 +1,77 @@ +"""The stories' shapes: what the rail reads, and what the stories agent reads and +writes.""" + +from __future__ import annotations + +from datetime import datetime +from typing import TypedDict + +from sqlmodel import SQLModel + + +class StoryError(ValueError): + """A save or close the agent can correct, returned as a message.""" + + +class StoryArticlePublic(SQLModel): + id: int + title: str + url: str + publisher: str + published_at: datetime | None = None + score: int + + +class StoryPublic(SQLModel): + slug: str + name: str + blurb: str + article_count: int + publisher_count: int + first_at: datetime | None = None + last_at: datetime | None = None + # An article joined it in the last 24 hours. + grew_today: bool + # Newest first, at most `stories.STORY_ARTICLES` of `article_count`. + articles: list[StoryArticlePublic] + + +class SavedStory(TypedDict): + slug: str + created: bool + # URLs this call added; ones already in the story are not counted again. + added: int + article_count: int + + +class CandidateArticle(TypedDict): + url: str + title: str + publisher: str + published_at: str + score: int + summary: str + + +class OpenStory(TypedDict): + slug: str + name: str + blurb: str + article_count: int + # Days since its newest article was published. + quiet_days: int + # The newest `CANDIDATE_TAIL`, newest first. + articles: list[CandidateArticle] + + +class Cluster(TypedDict): + # Open stories one of these articles sits close to: add to one of them + # before starting a new story. + near: list[str] + publishers: list[str] + articles: list[CandidateArticle] + + +class Candidates(TypedDict): + open: list[OpenStory] + clusters: list[Cluster] diff --git a/backend/app/catalog/stories.py b/backend/app/features/stories/service.py similarity index 92% rename from backend/app/catalog/stories.py rename to backend/app/features/stories/service.py index 95a853ab..2066815b 100644 --- a/backend/app/catalog/stories.py +++ b/backend/app/features/stories/service.py @@ -3,7 +3,7 @@ A story is wider than coverage. Coverage is one event told by several outlets, within `DEDUP_DIST_THRESHOLD`; a story is a theme a builder follows, which takes judgment, so an agent names and keeps them (ADR 16). This module is what -that agent drives through the MCP tools in `app/mcp/tools.py`, and what the +that agent drives through the MCP tools in `app/entry/mcp/`, and what the public rail reads. An article sits in at most one open story. Postgres cannot say that without a @@ -16,20 +16,24 @@ import re from collections import defaultdict from datetime import UTC, datetime, timedelta -from typing import TypedDict import numpy as np from sqlmodel import Session, col, func, select -from app.catalog.models import ( - Article, - Publisher, - Story, - StoryArticle, +from app.core.db.article import Article +from app.core.db.publisher import Publisher +from app.core.db.story import Story, StoryArticle +from app.core.embedding import cosine_distances, story_groups +from app.features.stories.model import ( + CandidateArticle, + Candidates, + Cluster, + OpenStory, + SavedStory, StoryArticlePublic, + StoryError, StoryPublic, ) -from app.core.embedding import cosine_distances, story_groups # What the rail shows: a story carried by fewer is coverage, or not yet a story. MIN_ARTICLES = 3 @@ -58,10 +62,6 @@ SENTENCE_BREAK = re.compile(r"(?<=[.!?])\s+(?=[A-Z0-9\"'“(])") -class StoryError(ValueError): - """A save or close the agent can correct, returned as a message.""" - - def _published(article_at: datetime | None, created_at: datetime) -> datetime: return article_at or created_at @@ -148,14 +148,6 @@ def _members(session: Session, story_ids: list[int]) -> list[tuple[int, Article, # ─── Agent writes ──────────────────────────────────────────────────────────── -class SavedStory(TypedDict): - slug: str - created: bool - # URLs this call added; ones already in the story are not counted again. - added: int - article_count: int - - def _validate(slug: str, name: str, blurb: str) -> tuple[str, str]: if len(slug) > SLUG_MAX or not SLUG.match(slug): raise StoryError( @@ -290,39 +282,6 @@ def close_story(session: Session, slug: str, now: datetime | None = None) -> Non # ─── Agent reads ───────────────────────────────────────────────────────────── -class CandidateArticle(TypedDict): - url: str - title: str - publisher: str - published_at: str - score: int - summary: str - - -class OpenStory(TypedDict): - slug: str - name: str - blurb: str - article_count: int - # Days since its newest article was published. - quiet_days: int - # The newest `CANDIDATE_TAIL`, newest first. - articles: list[CandidateArticle] - - -class Cluster(TypedDict): - # Open stories one of these articles sits close to: add to one of them - # before starting a new story. - near: list[str] - publishers: list[str] - articles: list[CandidateArticle] - - -class Candidates(TypedDict): - open: list[OpenStory] - clusters: list[Cluster] - - def _candidate(article: Article, publisher: str) -> CandidateArticle: return { "url": article.url, diff --git a/backend/app/catalog/tests/test_stories.py b/backend/app/features/stories/test_service.py similarity index 73% rename from backend/app/catalog/tests/test_stories.py rename to backend/app/features/stories/test_service.py index 80f5ac4e..2ea1c7d0 100644 --- a/backend/app/catalog/tests/test_stories.py +++ b/backend/app/features/stories/test_service.py @@ -1,23 +1,25 @@ -"""Stories: the schema, what the rail lists, what the agent may save, and the -story tools' write gate.""" +"""Stories: the schema, what the rail lists, and what the agent may save.""" from __future__ import annotations -from collections.abc import Generator from datetime import UTC, datetime, timedelta import numpy as np import pytest -from fastapi.testclient import TestClient from sqlalchemy import inspect -from sqlmodel import Session, col, delete, func, select - -from app.catalog import stories -from app.catalog.models import Article, Publisher, Story, StoryArticle -from app.catalog.tests.factories import create_random_article, create_random_publisher -from app.core.db import engine -from app.core.settings import settings -from tests.random_data import random_lower_string +from sqlmodel import Session, select + +from app.core.db.article import Article +from app.core.db.engine import engine +from app.core.db.factories import ( + create_random_article, + create_random_publisher, + random_lower_string, +) +from app.core.db.publisher import Publisher +from app.core.db.story import Story +from app.features.stories import service as stories +from app.features.stories.model import SavedStory, StoryError, StoryPublic NOW = datetime(2026, 10, 10, 12, tzinfo=UTC) # Slugs are unique per run: the tests share a database with whatever a @@ -32,22 +34,6 @@ def S(slug: str) -> str: DIM = 256 -@pytest.fixture -def made(db: Session) -> Generator[dict[str, list[int]]]: - """Ids of what a test created, deleted afterwards with every story made - meanwhile, so the next test starts from the same rows.""" - created: dict[str, list[int]] = {"articles": [], "publishers": []} - before = db.exec(select(func.max(Story.id))).one() or 0 - yield created - db.rollback() - ours = select(Story.id).where(col(Story.id) > before) - db.exec(delete(StoryArticle).where(col(StoryArticle.story_id).in_(ours))) # type: ignore[call-overload] - db.exec(delete(Story).where(col(Story.id) > before)) # type: ignore[call-overload] - db.exec(delete(Article).where(col(Article.id).in_(created["articles"]))) # type: ignore[call-overload] - db.exec(delete(Publisher).where(col(Publisher.id).in_(created["publishers"]))) # type: ignore[call-overload] - db.commit() - - def _publisher(db: Session, made: dict[str, list[int]]) -> Publisher: p = create_random_publisher(db) assert p.id is not None @@ -81,7 +67,7 @@ def _save( name: str | None = None, blurb: str = "What it is. Why it matters now.", now: datetime = NOW, -) -> stories.SavedStory: +) -> SavedStory: name = name or slug.replace("-", " ").title() return stories.save_story(db, slug, name, blurb, urls, now=now) @@ -100,14 +86,14 @@ def _story( return articles -def _mine(listed: list, slugs: set[str]) -> list[str]: +def _mine(listed: list[StoryPublic], slugs: set[str]) -> list[str]: return [s.slug for s in listed if s.slug in slugs] # ─── Migration ─────────────────────────────────────────────────────────────── -def test_the_migration_creates_both_tables(): +def test_the_migration_creates_both_tables() -> None: schema = inspect(engine) story = {c["name"] for c in schema.get_columns("story")} assert story >= { @@ -137,7 +123,7 @@ def test_the_migration_creates_both_tables(): def test_list_needs_three_articles_from_two_publishers( db: Session, made: dict[str, list[int]] -): +) -> None: one_publisher = _publisher(db, made) p1, p2 = _publisher(db, made), _publisher(db, made) _save( @@ -156,7 +142,7 @@ def test_list_needs_three_articles_from_two_publishers( ] -def test_list_hides_closed_stories(db: Session, made: dict[str, list[int]]): +def test_list_hides_closed_stories(db: Session, made: dict[str, list[int]]) -> None: _story(db, made, S("closed-one")) stories.close_story(db, S("closed-one"), now=NOW) @@ -165,7 +151,7 @@ def test_list_hides_closed_stories(db: Session, made: dict[str, list[int]]): def test_list_orders_by_newest_article_and_shapes_each_story( db: Session, made: dict[str, list[int]] -): +) -> None: older = _story(db, made, S("older"), days_ago=5) _story(db, made, S("newer"), days_ago=1) @@ -182,7 +168,7 @@ def test_list_orders_by_newest_article_and_shapes_each_story( def test_grew_today_reads_when_an_article_was_added( db: Session, made: dict[str, list[int]] -): +) -> None: _story(db, made, S("quiet")) later = NOW + timedelta(days=2) @@ -196,7 +182,7 @@ def test_grew_today_reads_when_an_article_was_added( def test_the_rail_caps_the_articles_it_sends( db: Session, made: dict[str, list[int]], monkeypatch: pytest.MonkeyPatch -): +) -> None: monkeypatch.setattr(stories, "STORY_ARTICLES", 8) p1, p2 = _publisher(db, made), _publisher(db, made) urls = [_article(db, made, p1 if i % 2 else p2, i).url for i in range(10)] @@ -211,28 +197,12 @@ def test_the_rail_caps_the_articles_it_sends( assert [a.url for a in shown.articles] == urls[:8] -def test_the_route_serves_the_list( - client: TestClient, db: Session, made: dict[str, list[int]] -): - _story(db, made, S("served")) - - r = client.get(f"{settings.API_V1_STR}/stories/", params={"limit": 20}) - assert r.status_code == 200 - story = next(s for s in r.json() if s["slug"] == S("served")) - assert set(story["articles"][0]) == { - "id", - "title", - "url", - "publisher", - "published_at", - "score", - } - - # ─── save_story ────────────────────────────────────────────────────────────── -def test_save_creates_then_updates_in_place(db: Session, made: dict[str, list[int]]): +def test_save_creates_then_updates_in_place( + db: Session, made: dict[str, list[int]] +) -> None: p = _publisher(db, made) first, second = _article(db, made, p), _article(db, made, p) @@ -282,17 +252,17 @@ def test_save_refuses_a_bad_label( name: str, blurb: str, message: str, -): +) -> None: url = _article(db, made, _publisher(db, made)).url - with pytest.raises(stories.StoryError, match=message): + with pytest.raises(StoryError, match=message): stories.save_story(db, slug, name, blurb, [url], now=NOW) def test_save_refuses_unknown_urls_and_writes_nothing( db: Session, made: dict[str, list[int]] -): +) -> None: known = _article(db, made, _publisher(db, made)).url - with pytest.raises(stories.StoryError, match="Not a published article"): + with pytest.raises(StoryError, match="Not a published article"): _save(db, S("nothing-written"), [known, "https://never.example/x"]) db.rollback() assert ( @@ -300,19 +270,19 @@ def test_save_refuses_unknown_urls_and_writes_nothing( ) -def test_save_refuses_a_new_story_with_no_articles(db: Session): - with pytest.raises(stories.StoryError, match="at least one"): +def test_save_refuses_a_new_story_with_no_articles(db: Session) -> None: + with pytest.raises(StoryError, match="at least one"): _save(db, S("empty"), []) -def test_an_article_sits_in_one_open_story(db: Session, made: dict[str, list[int]]): +def test_an_article_sits_in_one_open_story( + db: Session, made: dict[str, list[int]] +) -> None: p = _publisher(db, made) shared, other = _article(db, made, p), _article(db, made, p) _save(db, S("first"), [shared.url]) - with pytest.raises( - stories.StoryError, match=rf"another open story.*\({S('first')}\)" - ): + with pytest.raises(StoryError, match=rf"another open story.*\({S('first')}\)"): _save(db, S("second"), [other.url, shared.url]) db.rollback() assert db.exec(select(Story).where(Story.slug == S("second"))).first() is None @@ -323,16 +293,16 @@ def test_an_article_sits_in_one_open_story(db: Session, made: dict[str, list[int def test_a_closed_story_cannot_be_saved_or_closed_again( db: Session, made: dict[str, list[int]] -): +) -> None: url = _article(db, made, _publisher(db, made)).url _save(db, S("done"), [url]) stories.close_story(db, S("done"), now=NOW) - with pytest.raises(stories.StoryError, match="closed"): + with pytest.raises(StoryError, match="closed"): _save(db, S("done"), [url]) - with pytest.raises(stories.StoryError, match="already closed"): + with pytest.raises(StoryError, match="already closed"): stories.close_story(db, S("done"), now=NOW) - with pytest.raises(stories.StoryError, match="No story"): + with pytest.raises(StoryError, match="No story"): stories.close_story(db, S("never-was"), now=NOW) @@ -349,7 +319,7 @@ def _near(axis: int, wobble: float = 0.0) -> list[float]: def test_candidates_cluster_free_articles_and_name_the_story_they_reach( db: Session, made: dict[str, list[int]] -): +) -> None: p1, p2 = _publisher(db, made), _publisher(db, made) held = _article(db, made, p1, embedding=_near(0)) _save(db, S("held"), [held.url]) @@ -373,26 +343,3 @@ def test_candidates_cluster_free_articles_and_name_the_story_they_reach( assert by_urls[frozenset({a.url, b.url})]["publishers"] == sorted( [p1.name, p2.name] ) - - -# ─── Tool gate ─────────────────────────────────────────────────────────────── - - -def test_save_story_tool_reports_what_it_did( - db: Session, made: dict[str, list[int]], monkeypatch: pytest.MonkeyPatch -): - from fastmcp.server.auth import AccessToken - - from app.mcp import tools - - monkeypatch.setattr( - tools, - "get_access_token", - lambda: AccessToken(token="t", client_id="c", scopes=[tools.WRITE_SCOPE]), - ) - url = _article(db, made, _publisher(db, made)).url - slug = S("tool-made") - assert tools.save_story(slug, "Tool made", "It works.", [url]) == ( - f"Created {slug}: added 1, 1 articles in all" - ) - assert tools.close_story(slug) == f"Closed {slug}" diff --git a/backend/app/main.py b/backend/app/main.py index d6e3be77..c452a631 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -1,22 +1,31 @@ -"""FastAPI application entry point: wires routers, CORS, and the health check.""" +"""FastAPI application entry point: wires the API routers, the MCP app, CORS, and the health check.""" from __future__ import annotations +import logging + import sentry_sdk -from fastapi import FastAPI +from fastapi import APIRouter, FastAPI from fastapi.routing import APIRoute from starlette.middleware.cors import CORSMiddleware from app.core.settings import settings -from app.mcp.server import mcp_app -from app.router import api_router +from app.entry.api import analytics, articles, likes, login, stories, users +from app.entry.mcp.server import mcp_app + +# uvicorn configures only its own loggers, and collect and report print to +# stdout, so the API's INFO lines (the dev sign-in link) need this. +logging.basicConfig(level=logging.INFO) +# They log every request at INFO. +logging.getLogger("httpx").setLevel(logging.WARNING) +logging.getLogger("httpcore").setLevel(logging.WARNING) def custom_generate_unique_id(route: APIRoute) -> str: return f"{route.tags[0]}-{route.name}" -if settings.SENTRY_DSN and settings.ENVIRONMENT != "local": +if settings.SENTRY_DSN and settings.ENVIRONMENT != "development": sentry_sdk.init(dsn=str(settings.SENTRY_DSN), enable_tracing=True) app = FastAPI( @@ -38,6 +47,16 @@ def custom_generate_unique_id(route: APIRoute) -> str: allow_headers=["*"], ) +# Registration order is the order the endpoints appear in the OpenAPI document, +# and the generated client follows that document. Reordering churns the client +# for no behaviour change. +api_router = APIRouter() +api_router.include_router(login.router) +api_router.include_router(users.router) +api_router.include_router(articles.router) +api_router.include_router(likes.router) +api_router.include_router(analytics.router) +api_router.include_router(stories.router) app.include_router(api_router, prefix=settings.API_V1_STR) # Outside the versioned API on purpose: it is an MCP endpoint, not a REST # resource, and it carries its own bearer-token auth rather than a user JWT. diff --git a/backend/app/mcp/server.py b/backend/app/mcp/server.py deleted file mode 100644 index d9c4dac4..00000000 --- a/backend/app/mcp/server.py +++ /dev/null @@ -1,112 +0,0 @@ -"""The MCP server: the tools, the bearer-token gate, and the ASGI app `main` mounts.""" - -from __future__ import annotations - -import secrets - -from fastmcp import FastMCP -from fastmcp.server.auth import AccessToken, TokenVerifier - -from app.core.settings import settings -from app.mcp.tools import ( - WRITE_SCOPE, - approve, - check_link, - close_story, - draft_issue, - get_content, - list_candidates, - reject, - reject_many, - related, - save_story, - similar, - sql_query, - stories, - story_candidates, - vocabulary, - web_fetch, - web_search, - week, -) - - -def _matches(token: str, expected: str | None) -> bool: - """Constant-time compare against a configured token, if there is one. - - Bytes, not str: compare_digest raises on non-ASCII str operands, and the - token is whatever someone put in the header. An unset value matches - nothing, so a deploy that forgets the variable serves 401s instead of an - open database. - """ - return bool(expected) and secrets.compare_digest( - token.encode(), (expected or "").encode() - ) - - -class SharedSecret(TokenVerifier): - """Two bearer tokens, compared in constant time, one scope between them. - - There are two clients — a read-only agent answering questions about the - feed, and the curation agent that publishes to it — so an OAuth provider - would be ceremony around two values that already live in the same `.env` as - every other credential. The write token is checked first and is the only one - that comes back carrying `WRITE_SCOPE`; the read token reaches the three - reading tools and is refused by the rest. - """ - - async def verify_token(self, token: str) -> AccessToken | None: - if _matches(token, settings.MCP_WRITE_TOKEN): - return AccessToken( - token=token, client_id="agentique-curator", scopes=[WRITE_SCOPE] - ) - if _matches(token, settings.MCP_TOKEN): - return AccessToken(token=token, client_id="agentique-mcp", scopes=[]) - return None - - -mcp: FastMCP = FastMCP( - name="agentique", - instructions=( - "Tools for the agentique article pipeline. Reading: `sql_query` reads " - "the Postgres database (publishers, articles, scores, pipeline runs), " - "`web_fetch` reads one page, `web_search` finds pages. Curation, with " - "the write token only: `list_candidates` lists what is waiting on a " - "verdict, `get_content` pages through the stored text for one of them, " - "`similar` and `stories` compare candidates with the feed, " - "`check_link` looks up a repo or model, `vocabulary` lists the labels, " - "and `approve`, `reject` and `reject_many` settle them. Weekly " - "newsletter, with the write token only: `week` ranks the week's " - "topics, `related` finds what else covers one, and `draft_issue` " - "drafts an issue for a person to send and says where to save it as a " - "blog post. Stories, with the write token only: `story_candidates` " - "lists open stories and unassigned clusters, `save_story` creates or " - "extends one, `close_story` closes one." - ), - auth=SharedSecret(), - tools=[ - sql_query, - web_fetch, - web_search, - list_candidates, - get_content, - similar, - stories, - check_link, - vocabulary, - approve, - reject, - reject_many, - week, - related, - draft_issue, - story_candidates, - save_story, - close_story, - ], -) - -# Mounted at /mcp by `app.main`, so the route inside this app is the bare root. -# Its lifespan starts the streamable-HTTP session manager and has to be handed -# to the parent app -- a mounted ASGI app never gets a lifespan of its own. -mcp_app = mcp.http_app(path="/") diff --git a/backend/app/mcp/tests/test_server.py b/backend/app/mcp/tests/test_server.py deleted file mode 100644 index cd08f223..00000000 --- a/backend/app/mcp/tests/test_server.py +++ /dev/null @@ -1,61 +0,0 @@ -"""The bearer-token gate on the mounted MCP endpoint.""" - -from __future__ import annotations - -import pytest -from fastapi.testclient import TestClient - -from app.core.settings import settings - -MCP_URL = "/mcp/" -HEADERS = {"Accept": "application/json, text/event-stream"} -INITIALIZE = { - "jsonrpc": "2.0", - "id": 1, - "method": "initialize", - "params": { - "protocolVersion": "2025-06-18", - "capabilities": {}, - "clientInfo": {"name": "test", "version": "1"}, - }, -} - - -@pytest.fixture -def token(monkeypatch: pytest.MonkeyPatch) -> str: - monkeypatch.setattr(settings, "MCP_TOKEN", "test-mcp-token") - return "test-mcp-token" - - -def test_the_right_token_gets_a_session(client: TestClient, token: str): - response = client.post( - MCP_URL, - json=INITIALIZE, - headers={**HEADERS, "Authorization": f"Bearer {token}"}, - ) - assert response.status_code == 200 - - -@pytest.mark.usefixtures("token") -def test_a_wrong_token_is_rejected(client: TestClient): - response = client.post( - MCP_URL, json=INITIALIZE, headers={**HEADERS, "Authorization": "Bearer nope"} - ) - assert response.status_code == 401 - - -@pytest.mark.usefixtures("token") -def test_no_token_is_rejected(client: TestClient): - response = client.post(MCP_URL, json=INITIALIZE, headers=HEADERS) - assert response.status_code == 401 - - -def test_an_unconfigured_server_rejects_every_token( - client: TestClient, monkeypatch: pytest.MonkeyPatch -): - # A deploy that forgets MCP_TOKEN must serve 401s, not an open database. - monkeypatch.setattr(settings, "MCP_TOKEN", None) - response = client.post( - MCP_URL, json=INITIALIZE, headers={**HEADERS, "Authorization": "Bearer "} - ) - assert response.status_code == 401 diff --git a/backend/app/mcp/tests/test_tools.py b/backend/app/mcp/tests/test_tools.py deleted file mode 100644 index 05b055f9..00000000 --- a/backend/app/mcp/tests/test_tools.py +++ /dev/null @@ -1,142 +0,0 @@ -"""The MCP tools: the read-only guard, the row cap, JSON coercion, and the two wrappers.""" - -from __future__ import annotations - -import json - -import pytest -from fastmcp.exceptions import ToolError - -from app.core.settings import settings -from app.mcp import tools - - -@pytest.fixture(autouse=True) -def _test_dsn(monkeypatch: pytest.MonkeyPatch) -> None: - """Point the tool at the test credentials instead of `agentique_ro`. - - The role does not exist outside the production box, but the DSN is the one - settings build, read-only options and all, so the guard under test is real. - """ - monkeypatch.setattr( - tools, - "DSN", - settings.model_copy( - update={ - "MCP_DB_USER": settings.POSTGRES_USER, - "MCP_DB_PASSWORD": settings.POSTGRES_PASSWORD, - } - ).MCP_DATABASE_URI, - ) - - -def test_select_returns_columns_and_rows(): - result = json.loads(tools.sql_query("SELECT 1 AS n, 'a' AS label")) - assert result == { - "columns": ["n", "label"], - "rows": [{"n": 1, "label": "a"}], - "truncated": False, - } - - -def test_rows_are_capped_and_flagged(): - result = json.loads( - tools.sql_query(f"SELECT i FROM generate_series(1, {tools.MAX_ROWS + 5}) AS i") - ) - assert len(result["rows"]) == tools.MAX_ROWS - assert result["truncated"] is True - - -def test_a_result_that_exactly_fills_the_cap_is_not_truncated(): - result = json.loads( - tools.sql_query(f"SELECT i FROM generate_series(1, {tools.MAX_ROWS}) AS i") - ) - assert len(result["rows"]) == tools.MAX_ROWS - assert result["truncated"] is False - - -def test_timestamps_and_numerics_survive_serialisation(): - result = json.loads( - tools.sql_query( - "SELECT TIMESTAMP '2026-01-02 03:04:05' AS ts, 1.5::numeric AS amount" - ) - ) - assert result["rows"] == [{"ts": "2026-01-02T03:04:05", "amount": "1.5"}] - - -def test_a_write_is_rejected_by_the_read_only_transaction(): - # Not a syntax check on the way in: Postgres refuses it, so anything the - # tool failed to anticipate is refused too. - with pytest.raises(ToolError, match="read-only transaction"): - tools.sql_query("UPDATE article SET title = title WHERE false") - - -def test_anything_postgres_cannot_plan_as_a_query_is_refused(): - # Planning is the filter: COPY, SET, DO and a second statement after a - # semicolon never reach execution. - with pytest.raises(ToolError): - tools.sql_query("SET application_name = 'mcp'") - - -def test_the_user_table_is_not_readable(): - with pytest.raises(ToolError, match="Not readable through this tool: user"): - tools.sql_query('SELECT email FROM "user"') - - -def test_the_user_table_is_not_readable_through_a_subquery(): - # The check walks the whole plan, so hiding it below the top node does not - # help -- nor would a join, a CTE or a view. - with pytest.raises(ToolError, match="Not readable through this tool: user"): - tools.sql_query('SELECT 1 WHERE EXISTS (SELECT 1 FROM "user")') - - -def test_a_second_statement_after_a_semicolon_cannot_smuggle_a_read(): - with pytest.raises(ToolError): - tools.sql_query('SELECT 1; SELECT email FROM "user"') - - -def test_a_statement_crafted_to_close_the_explain_prefix_is_still_one_statement(): - # The prefix is text, so this would be three valid statements if anything - # but a prepared statement ran it -- and the middle one is the read. - with pytest.raises(ToolError): - tools.sql_query( - 'SELECT 1) AS x; SELECT email FROM "user"; SELECT * FROM (SELECT 1' - ) - - -def test_web_fetch_returns_extracted_text_and_forwards_the_cap( - monkeypatch: pytest.MonkeyPatch, -): - seen: list[tuple[str, int]] = [] - - def fake(url: str, max_length: int) -> str: - seen.append((url, max_length)) - return f"text of {url}" - - monkeypatch.setattr(tools, "fetch_and_extract", fake) - text = tools.web_fetch("https://example.com/post", max_length=50) - assert text == "text of https://example.com/post" - assert seen == [("https://example.com/post", 50)] - - -def test_web_fetch_fails_loudly_when_nothing_is_readable( - monkeypatch: pytest.MonkeyPatch, -): - monkeypatch.setattr(tools, "fetch_and_extract", lambda url, max_length: "") - with pytest.raises(ToolError, match="No readable content"): - tools.web_fetch("https://example.com/paywalled") - - -def test_web_search_forwards_every_argument(monkeypatch: pytest.MonkeyPatch): - calls: list[tuple[str, int, list[str] | None]] = [] - - def fake( - query: str, max_results: int, include_domains: list[str] | None - ) -> list[dict[str, str | None]]: - calls.append((query, max_results, include_domains)) - return [{"title": "t", "url": "u", "description": "d", "published_date": None}] - - monkeypatch.setattr(tools, "tavily_search", fake) - results = tools.web_search("gpt-5", max_results=3, include_domains=["openai.com"]) - assert calls == [("gpt-5", 3, ["openai.com"])] - assert results[0]["url"] == "u" diff --git a/backend/app/mcp/tools.py b/backend/app/mcp/tools.py deleted file mode 100644 index 3bbfe839..00000000 --- a/backend/app/mcp/tools.py +++ /dev/null @@ -1,502 +0,0 @@ -"""What an agent can do against agentique: read the db and the web, curate, -draft the weekly newsletter, and keep the rail's stories. - -Two groups, two tokens. `sql_query`, `web_fetch` and `web_search` read and are -reachable with `MCP_TOKEN`. The curation tools publish and reject articles, and -the newsletter tools draft emails addressed to every subscriber, and the story -tools at the bottom name what the right rail shows; all three need `MCP_WRITE_TOKEN` — see `WRITE_SCOPE` below. -""" - -from __future__ import annotations - -import json -from collections.abc import Iterator -from datetime import date, datetime - -import psycopg -from fastmcp.exceptions import ToolError -from fastmcp.server.dependencies import get_access_token -from sqlmodel import Session - -from app.catalog import stories as story_domain -from app.core.db import engine -from app.core.extract import fetch_and_extract -from app.core.settings import settings -from app.core.web_search import tavily_search -from app.newsletter import broadcast, digest -from pipeline import curation -from pipeline.fetching import page_facts - -# The scope `MCP_WRITE_TOKEN` carries and the read token does not. Named for -# what it lets a caller do, not for the token that carries it: `server.py` mints -# it, the curation tools below demand it. -WRITE_SCOPE = "curate" - -# A cap, not a page size: an agent that wants more should narrow the query or -# add its own LIMIT. Anything larger is context an agent cannot read anyway. -MAX_ROWS = 1000 - -# `user` holds emails and sign-in link tokens, and no question worth asking this -# tool needs either. The role's grants are the real boundary (see -# deploy/sql-roles.sql); the plan check below holds even where the tool runs as -# a role that has the grant anyway, which is every local database. -BLOCKED_TABLES = frozenset({"user"}) - -# Its own connection, not `app.core.db` — that one logs in as POSTGRES_USER -# and would happily run a DELETE. One per call rather than a pool: the cost is -# nothing at this call rate, and no session state survives a query. -DSN = settings.MCP_DATABASE_URI - - -def _json_safe(value: object) -> str: - """Fallback for the column types `json.dumps` has no encoding for. - - Timestamps get ISO-8601; Decimal, UUID and pgvector embeddings get `str`, - which is lossless for all three and keeps one unreadable column from - failing the whole query. - """ - if isinstance(value, datetime | date): - return value.isoformat() - return str(value) - - -def _plan_tables(node: object) -> Iterator[str]: - """Every table an EXPLAIN plan reads, however deeply nested. - - Walks the whole JSON rather than the top node: the table can sit under a - join, a subplan or a CTE. Views are expanded during planning, so what comes - back is the tables actually read, not the names the query used. - """ - if isinstance(node, dict): - name = node.get("Relation Name") - if isinstance(name, str): - yield name - for value in node.values(): - yield from _plan_tables(value) - elif isinstance(node, list): - for item in node: - yield from _plan_tables(item) - - -def _reject_blocked_tables( - cursor: psycopg.Cursor[tuple[object, ...]], sql: str -) -> None: - """Plan the statement, and refuse it if the plan reads a blocked table. - - EXPLAIN reports the tables actually read -- views expanded, joins, CTEs and - subplans included -- and refusing to plan is what keeps this a query tool, - since COPY, SET and DO never get that far. - - `prepare` is what makes the check honest. Given no parameters psycopg falls - back to the simple query protocol, which runs every command in the string, - so `SELECT 1; SELECT * FROM "user"` would be two commands and only the - first would be the one planned here. A prepared statement takes exactly - one, so the server rejects the second before anything executes. - - psycopg types a `str` query as LiteralString to stop callers interpolating - input into SQL. Doing precisely that is this tool's job, hence the - suppression; what keeps it safe is the role, this check, and `prepare`. - """ - cursor.execute(f"EXPLAIN (FORMAT JSON) {sql}", prepare=True) # ty: ignore[no-matching-overload] - plan = cursor.fetchone() - blocked = BLOCKED_TABLES.intersection(_plan_tables(plan[0] if plan else None)) - if blocked: - raise ToolError(f"Not readable through this tool: {', '.join(sorted(blocked))}") - - -def sql_query(sql: str) -> str: - """Run a read-only SQL query against the agentique Postgres database. - - Accepts one statement, and only one Postgres can plan as a query. The - connection is a read-only transaction held by a role with SELECT grants - only, so writes and DDL fail rather than being filtered out beforehand, and - statements are cancelled after 10 seconds. The `user` table is not - readable, directly or through a join. A rejected statement comes back as - Postgres's own error message. - - Returns JSON: `columns` (in select order), `rows` (a list of objects), and - `truncated` (true when the result hit the 1000-row cap and rows are - missing). Timestamps are ISO-8601 strings; numerics, UUIDs and embedding - vectors are stringified. - """ - try: - with psycopg.connect(DSN) as conn, conn.cursor() as cursor: - _reject_blocked_tables(cursor, sql) - - cursor.execute(sql, prepare=True) # ty: ignore[no-matching-overload] - if cursor.description is None: - # Unreachable while the plan check stands -- nothing that gets - # past it returns without a result set. Kept as an error rather - # than an empty result, which would read as "no rows found". - raise ToolError("Statement returned no result set") - columns = [column.name for column in cursor.description] - # One past the cap, so a full page can be told from an exact fit. - rows = cursor.fetchmany(MAX_ROWS + 1) - return json.dumps( - { - "columns": columns, - "rows": [ - dict(zip(columns, row, strict=True)) for row in rows[:MAX_ROWS] - ], - "truncated": len(rows) > MAX_ROWS, - }, - default=_json_safe, - ) - except psycopg.Error as exc: - # The agent wrote this SQL and can fix it, but only if it sees what - # Postgres actually said; anything but a ToolError reaches the client - # as a bare "internal error". - raise ToolError(str(exc)) - - -def web_fetch(url: str, max_length: int = 20000) -> str: - """Fetch a web page and return its readable text, boilerplate stripped. - - Retries through a residential proxy when the direct fetch is blocked by a - 403, a paywall or a login wall. Fails if the page yields nothing readable — - an empty string would look like an empty page rather than a failed fetch. - """ - content = fetch_and_extract(url, max_length) - if not content: - raise ToolError(f"No readable content at {url}") - return content - - -def web_search( - query: str, max_results: int = 5, include_domains: list[str] | None = None -) -> list[dict[str, str | None]]: - """Search the web (Tavily) and return ranked results. - - Each result has `title`, `url`, `description` and `published_date` - (null unless the source dated the page). `include_domains` restricts hits - to those hosts, e.g. `["openai.com"]` for first-party announcements only. - """ - return tavily_search( - query, max_results=max_results, include_domains=include_domains - ) - - -# ─── Curation ──────────────────────────────────────────────────────────────── -# The verbs the curation agent drives the pipeline's tail with. Some write and -# all of them read the queue, so each one checks for the write scope first: the -# read token reaches `sql_query`, `web_fetch` and `web_search` and nothing else. - - -def _require_write() -> None: - """Refuse a write to a caller holding only the read token.""" - token = get_access_token() - if token is None or WRITE_SCOPE not in token.scopes: - raise ToolError( - "This tool needs the curation token — the read token cannot write." - ) - - -def _curation_session() -> Session: - """A read/write session, not the read-only DSN `sql_query` uses.""" - return Session(engine) - - -def list_candidates( - offset: int = 0, limit: int = curation.LIST_LIMIT -) -> curation.CandidatePage: - """List one page of the articles waiting on a curation verdict, freshest first. - - Returns `rows`, `offset`, `next_offset` (null on the last page — pass it - back as `offset` for the next one) and `total` pending. Pages hold at most - 50 rows. Read every page before settling any candidate: a verdict removes - its row and shifts the pages after it. - - One row per candidate: `url`, `title`, `source` (where the link was found), - `publisher`, `publisher_kind` (`unknown` for a site the pipeline has just - created a publisher for: `approve` then needs one), `approved` (the - publisher's approvals / decisions over the - last 90 days, e.g. "3/10", or "new" when it has none), `traction`, - `published_at`, `queued_at`, and a 200-character `snippet`. Deliberately not the full text — call - `get_content` for a candidate worth a closer look. - - A candidate stays listed until `approve` or `reject` is called on it, so a - session that stops halfway leaves the rest for the next one. - """ - _require_write() - with _curation_session() as session: - return curation.list_candidates(session, offset, limit) - - -def get_content( - url: str, offset: int = 0, limit: int = curation.PAGE_LIMIT -) -> curation.ContentPage: - """Return one page of the article text the pipeline extracted for a candidate. - - The stored text is what a page fetch returns, cut at 12000 characters, so - read here first. Returns `text`, `offset`, `next_offset` (null on the last - page — pass it back as `offset` to read on), `total` characters, and, on - the first page only, `links`: the repo / model / paper / docs URLs the - article body links, grouped. Fetch the page only when this is empty, a - teaser, or cut off before the part you need. - """ - _require_write() - with _curation_session() as session: - try: - return curation.get_content(session, url, offset, limit) - except curation.CandidateError as exc: - raise ToolError(str(exc)) - - -def similar(url: str, days: int = 7, limit: int = 8) -> curation.Similar: - """Show what the feed and the ledger hold that is close to one candidate. - - Compares by embedding against every published article and every candidate - or reject from the last `days`. `rows` lists up to `limit` neighbours, the - closest first: `url`, `title`, `publisher`, `stage` (`published`, - `pending`, or the reject stage), `score` and `distance` (under 0.30 is the - same story). `coverage` is how many distinct publishers carry this story, - this candidate's own included, with their names in `publishers`. - """ - _require_write() - with _curation_session() as session: - try: - return curation.similar(session, url, days, limit) - except curation.CandidateError as exc: - raise ToolError(str(exc)) - - -def stories(days: int = 7) -> list[curation.Story]: - """Group the pending candidates that are one story, with each other or with - an article published in the last `days`. - - Only groups of two or more come back, largest first, each with its - `members` (same row shape as `similar`, without the distance), `coverage` - (distinct publishers) and `publishers`. A member at stage `published` - means the feed already carries the story. Call it once, after triage. - """ - _require_write() - with _curation_session() as session: - return curation.stories(session, days) - - -def check_link(url: str) -> dict[str, object]: - """Look up a GitHub or GitLab repo, or a Hugging Face model or dataset. - - Repos: `stars`, `forks`, `last_push`, `license`, `archived`, `has_readme`, - `description`. Models and datasets: `downloads`, `likes`, `last_modified`, - `license`, `files` (count), and for a model `weights` (true when weight - files are present). A value the lookup could not get reads `unknown`, never - 0; `lookup: unknown` means the whole lookup failed. Any other URL is - refused — read it with `web_fetch`. - """ - _require_write() - try: - return page_facts.check_link(url) - except page_facts.LinkError as exc: - raise ToolError(str(exc)) - - -def vocabulary() -> dict[str, object]: - """The labels `approve` accepts: `kinds`, `publisher_kinds`, and `tags` as - slug -> description. Call it once per session, before the first approve.""" - _require_write() - with _curation_session() as session: - return curation.vocabulary(session) - - -def approve( - url: str, - score: int, - reason: str, - summary: str, - kind: str, - tags: list[str], - publisher_kind: str | None = None, - publisher_name: str | None = None, -) -> str: - """Publish a candidate with the labels you chose, then embed it. - - `score` is 1-100 on the same scale the rubric describes, `reason` one short - sentence naming what decided it, and `summary` the text a reader sees under - the title. `kind` and `tags` (up to 3) come from `vocabulary`; a github, - huggingface or arxiv URL sets its own kind. When the candidate's - `publisher_kind` is `unknown`, pass `publisher_kind` (from - `vocabulary`'s `publisher_kinds`) and, if its host is not a readable name, - `publisher_name`; both are ignored for a publisher that has a kind. A label - that is not in the vocabulary is refused and the candidate stays pending. - The candidate stops being pending in the same transaction that inserts the - article, so nothing is ever published twice. - """ - _require_write() - with _curation_session() as session: - try: - article_id = curation.approve( - session, - url, - score, - reason, - summary, - kind, - tags, - publisher_kind, - publisher_name, - ) - except curation.CandidateError as exc: - raise ToolError(str(exc)) - return f"Published article #{article_id}: {url}" - - -def reject_many(verdicts: list[curation.Verdict]) -> str: - """Turn down many candidates at once: each of `verdicts` is `{url, score, - reason}`, the same fields `reject` takes. - - Each is settled on its own, so one URL that is not pending costs that line - and not the batch. Returns one line per verdict, in order, saying whether - it was rejected or why not. - """ - _require_write() - with _curation_session() as session: - return "\n".join(curation.reject_many(session, verdicts)) - - -def reject(url: str, score: int, reason: str) -> str: - """Turn a candidate down, keeping the score and the reason for it. - - The row stays in the ledger so the URL is never judged twice, and so the - verdicts can be read back as labels. Write `reason` for a human reading a - hundred of them later, not for a log line. - """ - _require_write() - with _curation_session() as session: - try: - curation.reject(session, url, score, reason) - except curation.CandidateError as exc: - raise ToolError(str(exc)) - return f"Rejected [{score}/100] {url}" - - -# ─── Newsletter ────────────────────────────────────────────────────────────── -# The weekly issue: find the week's topics, what backs each one, draft it. -# Behind the same write scope as curation, because a draft lands in the audience -# subscribers are mailed from. Sending is a person's, from the Resend dashboard. - - -def week(days: int = 7, limit: int = digest.WEEK_LIMIT) -> list[digest.WeekTopic]: - """The weekly newsletter's material: the last `days` of published articles - grouped into topics, the most talked-about first. - - Ranked by `coverage` (distinct publishers carrying it, counting the copies - curation rejected as retellings), then `likes`, then `top_score`; up to - `limit` topics. Each topic's `articles` are the published ones - `url`, - `title`, `publisher`, `publisher_kind` (`lab`, `company`, `individual`, - `media`, `community`, `unknown`), `kind`, `score`, `likes`, and the `summary` curation - wrote after reading it - and `covered_by` the other outlets' copies from - the ledger, with curation's `reason`. - """ - _require_write() - with _curation_session() as session: - return digest.week(session, days, limit) - - -def related( - url: str, days: int = digest.RELATED_DAYS, limit: int = digest.RELATED_LIMIT -) -> digest.Related: - """What else covers one published article's subject: the individual writer - who tried it, the repo built on it, the commentary on it. - - Compares by embedding against every article from the last `days` and every - reject from that window that scored 35 or more. Up to `limit` rows, the - closest first: `url`, `title`, `publisher`, `publisher_kind`, `kind`, - `stage` (`published` or `rejected`), `score`, `distance` (under 0.30 is the - same story, up to 0.45 is the same subject), and `summary` for a published - row or curation's `reason` for a rejected one. - """ - _require_write() - with _curation_session() as session: - try: - return digest.related(session, url, days, limit) - except digest.DigestError as exc: - raise ToolError(str(exc)) - - -def draft_issue(post: str) -> str: - """Draft one weekly issue as a Resend broadcast. - - `post` is the whole blog post file: a frontmatter block, then the body. - - --- - title: What does Ollama's MLX backend actually do? - description: One line that makes someone open it. - topic: local-ai - --- - The essay... - - `title` (1-55 characters) is the subject and the post's headline, - `description` (1-160) the line an inbox shows after it, `topic` a kebab - slug that names the draft in Resend. Quote a value in double quotes if it - holds ": " or starts with a quote or a symbol. The body is plain text, - 150-600 words: a blank line between paragraphs, `- ` to start a list line, - `[text](url)` for a link (at least 3 distinct sources), backticks for code. - The server renders it into the site's template. Returns the broadcast id - and the path to save the post at, exactly as passed. Nothing is sent: a - person reads the drafts and sends one from the Resend dashboard. - """ - _require_write() - try: - return broadcast.draft(post) - except broadcast.IssueError as exc: - raise ToolError(str(exc)) - - -# ─── Stories ───────────────────────────────────────────────────────────────── -# The right rail's named threads, kept by their own daily routine (ADR 16). -# Behind the write scope: a saved story is on the public site at once. - - -def story_candidates(days: int = 14) -> story_domain.Candidates: - """What the stories routine decides from. - - `open`: every open story, the most recently active first, with `slug`, - `name`, `blurb`, `article_count`, `quiet_days` (since its newest article - was published) and its newest 5 `articles`. `clusters`: the last `days` - of published articles that sit in no open story, grouped by embedding at - a looser cut than coverage, the most publishers first. Each cluster has - `articles` (`url`, `title`, `publisher`, `published_at`, `score`, - `summary`), its `publishers`, and `near`: the open stories one of its - articles is close to. A single article close to a story comes back as a - cluster of one with that story in `near`. - """ - _require_write() - with _curation_session() as session: - return story_domain.story_candidates(session, days) - - -def save_story(slug: str, name: str, blurb: str, urls: list[str]) -> str: - """Create a story, or update an open one: set its `name` and `blurb` and - add `urls` to it (ones already in it are skipped). - - `slug` is lowercase words joined by hyphens and never changes. `name` is - at most 4 words, `blurb` at most 2 sentences and 280 characters. Every URL - must be a published article and in no other open story. Any problem - refuses the whole call, saying which, and nothing is written. A closed - story cannot be saved to. The rail shows a story once it has 3 articles - from 2 publishers. - """ - _require_write() - with _curation_session() as session: - try: - saved = story_domain.save_story(session, slug, name, blurb, urls) - except story_domain.StoryError as exc: - raise ToolError(str(exc)) - verb = "Created" if saved["created"] else "Updated" - return ( - f"{verb} {saved['slug']}: added {saved['added']}, " - f"{saved['article_count']} articles in all" - ) - - -def close_story(slug: str) -> str: - """Close an open story: it leaves the rail, and its articles are free to - join another.""" - _require_write() - with _curation_session() as session: - try: - story_domain.close_story(session, slug) - except story_domain.StoryError as exc: - raise ToolError(str(exc)) - return f"Closed {slug}" diff --git a/backend/app/newsletter/models.py b/backend/app/newsletter/models.py deleted file mode 100644 index b7b2b3cc..00000000 --- a/backend/app/newsletter/models.py +++ /dev/null @@ -1,22 +0,0 @@ -"""Newsletter schema: the subscriber row.""" - -from __future__ import annotations - -from datetime import datetime - -from sqlalchemy import JSON, Column -from sqlmodel import Field, SQLModel - -from app.core.dates import get_datetime_utc - - -class NewsletterSubscriber(SQLModel, table=True): - __tablename__ = "newsletter_subscriber" - email: str = Field(primary_key=True) - categories: list[str] = Field( - default_factory=lambda: ["all"], sa_column=Column(JSON, nullable=False) - ) - custom_category: str = Field(default="") - utm_source: str | None = Field(default=None) - created_at: datetime = Field(default_factory=get_datetime_utc) - updated_at: datetime = Field(default_factory=get_datetime_utc) diff --git a/backend/app/newsletter/subscribers.py b/backend/app/newsletter/subscribers.py deleted file mode 100644 index 5075fe08..00000000 --- a/backend/app/newsletter/subscribers.py +++ /dev/null @@ -1,30 +0,0 @@ -"""Adding an email to the newsletter: the subscriber row and the Resend contact.""" - -from __future__ import annotations - -import logging -import os - -import resend -from sqlmodel import Session - -from app.newsletter.models import NewsletterSubscriber - -logger = logging.getLogger(__name__) - - -def subscribe(*, session: Session, email: str, utm_source: str | None = None) -> None: - """Add an email to the newsletter. An address already on it is left as is.""" - if session.get(NewsletterSubscriber, email) is None: - session.add(NewsletterSubscriber(email=email, utm_source=utm_source)) - session.commit() - - api_key = os.getenv("RESEND_API_KEY") - audience_id = os.getenv("RESEND_AUDIENCE_ID") - if api_key and audience_id: - resend.api_key = api_key - try: - resend.Contacts.create({"email": email, "audience_id": audience_id}) - except Exception as e: - if "already exists" not in str(e): - logger.error(f"Resend contact create failed for {email}: {e}") diff --git a/backend/app/router.py b/backend/app/router.py deleted file mode 100644 index 6d12b130..00000000 --- a/backend/app/router.py +++ /dev/null @@ -1,21 +0,0 @@ -"""Aggregates every domain router into the versioned API router mounted by app.main.""" - -from __future__ import annotations - -from fastapi import APIRouter - -from app.analytics.routes import router as analytics_router -from app.audience.routes import likes_router, login_router, users_router -from app.catalog.routes import router as catalog_router -from app.catalog.routes import stories_router - -# Registration order is the order the endpoints appear in the OpenAPI document, -# and the generated client follows that document. Reordering churns the client -# for no behaviour change. -api_router = APIRouter() -api_router.include_router(login_router) -api_router.include_router(users_router) -api_router.include_router(catalog_router) -api_router.include_router(likes_router) -api_router.include_router(analytics_router) -api_router.include_router(stories_router) diff --git a/backend/app/test_main.py b/backend/app/test_main.py new file mode 100644 index 00000000..2bf6cab3 --- /dev/null +++ b/backend/app/test_main.py @@ -0,0 +1,63 @@ +"""What only the whole app can show: every table is declared in `app.core.db`, +the routes and the collect run share one engine, and the API process logs at +INFO.""" + +from __future__ import annotations + +import os +import subprocess +import sys +from pathlib import Path +from typing import get_args + +from sqlmodel import SQLModel + +from app.core import db +from app.core.db.engine import engine, get_session +from app.entry.api.deps import SessionDep +from app.features.collect import service as collect +from app.main import app # noqa: F401 # every module that could declare a table + + +def test_every_table_lives_in_core_db() -> None: + """The app is fully imported here, so a table declared anywhere else shows + up in the metadata and not in `app.core.db`.""" + declared = set() + for name in db.__all__: + for cls in vars(getattr(db, name)).values(): + if isinstance(cls, type) and issubclass(cls, SQLModel): + table = getattr(cls, "__table__", None) + if table is not None: + declared.add(table.name) + assert declared == set(SQLModel.metadata.tables) + + +def test_routes_and_the_collect_run_share_the_engine() -> None: + assert vars(collect)["engine"] is engine + assert get_args(SessionDep)[1].dependency is get_session + + +def test_the_api_process_prints_the_dev_sign_in_link() -> None: + """A fresh interpreter, because pytest installs its own log handlers. Without + a logging config the link is dropped and a developer can't sign in.""" + # Empty env vars don't override `.env`, so the send is stubbed out instead: + # a developer's `.env` may hold a real Resend key. + script = ( + "import logging\n" + "import app.main\n" + "assert logging.getLogger('httpx').getEffectiveLevel() == logging.WARNING\n" + "from app.features.audience import email\n" + "email.send_email = lambda **_: None\n" + "email.send_link_email('dev@example.com', " + "email.generate_sign_in_email('t0ken'), 't0ken')\n" + ) + result = subprocess.run( + [sys.executable, "-c", script], + cwd=Path(__file__).parents[1], + env=os.environ | {"ENVIRONMENT": "development"}, + capture_output=True, + text=True, + ) + assert result.returncode == 0, result.stderr + assert "[dev] Sign-in link for dev@example.com" in result.stderr + assert "token=t0ken" in result.stderr diff --git a/backend/baml_client/inlinedbaml.py b/backend/baml_client/inlinedbaml.py index ee744bec..48cba6d1 100644 --- a/backend/baml_client/inlinedbaml.py +++ b/backend/baml_client/inlinedbaml.py @@ -14,7 +14,7 @@ "clients.baml": "// The provider caps how many requests may be in flight at once, far below its\n// per-minute quota: measured 2026-09-03, 5 parallel calls all answered while 10\n// returned 429 in ~90ms with x-ratelimit-remaining still at 1980/2000, and the\n// cap refills within ~10s. So a 429 here means \"too many at once\", never \"out\n// of budget\", and waiting is the correct response. Without this policy those\n// bursts fell through to the fallback client instead.\nretry_policy Burst {\n max_retries 4\n strategy {\n type exponential_backoff\n delay_ms 500\n multiplier 2\n max_delay_ms 8000\n }\n}\n\n// First in the Nvidia wrapper below since 2026-09-13. Second in line from 2026-09-03, replacing\n// NVIDIA's gpt-oss-120b, which reached end of life that morning and answered\n// every call with 410 Gone - dead weight for hours before anyone noticed,\n// because a fallback only speaks up when the primary fails.\n//\n// Measured 2026-09-03 on a SelectProductLink-shaped prompt: 1.5-6.8s across\n// sequential calls, all answering with clean JSON and the right pick — no sign\n// of the 52-285s latency that made gpt-oss-120b unusable, though that was a\n// property of the endpoint under load, so it is worth re-measuring if runs\n// start crawling. Past 8 parallel calls NIM starts returning 503; the retry\n// policy covers those.\n//\n// Qwen3.5-397B was the other candidate and judged just as well,\n// but it is a reasoning model: it spends its budget in a `reasoning` field and\n// emits an EMPTY content string if it runs out before it stops thinking (427\n// completion tokens for a two-element array). That needs a large explicit\n// max_tokens to be safe.\nclient Nemotron {\n provider openai-generic\n retry_policy Burst\n options {\n base_url env.NVIDIA_NIM_BASE_URL\n api_key env.NVIDIA_NIM_API_KEY\n model \"nvidia/nemotron-3-super-120b-a12b\"\n // The timeout lives in `http`, which BAML applies itself. A bare `timeout`\n // option is sent as a body param, and NIM answers that with 400\n // \"Unsupported parameter(s)\": every call here failed that way from\n // 2026-09-04 until this block replaced it.\n http {\n request_timeout_ms 120000\n }\n }\n}\n\n// Second in the Nvidia fallback. The smaller sibling of the gpt-oss-120b that\n// led scoring until its endpoint reached end of life. Measured on the old\n// scoring prompt (2026-09-13): clean JSON, but it read a rubric floor as a cap.\nclient GptOss20b {\n provider openai-generic\n retry_policy Burst\n options {\n base_url env.NVIDIA_NIM_BASE_URL\n api_key env.NVIDIA_NIM_API_KEY\n model \"openai/gpt-oss-20b\"\n http {\n request_timeout_ms 120000\n }\n }\n}\n\n// Last in the Nvidia fallback since 2026-09-13; primary from 2026-08-06 until\n// then. Fast (2-3s) and reliable, but a lower-quality judge: ±15 run-to-run\n// variance on identical scoring input, and it intermittently returned fewer\n// results than it was given. It stays as\n// the last resort because it is the only client not on NIM: when NIM refuses a\n// burst, the two clients above it fail together.\nclient Ministral {\n provider openai-generic\n retry_policy Burst\n options {\n base_url env.MISTRAL_AI_BASE_URL\n api_key env.MISTRAL_AI_KEY\n model \"mistralai/Ministral-3-14B-Instruct-2512\"\n // A real client-side timeout. The old `timeout 120` went out as a body\n // param, which this endpoint silently ignored: it never timed anything out.\n http {\n request_timeout_ms 120000\n }\n }\n}\n\n// The name every function references. A fallback wrapper: Nemotron first, then\n// gpt-oss-20b, then Ministral. The name stays - with two NIM models in front it\n// is mostly true.\n//\n// A Gemini Flash client sat third here until 2026-09-03, when the run that\n// needed it got 400 FAILED_PRECONDITION \"User location is not supported for\n// the API use\" - the Google AI API does not serve the prod VPS's region, so\n// the last resort had never been able to answer. Removed rather than left in\n// place: a client that always fails is not a fallback, it just hides how thin\n// the chain really is. A replacement has to be reachable from the box.\nclient Nvidia {\n provider fallback\n options {\n strategy [Nemotron, GptOss20b, Ministral]\n }\n}\n", "discover.baml": "// Batched relevance classifier for the Substack discovery crawler.\n// One LLM call judges many candidate profiles at once (array in, array out).\n// Classification runs on cheap\n// metadata (name + description) BEFORE any per-profile post fetch, and gates\n// both DB insertion and whether the crawler recurses into that profile.\n\nclass ProfileInput {\n handle string @description(\"Substack owner handle, echoed back in the verdict\")\n name string @description(\"Publication name\")\n description string? @description(\"Publication hero text and/or author bio, if known\")\n}\n\nclass ProfileVerdict {\n handle string @description(\"Must exactly match the input handle\")\n isAiRelated bool\n confidence int @description(\"0-100 confidence that AI/ML/LLMs is the PRIMARY focus\")\n}\n\nfunction ClassifyProfiles(profiles: ProfileInput[]) -> ProfileVerdict[] {\n client Nvidia\n prompt #\"\n {{ _.role(\"system\") }}\n You curate sources for a developer-focused AI news platform. For each Substack\n publication, decide whether artificial intelligence is its PRIMARY subject:\n AI/ML/LLMs, foundation models, AI engineering, AI products, or AI research.\n\n Mark isAiRelated = true ONLY when AI is clearly the main topic. Mark false for\n general tech, productivity, marketing, business, crypto, politics, lifestyle, or\n publications that merely mention AI in passing. When the signal is thin (sparse\n description, generic name), lean false and assign low confidence - the crawler\n prunes everything you reject, so false positives pollute the whole subtree.\n\n {{ _.role(\"user\") }}\n Classify every profile. Echo each handle back EXACTLY as given.\n\n {% for p in profiles %}\n {{ loop.index }}. handle={{ p.handle }} | {{ p.name }}{% if p.description %} — {{ p.description }}{% endif %}\n {% endfor %}\n\n {{ ctx.output_format }}\n \"#\n}\n", - "extract_items.baml": "// Newsletter extraction.\n//\n// A newsletter reports two kinds of thing worth ingesting, and they need\n// different URL resolution:\n//\n// - Product: a named launch (model, tool, SDK). Its canonical URL is the\n// vendor's own page, which the email rarely links to directly, so the URL is\n// rediscovered by web search (SelectProductLink) rather than trusted from the\n// email. This replaced the per-newsletter tracking-URL decoders.\n// - Article: a blog post, essay, paper, or write-up. Here the link IS the\n// subject — there is nothing to search for, and a search would land on some\n// other page about the same topic. Its href is kept and followed instead\n// (see ``_resolve_article`` in pipeline/sources/email.py), which turns a\n// tracking redirect into the real destination.\n//\n// Hence NewsletterItem.url: the model must copy an article's href verbatim\n// even though it is usually a redirect, because following it is exactly how\n// the real URL is recovered.\n\nenum NewsletterItemKind {\n Product // A named product, tool, model, SDK, dataset, or launch\n Article // A blog post, essay, paper, guide, or write-up\n}\n\nclass NewsletterItem {\n kind NewsletterItemKind\n name string @description(\"Product name, or the article's title\")\n description string @description(\"Concise phrase: what it is and does, usable verbatim as a web search query\")\n url string? @description(\"For an Article, the href of its link, copied verbatim including any tracking redirect. Omit for a Product.\")\n}\n\nfunction ExtractItems(newsletterText: string) -> NewsletterItem[] {\n client Nvidia\n prompt #\"\n {{ _.role(\"system\") }}\n You extract the AI/tech items a newsletter is reporting on: named products, and the articles it links to. Work only from the text; never invent items that are not mentioned.\n\n {{ _.role(\"user\") }}\n Below is the text of a newsletter email. Links appear as [anchor text](url).\n\n Identify every distinct item the newsletter reports as news, and classify each:\n\n - Product — a named product, tool, model, SDK, dataset, or launch a reader could go use. Judge it by the words, not the link: its URL is looked up separately, so do NOT return a url for a Product.\n - Article — a blog post, essay, paper, guide, benchmark write-up, or analysis the newsletter is pointing at. For these the link is the subject, so ALWAYS return its url, copied verbatim from the text — including tracking redirects, which are followed later to recover the real destination. An item with no link is not an Article.\n\n For each item return:\n - kind: Product or Article\n - name: the exact product name, or the article's title\n - description: a concise phrase describing what it is and does, suitable as a web search query\n - url: the article's href (Article only)\n\n A single story often yields one item, not two: if a link is the vendor's own announcement of a named launch, return it as a Product. Return an Article when the piece itself is the point — someone's write-up, teardown, guide, or opinion.\n\n INCLUDE only real, named launches and substantive articles a reader could go use or read.\n\n IGNORE:\n - the author's intro letter and personal commentary\n - sponsor blurbs, advertisements, \"brought to you by\", upgrade / subscribe / manage-preferences prompts\n - social-media embeds (an @handle with like / repost / reply counts) and the newsletter's own social links\n - section headers, navigation, and unsubscribe links\n - job postings\n\n Newsletter text:\n {{ newsletterText }}\n\n If the text names no qualifying items, return an empty list.\n\n {{ ctx.output_format }}\n \"#\n}\n\n// Notability gate: trims the extracted list to the items actually worth\n// resolving, so we don't spend a web search on every incremental SDK bump.\n// Returns the 1-based indices of the items to KEEP.\nfunction SelectNotableItems(items: NewsletterItem[]) -> int[] {\n client Nvidia\n prompt #\"\n {{ _.role(\"user\") }}\n Below is a numbered list of items a newsletter mentioned. Return the numbers of the ones that are NOTABLE enough to feature in an AI-news aggregator.\n\n KEEP an item if it is distinct, real, and individually newsworthy: new models, significant new products or platforms, notable agents/tools, major releases, and articles with real substance — technical write-ups, benchmarks, teardowns, engineering posts, or arguments a developer would want to read.\n\n DROP:\n - minor or incremental items: small SDK/utility bumps, niche helper libraries, design skills, config tweaks\n - vague or marketing-only entries with no concrete product behind them\n - listicles, SEO filler, press-release rewrites, and thin news blurbs that only restate an announcement\n - duplicates or sub-features of another item in the list\n\n Items:\n {% for item in items %}\n {{ loop.index }}. [{{ item.kind }}] {{ item.name }} — {{ item.description }}\n {% endfor %}\n\n Return only the numbers to keep, as a JSON array of integers. If none qualify, return [].\n\n {{ ctx.output_format }}\n \"#\n}\n\nclass SearchCandidate {\n title string\n url string\n snippet string\n}\n\nclass ProductLinkChoice {\n index int @description(\"1-based index of the best first-party result, or 0 if none qualifies\")\n confidence float @description(\"0.0 - 1.0\")\n}\n\n// Picks the best canonical URL for a product among search candidates, biased\n// hard toward the official / first-party source and away from social posts and\n// SEO/aggregator pages. Returns index 0 when nothing is clearly the product.\n// Products only — an article already carries its own link.\nfunction SelectProductLink(\n productName: string,\n productDescription: string,\n candidates: SearchCandidate[]\n) -> ProductLinkChoice {\n client Nvidia\n prompt #\"\n {{ _.role(\"user\") }}\n A newsletter featured this item:\n name: {{ productName }}\n description: {{ productDescription }}\n\n Web search returned these candidate results:\n {% for c in candidates %}\n {{ loop.index }}. {{ c.title }}\n url: {{ c.url }}\n snippet: {{ c.snippet }}\n {% endfor %}\n\n Pick the ONE candidate that is the canonical, first-party source for this exact product — its official site, product page, model card, repo, or the vendor's own announcement.\n\n PREFER the official / first-party domain that matches the product or its maker.\n REJECT (do not pick) when it is the best available:\n - social-media posts (x.com, twitter.com, linkedin.com, reddit.com, threads, mastodon)\n - videos (youtube.com)\n - content-farm / SEO / generic aggregator or listicle pages that merely mention the product\n - a DIFFERENT product that happens to share the name\n\n Return the 1-based index of the best first-party result. If none of the candidates is clearly a first-party source for THIS product, return index 0.\n\n {{ ctx.output_format }}\n \"#\n}\n\ntest extract_items_basic {\n functions [ExtractItems]\n args {\n newsletterText \"Headlines: [OpenAI ships GPT-5](https://sub.stack/redirect/2/abc), a new flagship model. Deep dive: [Why agent harnesses fail](https://stencil.so/blog/harness-playbook?utm_source=tldrai) walks through production agent design. Brought to you by Acme. [Unsubscribe](https://newsletter.com/unsub).\"\n }\n}\n", + "extract_items.baml": "// Newsletter extraction.\n//\n// A newsletter reports two kinds of thing worth ingesting, and they need\n// different URL resolution:\n//\n// - Product: a named launch (model, tool, SDK). Its canonical URL is the\n// vendor's own page, which the email rarely links to directly, so the URL is\n// rediscovered by web search (SelectProductLink) rather than trusted from the\n// email. This replaced the per-newsletter tracking-URL decoders.\n// - Article: a blog post, essay, paper, or write-up. Here the link IS the\n// subject — there is nothing to search for, and a search would land on some\n// other page about the same topic. Its href is kept and followed instead\n// (see ``_resolve_article`` in app/features/collect/newsletters.py), which turns a\n// tracking redirect into the real destination.\n//\n// Hence NewsletterItem.url: the model must copy an article's href verbatim\n// even though it is usually a redirect, because following it is exactly how\n// the real URL is recovered.\n\nenum NewsletterItemKind {\n Product // A named product, tool, model, SDK, dataset, or launch\n Article // A blog post, essay, paper, guide, or write-up\n}\n\nclass NewsletterItem {\n kind NewsletterItemKind\n name string @description(\"Product name, or the article's title\")\n description string @description(\"Concise phrase: what it is and does, usable verbatim as a web search query\")\n url string? @description(\"For an Article, the href of its link, copied verbatim including any tracking redirect. Omit for a Product.\")\n}\n\nfunction ExtractItems(newsletterText: string) -> NewsletterItem[] {\n client Nvidia\n prompt #\"\n {{ _.role(\"system\") }}\n You extract the AI/tech items a newsletter is reporting on: named products, and the articles it links to. Work only from the text; never invent items that are not mentioned.\n\n {{ _.role(\"user\") }}\n Below is the text of a newsletter email. Links appear as [anchor text](url).\n\n Identify every distinct item the newsletter reports as news, and classify each:\n\n - Product — a named product, tool, model, SDK, dataset, or launch a reader could go use. Judge it by the words, not the link: its URL is looked up separately, so do NOT return a url for a Product.\n - Article — a blog post, essay, paper, guide, benchmark write-up, or analysis the newsletter is pointing at. For these the link is the subject, so ALWAYS return its url, copied verbatim from the text — including tracking redirects, which are followed later to recover the real destination. An item with no link is not an Article.\n\n For each item return:\n - kind: Product or Article\n - name: the exact product name, or the article's title\n - description: a concise phrase describing what it is and does, suitable as a web search query\n - url: the article's href (Article only)\n\n A single story often yields one item, not two: if a link is the vendor's own announcement of a named launch, return it as a Product. Return an Article when the piece itself is the point — someone's write-up, teardown, guide, or opinion.\n\n INCLUDE only real, named launches and substantive articles a reader could go use or read.\n\n IGNORE:\n - the author's intro letter and personal commentary\n - sponsor blurbs, advertisements, \"brought to you by\", upgrade / subscribe / manage-preferences prompts\n - social-media embeds (an @handle with like / repost / reply counts) and the newsletter's own social links\n - section headers, navigation, and unsubscribe links\n - job postings\n\n Newsletter text:\n {{ newsletterText }}\n\n If the text names no qualifying items, return an empty list.\n\n {{ ctx.output_format }}\n \"#\n}\n\n// Notability gate: trims the extracted list to the items actually worth\n// resolving, so we don't spend a web search on every incremental SDK bump.\n// Returns the 1-based indices of the items to KEEP.\nfunction SelectNotableItems(items: NewsletterItem[]) -> int[] {\n client Nvidia\n prompt #\"\n {{ _.role(\"user\") }}\n Below is a numbered list of items a newsletter mentioned. Return the numbers of the ones that are NOTABLE enough to feature in an AI-news aggregator.\n\n KEEP an item if it is distinct, real, and individually newsworthy: new models, significant new products or platforms, notable agents/tools, major releases, and articles with real substance — technical write-ups, benchmarks, teardowns, engineering posts, or arguments a developer would want to read.\n\n DROP:\n - minor or incremental items: small SDK/utility bumps, niche helper libraries, design skills, config tweaks\n - vague or marketing-only entries with no concrete product behind them\n - listicles, SEO filler, press-release rewrites, and thin news blurbs that only restate an announcement\n - duplicates or sub-features of another item in the list\n\n Items:\n {% for item in items %}\n {{ loop.index }}. [{{ item.kind }}] {{ item.name }} — {{ item.description }}\n {% endfor %}\n\n Return only the numbers to keep, as a JSON array of integers. If none qualify, return [].\n\n {{ ctx.output_format }}\n \"#\n}\n\nclass SearchCandidate {\n title string\n url string\n snippet string\n}\n\nclass ProductLinkChoice {\n index int @description(\"1-based index of the best first-party result, or 0 if none qualifies\")\n confidence float @description(\"0.0 - 1.0\")\n}\n\n// Picks the best canonical URL for a product among search candidates, biased\n// hard toward the official / first-party source and away from social posts and\n// SEO/aggregator pages. Returns index 0 when nothing is clearly the product.\n// Products only — an article already carries its own link.\nfunction SelectProductLink(\n productName: string,\n productDescription: string,\n candidates: SearchCandidate[]\n) -> ProductLinkChoice {\n client Nvidia\n prompt #\"\n {{ _.role(\"user\") }}\n A newsletter featured this item:\n name: {{ productName }}\n description: {{ productDescription }}\n\n Web search returned these candidate results:\n {% for c in candidates %}\n {{ loop.index }}. {{ c.title }}\n url: {{ c.url }}\n snippet: {{ c.snippet }}\n {% endfor %}\n\n Pick the ONE candidate that is the canonical, first-party source for this exact product — its official site, product page, model card, repo, or the vendor's own announcement.\n\n PREFER the official / first-party domain that matches the product or its maker.\n REJECT (do not pick) when it is the best available:\n - social-media posts (x.com, twitter.com, linkedin.com, reddit.com, threads, mastodon)\n - videos (youtube.com)\n - content-farm / SEO / generic aggregator or listicle pages that merely mention the product\n - a DIFFERENT product that happens to share the name\n\n Return the 1-based index of the best first-party result. If none of the candidates is clearly a first-party source for THIS product, return index 0.\n\n {{ ctx.output_format }}\n \"#\n}\n\ntest extract_items_basic {\n functions [ExtractItems]\n args {\n newsletterText \"Headlines: [OpenAI ships GPT-5](https://sub.stack/redirect/2/abc), a new flagship model. Deep dive: [Why agent harnesses fail](https://stencil.so/blog/harness-playbook?utm_source=tldrai) walks through production agent design. Brought to you by Acme. [Unsubscribe](https://newsletter.com/unsub).\"\n }\n}\n", "generators.baml": "generator python {\n output_type \"python/pydantic\"\n output_dir \"../\"\n version \"0.223.0\"\n}\n", } diff --git a/backend/baml_src/extract_items.baml b/backend/baml_src/extract_items.baml index 0b55e5d9..8af704f2 100644 --- a/backend/baml_src/extract_items.baml +++ b/backend/baml_src/extract_items.baml @@ -10,7 +10,7 @@ // - Article: a blog post, essay, paper, or write-up. Here the link IS the // subject — there is nothing to search for, and a search would land on some // other page about the same topic. Its href is kept and followed instead -// (see ``_resolve_article`` in pipeline/sources/email.py), which turns a +// (see ``_resolve_article`` in app/features/collect/newsletters.py), which turns a // tracking redirect into the real destination. // // Hence NewsletterItem.url: the model must copy an article's href verbatim diff --git a/backend/conftest.py b/backend/conftest.py index 0c476fef..ce735dc1 100644 --- a/backend/conftest.py +++ b/backend/conftest.py @@ -1,4 +1,5 @@ -"""Shared pytest fixtures: test client, db session, and auth token headers.""" +"""Shared pytest fixtures: test clients, db session, auth token headers, and the +MCP write scope.""" from __future__ import annotations @@ -6,17 +7,26 @@ import pytest from fastapi.testclient import TestClient -from sqlmodel import Session, delete +from fastmcp.server.auth import AccessToken +from sqlmodel import Session, col, delete, func, select -from app.analytics.models import AnalyticsEvent -from app.audience.models import ArticleLike, User -from app.audience.service import init_db -from app.audience.tests.factories import ( +from app.core.db.article import Article +from app.core.db.engine import engine +from app.core.db.event import AnalyticsEvent +from app.core.db.factories import ( authentication_token_from_email, get_superuser_token_headers, + random_email, + random_lower_string, ) -from app.core.db import engine +from app.core.db.ledger import Reject, RejectStage +from app.core.db.like import ArticleLike +from app.core.db.publisher import Publisher, PublisherKind +from app.core.db.story import Story, StoryArticle +from app.core.db.user import User from app.core.settings import settings +from app.entry.mcp import auth +from app.features.audience.service import init_db from app.main import app @@ -48,3 +58,65 @@ def superuser_token_headers(db: Session) -> dict[str, str]: @pytest.fixture(scope="module") def normal_user_token_headers(db: Session) -> dict[str, str]: return authentication_token_from_email(email=settings.EMAIL_TEST_USER, db=db) + + +@pytest.fixture(scope="module") +def auth_client(db: Session) -> Generator[TestClient]: + """A TestClient that sends a logged-in reader's bearer token on every request. + + Reads are public; the token only decides whether `liked_by_me` is filled in. + """ + headers = authentication_token_from_email(email=random_email(), db=db) + with TestClient(app) as c: + c.headers.update(headers) + yield c + + +@pytest.fixture +def curator(monkeypatch: pytest.MonkeyPatch) -> None: + """Answer the MCP tools' scope check as the curation agent's write token.""" + monkeypatch.setattr( + auth, + "get_access_token", + lambda: AccessToken(token="t", client_id="c", scopes=[auth.WRITE_SCOPE]), + ) + + +@pytest.fixture +def made(db: Session) -> Generator[dict[str, list[int]]]: + """Ids of what a test created, deleted afterwards with every story made + meanwhile, so the next test starts from the same rows.""" + created: dict[str, list[int]] = {"articles": [], "publishers": []} + before = db.exec(select(func.max(Story.id))).one() or 0 + yield created + db.rollback() + ours = select(Story.id).where(col(Story.id) > before) + db.exec(delete(StoryArticle).where(col(StoryArticle.story_id).in_(ours))) + db.exec(delete(Story).where(col(Story.id) > before)) + db.exec(delete(Article).where(col(Article.id).in_(created["articles"]))) + db.exec(delete(Publisher).where(col(Publisher.id).in_(created["publishers"]))) + db.commit() + + +@pytest.fixture +def candidate(db: Session) -> Generator[Reject]: + """One pending candidate from a publisher the run has just created.""" + name = random_lower_string() + publisher = Publisher(name=f"{name}.dev", slug=name, kind=PublisherKind.unknown) + db.add(publisher) + db.commit() + row = Reject( + url=f"https://{name}.dev/post", + stage=RejectStage.pending, + title="An agent harness, measured", + source="Hacker News", + publisher_id=publisher.id, + content="x" * 5000, + ) + db.add(row) + db.commit() + yield row + db.rollback() + db.exec(delete(Reject).where(col(Reject.url) == row.url)) + db.exec(delete(Publisher).where(col(Publisher.id) == publisher.id)) + db.commit() diff --git a/backend/migrations/env.py b/backend/migrations/env.py new file mode 100755 index 00000000..519e2b87 --- /dev/null +++ b/backend/migrations/env.py @@ -0,0 +1,44 @@ +"""Alembic's entry point: run the migrations against the app's database.""" + +from __future__ import annotations + +import logging + +from alembic import context +from sqlalchemy import create_engine, pool +from sqlmodel import SQLModel + +import app.core.db # noqa: F401 # declares every table on SQLModel.metadata +from app.core.settings import settings + +# pyproject.toml's [tool.alembic] has no logging section, so it is set here. +logging.basicConfig(format="%(levelname)-5.5s [%(name)s] %(message)s") +logging.getLogger("alembic").setLevel(logging.INFO) + +target_metadata = SQLModel.metadata +url = str(settings.SQLALCHEMY_DATABASE_URI) + + +def run_migrations_offline() -> None: + """Emit the SQL to stdout instead of running it.""" + context.configure( + url=url, target_metadata=target_metadata, literal_binds=True, compare_type=True + ) + with context.begin_transaction(): + context.run_migrations() + + +def run_migrations_online() -> None: + connectable = create_engine(url, poolclass=pool.NullPool) + with connectable.connect() as connection: + context.configure( + connection=connection, target_metadata=target_metadata, compare_type=True + ) + with context.begin_transaction(): + context.run_migrations() + + +if context.is_offline_mode(): + run_migrations_offline() +else: + run_migrations_online() diff --git a/backend/app/alembic/script.py.mako b/backend/migrations/script.py.mako similarity index 100% rename from backend/app/alembic/script.py.mako rename to backend/migrations/script.py.mako diff --git a/backend/app/alembic/versions/.keep b/backend/migrations/versions/.keep similarity index 100% rename from backend/app/alembic/versions/.keep rename to backend/migrations/versions/.keep diff --git a/backend/app/alembic/versions/1a31ce608336_add_cascade_delete_relationships.py b/backend/migrations/versions/1a31ce608336_add_cascade_delete_relationships.py similarity index 100% rename from backend/app/alembic/versions/1a31ce608336_add_cascade_delete_relationships.py rename to backend/migrations/versions/1a31ce608336_add_cascade_delete_relationships.py diff --git a/backend/app/alembic/versions/3b3ed3a25c85_drop_item_table.py b/backend/migrations/versions/3b3ed3a25c85_drop_item_table.py similarity index 100% rename from backend/app/alembic/versions/3b3ed3a25c85_drop_item_table.py rename to backend/migrations/versions/3b3ed3a25c85_drop_item_table.py diff --git a/backend/app/alembic/versions/9c0a54914c78_add_max_length_for_string_varchar_.py b/backend/migrations/versions/9c0a54914c78_add_max_length_for_string_varchar_.py similarity index 100% rename from backend/app/alembic/versions/9c0a54914c78_add_max_length_for_string_varchar_.py rename to backend/migrations/versions/9c0a54914c78_add_max_length_for_string_varchar_.py diff --git a/backend/app/alembic/versions/a0b1c2d3e4f5_normalize_substack_links_to_rss.py b/backend/migrations/versions/a0b1c2d3e4f5_normalize_substack_links_to_rss.py similarity index 100% rename from backend/app/alembic/versions/a0b1c2d3e4f5_normalize_substack_links_to_rss.py rename to backend/migrations/versions/a0b1c2d3e4f5_normalize_substack_links_to_rss.py diff --git a/backend/app/alembic/versions/a1b2c3d4e5f6_add_articles_table.py b/backend/migrations/versions/a1b2c3d4e5f6_add_articles_table.py similarity index 100% rename from backend/app/alembic/versions/a1b2c3d4e5f6_add_articles_table.py rename to backend/migrations/versions/a1b2c3d4e5f6_add_articles_table.py diff --git a/backend/app/alembic/versions/a3b4c5d6e7f8_passwordless_user.py b/backend/migrations/versions/a3b4c5d6e7f8_passwordless_user.py similarity index 100% rename from backend/app/alembic/versions/a3b4c5d6e7f8_passwordless_user.py rename to backend/migrations/versions/a3b4c5d6e7f8_passwordless_user.py diff --git a/backend/app/alembic/versions/a5b6c7d8e9f0_add_reject_evidence_to_scored_url.py b/backend/migrations/versions/a5b6c7d8e9f0_add_reject_evidence_to_scored_url.py similarity index 100% rename from backend/app/alembic/versions/a5b6c7d8e9f0_add_reject_evidence_to_scored_url.py rename to backend/migrations/versions/a5b6c7d8e9f0_add_reject_evidence_to_scored_url.py diff --git a/backend/app/alembic/versions/a7b8c9d0e1f2_add_analytics_event_table.py b/backend/migrations/versions/a7b8c9d0e1f2_add_analytics_event_table.py similarity index 100% rename from backend/app/alembic/versions/a7b8c9d0e1f2_add_analytics_event_table.py rename to backend/migrations/versions/a7b8c9d0e1f2_add_analytics_event_table.py diff --git a/backend/app/alembic/versions/b1c2d3e4f5a6_drop_pro_until_from_user.py b/backend/migrations/versions/b1c2d3e4f5a6_drop_pro_until_from_user.py similarity index 100% rename from backend/app/alembic/versions/b1c2d3e4f5a6_drop_pro_until_from_user.py rename to backend/migrations/versions/b1c2d3e4f5a6_drop_pro_until_from_user.py diff --git a/backend/app/alembic/versions/b2c3d4e5f6a7_add_publisher_model.py b/backend/migrations/versions/b2c3d4e5f6a7_add_publisher_model.py similarity index 100% rename from backend/app/alembic/versions/b2c3d4e5f6a7_add_publisher_model.py rename to backend/migrations/versions/b2c3d4e5f6a7_add_publisher_model.py diff --git a/backend/app/alembic/versions/b4c5d6e7f8a9_drop_analytics_query_strings.py b/backend/migrations/versions/b4c5d6e7f8a9_drop_analytics_query_strings.py similarity index 100% rename from backend/app/alembic/versions/b4c5d6e7f8a9_drop_analytics_query_strings.py rename to backend/migrations/versions/b4c5d6e7f8a9_drop_analytics_query_strings.py diff --git a/backend/app/alembic/versions/b6c7d8e9f0a1_add_score_reason_to_article.py b/backend/migrations/versions/b6c7d8e9f0a1_add_score_reason_to_article.py similarity index 100% rename from backend/app/alembic/versions/b6c7d8e9f0a1_add_score_reason_to_article.py rename to backend/migrations/versions/b6c7d8e9f0a1_add_score_reason_to_article.py diff --git a/backend/app/alembic/versions/b7c8d9e0f1a2_add_scored_url_table.py b/backend/migrations/versions/b7c8d9e0f1a2_add_scored_url_table.py similarity index 100% rename from backend/app/alembic/versions/b7c8d9e0f1a2_add_scored_url_table.py rename to backend/migrations/versions/b7c8d9e0f1a2_add_scored_url_table.py diff --git a/backend/app/alembic/versions/c1d2e3f4a5b6_add_publishers_to_pipeline_run.py b/backend/migrations/versions/c1d2e3f4a5b6_add_publishers_to_pipeline_run.py similarity index 100% rename from backend/app/alembic/versions/c1d2e3f4a5b6_add_publishers_to_pipeline_run.py rename to backend/migrations/versions/c1d2e3f4a5b6_add_publishers_to_pipeline_run.py diff --git a/backend/app/alembic/versions/c5d6e7f8a9b0_expiring_login_token.py b/backend/migrations/versions/c5d6e7f8a9b0_expiring_login_token.py similarity index 100% rename from backend/app/alembic/versions/c5d6e7f8a9b0_expiring_login_token.py rename to backend/migrations/versions/c5d6e7f8a9b0_expiring_login_token.py diff --git a/backend/app/alembic/versions/c7d8e9f0a1b2_add_rescore_marks_to_article.py b/backend/migrations/versions/c7d8e9f0a1b2_add_rescore_marks_to_article.py similarity index 100% rename from backend/app/alembic/versions/c7d8e9f0a1b2_add_rescore_marks_to_article.py rename to backend/migrations/versions/c7d8e9f0a1b2_add_rescore_marks_to_article.py diff --git a/backend/app/alembic/versions/c8d9e0f1a2b3_add_newsletter_subscriber_table.py b/backend/migrations/versions/c8d9e0f1a2b3_add_newsletter_subscriber_table.py similarity index 100% rename from backend/app/alembic/versions/c8d9e0f1a2b3_add_newsletter_subscriber_table.py rename to backend/migrations/versions/c8d9e0f1a2b3_add_newsletter_subscriber_table.py diff --git a/backend/app/alembic/versions/d1e2f3a4b5c6_add_story_tables.py b/backend/migrations/versions/d1e2f3a4b5c6_add_story_tables.py similarity index 100% rename from backend/app/alembic/versions/d1e2f3a4b5c6_add_story_tables.py rename to backend/migrations/versions/d1e2f3a4b5c6_add_story_tables.py diff --git a/backend/app/alembic/versions/d4e5f6a7b8c9_add_article_like_table.py b/backend/migrations/versions/d4e5f6a7b8c9_add_article_like_table.py similarity index 100% rename from backend/app/alembic/versions/d4e5f6a7b8c9_add_article_like_table.py rename to backend/migrations/versions/d4e5f6a7b8c9_add_article_like_table.py diff --git a/backend/app/alembic/versions/d8e9f0a1b2c3_add_links_to_scored_url.py b/backend/migrations/versions/d8e9f0a1b2c3_add_links_to_scored_url.py similarity index 100% rename from backend/app/alembic/versions/d8e9f0a1b2c3_add_links_to_scored_url.py rename to backend/migrations/versions/d8e9f0a1b2c3_add_links_to_scored_url.py diff --git a/backend/app/alembic/versions/d98dd8ec85a3_edit_replace_id_integers_in_all_models_.py b/backend/migrations/versions/d98dd8ec85a3_edit_replace_id_integers_in_all_models_.py similarity index 100% rename from backend/app/alembic/versions/d98dd8ec85a3_edit_replace_id_integers_in_all_models_.py rename to backend/migrations/versions/d98dd8ec85a3_edit_replace_id_integers_in_all_models_.py diff --git a/backend/app/alembic/versions/e2412789c190_initialize_models.py b/backend/migrations/versions/e2412789c190_initialize_models.py similarity index 100% rename from backend/app/alembic/versions/e2412789c190_initialize_models.py rename to backend/migrations/versions/e2412789c190_initialize_models.py diff --git a/backend/app/alembic/versions/e3f4a5b6c7d8_add_publisher_topic_gated.py b/backend/migrations/versions/e3f4a5b6c7d8_add_publisher_topic_gated.py similarity index 100% rename from backend/app/alembic/versions/e3f4a5b6c7d8_add_publisher_topic_gated.py rename to backend/migrations/versions/e3f4a5b6c7d8_add_publisher_topic_gated.py diff --git a/backend/app/alembic/versions/e7f8a9b0c1d2_add_pro_until_to_user.py b/backend/migrations/versions/e7f8a9b0c1d2_add_pro_until_to_user.py similarity index 100% rename from backend/app/alembic/versions/e7f8a9b0c1d2_add_pro_until_to_user.py rename to backend/migrations/versions/e7f8a9b0c1d2_add_pro_until_to_user.py diff --git a/backend/app/alembic/versions/e9f0a1b2c3d4_add_fetch_health_to_publisher.py b/backend/migrations/versions/e9f0a1b2c3d4_add_fetch_health_to_publisher.py similarity index 100% rename from backend/app/alembic/versions/e9f0a1b2c3d4_add_fetch_health_to_publisher.py rename to backend/migrations/versions/e9f0a1b2c3d4_add_fetch_health_to_publisher.py diff --git a/backend/app/alembic/versions/f1a2b3c4d5e6_add_pipeline_run_table.py b/backend/migrations/versions/f1a2b3c4d5e6_add_pipeline_run_table.py similarity index 100% rename from backend/app/alembic/versions/f1a2b3c4d5e6_add_pipeline_run_table.py rename to backend/migrations/versions/f1a2b3c4d5e6_add_pipeline_run_table.py diff --git a/backend/app/alembic/versions/f4a5b6c7d8e9_add_publisher_type_reddit.py b/backend/migrations/versions/f4a5b6c7d8e9_add_publisher_type_reddit.py similarity index 100% rename from backend/app/alembic/versions/f4a5b6c7d8e9_add_publisher_type_reddit.py rename to backend/migrations/versions/f4a5b6c7d8e9_add_publisher_type_reddit.py diff --git a/backend/app/alembic/versions/f9a0b1c2d3e4_add_publisher_type.py b/backend/migrations/versions/f9a0b1c2d3e4_add_publisher_type.py similarity index 100% rename from backend/app/alembic/versions/f9a0b1c2d3e4_add_publisher_type.py rename to backend/migrations/versions/f9a0b1c2d3e4_add_publisher_type.py diff --git a/backend/app/alembic/versions/fa1b2c3d4e5f_publisher_origin.py b/backend/migrations/versions/fa1b2c3d4e5f_publisher_origin.py similarity index 100% rename from backend/app/alembic/versions/fa1b2c3d4e5f_publisher_origin.py rename to backend/migrations/versions/fa1b2c3d4e5f_publisher_origin.py diff --git a/backend/app/alembic/versions/fe56fa70289e_add_created_at_to_user_and_item.py b/backend/migrations/versions/fe56fa70289e_add_created_at_to_user_and_item.py similarity index 100% rename from backend/app/alembic/versions/fe56fa70289e_add_created_at_to_user_and_item.py rename to backend/migrations/versions/fe56fa70289e_add_created_at_to_user_and_item.py diff --git a/backend/pipeline/db.py b/backend/pipeline/db.py deleted file mode 100644 index ed2bb32f..00000000 --- a/backend/pipeline/db.py +++ /dev/null @@ -1,33 +0,0 @@ -"""Database engine for the pipeline. - -The engine is built lazily so importing pipeline modules — in tests, or to -poke at a pure function — does not require a database. -""" - -from __future__ import annotations - -import os -from urllib.parse import quote - -from sqlalchemy import Engine -from sqlmodel import create_engine - -_engine: Engine | None = None - - -def postgres_url() -> str: - """Read straight from os.environ rather than `app.core.settings`: the - pipeline stays decoupled from the backend's full Settings object.""" - server = os.environ["POSTGRES_SERVER"] - port = os.environ.get("POSTGRES_PORT", "5432") - user = quote(os.environ["POSTGRES_USER"], safe="") - password = quote(os.environ.get("POSTGRES_PASSWORD", ""), safe="") - db = os.environ.get("POSTGRES_DB", "") - return f"postgresql+psycopg://{user}:{password}@{server}:{port}/{db}" - - -def get_engine() -> Engine: - global _engine - if _engine is None: - _engine = create_engine(postgres_url()) - return _engine diff --git a/backend/pipeline/fetching/__init__.py b/backend/pipeline/fetching/__init__.py deleted file mode 100644 index 3a1fe2b9..00000000 --- a/backend/pipeline/fetching/__init__.py +++ /dev/null @@ -1,7 +0,0 @@ -"""Lookups that are not a source: the curation agent's ``check_link``, and -unwrapping tweet links. - -HTTP, search and text extraction live in ``app.core``. -""" - -from __future__ import annotations diff --git a/backend/pipeline/models.py b/backend/pipeline/models.py deleted file mode 100644 index ae75a2aa..00000000 --- a/backend/pipeline/models.py +++ /dev/null @@ -1,99 +0,0 @@ -"""Pipeline schema: the run record and the reject ledger the funnel writes.""" - -from __future__ import annotations - -from datetime import datetime -from enum import StrEnum - -from sqlalchemy import JSON, Column, DateTime, String -from sqlmodel import Field, SQLModel - -from app.core.dates import get_datetime_utc - - -class RejectStage(StrEnum): - """The step that turned a URL down, or ``pending`` for one still waiting. - - ``pending`` is the one stage that is not a rejection. The nightly run ends - by writing it, and the curation agent turns it into an Article or into - ``below_threshold``. It lives in this table because the row already holds - everything a judge needs -- title, source, publisher, content, traction -- - and because ``filter_known_urls`` reads the table, so a candidate waiting - for the agent is not fetched again the next night. - """ - - pending = "pending" - thin_repo = "thin_repo" - prefilter = "prefilter" - duplicate = "duplicate" - below_threshold = "below_threshold" - unusable_summary = "unusable_summary" - - -class Reject(SQLModel, table=True): - """A URL the funnel turned down, or one waiting on the curation agent. - - The name is the common case. A ``pending`` row is the exception: the same - columns, holding a candidate the agent has not read yet. - - Every column but ``url`` and ``created_at`` is nullable: rows written before - the ledger kept evidence carry only the URL. - """ - - # Still named scored_url: deploy runs additive migrations only, and a - # rename is not one. - __tablename__ = "scored_url" - url: str = Field(primary_key=True) - created_at: datetime = Field(default_factory=get_datetime_utc) - # A plain string, not a Postgres enum, so a new stage needs no migration. - stage: RejectStage | None = Field(default=None, sa_type=String) - title: str | None = None - source: str | None = None - publisher_id: int | None = Field(default=None, foreign_key="publisher.id") - published_at: datetime | None = Field( - default=None, - sa_type=DateTime(timezone=True), # type: ignore - ) - content: str | None = None - traction: str | None = None - score: int | None = None - # The judge's one-sentence account of the score; below_threshold only. - reason: str | None = None - # Stage-specific evidence: {"stars": 12}, {"keep_proba": 0.08}, - # {"dup_of": ""}. - # none_as_null: without it an absent detail is stored as JSON 'null', which - # `detail IS NULL` does not match. - detail: dict[str, object] | None = Field( - default=None, sa_column=Column(JSON(none_as_null=True), nullable=True) - ) - # The repo / model / paper / docs links the article body makes, - # {"repo": [...], "paper": [...]}; see ``app.core.extract``. - links: dict[str, list[str]] | None = Field( - default=None, sa_column=Column(JSON(none_as_null=True), nullable=True) - ) - - -class PipelineRun(SQLModel, table=True): - """One row per nightly pipeline run: did it finish, how long it took, and - what each source fetched and queued.""" - - __tablename__ = "pipeline_run" - id: int | None = Field(default=None, primary_key=True) - started_at: datetime = Field( - default_factory=get_datetime_utc, - sa_type=DateTime(timezone=True), # type: ignore - ) - finished_at: datetime | None = Field( - default=None, - sa_type=DateTime(timezone=True), # type: ignore - ) - duration_ms: int | None = None - ok: bool = Field(default=False) - # [{source, fetched, queued, error}]. Rows from before 2026-09-23 carry - # the older, longer funnel shape. - sources: list[dict] = Field( - default_factory=list, sa_column=Column(JSON, nullable=False) - ) - # The table also has a `publishers` column, per-publisher counts from before - # the Publisher row kept its own (last_fetched_at, last_new_at, last_error). - # No longer written; its default fills it. diff --git a/backend/pipeline/rejects.py b/backend/pipeline/rejects.py deleted file mode 100644 index fa11aabc..00000000 --- a/backend/pipeline/rejects.py +++ /dev/null @@ -1,64 +0,0 @@ -"""The reject ledger: every URL the funnel turned down, with what it saw and why. - -Two jobs. ``filter_known_urls`` reads it so a rejected URL is never fetched and -judged again. And it keeps the evidence - the text the judge saw, the links, the -score, the judge's reason - so a reject can be audited, re-judged under a new -rubric, or used as a training negative instead of vanishing. - -The embedding is deliberately not stored: it is a pure function of the stored -title and ``content[:SNIPPET_CAP]``, so it can be rebuilt offline in seconds. -""" - -from __future__ import annotations - -from sqlmodel import Session - -from app.core.dates import parse_date -from pipeline.models import Reject, RejectStage -from pipeline.types import Candidate - -# What a settled reject keeps: enough to re-score with a longer input or -# re-embed with a better model later; the full body would make the ledger most -# of the database. -CONTENT_CAP = 2000 -# What a pending candidate keeps, so the curation agent reads the article from -# the db instead of fetching the page again. Trimmed to `CONTENT_CAP` once the -# agent rejects it (`curation.reject`), so only the night's queue is ever this -# large. -CANDIDATE_CAP = 12000 - - -def _pg_text(text: str) -> str: - # Postgres text columns reject NUL bytes, and fetched pages occasionally - # carry one. The failed commit would take the whole source down with it. - return text.replace("\x00", "") - - -def record_reject( - session: Session, - a: Candidate, - stage: RejectStage, - *, - score: int | None = None, - reason: str | None = None, - detail: dict[str, object] | None = None, -) -> None: - """Stage a reject row. The caller commits, as it did for the bare URL.""" - cap = CANDIDATE_CAP if stage == RejectStage.pending else CONTENT_CAP - content = _pg_text(a["content"][:cap]) - session.merge( - Reject( - url=a["url"], - stage=stage, - title=_pg_text(a["title"]), - source=a["source"], - publisher_id=a["publisher_id"], - published_at=parse_date(a["published_date"]), - content=content or None, - traction=a.get("traction"), - links=a.get("links") or None, - score=score, - reason=reason, - detail=detail, - ) - ) diff --git a/backend/pipeline/steps/__init__.py b/backend/pipeline/steps/__init__.py deleted file mode 100644 index 49f290b3..00000000 --- a/backend/pipeline/steps/__init__.py +++ /dev/null @@ -1,20 +0,0 @@ -"""The pipeline's steps, in the order run.py applies them. - - fetch - poll each source, fill content, stamp each item's publisher - filter - drop URLs already judged or already waiting - queue - park each survivor as a pending candidate. The last step. - -The curation agent takes it from there (pipeline/curation.py): approving a -candidate runs persist and enrich (the embedding) on that one article, with -the summary, kind and tags the agent wrote -- see -docs/adr/0009-agent-curation.md. - -Every step takes the session and a list of articles and returns the survivors, -so run.py reads as the funnel it is. Steps own their own logging and commits. - -The article changes type as it goes -- RawItem -> Candidate -> Scored -> -Summarized -> Persisted, see pipeline.types -- so a step's signature says where -in the funnel it belongs. -""" - -from __future__ import annotations diff --git a/backend/pipeline/steps/enrich.py b/backend/pipeline/steps/enrich.py deleted file mode 100644 index 39652259..00000000 --- a/backend/pipeline/steps/enrich.py +++ /dev/null @@ -1,36 +0,0 @@ -"""The one piece of enrichment left after the curation agent writes the labels: -the embedding. - -Local and best-effort: a failed embedding costs the field, never the article. -""" - -from __future__ import annotations - -from sqlmodel import Session - -from app.catalog.models import Article -from app.core.embedding import embed_batch, to_embedding_text -from app.core.logging import log -from pipeline.types import ProcessedArticle - - -def embed_articles(session: Session, items: list[ProcessedArticle]) -> None: - if not items: - return - - log(f" Embedding {len(items)} articles...") - - texts = [to_embedding_text(item["title"], item.get("snippet")) for item in items] - try: - vecs = embed_batch(texts) - except Exception as e: - log(f" Embed batch failed, continuing: {e}") - return - - for item, vec in zip(items, vecs, strict=True): - article = session.get(Article, item["id"]) - if article: - article.embedding = vec - session.add(article) - - session.commit() diff --git a/backend/pipeline/types.py b/backend/pipeline/types.py deleted file mode 100644 index 1f1b6efd..00000000 --- a/backend/pipeline/types.py +++ /dev/null @@ -1,91 +0,0 @@ -"""The article as it changes shape on its way down the funnel. - -One type per stage, each fully populated by the step that produces it: - - RawItem what a source emits -- title, url, content, date, source - Candidate + the publisher it belongs to (fetch.resolve_publishers) - Scored + the curation agent's score (curation.approve) - Summarized + the summary a reader sees (curation.approve) - Persisted + the id of the row it landed in (persist.insert_articles) - -Each stage subclasses the one before, so a step that only needs a ``Candidate`` -also accepts a ``Scored`` or a ``Persisted``, while a step that needs a score -cannot be handed something that has not been judged. The signatures -are the funnel: reading them tells you the order without opening ``run.py``. - -This replaced a single ``FetchedArticle`` where every key was optional at every -stage. That shape could not say which step filled what, so each step's contract -lived in a docstring and a misordered call failed at runtime with a KeyError -instead of at the call site. - -``curation.approve`` reshapes a ``Persisted`` into the narrower -``ProcessedArticle`` that ``embed_articles`` consumes. -""" - -from __future__ import annotations - -from typing import NotRequired, TypedDict - - -class RawItem(TypedDict): - """One item as a source adapter emits it, before anything is known about it. - - ``content`` may be empty at this point: the thin sources (Hacker News, - Lab Watch, Newsletter) arrive with a blurb or nothing at all, and - ``fetch._with_content`` fills it before the item goes any further. - """ - - title: str - url: str - content: str - published_date: str | None - source: str - # Only the aggregator sources that carry a public reception signal (Hacker - # News points/comments) have one. Shown to the agent as evidence, not - # merely used as a gate -- see sources/hn.py. - traction: NotRequired[str] - # The repo / model / paper / docs links the article body makes, grouped by - # ``app.core.extract``. Absent when the text came from nowhere with HTML. - links: NotRequired[dict[str, list[str]]] - - -class Candidate(RawItem): - """An item resolved to the Publisher it came from.""" - - publisher_id: int - - -class Scored(Candidate): - """A candidate the curation agent approved, with the score it gave.""" - - score: int - # The agent's one-sentence account of the score. - score_reason: str | None - - -class Summarized(Scored): - """A scored article with the summary the agent wrote. Nothing is inserted - without one.""" - - summary: str - - -class Persisted(Summarized): - """A scored article that is now a row: ``id`` is the Article's primary key. - - ``title`` and ``content`` are the sanitized values actually stored, not the - raw ones the source emitted. - """ - - id: int - - -class ProcessedArticle(TypedDict): - """What enrichment hands the embedder: a stored article reduced to the - fields the embedding text is built from.""" - - id: int - url: str - title: str - score: int - snippet: str diff --git a/backend/pipeline/url_kind.py b/backend/pipeline/url_kind.py deleted file mode 100644 index 7495eedc..00000000 --- a/backend/pipeline/url_kind.py +++ /dev/null @@ -1,23 +0,0 @@ -"""ArticleKind straight from a URL's host, for the hosts where it is settled. - -A github.com link is a repo and an arxiv.org link is a paper whatever the LLM -would say about the text, so the host is asked first and the categorizer is -only consulted when it comes back inconclusive. -""" - -from __future__ import annotations - -from app.catalog.models import ArticleKind -from app.core.urls import hostname - - -def kind_from_url(url: str) -> ArticleKind | None: - """Deterministic ArticleKind from a URL host, or None if inconclusive.""" - host = hostname(url) - if host in ("github.com", "gitlab.com"): - return ArticleKind.repo - if host in ("huggingface.co", "hf.co"): - return ArticleKind.model - if host in ("arxiv.org", "ar5iv.labs.arxiv.org"): - return ArticleKind.paper - return None diff --git a/backend/pyproject.toml b/backend/pyproject.toml index 304402e2..125bc322 100644 --- a/backend/pyproject.toml +++ b/backend/pyproject.toml @@ -11,7 +11,7 @@ dependencies = [ "pydantic>2.0", "emails>=1.1.2,<2.0", "jinja2<4.0.0,>=3.1.4", - "alembic<2.0.0,>=1.12.1", + "alembic<2.0.0,>=1.18.0", "httpx<1.0.0,>=0.25.1", "psycopg[binary]<4.0.0,>=3.1.13", "sqlmodel>=0.0.39,<1.0.0", @@ -38,16 +38,20 @@ dev = [ "ruff<1.0.0,>=0.2.2", "coverage<8.0.0,>=7.4.3", "types-regex>=2026.9.29.20261017", + "import-linter>=2.15,<3.0.0", ] [build-system] requires = ["hatchling"] build-backend = "hatchling.build" +[tool.alembic] +script_location = "%(here)s/migrations" + [tool.mypy] strict = true disallow_any_explicit = true -exclude = ["venv", ".venv", "alembic"] +exclude = ["venv", ".venv", "migrations"] [tool.ruff] # Must match requires-python above. py314 lets the formatter emit PEP 758 @@ -55,7 +59,7 @@ exclude = ["venv", ".venv", "alembic"] # actually runs on - and the formatter rewrites silently. target-version = "py313" # baml_client is generated by `baml-cli generate` - never hand-edited. -exclude = ["alembic", "baml_client"] +exclude = ["migrations", "baml_client"] [tool.ruff.lint] select = [ @@ -87,29 +91,25 @@ keep-runtime-typing = true required-imports = ["from __future__ import annotations"] [tool.ruff.lint.per-file-ignores] -# The pipeline and ops scripts log to stdout via print() so systemd/journald -# captures it. -"pipeline/**" = ["T201"] -"scripts/**" = ["T201"] +# The collect run and the command-line entrypoints log to stdout via print() so +# systemd/journald captures it. +"app/features/collect/**" = ["T201"] +"app/entry/cli/**" = ["T201"] [tool.pytest.ini_options] -# Domain tests live inside the domain (ADR 4); `tests/` keeps what belongs to -# no single domain - the pipeline, the schema-wide enum guard, ops scripts. -testpaths = ["app", "pipeline", "tests"] +# Each test sits beside the module it tests (ADR 4). +testpaths = ["app"] [tool.coverage.run] -source = ["app", "pipeline"] +source = ["app"] # Colocated tests are inside `source`; measuring the tests themselves would # only inflate the number. -omit = ["*/tests/*"] +omit = ["*/test_*.py", "*/conftest.py", "*/factories.py"] dynamic_context = "test_function" [tool.coverage.report] show_missing = true sort = "-Cover" -# `scripts/` is deliberately outside `source`: those are ops entrypoints run by -# prestart.sh and by hand, not by pytest, so measuring them would only report a -# number nobody can act on. [tool.coverage.html] show_contexts = true @@ -119,3 +119,45 @@ error-on-warning = true [tool.fastapi] entrypoint = "app.main:app" + +[tool.importlinter] +root_package = "app" + +[[tool.importlinter.contracts]] +name = "main > entry > features > core" +type = "layers" +containers = ["app"] +layers = ["main", "entry", "features", "core"] +# A new top-level module under app/ has to join a layer. +exhaustive = true +exhaustive_ignores = ["test_main"] + +[[tool.importlinter.contracts]] +name = "Features never import each other" +type = "independence" +modules = ["app.features.*"] + +[[tool.importlinter.contracts]] +name = "api, mcp and cli never import each other" +type = "independence" +modules = ["app.entry.api", "app.entry.mcp", "app.entry.cli"] + +[[tool.importlinter.contracts]] +name = "Flat core modules never import core.db" +type = "forbidden" +# app.core.* matches app.core.db too; import-linter skips that overlap. +source_modules = ["app.core.*"] +forbidden_modules = ["app.core.db"] + +[[tool.importlinter.contracts]] +name = "Entry reaches a feature only through its service and model" +type = "forbidden" +source_modules = ["app.entry"] +forbidden_modules = ["app.features.*.*"] +allow_indirect_imports = true +ignore_imports = [ + "app.entry.** -> app.features.*.service", + "app.entry.** -> app.features.*.model", + # The search tests stub the query embedding where it is computed. + "app.entry.api.test_articles -> app.features.catalog.semantic_search", +] diff --git a/backend/scripts/__init__.py b/backend/scripts/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/backend/scripts/tests-start.sh b/backend/scripts/tests-start.sh deleted file mode 100755 index 2e643dc4..00000000 --- a/backend/scripts/tests-start.sh +++ /dev/null @@ -1,7 +0,0 @@ -#! /usr/bin/env bash -set -e -set -x - -python scripts/wait_for_db.py - -bash scripts/test.sh "$@" diff --git a/backend/tests/__init__.py b/backend/tests/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/backend/tests/core/__init__.py b/backend/tests/core/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/backend/tests/pipeline/__init__.py b/backend/tests/pipeline/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/backend/tests/pipeline/test_curation_db.py b/backend/tests/pipeline/test_curation_db.py deleted file mode 100644 index 32369929..00000000 --- a/backend/tests/pipeline/test_curation_db.py +++ /dev/null @@ -1,218 +0,0 @@ -"""The curation queries against the real schema, with the embedding stubbed. - -The pure parts are pinned in test_curation.py. This pins what those cannot: -that `similar` and `stories` read the right rows (the feed's stored vectors, -every ledger stage, the window) and that `list_candidates` counts a publisher's -record from the right ones. -""" - -from __future__ import annotations - -import hashlib -from collections.abc import Generator -from datetime import UTC, datetime, timedelta - -import numpy as np -import pytest -from sqlmodel import Session, col, delete, select - -from app.catalog.models import Article, Publisher, PublisherKind, PublisherType -from pipeline import curation -from pipeline.models import Reject, RejectStage -from tests.random_data import random_lower_string - -DIM = 256 -TAG = random_lower_string()[:8] - - -def _axis(i: int, wobble: float = 0.0) -> list[float]: - """A unit-ish vector along one of the high axes, far from anything real.""" - v = np.zeros(DIM, dtype=np.float32) - v[200 + i] = 1.0 - v[250] = wobble - return v.tolist() - - -def _noise(text: str) -> list[float]: - seed = int(hashlib.sha256(text.encode()).hexdigest()[:8], 16) - return np.random.default_rng(seed).standard_normal(DIM).tolist() - - -# Title -> vector. Everything else embeds to deterministic noise. -VECTORS = { - f"{TAG} GPT-6 Astra ships": _axis(0), - f"{TAG} OpenAI releases GPT-6 Astra": _axis(0, 0.05), - f"{TAG} Astra is here, a retelling": _axis(0, 0.10), - f"{TAG} An old Astra rumour": _axis(0, 0.02), - f"{TAG} A lone post about kernels": _axis(1), -} - - -def _fake_embed(texts: list[str]) -> list[list[float]]: - return [VECTORS.get(t.split("\n\n")[0], _noise(t)) for t in texts] - - -@pytest.fixture -def world(db: Session, monkeypatch: pytest.MonkeyPatch) -> Generator[dict]: - monkeypatch.setattr(curation, "embed_batch", _fake_embed) - - publishers = [ - Publisher( - name=f"{TAG} {name}", - slug=f"{TAG}-{name.lower()}", - kind=PublisherKind.media, - type=PublisherType.rss, - ) - for name in ("OpenAI", "Verge", "Wired", "Blog") - ] - db.add_all(publishers) - db.commit() - openai, verge, wired, blog = publishers - now = datetime.now(UTC) - - feed = Article( - title=f"{TAG} OpenAI releases GPT-6 Astra", - url=f"https://openai.example/{TAG}/astra", - publisher_id=openai.id or 0, - score=90, - embedding=_axis(0, 0.05), - ) - old = Article( - title=f"{TAG} An old Astra rumour", - url=f"https://old.example/{TAG}/rumour", - publisher_id=blog.id or 0, - score=60, - embedding=_axis(0, 0.02), - created_at=now - timedelta(days=30), - ) - pending = [ - Reject( - url=f"https://verge.example/{TAG}/astra", - stage=RejectStage.pending, - title=f"{TAG} GPT-6 Astra ships", - publisher_id=verge.id, - content="body", - ), - Reject( - url=f"https://blog.example/{TAG}/kernels", - stage=RejectStage.pending, - title=f"{TAG} A lone post about kernels", - publisher_id=blog.id, - content="body", - ), - ] - rejected = Reject( - url=f"https://wired.example/{TAG}/astra", - stage=RejectStage.below_threshold, - title=f"{TAG} Astra is here, a retelling", - publisher_id=wired.id, - content="body", - score=30, - ) - db.add_all([feed, old, *pending, rejected]) - db.commit() - - yield {"verge": pending[0].url, "lone": pending[1].url, "feed": feed.url} - - db.exec(delete(Reject).where(col(Reject.url).contains(TAG))) - db.exec(delete(Article).where(col(Article.url).contains(TAG))) - db.exec(delete(Publisher).where(col(Publisher.slug).startswith(TAG))) - db.commit() - - -def test_similar_lists_the_feed_and_every_ledger_stage(db: Session, world: dict): - result = curation.similar(db, world["verge"]) - - urls = [r["url"] for r in result["rows"]] - assert urls[:2] == [world["feed"], f"https://wired.example/{TAG}/astra"] - stages = {r["url"]: r["stage"] for r in result["rows"]} - assert stages[world["feed"]] == curation.PUBLISHED - assert stages[f"https://wired.example/{TAG}/astra"] == "below_threshold" - # Outside the window: the 30-day-old article is not a neighbour. - assert f"https://old.example/{TAG}/rumour" not in urls - # Never its own neighbour. - assert world["verge"] not in urls - - -def test_similar_counts_coverage_by_publisher(db: Session, world: dict): - result = curation.similar(db, world["verge"]) - assert result["publishers"] == [f"{TAG} OpenAI", f"{TAG} Verge", f"{TAG} Wired"] - assert result["coverage"] == 3 - - -def test_a_story_nobody_else_carries_has_coverage_one(db: Session, world: dict): - result = curation.similar(db, world["lone"]) - assert result["coverage"] == 1 - assert result["publishers"] == [f"{TAG} Blog"] - - -def test_similar_on_an_unknown_url_is_a_message(db: Session, world: dict): - with pytest.raises(curation.CandidateError, match="No candidate or article"): - curation.similar(db, f"https://nowhere.example/{TAG}") - assert world - - -def test_stories_groups_the_queue_with_the_feed(db: Session, world: dict): - groups = [ - g for g in curation.stories(db) if any(TAG in m["url"] for m in g["members"]) - ] - - [story] = groups - assert {m["url"] for m in story["members"]} == {world["verge"], world["feed"]} - assert story["coverage"] == 2 - - -def _all_candidates(db: Session, limit: int = curation.LIST_LIMIT) -> list[dict]: - """Every pending row, read page by page as the agent does.""" - page = curation.list_candidates(db, limit=limit) - rows = page["rows"] - while page["next_offset"] is not None: - page = curation.list_candidates(db, offset=page["next_offset"], limit=limit) - rows += page["rows"] - return rows - - -def test_list_candidates_shows_each_publishers_approval_rate(db: Session, world: dict): - rows = {r["url"]: r for r in _all_candidates(db)} - # Verge: nothing published, nothing turned down yet. - assert rows[world["verge"]]["approved"] == "new" - assert "trust" not in rows[world["verge"]] - # Blog's one article is 30 days old: outside `similar`'s week, inside the - # 90 days a record reaches. - assert rows[world["lone"]]["approved"] == "1/1" - - -def test_list_candidates_pages_through_the_whole_queue(db: Session, world: dict): - first = curation.list_candidates(db, limit=1) - assert first["offset"] == 0 - assert len(first["rows"]) == 1 - - urls = [r["url"] for r in _all_candidates(db, limit=1)] - - # Every pending row once, and only pending rows. - assert len(urls) == len(set(urls)) == first["total"] - assert {world["verge"], world["lone"]} <= set(urls) - assert f"https://wired.example/{TAG}/astra" not in urls - - -def test_list_candidates_past_the_end_is_an_empty_last_page(db: Session, world: dict): - page = curation.list_candidates(db, offset=10**6) - assert page["rows"] == [] - assert page["next_offset"] is None - assert page["offset"] == page["total"] - assert world - - -def test_approval_counts_reads_articles_and_agent_rejects(db: Session, world: dict): - ids = { - p.name: p.id - for p in db.exec(select(Publisher).where(col(Publisher.slug).startswith(TAG))) - } - since = datetime.now(UTC) - timedelta(days=90) - counts = curation.approval_counts(db, [i for i in ids.values() if i], since) - - assert counts[ids[f"{TAG} OpenAI"]] == (1, 0) - assert counts[ids[f"{TAG} Wired"]] == (0, 1) - # A pending row is not a decision. - assert ids[f"{TAG} Verge"] not in counts - assert world diff --git a/backend/tests/pipeline/test_email.py b/backend/tests/pipeline/test_email.py deleted file mode 100644 index c61cdecb..00000000 --- a/backend/tests/pipeline/test_email.py +++ /dev/null @@ -1,148 +0,0 @@ -"""The IMAP newsletter source's two decisions that are pure functions of text: -which publisher sent an email, and what URL an item actually points at. Wrong -in the first means a real newsletter is silently skipped or a stranger's mail -is attributed to a publisher; wrong in the second means the pipeline stores a -tracking link that dies with the campaign, or stores one post twice. -""" - -from __future__ import annotations - -import pytest - -from pipeline.sources.email import ( - _canonical_url, - _match_sender, - _unwrap_tracking_url, -) - -SOURCES = [ - ("@substack.com", "Some Substack"), - ("news@bensbites.com", "Ben's Bites"), -] - - -def test_at_prefix_matches_any_local_part_at_that_exact_domain(): - assert _match_sender("writer@substack.com", SOURCES) == "Some Substack" - - -def test_at_prefix_does_not_match_a_deeper_subdomain(): - """ "@substack.com" matches "anyone@substack.com" only — the check is a - string suffix on the whole address, so a subdomain like "foo.substack.com" - changes what precedes "substack.com" and no longer ends in "@substack.com".""" - assert _match_sender("writer@foo.substack.com", SOURCES) is None - - -@pytest.mark.parametrize( - "address", - ["writer@notsubstack.com", "writer@evilsubstack.com"], -) -def test_at_prefix_does_not_match_a_lookalike_domain(address): - """Confirms the check is a suffix match requiring the literal "@" right - before the domain, not a raw substring search for "substack.com".""" - assert _match_sender(address, SOURCES) is None - - -def test_exact_pattern_requires_an_exact_address_match(): - assert _match_sender("news@bensbites.com", SOURCES) == "Ben's Bites" - - -def test_exact_pattern_does_not_match_a_different_local_part_at_the_same_domain(): - assert _match_sender("other@bensbites.com", SOURCES) is None - - -def test_matching_is_case_insensitive(): - assert _match_sender("Writer@Substack.COM", SOURCES) == "Some Substack" - - -def test_no_match_returns_none(): - assert _match_sender("someone@example.com", SOURCES) is None - - -def test_empty_sources_never_matches(): - assert _match_sender("writer@substack.com", []) is None - - -# ─── tracker unwrapping ────────────────────────────────────────────────────── -# A newsletter links through its own redirector. The destination is what the -# pipeline stores, so recovering it from the wrapper — without a request, when -# the wrapper spells it out — is the whole job of this half of the source. - - -def test_unwraps_a_percent_encoded_destination(): - assert ( - _unwrap_tracking_url( - "https://tracking.tldrnewsletter.com/CL0/" - "https:%2F%2Fexample.com%2Fposts%2Fone/1/0100019a626e0311" - ) - == "https://example.com/posts/one" - ) - - -def test_unwraps_a_literal_destination_with_its_own_path_separators(): - assert ( - _unwrap_tracking_url( - "https://tracking.tldrnewsletter.com/CL0/" - "https://aws.amazon.com/build/ai-series/1/0100019a626e0311" - ) - == "https://aws.amazon.com/build/ai-series" - ) - - -def test_keeps_the_destination_query_while_dropping_the_tracker_tail(): - """The tail is the tracker's message id and click token, appended after the - destination; the destination's own query must survive it.""" - assert ( - _unwrap_tracking_url( - "https://tracking.tldrnewsletter.com/CL0/" - "https:%2F%2Fexample.com%2Fp%3Fid%3D7/1/0100019a626e0311" - ) - == "https://example.com/p?id=7" - ) - - -def test_an_opaque_token_cannot_be_unwrapped(): - """Nothing to recover without a request — the caller follows the redirect.""" - assert _unwrap_tracking_url("https://links.tldrnewsletter.com/zPfy8Y") is None - - -def test_a_direct_link_is_not_a_wrapper(): - assert _unwrap_tracking_url("https://example.com/blog/post") is None - - -# ─── URL canonicalization ──────────────────────────────────────────────────── -# Two links to one post must reduce to one string, or dedup stores it twice. - - -def test_drops_campaign_params_and_keeps_the_rest(): - assert ( - _canonical_url("https://example.com/p?id=7&utm_source=tldr&page=2") - == "https://example.com/p?id=7&page=2" - ) - - -def test_drops_campaign_params_hidden_after_a_fragment(): - """Newsletters append their params after the anchor, where a query parser - never looks.""" - assert ( - _canonical_url("https://example.com/p/#section?utm_source=tldrai") - == "https://example.com/p#section" - ) - - -def test_a_trailing_slash_does_not_make_a_second_url(): - assert _canonical_url("https://example.com/p/") == _canonical_url( - "https://example.com/p" - ) - - -def test_the_host_is_lowercased_but_the_path_is_not(): - """Hosts are case-insensitive; paths are not, and lowercasing one would - 404.""" - assert ( - _canonical_url("https://Example.COM/Posts/One") - == "https://example.com/Posts/One" - ) - - -def test_a_bare_root_url_keeps_its_slash(): - assert _canonical_url("https://example.com/") == "https://example.com/" diff --git a/backend/tests/pipeline/test_llm_text.py b/backend/tests/pipeline/test_llm_text.py deleted file mode 100644 index 8b582d35..00000000 --- a/backend/tests/pipeline/test_llm_text.py +++ /dev/null @@ -1,21 +0,0 @@ -"""Normalising an enum-typed BAML field to a plain string.""" - -from __future__ import annotations - -from enum import StrEnum - -from app.core.llm_text import enum_value - - -class _Colour(StrEnum): - red = "red" - - -def test_enum_value_unwraps_an_enum(): - assert enum_value(_Colour.red) == "red" - - -def test_enum_value_passes_a_plain_string_through(): - """BAML returns plain strings for some fields and enum members for others, - and which one has changed across regenerations.""" - assert enum_value("red") == "red" diff --git a/backend/tests/pipeline/test_newsletter_links.py b/backend/tests/pipeline/test_newsletter_links.py deleted file mode 100644 index 457f2bfd..00000000 --- a/backend/tests/pipeline/test_newsletter_links.py +++ /dev/null @@ -1,284 +0,0 @@ -"""Newsletter link extraction, rule prefilter and the Jev verdict. - -Everything here is a pure function of a string or a dict: what counts as a -link and what text surrounds it, which links are plumbing by rule, and where -the classification thresholds cut. The thresholds were measured on two real -issues; the table below is their specification. -""" - -from __future__ import annotations - -import pytest - -from pipeline.sources import email -from pipeline.sources.email import Link, extract, prefilter, verdict - -ZW = "​‌‍­" * 200 - -FIXTURE = f""" - -
Today's issue‌​{ZW}
- - - -
- - Why agent harnesses fail (6 minute read) - A walk through production agent design and where retries go wrong. -
Acme ships Agent SDK - lets you build agents in ten lines.
-""" - - -def _by_anchor(links: list[Link]) -> dict[str, Link]: - return {link.anchor: link for link in links} - - -def test_extract_finds_every_anchor_in_order(): - links = extract(FIXTURE) - assert [link.anchor for link in links] == [ - "Why agent harnesses fail (6 minute read)", - "", - "Acme ships Agent SDK", - ] - - -def test_the_context_window_reaches_the_description_after_the_link(): - link = _by_anchor(extract(FIXTURE))["Why agent harnesses fail (6 minute read)"] - assert "where retries go wrong" in link.context - assert "where retries go wrong" in link.blurb - - -def test_zero_width_padding_never_reaches_the_context(): - for link in extract(FIXTURE): - assert not email._ZERO_WIDTH_RE.search(link.context) - first = extract(FIXTURE)[0] - assert first.context.startswith("Today's issue Why agent harnesses fail") - - -def test_nested_tags_collapse_to_their_text(): - link = _by_anchor(extract(FIXTURE))["Acme ships Agent SDK"] - assert link.href == "https://acme.dev/launch" - assert link.blurb.startswith("lets you build agents in ten lines.") - - -def test_an_empty_anchor_is_still_extracted_with_its_surroundings(): - link = _by_anchor(extract(FIXTURE))[""] - assert link.href == "https://example.com/post" - assert "where retries go wrong" in link.context - - -def test_entities_in_hrefs_are_decoded(): - [link] = extract('Post') - assert link.href == "https://example.com/p?a=1&b=2" - - -def test_the_window_is_bounded_on_both_sides(): - html = f"{'x ' * 400}A{' y' * 400}" - html = html.replace("'", '"') - [link] = extract(html) - assert len(link.context) <= email.CONTEXT_BEFORE + len("A") + email.CONTEXT_AFTER - - -# ─── prefilter ─────────────────────────────────────────────────────────────── - - -def _link(href: str, anchor: str = "A story") -> Link: - return Link(href=href, anchor=anchor, context="") - - -def test_prefilter_drops_plumbing_and_keeps_stories(): - links = [ - _link("mailto:editor@tldrnewsletter.com", "Email us"), - _link("#top", "Back to top"), - _link("https://tldrnewsletter.com/unsubscribe?ep=1", "Unsubscribe"), - _link("https://example.org/manage", "Manage your preferences"), - _link("https://x.com/tldrnewsletter", "Follow us"), - _link("https://tldrnewsletter.com/ai", "Our AI edition"), - _link("https://example.com/post?utm_source=tldr", "Post"), - _link("https://example.com/post?utm_medium=email&utm_campaign=x", "Post"), - _link( - "https://tracking.tldrnewsletter.com/CL0/" - "https:%2F%2Fwww.youtube.com%2Fwatch%3Fv%3Dabc/1/0100019a626e0311", - "Watch the demo", - ), - _link("https://acme.dev/launch", ""), - _link("https://acme.dev/launch", "Acme ships Agent SDK"), - ] - - kept, dropped = prefilter(links, "tldrnewsletter.com") - - assert [link.anchor for link in kept] == ["Post", "Acme ships Agent SDK"] - assert kept[0].href == "https://example.com/post?utm_source=tldr" - reasons = {r: [link.anchor for link in ls] for r, ls in dropped.items()} - assert reasons == { - "non_http": ["Email us", "Back to top"], - "admin": ["Unsubscribe", "Manage your preferences"], - "denied": ["Follow us", "Watch the demo"], - "own_host": ["Our AI edition"], - "duplicate": ["Post"], - "no_text": [""], - } - - -def test_the_tracker_is_unwrapped_before_the_host_check(): - """The tracker lives on the newsletter's own domain; its destination does - not, and the destination is what decides.""" - tracked = _link( - "https://tracking.tldrnewsletter.com/CL0/" - "https:%2F%2Fexample.com%2Fpost/1/0100019a626e0311" - ) - kept, dropped = prefilter([tracked], "tldrnewsletter.com") - assert kept == [tracked] - assert dropped == {} - - -def test_a_tracked_link_and_a_direct_one_to_the_same_page_are_one(): - kept, dropped = prefilter( - [ - _link( - "https://tracking.tldrnewsletter.com/CL0/" - "https:%2F%2Fexample.com%2Fpost%3Futm_source%3Dtldr/1/0100019a626e0311" - ), - _link("https://example.com/post/"), - ], - "tldrnewsletter.com", - ) - assert len(kept) == 1 - assert len(dropped["duplicate"]) == 1 - - -def test_an_opaque_tracker_is_kept_for_jev_to_judge(): - """Beehiiv hides the destination behind a token: no host to check, and the - tracker's own host must not read as the sender's.""" - opaque = _link("https://link.mail.beehiiv.com/ss/c/u001.abcdef") - kept, _ = prefilter([opaque], "mail.beehiiv.com") - assert kept == [opaque] - - -# ─── verdict ───────────────────────────────────────────────────────────────── - -_EDITORIAL = { - "is_admin": 0.02, - "is_sponsor": 0.05, - "is_ai_topic": 0.95, - "kind": "article", - "is_first_party": 0.3, -} - - -@pytest.mark.parametrize( - ("overrides", "expected"), - [ - ({}, "keep"), - ({"is_sponsor": 0.74}, "keep"), - ({"is_sponsor": 0.76}, "sponsor"), - ({"is_admin": 0.54}, "keep"), - ({"is_admin": 0.56}, "admin"), - ({"is_ai_topic": 0.51}, "keep"), - ({"is_ai_topic": 0.49}, "off_topic"), - ({"kind": "product"}, "keep"), - ({"kind": "neither"}, "neither"), - # The measured false positive at 0.5: a launch post that reads promotional. - ({"is_sponsor": 0.64}, "keep"), - ({"is_admin": 0.9, "is_sponsor": 0.9}, "admin"), - ], -) -def test_verdict_thresholds(overrides, expected): - assert verdict({**_EDITORIAL, **overrides}) == expected - - -# ─── classification wiring ─────────────────────────────────────────────────── - - -def test_extract_items_keeps_only_what_the_verdict_keeps(monkeypatch): - answers = { - "Sponsored: Try Acme free": {**_EDITORIAL, "is_sponsor": 0.96}, - "Acme ships Agent SDK": { - **_EDITORIAL, - "kind": "product", - "is_first_party": 0.9, - }, - "The Senior Engineer Death Spiral": {**_EDITORIAL, "is_ai_topic": 0.05}, - } - seen_states = [] - - def ask(state, _questions): - seen_states.append(state) - return email.jev.Reply(answers=answers[state["link_text"]], cost=0.00002) - - monkeypatch.setattr(email.jev, "ask", ask) - html = "".join( - f'

{anchor} blurb {i}.

' - for i, anchor in enumerate(answers) - ) - - items = email._extract_items(html, "TLDR AI", "2026-09-23", "tldrnewsletter.com") - - assert len(seen_states) == 3 - assert {s["destination_host"] for s in seen_states} == { - "site0.com", - "site1.com", - "site2.com", - } - [item] = items - assert item["name"] == "Acme ships Agent SDK" - assert item["kind"] == email.NewsletterItemKind.Product - assert item["first_party"] is True - assert item["description"].startswith("blurb 1.") - - -def test_a_failed_classification_drops_only_that_link(monkeypatch): - def ask(state, _questions): - if state["link_text"] == "Broken": - raise email.jev.JevError("unanswered: is_sponsor") - return email.jev.Reply(answers=_EDITORIAL, cost=0.0) - - monkeypatch.setattr(email.jev, "ask", ask) - html = 'Broken Fine' - - items = email._extract_items(html, "TLDR AI", "2026-09-23", "tldr.tech") - - assert [i["name"] for i in items] == ["Fine"] - - -def test_an_opaque_tracker_is_never_first_party(monkeypatch): - """Jev saw "unknown" for the host, so its first-party answer is a guess.""" - monkeypatch.setattr( - email.jev, - "ask", - lambda state, questions: email.jev.Reply( - answers={**_EDITORIAL, "kind": "product", "is_first_party": 0.99}, cost=0 - ), - ) - html = 'Acme SDK' - - [item] = email._extract_items(html, "Pointer", "2026-09-23", "pointer.io") - - assert item["first_party"] is False - - -def test_an_article_about_privacy_is_not_a_privacy_policy(): - kept, _ = prefilter( - [ - _link("https://example.com/blog/privacy-in-llms", "Privacy in LLMs"), - _link("https://example.com/privacy", "Read more"), - ], - "tldrnewsletter.com", - ) - assert [link.anchor for link in kept] == ["Privacy in LLMs"] - - -@pytest.mark.parametrize( - ("sender", "own"), - [ - ("dan@tldrnewsletter.com", "tldrnewsletter.com"), - ("News@WWW.Example.com", "example.com"), - ("writer@substack.com", ""), - ("pointer@mail.beehiiv.com", ""), - ], -) -def test_own_host_ignores_shared_sending_platforms(sender, own): - """A Substack sender's own host would otherwise drop every post on every - other Substack.""" - assert email._own_host(sender) == own diff --git a/backend/tests/pipeline/test_publishers.py b/backend/tests/pipeline/test_publishers.py deleted file mode 100644 index 54a0d3ab..00000000 --- a/backend/tests/pipeline/test_publishers.py +++ /dev/null @@ -1,101 +0,0 @@ -"""Crediting a URL to the publisher whose site it is on.""" - -from __future__ import annotations - -from types import SimpleNamespace - -import pytest - -from app.catalog.models import PublisherKind -from pipeline.publishers import ( - host_kind, - hosts_to_publishers, - match_publisher, - owner_key, -) - - -def _pub(id_: int, **links: str) -> SimpleNamespace: - return SimpleNamespace(id=id_, links=links) - - -def _credited(url: str, *publishers: SimpleNamespace) -> int | None: - match = match_publisher(url, hosts_to_publishers(publishers)) # type: ignore[arg-type] - return match.id if match else None - - -def test_credits_a_url_to_the_publisher_whose_site_it_is_on(): - simon = _pub(1, website="https://simonwillison.net") - assert _credited("https://simonwillison.net/2026/Aug/8/auto-mode/", simon) == 1 - - -def test_a_feed_link_names_the_host_too(): - hamel = _pub(2, rss="https://hamelhusain.substack.com/feed") - assert _credited("https://hamelhusain.substack.com/p/evals", hamel) == 2 - - -def test_climbs_to_the_parent_domain(): - cloudflare = _pub(3, website="https://www.cloudflare.com") - assert _credited("https://blog.cloudflare.com/ai-code-review/", cloudflare) == 3 - - -def test_a_search_link_is_a_bare_host(): - anthropic = _pub(4, search="anthropic.com") - assert _credited("https://www.anthropic.com/news/claude-opus-5", anthropic) == 4 - - -def test_a_shared_platform_host_credits_nobody(): - """A publisher whose site is a GitHub or Hugging Face page must not claim - every repo or model card on the platform.""" - hf_blog = _pub(5, website="https://huggingface.co/blog") - assert _credited("https://huggingface.co/Qwen/Qwen3.8-27B-FP8", hf_blog) is None - - -def test_a_host_two_publishers_claim_credits_neither(): - a = _pub(6, website="https://example.com") - b = _pub(7, rss="https://example.com/feed") - assert _credited("https://example.com/post", a, b) is None - - -def test_an_unknown_host_credits_nobody(): - assert ( - _credited("https://nobody-we-know.dev/post", _pub(8, website="https://a.dev")) - is None - ) - - -def test_a_platform_owner_is_credited_by_path(): - anthropic = _pub( - 9, website="https://anthropic.com", github="https://github.com/anthropics" - ) - assert _credited("https://github.com/anthropics/claude-code/pull/1", anthropic) == 9 - assert _credited("https://github.com/someone-else/repo", anthropic) is None - - -def test_a_hugging_face_blog_feed_claims_the_blog_not_the_models(): - hf = _pub(10, rss="https://huggingface.co/blog/feed.xml") - assert _credited("https://huggingface.co/blog/smolagents", hf) == 10 - assert _credited("https://huggingface.co/Qwen/Qwen3", hf) is None - - -@pytest.mark.parametrize( - "url, key", - [ - ("https://github.com/MattPocock/skills", "github.com/mattpocock"), - ("https://github.com/features/copilot", None), - ("https://huggingface.co/datasets/allenai/tulu", "huggingface.co/allenai"), - ("https://hf.co/Qwen/Qwen3", "huggingface.co/qwen"), - ("https://huggingface.co/papers/2609.1", None), - ("https://medium.com/feed/airbnb-engineering", "medium.com/airbnb-engineering"), - ("https://medium.com/@someone/a-post-123", "medium.com/@someone"), - ("https://medium.com/tag/ai", None), - ("https://simonwillison.net/2026/", None), - ], -) -def test_owner_key(url: str, key: str | None): - assert owner_key(url) == key - - -def test_a_personal_host_is_an_individual(): - assert host_kind("someone.github.io") == PublisherKind.individual - assert host_kind("goodfire.ai") == PublisherKind.unknown diff --git a/backend/tests/pipeline/test_publishers_db.py b/backend/tests/pipeline/test_publishers_db.py deleted file mode 100644 index 013e5ef4..00000000 --- a/backend/tests/pipeline/test_publishers_db.py +++ /dev/null @@ -1,108 +0,0 @@ -"""Who an item is credited to, against the real schema: an aggregator's links -go to their authors, made on the spot when we have not seen the site.""" - -from __future__ import annotations - -from collections.abc import Generator - -import pytest -from sqlmodel import Session, col, delete - -from app.catalog.models import LinkPlatform, Publisher, PublisherKind, PublisherType -from pipeline import publishers as publishers_module -from pipeline.publishers import PublisherResolver -from tests.random_data import random_lower_string - -TAG = random_lower_string()[:8] - - -@pytest.fixture -def sources( - db: Session, monkeypatch: pytest.MonkeyPatch -) -> Generator[dict[str, Publisher]]: - monkeypatch.setattr(publishers_module, "account_type", lambda owner: "User") - made = { - "aggregator": Publisher( - name=f"{TAG} News", - slug=f"{TAG}-news", - kind=PublisherKind.community, - type=PublisherType.hn, - links={LinkPlatform.website: f"https://news.{TAG}.com"}, - ), - "feed": Publisher( - name=f"{TAG} Feed", - slug=f"{TAG}-feed", - kind=PublisherKind.company, - type=PublisherType.rss, - links={LinkPlatform.website: f"https://{TAG}-feed.com"}, - ), - } - db.add_all(made.values()) - db.commit() - yield made - db.exec(delete(Publisher).where(col(Publisher.slug).contains(TAG))) - db.commit() - - -def test_an_aggregators_link_to_a_new_site_gets_a_publisher_of_its_own( - db: Session, sources: dict[str, Publisher] -): - resolver = PublisherResolver(db) - url = f"https://blog.{TAG}.dev/a-post" - - publisher = resolver.publisher_for(url, sources["aggregator"].name) - - assert publisher.id not in (sources["aggregator"].id, sources["feed"].id) - assert publisher.name == f"blog.{TAG}.dev" - assert publisher.kind == PublisherKind.unknown - # Never polled: inactive, and only a website link. - assert publisher.is_active is False - assert publisher.links == {"website": f"https://blog.{TAG}.dev"} - # The next post on the site is credited to the same one, even in a new run. - again = PublisherResolver(db).publisher_for(f"https://blog.{TAG}.dev/b", "x") - assert again.id == publisher.id - - -def test_a_github_user_is_an_individual(db: Session, sources: dict[str, Publisher]): - resolver = PublisherResolver(db) - url = f"https://github.com/{TAG}owner/repo" - - publisher = resolver.publisher_for(url, sources["aggregator"].name) - - assert (publisher.name, publisher.kind) == (f"{TAG}owner", PublisherKind.individual) - assert ( - resolver.publisher_for(f"https://github.com/{TAG}owner/other", "x").id - == publisher.id - ) - - -def test_the_aggregators_own_page_stays_with_it( - db: Session, sources: dict[str, Publisher] -): - resolver = PublisherResolver(db) - url = f"https://news.{TAG}.com/item?id=1" - assert ( - resolver.publisher_for(url, sources["aggregator"].name).id - == sources["aggregator"].id - ) - - -def test_a_shared_host_that_names_no_author_stays_with_the_aggregator( - db: Session, sources: dict[str, Publisher] -): - resolver = PublisherResolver(db) - url = "https://x.com/someone/status/1" - assert ( - resolver.publisher_for(url, sources["aggregator"].name).id - == sources["aggregator"].id - ) - - -def test_a_feeds_own_item_stays_with_the_feed( - db: Session, sources: dict[str, Publisher] -): - """A feed's posts may live on a host its links do not name; that is not a - new author.""" - resolver = PublisherResolver(db) - url = f"https://cdn.{TAG}-elsewhere.com/post" - assert resolver.publisher_for(url, sources["feed"].name).id == sources["feed"].id diff --git a/backend/tests/pipeline/test_rejects.py b/backend/tests/pipeline/test_rejects.py deleted file mode 100644 index 7f22b261..00000000 --- a/backend/tests/pipeline/test_rejects.py +++ /dev/null @@ -1,80 +0,0 @@ -"""What a reject row keeps, and what it must strip to be insertable at all.""" - -from __future__ import annotations - -from pipeline.models import RejectStage -from pipeline.rejects import CANDIDATE_CAP, CONTENT_CAP, record_reject -from pipeline.types import Candidate - - -class _FakeSession: - def __init__(self) -> None: - self.rows: list = [] - - def merge(self, obj) -> None: - self.rows.append(obj) - - -def _candidate(**overrides) -> Candidate: - a: Candidate = { - "title": "A repo", - "url": "https://github.com/someone/thing", - "content": "body", - "published_date": "2026-09-12T10:00:00+00:00", - "source": "Hacker News", - "publisher_id": 7, - "traction": "3 points, 0 comments on Hacker News", - } - return {**a, **overrides} - - -def test_keeps_what_the_step_saw(): - session = _FakeSession() - record_reject(session, _candidate(), RejectStage.thin_repo, detail={"stars": 4}) - [row] = session.rows - assert row.stage == RejectStage.thin_repo - assert (row.title, row.source, row.publisher_id) == ("A repo", "Hacker News", 7) - assert row.traction == "3 points, 0 comments on Hacker News" - assert row.published_at.year == 2026 - assert row.detail == {"stars": 4} - - -def test_caps_content_and_strips_nul_bytes(): - """Postgres refuses a NUL in a text column, and the failed commit would take - the whole source down with it.""" - session = _FakeSession() - content = "a\x00b" + "x" * (CONTENT_CAP * 2) - record_reject( - session, _candidate(title="t\x00", content=content), RejectStage.prefilter - ) - [row] = session.rows - assert row.title == "t" - assert "\x00" not in row.content - assert len(row.content) < CONTENT_CAP - - -def test_a_pending_candidate_keeps_the_article_for_the_agent_to_read(): - session = _FakeSession() - record_reject( - session, _candidate(content="x" * (CANDIDATE_CAP * 2)), RejectStage.pending - ) - assert len(session.rows[0].content) == CANDIDATE_CAP - - -def test_empty_content_is_stored_as_null(): - session = _FakeSession() - record_reject(session, _candidate(content=""), RejectStage.prefilter) - assert session.rows[0].content is None - - -def test_keeps_the_links_extraction_found(): - session = _FakeSession() - links = {"repo": ["https://github.com/someone/thing"]} - record_reject(session, _candidate(links=links), RejectStage.pending) - assert session.rows[0].links == links - - -def test_no_links_is_stored_as_null(): - session = _FakeSession() - record_reject(session, _candidate(links={}), RejectStage.pending) - assert session.rows[0].links is None diff --git a/backend/tests/pipeline/test_run_stats.py b/backend/tests/pipeline/test_run_stats.py deleted file mode 100644 index f93581e9..00000000 --- a/backend/tests/pipeline/test_run_stats.py +++ /dev/null @@ -1,153 +0,0 @@ -"""What the run records: per-source counts, and the publishers it polled. - -On 2026-09-03 the Feeds source died after the fetch and took the per-publisher -record with it, so every feed was invisible on exactly the night something was -wrong. These pin that the publisher bookkeeping survives a failure after the -fetch, that a failed fetch stamps nothing, and that the stored error stays a -readable length. -""" - -from __future__ import annotations - -from pipeline import run as run_module -from pipeline.runs import RunStats -from pipeline.steps.fetch import Source - - -class _FakeSession: - def __enter__(self): - return self - - def __exit__(self, *exc): - return False - - def rollback(self) -> None: - pass - - -class _FakeResolver: - quarantined: list[str] = [] - created: list[str] = [] - - def __init__(self, **kwargs) -> None: - pass - - -def _article(url: str, source: str, publisher_id: int = 1) -> dict: - return { - "title": "t", - "url": url, - "content": "body", - "source": source, - "publisher_id": publisher_id, - } - - -def _stub_run(monkeypatch, sources, **overrides): - """Run the pipeline over ``sources`` with every step a no-op passthrough. - Returns the stats and what the publisher bookkeeping was handed.""" - seen: dict[str, list] = {"polled": [], "new": []} - monkeypatch.setattr(run_module, "get_engine", lambda: None) - monkeypatch.setattr(run_module, "Session", lambda engine: _FakeSession()) - monkeypatch.setattr(run_module, "PublisherResolver", _FakeResolver) - monkeypatch.setattr(run_module, "build_sources", lambda session: sources) - monkeypatch.setattr(run_module, "resolve_publishers", lambda a, r: a) - monkeypatch.setattr(run_module, "filter_known_urls", lambda s, a, label: a) - monkeypatch.setattr(run_module, "queue_candidates", lambda s, a: a) - monkeypatch.setattr( - run_module, "fetch_source", lambda source: (source.fetcher(), {}) - ) - monkeypatch.setattr( - run_module, - "record_polled", - lambda s, names, errors: seen["polled"].append((names, errors)), - ) - monkeypatch.setattr( - run_module, "record_new", lambda s, ids: seen["new"].append(sorted(ids)) - ) - for name, value in overrides.items(): - monkeypatch.setattr(run_module, name, value) - - stats = RunStats() - run_module.run_pipeline(stats) - return stats, seen - - -def _feeds_source() -> Source: - return Source( - "Feeds", - lambda: [_article("u1", "Feed A", publisher_id=7)], - publisher_names=("Feed A", "Feed B"), - ) - - -def test_a_clean_run_counts_and_stamps(monkeypatch): - stats, seen = _stub_run(monkeypatch, [_feeds_source()]) - - [source] = stats.sources - assert (source.fetched, source.queued, source.error) == (1, 1, None) - assert seen["polled"] == [(("Feed A", "Feed B"), {})] - assert seen["new"] == [[7]] - - -def test_polled_publishers_are_stamped_after_a_failure_past_the_fetch(monkeypatch): - def boom(_session, _articles): - raise RuntimeError("queue down") - - stats, seen = _stub_run(monkeypatch, [_feeds_source()], queue_candidates=boom) - - assert seen["polled"] == [(("Feed A", "Feed B"), {})] - # Nothing was queued, so nothing is new. - assert seen["new"] == [[]] - assert stats.sources[0].error == "RuntimeError: queue down" - - -def test_a_fetch_that_failed_stamps_no_publisher(monkeypatch): - """Nothing was polled, so there was no answer to record — stamping every - publisher as fetched would hide a source that is down.""" - - def boom(_source): - raise RuntimeError("feeds unreachable") - - stats, seen = _stub_run(monkeypatch, [_feeds_source()], fetch_source=boom) - - assert seen["polled"] == [] - assert stats.sources[0].error == "RuntimeError: feeds unreachable" - - -def test_bookkeeping_that_fails_does_not_fail_the_source(monkeypatch): - def boom(*_args): - raise RuntimeError("db gone") - - stats, _ = _stub_run(monkeypatch, [_feeds_source()], record_polled=boom) - - assert (stats.sources[0].queued, stats.sources[0].error) == (1, None) - - -def test_one_source_failing_does_not_stop_the_next(monkeypatch): - def boom(_session, articles): - if articles[0]["source"] == "Feed A": - raise RuntimeError("queue down") - return articles - - stats, _ = _stub_run( - monkeypatch, - [_feeds_source(), Source("Hacker News", lambda: [_article("u2", "HN")])], - queue_candidates=boom, - ) - - assert [(s.source, s.queued) for s in stats.sources] == [ - ("Feeds", 0), - ("Hacker News", 1), - ] - - -def test_the_stored_error_is_truncated(monkeypatch): - def boom(_session, _articles): - raise RuntimeError("x" * 50_000) - - stats, _ = _stub_run(monkeypatch, [_feeds_source()], queue_candidates=boom) - - error = stats.sources[0].error or "" - assert len(error) < 1_000 - assert error.endswith("chars]") diff --git a/backend/tests/random_data.py b/backend/tests/random_data.py deleted file mode 100644 index dd00b236..00000000 --- a/backend/tests/random_data.py +++ /dev/null @@ -1,14 +0,0 @@ -"""Random strings and emails, so two tests never collide on a unique column.""" - -from __future__ import annotations - -import random -import string - - -def random_lower_string() -> str: - return "".join(random.choices(string.ascii_lowercase, k=32)) - - -def random_email() -> str: - return f"{random_lower_string()}@{random_lower_string()}.com" diff --git a/backend/tests/scripts/__init__.py b/backend/tests/scripts/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/deploy/README.md b/deploy/README.md index 53dc16cc..7cfcc67f 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -14,13 +14,30 @@ How agentique runs in production. Docker runs only the database | `agentique-backup.service` + `.timer` | nightly DB dump to off-box storage at 03:30 | | `agentique-sql` | run SQL against the prod DB from stdin, inside the `db` container | | `sql-roles.sql` | one-time: creates the read-only and read/write login roles `agentique-sql` uses | +| `rsync-filter` | what the deploy copies to `/opt/agentique`, and what it never deletes there | ## Shape - A push to `master` triggers `.github/workflows/deploy-production.yml`: build the frontend on a GitHub-hosted runner, then a self-hosted runner on the box - rsyncs the source, syncs the venv, runs migrations + prestart, and restarts - the backend. Additive migrations only; rollback is `git revert` + push. + rsyncs what the box runs, syncs the venv, runs migrations + prestart, and + restarts the backend. Additive migrations only; rollback is `git revert` + + push. +- The rsync copies only what `rsync-filter` lists: the backend's `app/` + without tests, `migrations/`, `baml_client/`, the uv workspace files, + `compose.yml`, `deploy/` and the built frontend. It deletes everything + else in `/opt/agentique`, except `.env`, `.venv` and `.git`, which the + deploy never sends. A file the box needs goes in the filter. + `app/entry/cli/test_deploy_files.py` runs the filter and checks every path + the units and the deploy job use. +- The units, the Caddyfile and `agentique-sql` are installed by hand. After + a deploy that changes one, copy it from `/opt/agentique/deploy/` and run + `sudo systemctl daemon-reload`. If an `ExecStart` path changed, the + deploy has already deleted the old file. The backend then fails from the + deploy's own restart until you copy its unit and restart it, and each + timer fails the next time it fires. The deploy job doesn't install units + because the runner can rewrite anything under `/opt/agentique`. Letting + it install units with sudo would give it root. - The pipeline needs no restart — the timer starts a fresh process each run. - The night runs in three steps an hour apart: 04:00 the pipeline queues candidates, 05:00 the agent judges them, 06:00 the report goes out. The box @@ -146,13 +163,17 @@ Finish with a short report: stories created, extended and closed, and anything a ## Newsletter environment -Read by the backend through `app/core/settings.py`. +Read by the backend through `app/core/settings.py`, with two exceptions that +read `os.environ`: `subscribe` (`app/features/newsletter/service.py`) reads +`RESEND_API_KEY` and `RESEND_AUDIENCE_ID`, and `send_mail` +(`app/core/email.py`), which sends the daily mail, reads `RESEND_API_KEY` and +`EMAILS_FROM_EMAIL`. | Variable | Read by | Effect | | --- | --- | --- | -| `RESEND_API_KEY` | `app/newsletter/broadcast.py` | Unset: `draft_issue` refuses, naming it. | -| `RESEND_AUDIENCE_ID` | `app/newsletter/broadcast.py`, `app/newsletter/routes.py` | The audience signups are synced to and the issue is broadcast to. | -| `EMAILS_FROM_EMAIL`, `EMAILS_FROM_NAME` | `app/newsletter/broadcast.py` | The issue's sender. | +| `RESEND_API_KEY` | `app/features/newsletter/broadcast.py`, `app/features/newsletter/service.py`, `app/core/email.py` | Unset: `draft_issue` refuses, naming it, signups are not synced to Resend, and the daily mail is only logged. | +| `RESEND_AUDIENCE_ID` | `app/features/newsletter/broadcast.py`, `app/features/newsletter/service.py` | The audience signups are synced to and the issue is broadcast to. | +| `EMAILS_FROM_EMAIL`, `EMAILS_FROM_NAME` | `app/features/newsletter/broadcast.py`, `app/core/email.py` (email only) | The issue's sender, and the daily mail's. | ## Pipeline environment @@ -163,3 +184,9 @@ Read as raw `os.environ` beside each consumer (ADR 7), so they belong in | --- | --- | --- | | `OPENROUTER_API_KEY` | `app/core/jev.py` | Jev, which classifies each newsletter link (sponsor, plumbing, on-topic, kind). Unset fails every link, so the newsletter channel yields nothing. | | `PIPELINE_ALERT_EMAIL` | `app/core/email.py` | Where the daily mail goes (liveness, failing sources and feeds, what landed); falls back to `EMAILS_FROM_EMAIL`. | +| `IMAP_HOST`, `IMAP_USER`, `IMAP_PASSWORD`, `IMAP_PORT` | `app/features/collect/newsletters.py` | The inbox the newsletter source reads. The first three are required by that source; the port defaults to 993. | +| `HN_MIN_POINTS`, `HN_MIN_COMMENTS`, `HN_GRACE_HOURS` | `app/features/collect/hn.py` | The HN traction gate: 10 points or 5 comments. A story under 6 hours old is held back for the next run. A first-party URL skips the gate, and `HN_MIN_POINTS=0` turns it off. | +| `TAVILY_API_KEY` | `app/core/web_search.py` | Search for the lab-watch source, the newsletter source and the `web_search` tool. | +| `GITHUB_TOKEN` | `app/core/github.py` | Optional. Raises the GitHub API rate limit for repo lookups. | +| `RESIDENTIAL_PROXY_URL` | `app/core/http.py` | Optional. A metered proxy, tried when a feed fetch or a page extraction is blocked direct. | +| `KDRIVE_API_TOKEN`, `KDRIVE_DRIVE_ID`, `KDRIVE_BACKUP_DIR`, `KDRIVE_BACKUP_KEEP` | `app/entry/cli/backup_db.py` | Where the nightly dump goes. The dir defaults to `/Private/agentique-db`, and 14 dumps are kept. | diff --git a/deploy/agentique-backup.service b/deploy/agentique-backup.service index 4e0d81cd..edce5cba 100644 --- a/deploy/agentique-backup.service +++ b/deploy/agentique-backup.service @@ -18,4 +18,4 @@ EnvironmentFile=/opt/agentique/.env # Root running python against the shared venv would otherwise leave root-owned # __pycache__ files behind, breaking `uv sync` on the next deploy. Environment=PYTHONDONTWRITEBYTECODE=1 -ExecStart=/opt/agentique/.venv/bin/python scripts/backup_db.py +ExecStart=/opt/agentique/.venv/bin/python app/entry/cli/backup_db.py diff --git a/deploy/agentique-pipeline.service b/deploy/agentique-pipeline.service index 8c0a1c9f..4c84bc16 100644 --- a/deploy/agentique-pipeline.service +++ b/deploy/agentique-pipeline.service @@ -11,12 +11,12 @@ Type=oneshot User=agentique Group=agentique WorkingDirectory=/opt/agentique/backend -# pipeline/run.py reads raw os.environ and loads no dotenv — EnvironmentFile -# is what feeds it. +# The database settings come through app.core.settings, the per-source knobs +# straight from os.environ (ADR 7). EnvironmentFile feeds both. EnvironmentFile=/opt/agentique/.env # agentique is a system user (adduser --system) with home=/nonexistent. # Anything that embeds via model2vec lazy-downloads through huggingface_hub and # expands ~ for its cache dir — fails with EACCES against /nonexistent. /var/lib/agentique is real and writable (vps-setup.sh) and # lives outside /opt/agentique so the deploy rsync's --delete never touches it. Environment=HOME=/var/lib/agentique -ExecStart=/opt/agentique/.venv/bin/python -m pipeline.run +ExecStart=/opt/agentique/.venv/bin/python -m app.entry.cli.collect diff --git a/deploy/agentique-report.service b/deploy/agentique-report.service index bc2d8a05..f4930710 100644 --- a/deploy/agentique-report.service +++ b/deploy/agentique-report.service @@ -11,8 +11,8 @@ Type=oneshot User=agentique Group=agentique WorkingDirectory=/opt/agentique/backend -# pipeline/report.py reads raw os.environ and loads no dotenv, same as -# pipeline/run.py -- EnvironmentFile is what feeds it. +# Same as agentique-pipeline.service: EnvironmentFile feeds the database +# settings and the raw os.environ reads alike. EnvironmentFile=/opt/agentique/.env Environment=HOME=/var/lib/agentique -ExecStart=/opt/agentique/.venv/bin/python -m pipeline.report +ExecStart=/opt/agentique/.venv/bin/python -m app.entry.cli.report diff --git a/deploy/rsync-filter b/deploy/rsync-filter new file mode 100644 index 00000000..50371b44 --- /dev/null +++ b/deploy/rsync-filter @@ -0,0 +1,35 @@ +# The deploy's rsync filter (.github/workflows/deploy-production.yml): what +# lands in /opt/agentique. The first matching rule wins. +# +# The deploy runs with --delete-excluded, so rsync deletes anything on the box +# that is not listed here, including files from earlier deploys. It never +# deletes the P paths. The deploy doesn't send them either. +P /.env +P /.venv/ +P /.git/ + +# Test-only code. backend/conftest.py falls under the final rule. +- __pycache__/ +- test_*.py +- /backend/app/core/db/factories.py + +# uv sync --frozen --package app reads the workspace root and the member. ++ /pyproject.toml ++ /uv.lock ++ /.python-version +# backup_db.py runs `docker compose -f /compose.yml`. ++ /compose.yml +# The units, agentique-sql, the Caddyfile and sql-roles.sql, for installing +# by hand. ++ /deploy/*** ++ /backend/ ++ /backend/pyproject.toml ++ /backend/.python-version ++ /backend/app/*** ++ /backend/migrations/*** ++ /backend/baml_client/*** +# Caddy serves the built frontend. The deploy downloads it into frontend/dist +# before the sync. ++ /frontend/ ++ /frontend/dist/*** +- * diff --git a/deploy/sql-roles.sql b/deploy/sql-roles.sql index 5c36a0d8..3612e928 100644 --- a/deploy/sql-roles.sql +++ b/deploy/sql-roles.sql @@ -1,5 +1,5 @@ -- Login roles for /usr/local/bin/agentique-sql (deploy/agentique-sql) and for --- the MCP `sql_query` tool (backend/app/mcp/tools.py). +-- the MCP `sql_query` tool (backend/app/entry/mcp/sql_query.py). -- One-time, on the box: -- docker exec -i agentique-db-1 psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" \ -- -v ON_ERROR_STOP=1 < /opt/agentique/deploy/sql-roles.sql diff --git a/docs/adr/0017-layered-features.md b/docs/adr/0017-layered-features.md new file mode 100644 index 00000000..ab7913c8 --- /dev/null +++ b/docs/adr/0017-layered-features.md @@ -0,0 +1,115 @@ +# 17. Three layers, one folder per feature + +## Status + +Accepted. Supersedes 1 and 4. Changes part of 7 and 8. + +## Context + +ADR 1 gave `app/` one folder per domain and left `pipeline/` beside it in its +own shape. The domains still imported each other. Every table sat in some +domain's `models.py`, and the pipeline and every domain read `Article` from +`catalog/models.py`. Routes in `audience/` called into `catalog` and +`newsletter`. `catalog/stories.py` used the embedding maths in +`pipeline/curation.py`, and the newsletter used the curation maths, the +ledger, `url_kind` and `SNIPPET_CAP`. The MCP tools reached into +`pipeline/fetching`. + +Shared code had no single home. Some was in `app/platform/`, some in +`pipeline/`, and there were two engines: `app/platform/db.py` and +`pipeline/db.py`. Tests sat in a `tests/` folder in each domain (ADR 4), plus +`backend/tests/` for the pipeline, the scripts and the enum guard. The deploy +copied the whole checkout to the box. + +## Decision + + app/main.py wires entry.api and entry.mcp into one FastAPI app + entry/ api/ mcp/ cli/ the ways in: HTTP, MCP, command line + features/ catalog/ stories/ curation/ collect/ newsletter/ audience/ analytics/ + core/ db/ plus flat modules tables, engine, shared mechanism + +- **Three layers.** `entry/` composes features, features use `core/`, and + imports only point down. One rule covers every case, so a reader can tell + where a file goes from what it does. +- **Features never import each other.** When two features need the same + thing, it moves to `core/`. When an API call spans features (sign-up + creates a user and subscribes them), `entry/api/` composes them. +- **Every feature folder uses the same names.** `model.py` holds its own + types, never tables. `service.py` holds the verbs entry calls, as + `session` plus plain arguments. A file named after a `core` module + (`audience/email.py`) adapts it. Other files are private helpers named + after what they hold. A template sits beside its one consumer. +- **Tables live in `core/db/`, one module per table.** The table graph is + shared. Foreign keys cross feature lines, and the migrations are already + one history over the whole schema. Tables in a feature would force + sideways imports. `core/db/__init__.py` imports every table module, so + `SQLModel.metadata` is complete for autogenerate. `core/db/` also holds + the one engine and the few table rules two features need + (`like_counts_subquery`, `kind_from_url`). Queries one feature owns stay + in that feature. +- **Shared mechanism is flat in `core/`.** `settings.py`, `http.py`, + `extract.py`, `embedding.py` and the rest sit beside `db/`. A flat module + imports stdlib, third-party packages and other flat modules, never + `core.db`. There is no `shared/` folder, so ADR 3 needs no exception. +- **`entry/` has one file per router, one file per MCP tool, one file per + command.** A route, a tool or a command is an adapter: parse the input, + call a feature's `service`, shape the result. `entry/cli/` replaces + `backend/scripts/`, `pipeline/run.py` and `pipeline/report.py`. +- **Tests sit beside their module as `test_.py`.** No `tests/` + folders. `backend/conftest.py` holds the shared fixtures, and + `core/db/factories.py` the model factories. +- **import-linter enforces it.** `lint-imports` runs in pre-commit and in + the test-backend workflow, with five contracts in `backend/pyproject.toml`: + 1. layers `app.main` > `app.entry` > `app.features` > `app.core`, exhaustive, + so a new top-level module has to join a layer + 2. independence over `app.features.*` + 3. independence over `app.entry.api`, `app.entry.mcp`, `app.entry.cli` + 4. forbidden: the flat `app.core.*` modules never import `app.core.db` + 5. forbidden: entry reaches a feature only through its `service` and `model` +- **The deploy ships only what the box runs.** `deploy/rsync-filter` lists + `backend/app/` without tests or factories, `backend/migrations/`, + `backend/baml_client/`, the uv workspace files, `compose.yml`, `deploy/` + and the built frontend. The sync deletes everything else on the box except + `.env`, `.venv` and `.git`. + +What changes in earlier ADRs: + +- **1** is superseded. Domain folders with `models.py` / `service.py` / + `routes.py` / `tests/` become feature folders with `model.py` / + `service.py` and named helpers. Routes move to `entry/api/`. `platform/` + becomes `core/`. `pipeline/` is no longer an exception. It becomes + `features/collect/`, with its curation half in `features/curation/`. +- **4** is superseded. Tests still sit beside the code they test, but as + `test_.py` files next to the module, not in a `tests/` folder. + `backend/tests/` is gone, and factories moved to `core/db/factories.py`. + The frontend half of 4 is unchanged. +- **7** changes in part. There is one engine, in `core/db/engine.py`, and it + takes the database URL from `Settings`, so the collect run builds Settings + too. The box feeds both processes the same `.env`. The per-source knobs + (IMAP, HN thresholds, the Tavily and GitHub keys, the proxy) still read + `os.environ` in the module that uses them, and `deploy/README.md` still lists them. +- **8** changes in part. The stage-type chain splits at the queue. collect + owns `RawItem` -> `Candidate` (`features/collect/model.py`). curation owns + `Scored` -> `Summarized` -> `Persisted` (`features/curation/model.py`) and + builds `Scored` from the ledger row the agent judged. `Scored` no longer + subclasses `Candidate`, since that would import one feature into another. + +## Consequences + +`ls app/features/` lists what the backend does, and `ls` on one feature +lists its verbs, its types and its helpers. An import that breaks the rules +fails pre-commit and CI rather than review. + +A rule two features share has to move to `core/` before either can use it. +That move gets its own commit. + +The `stories` MCP tool (curation coverage) keeps its name although a +`stories/` feature now exists. Renaming it would change the curate rubric's +tools. + +Contract 5 checks import paths, not names. A feature that re-exports a helper +from its `__init__.py` would let entry reach it without a failure. + +The systemd units now run `app.entry.cli.*`. They are installed by hand, so +the deploy that changed `ExecStart` needed the units copied and +`daemon-reload` run before that night's timers. diff --git a/docs/backend.md b/docs/backend.md index 80db691f..0b86100b 100644 --- a/docs/backend.md +++ b/docs/backend.md @@ -9,20 +9,20 @@ - Needs a root `.env` (gitignored): `PROJECT_NAME`, `SECRET_KEY`, `FIRST_SUPERUSER`, `POSTGRES_SERVER=localhost`, `POSTGRES_USER`, `POSTGRES_PASSWORD`, `POSTGRES_DB` - Backend loads it via pydantic-settings (`env_file="../.env"`) -- Pipeline reads raw `os.environ` directly, hence `--env-file` below +- The per-source knobs (IMAP, HN thresholds, API keys) are read from raw `os.environ`, hence `--env-file` below - `frontend/.env` is separate (Vite-only, `VITE_API_URL`) - keeps secrets out of the frontend build ```bash docker compose up -d # from repo root: starts the db cd backend -uv run bash scripts/prestart.sh # migrations + initial data +uv run bash app/entry/cli/prestart.sh # migrations + initial data uv run fastapi dev app/main.py ``` -Pipeline, manual run: +The nightly collect, run by hand: ```bash -cd backend && uv run --env-file ../.env python -m pipeline.run +cd backend && uv run --env-file ../.env python -m app.entry.cli.collect ``` The run ends at pending candidates and publishes nothing — the curation agent @@ -30,7 +30,7 @@ does that (ADR 9). Judge them, then send the night's report: ```bash claude "/curate" # from the repo root -cd backend && uv run --env-file ../.env python -m pipeline.report +cd backend && uv run --env-file ../.env python -m app.entry.cli.report ``` `AGENTIQUE_MCP_TOKEN` must be the write token for `/curate` to publish anything; @@ -45,53 +45,64 @@ nothing. Uses [prek](https://prek.j178.dev/). Config: `.pre-commit-config.yaml`. ```bash -uv run prek install -f # install the git hook, from backend/ +uv run prek install -f # install the git hook uv run prek run --all-files # run manually on everything ``` +- Runs ruff (lint and format) on every Python file, `mypy --strict` and `ty` over `backend/app`, typos, biome and the banned-module-names check +- Runs `lint-imports`, which checks the five import contracts in `backend/pyproject.toml` (ADR 17). The test-backend workflow runs it too +- Regenerates the frontend client when anything under `backend/` changes + ## Setup ```bash -cd backend uv sync source .venv/bin/activate ``` -- Point your editor's Python interpreter at `backend/.venv/bin/python` -- SQLModel models live with their domain: `app/catalog/models.py`, `app/audience/models.py`, `app/newsletter/models.py`, `app/analytics/models.py`, `pipeline/models.py` for the run record -- Each domain folder holds its own `routes.py` and service module +- uv workspace: the root `pyproject.toml` lists `backend/` as its one member, so the venv is the root `.venv` +- Point your editor's Python interpreter at `.venv/bin/python` + +## Layout -VS Code: debugger and Python-tests-tab configs are already set up. +- `app/main.py` - the FastAPI app: the routers, then the MCP server at `/mcp` +- `app/entry/` - the ways in: `api/` one file per router, `mcp/` one file per tool, `cli/` one file per command +- `app/features/` - one folder per feature: catalog, stories, curation, collect, newsletter, audience, analytics. Each has `model.py` and `service.py` +- `app/core/` - `db/` (the tables, one module each, and the engine) plus flat shared modules (`settings.py`, `http.py`, `extract.py`, `embedding.py`, ...) +- Imports point down, entry to features to core, and features never import each other (ADR 17) ## Tests With the db up: ```bash -uv run bash scripts/tests-start.sh +cd backend +uv run bash ../scripts/tests-start.sh ``` -- Domain tests live in that domain's `tests/` folder (`app/catalog/tests/`, `app/audience/tests/`, ...) -- `backend/tests/` holds what belongs to no single domain: pipeline, schema-wide enum guard, ops scripts -- Shared fixtures: `backend/conftest.py` -- `tests-start.sh` waits for the db, then runs `scripts/test.sh` (pytest + coverage). Extra args forward to pytest: +- Each test sits beside its module as `test_.py` (`app/features/catalog/service.py` and `test_service.py`). No `tests/` folders +- Shared fixtures: `backend/conftest.py`. Model factories: `app/core/db/factories.py` +- `tests-start.sh` waits for the db, then runs `scripts/test.sh` beside it (pytest + coverage). An argument becomes the coverage report's title, not a pytest flag: ```bash -uv run bash scripts/tests-start.sh -x +uv run bash ../scripts/tests-start.sh "my branch" ``` -- Coverage report: `htmlcov/index.html` +- Coverage report: `htmlcov/index.html`. Test files, `conftest.py` and `factories.py` are left out of the count ## Migrations - After changing a model, create a revision and apply it, or the schema and the app disagree -- Alembic imports every model module in `app/alembic/env.py` - a new domain needs a line there or its tables are invisible to autogenerate +- `app/core/db/__init__.py` imports every table module. A new table module needs a line there or autogenerate can't see it +- Alembic's config is `[tool.alembic]` in `backend/pyproject.toml`. Revisions are in `backend/migrations/versions/` + +From `backend/`: ```bash uv run alembic revision --autogenerate -m "Add column last_name to User model" ``` -Commit the generated file under `app/alembic/versions/`, then: +Commit the generated file under `migrations/versions/`, then: ```bash uv run alembic upgrade head @@ -100,6 +111,6 @@ uv run alembic upgrade head ## Email templates - Two transactional emails: `welcome.html` on signup and `sign_in.html` on a sign-in request, both carrying a fresh sign-in link -- Templates: `app/core/email-templates/`, next to `app/core/email.py`, which renders and sends them -- The weekly newsletter issue: `weekly_issue.html` in the same folder, the site's wordmark over plain prose, rendered (autoescaped) and drafted by `app/newsletter/broadcast.py` +- They sit in `app/features/audience/`, beside `email.py`, which builds them and sends them through `app/core/email.py` +- The weekly newsletter issue: `app/features/newsletter/weekly_issue.html`, the site's wordmark over plain prose, rendered (autoescaped) and drafted by `broadcast.py` beside it - No MJML source - edit the HTML directly diff --git a/docs/glossary.md b/docs/glossary.md index 14de0ff3..57d6a847 100644 --- a/docs/glossary.md +++ b/docs/glossary.md @@ -8,11 +8,11 @@ Glossary for words the code uses. If a term below and the code disagree, the cod - `kind` (`PublisherKind`) - what the publisher is: `lab` (makes models, writing on its own site), `company`, `individual`, `media`, `community` (an aggregator, credited only for its own pages), or `unknown` (a site the pipeline just made a publisher for; the curation agent names it on its first approval, and a kind once set stays). Shown to readers. - `type` (`PublisherType`) - how the pipeline finds its articles: `rss`, `substack`, `search`, `hn`, `reddit`, `email`, `ainews`, `other`. Internal ingestion detail, not on the public API. -- **Source** - one of the pipeline's fetch adapters (`pipeline/sources/*.py`): Hacker News, the RSS/Substack feeds, a lab-watch crawler, email newsletters. A source yields raw items for one or more publishers. Pipeline concept, not a database column. +- **Source** - one of the pipeline's fetch adapters (`hn.py`, `feeds.py`, `lab_watch.py` and `newsletters.py` in `app/features/collect/`): Hacker News, the RSS/Substack feeds, a lab-watch crawler, email newsletters. A source yields raw items for one or more publishers. Pipeline concept, not a database column. - **Trust** (`Publisher.trust`, `TrustLevel`) - `low` / `medium` / `high`, hand-set per publisher. No longer read (ADR 11): the curation agent sees each publisher's approval rate instead (`approved` in `list_candidates`: approvals / decisions over 90 days, or "new"). The column stays. Not on the public API. -- **Topic-gated** (`Publisher.topic_gated`) - a publisher flagged as mostly off-topic (a general engineering blog, not an AI one). No longer read: the fetch step used to drop its off-topic titles, and the curation agent judges topic now (ADR 11). The column stays. Hacker News keeps its own title keyword gate (`is_on_topic` in `pipeline/sources/hn.py`). +- **Topic-gated** (`Publisher.topic_gated`) - a publisher flagged as mostly off-topic (a general engineering blog, not an AI one). No longer read: the fetch step used to drop its off-topic titles, and the curation agent judges topic now (ADR 11). The column stays. Hacker News keeps its own title keyword gate (`is_on_topic` in `app/features/collect/hn.py`). - **Found via** (`Article.found_via`) - the aggregator (Hacker News, a newsletter) that found an article, when that is not its publisher. Null for an article from its publisher's own feed. @@ -21,24 +21,24 @@ Glossary for words the code uses. If a term below and the code disagree, the cod - **Score** (`Article.score`) - a 1-100 rating of how actionable the article is for a developer building with AI right now. Written by the curation agent, which reads the page before it decides (see **Candidate** below and ADR 9); an LLM in the pipeline used to write it from a title and a 200-char snippet. The one-sentence reason is kept in `Article.score_reason` (internal, not on the public API) and, for rejects, in `Reject.reason`. - **Candidate** - overloaded, and the two senses are one step apart. - - The *type* `Candidate` (`pipeline/types.py`) - a raw item resolved to its Publisher, mid-funnel. + - The *type* `Candidate` (`app/features/collect/model.py`) - a raw item resolved to its Publisher, mid-funnel. - A *candidate* in the feed's sense - an article the nightly run has queued and the curation agent has not judged yet: a `Reject` row at stage `pending`. It becomes an `Article` on approval and a `below_threshold` reject otherwise. Until then it is not visible anywhere a reader can see. - **Curation agent** - the Claude Code session that runs at 05:00, an hour after the pipeline, reads the pending candidates through the MCP tools (`list_candidates`, `get_content`, `similar`, `stories`, `check_link`, `approve`, `reject`, `reject_many`) and decides each one. Its rubric is `.claude/skills/curate/SKILL.md`. It is the only judge the feed has and the only thing that writes an `Article`. -- **Coverage** - how many distinct publishers carry one story, counted by `similar` and `stories` (`pipeline/curation.py`): rows within 0.30 cosine distance of each other across the feed and the ledger. Reach evidence for the curation agent, like traction. +- **Coverage** - how many distinct publishers carry one story, counted by `similar` and `stories` (`app/features/curation/service.py`): rows within 0.30 cosine distance of each other across the feed and the ledger. Reach evidence for the curation agent, like traction. -- **Story** (`Story`, `app/catalog/stories.py`) - a named thread that runs over days or weeks, like "Codemode": a name of 1-3 words, a two-sentence blurb, and its articles (`story_article`). Wider than coverage: coverage is one event told by several outlets, a story is a theme a builder follows across several events, which takes judgment. The stories routine (`.claude/skills/stories/SKILL.md`, ADR 16) names and keeps them through `story_candidates`, `save_story` and `close_story`. An article sits in at most one open story. The right rail (`GET /api/v1/stories`) shows open stories with 3+ articles from 2+ publishers. +- **Story** (`Story` in `app/core/db/story.py`, `app/features/stories/`) - a named thread that runs over days or weeks, like "Codemode": a name of 1-3 words, a two-sentence blurb, and its articles (`story_article`). Wider than coverage: coverage is one event told by several outlets, a story is a theme a builder follows across several events, which takes judgment. The stories routine (`.claude/skills/stories/SKILL.md`, ADR 16) names and keeps them through `story_candidates`, `save_story` and `close_story`. An article sits in at most one open story. The right rail (`GET /api/v1/stories`) shows open stories with 3+ articles from 2+ publishers. -- **Issue** - one weekly newsletter email and the blog post with the same text: a 250-450 word explainer of one topic, what it is, how it works and how people are taking it, linking the sources it was written from. The newsletter agent (`.claude/skills/newsletter/SKILL.md`) drafts three a week, one per topic, through `week`, `related` and `draft_issue` (`app/newsletter/`), and a person sends one. Each is a file in `frontend/content/blog/-.md`, the frontmatter giving the title (the subject), the description (the preheader) and the topic; the routine opens one PR with the three, and git stores them. +- **Issue** - one weekly newsletter email and the blog post with the same text: a 250-450 word explainer of one topic, what it is, how it works and how people are taking it, linking the sources it was written from. The newsletter agent (`.claude/skills/newsletter/SKILL.md`) drafts three a week, one per topic, through `week`, `related` and `draft_issue` (`app/features/newsletter/`), and a person sends one. Each is a file in `frontend/content/blog/-.md`, the frontmatter giving the title (the subject), the description (the preheader) and the topic; the routine opens one PR with the three, and git stores them. - **Traction** - outside signal that people found a story worth reading, independent of the agent's score: Hacker News points/comments past a minimum, which hold back low-traction submissions from the "everyone can post" source; a repo's stars or a model's downloads, which the curation agent reads with `check_link`; and coverage (above). A first-party URL (`app/core/urls.py`) skips the gate - a lab's own announcement counts as news at zero votes. - **Reject** (`Reject`, table `scored_url`) - a URL the funnel turned down, one row per URL. `stage` says which step dropped it - `below_threshold` for the agent's rejects; `thin_repo`, `prefilter`, `duplicate` and `unusable_summary` are on old rows only, from gates that are gone - or `pending`, the one stage that is not a rejection at all: a candidate still waiting on the agent. It keeps what that step saw: title, source, publisher, the first 2000 chars of content, traction, the repo / model / paper / docs `links` the article body makes, score, and the judge's `reason`. `filter_known_urls` reads it so a reject is never judged twice. Holds in a source (HN traction, recency) and a failed LLM call are deliberately not rejects: those come back next run. Rows from before 2026-09-13 carry only the URL. -- **Run** (`PipelineRun`) - one row per nightly pipeline execution: start/end time, duration, ok/fail, and per source `{fetched, queued, error}`. The daily mail (`pipeline/report.py`) reads the latest one for liveness. Per-publisher history lives on the Publisher row instead: `last_fetched_at` (its feed last answered), `last_new_at` (it last gave a new candidate), `last_error` (its last fetch failed, cleared by the next good one). +- **Run** (`PipelineRun`) - one row per nightly pipeline execution: start/end time, duration, ok/fail, and per source `{fetched, queued, error}`. The daily mail (`app/features/collect/report.py`) reads the latest one for liveness. Per-publisher history lives on the Publisher row instead: `last_fetched_at` (its feed last answered), `last_new_at` (it last gave a new candidate), `last_error` (its last fetch failed, cleared by the next good one). -- **RawItem / Candidate / Scored / Persisted** (`pipeline/types.py`) - the four shapes an article takes going down the funnel, one per stage. +- **RawItem / Candidate / Scored / Persisted** (`app/features/collect/model.py`, `app/features/curation/model.py`) - the four shapes an article takes going down the funnel, one per stage. - **RawItem** - what a source adapter emitted. Title, url, content, date, source name. Nothing else known yet. - **Candidate** - a raw item resolved to its Publisher: carries `publisher_id`. - **Scored** - a candidate the curation agent approved, with its score. The nightly run never produces one: it ends at a pending candidate, and `curation.approve` builds the shape from the row the agent judged. diff --git a/docs/product.md b/docs/product.md index 5b57b1b6..ad52f541 100644 --- a/docs/product.md +++ b/docs/product.md @@ -102,11 +102,12 @@ aren't wired into the nightly run yet — candidates for whoever picks up sourci ## Under the hood, briefly - **Backend**: FastAPI (Python), auto-generates the OpenAPI spec the frontend client is - built from. + built from. One package in three layers: `entry/` (the API, the MCP server, the + commands) calls `features/`, which call `core/` (docs/adr/0017). - **Database**: Postgres with the pgvector extension — one table holds articles plus their embedding vectors, so relevance search is just a SQL query. -- **Pipeline**: a separate scheduled Python process (once a day, 04:00) that fetches - and filters, plus a Claude Code session an hour later that judges what it queued and +- **Pipeline**: a separate scheduled Python process (once a day, 04:00, + `app.entry.cli.collect`) that fetches and filters, plus a Claude Code session an hour later that judges what it queued and a report email an hour after that. A second Claude Code session drafts the newsletter on Friday mornings. - **Embeddings**: model2vec — a tiny, fast, CPU-only static embedding model, no GPU or @@ -129,4 +130,4 @@ above, and a real load/scale test (current data volume is small — this has bee architecture spike more than a performance one so far). If you want the ground-level detail — endpoints, models, exact commands — see -`backend/README.md` and `frontend/README.md`. +`docs/backend.md` and `docs/frontend.md`. diff --git a/docs/style.md b/docs/style.md index a7d9337b..43843d36 100644 --- a/docs/style.md +++ b/docs/style.md @@ -1,47 +1,64 @@ # Style -Rules a linter can enforce live in `backend/pyproject.toml` (ruff, mypy) and -`biome.json`, and fail CI. This file is the rest — style a linter cannot -express, enforced by review. +Rules a linter can enforce live in `backend/pyproject.toml` (ruff, mypy, +import-linter) and `biome.json`, and fail CI. This file holds the rest, the +style a linter cannot express, enforced by review. ## Backend -- **A route handler is an adapter.** Parse input, call one domain function, - shape the response. No SQL, no model loading, no branching business rules - in a handler body. -- **Domain functions take an explicit `session` plus plain arguments and - return domain types** — the same signature discipline as - `step(session, articles) -> survivors` in the pipeline. No hidden - globals. +- **Three layers, imports point down.** `app/entry/` (api, mcp, cli) calls + `app/features/`, and features call `app/core/`. Features never import each + other, and api, mcp and cli never import each other. `lint-imports` + enforces this (ADR 17). +- **A feature folder uses the same file names.** `model.py` for its own + types (no tables), `service.py` for the verbs entry calls, a file named + after a `core` module for the adapter over it (`audience/email.py`), and + private helpers named after what they hold (`facets.py`, `coverage.py`). + Entry imports a feature's `service` and `model` only. +- **Tables live in `app/core/db/`, one module per table.** A query only one + feature runs stays in that feature. A rule two features need on a table + goes beside the table (`like_counts_subquery` in `like.py`). +- **Shared mechanism is flat in `app/core/`.** A flat `core` module imports + stdlib, third-party packages and other flat `core` modules, never + `core.db`. When a second feature needs something from the first, it + moves to `core/`. +- **A route handler, an MCP tool and a command are adapters.** Parse input, + call feature verbs, shape the response or set the exit code. When one call + spans two features (sign-up creates a user and subscribes them), the + handler composes them; the features don't import each other. No SQL, no + model loading, no branching business rules in their bodies. One file per + router in `entry/api/`, per tool in `entry/mcp/`, per command in + `entry/cli/`. +- **Feature verbs take an explicit `session` plus plain arguments and return + domain types.** No hidden globals. - **Comments explain why, not what.** Well-named identifiers already say what the code does; a comment earns its place only for a hidden constraint, a subtle invariant, or a workaround for a specific bug. -- **Pipeline stage modules are verb-named** (`fetch`, `filter`, `queue`, - `persist`, `enrich`) — they are stages in a funnel. **Domain modules are - noun-named** (`articles`, `likes`, `publishers`) — they are resources. - This is a deliberate, permanent difference: forcing one naming scheme on - both would make each half read wrong. +- **collect's stage modules are verb-named** (`fetch`, `filter`, `queue`), + because they are stages in a funnel. **Other modules are noun-named** + (`facets`, `publishers`, `coverage`), because they are resources. One + naming scheme for both would make each half read wrong. - **Constants live with their single consumer.** A constant read by exactly - one module is an implementation detail of that module — it does not move - to a shared file. A constant read by two or more modules is a real - interface and lives at that seam (see `SNIPPET_CAP` in - `app/core/embedding.py`, read by curation and by the newsletter, for the - two-reader case). See the constants-vs-config discussion in the refactor plan for - the reasoning. -- **`app/` holds the running service; `backend/scripts/` holds ops - entrypoints.** A module that exists to be run by hand or by `prestart.sh` — - waiting for the DB, creating the superuser, seeding — is an entrypoint, not - part of the service, and lives in `backend/scripts/`. They are still - type-checked (mypy and ty run over `backend/scripts` too); they are - deliberately outside coverage's `source`, because pytest never runs them. + one module is an implementation detail of that module and does not move + to a shared file. A constant read by two or more features is an interface + and lives in `core` (`SNIPPET_CAP` in `app/core/embedding.py`, read by + curation and by the newsletter). +- **Command-line entrypoints live in `app/entry/cli/`.** A module run by + systemd, by `prestart.sh` or by hand (the nightly collect, the daily mail, + waiting for the DB, seeding) calls feature verbs and sets the exit code. + Run it with `python -m app.entry.cli.` from `backend/`. It is + type-checked and counted in coverage like the rest of `app/`. - **Config is different from constants.** Deployment-varying, env-provided values (database URL, API keys) live in one settings object (`app/core/settings.py`). Tuning knobs compiled into the code (`CANDIDATE_CAP`, `SNIPPET_CAP`) never move there. -- **The pipeline is the exception, on purpose.** It reads `os.environ` beside - each consumer rather than through that settings object — see ADR 7. The index - of what it reads lives in `deploy/README.md`; add a row there when you add a - var. +- **Per-source knobs are the exception, on purpose.** The IMAP account, the + HN thresholds, the Tavily and GitHub keys and the proxy are read from + `os.environ` in the module that uses them (ADR 7). The index of what is read + lives in `deploy/README.md`; add a row there when you add a var. +- **Tests sit beside their module** as `test_.py`. No `tests/` + folders. Shared fixtures are in `backend/conftest.py`, model factories in + `app/core/db/factories.py` (ADR 17). ## Frontend @@ -58,7 +75,6 @@ express, enforced by review. - No module named `utils`, `helpers`, `common`, `shared`, or `misc` — see ADR 3. One documented exception: `frontend/src/lib/utils.ts`'s `cn()`, which is shadcn convention. -- A domain folder holds only `models.py`, `service.py` (or a domain-named - module), `routes.py`, `tests/` — see ADR 1. - Hide mechanism, never hide capability — see ADR 2. -- Tests colocate with the domain they test — see ADR 4. +- Tests sit beside the code they test. Backend: see ADR 17. Frontend: see + ADR 4. diff --git a/frontend/scripts/blog-posts.ts b/frontend/scripts/blog-posts.ts index 6a5d18fa..2d9d59b6 100644 --- a/frontend/scripts/blog-posts.ts +++ b/frontend/scripts/blog-posts.ts @@ -5,7 +5,7 @@ * * Only structural checks live here: what a page needs to render. The content * rules for an issue (lengths, word count, sources) live in - * backend/app/newsletter/broadcast.py, which refuses a post before it is ever + * backend/app/features/newsletter/broadcast.py, which refuses a post before it is ever * drafted. */ diff --git a/frontend/scripts/prerender-blog.ts b/frontend/scripts/prerender-blog.ts index ce1ba608..5973d57e 100644 --- a/frontend/scripts/prerender-blog.ts +++ b/frontend/scripts/prerender-blog.ts @@ -10,7 +10,7 @@ * The checks here are structural only (see blog-posts.ts): frontmatter parses, * title/description/topic present, filename is YYYY-MM-DD-slug.md, date not in * the future, slugs unique. Any violation exits non-zero and fails the build. - * The content rules for an issue live in backend/app/newsletter/broadcast.py + * The content rules for an issue live in backend/app/features/newsletter/broadcast.py * (draft_issue refuses a post that breaks them); the writing guide is * .claude/skills/newsletter/SKILL.md; the PR is the editorial review. * diff --git a/frontend/src/client/schemas.gen.ts b/frontend/src/client/schemas.gen.ts index 7f6fd0f4..cbee545c 100644 --- a/frontend/src/client/schemas.gen.ts +++ b/frontend/src/client/schemas.gen.ts @@ -322,7 +322,7 @@ export const OriginSchema = { title: 'Origin', description: `The feed's "From" filter: what the article is when that is settled (a repo, a paper, a model), otherwise who published it. Not stored; each -article falls under exactly one, see \`catalog.articles.origin_condition\`.` +article falls under exactly one, see \`catalog.rows.origin_condition\`.` } as const; export const OriginCountSchema = { diff --git a/frontend/src/client/types.gen.ts b/frontend/src/client/types.gen.ts index 0e6de889..9b947054 100644 --- a/frontend/src/client/types.gen.ts +++ b/frontend/src/client/types.gen.ts @@ -69,7 +69,7 @@ export type Message = { /** * The feed's "From" filter: what the article is when that is settled (a * repo, a paper, a model), otherwise who published it. Not stored; each - * article falls under exactly one, see `catalog.articles.origin_condition`. + * article falls under exactly one, see `catalog.rows.origin_condition`. */ export type Origin = 'lab' | 'company' | 'individual' | 'media' | 'repo' | 'paper' | 'model'; diff --git a/scripts/restore-db.sh b/scripts/restore-db.sh index d0633e4b..d0b7ec80 100755 --- a/scripts/restore-db.sh +++ b/scripts/restore-db.sh @@ -18,7 +18,7 @@ set +a dump=$(ls -1 .ignored/dumps/agentique-db-*.sql.gz 2>/dev/null | sort | tail -1) if [ -z "$dump" ]; then - echo "no dump in .ignored/dumps/ — drop one there or run backend/scripts/seed_articles.py" >&2 + echo "no dump in .ignored/dumps/ — drop one there or run python -m app.entry.cli.seed_articles in backend/" >&2 exit 1 fi diff --git a/backend/scripts/test.sh b/scripts/test.sh similarity index 100% rename from backend/scripts/test.sh rename to scripts/test.sh diff --git a/scripts/tests-start.sh b/scripts/tests-start.sh new file mode 100755 index 00000000..02022138 --- /dev/null +++ b/scripts/tests-start.sh @@ -0,0 +1,7 @@ +#! /usr/bin/env bash +set -e +set -x + +python -m app.entry.cli.wait_for_db + +bash "$(dirname "$0")/test.sh" "$@" diff --git a/uv.lock b/uv.lock index 2489d12a..87093778 100644 --- a/uv.lock +++ b/uv.lock @@ -108,6 +108,7 @@ dependencies = [ [package.dev-dependencies] dev = [ { name = "coverage" }, + { name = "import-linter" }, { name = "mypy" }, { name = "pytest" }, { name = "ruff" }, @@ -117,7 +118,7 @@ dev = [ [package.metadata] requires-dist = [ - { name = "alembic", specifier = ">=1.12.1,<2.0.0" }, + { name = "alembic", specifier = ">=1.18.0,<2.0.0" }, { name = "baml-py", specifier = ">=0.223.0" }, { name = "email-validator", specifier = ">=2.1.0.post1,<3.0.0.0" }, { name = "emails", specifier = ">=1.1.2,<2.0" }, @@ -146,6 +147,7 @@ requires-dist = [ [package.metadata.requires-dev] dev = [ { name = "coverage", specifier = ">=7.4.3,<8.0.0" }, + { name = "import-linter", specifier = ">=2.15,<3.0.0" }, { name = "mypy", specifier = ">=1.8.0,<3.0.0" }, { name = "pytest", specifier = ">=7.4.3,<10.0.0" }, { name = "ruff", specifier = ">=0.2.2,<1.0.0" }, @@ -909,6 +911,71 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/41/63/e876e789525063c840ccfa8857febdabd6523bcef9ce7eb979b9305ea895/griffelib-2.3.0-py3-none-any.whl", hash = "sha256:1b8f9cd525681c26b1d6d574faa1371651e8459ca51d209684f50b8096ae06e0", size = 169423, upload-time = "2026-09-04T15:08:12.956Z" }, ] +[[package]] +name = "grimp" +version = "3.17" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ad/2e/95736b7c984705ff5ee376f07c6e71bb80fcbb22c8fa3b8c1cfbfa919d5c/grimp-3.17.tar.gz", hash = "sha256:5161e03c6f518fe4da43b06c97ff49a1e224ecb0e4b9fd6ef48235373d1b7b33", size = 832923, upload-time = "2026-09-04T11:28:54.609Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/10/7a/b1c76982a239347bfccd2ae95b9f79d3390cd14170f62e7eb822a17d786d/grimp-3.17-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:eb8f09da4302f05fc92fe7564f27731c0b43789d8893ae5e6be5df2b01668923", size = 2145452, upload-time = "2026-09-04T11:28:12.121Z" }, + { url = "https://files.pythonhosted.org/packages/cb/ab/2a1a9574b391c0ee62cb33a633c0d377dce1b7d19d04a7ac5d5e6872d395/grimp-3.17-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:0f11566bcf388db4c79f1c7f1051f6ef150f778306350bb7d86d0c9aab32d297", size = 2091654, upload-time = "2026-09-04T11:28:04.635Z" }, + { url = "https://files.pythonhosted.org/packages/e6/12/7dc086132ac10a2409f158f0b0d7cc528a2ebe0a4e95bca895afe7452fa6/grimp-3.17-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bfda3e2ebb4e51beec4a3cb09bb771440f3192163107d9320e6d9ce8fe064e64", size = 2277300, upload-time = "2026-09-04T11:27:01.844Z" }, + { url = "https://files.pythonhosted.org/packages/5b/62/2e16a54580fa76fb7e64e8e8e2ebb899c709a764cb29149a88eb03f2eea7/grimp-3.17-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:b0ab1ab71d807fc6f56d82f1ab24acf2f2e0e5000e37b5540af9449e28c6830c", size = 2220417, upload-time = "2026-09-04T11:27:11.418Z" }, + { url = "https://files.pythonhosted.org/packages/73/c0/e1b0ad0d0b3d9dff996bad306542f3cf000cb86c368bfe3ac6a8bb1e2e2f/grimp-3.17-cp313-cp313-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:187b092664e882d10b9f2c3da99251d0aeb99186d407ed5d20f52005832045f6", size = 2370754, upload-time = "2026-09-04T11:27:39.995Z" }, + { url = "https://files.pythonhosted.org/packages/36/0f/015e48ad8029f788eaa3df71b66859b2b5dc63abb8d7a2014a2b5a581040/grimp-3.17-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:75da770aeee97d26b202d7251fe7f6562e7f2793e348501b0a64ca5c50e881b5", size = 2628257, upload-time = "2026-09-04T11:27:20.491Z" }, + { url = "https://files.pythonhosted.org/packages/eb/d3/66da78504a1731237ed8befd9c66fb91037a7b82816443ca772107acf8fe/grimp-3.17-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f5cc64f891f71f538d3d032d6e7193f69c3a2e0ceac72c55114faee5aa161f9f", size = 2347569, upload-time = "2026-09-04T11:27:30.282Z" }, + { url = "https://files.pythonhosted.org/packages/cb/a2/6474d008fa49cc650d626798cb3b6e3fa7b0daaad24884fd6bb4288f6154/grimp-3.17-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6dbf2d93b0ba7f6c1efba9f5b3e07a3da0a157392c650316a5c1478b0ef7e503", size = 2284227, upload-time = "2026-09-04T11:27:52.462Z" }, + { url = "https://files.pythonhosted.org/packages/c5/b5/91cd378c08494fd0130e5cc2e54d60d12f53b21850e0dbccf668b6e73d91/grimp-3.17-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:7a22b5dcbf55f1c348885e09cbe91e63baedf6ce0f89c60bb01a0475c7d0a90b", size = 2454655, upload-time = "2026-09-04T11:28:19.837Z" }, + { url = "https://files.pythonhosted.org/packages/f1/d0/b582238247ce51857c7ac770c3be10203c992696f3dc9153f1a979c011dc/grimp-3.17-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:73aa9459a604554b1dc1618148312036679f3974bf8b0285588220307147060b", size = 2495852, upload-time = "2026-09-04T11:28:29.606Z" }, + { url = "https://files.pythonhosted.org/packages/8f/99/e0690b02be70f346aadfe2aa8bd7e496c773298ac58c0d0fa6376c1298b2/grimp-3.17-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:c31660d30e79569faac05033637c1eef4595542f44ec2323af54e900030164a6", size = 2527887, upload-time = "2026-09-04T11:28:38.828Z" }, + { url = "https://files.pythonhosted.org/packages/69/7a/33e41d9b08e63cf398d8194a9208b21788c76916b5ac6d90146e8e2ebf95/grimp-3.17-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:8f395c5ab2cf554504f46680b9269b769939ab5d7ea58f93e5c341e55b1affbd", size = 2539000, upload-time = "2026-09-04T11:28:49.037Z" }, + { url = "https://files.pythonhosted.org/packages/69/f5/7af1dd74b9608a85274414fc423a7162844492b198ece3fe31f4e1f88237/grimp-3.17-cp313-cp313-win32.whl", hash = "sha256:34369739ed293accbb674494ae932351516b316e8e85d1d333d4c51796af846a", size = 1874244, upload-time = "2026-09-04T11:29:21.693Z" }, + { url = "https://files.pythonhosted.org/packages/8b/d4/af497adddfcf11f9bd4ab273625a49022bf5ba4f5fc389589f4e6edc67b9/grimp-3.17-cp313-cp313-win_amd64.whl", hash = "sha256:d21b859c50418bdb48b403a84450650fa7aab3102f3212d00d3c21c76a982167", size = 1998225, upload-time = "2026-09-04T11:29:10.5Z" }, + { url = "https://files.pythonhosted.org/packages/e2/2a/07fd537997281049af2390fc8641de5ae063ae4972cb6fcec85873674595/grimp-3.17-cp313-cp313-win_arm64.whl", hash = "sha256:206cd9f7c757b97934f48ba2b8ec6d649725fd6314d6b182dd327b19b4f35707", size = 1922071, upload-time = "2026-09-04T11:28:58.34Z" }, + { url = "https://files.pythonhosted.org/packages/39/e0/e11ea5aeb62534c9706e17952d142aa844ce5659e2d66b4b8ba9377407d6/grimp-3.17-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:0100ff6f08067c8eb8a77dd1433af5e173aea8dbf6a6b76f5372a3b49f2e0a43", size = 2161696, upload-time = "2026-09-11T10:36:49.104Z" }, + { url = "https://files.pythonhosted.org/packages/4b/d6/ddf82f4428d8bdc54ced0db244cbf5d8e5638490c3ed6a8b9f5db55dfa51/grimp-3.17-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:67ada40a5c1cf4321183c0b0cd0393658ccd04b845caed002ac81dc2fdef828e", size = 2107301, upload-time = "2026-09-11T10:36:46.459Z" }, + { url = "https://files.pythonhosted.org/packages/86/1a/1dd0d9d2751e8078547e1cb7131e13d9624a2ea60700d926b3d624c15825/grimp-3.17-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:baa385915a4c17948bfb25f77509cbbfb33795f321d1c5ca7cc602e34ed58e99", size = 2278433, upload-time = "2026-09-04T11:27:03.075Z" }, + { url = "https://files.pythonhosted.org/packages/58/71/3768fd253e7455a83d668193055691c82d3a4d434047116c572a27d2bd39/grimp-3.17-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:8322a435b5ee97ff3db823df36cbefea569faec505cc0ca66223c27b4b44cca6", size = 2221152, upload-time = "2026-09-04T11:27:12.642Z" }, + { url = "https://files.pythonhosted.org/packages/b4/4f/907a130e87554d57be14c2a7cedeb818d0ef519064877297d19636735f6b/grimp-3.17-cp314-cp314-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:cb953c21beb6ac85a59b5b54128be7fdddd89396d34b3c5705cf59829a85b431", size = 2371546, upload-time = "2026-09-04T11:27:41.195Z" }, + { url = "https://files.pythonhosted.org/packages/7e/4f/a3bf534aa7c582a47d8cfefab5fccb50e6f4dcd327ed6a8552d344a04ffe/grimp-3.17-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c34b904c25359f98e3f20771256d62329ed17f1e26a254a4f5c74bacc1f3bd21", size = 2628467, upload-time = "2026-09-04T11:27:21.818Z" }, + { url = "https://files.pythonhosted.org/packages/32/d3/2b2ab7b3a4e2c17bb1e10ec37e7ee8eecce7f72b1bf6c8019dd439943884/grimp-3.17-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f69ce8be301299f15ca99843e89991c240a231836cf45a11c6bfc54fb9fbeaa9", size = 2347520, upload-time = "2026-09-04T11:27:31.73Z" }, + { url = "https://files.pythonhosted.org/packages/72/da/6bc1f783a739afa51ffa2e235791487b5d09202501b25843ecae9833a3ff/grimp-3.17-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:566411d1ac97e1f3a3f00a13ae33aebb9abda9787d3d9914f62f73b589d84fe2", size = 2284007, upload-time = "2026-09-04T11:27:53.889Z" }, + { url = "https://files.pythonhosted.org/packages/63/06/a08f1b16aec11f32bbc86f60e24986659ab4cfab79388c8d4b79f6a6723e/grimp-3.17-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:5f89948029f3c7339a20bab007977194d3dbdddb27edae5c74b948d8f13191c7", size = 2455301, upload-time = "2026-09-04T11:28:21.177Z" }, + { url = "https://files.pythonhosted.org/packages/ff/c1/d55e9686f3797d1a2c6403d2ffec3020c67591c004304a414fe8d7a9755c/grimp-3.17-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:5a895bc27105ada9b0e76a80bf9cc898409e5cc3689e73e64ca1cbd71dc87dce", size = 2497158, upload-time = "2026-09-04T11:28:30.984Z" }, + { url = "https://files.pythonhosted.org/packages/7f/85/a2d2e70c4f272752c395cc35b3732d82ff92c55e229d7bbd6777fd9868ff/grimp-3.17-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:e80b57a7d03f53e96eb0a5b0844340f99f17eede184042fb73e3ef010176ab35", size = 2528862, upload-time = "2026-09-04T11:28:40.251Z" }, + { url = "https://files.pythonhosted.org/packages/cb/03/fc3a58d9b22b011c1c3377c1ecee127a5f642de9b20de32d3ad7213ab71a/grimp-3.17-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5c0cac76283679fe9c1fd761e63deb9bbc7ba46b32988113ecef4b13742f31b9", size = 2538240, upload-time = "2026-09-04T11:28:50.248Z" }, + { url = "https://files.pythonhosted.org/packages/58/13/d7666cf03c79c275020bad40ea1df8597af7877083ef5a84b5d8b566029a/grimp-3.17-cp314-cp314-win32.whl", hash = "sha256:c8982f4fa1276e059665947156740a55a4e8e6b5a4d7f40730a7511afb608762", size = 1874930, upload-time = "2026-09-04T11:29:23.152Z" }, + { url = "https://files.pythonhosted.org/packages/59/ce/abb9ccb50972351461065ca72cd0f528837a2f398621147dfaf1013698f0/grimp-3.17-cp314-cp314-win_amd64.whl", hash = "sha256:0b97dc29bdca523932bbbcb425fc3df3547cf970396b293bfa960b910f3932d1", size = 1999405, upload-time = "2026-09-04T11:29:11.832Z" }, + { url = "https://files.pythonhosted.org/packages/9f/1d/93c12ff4ce9ede9f91aa03df66128d223a4c824a13de6ab702c973fcae0d/grimp-3.17-cp314-cp314-win_arm64.whl", hash = "sha256:80b5b25b8d8ed6cc68dc39ae9e4c2fb64acd1a3e66434d9cf4d68c72e46d505d", size = 1923324, upload-time = "2026-09-04T11:29:00.004Z" }, + { url = "https://files.pythonhosted.org/packages/d2/59/f94dd90757a2195415736f0e2be667ac4cd375fe86d875d4298661751fce/grimp-3.17-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:e15340ee11e34bfaaee496cff53f8287dc9f417a94e5bdf3d6b4204305d004cc", size = 2145199, upload-time = "2026-09-04T11:28:13.287Z" }, + { url = "https://files.pythonhosted.org/packages/ce/83/9d7cb22fc136ad6a61bff3b74cf0b31b13fdf94e224d46ff1edd9b5b8c86/grimp-3.17-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:cbccf2a6cd3f89bfdfd683d9b000bd61f6936e16b6d4d6619da592c54f79f272", size = 2090761, upload-time = "2026-09-04T11:28:05.984Z" }, + { url = "https://files.pythonhosted.org/packages/e2/6b/b70add79e9e315d111bb1643b44e1b9e1ded82888856e1ad68bc484cd635/grimp-3.17-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5332893955688759a9f7868c3c9e56b19bc1694ee251852d3daf54876b35c115", size = 2279005, upload-time = "2026-09-04T11:27:04.543Z" }, + { url = "https://files.pythonhosted.org/packages/d3/4c/f5f001e592f8e8820086604db98373ca129e2831facad81ec0d1820a749f/grimp-3.17-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:848a92d8a63a38d79c89ab1e68fb5429d90079f39c07533ffd45ce756e7f210f", size = 2220430, upload-time = "2026-09-04T11:27:13.814Z" }, + { url = "https://files.pythonhosted.org/packages/83/13/cc57724307c22060e193a1dc9bc99fa14d46584c534a3936319ec579c599/grimp-3.17-cp314-cp314t-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c89cfe49f822a2cc9c42cc7b0f3975ecb1281970735ea31610bfa06a7e04c460", size = 2370318, upload-time = "2026-09-04T11:27:42.535Z" }, + { url = "https://files.pythonhosted.org/packages/50/1b/05d9964d4b47ffe0ef826bfe04b3f4b713fd623c90e424ff002cb0d8a917/grimp-3.17-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:541c9a2b7f1ad5d4f5d1597740e51058fba8ca0c1a225d6ccc6d11a8364fa563", size = 2629127, upload-time = "2026-09-04T11:27:23.18Z" }, + { url = "https://files.pythonhosted.org/packages/ce/ea/186a3087623f1556eb8b9d02baa00300003bb84b7b35677e3fcec34691b6/grimp-3.17-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:66b4a13edb3fe63be6f7dbbc2576786f501af9d386a5a1f60b2938f270aef97c", size = 2348453, upload-time = "2026-09-04T11:27:32.964Z" }, + { url = "https://files.pythonhosted.org/packages/e8/6c/a718328adc50529135510b515c5ff1d9f94ab26ab8df9057bb1ef5ab8993/grimp-3.17-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:160abad142e168545be8d4cc91b2a95b4565f7571fc56f7fced69ce3589e5111", size = 2287338, upload-time = "2026-09-04T11:27:55.16Z" }, + { url = "https://files.pythonhosted.org/packages/ea/ed/bf67bc6749abee0a1e97f8f14ee2cb19ccc435b306e22bcaac6118d402a4/grimp-3.17-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:cd76fb13a534209c1777c807092f51b1570854eb49948d65cd5e1d94d80e981d", size = 2455299, upload-time = "2026-09-04T11:28:22.505Z" }, + { url = "https://files.pythonhosted.org/packages/83/8a/6c954092306e21c93c744432c3d47491c3264d3cc4b465941de5164a1cb3/grimp-3.17-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:1688e2dbe382b6644ecc3f55053e319de70034d9dfe0809d21d441a590c320e6", size = 2496813, upload-time = "2026-09-04T11:28:32.303Z" }, + { url = "https://files.pythonhosted.org/packages/ea/a8/cf0aa245ace55dd2f6bdd3bb646ec50c294bdfe623fd51327bd27efaa585/grimp-3.17-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:0433d177f327c31410f21845322282c832d887f1a42cbdfa176133d8422d84b2", size = 2527052, upload-time = "2026-09-04T11:28:41.869Z" }, + { url = "https://files.pythonhosted.org/packages/75/b8/90cd11e346f12fb5269ff050ae825083410d3bdb54d82a3b8e955e4f9d26/grimp-3.17-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:790d68157e034e7848257ef46b37550040c9e4692cea17372d3d9f87cef1d84b", size = 2541238, upload-time = "2026-09-04T11:28:51.786Z" }, + { url = "https://files.pythonhosted.org/packages/48/92/10cc37db6149030c8cfc80fa5287f739c1e9620c8a05d1ba8dfdfc0a6c4f/grimp-3.17-cp314-cp314t-win32.whl", hash = "sha256:37e6333ca54d4d80b5c4f2eae324774b223dbdc65994dc9bccda2985a9261dd8", size = 1872673, upload-time = "2026-09-04T11:29:24.514Z" }, + { url = "https://files.pythonhosted.org/packages/ac/cc/b257d8da079a87e65d0f1b0e004fd50f31ba25e7136a3b36f896e0e338f8/grimp-3.17-cp314-cp314t-win_amd64.whl", hash = "sha256:91e4bbbed4d934369b83071380e1ad5f37b7f5c80b80ffa924627db8d66fea11", size = 2000197, upload-time = "2026-09-04T11:29:13.138Z" }, + { url = "https://files.pythonhosted.org/packages/29/d0/feebddf766448932f2f1e40324fe75cf50ce6d3a21bdf87fc0eae263218a/grimp-3.17-cp314-cp314t-win_arm64.whl", hash = "sha256:28ae2ff57a9ef535c2e7054dcb18baf0cdc2dee3982367512bff34eaecb85cf0", size = 1920856, upload-time = "2026-09-04T11:29:01.498Z" }, + { url = "https://files.pythonhosted.org/packages/83/39/bb07084f0b97da66b0aa38077bd656c4f831b4aa3f167b1c060f6a65ea5e/grimp-3.17-cp315-cp315-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:fe3e2285a2bd4e62ddb8825de37152d2dc842394d419786d3d5a803b07bb84eb", size = 2372594, upload-time = "2026-09-04T11:27:43.884Z" }, + { url = "https://files.pythonhosted.org/packages/e6/a9/905a484659749f40ed797963147202cb74c3b68d8e90a7a600af55de47ee/grimp-3.17-cp315-cp315-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a377d5935ae9a62ccbb59e492e16624cec97cc7244c4a964b0c6f83940440626", size = 2284577, upload-time = "2026-09-04T11:27:56.428Z" }, + { url = "https://files.pythonhosted.org/packages/30/6a/40fb12b111c95b94db4930f35661a629532d9d12fb484db5b5ef1a6c3da3/grimp-3.17-cp315-cp315-win32.whl", hash = "sha256:65a1dc2741bef8a96ff3ff71252db8e99eac2a32e09d5e2b63597474706186ca", size = 1876437, upload-time = "2026-09-04T11:29:25.725Z" }, + { url = "https://files.pythonhosted.org/packages/71/98/79af8d51961611e13b3dabc7a3b356b27aa696e5549bb439f1d6002be31a/grimp-3.17-cp315-cp315-win_amd64.whl", hash = "sha256:d20fbe5aab8b013b351313f8eaa3a46f4ac57677e357cb1c3d1012c606b565b4", size = 1999947, upload-time = "2026-09-04T11:29:14.623Z" }, + { url = "https://files.pythonhosted.org/packages/96/d6/e1922c4c7e001210f98e0cf89bd76fea8564f590be3e9ccfe99435cfcd59/grimp-3.17-cp315-cp315-win_arm64.whl", hash = "sha256:a0288dc0e8947dead5c02a246e16f3f5e5cb486348171bdf1289885a13de36c1", size = 1923774, upload-time = "2026-09-04T11:29:02.897Z" }, + { url = "https://files.pythonhosted.org/packages/85/f6/f4f84e44a9237350c41e7c77343498cb1ad843491c18f29d6811021c617d/grimp-3.17-cp315-cp315t-macosx_10_12_x86_64.whl", hash = "sha256:1bf67cb05a80796be99d1bf87e12d4421de101950c5246a8220b6c839cbf27c2", size = 2145245, upload-time = "2026-09-04T11:28:14.537Z" }, + { url = "https://files.pythonhosted.org/packages/71/98/822287270c41360f66c4b5638e040afc6bbf11e9f04cf5005bbd5f3e4830/grimp-3.17-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:d058fdb980475eb9e0143792d63757ddff86c5776e792866e8208cfe5e8413f9", size = 2090367, upload-time = "2026-09-04T11:28:07.277Z" }, + { url = "https://files.pythonhosted.org/packages/ac/76/32d303603157a0d98cb8bde8757e6ae8fa02d41f1d62cdf843fb2e7fe727/grimp-3.17-cp315-cp315t-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:92f1989ce9f9ce26b7d45381df164c62aff783add84cd7a4216a6ebba0f9f03a", size = 2371543, upload-time = "2026-09-04T11:27:45.108Z" }, + { url = "https://files.pythonhosted.org/packages/7b/68/4551e23ab9422e2ead806c1fec9f4b3acd21a267e43bfe8e8cd7f4640dce/grimp-3.17-cp315-cp315t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e275cee07826cf36db1914427e68afac87588f759e1dec676870234d4a250ae", size = 2288003, upload-time = "2026-09-04T11:27:57.605Z" }, + { url = "https://files.pythonhosted.org/packages/20/64/79161f05c900f7f7f1b0acd22a730b1be9f96680dd44efeef90aa45e6998/grimp-3.17-cp315-cp315t-win32.whl", hash = "sha256:a7235ccffb0350647ae2b01969f552a9859fea7027032f6cfed110801269a439", size = 1873249, upload-time = "2026-09-04T11:29:27.094Z" }, + { url = "https://files.pythonhosted.org/packages/94/b7/6f9cdeba2c1263fb91b791ddd5e924d6c91c2485f4f16b56698802fdd540/grimp-3.17-cp315-cp315t-win_amd64.whl", hash = "sha256:473d0e0c016ca429b9ec17f34d276db454ba4d467e3c915a46279d86ab004716", size = 2000579, upload-time = "2026-09-04T11:29:15.961Z" }, + { url = "https://files.pythonhosted.org/packages/ac/16/8ae9028f046bf8f48cc13f6de2de40adbcbf38fa79a3a5f265753b248da0/grimp-3.17-cp315-cp315t-win_arm64.whl", hash = "sha256:478aa0b696e5dd79373943e75b998fbb4ef3d146f24fe3569fa9d106ce84991b", size = 1921829, upload-time = "2026-09-04T11:29:04.403Z" }, +] + [[package]] name = "h11" version = "0.16.0" @@ -1099,6 +1166,21 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/19/a8/0f58c13d2660d5fc8f808ce8b46828d2941752ec21e4015bde99c08b37d7/imap_tools-1.13.0-py3-none-any.whl", hash = "sha256:656c37beba22ab2929b73c07d0ca397ae8805b670d390b1127723e3335244e6d", size = 35849, upload-time = "2026-05-12T07:14:52.669Z" }, ] +[[package]] +name = "import-linter" +version = "2.15" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "click" }, + { name = "grimp" }, + { name = "rich" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/3c/81/1bb414f8c8bcd024499e91f112d317c36a86bc264da1fd43f9797fc2b7b0/import_linter-2.15.tar.gz", hash = "sha256:1da912bea5e172a82a3ce617b5543f75cf64dc0d8f4d9b46c5578b68ccb81590", size = 1405973, upload-time = "2026-09-04T14:47:56.097Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/08/31/4571c20652a4002ed9e500046386f3731a77e23d95a608ae8a87f1fbb2da/import_linter-2.15-py3-none-any.whl", hash = "sha256:9aaf16a88ac1e99d5a464cd7f66b6a05f7060bfa761162e0ed441773a267ed3b", size = 640512, upload-time = "2026-09-04T14:47:54.878Z" }, +] + [[package]] name = "iniconfig" version = "2.3.0"