ci: enforce backend import layers

.env.dev

@@ -28,6 +28,6 @@ MCP_WRITE_TOKEN=agentique-dev-write
 MCP_DB_USER=postgres
 MCP_DB_PASSWORD=agentique-dev
 
-# Jev (pipeline/jev.py), the newsletter link classifier. Empty locally: set a
+# Jev (backend/app/core/jev.py), the newsletter link classifier. Empty locally: set a
 # real key only in .env, never here.
 OPENROUTER_API_KEY=

.github/workflows/claude-code-review.yml

@@ -34,7 +34,7 @@ jobs:
 
       - name: Run Claude Code Review
         id: claude-review
-        uses: anthropics/claude-code-action@f27be6b65a4652f08a2ba3cff79114829b588967 # v1
+        uses: anthropics/claude-code-action@f27be6b65a4652f08a2ba3cff79114829b588967 # v1.0.252
         with:
           claude_code_oauth_token: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }}
           plugin_marketplaces: 'https://github.com/anthropics/claude-code.git'

.github/workflows/claude.yml

@@ -33,7 +33,7 @@ jobs:
 
       - name: Run Claude Code
         id: claude
-        uses: anthropics/claude-code-action@f27be6b65a4652f08a2ba3cff79114829b588967 # v1
+        uses: anthropics/claude-code-action@f27be6b65a4652f08a2ba3cff79114829b588967 # v1.0.252
         with:
           claude_code_oauth_token: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }}
 

.github/workflows/deploy-production.yml

@@ -49,25 +49,21 @@ jobs:
         uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0
         with:
           version: "0.11.18"
+      # Straight into frontend/dist, so the one sync below ships it.
       - name: Download dist
         uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
         with:
           name: frontend-dist
-          path: frontend-dist
+          path: frontend/dist
       # The workspace is a staging area only — actions/checkout resets it every
-      # run, so the services live in /opt/agentique. .env and .venv belong to
-      # the box; frontend/dist is replaced separately below.
-      - name: Sync source to /opt/agentique
-        run: >
-          rsync -a --delete
-          --exclude '.git'
-          --exclude '.env'
-          --exclude '.venv'
-          --exclude 'frontend/dist'
-          --exclude 'frontend-dist'
-          ./ /opt/agentique/
-      - name: Sync frontend dist
-        run: rsync -a --delete frontend-dist/ /opt/agentique/frontend/dist/
+      # run, so the services live in /opt/agentique. deploy/rsync-filter lists
+      # what the box runs; --delete-excluded removes everything else there,
+      # except what the filter protects (.env, .venv, .git). A missing dist
+      # would delete the live site, so check for it first.
+      - name: Sync to /opt/agentique
+        run: |
+          test -f frontend/dist/index.html
+          rsync -a --delete --delete-excluded --filter='merge deploy/rsync-filter' ./ /opt/agentique/
       # --package app: the workspace root is virtual, a bare sync would not
       # install the backend (same flag the old Dockerfile used).
       - name: Sync venv

.github/workflows/test-backend.yml

@@ -39,6 +39,9 @@ jobs:
           POSTGRES_USER=postgres
           POSTGRES_PASSWORD=ci-test-password
           EOF
+      - name: Check import layers
+        run: uv run lint-imports
+        working-directory: backend
       - run: docker compose up -d
       - name: Migrate DB
         run: uv run bash app/entry/cli/prestart.sh

.pre-commit-config.yaml

@@ -53,14 +53,21 @@ repos:
 
       - id: local-mypy
         name: mypy check
-        entry: uv run mypy backend/app
+        entry: uv run mypy --config-file backend/pyproject.toml backend/app
         require_serial: true
         language: unsupported
         pass_filenames: false
 
       - id: local-ty
         name: ty check
-        entry: uv run ty check backend/app
+        entry: uv run ty check --project backend backend/app
+        require_serial: true
+        language: unsupported
+        pass_filenames: false
+
+      - id: local-lint-imports
+        name: import-linter
+        entry: uv run lint-imports --config backend/pyproject.toml
         require_serial: true
         language: unsupported
         pass_filenames: false

backend/app/core/db/story.py

@@ -41,7 +41,7 @@ class Story(SQLModel, table=True):
 
 class StoryArticle(SQLModel, table=True):
     """An article in a story. At most one open story per article, enforced in
-    `catalog.stories.save_story`: a closed story keeps its articles."""
+    `stories.service.save_story`: a closed story keeps its articles."""
 
     __tablename__ = "story_article"
     # The primary key leads with story_id; "which story holds this article" is

backend/app/core/jev.py

@@ -107,6 +107,14 @@ def _answers(body: dict[str, Json], questions: dict[str, Question]) -> dict[str,
     missing = [qid for qid, a in answers.items() if a is None]
     if missing:
         raise JevError(f"unanswered: {', '.join(missing)}")
+    # "0.93" as a string would read as 0 the same way.
+    odd = [
+        qid
+        for qid, q in questions.items()
+        if q["type"] == "noul" and not isinstance(answers[qid], int | float)
+    ]
+    if odd:
+        raise JevError(f"non-numeric: {', '.join(odd)}")
     return answers
 
 

backend/app/core/test_jev.py

@@ -1,6 +1,6 @@
 """The Jev transport's two guarantees: a 429 waits as long as it is told to
-and tries again, and an answer set with a hole in it is an error, never a
-quiet zero that reads as "not a sponsor" or "off topic".
+and tries again, and an answer set with a hole or a non-numeric probability in
+it is an error, never a quiet zero that reads as "not a sponsor" or "off topic".
 """
 
 from __future__ import annotations
@@ -107,6 +107,16 @@ def test_a_missing_answer_raises_instead_of_reading_as_zero(
         jev.ask({}, QUESTIONS)
 
 
+def test_a_non_numeric_probability_raises_instead_of_reading_as_zero(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    body = {"answers": {"is_sponsor": {"noul": "0.93"}, "kind": "article"}}
+    _stub(monkeypatch, [_resp(200, body)])
+
+    with pytest.raises(jev.JevError, match="is_sponsor"):
+        jev.ask({}, QUESTIONS)
+
+
 def test_a_response_with_no_answers_raises(monkeypatch: pytest.MonkeyPatch) -> None:
     _stub(monkeypatch, [_resp(200, {"error": "nope"})])
 

backend/app/entry/cli/test_deploy_files.py

@@ -0,0 +1,155 @@
+"""What the deploy ships: the workflow's rsync run against this checkout's
+tracked files, then every path the units and the deploy job use looked up in
+the result."""
+
+from __future__ import annotations
+
+import re
+import shlex
+import shutil
+import subprocess
+import sys
+import tomllib
+from pathlib import Path
+
+import pytest
+
+BACKEND = Path(__file__).resolve().parents[3]
+REPO = BACKEND.parent
+BOX = Path("/opt/agentique")
+UNITS = sorted((REPO / "deploy").glob("*.service"))
+WORKFLOW = REPO / ".github/workflows/deploy-production.yml"
+# Never shipped: the box keeps its own copy, and the filter protects it.
+BOX_OWNED = (BOX / ".env", BOX / ".venv")
+
+
+def rsync_command() -> list[str]:
+    lines = [
+        line.strip()
+        for line in WORKFLOW.read_text().splitlines()
+        if line.strip().startswith("rsync ")
+    ]
+    assert len(lines) == 1, lines
+    return shlex.split(lines[0])
+
+
+@pytest.fixture(scope="module")
+def shipped(tmp_path_factory: pytest.TempPathFactory) -> Path:
+    version = subprocess.run(
+        ["rsync", "--version"], capture_output=True, text=True, check=False
+    ).stdout
+    if "version 3." not in version:
+        pytest.skip("the filter needs rsync 3, as on the box")
+    # The deploy syncs a clean checkout, so untracked files must not count.
+    source = tmp_path_factory.mktemp("checkout")
+    tracked = subprocess.run(
+        ["git", "ls-files", "-z"], cwd=REPO, capture_output=True, check=True
+    ).stdout.decode()
+    for name in filter(None, tracked.split("\0")):
+        if (REPO / name).is_file():
+            (source / name).parent.mkdir(parents=True, exist_ok=True)
+            shutil.copy2(REPO / name, source / name)
+    (source / "frontend/dist").mkdir(parents=True)
+    (source / "frontend/dist/index.html").touch()
+
+    dest = tmp_path_factory.mktemp("box")
+    command = rsync_command()
+    assert command[-1] == f"{BOX}/", command
+    subprocess.run([*command[:-1], f"{dest}/"], cwd=source, check=True)
+    return dest
+
+
+def on_box(shipped: Path, path: Path) -> Path:
+    return shipped / path.relative_to(BOX)
+
+
+def box_owned(path: Path) -> bool:
+    return any(path == owned or owned in path.parents for owned in BOX_OWNED)
+
+
+@pytest.mark.parametrize("unit", UNITS, ids=lambda p: p.name)
+def test_every_path_a_unit_uses_is_shipped(shipped: Path, unit: Path) -> None:
+    config = dict(
+        line.split("=", 1)
+        for line in unit.read_text().splitlines()
+        if "=" in line and not line.startswith("#")
+    )
+    workdir = on_box(shipped, Path(config["WorkingDirectory"]))
+    assert workdir.is_dir(), workdir
+    assert box_owned(Path(config["EnvironmentFile"]))
+    # systemd's -, @, + and ! prefixes change how it runs, not what.
+    command, *args = config["ExecStart"].lstrip("-@+!:").split()
+    assert box_owned(Path(command)), command
+    if args[:1] == ["-m"]:
+        module = workdir / args[1].replace(".", "/")
+        assert module.with_suffix(".py").is_file() or (module / "__main__.py").is_file()
+    for arg in args:
+        if arg.endswith(".py"):
+            assert (workdir / arg).is_file(), arg
+
+
+def test_every_directory_the_deploy_job_works_in_is_shipped(shipped: Path) -> None:
+    dirs = re.findall(r"working-directory: (/opt/agentique\S*)", WORKFLOW.read_text())
+    assert dirs
+    for workdir in dirs:
+        assert on_box(shipped, Path(workdir)).is_dir(), workdir
+    for script in re.findall(r"bash (\S+\.sh)", WORKFLOW.read_text()):
+        assert (on_box(shipped, BOX / "backend") / script).is_file(), script
+
+
+def test_prestart_runs_only_shipped_modules(shipped: Path) -> None:
+    prestart = shipped / "backend/app/entry/cli/prestart.sh"
+    for module in re.findall(r"python -m ([\w.]+)", prestart.read_text()):
+        assert (shipped / "backend" / (module.replace(".", "/") + ".py")).is_file()
+
+
+def test_uv_sync_and_alembic_find_their_files(shipped: Path) -> None:
+    root = tomllib.loads((shipped / "pyproject.toml").read_text())
+    for member in root["tool"]["uv"]["workspace"]["members"]:
+        assert (shipped / member / "pyproject.toml").is_file(), member
+    assert (shipped / "uv.lock").is_file()
+    assert (shipped / ".python-version").is_file()
+    assert (shipped / "backend/migrations/env.py").is_file()
+    # backup_db.py runs docker compose against the repo root's compose.yml.
+    assert (shipped / "compose.yml").is_file()
+
+
+def test_every_data_file_in_app_is_shipped(shipped: Path) -> None:
+    data = [
+        p.relative_to(REPO)
+        for p in (BACKEND / "app").rglob("*")
+        if p.is_file()
+        and p.suffix not in {".py", ".pyc"}
+        and not p.name.startswith("test_")
+    ]
+    assert data
+    for path in data:
+        assert (shipped / path).is_file(), path
+
+
+def test_no_test_code_is_shipped(shipped: Path) -> None:
+    names = {p.name for p in (shipped / "backend").rglob("*.py")}
+    assert not [n for n in names if n.startswith("test_")]
+    assert "conftest.py" not in names
+    assert "factories.py" not in names
+
+
+def test_every_shipped_module_imports_from_the_shipped_tree(shipped: Path) -> None:
+    # Run from the shipped backend/, as the units do. The venv's editable
+    # install still points at the checkout, so a module the filter drops would
+    # load from there instead. Fail on any module that did.
+    env_file = REPO / ".env"
+    if env_file.is_file():
+        shutil.copy(env_file, shipped / ".env")
+    script = f"""
+import importlib, pkgutil, sys
+import app
+for module in pkgutil.walk_packages(app.__path__, "app."):
+    importlib.import_module(module.name)
+checkout = [
+    name for name, module in sys.modules.items()
+    if (getattr(module, "__file__", None) or "").startswith({f"{BACKEND}/"!r})
+]
+assert not checkout, checkout
+"""
+    subprocess.run([sys.executable, "-c", script], cwd=shipped / "backend", check=True)

backend/app/features/catalog/model.py

@@ -14,7 +14,7 @@
 class Origin(StrEnum):
     """The feed's "From" filter: what the article is when that is settled (a
     repo, a paper, a model), otherwise who published it. Not stored; each
-    article falls under exactly one, see `catalog.articles.origin_condition`."""
+    article falls under exactly one, see `catalog.rows.origin_condition`."""
 
     lab = "lab"
     company = "company"

backend/app/features/collect/hn.py

@@ -45,7 +45,7 @@ def hn_grace_hours() -> float:
     return float(os.environ.get("HN_GRACE_HOURS", "6"))
 
 
-# The title-only AI gate. Moved here from pipeline/topic_gate.py when the
+# The title-only AI gate. Moved here from the old topic gate module when the
 # pipeline stopped gating feeds by topic: HN is the one source whose firehose
 # is mostly not about AI, so it keeps the gate.
 AI_TITLE_KEYWORDS = re.compile(

backend/app/features/collect/newsletters.py

@@ -374,7 +374,8 @@ def prefilter(
 
 
 def _probability(answers: Answers, key: str) -> float:
-    """A yes/no answer as a number. A missing or non-numeric answer is 0."""
+    """A yes/no answer as a number. `jev.ask` has already failed the link on a
+    missing or non-numeric one."""
     value = answers.get(key)
     return float(value) if isinstance(value, int | float) else 0.0
 

backend/app/features/collect/publishers.py

@@ -196,7 +196,8 @@ def _quarantine(self, slug: str, name: str) -> Publisher:
 )
 # Shared hosts whose subdomains are one author's site (writer.substack.com).
 # Any other shared host's subdomain is the platform itself (old.reddit.com,
-# m.youtube.com).
+# m.youtube.com), unless a publisher's own link names it
+# (engineering.linkedin.com, see `hosts_to_publishers`).
 PERSONAL_SUBDOMAIN_HOSTS = frozenset({"medium.com", "substack.com"})
 
 _HOST_PLATFORMS = (
@@ -313,7 +314,9 @@ def hosts_to_publishers(publishers: Iterable[Publisher]) -> dict[str, Publisher]
                 host = owner_key(link) or hostname(link)
             else:
                 continue
-            if host and not is_shared_host(host):
+            # Only the bare platform: a subdomain a publisher links to
+            # (engineering.linkedin.com) is that publisher's own site.
+            if host and host not in SHARED_HOSTS:
                 claims.setdefault(host, {})[p.id] = p
     return {
         host: next(iter(pubs.values()))

backend/app/features/collect/test_publishers.py

@@ -130,9 +130,18 @@ def test_a_shared_platform_includes_its_subdomains(host: str, shared: bool) -> N
     assert is_shared_host(host) is shared
 
 
-def test_a_publisher_on_a_platform_subdomain_claims_nothing() -> None:
-    sub = _pub(11, website="https://old.reddit.com/r/LocalLLaMA")
-    assert _credited("https://old.reddit.com/r/LocalLLaMA/comments/x", sub) is None
+@pytest.mark.parametrize(
+    ("link", "url"),
+    [
+        ("https://engineering.linkedin.com", "https://engineering.linkedin.com/blog/a"),
+        ("https://docs.github.com/en", "https://docs.github.com/en/copilot/agents"),
+        ("https://blog.x.com", "https://blog.x.com/engineering/en_us/a-post"),
+    ],
+)
+def test_a_publishers_own_link_claims_a_platform_subdomain(link: str, url: str) -> None:
+    """A platform's first-party site is the publisher's own, unlike the bare
+    platform host."""
+    assert _credited(url, _pub(11, website=link)) == 11
 
 
 def test_a_substack_writer_still_claims_their_own_subdomain() -> None: