From 01fb02cf339d63f9a9e73ef6346f954ad442e8d1 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 18 Oct 2026 22:32:41 +0000 Subject: [PATCH 1/4] fix: run mypy and ty in pre-commit with the backend config --- .pre-commit-config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 0826cf5c..1e713644 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -53,14 +53,14 @@ repos: - id: local-mypy name: mypy check - entry: uv run mypy backend/app + entry: uv run mypy --config-file backend/pyproject.toml backend/app require_serial: true language: unsupported pass_filenames: false - id: local-ty name: ty check - entry: uv run ty check backend/app + entry: uv run ty check --project backend backend/app require_serial: true language: unsupported pass_filenames: false From 29bf42183fdeae78769a60494f7411132cce4a36 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 18 Oct 2026 22:32:41 +0000 Subject: [PATCH 2/4] ci: enforce backend import layers --- .github/workflows/test-backend.yml | 3 ++ .pre-commit-config.yaml | 7 +++ backend/pyproject.toml | 43 ++++++++++++++++ uv.lock | 82 ++++++++++++++++++++++++++++++ 4 files changed, 135 insertions(+) diff --git a/.github/workflows/test-backend.yml b/.github/workflows/test-backend.yml index ded53141..48323c5f 100644 --- a/.github/workflows/test-backend.yml +++ b/.github/workflows/test-backend.yml @@ -39,6 +39,9 @@ jobs: POSTGRES_USER=postgres POSTGRES_PASSWORD=ci-test-password EOF + - name: Check import layers + run: uv run lint-imports + working-directory: backend - run: docker compose up -d - name: Migrate DB run: uv run bash app/entry/cli/prestart.sh diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 1e713644..5d8f43c7 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -65,6 +65,13 @@ repos: language: unsupported pass_filenames: false + - id: local-lint-imports + name: import-linter + entry: uv run lint-imports --config backend/pyproject.toml + require_serial: true + language: unsupported + pass_filenames: false + - id: generate-frontend-sdk name: Generate Frontend SDK entry: bash ./scripts/generate-client.sh diff --git a/backend/pyproject.toml b/backend/pyproject.toml index f0775350..125bc322 100644 --- a/backend/pyproject.toml +++ b/backend/pyproject.toml @@ -38,6 +38,7 @@ dev = [ "ruff<1.0.0,>=0.2.2", "coverage<8.0.0,>=7.4.3", "types-regex>=2026.9.29.20261017", + "import-linter>=2.15,<3.0.0", ] [build-system] @@ -118,3 +119,45 @@ error-on-warning = true [tool.fastapi] entrypoint = "app.main:app" + +[tool.importlinter] +root_package = "app" + +[[tool.importlinter.contracts]] +name = "main > entry > features > core" +type = "layers" +containers = ["app"] +layers = ["main", "entry", "features", "core"] +# A new top-level module under app/ has to join a layer. +exhaustive = true +exhaustive_ignores = ["test_main"] + +[[tool.importlinter.contracts]] +name = "Features never import each other" +type = "independence" +modules = ["app.features.*"] + +[[tool.importlinter.contracts]] +name = "api, mcp and cli never import each other" +type = "independence" +modules = ["app.entry.api", "app.entry.mcp", "app.entry.cli"] + +[[tool.importlinter.contracts]] +name = "Flat core modules never import core.db" +type = "forbidden" +# app.core.* matches app.core.db too; import-linter skips that overlap. +source_modules = ["app.core.*"] +forbidden_modules = ["app.core.db"] + +[[tool.importlinter.contracts]] +name = "Entry reaches a feature only through its service and model" +type = "forbidden" +source_modules = ["app.entry"] +forbidden_modules = ["app.features.*.*"] +allow_indirect_imports = true +ignore_imports = [ + "app.entry.** -> app.features.*.service", + "app.entry.** -> app.features.*.model", + # The search tests stub the query embedding where it is computed. + "app.entry.api.test_articles -> app.features.catalog.semantic_search", +] diff --git a/uv.lock b/uv.lock index c657a6ce..87093778 100644 --- a/uv.lock +++ b/uv.lock @@ -108,6 +108,7 @@ dependencies = [ [package.dev-dependencies] dev = [ { name = "coverage" }, + { name = "import-linter" }, { name = "mypy" }, { name = "pytest" }, { name = "ruff" }, @@ -146,6 +147,7 @@ requires-dist = [ [package.metadata.requires-dev] dev = [ { name = "coverage", specifier = ">=7.4.3,<8.0.0" }, + { name = "import-linter", specifier = ">=2.15,<3.0.0" }, { name = "mypy", specifier = ">=1.8.0,<3.0.0" }, { name = "pytest", specifier = ">=7.4.3,<10.0.0" }, { name = "ruff", specifier = ">=0.2.2,<1.0.0" }, @@ -909,6 +911,71 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/41/63/e876e789525063c840ccfa8857febdabd6523bcef9ce7eb979b9305ea895/griffelib-2.3.0-py3-none-any.whl", hash = "sha256:1b8f9cd525681c26b1d6d574faa1371651e8459ca51d209684f50b8096ae06e0", size = 169423, upload-time = "2026-09-04T15:08:12.956Z" }, ] +[[package]] +name = "grimp" +version = "3.17" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ad/2e/95736b7c984705ff5ee376f07c6e71bb80fcbb22c8fa3b8c1cfbfa919d5c/grimp-3.17.tar.gz", hash = "sha256:5161e03c6f518fe4da43b06c97ff49a1e224ecb0e4b9fd6ef48235373d1b7b33", size = 832923, upload-time = "2026-09-04T11:28:54.609Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/10/7a/b1c76982a239347bfccd2ae95b9f79d3390cd14170f62e7eb822a17d786d/grimp-3.17-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:eb8f09da4302f05fc92fe7564f27731c0b43789d8893ae5e6be5df2b01668923", size = 2145452, upload-time = "2026-09-04T11:28:12.121Z" }, + { url = "https://files.pythonhosted.org/packages/cb/ab/2a1a9574b391c0ee62cb33a633c0d377dce1b7d19d04a7ac5d5e6872d395/grimp-3.17-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:0f11566bcf388db4c79f1c7f1051f6ef150f778306350bb7d86d0c9aab32d297", size = 2091654, upload-time = "2026-09-04T11:28:04.635Z" }, + { url = "https://files.pythonhosted.org/packages/e6/12/7dc086132ac10a2409f158f0b0d7cc528a2ebe0a4e95bca895afe7452fa6/grimp-3.17-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bfda3e2ebb4e51beec4a3cb09bb771440f3192163107d9320e6d9ce8fe064e64", size = 2277300, upload-time = "2026-09-04T11:27:01.844Z" }, + { url = "https://files.pythonhosted.org/packages/5b/62/2e16a54580fa76fb7e64e8e8e2ebb899c709a764cb29149a88eb03f2eea7/grimp-3.17-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:b0ab1ab71d807fc6f56d82f1ab24acf2f2e0e5000e37b5540af9449e28c6830c", size = 2220417, upload-time = "2026-09-04T11:27:11.418Z" }, + { url = "https://files.pythonhosted.org/packages/73/c0/e1b0ad0d0b3d9dff996bad306542f3cf000cb86c368bfe3ac6a8bb1e2e2f/grimp-3.17-cp313-cp313-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:187b092664e882d10b9f2c3da99251d0aeb99186d407ed5d20f52005832045f6", size = 2370754, upload-time = "2026-09-04T11:27:39.995Z" }, + { url = "https://files.pythonhosted.org/packages/36/0f/015e48ad8029f788eaa3df71b66859b2b5dc63abb8d7a2014a2b5a581040/grimp-3.17-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:75da770aeee97d26b202d7251fe7f6562e7f2793e348501b0a64ca5c50e881b5", size = 2628257, upload-time = "2026-09-04T11:27:20.491Z" }, + { url = "https://files.pythonhosted.org/packages/eb/d3/66da78504a1731237ed8befd9c66fb91037a7b82816443ca772107acf8fe/grimp-3.17-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f5cc64f891f71f538d3d032d6e7193f69c3a2e0ceac72c55114faee5aa161f9f", size = 2347569, upload-time = "2026-09-04T11:27:30.282Z" }, + { url = "https://files.pythonhosted.org/packages/cb/a2/6474d008fa49cc650d626798cb3b6e3fa7b0daaad24884fd6bb4288f6154/grimp-3.17-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6dbf2d93b0ba7f6c1efba9f5b3e07a3da0a157392c650316a5c1478b0ef7e503", size = 2284227, upload-time = "2026-09-04T11:27:52.462Z" }, + { url = "https://files.pythonhosted.org/packages/c5/b5/91cd378c08494fd0130e5cc2e54d60d12f53b21850e0dbccf668b6e73d91/grimp-3.17-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:7a22b5dcbf55f1c348885e09cbe91e63baedf6ce0f89c60bb01a0475c7d0a90b", size = 2454655, upload-time = "2026-09-04T11:28:19.837Z" }, + { url = "https://files.pythonhosted.org/packages/f1/d0/b582238247ce51857c7ac770c3be10203c992696f3dc9153f1a979c011dc/grimp-3.17-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:73aa9459a604554b1dc1618148312036679f3974bf8b0285588220307147060b", size = 2495852, upload-time = "2026-09-04T11:28:29.606Z" }, + { url = "https://files.pythonhosted.org/packages/8f/99/e0690b02be70f346aadfe2aa8bd7e496c773298ac58c0d0fa6376c1298b2/grimp-3.17-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:c31660d30e79569faac05033637c1eef4595542f44ec2323af54e900030164a6", size = 2527887, upload-time = "2026-09-04T11:28:38.828Z" }, + { url = "https://files.pythonhosted.org/packages/69/7a/33e41d9b08e63cf398d8194a9208b21788c76916b5ac6d90146e8e2ebf95/grimp-3.17-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:8f395c5ab2cf554504f46680b9269b769939ab5d7ea58f93e5c341e55b1affbd", size = 2539000, upload-time = "2026-09-04T11:28:49.037Z" }, + { url = "https://files.pythonhosted.org/packages/69/f5/7af1dd74b9608a85274414fc423a7162844492b198ece3fe31f4e1f88237/grimp-3.17-cp313-cp313-win32.whl", hash = "sha256:34369739ed293accbb674494ae932351516b316e8e85d1d333d4c51796af846a", size = 1874244, upload-time = "2026-09-04T11:29:21.693Z" }, + { url = "https://files.pythonhosted.org/packages/8b/d4/af497adddfcf11f9bd4ab273625a49022bf5ba4f5fc389589f4e6edc67b9/grimp-3.17-cp313-cp313-win_amd64.whl", hash = "sha256:d21b859c50418bdb48b403a84450650fa7aab3102f3212d00d3c21c76a982167", size = 1998225, upload-time = "2026-09-04T11:29:10.5Z" }, + { url = "https://files.pythonhosted.org/packages/e2/2a/07fd537997281049af2390fc8641de5ae063ae4972cb6fcec85873674595/grimp-3.17-cp313-cp313-win_arm64.whl", hash = "sha256:206cd9f7c757b97934f48ba2b8ec6d649725fd6314d6b182dd327b19b4f35707", size = 1922071, upload-time = "2026-09-04T11:28:58.34Z" }, + { url = "https://files.pythonhosted.org/packages/39/e0/e11ea5aeb62534c9706e17952d142aa844ce5659e2d66b4b8ba9377407d6/grimp-3.17-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:0100ff6f08067c8eb8a77dd1433af5e173aea8dbf6a6b76f5372a3b49f2e0a43", size = 2161696, upload-time = "2026-09-11T10:36:49.104Z" }, + { url = "https://files.pythonhosted.org/packages/4b/d6/ddf82f4428d8bdc54ced0db244cbf5d8e5638490c3ed6a8b9f5db55dfa51/grimp-3.17-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:67ada40a5c1cf4321183c0b0cd0393658ccd04b845caed002ac81dc2fdef828e", size = 2107301, upload-time = "2026-09-11T10:36:46.459Z" }, + { url = "https://files.pythonhosted.org/packages/86/1a/1dd0d9d2751e8078547e1cb7131e13d9624a2ea60700d926b3d624c15825/grimp-3.17-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:baa385915a4c17948bfb25f77509cbbfb33795f321d1c5ca7cc602e34ed58e99", size = 2278433, upload-time = "2026-09-04T11:27:03.075Z" }, + { url = "https://files.pythonhosted.org/packages/58/71/3768fd253e7455a83d668193055691c82d3a4d434047116c572a27d2bd39/grimp-3.17-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:8322a435b5ee97ff3db823df36cbefea569faec505cc0ca66223c27b4b44cca6", size = 2221152, upload-time = "2026-09-04T11:27:12.642Z" }, + { url = "https://files.pythonhosted.org/packages/b4/4f/907a130e87554d57be14c2a7cedeb818d0ef519064877297d19636735f6b/grimp-3.17-cp314-cp314-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:cb953c21beb6ac85a59b5b54128be7fdddd89396d34b3c5705cf59829a85b431", size = 2371546, upload-time = "2026-09-04T11:27:41.195Z" }, + { url = "https://files.pythonhosted.org/packages/7e/4f/a3bf534aa7c582a47d8cfefab5fccb50e6f4dcd327ed6a8552d344a04ffe/grimp-3.17-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c34b904c25359f98e3f20771256d62329ed17f1e26a254a4f5c74bacc1f3bd21", size = 2628467, upload-time = "2026-09-04T11:27:21.818Z" }, + { url = "https://files.pythonhosted.org/packages/32/d3/2b2ab7b3a4e2c17bb1e10ec37e7ee8eecce7f72b1bf6c8019dd439943884/grimp-3.17-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f69ce8be301299f15ca99843e89991c240a231836cf45a11c6bfc54fb9fbeaa9", size = 2347520, upload-time = "2026-09-04T11:27:31.73Z" }, + { url = "https://files.pythonhosted.org/packages/72/da/6bc1f783a739afa51ffa2e235791487b5d09202501b25843ecae9833a3ff/grimp-3.17-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:566411d1ac97e1f3a3f00a13ae33aebb9abda9787d3d9914f62f73b589d84fe2", size = 2284007, upload-time = "2026-09-04T11:27:53.889Z" }, + { url = "https://files.pythonhosted.org/packages/63/06/a08f1b16aec11f32bbc86f60e24986659ab4cfab79388c8d4b79f6a6723e/grimp-3.17-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:5f89948029f3c7339a20bab007977194d3dbdddb27edae5c74b948d8f13191c7", size = 2455301, upload-time = "2026-09-04T11:28:21.177Z" }, + { url = "https://files.pythonhosted.org/packages/ff/c1/d55e9686f3797d1a2c6403d2ffec3020c67591c004304a414fe8d7a9755c/grimp-3.17-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:5a895bc27105ada9b0e76a80bf9cc898409e5cc3689e73e64ca1cbd71dc87dce", size = 2497158, upload-time = "2026-09-04T11:28:30.984Z" }, + { url = "https://files.pythonhosted.org/packages/7f/85/a2d2e70c4f272752c395cc35b3732d82ff92c55e229d7bbd6777fd9868ff/grimp-3.17-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:e80b57a7d03f53e96eb0a5b0844340f99f17eede184042fb73e3ef010176ab35", size = 2528862, upload-time = "2026-09-04T11:28:40.251Z" }, + { url = "https://files.pythonhosted.org/packages/cb/03/fc3a58d9b22b011c1c3377c1ecee127a5f642de9b20de32d3ad7213ab71a/grimp-3.17-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5c0cac76283679fe9c1fd761e63deb9bbc7ba46b32988113ecef4b13742f31b9", size = 2538240, upload-time = "2026-09-04T11:28:50.248Z" }, + { url = "https://files.pythonhosted.org/packages/58/13/d7666cf03c79c275020bad40ea1df8597af7877083ef5a84b5d8b566029a/grimp-3.17-cp314-cp314-win32.whl", hash = "sha256:c8982f4fa1276e059665947156740a55a4e8e6b5a4d7f40730a7511afb608762", size = 1874930, upload-time = "2026-09-04T11:29:23.152Z" }, + { url = "https://files.pythonhosted.org/packages/59/ce/abb9ccb50972351461065ca72cd0f528837a2f398621147dfaf1013698f0/grimp-3.17-cp314-cp314-win_amd64.whl", hash = "sha256:0b97dc29bdca523932bbbcb425fc3df3547cf970396b293bfa960b910f3932d1", size = 1999405, upload-time = "2026-09-04T11:29:11.832Z" }, + { url = "https://files.pythonhosted.org/packages/9f/1d/93c12ff4ce9ede9f91aa03df66128d223a4c824a13de6ab702c973fcae0d/grimp-3.17-cp314-cp314-win_arm64.whl", hash = "sha256:80b5b25b8d8ed6cc68dc39ae9e4c2fb64acd1a3e66434d9cf4d68c72e46d505d", size = 1923324, upload-time = "2026-09-04T11:29:00.004Z" }, + { url = "https://files.pythonhosted.org/packages/d2/59/f94dd90757a2195415736f0e2be667ac4cd375fe86d875d4298661751fce/grimp-3.17-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:e15340ee11e34bfaaee496cff53f8287dc9f417a94e5bdf3d6b4204305d004cc", size = 2145199, upload-time = "2026-09-04T11:28:13.287Z" }, + { url = "https://files.pythonhosted.org/packages/ce/83/9d7cb22fc136ad6a61bff3b74cf0b31b13fdf94e224d46ff1edd9b5b8c86/grimp-3.17-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:cbccf2a6cd3f89bfdfd683d9b000bd61f6936e16b6d4d6619da592c54f79f272", size = 2090761, upload-time = "2026-09-04T11:28:05.984Z" }, + { url = "https://files.pythonhosted.org/packages/e2/6b/b70add79e9e315d111bb1643b44e1b9e1ded82888856e1ad68bc484cd635/grimp-3.17-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5332893955688759a9f7868c3c9e56b19bc1694ee251852d3daf54876b35c115", size = 2279005, upload-time = "2026-09-04T11:27:04.543Z" }, + { url = "https://files.pythonhosted.org/packages/d3/4c/f5f001e592f8e8820086604db98373ca129e2831facad81ec0d1820a749f/grimp-3.17-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:848a92d8a63a38d79c89ab1e68fb5429d90079f39c07533ffd45ce756e7f210f", size = 2220430, upload-time = "2026-09-04T11:27:13.814Z" }, + { url = "https://files.pythonhosted.org/packages/83/13/cc57724307c22060e193a1dc9bc99fa14d46584c534a3936319ec579c599/grimp-3.17-cp314-cp314t-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c89cfe49f822a2cc9c42cc7b0f3975ecb1281970735ea31610bfa06a7e04c460", size = 2370318, upload-time = "2026-09-04T11:27:42.535Z" }, + { url = "https://files.pythonhosted.org/packages/50/1b/05d9964d4b47ffe0ef826bfe04b3f4b713fd623c90e424ff002cb0d8a917/grimp-3.17-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:541c9a2b7f1ad5d4f5d1597740e51058fba8ca0c1a225d6ccc6d11a8364fa563", size = 2629127, upload-time = "2026-09-04T11:27:23.18Z" }, + { url = "https://files.pythonhosted.org/packages/ce/ea/186a3087623f1556eb8b9d02baa00300003bb84b7b35677e3fcec34691b6/grimp-3.17-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:66b4a13edb3fe63be6f7dbbc2576786f501af9d386a5a1f60b2938f270aef97c", size = 2348453, upload-time = "2026-09-04T11:27:32.964Z" }, + { url = "https://files.pythonhosted.org/packages/e8/6c/a718328adc50529135510b515c5ff1d9f94ab26ab8df9057bb1ef5ab8993/grimp-3.17-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:160abad142e168545be8d4cc91b2a95b4565f7571fc56f7fced69ce3589e5111", size = 2287338, upload-time = "2026-09-04T11:27:55.16Z" }, + { url = "https://files.pythonhosted.org/packages/ea/ed/bf67bc6749abee0a1e97f8f14ee2cb19ccc435b306e22bcaac6118d402a4/grimp-3.17-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:cd76fb13a534209c1777c807092f51b1570854eb49948d65cd5e1d94d80e981d", size = 2455299, upload-time = "2026-09-04T11:28:22.505Z" }, + { url = "https://files.pythonhosted.org/packages/83/8a/6c954092306e21c93c744432c3d47491c3264d3cc4b465941de5164a1cb3/grimp-3.17-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:1688e2dbe382b6644ecc3f55053e319de70034d9dfe0809d21d441a590c320e6", size = 2496813, upload-time = "2026-09-04T11:28:32.303Z" }, + { url = "https://files.pythonhosted.org/packages/ea/a8/cf0aa245ace55dd2f6bdd3bb646ec50c294bdfe623fd51327bd27efaa585/grimp-3.17-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:0433d177f327c31410f21845322282c832d887f1a42cbdfa176133d8422d84b2", size = 2527052, upload-time = "2026-09-04T11:28:41.869Z" }, + { url = "https://files.pythonhosted.org/packages/75/b8/90cd11e346f12fb5269ff050ae825083410d3bdb54d82a3b8e955e4f9d26/grimp-3.17-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:790d68157e034e7848257ef46b37550040c9e4692cea17372d3d9f87cef1d84b", size = 2541238, upload-time = "2026-09-04T11:28:51.786Z" }, + { url = "https://files.pythonhosted.org/packages/48/92/10cc37db6149030c8cfc80fa5287f739c1e9620c8a05d1ba8dfdfc0a6c4f/grimp-3.17-cp314-cp314t-win32.whl", hash = "sha256:37e6333ca54d4d80b5c4f2eae324774b223dbdc65994dc9bccda2985a9261dd8", size = 1872673, upload-time = "2026-09-04T11:29:24.514Z" }, + { url = "https://files.pythonhosted.org/packages/ac/cc/b257d8da079a87e65d0f1b0e004fd50f31ba25e7136a3b36f896e0e338f8/grimp-3.17-cp314-cp314t-win_amd64.whl", hash = "sha256:91e4bbbed4d934369b83071380e1ad5f37b7f5c80b80ffa924627db8d66fea11", size = 2000197, upload-time = "2026-09-04T11:29:13.138Z" }, + { url = "https://files.pythonhosted.org/packages/29/d0/feebddf766448932f2f1e40324fe75cf50ce6d3a21bdf87fc0eae263218a/grimp-3.17-cp314-cp314t-win_arm64.whl", hash = "sha256:28ae2ff57a9ef535c2e7054dcb18baf0cdc2dee3982367512bff34eaecb85cf0", size = 1920856, upload-time = "2026-09-04T11:29:01.498Z" }, + { url = "https://files.pythonhosted.org/packages/83/39/bb07084f0b97da66b0aa38077bd656c4f831b4aa3f167b1c060f6a65ea5e/grimp-3.17-cp315-cp315-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:fe3e2285a2bd4e62ddb8825de37152d2dc842394d419786d3d5a803b07bb84eb", size = 2372594, upload-time = "2026-09-04T11:27:43.884Z" }, + { url = "https://files.pythonhosted.org/packages/e6/a9/905a484659749f40ed797963147202cb74c3b68d8e90a7a600af55de47ee/grimp-3.17-cp315-cp315-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a377d5935ae9a62ccbb59e492e16624cec97cc7244c4a964b0c6f83940440626", size = 2284577, upload-time = "2026-09-04T11:27:56.428Z" }, + { url = "https://files.pythonhosted.org/packages/30/6a/40fb12b111c95b94db4930f35661a629532d9d12fb484db5b5ef1a6c3da3/grimp-3.17-cp315-cp315-win32.whl", hash = "sha256:65a1dc2741bef8a96ff3ff71252db8e99eac2a32e09d5e2b63597474706186ca", size = 1876437, upload-time = "2026-09-04T11:29:25.725Z" }, + { url = "https://files.pythonhosted.org/packages/71/98/79af8d51961611e13b3dabc7a3b356b27aa696e5549bb439f1d6002be31a/grimp-3.17-cp315-cp315-win_amd64.whl", hash = "sha256:d20fbe5aab8b013b351313f8eaa3a46f4ac57677e357cb1c3d1012c606b565b4", size = 1999947, upload-time = "2026-09-04T11:29:14.623Z" }, + { url = "https://files.pythonhosted.org/packages/96/d6/e1922c4c7e001210f98e0cf89bd76fea8564f590be3e9ccfe99435cfcd59/grimp-3.17-cp315-cp315-win_arm64.whl", hash = "sha256:a0288dc0e8947dead5c02a246e16f3f5e5cb486348171bdf1289885a13de36c1", size = 1923774, upload-time = "2026-09-04T11:29:02.897Z" }, + { url = "https://files.pythonhosted.org/packages/85/f6/f4f84e44a9237350c41e7c77343498cb1ad843491c18f29d6811021c617d/grimp-3.17-cp315-cp315t-macosx_10_12_x86_64.whl", hash = "sha256:1bf67cb05a80796be99d1bf87e12d4421de101950c5246a8220b6c839cbf27c2", size = 2145245, upload-time = "2026-09-04T11:28:14.537Z" }, + { url = "https://files.pythonhosted.org/packages/71/98/822287270c41360f66c4b5638e040afc6bbf11e9f04cf5005bbd5f3e4830/grimp-3.17-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:d058fdb980475eb9e0143792d63757ddff86c5776e792866e8208cfe5e8413f9", size = 2090367, upload-time = "2026-09-04T11:28:07.277Z" }, + { url = "https://files.pythonhosted.org/packages/ac/76/32d303603157a0d98cb8bde8757e6ae8fa02d41f1d62cdf843fb2e7fe727/grimp-3.17-cp315-cp315t-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:92f1989ce9f9ce26b7d45381df164c62aff783add84cd7a4216a6ebba0f9f03a", size = 2371543, upload-time = "2026-09-04T11:27:45.108Z" }, + { url = "https://files.pythonhosted.org/packages/7b/68/4551e23ab9422e2ead806c1fec9f4b3acd21a267e43bfe8e8cd7f4640dce/grimp-3.17-cp315-cp315t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e275cee07826cf36db1914427e68afac87588f759e1dec676870234d4a250ae", size = 2288003, upload-time = "2026-09-04T11:27:57.605Z" }, + { url = "https://files.pythonhosted.org/packages/20/64/79161f05c900f7f7f1b0acd22a730b1be9f96680dd44efeef90aa45e6998/grimp-3.17-cp315-cp315t-win32.whl", hash = "sha256:a7235ccffb0350647ae2b01969f552a9859fea7027032f6cfed110801269a439", size = 1873249, upload-time = "2026-09-04T11:29:27.094Z" }, + { url = "https://files.pythonhosted.org/packages/94/b7/6f9cdeba2c1263fb91b791ddd5e924d6c91c2485f4f16b56698802fdd540/grimp-3.17-cp315-cp315t-win_amd64.whl", hash = "sha256:473d0e0c016ca429b9ec17f34d276db454ba4d467e3c915a46279d86ab004716", size = 2000579, upload-time = "2026-09-04T11:29:15.961Z" }, + { url = "https://files.pythonhosted.org/packages/ac/16/8ae9028f046bf8f48cc13f6de2de40adbcbf38fa79a3a5f265753b248da0/grimp-3.17-cp315-cp315t-win_arm64.whl", hash = "sha256:478aa0b696e5dd79373943e75b998fbb4ef3d146f24fe3569fa9d106ce84991b", size = 1921829, upload-time = "2026-09-04T11:29:04.403Z" }, +] + [[package]] name = "h11" version = "0.16.0" @@ -1099,6 +1166,21 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/19/a8/0f58c13d2660d5fc8f808ce8b46828d2941752ec21e4015bde99c08b37d7/imap_tools-1.13.0-py3-none-any.whl", hash = "sha256:656c37beba22ab2929b73c07d0ca397ae8805b670d390b1127723e3335244e6d", size = 35849, upload-time = "2026-05-12T07:14:52.669Z" }, ] +[[package]] +name = "import-linter" +version = "2.15" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "click" }, + { name = "grimp" }, + { name = "rich" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/3c/81/1bb414f8c8bcd024499e91f112d317c36a86bc264da1fd43f9797fc2b7b0/import_linter-2.15.tar.gz", hash = "sha256:1da912bea5e172a82a3ce617b5543f75cf64dc0d8f4d9b46c5578b68ccb81590", size = 1405973, upload-time = "2026-09-04T14:47:56.097Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/08/31/4571c20652a4002ed9e500046386f3731a77e23d95a608ae8a87f1fbb2da/import_linter-2.15-py3-none-any.whl", hash = "sha256:9aaf16a88ac1e99d5a464cd7f66b6a05f7060bfa761162e0ed441773a267ed3b", size = 640512, upload-time = "2026-09-04T14:47:54.878Z" }, +] + [[package]] name = "iniconfig" version = "2.3.0" From 3e98f09593f2f10e6f1f188f471c3ed3522652ee Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 18 Oct 2026 22:39:18 +0000 Subject: [PATCH 3/4] fix: name the exact claude-code-action tag in its pin comments --- .github/workflows/claude-code-review.yml | 2 +- .github/workflows/claude.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/claude-code-review.yml b/.github/workflows/claude-code-review.yml index 1a422372..d4a8cb7b 100644 --- a/.github/workflows/claude-code-review.yml +++ b/.github/workflows/claude-code-review.yml @@ -34,7 +34,7 @@ jobs: - name: Run Claude Code Review id: claude-review - uses: anthropics/claude-code-action@f27be6b65a4652f08a2ba3cff79114829b588967 # v1 + uses: anthropics/claude-code-action@f27be6b65a4652f08a2ba3cff79114829b588967 # v1.0.252 with: claude_code_oauth_token: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }} plugin_marketplaces: 'https://github.com/anthropics/claude-code.git' diff --git a/.github/workflows/claude.yml b/.github/workflows/claude.yml index 04a576af..c5b76092 100644 --- a/.github/workflows/claude.yml +++ b/.github/workflows/claude.yml @@ -33,7 +33,7 @@ jobs: - name: Run Claude Code id: claude - uses: anthropics/claude-code-action@f27be6b65a4652f08a2ba3cff79114829b588967 # v1 + uses: anthropics/claude-code-action@f27be6b65a4652f08a2ba3cff79114829b588967 # v1.0.252 with: claude_code_oauth_token: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }} From b0ddf956cb3a51e1312674413a13a0b8ff8ecd3d Mon Sep 17 00:00:00 2001 From: DS Date: Tue, 20 Oct 2026 00:37:34 +0200 Subject: [PATCH 4/4] ci: deploy only what the box runs (#88) * docs: point the deploy README's env table at app/features * ci: deploy only what the box runs * test: check the deploy filter ships every path the units and the job use * docs: record the layered backend (#89) * docs: record the layered backend * fix: address the backend migration review (#91) * fix: configure logging for the api process * fix: let a publisher's link claim a platform subdomain * fix: fail a link loudly on a non-numeric jev answer * docs: point stale docstrings at where the code moved --------- Co-authored-by: Claude --------- Co-authored-by: Claude --------- Co-authored-by: Claude --- .env.dev | 2 +- .github/workflows/deploy-production.yml | 24 ++- backend/app/core/db/story.py | 2 +- backend/app/core/jev.py | 8 + backend/app/core/test_jev.py | 14 +- backend/app/entry/cli/test_deploy_files.py | 155 ++++++++++++++++++ backend/app/features/catalog/model.py | 2 +- backend/app/features/collect/hn.py | 2 +- backend/app/features/collect/newsletters.py | 3 +- backend/app/features/collect/publishers.py | 7 +- .../app/features/collect/test_publishers.py | 15 +- .../features/collect/test_publishers_db.py | 21 +++ backend/app/features/curation/test_tags.py | 2 +- backend/app/main.py | 9 + backend/app/test_main.py | 33 +++- backend/baml_client/inlinedbaml.py | 2 +- backend/baml_src/extract_items.baml | 2 +- deploy/README.md | 39 ++++- deploy/rsync-filter | 35 ++++ docs/adr/0017-layered-features.md | 115 +++++++++++++ docs/backend.md | 40 +++-- docs/glossary.md | 16 +- docs/product.md | 9 +- docs/style.md | 80 +++++---- frontend/scripts/blog-posts.ts | 2 +- frontend/scripts/prerender-blog.ts | 2 +- frontend/src/client/schemas.gen.ts | 2 +- frontend/src/client/types.gen.ts | 2 +- 28 files changed, 546 insertions(+), 99 deletions(-) create mode 100644 backend/app/entry/cli/test_deploy_files.py create mode 100644 deploy/rsync-filter create mode 100644 docs/adr/0017-layered-features.md diff --git a/.env.dev b/.env.dev index ca0e7415..c257ad91 100644 --- a/.env.dev +++ b/.env.dev @@ -28,6 +28,6 @@ MCP_WRITE_TOKEN=agentique-dev-write MCP_DB_USER=postgres MCP_DB_PASSWORD=agentique-dev -# Jev (pipeline/jev.py), the newsletter link classifier. Empty locally: set a +# Jev (backend/app/core/jev.py), the newsletter link classifier. Empty locally: set a # real key only in .env, never here. OPENROUTER_API_KEY= diff --git a/.github/workflows/deploy-production.yml b/.github/workflows/deploy-production.yml index 31e42519..2e46a075 100644 --- a/.github/workflows/deploy-production.yml +++ b/.github/workflows/deploy-production.yml @@ -49,25 +49,21 @@ jobs: uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0 with: version: "0.11.18" + # Straight into frontend/dist, so the one sync below ships it. - name: Download dist uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: frontend-dist - path: frontend-dist + path: frontend/dist # The workspace is a staging area only — actions/checkout resets it every - # run, so the services live in /opt/agentique. .env and .venv belong to - # the box; frontend/dist is replaced separately below. - - name: Sync source to /opt/agentique - run: > - rsync -a --delete - --exclude '.git' - --exclude '.env' - --exclude '.venv' - --exclude 'frontend/dist' - --exclude 'frontend-dist' - ./ /opt/agentique/ - - name: Sync frontend dist - run: rsync -a --delete frontend-dist/ /opt/agentique/frontend/dist/ + # run, so the services live in /opt/agentique. deploy/rsync-filter lists + # what the box runs; --delete-excluded removes everything else there, + # except what the filter protects (.env, .venv, .git). A missing dist + # would delete the live site, so check for it first. + - name: Sync to /opt/agentique + run: | + test -f frontend/dist/index.html + rsync -a --delete --delete-excluded --filter='merge deploy/rsync-filter' ./ /opt/agentique/ # --package app: the workspace root is virtual, a bare sync would not # install the backend (same flag the old Dockerfile used). - name: Sync venv diff --git a/backend/app/core/db/story.py b/backend/app/core/db/story.py index 85e08b35..5458cc3d 100644 --- a/backend/app/core/db/story.py +++ b/backend/app/core/db/story.py @@ -41,7 +41,7 @@ class Story(SQLModel, table=True): class StoryArticle(SQLModel, table=True): """An article in a story. At most one open story per article, enforced in - `catalog.stories.save_story`: a closed story keeps its articles.""" + `stories.service.save_story`: a closed story keeps its articles.""" __tablename__ = "story_article" # The primary key leads with story_id; "which story holds this article" is diff --git a/backend/app/core/jev.py b/backend/app/core/jev.py index 0c4f3b41..1a1cea5d 100644 --- a/backend/app/core/jev.py +++ b/backend/app/core/jev.py @@ -107,6 +107,14 @@ def _answers(body: dict[str, Json], questions: dict[str, Question]) -> dict[str, missing = [qid for qid, a in answers.items() if a is None] if missing: raise JevError(f"unanswered: {', '.join(missing)}") + # "0.93" as a string would read as 0 the same way. + odd = [ + qid + for qid, q in questions.items() + if q["type"] == "noul" and not isinstance(answers[qid], int | float) + ] + if odd: + raise JevError(f"non-numeric: {', '.join(odd)}") return answers diff --git a/backend/app/core/test_jev.py b/backend/app/core/test_jev.py index 384b5140..edd0f3ca 100644 --- a/backend/app/core/test_jev.py +++ b/backend/app/core/test_jev.py @@ -1,6 +1,6 @@ """The Jev transport's two guarantees: a 429 waits as long as it is told to -and tries again, and an answer set with a hole in it is an error, never a -quiet zero that reads as "not a sponsor" or "off topic". +and tries again, and an answer set with a hole or a non-numeric probability in +it is an error, never a quiet zero that reads as "not a sponsor" or "off topic". """ from __future__ import annotations @@ -107,6 +107,16 @@ def test_a_missing_answer_raises_instead_of_reading_as_zero( jev.ask({}, QUESTIONS) +def test_a_non_numeric_probability_raises_instead_of_reading_as_zero( + monkeypatch: pytest.MonkeyPatch, +) -> None: + body = {"answers": {"is_sponsor": {"noul": "0.93"}, "kind": "article"}} + _stub(monkeypatch, [_resp(200, body)]) + + with pytest.raises(jev.JevError, match="is_sponsor"): + jev.ask({}, QUESTIONS) + + def test_a_response_with_no_answers_raises(monkeypatch: pytest.MonkeyPatch) -> None: _stub(monkeypatch, [_resp(200, {"error": "nope"})]) diff --git a/backend/app/entry/cli/test_deploy_files.py b/backend/app/entry/cli/test_deploy_files.py new file mode 100644 index 00000000..ba60b62c --- /dev/null +++ b/backend/app/entry/cli/test_deploy_files.py @@ -0,0 +1,155 @@ +"""What the deploy ships: the workflow's rsync run against this checkout's +tracked files, then every path the units and the deploy job use looked up in +the result.""" + +from __future__ import annotations + +import re +import shlex +import shutil +import subprocess +import sys +import tomllib +from pathlib import Path + +import pytest + +BACKEND = Path(__file__).resolve().parents[3] +REPO = BACKEND.parent +BOX = Path("/opt/agentique") +UNITS = sorted((REPO / "deploy").glob("*.service")) +WORKFLOW = REPO / ".github/workflows/deploy-production.yml" +# Never shipped: the box keeps its own copy, and the filter protects it. +BOX_OWNED = (BOX / ".env", BOX / ".venv") + + +def rsync_command() -> list[str]: + lines = [ + line.strip() + for line in WORKFLOW.read_text().splitlines() + if line.strip().startswith("rsync ") + ] + assert len(lines) == 1, lines + return shlex.split(lines[0]) + + +@pytest.fixture(scope="module") +def shipped(tmp_path_factory: pytest.TempPathFactory) -> Path: + version = subprocess.run( + ["rsync", "--version"], capture_output=True, text=True, check=False + ).stdout + if "version 3." not in version: + pytest.skip("the filter needs rsync 3, as on the box") + # The deploy syncs a clean checkout, so untracked files must not count. + source = tmp_path_factory.mktemp("checkout") + tracked = subprocess.run( + ["git", "ls-files", "-z"], cwd=REPO, capture_output=True, check=True + ).stdout.decode() + for name in filter(None, tracked.split("\0")): + if (REPO / name).is_file(): + (source / name).parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(REPO / name, source / name) + (source / "frontend/dist").mkdir(parents=True) + (source / "frontend/dist/index.html").touch() + + dest = tmp_path_factory.mktemp("box") + command = rsync_command() + assert command[-1] == f"{BOX}/", command + subprocess.run([*command[:-1], f"{dest}/"], cwd=source, check=True) + return dest + + +def on_box(shipped: Path, path: Path) -> Path: + return shipped / path.relative_to(BOX) + + +def box_owned(path: Path) -> bool: + return any(path == owned or owned in path.parents for owned in BOX_OWNED) + + +@pytest.mark.parametrize("unit", UNITS, ids=lambda p: p.name) +def test_every_path_a_unit_uses_is_shipped(shipped: Path, unit: Path) -> None: + config = dict( + line.split("=", 1) + for line in unit.read_text().splitlines() + if "=" in line and not line.startswith("#") + ) + workdir = on_box(shipped, Path(config["WorkingDirectory"])) + assert workdir.is_dir(), workdir + assert box_owned(Path(config["EnvironmentFile"])) + # systemd's -, @, + and ! prefixes change how it runs, not what. + command, *args = config["ExecStart"].lstrip("-@+!:").split() + assert box_owned(Path(command)), command + if args[:1] == ["-m"]: + module = workdir / args[1].replace(".", "/") + assert module.with_suffix(".py").is_file() or (module / "__main__.py").is_file() + for arg in args: + if arg.endswith(".py"): + assert (workdir / arg).is_file(), arg + + +def test_every_directory_the_deploy_job_works_in_is_shipped(shipped: Path) -> None: + dirs = re.findall(r"working-directory: (/opt/agentique\S*)", WORKFLOW.read_text()) + assert dirs + for workdir in dirs: + assert on_box(shipped, Path(workdir)).is_dir(), workdir + for script in re.findall(r"bash (\S+\.sh)", WORKFLOW.read_text()): + assert (on_box(shipped, BOX / "backend") / script).is_file(), script + + +def test_prestart_runs_only_shipped_modules(shipped: Path) -> None: + prestart = shipped / "backend/app/entry/cli/prestart.sh" + for module in re.findall(r"python -m ([\w.]+)", prestart.read_text()): + assert (shipped / "backend" / (module.replace(".", "/") + ".py")).is_file() + + +def test_uv_sync_and_alembic_find_their_files(shipped: Path) -> None: + root = tomllib.loads((shipped / "pyproject.toml").read_text()) + for member in root["tool"]["uv"]["workspace"]["members"]: + assert (shipped / member / "pyproject.toml").is_file(), member + assert (shipped / "uv.lock").is_file() + assert (shipped / ".python-version").is_file() + assert (shipped / "backend/migrations/env.py").is_file() + # backup_db.py runs docker compose against the repo root's compose.yml. + assert (shipped / "compose.yml").is_file() + + +def test_every_data_file_in_app_is_shipped(shipped: Path) -> None: + data = [ + p.relative_to(REPO) + for p in (BACKEND / "app").rglob("*") + if p.is_file() + and p.suffix not in {".py", ".pyc"} + and not p.name.startswith("test_") + ] + assert data + for path in data: + assert (shipped / path).is_file(), path + + +def test_no_test_code_is_shipped(shipped: Path) -> None: + names = {p.name for p in (shipped / "backend").rglob("*.py")} + assert not [n for n in names if n.startswith("test_")] + assert "conftest.py" not in names + assert "factories.py" not in names + + +def test_every_shipped_module_imports_from_the_shipped_tree(shipped: Path) -> None: + # Run from the shipped backend/, as the units do. The venv's editable + # install still points at the checkout, so a module the filter drops would + # load from there instead. Fail on any module that did. + env_file = REPO / ".env" + if env_file.is_file(): + shutil.copy(env_file, shipped / ".env") + script = f""" +import importlib, pkgutil, sys +import app +for module in pkgutil.walk_packages(app.__path__, "app."): + importlib.import_module(module.name) +checkout = [ + name for name, module in sys.modules.items() + if (getattr(module, "__file__", None) or "").startswith({f"{BACKEND}/"!r}) +] +assert not checkout, checkout +""" + subprocess.run([sys.executable, "-c", script], cwd=shipped / "backend", check=True) diff --git a/backend/app/features/catalog/model.py b/backend/app/features/catalog/model.py index 4a6de95b..8b95ad2b 100644 --- a/backend/app/features/catalog/model.py +++ b/backend/app/features/catalog/model.py @@ -14,7 +14,7 @@ class Origin(StrEnum): """The feed's "From" filter: what the article is when that is settled (a repo, a paper, a model), otherwise who published it. Not stored; each - article falls under exactly one, see `catalog.articles.origin_condition`.""" + article falls under exactly one, see `catalog.rows.origin_condition`.""" lab = "lab" company = "company" diff --git a/backend/app/features/collect/hn.py b/backend/app/features/collect/hn.py index 4a22c1f3..0a7f5cb5 100644 --- a/backend/app/features/collect/hn.py +++ b/backend/app/features/collect/hn.py @@ -45,7 +45,7 @@ def hn_grace_hours() -> float: return float(os.environ.get("HN_GRACE_HOURS", "6")) -# The title-only AI gate. Moved here from pipeline/topic_gate.py when the +# The title-only AI gate. Moved here from the old topic gate module when the # pipeline stopped gating feeds by topic: HN is the one source whose firehose # is mostly not about AI, so it keeps the gate. AI_TITLE_KEYWORDS = re.compile( diff --git a/backend/app/features/collect/newsletters.py b/backend/app/features/collect/newsletters.py index 55bf8681..17dafed5 100644 --- a/backend/app/features/collect/newsletters.py +++ b/backend/app/features/collect/newsletters.py @@ -374,7 +374,8 @@ def prefilter( def _probability(answers: Answers, key: str) -> float: - """A yes/no answer as a number. A missing or non-numeric answer is 0.""" + """A yes/no answer as a number. `jev.ask` has already failed the link on a + missing or non-numeric one.""" value = answers.get(key) return float(value) if isinstance(value, int | float) else 0.0 diff --git a/backend/app/features/collect/publishers.py b/backend/app/features/collect/publishers.py index c958f345..2e0310a9 100644 --- a/backend/app/features/collect/publishers.py +++ b/backend/app/features/collect/publishers.py @@ -196,7 +196,8 @@ def _quarantine(self, slug: str, name: str) -> Publisher: ) # Shared hosts whose subdomains are one author's site (writer.substack.com). # Any other shared host's subdomain is the platform itself (old.reddit.com, -# m.youtube.com). +# m.youtube.com), unless a publisher's own link names it +# (engineering.linkedin.com, see `hosts_to_publishers`). PERSONAL_SUBDOMAIN_HOSTS = frozenset({"medium.com", "substack.com"}) _HOST_PLATFORMS = ( @@ -313,7 +314,9 @@ def hosts_to_publishers(publishers: Iterable[Publisher]) -> dict[str, Publisher] host = owner_key(link) or hostname(link) else: continue - if host and not is_shared_host(host): + # Only the bare platform: a subdomain a publisher links to + # (engineering.linkedin.com) is that publisher's own site. + if host and host not in SHARED_HOSTS: claims.setdefault(host, {})[p.id] = p return { host: next(iter(pubs.values())) diff --git a/backend/app/features/collect/test_publishers.py b/backend/app/features/collect/test_publishers.py index baa35bcf..47b8ca1f 100644 --- a/backend/app/features/collect/test_publishers.py +++ b/backend/app/features/collect/test_publishers.py @@ -130,9 +130,18 @@ def test_a_shared_platform_includes_its_subdomains(host: str, shared: bool) -> N assert is_shared_host(host) is shared -def test_a_publisher_on_a_platform_subdomain_claims_nothing() -> None: - sub = _pub(11, website="https://old.reddit.com/r/LocalLLaMA") - assert _credited("https://old.reddit.com/r/LocalLLaMA/comments/x", sub) is None +@pytest.mark.parametrize( + ("link", "url"), + [ + ("https://engineering.linkedin.com", "https://engineering.linkedin.com/blog/a"), + ("https://docs.github.com/en", "https://docs.github.com/en/copilot/agents"), + ("https://blog.x.com", "https://blog.x.com/engineering/en_us/a-post"), + ], +) +def test_a_publishers_own_link_claims_a_platform_subdomain(link: str, url: str) -> None: + """A platform's first-party site is the publisher's own, unlike the bare + platform host.""" + assert _credited(url, _pub(11, website=link)) == 11 def test_a_substack_writer_still_claims_their_own_subdomain() -> None: diff --git a/backend/app/features/collect/test_publishers_db.py b/backend/app/features/collect/test_publishers_db.py index a1dec259..c91173cb 100644 --- a/backend/app/features/collect/test_publishers_db.py +++ b/backend/app/features/collect/test_publishers_db.py @@ -112,6 +112,7 @@ def test_a_shared_host_that_names_no_author_stays_with_the_aggregator( "https://m.youtube.com/watch?v=abc", "https://youtu.be/abc", "https://mobile.twitter.com/someone/status/1", + "https://gist.github.com/someone/abc", ], ) def test_a_shared_platforms_subdomain_names_no_author_either( @@ -127,6 +128,26 @@ def test_a_shared_platforms_subdomain_names_no_author_either( assert resolver.created == [] +def test_a_platform_subdomain_a_publisher_links_goes_to_that_publisher( + db: Session, sources: dict[str, Publisher] +) -> None: + """The explicit link wins over the rule that keeps a platform's subdomains + with the aggregator.""" + engineering = Publisher( + name=f"{TAG} Engineering", + slug=f"{TAG}-engineering", + kind=PublisherKind.company, + type=PublisherType.rss, + links={LinkPlatform.website: f"https://{TAG}.linkedin.com"}, + ) + db.add(engineering) + db.commit() + resolver = PublisherResolver(db) + url = f"https://{TAG}.linkedin.com/blog/a-post" + assert resolver.publisher_for(url, sources["aggregator"].name).id == engineering.id + assert resolver.created == [] + + def test_a_feeds_own_item_stays_with_the_feed( db: Session, sources: dict[str, Publisher] ) -> None: diff --git a/backend/app/features/curation/test_tags.py b/backend/app/features/curation/test_tags.py index c5f21f72..f5e57404 100644 --- a/backend/app/features/curation/test_tags.py +++ b/backend/app/features/curation/test_tags.py @@ -1,6 +1,6 @@ """validate_tags: the last line of defence between whatever the LLM returns and what gets written as ArticleTag rows. Off-list tags must never be written, and -no tag is ever minted at runtime — see pipeline/tags.py. +no tag is ever minted at runtime. See app/features/curation/tags.py. """ from __future__ import annotations diff --git a/backend/app/main.py b/backend/app/main.py index 33535303..c452a631 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -2,6 +2,8 @@ from __future__ import annotations +import logging + import sentry_sdk from fastapi import APIRouter, FastAPI from fastapi.routing import APIRoute @@ -11,6 +13,13 @@ from app.entry.api import analytics, articles, likes, login, stories, users from app.entry.mcp.server import mcp_app +# uvicorn configures only its own loggers, and collect and report print to +# stdout, so the API's INFO lines (the dev sign-in link) need this. +logging.basicConfig(level=logging.INFO) +# They log every request at INFO. +logging.getLogger("httpx").setLevel(logging.WARNING) +logging.getLogger("httpcore").setLevel(logging.WARNING) + def custom_generate_unique_id(route: APIRoute) -> str: return f"{route.tags[0]}-{route.name}" diff --git a/backend/app/test_main.py b/backend/app/test_main.py index 7a2f3651..2bf6cab3 100644 --- a/backend/app/test_main.py +++ b/backend/app/test_main.py @@ -1,8 +1,13 @@ """What only the whole app can show: every table is declared in `app.core.db`, -and the routes and the collect run share one engine.""" +the routes and the collect run share one engine, and the API process logs at +INFO.""" from __future__ import annotations +import os +import subprocess +import sys +from pathlib import Path from typing import get_args from sqlmodel import SQLModel @@ -30,3 +35,29 @@ def test_every_table_lives_in_core_db() -> None: def test_routes_and_the_collect_run_share_the_engine() -> None: assert vars(collect)["engine"] is engine assert get_args(SessionDep)[1].dependency is get_session + + +def test_the_api_process_prints_the_dev_sign_in_link() -> None: + """A fresh interpreter, because pytest installs its own log handlers. Without + a logging config the link is dropped and a developer can't sign in.""" + # Empty env vars don't override `.env`, so the send is stubbed out instead: + # a developer's `.env` may hold a real Resend key. + script = ( + "import logging\n" + "import app.main\n" + "assert logging.getLogger('httpx').getEffectiveLevel() == logging.WARNING\n" + "from app.features.audience import email\n" + "email.send_email = lambda **_: None\n" + "email.send_link_email('dev@example.com', " + "email.generate_sign_in_email('t0ken'), 't0ken')\n" + ) + result = subprocess.run( + [sys.executable, "-c", script], + cwd=Path(__file__).parents[1], + env=os.environ | {"ENVIRONMENT": "development"}, + capture_output=True, + text=True, + ) + assert result.returncode == 0, result.stderr + assert "[dev] Sign-in link for dev@example.com" in result.stderr + assert "token=t0ken" in result.stderr diff --git a/backend/baml_client/inlinedbaml.py b/backend/baml_client/inlinedbaml.py index ee744bec..48cba6d1 100644 --- a/backend/baml_client/inlinedbaml.py +++ b/backend/baml_client/inlinedbaml.py @@ -14,7 +14,7 @@ "clients.baml": "// The provider caps how many requests may be in flight at once, far below its\n// per-minute quota: measured 2026-09-03, 5 parallel calls all answered while 10\n// returned 429 in ~90ms with x-ratelimit-remaining still at 1980/2000, and the\n// cap refills within ~10s. So a 429 here means \"too many at once\", never \"out\n// of budget\", and waiting is the correct response. Without this policy those\n// bursts fell through to the fallback client instead.\nretry_policy Burst {\n max_retries 4\n strategy {\n type exponential_backoff\n delay_ms 500\n multiplier 2\n max_delay_ms 8000\n }\n}\n\n// First in the Nvidia wrapper below since 2026-09-13. Second in line from 2026-09-03, replacing\n// NVIDIA's gpt-oss-120b, which reached end of life that morning and answered\n// every call with 410 Gone - dead weight for hours before anyone noticed,\n// because a fallback only speaks up when the primary fails.\n//\n// Measured 2026-09-03 on a SelectProductLink-shaped prompt: 1.5-6.8s across\n// sequential calls, all answering with clean JSON and the right pick — no sign\n// of the 52-285s latency that made gpt-oss-120b unusable, though that was a\n// property of the endpoint under load, so it is worth re-measuring if runs\n// start crawling. Past 8 parallel calls NIM starts returning 503; the retry\n// policy covers those.\n//\n// Qwen3.5-397B was the other candidate and judged just as well,\n// but it is a reasoning model: it spends its budget in a `reasoning` field and\n// emits an EMPTY content string if it runs out before it stops thinking (427\n// completion tokens for a two-element array). That needs a large explicit\n// max_tokens to be safe.\nclient Nemotron {\n provider openai-generic\n retry_policy Burst\n options {\n base_url env.NVIDIA_NIM_BASE_URL\n api_key env.NVIDIA_NIM_API_KEY\n model \"nvidia/nemotron-3-super-120b-a12b\"\n // The timeout lives in `http`, which BAML applies itself. A bare `timeout`\n // option is sent as a body param, and NIM answers that with 400\n // \"Unsupported parameter(s)\": every call here failed that way from\n // 2026-09-04 until this block replaced it.\n http {\n request_timeout_ms 120000\n }\n }\n}\n\n// Second in the Nvidia fallback. The smaller sibling of the gpt-oss-120b that\n// led scoring until its endpoint reached end of life. Measured on the old\n// scoring prompt (2026-09-13): clean JSON, but it read a rubric floor as a cap.\nclient GptOss20b {\n provider openai-generic\n retry_policy Burst\n options {\n base_url env.NVIDIA_NIM_BASE_URL\n api_key env.NVIDIA_NIM_API_KEY\n model \"openai/gpt-oss-20b\"\n http {\n request_timeout_ms 120000\n }\n }\n}\n\n// Last in the Nvidia fallback since 2026-09-13; primary from 2026-08-06 until\n// then. Fast (2-3s) and reliable, but a lower-quality judge: ±15 run-to-run\n// variance on identical scoring input, and it intermittently returned fewer\n// results than it was given. It stays as\n// the last resort because it is the only client not on NIM: when NIM refuses a\n// burst, the two clients above it fail together.\nclient Ministral {\n provider openai-generic\n retry_policy Burst\n options {\n base_url env.MISTRAL_AI_BASE_URL\n api_key env.MISTRAL_AI_KEY\n model \"mistralai/Ministral-3-14B-Instruct-2512\"\n // A real client-side timeout. The old `timeout 120` went out as a body\n // param, which this endpoint silently ignored: it never timed anything out.\n http {\n request_timeout_ms 120000\n }\n }\n}\n\n// The name every function references. A fallback wrapper: Nemotron first, then\n// gpt-oss-20b, then Ministral. The name stays - with two NIM models in front it\n// is mostly true.\n//\n// A Gemini Flash client sat third here until 2026-09-03, when the run that\n// needed it got 400 FAILED_PRECONDITION \"User location is not supported for\n// the API use\" - the Google AI API does not serve the prod VPS's region, so\n// the last resort had never been able to answer. Removed rather than left in\n// place: a client that always fails is not a fallback, it just hides how thin\n// the chain really is. A replacement has to be reachable from the box.\nclient Nvidia {\n provider fallback\n options {\n strategy [Nemotron, GptOss20b, Ministral]\n }\n}\n", "discover.baml": "// Batched relevance classifier for the Substack discovery crawler.\n// One LLM call judges many candidate profiles at once (array in, array out).\n// Classification runs on cheap\n// metadata (name + description) BEFORE any per-profile post fetch, and gates\n// both DB insertion and whether the crawler recurses into that profile.\n\nclass ProfileInput {\n handle string @description(\"Substack owner handle, echoed back in the verdict\")\n name string @description(\"Publication name\")\n description string? @description(\"Publication hero text and/or author bio, if known\")\n}\n\nclass ProfileVerdict {\n handle string @description(\"Must exactly match the input handle\")\n isAiRelated bool\n confidence int @description(\"0-100 confidence that AI/ML/LLMs is the PRIMARY focus\")\n}\n\nfunction ClassifyProfiles(profiles: ProfileInput[]) -> ProfileVerdict[] {\n client Nvidia\n prompt #\"\n {{ _.role(\"system\") }}\n You curate sources for a developer-focused AI news platform. For each Substack\n publication, decide whether artificial intelligence is its PRIMARY subject:\n AI/ML/LLMs, foundation models, AI engineering, AI products, or AI research.\n\n Mark isAiRelated = true ONLY when AI is clearly the main topic. Mark false for\n general tech, productivity, marketing, business, crypto, politics, lifestyle, or\n publications that merely mention AI in passing. When the signal is thin (sparse\n description, generic name), lean false and assign low confidence - the crawler\n prunes everything you reject, so false positives pollute the whole subtree.\n\n {{ _.role(\"user\") }}\n Classify every profile. Echo each handle back EXACTLY as given.\n\n {% for p in profiles %}\n {{ loop.index }}. handle={{ p.handle }} | {{ p.name }}{% if p.description %} — {{ p.description }}{% endif %}\n {% endfor %}\n\n {{ ctx.output_format }}\n \"#\n}\n", - "extract_items.baml": "// Newsletter extraction.\n//\n// A newsletter reports two kinds of thing worth ingesting, and they need\n// different URL resolution:\n//\n// - Product: a named launch (model, tool, SDK). Its canonical URL is the\n// vendor's own page, which the email rarely links to directly, so the URL is\n// rediscovered by web search (SelectProductLink) rather than trusted from the\n// email. This replaced the per-newsletter tracking-URL decoders.\n// - Article: a blog post, essay, paper, or write-up. Here the link IS the\n// subject — there is nothing to search for, and a search would land on some\n// other page about the same topic. Its href is kept and followed instead\n// (see ``_resolve_article`` in pipeline/sources/email.py), which turns a\n// tracking redirect into the real destination.\n//\n// Hence NewsletterItem.url: the model must copy an article's href verbatim\n// even though it is usually a redirect, because following it is exactly how\n// the real URL is recovered.\n\nenum NewsletterItemKind {\n Product // A named product, tool, model, SDK, dataset, or launch\n Article // A blog post, essay, paper, guide, or write-up\n}\n\nclass NewsletterItem {\n kind NewsletterItemKind\n name string @description(\"Product name, or the article's title\")\n description string @description(\"Concise phrase: what it is and does, usable verbatim as a web search query\")\n url string? @description(\"For an Article, the href of its link, copied verbatim including any tracking redirect. Omit for a Product.\")\n}\n\nfunction ExtractItems(newsletterText: string) -> NewsletterItem[] {\n client Nvidia\n prompt #\"\n {{ _.role(\"system\") }}\n You extract the AI/tech items a newsletter is reporting on: named products, and the articles it links to. Work only from the text; never invent items that are not mentioned.\n\n {{ _.role(\"user\") }}\n Below is the text of a newsletter email. Links appear as [anchor text](url).\n\n Identify every distinct item the newsletter reports as news, and classify each:\n\n - Product — a named product, tool, model, SDK, dataset, or launch a reader could go use. Judge it by the words, not the link: its URL is looked up separately, so do NOT return a url for a Product.\n - Article — a blog post, essay, paper, guide, benchmark write-up, or analysis the newsletter is pointing at. For these the link is the subject, so ALWAYS return its url, copied verbatim from the text — including tracking redirects, which are followed later to recover the real destination. An item with no link is not an Article.\n\n For each item return:\n - kind: Product or Article\n - name: the exact product name, or the article's title\n - description: a concise phrase describing what it is and does, suitable as a web search query\n - url: the article's href (Article only)\n\n A single story often yields one item, not two: if a link is the vendor's own announcement of a named launch, return it as a Product. Return an Article when the piece itself is the point — someone's write-up, teardown, guide, or opinion.\n\n INCLUDE only real, named launches and substantive articles a reader could go use or read.\n\n IGNORE:\n - the author's intro letter and personal commentary\n - sponsor blurbs, advertisements, \"brought to you by\", upgrade / subscribe / manage-preferences prompts\n - social-media embeds (an @handle with like / repost / reply counts) and the newsletter's own social links\n - section headers, navigation, and unsubscribe links\n - job postings\n\n Newsletter text:\n {{ newsletterText }}\n\n If the text names no qualifying items, return an empty list.\n\n {{ ctx.output_format }}\n \"#\n}\n\n// Notability gate: trims the extracted list to the items actually worth\n// resolving, so we don't spend a web search on every incremental SDK bump.\n// Returns the 1-based indices of the items to KEEP.\nfunction SelectNotableItems(items: NewsletterItem[]) -> int[] {\n client Nvidia\n prompt #\"\n {{ _.role(\"user\") }}\n Below is a numbered list of items a newsletter mentioned. Return the numbers of the ones that are NOTABLE enough to feature in an AI-news aggregator.\n\n KEEP an item if it is distinct, real, and individually newsworthy: new models, significant new products or platforms, notable agents/tools, major releases, and articles with real substance — technical write-ups, benchmarks, teardowns, engineering posts, or arguments a developer would want to read.\n\n DROP:\n - minor or incremental items: small SDK/utility bumps, niche helper libraries, design skills, config tweaks\n - vague or marketing-only entries with no concrete product behind them\n - listicles, SEO filler, press-release rewrites, and thin news blurbs that only restate an announcement\n - duplicates or sub-features of another item in the list\n\n Items:\n {% for item in items %}\n {{ loop.index }}. [{{ item.kind }}] {{ item.name }} — {{ item.description }}\n {% endfor %}\n\n Return only the numbers to keep, as a JSON array of integers. If none qualify, return [].\n\n {{ ctx.output_format }}\n \"#\n}\n\nclass SearchCandidate {\n title string\n url string\n snippet string\n}\n\nclass ProductLinkChoice {\n index int @description(\"1-based index of the best first-party result, or 0 if none qualifies\")\n confidence float @description(\"0.0 - 1.0\")\n}\n\n// Picks the best canonical URL for a product among search candidates, biased\n// hard toward the official / first-party source and away from social posts and\n// SEO/aggregator pages. Returns index 0 when nothing is clearly the product.\n// Products only — an article already carries its own link.\nfunction SelectProductLink(\n productName: string,\n productDescription: string,\n candidates: SearchCandidate[]\n) -> ProductLinkChoice {\n client Nvidia\n prompt #\"\n {{ _.role(\"user\") }}\n A newsletter featured this item:\n name: {{ productName }}\n description: {{ productDescription }}\n\n Web search returned these candidate results:\n {% for c in candidates %}\n {{ loop.index }}. {{ c.title }}\n url: {{ c.url }}\n snippet: {{ c.snippet }}\n {% endfor %}\n\n Pick the ONE candidate that is the canonical, first-party source for this exact product — its official site, product page, model card, repo, or the vendor's own announcement.\n\n PREFER the official / first-party domain that matches the product or its maker.\n REJECT (do not pick) when it is the best available:\n - social-media posts (x.com, twitter.com, linkedin.com, reddit.com, threads, mastodon)\n - videos (youtube.com)\n - content-farm / SEO / generic aggregator or listicle pages that merely mention the product\n - a DIFFERENT product that happens to share the name\n\n Return the 1-based index of the best first-party result. If none of the candidates is clearly a first-party source for THIS product, return index 0.\n\n {{ ctx.output_format }}\n \"#\n}\n\ntest extract_items_basic {\n functions [ExtractItems]\n args {\n newsletterText \"Headlines: [OpenAI ships GPT-5](https://sub.stack/redirect/2/abc), a new flagship model. Deep dive: [Why agent harnesses fail](https://stencil.so/blog/harness-playbook?utm_source=tldrai) walks through production agent design. Brought to you by Acme. [Unsubscribe](https://newsletter.com/unsub).\"\n }\n}\n", + "extract_items.baml": "// Newsletter extraction.\n//\n// A newsletter reports two kinds of thing worth ingesting, and they need\n// different URL resolution:\n//\n// - Product: a named launch (model, tool, SDK). Its canonical URL is the\n// vendor's own page, which the email rarely links to directly, so the URL is\n// rediscovered by web search (SelectProductLink) rather than trusted from the\n// email. This replaced the per-newsletter tracking-URL decoders.\n// - Article: a blog post, essay, paper, or write-up. Here the link IS the\n// subject — there is nothing to search for, and a search would land on some\n// other page about the same topic. Its href is kept and followed instead\n// (see ``_resolve_article`` in app/features/collect/newsletters.py), which turns a\n// tracking redirect into the real destination.\n//\n// Hence NewsletterItem.url: the model must copy an article's href verbatim\n// even though it is usually a redirect, because following it is exactly how\n// the real URL is recovered.\n\nenum NewsletterItemKind {\n Product // A named product, tool, model, SDK, dataset, or launch\n Article // A blog post, essay, paper, guide, or write-up\n}\n\nclass NewsletterItem {\n kind NewsletterItemKind\n name string @description(\"Product name, or the article's title\")\n description string @description(\"Concise phrase: what it is and does, usable verbatim as a web search query\")\n url string? @description(\"For an Article, the href of its link, copied verbatim including any tracking redirect. Omit for a Product.\")\n}\n\nfunction ExtractItems(newsletterText: string) -> NewsletterItem[] {\n client Nvidia\n prompt #\"\n {{ _.role(\"system\") }}\n You extract the AI/tech items a newsletter is reporting on: named products, and the articles it links to. Work only from the text; never invent items that are not mentioned.\n\n {{ _.role(\"user\") }}\n Below is the text of a newsletter email. Links appear as [anchor text](url).\n\n Identify every distinct item the newsletter reports as news, and classify each:\n\n - Product — a named product, tool, model, SDK, dataset, or launch a reader could go use. Judge it by the words, not the link: its URL is looked up separately, so do NOT return a url for a Product.\n - Article — a blog post, essay, paper, guide, benchmark write-up, or analysis the newsletter is pointing at. For these the link is the subject, so ALWAYS return its url, copied verbatim from the text — including tracking redirects, which are followed later to recover the real destination. An item with no link is not an Article.\n\n For each item return:\n - kind: Product or Article\n - name: the exact product name, or the article's title\n - description: a concise phrase describing what it is and does, suitable as a web search query\n - url: the article's href (Article only)\n\n A single story often yields one item, not two: if a link is the vendor's own announcement of a named launch, return it as a Product. Return an Article when the piece itself is the point — someone's write-up, teardown, guide, or opinion.\n\n INCLUDE only real, named launches and substantive articles a reader could go use or read.\n\n IGNORE:\n - the author's intro letter and personal commentary\n - sponsor blurbs, advertisements, \"brought to you by\", upgrade / subscribe / manage-preferences prompts\n - social-media embeds (an @handle with like / repost / reply counts) and the newsletter's own social links\n - section headers, navigation, and unsubscribe links\n - job postings\n\n Newsletter text:\n {{ newsletterText }}\n\n If the text names no qualifying items, return an empty list.\n\n {{ ctx.output_format }}\n \"#\n}\n\n// Notability gate: trims the extracted list to the items actually worth\n// resolving, so we don't spend a web search on every incremental SDK bump.\n// Returns the 1-based indices of the items to KEEP.\nfunction SelectNotableItems(items: NewsletterItem[]) -> int[] {\n client Nvidia\n prompt #\"\n {{ _.role(\"user\") }}\n Below is a numbered list of items a newsletter mentioned. Return the numbers of the ones that are NOTABLE enough to feature in an AI-news aggregator.\n\n KEEP an item if it is distinct, real, and individually newsworthy: new models, significant new products or platforms, notable agents/tools, major releases, and articles with real substance — technical write-ups, benchmarks, teardowns, engineering posts, or arguments a developer would want to read.\n\n DROP:\n - minor or incremental items: small SDK/utility bumps, niche helper libraries, design skills, config tweaks\n - vague or marketing-only entries with no concrete product behind them\n - listicles, SEO filler, press-release rewrites, and thin news blurbs that only restate an announcement\n - duplicates or sub-features of another item in the list\n\n Items:\n {% for item in items %}\n {{ loop.index }}. [{{ item.kind }}] {{ item.name }} — {{ item.description }}\n {% endfor %}\n\n Return only the numbers to keep, as a JSON array of integers. If none qualify, return [].\n\n {{ ctx.output_format }}\n \"#\n}\n\nclass SearchCandidate {\n title string\n url string\n snippet string\n}\n\nclass ProductLinkChoice {\n index int @description(\"1-based index of the best first-party result, or 0 if none qualifies\")\n confidence float @description(\"0.0 - 1.0\")\n}\n\n// Picks the best canonical URL for a product among search candidates, biased\n// hard toward the official / first-party source and away from social posts and\n// SEO/aggregator pages. Returns index 0 when nothing is clearly the product.\n// Products only — an article already carries its own link.\nfunction SelectProductLink(\n productName: string,\n productDescription: string,\n candidates: SearchCandidate[]\n) -> ProductLinkChoice {\n client Nvidia\n prompt #\"\n {{ _.role(\"user\") }}\n A newsletter featured this item:\n name: {{ productName }}\n description: {{ productDescription }}\n\n Web search returned these candidate results:\n {% for c in candidates %}\n {{ loop.index }}. {{ c.title }}\n url: {{ c.url }}\n snippet: {{ c.snippet }}\n {% endfor %}\n\n Pick the ONE candidate that is the canonical, first-party source for this exact product — its official site, product page, model card, repo, or the vendor's own announcement.\n\n PREFER the official / first-party domain that matches the product or its maker.\n REJECT (do not pick) when it is the best available:\n - social-media posts (x.com, twitter.com, linkedin.com, reddit.com, threads, mastodon)\n - videos (youtube.com)\n - content-farm / SEO / generic aggregator or listicle pages that merely mention the product\n - a DIFFERENT product that happens to share the name\n\n Return the 1-based index of the best first-party result. If none of the candidates is clearly a first-party source for THIS product, return index 0.\n\n {{ ctx.output_format }}\n \"#\n}\n\ntest extract_items_basic {\n functions [ExtractItems]\n args {\n newsletterText \"Headlines: [OpenAI ships GPT-5](https://sub.stack/redirect/2/abc), a new flagship model. Deep dive: [Why agent harnesses fail](https://stencil.so/blog/harness-playbook?utm_source=tldrai) walks through production agent design. Brought to you by Acme. [Unsubscribe](https://newsletter.com/unsub).\"\n }\n}\n", "generators.baml": "generator python {\n output_type \"python/pydantic\"\n output_dir \"../\"\n version \"0.223.0\"\n}\n", } diff --git a/backend/baml_src/extract_items.baml b/backend/baml_src/extract_items.baml index 0b55e5d9..8af704f2 100644 --- a/backend/baml_src/extract_items.baml +++ b/backend/baml_src/extract_items.baml @@ -10,7 +10,7 @@ // - Article: a blog post, essay, paper, or write-up. Here the link IS the // subject — there is nothing to search for, and a search would land on some // other page about the same topic. Its href is kept and followed instead -// (see ``_resolve_article`` in pipeline/sources/email.py), which turns a +// (see ``_resolve_article`` in app/features/collect/newsletters.py), which turns a // tracking redirect into the real destination. // // Hence NewsletterItem.url: the model must copy an article's href verbatim diff --git a/deploy/README.md b/deploy/README.md index 53dc16cc..7cfcc67f 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -14,13 +14,30 @@ How agentique runs in production. Docker runs only the database | `agentique-backup.service` + `.timer` | nightly DB dump to off-box storage at 03:30 | | `agentique-sql` | run SQL against the prod DB from stdin, inside the `db` container | | `sql-roles.sql` | one-time: creates the read-only and read/write login roles `agentique-sql` uses | +| `rsync-filter` | what the deploy copies to `/opt/agentique`, and what it never deletes there | ## Shape - A push to `master` triggers `.github/workflows/deploy-production.yml`: build the frontend on a GitHub-hosted runner, then a self-hosted runner on the box - rsyncs the source, syncs the venv, runs migrations + prestart, and restarts - the backend. Additive migrations only; rollback is `git revert` + push. + rsyncs what the box runs, syncs the venv, runs migrations + prestart, and + restarts the backend. Additive migrations only; rollback is `git revert` + + push. +- The rsync copies only what `rsync-filter` lists: the backend's `app/` + without tests, `migrations/`, `baml_client/`, the uv workspace files, + `compose.yml`, `deploy/` and the built frontend. It deletes everything + else in `/opt/agentique`, except `.env`, `.venv` and `.git`, which the + deploy never sends. A file the box needs goes in the filter. + `app/entry/cli/test_deploy_files.py` runs the filter and checks every path + the units and the deploy job use. +- The units, the Caddyfile and `agentique-sql` are installed by hand. After + a deploy that changes one, copy it from `/opt/agentique/deploy/` and run + `sudo systemctl daemon-reload`. If an `ExecStart` path changed, the + deploy has already deleted the old file. The backend then fails from the + deploy's own restart until you copy its unit and restart it, and each + timer fails the next time it fires. The deploy job doesn't install units + because the runner can rewrite anything under `/opt/agentique`. Letting + it install units with sudo would give it root. - The pipeline needs no restart — the timer starts a fresh process each run. - The night runs in three steps an hour apart: 04:00 the pipeline queues candidates, 05:00 the agent judges them, 06:00 the report goes out. The box @@ -146,13 +163,17 @@ Finish with a short report: stories created, extended and closed, and anything a ## Newsletter environment -Read by the backend through `app/core/settings.py`. +Read by the backend through `app/core/settings.py`, with two exceptions that +read `os.environ`: `subscribe` (`app/features/newsletter/service.py`) reads +`RESEND_API_KEY` and `RESEND_AUDIENCE_ID`, and `send_mail` +(`app/core/email.py`), which sends the daily mail, reads `RESEND_API_KEY` and +`EMAILS_FROM_EMAIL`. | Variable | Read by | Effect | | --- | --- | --- | -| `RESEND_API_KEY` | `app/newsletter/broadcast.py` | Unset: `draft_issue` refuses, naming it. | -| `RESEND_AUDIENCE_ID` | `app/newsletter/broadcast.py`, `app/newsletter/routes.py` | The audience signups are synced to and the issue is broadcast to. | -| `EMAILS_FROM_EMAIL`, `EMAILS_FROM_NAME` | `app/newsletter/broadcast.py` | The issue's sender. | +| `RESEND_API_KEY` | `app/features/newsletter/broadcast.py`, `app/features/newsletter/service.py`, `app/core/email.py` | Unset: `draft_issue` refuses, naming it, signups are not synced to Resend, and the daily mail is only logged. | +| `RESEND_AUDIENCE_ID` | `app/features/newsletter/broadcast.py`, `app/features/newsletter/service.py` | The audience signups are synced to and the issue is broadcast to. | +| `EMAILS_FROM_EMAIL`, `EMAILS_FROM_NAME` | `app/features/newsletter/broadcast.py`, `app/core/email.py` (email only) | The issue's sender, and the daily mail's. | ## Pipeline environment @@ -163,3 +184,9 @@ Read as raw `os.environ` beside each consumer (ADR 7), so they belong in | --- | --- | --- | | `OPENROUTER_API_KEY` | `app/core/jev.py` | Jev, which classifies each newsletter link (sponsor, plumbing, on-topic, kind). Unset fails every link, so the newsletter channel yields nothing. | | `PIPELINE_ALERT_EMAIL` | `app/core/email.py` | Where the daily mail goes (liveness, failing sources and feeds, what landed); falls back to `EMAILS_FROM_EMAIL`. | +| `IMAP_HOST`, `IMAP_USER`, `IMAP_PASSWORD`, `IMAP_PORT` | `app/features/collect/newsletters.py` | The inbox the newsletter source reads. The first three are required by that source; the port defaults to 993. | +| `HN_MIN_POINTS`, `HN_MIN_COMMENTS`, `HN_GRACE_HOURS` | `app/features/collect/hn.py` | The HN traction gate: 10 points or 5 comments. A story under 6 hours old is held back for the next run. A first-party URL skips the gate, and `HN_MIN_POINTS=0` turns it off. | +| `TAVILY_API_KEY` | `app/core/web_search.py` | Search for the lab-watch source, the newsletter source and the `web_search` tool. | +| `GITHUB_TOKEN` | `app/core/github.py` | Optional. Raises the GitHub API rate limit for repo lookups. | +| `RESIDENTIAL_PROXY_URL` | `app/core/http.py` | Optional. A metered proxy, tried when a feed fetch or a page extraction is blocked direct. | +| `KDRIVE_API_TOKEN`, `KDRIVE_DRIVE_ID`, `KDRIVE_BACKUP_DIR`, `KDRIVE_BACKUP_KEEP` | `app/entry/cli/backup_db.py` | Where the nightly dump goes. The dir defaults to `/Private/agentique-db`, and 14 dumps are kept. | diff --git a/deploy/rsync-filter b/deploy/rsync-filter new file mode 100644 index 00000000..50371b44 --- /dev/null +++ b/deploy/rsync-filter @@ -0,0 +1,35 @@ +# The deploy's rsync filter (.github/workflows/deploy-production.yml): what +# lands in /opt/agentique. The first matching rule wins. +# +# The deploy runs with --delete-excluded, so rsync deletes anything on the box +# that is not listed here, including files from earlier deploys. It never +# deletes the P paths. The deploy doesn't send them either. +P /.env +P /.venv/ +P /.git/ + +# Test-only code. backend/conftest.py falls under the final rule. +- __pycache__/ +- test_*.py +- /backend/app/core/db/factories.py + +# uv sync --frozen --package app reads the workspace root and the member. ++ /pyproject.toml ++ /uv.lock ++ /.python-version +# backup_db.py runs `docker compose -f /compose.yml`. ++ /compose.yml +# The units, agentique-sql, the Caddyfile and sql-roles.sql, for installing +# by hand. ++ /deploy/*** ++ /backend/ ++ /backend/pyproject.toml ++ /backend/.python-version ++ /backend/app/*** ++ /backend/migrations/*** ++ /backend/baml_client/*** +# Caddy serves the built frontend. The deploy downloads it into frontend/dist +# before the sync. ++ /frontend/ ++ /frontend/dist/*** +- * diff --git a/docs/adr/0017-layered-features.md b/docs/adr/0017-layered-features.md new file mode 100644 index 00000000..ab7913c8 --- /dev/null +++ b/docs/adr/0017-layered-features.md @@ -0,0 +1,115 @@ +# 17. Three layers, one folder per feature + +## Status + +Accepted. Supersedes 1 and 4. Changes part of 7 and 8. + +## Context + +ADR 1 gave `app/` one folder per domain and left `pipeline/` beside it in its +own shape. The domains still imported each other. Every table sat in some +domain's `models.py`, and the pipeline and every domain read `Article` from +`catalog/models.py`. Routes in `audience/` called into `catalog` and +`newsletter`. `catalog/stories.py` used the embedding maths in +`pipeline/curation.py`, and the newsletter used the curation maths, the +ledger, `url_kind` and `SNIPPET_CAP`. The MCP tools reached into +`pipeline/fetching`. + +Shared code had no single home. Some was in `app/platform/`, some in +`pipeline/`, and there were two engines: `app/platform/db.py` and +`pipeline/db.py`. Tests sat in a `tests/` folder in each domain (ADR 4), plus +`backend/tests/` for the pipeline, the scripts and the enum guard. The deploy +copied the whole checkout to the box. + +## Decision + + app/main.py wires entry.api and entry.mcp into one FastAPI app + entry/ api/ mcp/ cli/ the ways in: HTTP, MCP, command line + features/ catalog/ stories/ curation/ collect/ newsletter/ audience/ analytics/ + core/ db/ plus flat modules tables, engine, shared mechanism + +- **Three layers.** `entry/` composes features, features use `core/`, and + imports only point down. One rule covers every case, so a reader can tell + where a file goes from what it does. +- **Features never import each other.** When two features need the same + thing, it moves to `core/`. When an API call spans features (sign-up + creates a user and subscribes them), `entry/api/` composes them. +- **Every feature folder uses the same names.** `model.py` holds its own + types, never tables. `service.py` holds the verbs entry calls, as + `session` plus plain arguments. A file named after a `core` module + (`audience/email.py`) adapts it. Other files are private helpers named + after what they hold. A template sits beside its one consumer. +- **Tables live in `core/db/`, one module per table.** The table graph is + shared. Foreign keys cross feature lines, and the migrations are already + one history over the whole schema. Tables in a feature would force + sideways imports. `core/db/__init__.py` imports every table module, so + `SQLModel.metadata` is complete for autogenerate. `core/db/` also holds + the one engine and the few table rules two features need + (`like_counts_subquery`, `kind_from_url`). Queries one feature owns stay + in that feature. +- **Shared mechanism is flat in `core/`.** `settings.py`, `http.py`, + `extract.py`, `embedding.py` and the rest sit beside `db/`. A flat module + imports stdlib, third-party packages and other flat modules, never + `core.db`. There is no `shared/` folder, so ADR 3 needs no exception. +- **`entry/` has one file per router, one file per MCP tool, one file per + command.** A route, a tool or a command is an adapter: parse the input, + call a feature's `service`, shape the result. `entry/cli/` replaces + `backend/scripts/`, `pipeline/run.py` and `pipeline/report.py`. +- **Tests sit beside their module as `test_.py`.** No `tests/` + folders. `backend/conftest.py` holds the shared fixtures, and + `core/db/factories.py` the model factories. +- **import-linter enforces it.** `lint-imports` runs in pre-commit and in + the test-backend workflow, with five contracts in `backend/pyproject.toml`: + 1. layers `app.main` > `app.entry` > `app.features` > `app.core`, exhaustive, + so a new top-level module has to join a layer + 2. independence over `app.features.*` + 3. independence over `app.entry.api`, `app.entry.mcp`, `app.entry.cli` + 4. forbidden: the flat `app.core.*` modules never import `app.core.db` + 5. forbidden: entry reaches a feature only through its `service` and `model` +- **The deploy ships only what the box runs.** `deploy/rsync-filter` lists + `backend/app/` without tests or factories, `backend/migrations/`, + `backend/baml_client/`, the uv workspace files, `compose.yml`, `deploy/` + and the built frontend. The sync deletes everything else on the box except + `.env`, `.venv` and `.git`. + +What changes in earlier ADRs: + +- **1** is superseded. Domain folders with `models.py` / `service.py` / + `routes.py` / `tests/` become feature folders with `model.py` / + `service.py` and named helpers. Routes move to `entry/api/`. `platform/` + becomes `core/`. `pipeline/` is no longer an exception. It becomes + `features/collect/`, with its curation half in `features/curation/`. +- **4** is superseded. Tests still sit beside the code they test, but as + `test_.py` files next to the module, not in a `tests/` folder. + `backend/tests/` is gone, and factories moved to `core/db/factories.py`. + The frontend half of 4 is unchanged. +- **7** changes in part. There is one engine, in `core/db/engine.py`, and it + takes the database URL from `Settings`, so the collect run builds Settings + too. The box feeds both processes the same `.env`. The per-source knobs + (IMAP, HN thresholds, the Tavily and GitHub keys, the proxy) still read + `os.environ` in the module that uses them, and `deploy/README.md` still lists them. +- **8** changes in part. The stage-type chain splits at the queue. collect + owns `RawItem` -> `Candidate` (`features/collect/model.py`). curation owns + `Scored` -> `Summarized` -> `Persisted` (`features/curation/model.py`) and + builds `Scored` from the ledger row the agent judged. `Scored` no longer + subclasses `Candidate`, since that would import one feature into another. + +## Consequences + +`ls app/features/` lists what the backend does, and `ls` on one feature +lists its verbs, its types and its helpers. An import that breaks the rules +fails pre-commit and CI rather than review. + +A rule two features share has to move to `core/` before either can use it. +That move gets its own commit. + +The `stories` MCP tool (curation coverage) keeps its name although a +`stories/` feature now exists. Renaming it would change the curate rubric's +tools. + +Contract 5 checks import paths, not names. A feature that re-exports a helper +from its `__init__.py` would let entry reach it without a failure. + +The systemd units now run `app.entry.cli.*`. They are installed by hand, so +the deploy that changed `ExecStart` needed the units copied and +`daemon-reload` run before that night's timers. diff --git a/docs/backend.md b/docs/backend.md index 5e11b727..0b86100b 100644 --- a/docs/backend.md +++ b/docs/backend.md @@ -9,7 +9,7 @@ - Needs a root `.env` (gitignored): `PROJECT_NAME`, `SECRET_KEY`, `FIRST_SUPERUSER`, `POSTGRES_SERVER=localhost`, `POSTGRES_USER`, `POSTGRES_PASSWORD`, `POSTGRES_DB` - Backend loads it via pydantic-settings (`env_file="../.env"`) -- Pipeline reads raw `os.environ` directly, hence `--env-file` below +- The per-source knobs (IMAP, HN thresholds, API keys) are read from raw `os.environ`, hence `--env-file` below - `frontend/.env` is separate (Vite-only, `VITE_API_URL`) - keeps secrets out of the frontend build ```bash @@ -19,7 +19,7 @@ uv run bash app/entry/cli/prestart.sh # migrations + initial data uv run fastapi dev app/main.py ``` -Pipeline, manual run: +The nightly collect, run by hand: ```bash cd backend && uv run --env-file ../.env python -m app.entry.cli.collect @@ -45,48 +45,58 @@ nothing. Uses [prek](https://prek.j178.dev/). Config: `.pre-commit-config.yaml`. ```bash -uv run prek install -f # install the git hook, from backend/ +uv run prek install -f # install the git hook uv run prek run --all-files # run manually on everything ``` +- Runs ruff (lint and format) on every Python file, `mypy --strict` and `ty` over `backend/app`, typos, biome and the banned-module-names check +- Runs `lint-imports`, which checks the five import contracts in `backend/pyproject.toml` (ADR 17). The test-backend workflow runs it too +- Regenerates the frontend client when anything under `backend/` changes + ## Setup ```bash -cd backend uv sync source .venv/bin/activate ``` -- Point your editor's Python interpreter at `backend/.venv/bin/python` -- Tables live in `app/core/db/`, one module per table. API shapes stay in each domain's `models.py` -- Each domain folder holds its own `routes.py` and service module +- uv workspace: the root `pyproject.toml` lists `backend/` as its one member, so the venv is the root `.venv` +- Point your editor's Python interpreter at `.venv/bin/python` + +## Layout -VS Code: debugger and Python-tests-tab configs are already set up. +- `app/main.py` - the FastAPI app: the routers, then the MCP server at `/mcp` +- `app/entry/` - the ways in: `api/` one file per router, `mcp/` one file per tool, `cli/` one file per command +- `app/features/` - one folder per feature: catalog, stories, curation, collect, newsletter, audience, analytics. Each has `model.py` and `service.py` +- `app/core/` - `db/` (the tables, one module each, and the engine) plus flat shared modules (`settings.py`, `http.py`, `extract.py`, `embedding.py`, ...) +- Imports point down, entry to features to core, and features never import each other (ADR 17) ## Tests With the db up: ```bash +cd backend uv run bash ../scripts/tests-start.sh ``` -- Domain tests live in that domain's `tests/` folder (`app/catalog/tests/`, `app/audience/tests/`, ...) -- `backend/tests/` holds what belongs to no single domain: pipeline, schema-wide enum guard, ops scripts -- Shared fixtures: `backend/conftest.py` +- Each test sits beside its module as `test_.py` (`app/features/catalog/service.py` and `test_service.py`). No `tests/` folders +- Shared fixtures: `backend/conftest.py`. Model factories: `app/core/db/factories.py` - `tests-start.sh` waits for the db, then runs `scripts/test.sh` beside it (pytest + coverage). An argument becomes the coverage report's title, not a pytest flag: ```bash uv run bash ../scripts/tests-start.sh "my branch" ``` -- Coverage report: `htmlcov/index.html` +- Coverage report: `htmlcov/index.html`. Test files, `conftest.py` and `factories.py` are left out of the count ## Migrations - After changing a model, create a revision and apply it, or the schema and the app disagree - `app/core/db/__init__.py` imports every table module. A new table module needs a line there or autogenerate can't see it -- Alembic's config is `[tool.alembic]` in `backend/pyproject.toml` +- Alembic's config is `[tool.alembic]` in `backend/pyproject.toml`. Revisions are in `backend/migrations/versions/` + +From `backend/`: ```bash uv run alembic revision --autogenerate -m "Add column last_name to User model" @@ -101,6 +111,6 @@ uv run alembic upgrade head ## Email templates - Two transactional emails: `welcome.html` on signup and `sign_in.html` on a sign-in request, both carrying a fresh sign-in link -- Templates: `app/core/email-templates/`, next to `app/core/email.py`, which renders and sends them -- The weekly newsletter issue: `weekly_issue.html` in the same folder, the site's wordmark over plain prose, rendered (autoescaped) and drafted by `app/newsletter/broadcast.py` +- They sit in `app/features/audience/`, beside `email.py`, which builds them and sends them through `app/core/email.py` +- The weekly newsletter issue: `app/features/newsletter/weekly_issue.html`, the site's wordmark over plain prose, rendered (autoescaped) and drafted by `broadcast.py` beside it - No MJML source - edit the HTML directly diff --git a/docs/glossary.md b/docs/glossary.md index 14de0ff3..57d6a847 100644 --- a/docs/glossary.md +++ b/docs/glossary.md @@ -8,11 +8,11 @@ Glossary for words the code uses. If a term below and the code disagree, the cod - `kind` (`PublisherKind`) - what the publisher is: `lab` (makes models, writing on its own site), `company`, `individual`, `media`, `community` (an aggregator, credited only for its own pages), or `unknown` (a site the pipeline just made a publisher for; the curation agent names it on its first approval, and a kind once set stays). Shown to readers. - `type` (`PublisherType`) - how the pipeline finds its articles: `rss`, `substack`, `search`, `hn`, `reddit`, `email`, `ainews`, `other`. Internal ingestion detail, not on the public API. -- **Source** - one of the pipeline's fetch adapters (`pipeline/sources/*.py`): Hacker News, the RSS/Substack feeds, a lab-watch crawler, email newsletters. A source yields raw items for one or more publishers. Pipeline concept, not a database column. +- **Source** - one of the pipeline's fetch adapters (`hn.py`, `feeds.py`, `lab_watch.py` and `newsletters.py` in `app/features/collect/`): Hacker News, the RSS/Substack feeds, a lab-watch crawler, email newsletters. A source yields raw items for one or more publishers. Pipeline concept, not a database column. - **Trust** (`Publisher.trust`, `TrustLevel`) - `low` / `medium` / `high`, hand-set per publisher. No longer read (ADR 11): the curation agent sees each publisher's approval rate instead (`approved` in `list_candidates`: approvals / decisions over 90 days, or "new"). The column stays. Not on the public API. -- **Topic-gated** (`Publisher.topic_gated`) - a publisher flagged as mostly off-topic (a general engineering blog, not an AI one). No longer read: the fetch step used to drop its off-topic titles, and the curation agent judges topic now (ADR 11). The column stays. Hacker News keeps its own title keyword gate (`is_on_topic` in `pipeline/sources/hn.py`). +- **Topic-gated** (`Publisher.topic_gated`) - a publisher flagged as mostly off-topic (a general engineering blog, not an AI one). No longer read: the fetch step used to drop its off-topic titles, and the curation agent judges topic now (ADR 11). The column stays. Hacker News keeps its own title keyword gate (`is_on_topic` in `app/features/collect/hn.py`). - **Found via** (`Article.found_via`) - the aggregator (Hacker News, a newsletter) that found an article, when that is not its publisher. Null for an article from its publisher's own feed. @@ -21,24 +21,24 @@ Glossary for words the code uses. If a term below and the code disagree, the cod - **Score** (`Article.score`) - a 1-100 rating of how actionable the article is for a developer building with AI right now. Written by the curation agent, which reads the page before it decides (see **Candidate** below and ADR 9); an LLM in the pipeline used to write it from a title and a 200-char snippet. The one-sentence reason is kept in `Article.score_reason` (internal, not on the public API) and, for rejects, in `Reject.reason`. - **Candidate** - overloaded, and the two senses are one step apart. - - The *type* `Candidate` (`pipeline/types.py`) - a raw item resolved to its Publisher, mid-funnel. + - The *type* `Candidate` (`app/features/collect/model.py`) - a raw item resolved to its Publisher, mid-funnel. - A *candidate* in the feed's sense - an article the nightly run has queued and the curation agent has not judged yet: a `Reject` row at stage `pending`. It becomes an `Article` on approval and a `below_threshold` reject otherwise. Until then it is not visible anywhere a reader can see. - **Curation agent** - the Claude Code session that runs at 05:00, an hour after the pipeline, reads the pending candidates through the MCP tools (`list_candidates`, `get_content`, `similar`, `stories`, `check_link`, `approve`, `reject`, `reject_many`) and decides each one. Its rubric is `.claude/skills/curate/SKILL.md`. It is the only judge the feed has and the only thing that writes an `Article`. -- **Coverage** - how many distinct publishers carry one story, counted by `similar` and `stories` (`pipeline/curation.py`): rows within 0.30 cosine distance of each other across the feed and the ledger. Reach evidence for the curation agent, like traction. +- **Coverage** - how many distinct publishers carry one story, counted by `similar` and `stories` (`app/features/curation/service.py`): rows within 0.30 cosine distance of each other across the feed and the ledger. Reach evidence for the curation agent, like traction. -- **Story** (`Story`, `app/catalog/stories.py`) - a named thread that runs over days or weeks, like "Codemode": a name of 1-3 words, a two-sentence blurb, and its articles (`story_article`). Wider than coverage: coverage is one event told by several outlets, a story is a theme a builder follows across several events, which takes judgment. The stories routine (`.claude/skills/stories/SKILL.md`, ADR 16) names and keeps them through `story_candidates`, `save_story` and `close_story`. An article sits in at most one open story. The right rail (`GET /api/v1/stories`) shows open stories with 3+ articles from 2+ publishers. +- **Story** (`Story` in `app/core/db/story.py`, `app/features/stories/`) - a named thread that runs over days or weeks, like "Codemode": a name of 1-3 words, a two-sentence blurb, and its articles (`story_article`). Wider than coverage: coverage is one event told by several outlets, a story is a theme a builder follows across several events, which takes judgment. The stories routine (`.claude/skills/stories/SKILL.md`, ADR 16) names and keeps them through `story_candidates`, `save_story` and `close_story`. An article sits in at most one open story. The right rail (`GET /api/v1/stories`) shows open stories with 3+ articles from 2+ publishers. -- **Issue** - one weekly newsletter email and the blog post with the same text: a 250-450 word explainer of one topic, what it is, how it works and how people are taking it, linking the sources it was written from. The newsletter agent (`.claude/skills/newsletter/SKILL.md`) drafts three a week, one per topic, through `week`, `related` and `draft_issue` (`app/newsletter/`), and a person sends one. Each is a file in `frontend/content/blog/-.md`, the frontmatter giving the title (the subject), the description (the preheader) and the topic; the routine opens one PR with the three, and git stores them. +- **Issue** - one weekly newsletter email and the blog post with the same text: a 250-450 word explainer of one topic, what it is, how it works and how people are taking it, linking the sources it was written from. The newsletter agent (`.claude/skills/newsletter/SKILL.md`) drafts three a week, one per topic, through `week`, `related` and `draft_issue` (`app/features/newsletter/`), and a person sends one. Each is a file in `frontend/content/blog/-.md`, the frontmatter giving the title (the subject), the description (the preheader) and the topic; the routine opens one PR with the three, and git stores them. - **Traction** - outside signal that people found a story worth reading, independent of the agent's score: Hacker News points/comments past a minimum, which hold back low-traction submissions from the "everyone can post" source; a repo's stars or a model's downloads, which the curation agent reads with `check_link`; and coverage (above). A first-party URL (`app/core/urls.py`) skips the gate - a lab's own announcement counts as news at zero votes. - **Reject** (`Reject`, table `scored_url`) - a URL the funnel turned down, one row per URL. `stage` says which step dropped it - `below_threshold` for the agent's rejects; `thin_repo`, `prefilter`, `duplicate` and `unusable_summary` are on old rows only, from gates that are gone - or `pending`, the one stage that is not a rejection at all: a candidate still waiting on the agent. It keeps what that step saw: title, source, publisher, the first 2000 chars of content, traction, the repo / model / paper / docs `links` the article body makes, score, and the judge's `reason`. `filter_known_urls` reads it so a reject is never judged twice. Holds in a source (HN traction, recency) and a failed LLM call are deliberately not rejects: those come back next run. Rows from before 2026-09-13 carry only the URL. -- **Run** (`PipelineRun`) - one row per nightly pipeline execution: start/end time, duration, ok/fail, and per source `{fetched, queued, error}`. The daily mail (`pipeline/report.py`) reads the latest one for liveness. Per-publisher history lives on the Publisher row instead: `last_fetched_at` (its feed last answered), `last_new_at` (it last gave a new candidate), `last_error` (its last fetch failed, cleared by the next good one). +- **Run** (`PipelineRun`) - one row per nightly pipeline execution: start/end time, duration, ok/fail, and per source `{fetched, queued, error}`. The daily mail (`app/features/collect/report.py`) reads the latest one for liveness. Per-publisher history lives on the Publisher row instead: `last_fetched_at` (its feed last answered), `last_new_at` (it last gave a new candidate), `last_error` (its last fetch failed, cleared by the next good one). -- **RawItem / Candidate / Scored / Persisted** (`pipeline/types.py`) - the four shapes an article takes going down the funnel, one per stage. +- **RawItem / Candidate / Scored / Persisted** (`app/features/collect/model.py`, `app/features/curation/model.py`) - the four shapes an article takes going down the funnel, one per stage. - **RawItem** - what a source adapter emitted. Title, url, content, date, source name. Nothing else known yet. - **Candidate** - a raw item resolved to its Publisher: carries `publisher_id`. - **Scored** - a candidate the curation agent approved, with its score. The nightly run never produces one: it ends at a pending candidate, and `curation.approve` builds the shape from the row the agent judged. diff --git a/docs/product.md b/docs/product.md index 5b57b1b6..ad52f541 100644 --- a/docs/product.md +++ b/docs/product.md @@ -102,11 +102,12 @@ aren't wired into the nightly run yet — candidates for whoever picks up sourci ## Under the hood, briefly - **Backend**: FastAPI (Python), auto-generates the OpenAPI spec the frontend client is - built from. + built from. One package in three layers: `entry/` (the API, the MCP server, the + commands) calls `features/`, which call `core/` (docs/adr/0017). - **Database**: Postgres with the pgvector extension — one table holds articles plus their embedding vectors, so relevance search is just a SQL query. -- **Pipeline**: a separate scheduled Python process (once a day, 04:00) that fetches - and filters, plus a Claude Code session an hour later that judges what it queued and +- **Pipeline**: a separate scheduled Python process (once a day, 04:00, + `app.entry.cli.collect`) that fetches and filters, plus a Claude Code session an hour later that judges what it queued and a report email an hour after that. A second Claude Code session drafts the newsletter on Friday mornings. - **Embeddings**: model2vec — a tiny, fast, CPU-only static embedding model, no GPU or @@ -129,4 +130,4 @@ above, and a real load/scale test (current data volume is small — this has bee architecture spike more than a performance one so far). If you want the ground-level detail — endpoints, models, exact commands — see -`backend/README.md` and `frontend/README.md`. +`docs/backend.md` and `docs/frontend.md`. diff --git a/docs/style.md b/docs/style.md index c4a39031..43843d36 100644 --- a/docs/style.md +++ b/docs/style.md @@ -1,47 +1,64 @@ # Style -Rules a linter can enforce live in `backend/pyproject.toml` (ruff, mypy) and -`biome.json`, and fail CI. This file is the rest — style a linter cannot -express, enforced by review. +Rules a linter can enforce live in `backend/pyproject.toml` (ruff, mypy, +import-linter) and `biome.json`, and fail CI. This file holds the rest, the +style a linter cannot express, enforced by review. ## Backend -- **A route handler is an adapter.** Parse input, call one domain function, - shape the response. No SQL, no model loading, no branching business rules - in a handler body. -- **Domain functions take an explicit `session` plus plain arguments and - return domain types** — the same signature discipline as - `step(session, articles) -> survivors` in the pipeline. No hidden - globals. +- **Three layers, imports point down.** `app/entry/` (api, mcp, cli) calls + `app/features/`, and features call `app/core/`. Features never import each + other, and api, mcp and cli never import each other. `lint-imports` + enforces this (ADR 17). +- **A feature folder uses the same file names.** `model.py` for its own + types (no tables), `service.py` for the verbs entry calls, a file named + after a `core` module for the adapter over it (`audience/email.py`), and + private helpers named after what they hold (`facets.py`, `coverage.py`). + Entry imports a feature's `service` and `model` only. +- **Tables live in `app/core/db/`, one module per table.** A query only one + feature runs stays in that feature. A rule two features need on a table + goes beside the table (`like_counts_subquery` in `like.py`). +- **Shared mechanism is flat in `app/core/`.** A flat `core` module imports + stdlib, third-party packages and other flat `core` modules, never + `core.db`. When a second feature needs something from the first, it + moves to `core/`. +- **A route handler, an MCP tool and a command are adapters.** Parse input, + call feature verbs, shape the response or set the exit code. When one call + spans two features (sign-up creates a user and subscribes them), the + handler composes them; the features don't import each other. No SQL, no + model loading, no branching business rules in their bodies. One file per + router in `entry/api/`, per tool in `entry/mcp/`, per command in + `entry/cli/`. +- **Feature verbs take an explicit `session` plus plain arguments and return + domain types.** No hidden globals. - **Comments explain why, not what.** Well-named identifiers already say what the code does; a comment earns its place only for a hidden constraint, a subtle invariant, or a workaround for a specific bug. -- **Pipeline stage modules are verb-named** (`fetch`, `filter`, `queue`, - `persist`, `enrich`) — they are stages in a funnel. **Domain modules are - noun-named** (`articles`, `likes`, `publishers`) — they are resources. - This is a deliberate, permanent difference: forcing one naming scheme on - both would make each half read wrong. +- **collect's stage modules are verb-named** (`fetch`, `filter`, `queue`), + because they are stages in a funnel. **Other modules are noun-named** + (`facets`, `publishers`, `coverage`), because they are resources. One + naming scheme for both would make each half read wrong. - **Constants live with their single consumer.** A constant read by exactly - one module is an implementation detail of that module — it does not move - to a shared file. A constant read by two or more modules is a real - interface and lives at that seam (see `SNIPPET_CAP` in - `app/core/embedding.py`, read by curation and by the newsletter, for the - two-reader case). See the constants-vs-config discussion in the refactor plan for - the reasoning. + one module is an implementation detail of that module and does not move + to a shared file. A constant read by two or more features is an interface + and lives in `core` (`SNIPPET_CAP` in `app/core/embedding.py`, read by + curation and by the newsletter). - **Command-line entrypoints live in `app/entry/cli/`.** A module run by systemd, by `prestart.sh` or by hand (the nightly collect, the daily mail, - waiting for the DB, seeding) calls feature verbs and sets the exit code, - the way a route handler does for HTTP. Run it with - `python -m app.entry.cli.` from `backend/`. It is type-checked and - counted in coverage like the rest of `app/`. + waiting for the DB, seeding) calls feature verbs and sets the exit code. + Run it with `python -m app.entry.cli.` from `backend/`. It is + type-checked and counted in coverage like the rest of `app/`. - **Config is different from constants.** Deployment-varying, env-provided values (database URL, API keys) live in one settings object (`app/core/settings.py`). Tuning knobs compiled into the code (`CANDIDATE_CAP`, `SNIPPET_CAP`) never move there. -- **The pipeline is the exception, on purpose.** It reads `os.environ` beside - each consumer rather than through that settings object — see ADR 7. The index - of what it reads lives in `deploy/README.md`; add a row there when you add a - var. +- **Per-source knobs are the exception, on purpose.** The IMAP account, the + HN thresholds, the Tavily and GitHub keys and the proxy are read from + `os.environ` in the module that uses them (ADR 7). The index of what is read + lives in `deploy/README.md`; add a row there when you add a var. +- **Tests sit beside their module** as `test_.py`. No `tests/` + folders. Shared fixtures are in `backend/conftest.py`, model factories in + `app/core/db/factories.py` (ADR 17). ## Frontend @@ -58,7 +75,6 @@ express, enforced by review. - No module named `utils`, `helpers`, `common`, `shared`, or `misc` — see ADR 3. One documented exception: `frontend/src/lib/utils.ts`'s `cn()`, which is shadcn convention. -- A domain folder holds only `models.py`, `service.py` (or a domain-named - module), `routes.py`, `tests/` — see ADR 1. - Hide mechanism, never hide capability — see ADR 2. -- Tests colocate with the domain they test — see ADR 4. +- Tests sit beside the code they test. Backend: see ADR 17. Frontend: see + ADR 4. diff --git a/frontend/scripts/blog-posts.ts b/frontend/scripts/blog-posts.ts index 6a5d18fa..2d9d59b6 100644 --- a/frontend/scripts/blog-posts.ts +++ b/frontend/scripts/blog-posts.ts @@ -5,7 +5,7 @@ * * Only structural checks live here: what a page needs to render. The content * rules for an issue (lengths, word count, sources) live in - * backend/app/newsletter/broadcast.py, which refuses a post before it is ever + * backend/app/features/newsletter/broadcast.py, which refuses a post before it is ever * drafted. */ diff --git a/frontend/scripts/prerender-blog.ts b/frontend/scripts/prerender-blog.ts index ce1ba608..5973d57e 100644 --- a/frontend/scripts/prerender-blog.ts +++ b/frontend/scripts/prerender-blog.ts @@ -10,7 +10,7 @@ * The checks here are structural only (see blog-posts.ts): frontmatter parses, * title/description/topic present, filename is YYYY-MM-DD-slug.md, date not in * the future, slugs unique. Any violation exits non-zero and fails the build. - * The content rules for an issue live in backend/app/newsletter/broadcast.py + * The content rules for an issue live in backend/app/features/newsletter/broadcast.py * (draft_issue refuses a post that breaks them); the writing guide is * .claude/skills/newsletter/SKILL.md; the PR is the editorial review. * diff --git a/frontend/src/client/schemas.gen.ts b/frontend/src/client/schemas.gen.ts index 7f6fd0f4..cbee545c 100644 --- a/frontend/src/client/schemas.gen.ts +++ b/frontend/src/client/schemas.gen.ts @@ -322,7 +322,7 @@ export const OriginSchema = { title: 'Origin', description: `The feed's "From" filter: what the article is when that is settled (a repo, a paper, a model), otherwise who published it. Not stored; each -article falls under exactly one, see \`catalog.articles.origin_condition\`.` +article falls under exactly one, see \`catalog.rows.origin_condition\`.` } as const; export const OriginCountSchema = { diff --git a/frontend/src/client/types.gen.ts b/frontend/src/client/types.gen.ts index 0e6de889..9b947054 100644 --- a/frontend/src/client/types.gen.ts +++ b/frontend/src/client/types.gen.ts @@ -69,7 +69,7 @@ export type Message = { /** * The feed's "From" filter: what the article is when that is settled (a * repo, a paper, a model), otherwise who published it. Not stored; each - * article falls under exactly one, see `catalog.articles.origin_condition`. + * article falls under exactly one, see `catalog.rows.origin_condition`. */ export type Origin = 'lab' | 'company' | 'individual' | 'media' | 'repo' | 'paper' | 'model';