From c3a0aa0f68ccd6934936819af55d0c475164a428 Mon Sep 17 00:00:00 2001 From: BADRI NARAYANAN Date: Tue, 22 Sep 2026 11:45:22 -0700 Subject: [PATCH 01/12] Count a partly read document as read, as the rest of the app does MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `READ_STATUSES` has been ("ok", "partial") since ADR 0017, and the plan card, `search_plans` and `plan_coverage` all honour it: a `partial` document's text is stored, searched and quoted. Six other places still said `== "ok"`. `make sbc-report` therefore reported 1,918 of 3,276 plans covered where the rest of the app counts 1,940 — the number Phase 4 was signed off with was this bug's. The same literal listed partly-read documents among the reasons a document could not be read, skipped their kept PDFs under VERIFY=1, and hid them from the outdated-parser count even though `_needs_reading` re-reads them. Ingestion counted a current `partial` as a recorded failure and pointed the operator at `make refresh-sbc` for something that had not failed. All six derive from `READ_STATUSES` now, and the run's non-failure outcomes are derived from it too rather than listed by hand a fourth time. `fetch.py`'s two `!= "ok"` checks are left: a fetch is never `partial`. --- CHANGELOG.md | 8 ++++++ src/ingestion/sbc/ingest.py | 10 +++++-- src/ingestion/sbc/report.py | 19 ++++++++----- tests/test_sbc_ingest.py | 17 ++++++++++++ tests/test_sbc_report.py | 53 +++++++++++++++++++++++++++++++------ 5 files changed, 91 insertions(+), 16 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2cee7c5..5e79c91 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -295,6 +295,14 @@ ### Fixed +- `make sbc-report` counted only `ok` as read, so it disagreed with the plan + card, `search_plans` and `plan_coverage`, which all count `partial` too: it + reported 1,918 of 3,276 plans covered where the rest of the app counts 1,940 + (59.2%). It also listed partly-read documents among the reasons a document + could not be read, never hash-checked their kept PDFs under `VERIFY=1`, and + left them out of the outdated-parser count. Ingestion likewise counted a + current `partial` document as a recorded failure and told the operator to + refresh it. All six sites now derive from `READ_STATUSES`. - A thread whose history failed to load was shown as an empty conversation, suggested prompts and all, because the transcript had no error branch and fell through to the empty state. It now names the failure and offers to load diff --git a/src/ingestion/sbc/ingest.py b/src/ingestion/sbc/ingest.py index 3e10f27..40d14c7 100644 --- a/src/ingestion/sbc/ingest.py +++ b/src/ingestion/sbc/ingest.py @@ -59,6 +59,12 @@ logger = get_logger(__name__) +# Outcomes that leave the plan with searchable text: the document was read, a +# refresh found it changed and read it again, or the host was unreachable and +# the text already stored was kept. Anything else is a failure worth naming. +KEPT_OUTCOMES = (*READ_STATUSES, *(f"changed:{status}" for status in READ_STATUSES), + "unchanged", "unreachable") + _HIOS_ISSUER_ID = re.compile(r"^\d{5}$") @@ -280,7 +286,7 @@ def execute(states: list[str], year: int, limit: int | None = None, pending = [url for url in documents if _needs_reading(url, stored.get(url), year, refresh)] urls = pending[:limit] skipped = sum(1 for url in documents - if url not in pending and stored.get(url) and stored[url].status != "ok") + if url not in pending and stored.get(url) and stored[url].status not in READ_STATUSES) print(f"{len(documents)} SBC documents behind the {', '.join(states)} plans for {year}; " f"{len(documents) - len(pending)} already current, " f"{'re-checking' if refresh else 'reading'} {len(urls)}") @@ -294,7 +300,7 @@ def execute(states: list[str], year: int, limit: int | None = None, status = refresh_document(url, year, row) if refresh and row and row.status in READ_STATUSES \ else ingest_document(url, year, row) statuses[status] += 1 - if status not in ("ok", "partial", "unchanged", "changed:ok", "changed:partial", "unreachable"): + if status not in KEPT_OUTCOMES: for plan in documents[url]: failures[(plan.issuer, status)].append(plan) diff --git a/src/ingestion/sbc/report.py b/src/ingestion/sbc/report.py index 24cef40..e8e7c3c 100644 --- a/src/ingestion/sbc/report.py +++ b/src/ingestion/sbc/report.py @@ -20,7 +20,7 @@ from src.core.db import get_session from src.models.plan import Issuer, Plan from src.models.sbc import SbcDocument -from src.services.sbc_status import plan_sbc_status, sbc_document_join +from src.services.sbc_status import READ_STATUSES, plan_sbc_status, sbc_document_join from ..constants import SBC_ARCHIVE, SBC_RAW, SBC_REJECTED from ..plans import resolve_states @@ -44,7 +44,13 @@ class Coverage: @property def read(self) -> int: - return self.statuses["ok"] + """Plans with searchable text, counted as the rest of the app counts it. + + `partial` is read: its text is stored, searched and quoted. Counting + only `ok` here made the report disagree with the plan card and the + answer the user sees. + """ + return sum(self.statuses[status] for status in READ_STATUSES) def add(self, status: str, plans: int) -> None: self.plans += plans @@ -93,7 +99,7 @@ def collect(session, year: int, states: list[str] | None = None, verify: bool = orphans=_orphans(session, year), outdated=session.scalar( select(func.count()).select_from(SbcDocument) - .where(SbcDocument.plan_year == year, SbcDocument.status == "ok", + .where(SbcDocument.plan_year == year, SbcDocument.status.in_(READ_STATUSES), SbcDocument.parser_version != PARSER_VERSION) ), stale_plans=_stale_plans(session, year, states), @@ -105,7 +111,7 @@ def collect(session, year: int, states: list[str] | None = None, verify: bool = def _reasons(session, year: int) -> Counter: rows = session.execute( select(SbcDocument.status, SbcDocument.detail, func.count()) - .where(SbcDocument.plan_year == year, SbcDocument.status != "ok") + .where(SbcDocument.plan_year == year, SbcDocument.status.notin_(READ_STATUSES)) .group_by(SbcDocument.status, SbcDocument.detail) ).all() return Counter({(status, detail): count for status, detail, count in rows}) @@ -152,7 +158,7 @@ def _verify(session, year: int) -> dict: missing, changed = [], [] documents = session.execute( select(SbcDocument.url, SbcDocument.sha256) - .where(SbcDocument.plan_year == year, SbcDocument.status == "ok") + .where(SbcDocument.plan_year == year, SbcDocument.status.in_(READ_STATUSES)) ).all() for url, sha256 in documents: path = cache_path(url, year) @@ -165,7 +171,8 @@ def _verify(session, year: int) -> dict: def _line(name: str, coverage: Coverage) -> str: share = f"{100 * coverage.read / coverage.plans:5.1f}%" if coverage.plans else " -" - missing = ", ".join(f"{count} {status}" for status, count in coverage.statuses.most_common() if status != "ok") + missing = ", ".join(f"{count} {status}" for status, count in coverage.statuses.most_common() + if status not in READ_STATUSES) return f" {name[:52]:<52} {coverage.plans:>6} {coverage.read:>6} {share} {missing}" diff --git a/tests/test_sbc_ingest.py b/tests/test_sbc_ingest.py index d56a2e1..b1a38bb 100644 --- a/tests/test_sbc_ingest.py +++ b/tests/test_sbc_ingest.py @@ -364,3 +364,20 @@ def test_a_document_missing_some_of_its_questions_says_how_many(sbc, session): detail = session.scalar(select(SbcDocument.detail).where(SbcDocument.url == GOLD)) assert detail == "missing 3 of the 7 questions" + + +def test_a_partly_read_document_is_neither_a_failure_nor_a_skipped_one(sbc, fetches, catalog, session, capsys): + """It was read. Listing it as a failure would send the operator to refresh what did not fail.""" + pages = _pages() + kept = [row for row in pages[0].rows if row.label not in QUESTION_HEADINGS[:3]] + sbc[GOLD] = [PdfPage(text=pages[0].text, rows=tuple(kept))] + + ingest.execute(["NH"], YEAR) + + assert _status(session, GOLD) == "partial" + assert "Plans without a usable SBC" not in capsys.readouterr().out + + fetches.clear() + assert ingest.execute(["NH"], YEAR) == Counter() + assert "recorded failures skipped" not in capsys.readouterr().out + assert fetches == [] diff --git a/tests/test_sbc_report.py b/tests/test_sbc_report.py index d84902f..aa42cc3 100644 --- a/tests/test_sbc_report.py +++ b/tests/test_sbc_report.py @@ -13,6 +13,7 @@ from src.ingestion.sbc.report import collect, render from src.models.plan import Issuer, Plan from src.models.sbc import SbcChunk, SbcDocument +from src.services.sbc_status import READ_STATUSES YEAR = 1999 HCSC = "33602" # in TOP_ISSUERS, under Health Care Service Corporation @@ -36,17 +37,21 @@ def plan(issuer_id, plan_id, url, state="TX"): plan(HCSC, "33602TX0010001", "https://sbc.example.com/read.pdf") plan(HCSC, "33602TX0010002", "https://sbc.example.com/read.pdf") plan(HCSC, "33602TX0010003", "https://sbc.example.com/blocked.pdf") + plan(HCSC, "33602TX0010004", "https://sbc.example.com/partial.pdf") plan("99999", "99999TX0010001", None) plan("99999", "99999TX0010002", "https://sbc.example.com/never-tried.pdf") plan("99999", "99999NH0010003", "https://sbc.example.com/read.pdf", state="NH") - for url, status in (("read.pdf", "ok"), ("blocked.pdf", "blocked")): + documents = (("read.pdf", "ok", None), + ("partial.pdf", "partial", "missing 2 of the 10 chart's groups"), + ("blocked.pdf", "blocked", "robots.txt disallows it")) + for url, status, detail in documents: document = SbcDocument(url=f"https://sbc.example.com/{url}", plan_year=YEAR, status=status, - detail=None if status == "ok" else "robots.txt disallows it", - sha256=hashlib.sha256(url.encode()).hexdigest(), parser_version=3) + detail=detail, sha256=hashlib.sha256(url.encode()).hexdigest(), + parser_version=3) session.add(document) session.flush() - if status == "ok": + if status in READ_STATUSES: session.add(SbcChunk(document_id=document.id, chunk_id=f"{url}#0", section="If you have a test", position=0, content="If you have a test\nImaging $100")) session.flush() @@ -56,15 +61,33 @@ def _issuer_line(report, name): return next(coverage for (_, issuer), coverage in report.by_issuer.items() if issuer.startswith(name)) +def _line_for(report, name): + return next(line for line in render(report).splitlines() if line.strip().startswith(name)) + + def test_every_plan_is_counted_by_whether_its_document_was_read(session, catalog): report = collect(session, YEAR) - assert (report.total.plans, report.total.read) == (6, 3) - assert report.total.statuses == {"ok": 3, "blocked": 1, "no_link": 1, "not_read": 1} - assert report.by_state["TX"].plans == 5 + assert (report.total.plans, report.total.read) == (7, 4) + assert report.total.statuses == {"ok": 3, "partial": 1, "blocked": 1, "no_link": 1, "not_read": 1} + assert report.by_state["TX"].plans == 6 assert report.by_state["NH"].statuses == {"ok": 1} +def test_a_partly_read_document_counts_as_read(session, catalog): + """`partial` text is stored, searched and quoted, so the report must not call it missing. + + Counting only `ok` made `make sbc-report` disagree with the plan card and + the answer: it reported 1,918 of 3,276 plans covered where the rest of the + app counted 1,940. + """ + report = collect(session, YEAR) + hcsc = _issuer_line(report, "BCBS of Texas") + + assert hcsc.read == 3 # two `ok` plans and the `partial` one + assert "partial" not in _line_for(report, "ALL") + + def test_a_plan_with_no_link_and_one_never_read_are_both_counted(session, catalog): """Plans with no link were left out of ingestion's report entirely.""" small = _issuer_line(collect(session, YEAR), "Small Health") @@ -85,6 +108,7 @@ def test_a_state_filter_counts_only_that_state(session, catalog): def test_why_each_document_could_not_be_read_is_counted(session, catalog): + """A `partial` document was read, so it does not belong among the reasons it wasn't.""" assert collect(session, YEAR).reasons == {("blocked", "robots.txt disallows it"): 1} @@ -96,9 +120,10 @@ def test_a_document_no_plan_points_at_is_reported_as_an_orphan(session, catalog) def test_documents_stored_by_an_older_parser_are_counted(session, catalog): + """Both read statuses: `_needs_reading` re-reads a `partial` document too.""" session.execute(SbcDocument.__table__.update().values(parser_version=1)) - assert collect(session, YEAR).outdated == 1 + assert collect(session, YEAR).outdated == 2 def test_plans_the_latest_catalog_run_did_not_return_are_reported(session, catalog): @@ -138,8 +163,20 @@ def test_kept_pdfs_are_counted_by_folder(session, catalog, monkeypatch, tmp_path def test_verifying_files_names_a_missing_one_and_one_whose_hash_changed(session, catalog, monkeypatch, tmp_path): monkeypatch.setattr(sbc_report, "cache_path", lambda url, year: tmp_path / f"{url.rsplit('/', 1)[1]}") (tmp_path / "read.pdf").write_bytes(b"a different file") + (tmp_path / "partial.pdf").write_bytes(b"partial.pdf") # the bytes its stored hash was taken from report = collect(session, YEAR, verify=True) assert (report.changed_files, report.missing_files) == (["https://sbc.example.com/read.pdf"], []) assert "files whose hash changed: 1" in render(report) + + +def test_verifying_files_checks_a_partly_read_documents_pdf_too(session, catalog, monkeypatch, tmp_path): + """Its text is quoted in answers, so its kept PDF is worth the same check as any other.""" + monkeypatch.setattr(sbc_report, "cache_path", lambda url, year: tmp_path / f"{url.rsplit('/', 1)[1]}") + (tmp_path / "read.pdf").write_bytes(b"read.pdf") + + report = collect(session, YEAR, verify=True) + + assert report.missing_files == ["https://sbc.example.com/partial.pdf"] + assert report.changed_files == [] From de4d4d932ace7bbecd6c6b42c42d799b791e958b Mon Sep 17 00:00:00 2001 From: BADRI NARAYANAN Date: Tue, 22 Sep 2026 11:52:41 -0700 Subject: [PATCH 02/12] Give the API logs, and answer 401 when a token names nobody MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two things the API process got wrong, both invisible until something went looking. `setup_logging()` is called by every command-line entry point and by all four eval scripts, but never by `create_app()`. The root logger therefore kept Python's default in the Flask process: WARNING, with no handler. Every logger.info in the request path — retrieval's hybrid-search diagnostics, plan search, tool outcomes, the rewrite decisions, answer stats — went nowhere, and `logs/backend/app.log` held only what the ingestion and eval runs wrote. The per-library WARNING pins went uninstalled too; they exist to keep the CMS api key out of urllib3 and a user's ZIP and age out of httpx and openai, and an absent safeguard is worth fixing before this runs behind a WSGI server that configures logging itself. `get_current_user()` is annotated `-> User` and guarded neither case that breaks it. A subject that is not one of our IDs raised ValueError out of `uuid.UUID`; an account deleted since its token was issued left `db.get` returning None, which every caller dereferences. Both surfaced as a generic 500. A token that names nobody is a dead session, so both are 401 now and the client signs out instead of seeing a server error. --- CHANGELOG.md | 10 ++++++++++ src/api/deps.py | 23 ++++++++++++++++++++--- src/api/main.py | 22 ++++++++++++++++++++-- tests/test_api_logging.py | 26 ++++++++++++++++++++++++++ tests/test_auth_api.py | 29 +++++++++++++++++++++++++++++ 5 files changed, 105 insertions(+), 5 deletions(-) create mode 100644 tests/test_api_logging.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 5e79c91..36a9291 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -295,6 +295,16 @@ ### Fixed +- The API never configured logging. Every command-line entry point calls + `setup_logging()`; `create_app()` did not, so in the Flask process the root + logger kept Python's default of WARNING with no handler: nothing the request + path logged reached `logs/backend/app.log`, and the WARNING pins that keep + the CMS api key out of `urllib3` and a user's ZIP and age out of `httpx` and + `openai` were never installed. +- A signed, unexpired token naming no usable account returned 500. A subject + that is not one of our IDs raised `ValueError`, and an account deleted since + the token was issued left `get_current_user()` returning `None` for callers + that dereference it. Both are now 401, so the client signs out cleanly. - `make sbc-report` counted only `ok` as read, so it disagreed with the plan card, `search_plans` and `plan_coverage`, which all count `partial` too: it reported 1,918 of 3,276 plans covered where the rest of the app counts 1,940 diff --git a/src/api/deps.py b/src/api/deps.py index 6f47172..b35b7f3 100644 --- a/src/api/deps.py +++ b/src/api/deps.py @@ -23,6 +23,10 @@ def __init__(self, retry_after: int): self.retry_after = retry_after +class InvalidSessionError(Exception): + """Raised when a signed token names no usable account (401). Carries no detail.""" + + def get_db() -> Session: """One SQLAlchemy session per request, reused across the request via flask.g.""" if "db" not in g: @@ -43,10 +47,23 @@ def close_db(_exception: BaseException | None = None) -> None: def get_current_user() -> User: - """The authenticated user for this request. Call only inside a @jwt_required() view.""" + """The authenticated user for this request. Call only inside a @jwt_required() view. + + A token can be signed and unexpired and still name nobody: its subject may + not be one of our IDs, or the account may have been deleted since. Callers + dereference the result, so returning None here became a 500 where the + honest answer is 401 — the session is no good, sign in again. + """ db = get_db() - user_id = uuid.UUID(get_jwt_identity()) - return db.get(User, user_id) + try: + user_id = uuid.UUID(get_jwt_identity()) + except (TypeError, ValueError) as exc: + raise InvalidSessionError from exc + + user = db.get(User, user_id) + if user is None: + raise InvalidSessionError + return user def parse_body[T: BaseModel](schema: type[T]) -> T: diff --git a/src/api/main.py b/src/api/main.py index 67e6952..1ff05ef 100644 --- a/src/api/main.py +++ b/src/api/main.py @@ -5,18 +5,29 @@ from flask_jwt_extended import JWTManager from werkzeug.exceptions import HTTPException -from src.api.deps import TokenBudgetExhaustedError, ValidationFailedError, close_db +from src.api.deps import ( + InvalidSessionError, + TokenBudgetExhaustedError, + ValidationFailedError, + close_db, +) from src.api.limiter import limiter from src.api.routes.auth import bp as auth_bp from src.api.routes.chat import bp as chat_bp from src.api.routes.profile import bp as profile_bp -from src.core.logging import get_logger +from src.core.logging import get_logger, setup_logging from src.policypal.config import settings logger = get_logger(__name__) def create_app() -> Flask: + # Every command-line entry point configures logging; the API never did, so + # in this process the root logger kept Python's default (WARNING, no + # handler): nothing the request path logged was written anywhere, and the + # WARNING pins that keep the CMS api key and a user's ZIP out of urllib3, + # httpx and openai output were never installed. + setup_logging() app = Flask(__name__) app.config["JWT_SECRET_KEY"] = settings.jwt_secret_key.get_secret_value() @@ -38,6 +49,13 @@ def create_app() -> Flask: def _handle_validation_error(exc: ValidationFailedError): return jsonify(error="validation failed", details=exc.errors), 422 + @app.errorhandler(InvalidSessionError) + def _handle_invalid_session(_exc: InvalidSessionError): + # The token is signed and unexpired but names no account we can use. + # 401 lets the client sign out cleanly; the body says nothing about + # which of the two reasons it was. + return jsonify(error="invalid session"), 401 + @app.errorhandler(TokenBudgetExhaustedError) def _handle_token_budget(exc: TokenBudgetExhaustedError): # 429 like the rate limiter, but a distinct error string: "slow down" diff --git a/tests/test_api_logging.py b/tests/test_api_logging.py new file mode 100644 index 0000000..9661502 --- /dev/null +++ b/tests/test_api_logging.py @@ -0,0 +1,26 @@ +"""The API process configures logging like every other entry point. + +It did not, and the cost was invisible: the root logger kept Python's default +of WARNING with no handler, so nothing the request path logged reached +`logs/backend/app.log`, and the third-party levels that keep the CMS api key +and a user's ZIP out of the log were never installed. +""" +import logging +from logging.handlers import RotatingFileHandler + + +def test_creating_the_app_installs_the_file_handler(app): + root = logging.getLogger() + + assert any(isinstance(handler, RotatingFileHandler) for handler in root.handlers) + + +def test_the_request_path_can_log_at_info(app): + """The default root level of WARNING discarded every logger.info in a request.""" + assert logging.getLogger("src.services.generation").isEnabledFor(logging.INFO) + + +def test_the_libraries_that_would_log_secrets_stay_at_warning(app): + """urllib3 logs the CMS URL with its ?apikey=; httpx and openai log the prompt.""" + for name in ("urllib3", "httpx", "openai"): + assert logging.getLogger(name).level == logging.WARNING diff --git a/tests/test_auth_api.py b/tests/test_auth_api.py index 2843c2e..977d81b 100644 --- a/tests/test_auth_api.py +++ b/tests/test_auth_api.py @@ -1,3 +1,8 @@ +from flask_jwt_extended import create_access_token +from sqlalchemy import delete + +from src.api import deps +from src.models.user import User from tests.helpers import PROFILE @@ -69,3 +74,27 @@ def test_login_validation_error(client): resp = client.post("/api/auth/login", json={"email": "not-an-email"}) assert resp.status_code == 422 + + +def test_a_token_naming_a_deleted_account_is_rejected_not_a_server_error(client, app): + """Every view dereferences get_current_user(), so returning None was a 500.""" + token = _register(client).get_json()["access_token"] + db = deps.SessionLocal() + db.execute(delete(User).where(User.email == "alice@example.com")) + db.commit() + db.close() + + resp = client.get("/api/auth/me", headers={"Authorization": f"Bearer {token}"}) + + assert resp.status_code == 401 + assert resp.get_json() == {"error": "invalid session"} + + +def test_a_token_whose_subject_is_not_a_user_id_is_rejected(client, app): + with app.app_context(): + token = create_access_token(identity="not-a-uuid") + + resp = client.get("/api/auth/me", headers={"Authorization": f"Bearer {token}"}) + + assert resp.status_code == 401 + assert resp.get_json() == {"error": "invalid session"} From 252cd2940c7e9a249c5c4bf4ae47849e57c25719 Mon Sep 17 00:00:00 2001 From: BADRI NARAYANAN Date: Tue, 22 Sep 2026 11:59:53 -0700 Subject: [PATCH 03/12] Make the documents say what the code and the database say MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An audit of the finished project found three documents stating counts the database contradicts. For a project whose Phase 4 thesis is saying honestly what was read, that is the wrong kind of error to leave. ADR 0018 owns parser decisions and stopped at version 6: its status line said "parser versions 3 to 6" and its body said "`PARSER_VERSION` is 3", while extract.py has been at 9 since the live run. Versions 7, 8 and 9 — the chart ruled across but not down, the header printed in pieces, and the template's closing sentence read as a chart row — were documented nowhere but the findings. They have sections now, and the ADR says where the number lives rather than restating it. ADR 0017 and the CHANGELOG still said 62 of 1,189 documents are partial. The parser work those 62 prompted brought it to 22 within the same phase, and the refresh took the corpus to 1,246. ADR 0017 also gains the point the audit turned up the hard way: `READ_STATUSES` is the single definition of "there is text for this plan", and spelling it `== "ok"` anywhere else reintroduces the silent disagreement that section exists to stop. The CHANGELOG had no entry for either of the two largest runs in the project — the eighteen-state ingest and the first full refresh — though Phase 3's much smaller run has one. Both are recorded now, with the settled OCR answer and the eval floor raised to full marks, which also shipped without an entry. README and CLAUDE.md called the plan comparison "in progress" when all five phases are complete; they also omitted docs/findings, docs/runbooks and eval_sbc_ranking.py from the structure, and never named the `partial` status. ADR 0005 said follow-up latency was "the thing to revisit first if generation moves behind an API". ADR 0008 moved it. Recorded as still open rather than quietly left. --- CHANGELOG.md | 22 +++++++- CLAUDE.md | 2 +- README.md | 22 ++++++-- docs/decisions/0005-conversation-history.md | 7 +++ ...017-saying-which-plans-have-no-document.md | 14 ++++- .../0018-chart-rows-from-the-ruled-grid.md | 55 +++++++++++++++++-- 6 files changed, 108 insertions(+), 14 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 36a9291..8753858 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,20 @@ ### Added +- Phase 4 live ingest: the catalog and SBCs for eighteen HealthCare.gov states + — 3,276 plans from 137 issuers, every issuer's documents rather than the top + parents'. 1,246 documents are stored and 1,940 plans (59.2%) have text behind + them; the rest are named and counted, 995 of them blocked by their issuer. + Six parser defects found in real PDFs were fixed along the way (ADR 0018), + and the results are in `docs/findings/sbc-documents.md`. +- First refresh at full scale: 2,344 documents re-checked across all eighteen + states (ADR 0019). 1,116 unchanged, 995 still blocked, 55 unreachable, and + **18 that the issuer had replaced at the same URL** — the case nothing else + would have noticed. Each superseded file moved to `data/sbc/archive/`, none + deleted. Retrying past failures recovered 57 documents. +- Settled ADR 0018's deferred OCR question by measurement: **zero** of the + documents read in eighteen states lack a text layer, so no issuer publishes + a scanned SBC and OCR would buy nothing. - A coverage answer names the plan it could not read even when that plan is the only one shown, and cites no glossary entry when asked what a plan does or charges. Measured over three runs, the MISSING DOCUMENTS set went from @@ -11,8 +25,9 @@ - A document read in part is recorded as `partial`, not `ok` (ADR 0017): its text is kept and searched, but the plan card says its costs chart is not all there, and an answer that finds nothing says the part that would answer - wasn't read here rather than implying the plan doesn't cover it. 62 of 1,189 - documents are partial (new migration). + wasn't read here rather than implying the plan doesn't cover it. 22 of the + 1,246 documents stored are partial (new migration); the run first found 62, + and the parser work that followed (versions 7 to 9) cleared 40 of them. - `make refresh-sbc`: asks every stored SBC whether the issuer has changed it, with a conditional request, and retries recorded failures (ADR 0019). A changed file is downloaded and the one it replaces is moved to @@ -186,6 +201,9 @@ ### Changed +- The COVERAGE eval floor is full marks, 10 of 10, raised from 9. Parser + version 3 reads CHRISTUS's wrapped imaging row as one line, so the answer + states its price in 6 tries of 6, and the set ran 10/10 in three runs. - `make ingest-sbc` no longer retries recorded failures; it reads what is new, what an older parser stored, and any stored document whose PDF has gone missing, and says how many failures it skipped (ADR 0019). Re-requesting a diff --git a/CLAUDE.md b/CLAUDE.md index e38eb5f..0f1f440 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -4,7 +4,7 @@ PolicyPal lets a logged-in user ask natural-language questions about any kind of insurance — health, auto, life, home, and more — and returns grounded answers generated by an LLM using context retrieved from a vector store. -**In progress:** comparison of real purchasable health plans from the CMS Marketplace API. That half is narrower than the Q&A — ACA individual and family plans, in the 30 states on HealthCare.gov. PolicyPal will compare plans, never recommend one. See [docs/plans](docs/plans/) for the phased roadmap. +**Built:** comparison of real purchasable health plans from the CMS Marketplace API. That half is narrower than the Q&A — ACA individual and family plans, in the 30 states on HealthCare.gov. PolicyPal compares plans, never recommends one. All five phases in [docs/plans](docs/plans/) are complete; keeping the plan documents current is a routine, in [docs/runbooks/sbc.md](docs/runbooks/sbc.md). ## Stack diff --git a/README.md b/README.md index 4a82acf..4f60dc1 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ PolicyPal lets a logged-in user ask natural-language questions about any kind of insurance — health, auto, life, home, and more — and returns grounded answers generated by an LLM using context retrieved from a vector store. Every answer cites the source passages it was built from. -**In progress:** comparison of real purchasable health plans drawn from the CMS Marketplace API. That half is deliberately narrower than the Q&A — ACA individual and family plans, in the 30 states served by HealthCare.gov. The 21 state-based exchanges, employer coverage, Medicare and Medicaid have no equivalent public catalog and are out of scope. [docs/plans](docs/plans/) records the phased roadmap and what it excludes. +**Built:** comparison of real purchasable health plans drawn from the CMS Marketplace API. That half is deliberately narrower than the Q&A — ACA individual and family plans, in the 30 states served by HealthCare.gov. The 21 state-based exchanges, employer coverage, Medicare and Medicaid have no equivalent public catalog and are out of scope. All five phases of [docs/plans](docs/plans/) are complete; the catalog is loaded for eighteen states (3,276 plans, 137 issuers), and 1,940 of those plans have their Summary of Benefits read here. Keeping the documents current is a routine now, not a phase: [docs/runbooks/sbc.md](docs/runbooks/sbc.md). PolicyPal will compare plans; it will not recommend one. Advice-shaped questions are refused by design, and that refusal is a measured property with its own regression floor (ADR 0004). @@ -72,9 +72,11 @@ PolicyPal/ ├── migrations/ # Alembic revisions ├── docs/decisions/ # ADRs for material architectural choices ├── docs/plans/ # phased roadmap for the plan-comparison expansion +├── docs/findings/ # what each live run actually measured +├── docs/runbooks/ # keeping the plan documents current ├── frontend/ # React + Vite chat UI -├── scripts/ # ask.py (manual RAG smoke test), -│ # eval_retrieval.py, eval_generation.py +├── scripts/ # ask.py (manual RAG smoke test), eval_retrieval.py, +│ # eval_generation.py, eval_sbc_ranking.py ├── tests/ ├── docker-compose.yml # PostgreSQL + pgvector └── pyproject.toml @@ -222,6 +224,7 @@ Verified API behaviour is recorded in ```bash make ingest-sbc STATES=NH,DE # after make ingest-plans for the same states make ingest-sbc STATES=FL,TX TOP_ISSUERS=1 # only the ten largest parent companies +make ingest-sbc STATES=NH YEAR=2027 # a run during open enrollment ``` Reads the **Summary of Benefits and Coverage** behind each catalog plan's `benefits_url`: @@ -237,8 +240,10 @@ sections into `sbc_chunks`, kept apart from the corpus that `make ingest` rebuil with no request. Bumping `PARSER_VERSION` in `src/ingestion/sbc/extract.py` reads every document again from its kept PDF (ADR 0015, 0016). - **One line per service.** Chart rows are rebuilt from the table's ruled grid, so a - wrapped service name stays beside its price (ADR 0018). A scanned PDF with no text - layer is recorded as `unparseable`; there is no OCR. + wrapped service name stays beside its price; a chart ruled across but not down falls + back to reading the band between two rules (ADR 0018). A scanned PDF with no text + layer would be recorded as `unparseable`, and there is no OCR — but measuring + eighteen states found **zero** of them, so no issuer publishes a scanned SBC. - **Every PDF is kept.** Downloaded SBCs stay in `data/sbc/raw/` (gitignored, never served or committed), about 0.75 MB each. A stored document whose PDF has gone missing is downloaded again, and one rejected as another year's is moved to `data/sbc/rejected/` @@ -293,6 +298,13 @@ comparison names every plan it has no document for, and each saved plan card records whether its SBC could be read (ADR 0017). Links in answers are clickable (https only). +A document can also be read only in part — the costs chart drawn in a way that +yields no table, or a questions column that interleaves its answers. Those are +stored as `partial`: the text that was read is kept and searched, the plan card +says the chart is not all there, and an answer that finds nothing says the part +that would answer wasn't among what was read, rather than implying the plan +doesn't cover it. 22 of the 1,246 documents stored are partial. + ## Testing ```bash diff --git a/docs/decisions/0005-conversation-history.md b/docs/decisions/0005-conversation-history.md index 35140e0..bab73a1 100644 --- a/docs/decisions/0005-conversation-history.md +++ b/docs/decisions/0005-conversation-history.md @@ -115,6 +115,13 @@ implied. Follow-ups cost roughly 2× a first message in latency — acceptable for a local small model, and the thing to revisit first if generation moves behind an API where a rewrite call is cheap and parallelisable. +**That move happened (ADR 0008) and this was not revisited.** Generation and +the rewrite both run against `gpt-5-mini`/`gpt-5-nano` now, and the rewrite is +still a serial call before retrieval rather than one issued alongside it. It +stays open, and recorded here rather than implied: the latency is acceptable +in use, so nothing has forced the question, but the reason given for deferring +it no longer holds. + Retrieval quality for follow-ups now depends on a generated artefact, so it is no longer fully deterministic. `scripts/eval_generation.py` grows a multi-turn case for exactly that reason; the deterministic parts — history diff --git a/docs/decisions/0017-saying-which-plans-have-no-document.md b/docs/decisions/0017-saying-which-plans-have-no-document.md index 879cb0d..31261e0 100644 --- a/docs/decisions/0017-saying-which-plans-have-no-document.md +++ b/docs/decisions/0017-saying-which-plans-have-no-document.md @@ -64,6 +64,11 @@ questions column interleaves its answers. Recorded as `ok`, they are the same silent failure this ADR exists to stop — an answer would search a document that has no prices in it and conclude the plan says nothing. +The parser work those 62 prompted brought the number down within the same +phase: reading a chart ruled across but not down, and completing a wrapped +header (ADR 0018, versions 7 and 8), left **22 of 1,189**. After the refresh +of all eighteen states the corpus stands at **22 partial of 1,246**. + - **Its text is kept and searched.** What was read is the plan's own words, and half a document beats none. - **`sbc_readable` stays true,** and `plan_coverage` still returns its @@ -72,8 +77,13 @@ that has no prices in it and conclude the plan says nothing. partial, the answer says the part that would answer isn't among what was read, and links the PDF — never that the plan doesn't cover it. - **The completeness rule is the template's own:** all seven Important - Questions, and at least the chart's ten "If you…" groups. 1,127 of 1,189 - documents meet it. + Questions, and at least the chart's ten "If you…" groups. 1,224 of the 1,246 + documents stored meet it. +- **`partial` is a read status.** `READ_STATUSES` in `src/services/sbc_status.py` + is the single definition, and everything that asks "is there text for this + plan?" derives from it — the plan card, `search_plans`, `plan_coverage` and + `make sbc-report` alike. Spelling the question as `== "ok"` anywhere else + reintroduces exactly the silent disagreement this section exists to stop. ### The plan card keeps the status it was shown with diff --git a/docs/decisions/0018-chart-rows-from-the-ruled-grid.md b/docs/decisions/0018-chart-rows-from-the-ruled-grid.md index c3f3817..d1ee30a 100644 --- a/docs/decisions/0018-chart-rows-from-the-ruled-grid.md +++ b/docs/decisions/0018-chart-rows-from-the-ruled-grid.md @@ -2,7 +2,7 @@ ## Status -Accepted. Phase 4 (parser versions 3 to 6). Amends ADR 0013's "table cells are not rebuilt one by +Accepted. Phase 4 (parser versions 3 to 9). Amends ADR 0013's "table cells are not rebuilt one by one", and ADR 0015's "`parser_version` is NULL unless `ok`". ## Context @@ -48,7 +48,9 @@ Inside one "If you…" group: The result is one line per service: `Imaging (CT/PET scans, MRIs) | No charge | Not covered | Preauthorization is required…`. -`PARSER_VERSION` is 3. +This is version 3. Each section below raises `PARSER_VERSION` by one, and each +bump re-reads every stored document from disk with no request to any issuer. +The parser is at **9**; `src/ingestion/sbc/extract.py` holds the number. ### A PDF with no text layer is unparseable, with the reason @@ -124,6 +126,46 @@ thousands of documents. - **A document the database still refuses is recorded as `unparseable`,** with that reason, and the run carries on. One file must not end a run. +### A chart ruled across but not down still yields rows (version 7) + +`find_tables()` needs both rules to make a cell. University of Utah Health +Plans draws its chart with twenty horizontal lines and no vertical ones, so +pdfplumber found no table at all and the whole costs chart was lost — the +document still parsed, so it was stored as `ok` with no prices in it. That is +the silent failure ADR 0017 exists to stop, arriving by a different route. + +- **When a page yields no table, a row is the band between two rules.** Its + label is whichever of the band's lines is one of the template's headings, + and the rest is its body: ADR 0013's reading, kept for exactly the layouts + that defeat the grid. +- **A band shorter than six points is skipped**, being a rule's own thickness + or a hairline, not a row. +- It is a fallback, not a new default. A page whose table is found is read + from the grid as before. + +### A table header printed in pieces is still that header (version 8) + +BCBS of Oklahoma prints "Common" and "Medical Event" as rows of their own, so +the chart's header never matched and the group headings after it were read as +chart rows. A label now counts as a header when it begins that header **or**, +at six characters or more, when the header begins with it. The length floor +keeps a short label like "If you" from matching half the template. + +### The template's closing sentence is not a chart row (version 9) + +"If your plan doesn't meet the Minimum Value Standards…" is the federal +template's own closing wording, but it begins "If you" like a chart group, so +it was stored as a row and could be cited as though it were a price. `If your +plan` joins the list of headings that end the chart. + +An earlier attempt narrowed `_event_heading` instead. It broke two tests that +documented intentional behaviour, and was reverted: the chart's end is a +better place to say this than the chart's rows. + +**Together, versions 7 to 9 took partly-read documents from 62 of 1,189 to +22** — measured by re-reading every stored document from disk, with no request +to any issuer. + ### Every file the parser judged records which file and which parser - `unparseable` and `wrong_year` rows now store the file's `sha256` and the @@ -137,10 +179,15 @@ thousands of documents. ## Consequences -- **Measured on the 436 stored documents:** all still yield their 25 sections. - The ranking and eval results are in the Phase 4 findings. +- **Measured at version 3 on the 436 documents then stored:** all still yield + their 25 sections. Re-measured after every bump since, now over 1,246 + documents; the ranking and eval results are in the Phase 4 findings, and + ranking did not move. - **Each parser bump costs a re-parse from disk,** about 1.5 s per document, and no download. +- **22 documents of 1,246 are still partial,** and say so. The last two fixes + had sharply diminishing returns, so they are marked honestly rather than + chased further. - **A merged cell's text lands in the row it is centred in.** A limitation shared by two services appears on one of them. - **Layouts can still defeat the grid.** A group without rules falls back to From 83c6c5cde18d8750144684115fb4e612bc66202f Mon Sep 17 00:00:00 2001 From: BADRI NARAYANAN Date: Tue, 22 Sep 2026 12:43:50 -0700 Subject: [PATCH 04/12] Run CI on every pull request, not only those targeting main MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The trigger filtered `pull_request: branches: [main]`, so a PR based on another branch got no backend, frontend or secret-scan job at all — only the advisory Claude review, which is `continue-on-error`. A stack of related changes is exactly where a missed regression is easiest to inherit, and the filter made those the only PRs with no gate. --- .github/workflows/ci.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5e27d98..389882b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,8 +1,10 @@ name: CI on: + # Every pull request, not only those targeting main: a stacked PR based on + # another branch needs the same gate as any other, and the branch filter + # silently gave it none. pull_request: - branches: [main] push: branches: [main] From 6714cae88c2170d636aa6e00b3bc8ef4e25c4e28 Mon Sep 17 00:00:00 2001 From: BADRI NARAYANAN Date: Tue, 22 Sep 2026 12:13:13 -0700 Subject: [PATCH 05/12] Read an unavailable robots.txt as the standard does, not as a refusal MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ADR 0013 recorded a site that would not serve /robots.txt as `blocked`, on the grounds that "a refused robots.txt counts as disallowing everything". That was written from two hosts in a two-state slice. At eighteen states it governs 482 documents, and it is stricter than RFC 9309, which puts every 400-499 in one "unavailable" class where a crawler may fetch. Only 5xx requires assuming a disallow. The cost of the rule had never been measured, so it was. All 482 documents come from seven hosts, so this is a census rather than a sample: one request each, two seconds apart, status and content type only, nothing stored. Five of the seven serve the document at 200 with application/pdf — Medica, Anthem, Wellpoint, AvMed and Mending. The prediction going in was the opposite, that a 403 on robots.txt meant a WAF refusing the client outright. That holds for UnitedHealthcare and Mercy, and they refuse on their own account: their 403 on the PDF still records the document as blocked, and is still never worked around. Nothing else moves. HTTPS only, public hosts re-checked at every redirect, two seconds between requests to one host, the 15 MB cap, and a real Disallow rule all stand. A 5xx robots.txt now blocks where it used to fall through to allow_all, which is the safer direction. A network error still fails open, as it did, but says so in the log instead of doing it silently. --- CHANGELOG.md | 7 ++ docs/decisions/0013-sbc-documents.md | 9 +- ...unavailable-robots-txt-is-not-a-refusal.md | 98 +++++++++++++++++++ src/ingestion/sbc/fetch.py | 37 ++++--- tests/test_sbc_fetch.py | 28 +++++- 5 files changed, 162 insertions(+), 17 deletions(-) create mode 100644 docs/decisions/0020-an-unavailable-robots-txt-is-not-a-refusal.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 8753858..c66245a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -201,6 +201,13 @@ ### Changed +- A `robots.txt` the server cannot serve is no longer read as a refusal + (ADR 0020, superseding ADR 0013). RFC 9309 puts every 4xx in one + "unavailable" class where a crawler may fetch, and probing all seven + affected hosts found five serving their PDF at 200 `application/pdf`. A 5xx + now disallows, which it did not before. A real `Disallow` rule, and a 401 or + 403 on the document itself, are still refusals and still never worked + around. - The COVERAGE eval floor is full marks, 10 of 10, raised from 9. Parser version 3 reads CHRISTUS's wrapped imaging row as one line, so the answer states its price in 6 tries of 6, and the set ran 10/10 in three runs. diff --git a/docs/decisions/0013-sbc-documents.md b/docs/decisions/0013-sbc-documents.md index 34379cb..388182a 100644 --- a/docs/decisions/0013-sbc-documents.md +++ b/docs/decisions/0013-sbc-documents.md @@ -8,6 +8,8 @@ not on every run. ADR 0015 also deleted each PDF once its text was stored; ADR 0016 reverses that, and every downloaded SBC is kept. Amended by ADR 0018: chart rows are rebuilt from the table's ruled grid, one line per service, and a PDF with no text layer is refused as scanned. +Its "a refused `robots.txt` counts as disallowing everything" is **superseded +by ADR 0020**, which follows RFC 9309's reading of an unavailable robots.txt. ## Context @@ -61,8 +63,11 @@ among the catalog plans of those states. every redirect, which is followed by hand rather than by `requests`. - **Refusals are respected.** - `robots.txt` is read and obeyed, using the standard library's parser. - - A 401 or 403, on the file or on `robots.txt`, records the document as - `blocked`. A refused `robots.txt` counts as disallowing everything. + - A 401 or 403 on the file records the document as `blocked`. + - ~~A refused `robots.txt` counts as disallowing everything.~~ **Superseded + by ADR 0020:** RFC 9309 puts 4xx in one "unavailable" class where a + crawler may fetch, and measuring all seven affected hosts found five + serving the document at 200. A 5xx now disallows, which this did not. - Nothing tries to get around a refusal: no browser user agent, no other route to the file. Those plans get no SBC answers; answers link their PDF instead. diff --git a/docs/decisions/0020-an-unavailable-robots-txt-is-not-a-refusal.md b/docs/decisions/0020-an-unavailable-robots-txt-is-not-a-refusal.md new file mode 100644 index 0000000..9c3a793 --- /dev/null +++ b/docs/decisions/0020-an-unavailable-robots-txt-is-not-a-refusal.md @@ -0,0 +1,98 @@ +# 0020 — An unavailable robots.txt is not a refusal + +## Status + +Accepted. Supersedes ADR 0013's "A refused `robots.txt` counts as disallowing +everything." + +## Context + +ADR 0013 read `robots.txt` before every SBC fetch and recorded a site that +refused it as `blocked`: + +> A 401 or 403, on the file or on `robots.txt`, records the document as +> `blocked`. A refused `robots.txt` counts as disallowing everything. + +That was written from two hosts in a two-state slice. At eighteen states it +governs **482 documents** — the second largest reason for missing coverage +after a real `Disallow` rule. + +It is also stricter than the standard it implements. RFC 9309 §2.3.1.3 puts +every 400–499 status in one class, "Unavailable", and says a crawler **may** +access the site's resources; only §2.3.1.4's "Unreachable" (5xx) requires +assuming a complete disallow. A 401 or 403 on `/robots.txt` is a web server +declining to serve one file. It is not a statement about the documents. + +The costs chart of this decision was never measured, so it was measured. All +482 documents come from **seven hosts**, so this is a census, not a sample: +one request each, two seconds apart, status and content type only, nothing +stored or parsed. + +| Host | Documents | `/robots.txt` | The PDF itself | +| --- | --- | --- | --- | +| www.uhc.com | 229 | 403 | **403** | +| portal.medica.com | 117 | 403 | 200 `application/pdf` | +| sbc.anthem.com | 76 | 401 | 200 `application/pdf` | +| sbc.wellpoint.com | 34 | 401 | 200 `application/pdf` | +| apps.avmed.com | 14 | 401 | 200 `application/pdf` | +| www.mercyhealthsystem.org | 6 | 403 | **403** | +| static.mending.com | 6 | 403 | 200 `application/pdf` | + +**Five of seven serve the document without objection.** The prediction going +in was the opposite — that a 403 on `robots.txt` meant a WAF refusing the +client outright, so the PDF would refuse too. That holds for exactly two +hosts, and they are already refusing on their own account. + +## Decision + +### The status of `robots.txt` decides, as RFC 9309 says + +- **2xx** — these are the rules, parsed and obeyed. Unchanged: a real + `Disallow` is still a refusal, and 483 documents remain blocked by one. +- **4xx** — "unavailable". No rules were published, and the fetch proceeds. +- **5xx** — "unreachable". A complete disallow, recorded as `robots.txt could + not be served`. This is *stricter* than before, where any non-200 that was + not 401/403 fell through to `allow_all`. + +Nothing else moves. HTTPS only, public hosts re-checked at every redirect, two +seconds between requests to one host, the 15 MB cap, and the PDF check all +stand. **A 401 or 403 on the document itself is still `blocked`, and is still +never worked around** — which is what keeps UnitedHealthcare's 229 documents +and Mercy's 6 out, on their own servers' say-so rather than on an inference +from a different file. + +### Why this is not a workaround + +ADR 0013's rule stands on "as robots.txt conventions go", and the convention +it names is not the one the standard states. Reading the standard correctly is +not getting past a refusal; it is noticing that no refusal was made. The +project's rule — a host that says no is left alone — is unchanged, and the two +hosts that say no are still left alone. + +The documents in question are Summaries of Benefits and Coverage, which +issuers are federally required to publish for exactly this audience. That is +context, not a licence: the deciding facts are the standard and the servers' +own answers. + +### A network error still fails open, and is now logged + +`requests` raising means we never learned anything. RFC 9309 would group that +with "unreachable" and disallow. It is left as it was, because changing it +would newly block hosts on a transient blip and that effect has not been +measured — but it now logs, instead of failing open in silence. Recorded here +rather than left implied. + +## Consequences + +- **247 documents behind 333 plans in 8 states** become readable, taking + coverage from 1,940 plans (59.2%) to about 2,273 (69.4%) — the largest + single gain available, and the only one that needed no parser work. +- **The reason a plan has no document gets more honest.** "The insurer blocks + automated access" now means the insurer's server refused the document, not + that it refused an unrelated file. +- **`robots.txt refused (HTTP 401/403)` disappears as a stored reason.** A + refresh re-judges those documents; the ones that still refuse land on the + document's own status. +- **A 5xx now blocks where it used to pass.** Rare, and the safer direction. +- **This is re-measurable.** Seven hosts, one request each — the same census + can be re-run whenever the question comes up again. diff --git a/src/ingestion/sbc/fetch.py b/src/ingestion/sbc/fetch.py index 0053e33..9db8cb1 100644 --- a/src/ingestion/sbc/fetch.py +++ b/src/ingestion/sbc/fetch.py @@ -3,8 +3,9 @@ The URLs come from CMS, not users, but they point at dozens of issuers' servers, so each is checked before it is fetched and again at every redirect: HTTPS only, and never an IP address or a local host name. A site that -refuses us — robots.txt, or a 401/403 — is recorded as `blocked` and left -alone. Nothing here tries to get past it. +refuses us — a robots.txt rule, or a 401/403 on the document itself — is +recorded as `blocked` and left alone. Nothing here tries to get past it. A +robots.txt that cannot be served is not a refusal (ADR 0020). `revalidate` asks whether a stored document has changed, with the validators the issuer gave us, and writes nothing. @@ -163,24 +164,36 @@ def _unsafe(url: str) -> str | None: @lru_cache(maxsize=256) -def _robots(origin: str) -> RobotFileParser | int: - """The site's robots.txt rules, read once per run, or the 401/403 it answered with. +def _robots(origin: str) -> RobotFileParser: + """The site's robots.txt rules, read once per run (RFC 9309 §2.3.1, ADR 0020). Read with our own client, not RobotFileParser.read(), for the timeout and - User-Agent. As robots.txt conventions go: 401/403 means stay out, any - other failure means no rules were published. + User-Agent. The status decides what the site told us: + + - 2xx: these are the rules, and they are obeyed. + - 4xx "unavailable": the file is not there to read, so no rules were + published and the standard says a crawler may fetch. A 401 or 403 on + `/robots.txt` says nothing about the documents themselves. + - 5xx "unreachable": the server is failing, and the standard says assume a + complete disallow rather than guess. + + A network error is the one place this still fails open, which RFC 9309 + would group with unreachable. It is left as it was and logged rather than + changed unmeasured (ADR 0020). """ parser = RobotFileParser() _pace(urlsplit(origin).hostname) try: response = requests.get(f"{origin}/robots.txt", headers={"User-Agent": USER_AGENT}, timeout=_TIMEOUT, allow_redirects=True) - except requests.RequestException: + except requests.RequestException as exc: + logger.info("robots.txt unreachable for %s (%s): proceeding", origin, type(exc).__name__) parser.allow_all = True return parser - if response.status_code in (401, 403): - return response.status_code - if response.status_code != 200: + + if response.status_code >= 500: + parser.disallow_all = True + elif response.status_code != 200: parser.allow_all = True else: parser.parse(response.text.splitlines()) @@ -191,8 +204,8 @@ def _robots_refusal(url: str) -> str | None: """Why the site's robots.txt keeps us from `url`, or None.""" parts = urlsplit(url) rules = _robots(f"{parts.scheme}://{parts.netloc}") - if isinstance(rules, int): - return f"robots.txt refused (HTTP {rules})" + if rules.disallow_all: + return "robots.txt could not be served" return None if rules.can_fetch(USER_AGENT, url) else "robots.txt disallows it" diff --git a/tests/test_sbc_fetch.py b/tests/test_sbc_fetch.py index ddfd3b0..ec21707 100644 --- a/tests/test_sbc_fetch.py +++ b/tests/test_sbc_fetch.py @@ -99,15 +99,37 @@ def test_robots_txt_is_honoured(_offline): assert [call.args[0] for call in _offline.call_args_list] == ["https://sbc.example.com/robots.txt"] -def test_a_site_refusing_robots_txt_is_left_alone(_offline): - _serve(_offline, _response(), robots=_response(403)) +@pytest.mark.parametrize("status", [401, 403, 404, 410]) +def test_a_robots_txt_that_is_not_there_publishes_no_rules(_offline, status): + """RFC 9309 §2.3.1.3: 4xx is "unavailable", and a crawler may fetch (ADR 0020). + + Measured over the seven hosts this affected, five served their PDF at 200 + with `application/pdf`. A 401 or 403 on /robots.txt is the web server + declining to serve one file, and says nothing about the documents. + """ + _serve(_offline, _response(), robots=_response(status)) + + assert fetch_pdf(URL, 2026).status == "ok" + + +@pytest.mark.parametrize("status", [500, 503]) +def test_a_robots_txt_the_server_cannot_serve_stops_the_fetch(_offline, status): + """RFC 9309 §2.3.1.4: 5xx is "unreachable", and a crawler assumes a full disallow.""" + _serve(_offline, _response(), robots=_response(status)) result = fetch_pdf(URL, 2026) - assert (result.status, result.detail) == ("blocked", "robots.txt refused (HTTP 403)") + assert (result.status, result.detail) == ("blocked", "robots.txt could not be served") assert [call.args[0] for call in _offline.call_args_list] == ["https://sbc.example.com/robots.txt"] +def test_a_site_that_refuses_the_document_itself_is_left_alone(_offline): + """The refusal that still counts: a 403 on the PDF, not on /robots.txt.""" + _serve(_offline, _response(403)) + + assert fetch_pdf(URL, 2026).status == "blocked" + + def test_a_body_past_the_size_cap_is_dropped_unsaved(_offline): _serve(_offline, _response(body=b"%PDF-" + b"x" * MAX_BYTES)) From 7b895a31db3b502351fc52bb7d525f5e407b6dd9 Mon Sep 17 00:00:00 2001 From: BADRI NARAYANAN Date: Tue, 22 Sep 2026 13:23:39 -0700 Subject: [PATCH 06/12] Record what the robots.txt change actually reached MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The eight affected states were refreshed, 1,496 documents re-checked. The estimate held: coverage went from 1,940 plans (59.2%) to 2,274 (69.4%). Two things worth saying plainly rather than reporting the headline alone. The documents this reaches parse less completely than the ones already read: `partial` went from 22 to 138, so 69.4% is 2,136 whole documents and 138 partial ones. They are honestly marked and their text is searched, which is what ADR 0017 is for, but the number is not 2,274 whole documents. UnitedHealthcare and Mercy now fail on their own status — www.uhc.com returns 403 for the document itself, so those records carry that reason instead of one inferred from a different file. Same blocked status, same sentence to the user, accurate cause. ADR 0016 verified: 1,264 PDFs before, 1,593 after, every earlier file still present, and the archive grew by exactly the 77 documents issuers had replaced. --- CHANGELOG.md | 6 +++- ...unavailable-robots-txt-is-not-a-refusal.md | 31 +++++++++++++++++-- 2 files changed, 33 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c66245a..5c1ebcb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -207,7 +207,11 @@ affected hosts found five serving their PDF at 200 `application/pdf`. A 5xx now disallows, which it did not before. A real `Disallow` rule, and a 401 or 403 on the document itself, are still refusals and still never worked - around. + around. Refreshing the eight affected states took coverage from 1,940 plans + (59.2%) to **2,274 (69.4%)**. Those documents parse less completely than the + ones already read, so `partial` went from 22 to 138: the figure is 2,136 + whole documents and 138 partial, not 2,274 whole. No PDF was deleted; the + archive grew by the 77 files issuers had replaced. - The COVERAGE eval floor is full marks, 10 of 10, raised from 9. Parser version 3 reads CHRISTUS's wrapped imaging row as one line, so the answer states its price in 6 tries of 6, and the set ran 10/10 in three runs. diff --git a/docs/decisions/0020-an-unavailable-robots-txt-is-not-a-refusal.md b/docs/decisions/0020-an-unavailable-robots-txt-is-not-a-refusal.md index 9c3a793..e353c09 100644 --- a/docs/decisions/0020-an-unavailable-robots-txt-is-not-a-refusal.md +++ b/docs/decisions/0020-an-unavailable-robots-txt-is-not-a-refusal.md @@ -84,9 +84,34 @@ rather than left implied. ## Consequences -- **247 documents behind 333 plans in 8 states** become readable, taking - coverage from 1,940 plans (59.2%) to about 2,273 (69.4%) — the largest - single gain available, and the only one that needed no parser work. +**Measured.** The eight affected states were refreshed — 1,496 documents +re-checked — and the estimate held: + +| | Before | After | +| --- | --- | --- | +| Plans with text | 1,940 (59.2%) | **2,274 (69.4%)** | +| Documents `ok` | 1,224 | 1,360 | +| Documents `partial` | 22 | **138** | +| Documents `blocked` | 995 | 748 | + +The run's outcomes were `698 unchanged, 434 blocked, 136 ok, 116 partial, +77 changed:ok, 31 http_error, 4 unreachable`. + +- **The documents this reaches parse less completely than the ones already + read.** Partial went from 22 to 138: 116 of this run's documents gave up + their text but not all of the template. They are honestly marked and their + text is searched (ADR 0017), but "69.4% covered" means 2,136 whole documents + and 138 partial ones, not 2,274 whole ones. Worth saying plainly rather than + reporting the headline alone. +- **UnitedHealthcare and Mercy now fail on their own status.** `www.uhc.com` + returns 403 for the document, so those 125 records moved from "robots.txt + refused (HTTP 403)" to "HTTP 403" — the same `blocked` status and the same + user-facing sentence, now for the accurate reason. 110 records in the ten + states not refreshed still carry the old wording; the next monthly refresh + clears them, and the status they resolve to is unchanged either way. +- **Nothing was deleted (ADR 0016), verified.** 1,264 PDFs before, 1,593 + after; `comm` confirms every file present before is still present, and the + archive grew by exactly 77 — the count of documents the issuer had replaced. - **The reason a plan has no document gets more honest.** "The insurer blocks automated access" now means the insurer's server refused the document, not that it refused an unrelated file. From cac565394a1931279b388fdb37adb85973cce136 Mon Sep 17 00:00:00 2001 From: BADRI NARAYANAN Date: Tue, 22 Sep 2026 13:42:24 -0700 Subject: [PATCH 07/12] Carry the new coverage figures into the README The robots.txt change moved them the moment it ran: 2,274 plans covered rather than 1,940, and 138 partly-read documents rather than 22. --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 4f60dc1..9dce3c9 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ PolicyPal lets a logged-in user ask natural-language questions about any kind of insurance — health, auto, life, home, and more — and returns grounded answers generated by an LLM using context retrieved from a vector store. Every answer cites the source passages it was built from. -**Built:** comparison of real purchasable health plans drawn from the CMS Marketplace API. That half is deliberately narrower than the Q&A — ACA individual and family plans, in the 30 states served by HealthCare.gov. The 21 state-based exchanges, employer coverage, Medicare and Medicaid have no equivalent public catalog and are out of scope. All five phases of [docs/plans](docs/plans/) are complete; the catalog is loaded for eighteen states (3,276 plans, 137 issuers), and 1,940 of those plans have their Summary of Benefits read here. Keeping the documents current is a routine now, not a phase: [docs/runbooks/sbc.md](docs/runbooks/sbc.md). +**Built:** comparison of real purchasable health plans drawn from the CMS Marketplace API. That half is deliberately narrower than the Q&A — ACA individual and family plans, in the 30 states served by HealthCare.gov. The 21 state-based exchanges, employer coverage, Medicare and Medicaid have no equivalent public catalog and are out of scope. All five phases of [docs/plans](docs/plans/) are complete; the catalog is loaded for eighteen states (3,276 plans, 137 issuers), and 2,274 of those plans (69.4%) have their Summary of Benefits read here — 2,136 read whole and 138 in part (ADR 0017). Keeping the documents current is a routine now, not a phase: [docs/runbooks/sbc.md](docs/runbooks/sbc.md). PolicyPal will compare plans; it will not recommend one. Advice-shaped questions are refused by design, and that refusal is a measured property with its own regression floor (ADR 0004). @@ -303,7 +303,7 @@ yields no table, or a questions column that interleaves its answers. Those are stored as `partial`: the text that was read is kept and searched, the plan card says the chart is not all there, and an answer that finds nothing says the part that would answer wasn't among what was read, rather than implying the plan -doesn't cover it. 22 of the 1,246 documents stored are partial. +doesn't cover it. 138 of the 1,498 documents stored are partial. ## Testing From 01ee2a326e48ab03523aba2e09c889fb57852077 Mon Sep 17 00:00:00 2001 From: BADRI NARAYANAN Date: Tue, 22 Sep 2026 12:15:05 -0700 Subject: [PATCH 08/12] Store the BM25 index in the database, beside the chunks it indexes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `_bm25_index()` read `data/corpus/indices/bm25.pkl` from local disk and raised FileNotFoundError when it was absent. Nothing catches that, so the chat route's safety net turns it into a 500: an app running anywhere but the machine that built the corpus answers no questions at all — not degraded, not sparse-only, every question fails. That never showed because the app had only ever run where the file was. It cannot be fixed where it stands: `data/` is gitignored and stays that way, so no image can be built containing it. The index is derived data — every byte of it comes from the `chunks` table, which already holds each chunk's text and id. So it moves into the database next to them, as one row of `search_indexes`, and `make build-index` rebuilds it from the chunks already stored with no fetching, chunking or embedding. Seeding a fresh database is a restore plus one command. A missing index now names that command instead of naming a path that will not exist wherever the error was raised. The payload also stops carrying a second copy of the corpus: `texts` was read by nothing but a test, and dropping it took the stored index from 3.4 MB to 1.8 MB for the same 1,568 chunks. The orchestrator's tests now check what reaches the indexer at the call rather than by reading back a pickle, which is what they were really asserting; storing it has its own tests. --- CHANGELOG.md | 7 ++ Makefile | 22 +++- README.md | 5 + ...-the-search-index-lives-in-the-database.md | 91 ++++++++++++++++ migrations/env.py | 1 + ...26_store_the_bm25_index_in_the_database.py | 42 +++++++ src/core/exceptions.py | 4 + src/ingestion/build_index.py | 37 +++++++ src/ingestion/chunking.py | 41 ++++--- src/ingestion/constants.py | 2 - src/models/search_index.py | 28 +++++ src/policypal/config.py | 5 +- src/services/retrieval.py | 28 +++-- tests/test_ingestion_pipeline.py | 40 +++---- tests/test_search_index.py | 103 ++++++++++++++++++ 15 files changed, 399 insertions(+), 57 deletions(-) create mode 100644 docs/decisions/0021-the-search-index-lives-in-the-database.md create mode 100644 migrations/versions/20260922_1909-f6e50c4aec26_store_the_bm25_index_in_the_database.py create mode 100644 src/ingestion/build_index.py create mode 100644 src/models/search_index.py create mode 100644 tests/test_search_index.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 5c1ebcb..eb45516 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -201,6 +201,13 @@ ### Changed +- The BM25 index is stored in the database, in `search_indexes`, instead of + `data/corpus/indices/bm25.pkl` (ADR 0021, new migration). A missing index + was a `FileNotFoundError` nothing caught, so an app running anywhere but the + machine that built the corpus answered no questions at all; it now raises an + error naming the command to run. `make build-index` rebuilds it from the + chunks already stored. Dropping the payload's unread `texts` key, a second + copy of the corpus, took it from 3.4 MB to 1.8 MB. - A `robots.txt` the server cannot serve is no longer read as a refusal (ADR 0020, superseding ADR 0013). RFC 9309 puts every 4xx in one "unavailable" class where a crawler may fetch, and probing all seven diff --git a/Makefile b/Makefile index fd6692d..5df78fd 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ -.PHONY: ingest ingest-plans ingest-sbc refresh-sbc sbc-report migrate api \ - ui-dev ui-build ui-lint ui-preview ui-test +.PHONY: ingest build-index ingest-plans ingest-sbc refresh-sbc sbc-report migrate api \ + ui-dev ui-build ui-lint ui-preview ui-test test lint check # The plan year to work on; the commands default to the calendar year, which # is the wrong one once the next year's plans are on sale. e.g. YEAR=2027 @@ -8,6 +8,12 @@ YEAR_ARG = $(if $(YEAR),--year $(YEAR)) ingest: uv run python -m src.ingestion.pipeline +# Rebuild the BM25 index from the chunks already stored, with no fetching, +# chunking or embedding (ADR 0021). `make ingest` does this as its last step; +# this is for a database seeded from a dump. +build-index: + uv run python -m src.ingestion.build_index + # STATES is required: comma-separated codes, or ALL for every HealthCare.gov # state. No default, so a bare `make ingest-plans` cannot start thousands of # requests. e.g. make ingest-plans STATES=TX,FL @@ -55,3 +61,15 @@ ui-preview: ui-test: cd frontend && npm test + +# What CI runs, in one command, so "it passed locally" means the same thing. +test: + uv run pytest -q --cov=src --cov-report=term-missing --cov-fail-under=85 + +lint: + uv run ruff check . + +check: lint test + uv run alembic check + uv run bandit -r src -ll + uv run pip-audit diff --git a/README.md b/README.md index 9dce3c9..4ec97f5 100644 --- a/README.md +++ b/README.md @@ -180,6 +180,11 @@ Runs in order: **fetch** (download each source) → **normalize** (convert to ma **chunk** (per-source splitting + a shared BM25 index) → **embed** (vectorize and upsert to pgvector). Every phase is idempotent, so a re-run only picks up what's missing. +The BM25 index is stored in the database beside the chunks it indexes, not on local disk +(ADR 0021), so the app can run somewhere with no `data/` directory. It is derived data: +`make build-index` rebuilds it from the chunks already stored, with no fetching, chunking +or embedding — which is what makes seeding a fresh database a restore plus one command. + Adding a corpus means writing one `Source` subclass in `src/ingestion/sources/` — `fetch`, `normalize`, `chunk_documents` — and registering it in that package's `__init__.py`. No pipeline, chunking, or embedding code changes. Chunking strategy lives with the source diff --git a/docs/decisions/0021-the-search-index-lives-in-the-database.md b/docs/decisions/0021-the-search-index-lives-in-the-database.md new file mode 100644 index 0000000..e7b7080 --- /dev/null +++ b/docs/decisions/0021-the-search-index-lives-in-the-database.md @@ -0,0 +1,91 @@ +# 0021 — The BM25 index lives in the database, not on local disk + +## Status + +Accepted. Amends ADR 0004's hybrid retrieval, which put the sparse index in +`data/corpus/indices/bm25.pkl`. + +## Context + +Hybrid retrieval (ADR 0004) needs a BM25 index at request time. +`_bm25_index()` read it from a pickle on local disk, cached per process, and +raised `FileNotFoundError` if it was absent. + +Nothing caught that exception. The chat route's catch-all turns it into a 500, +so **a deployment without the file answers no questions at all** — not +degraded, not sparse-only: every question fails. + +That never showed up because the app had only ever run on the machine that +built the corpus. Containerising it makes the gap immediate, and unfixable +where it stands: + +- `data/` is gitignored and stays that way, so CI cannot build an image + containing the file; +- the index is built by `make ingest`, which is a local, long-running job; +- the file is small (3.4 MB), but the constraint is provenance, not size. + +The alternatives were object storage — a storage account, a credential, an SDK +and a new startup failure mode — or building the image locally, which is not +CD at all. + +Meanwhile the index is **derived data**: everything it contains comes from the +`chunks` table, which already holds each chunk's text and id. + +## Decision + +### One row, in the database that holds the chunks it indexes + +`search_indexes` — `name` (primary key), `payload`, `chunks`, `built_at`. The +BM25 index is the only row today; a second index would be a second row, not a +second table. + +- **The request path reads it from Postgres**, still cached per process with + `lru_cache`, so the cost is one query per worker. +- **A missing index raises `MissingSearchIndexError`** naming the command to + run, instead of a `FileNotFoundError` naming a path that will not exist in + the environment that hit it. +- **Nothing writes to local disk.** `bm25_index_path` and `INDEX_DIR` are + gone; there is no path to configure. + +### It is rebuilt from the chunks, not restored with them + +`make build-index` reads `chunks` and rebuilds the index: no fetching, no +chunking, no embedding. `make ingest` still builds it as the last step of a +full run. + +This is what makes seeding a fresh database one restore plus one command, and +it means the index can never be silently out of date with a corpus that was +loaded some other way. + +### The payload no longer carries a second copy of the corpus + +The pickle held `{"bm25", "chunk_ids", "texts"}`. `texts` was read by nothing +outside a test: retrieval uses `bm25` and `chunk_ids`, and the text itself is +a column away in the table being indexed. Dropping it took the payload from +3.4 MB to 1.8 MB for the same 1,568 chunks. + +### Unpickling stays, and why that is not a new risk + +The payload is a pickle because `BM25Okapi` is a Python object. It is written +only by this project's own ingestion, into a table only this application +writes to, and it was already being unpickled from a file. Moving the bytes +from a file the app owns to a row the app owns does not add a source of +untrusted input. Anyone who can write that row can already run code as the +application. + +If that changes — a shared database, or a second writer — this is the thing to +revisit, and the answer is a schema-validated format rather than tighter +permissions. + +## Consequences + +- **The app runs where there is no `data/` directory.** This is the change + that makes a container work, and it is testable: a container with no `data/` + answering a question is the proof. +- **One more thing in the database dump,** and one fewer thing to copy beside + it. +- **A re-chunk must be followed by an index build,** as before — but now the + two are in the same database, so `built_at` and `chunks` say when the index + was made and from how many, which a file's mtime did not. +- **`make ingest` now needs a database.** It always did, at the embed stage; + the chunk stage no longer stands alone. diff --git a/migrations/env.py b/migrations/env.py index e95feec..b7da31a 100644 --- a/migrations/env.py +++ b/migrations/env.py @@ -9,6 +9,7 @@ chunk, plan, sbc, + search_index, usage, user, ) diff --git a/migrations/versions/20260922_1909-f6e50c4aec26_store_the_bm25_index_in_the_database.py b/migrations/versions/20260922_1909-f6e50c4aec26_store_the_bm25_index_in_the_database.py new file mode 100644 index 0000000..37c6b07 --- /dev/null +++ b/migrations/versions/20260922_1909-f6e50c4aec26_store_the_bm25_index_in_the_database.py @@ -0,0 +1,42 @@ +"""Store the BM25 index in the database (ADR 0021) + +One row per built index. It moves off local disk so the request path does +not need a `data/` directory, and so the index travels with the chunks it +indexes. Derived data: `make build-index` rebuilds it from `chunks`. + +Revision ID: f6e50c4aec26 +Revises: c4e91a7d6b28 +Create Date: 2026-09-22 19:09:02.890934+00:00 + +""" + +from collections.abc import Sequence + +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision: str = "f6e50c4aec26" +down_revision: str | Sequence[str] | None = "c4e91a7d6b28" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +def upgrade() -> None: + op.create_table( + "search_indexes", + sa.Column("name", sa.String(length=50), nullable=False), + sa.Column("payload", sa.LargeBinary(), nullable=False), + sa.Column("chunks", sa.Integer(), nullable=False), + sa.Column( + "built_at", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=False, + ), + sa.PrimaryKeyConstraint("name"), + ) + + +def downgrade() -> None: + op.drop_table("search_indexes") diff --git a/src/core/exceptions.py b/src/core/exceptions.py index c887260..e601756 100644 --- a/src/core/exceptions.py +++ b/src/core/exceptions.py @@ -8,3 +8,7 @@ class InvalidCredentialsError(Exception): class MarketplaceApiKeyMissingError(Exception): """Raised when a Marketplace API call is made without CMS_MARKETPLACE_API_KEY set.""" + + +class MissingSearchIndexError(Exception): + """Raised when the BM25 index has not been built into the database yet.""" diff --git a/src/ingestion/build_index.py b/src/ingestion/build_index.py new file mode 100644 index 0000000..6595183 --- /dev/null +++ b/src/ingestion/build_index.py @@ -0,0 +1,37 @@ +"""Rebuild the BM25 index from the chunks already in the database: `make build-index`. + +The index is derived data (ADR 0021): everything it needs — each chunk's text +and its id — is a column away. So it can be rebuilt without re-fetching, +re-chunking or re-embedding anything, which is what makes seeding a fresh +database a restore plus one command. + +`make ingest` still builds it as the last step of a full run; this is for when +the chunks are already there. +""" +import sys + +from sqlalchemy import select + +from src.core.db import get_session +from src.core.logging import get_logger, setup_logging +from src.models.chunk import Chunk + +from .chunking import build_and_store_index + +logger = get_logger(__name__) + + +def main() -> None: + setup_logging() + with get_session() as session: + rows = session.execute(select(Chunk.chunk_id, Chunk.content).order_by(Chunk.id)).all() + + if not rows: + sys.exit("No chunks in the database. Run `make ingest` first.") + + build_and_store_index([content for _, content in rows], [chunk_id for chunk_id, _ in rows]) + print(f"BM25 index rebuilt from {len(rows)} chunks.") + + +if __name__ == "__main__": + main() diff --git a/src/ingestion/chunking.py b/src/ingestion/chunking.py index a94bac2..7ef6b00 100644 --- a/src/ingestion/chunking.py +++ b/src/ingestion/chunking.py @@ -10,11 +10,13 @@ from langchain_text_splitters import RecursiveCharacterTextSplitter from rank_bm25 import BM25Okapi +from sqlalchemy import func +from sqlalchemy.dialects.postgresql import insert +from src.core.db import get_session from src.core.logging import get_logger from src.core.text import count_tokens - -from .constants import INDEX_DIR +from src.models.search_index import BM25_INDEX, SearchIndex logger = get_logger(__name__) @@ -126,20 +128,23 @@ def make_chunk( def build_and_store_index(texts: list[str], chunk_ids: list[str]) -> None: - INDEX_DIR.mkdir(parents=True, exist_ok=True) - - index_file_path = INDEX_DIR / "bm25.pkl" - - tokenized_texts = [text.lower().split() for text in texts] - - bm25 = BM25Okapi(tokenized_texts) + """Build the BM25 index and store it in the database (ADR 0021). - with index_file_path.open("wb") as file: - pickle.dump({ - "bm25": bm25, - "chunk_ids": chunk_ids, - "texts": texts, - }, - file) - - logger.info(f"BM25 index -> {len(tokenized_texts)} docs saved.") + `texts` is not kept in the payload: retrieval reads only `bm25` and + `chunk_ids`, and the chunk text is a column away in the table this + indexes. + """ + bm25 = BM25Okapi([text.lower().split() for text in texts]) + payload = pickle.dumps({"bm25": bm25, "chunk_ids": chunk_ids}) + + with get_session() as session: + session.execute( + insert(SearchIndex) + .values(name=BM25_INDEX, payload=payload, chunks=len(chunk_ids), built_at=func.now()) + .on_conflict_do_update( + index_elements=[SearchIndex.name], + set_={"payload": payload, "chunks": len(chunk_ids), "built_at": func.now()}, + ) + ) + + logger.info("BM25 index -> %d docs stored (%d KB).", len(chunk_ids), len(payload) // 1024) diff --git a/src/ingestion/constants.py b/src/ingestion/constants.py index 19d4ce0..fc4d9e7 100644 --- a/src/ingestion/constants.py +++ b/src/ingestion/constants.py @@ -65,5 +65,3 @@ MARKDOWN = Path("data/corpus/markdown") CHUNKS_DIR = Path("data/corpus/chunks") - -INDEX_DIR = Path("data/corpus/indices") diff --git a/src/models/search_index.py b/src/models/search_index.py new file mode 100644 index 0000000..0cc3a5c --- /dev/null +++ b/src/models/search_index.py @@ -0,0 +1,28 @@ +from datetime import datetime + +from sqlalchemy import DateTime, Integer, LargeBinary, String, func +from sqlalchemy.orm import Mapped, mapped_column + +from src.core.db import Base + +# The one index there is. A second would be a second row, not a second table. +BM25_INDEX = "bm25" + + +class SearchIndex(Base): + """A built search index, stored beside the chunks it indexes (ADR 0021). + + One row per index; `bm25` is the only one today. It lives here rather than + on disk because the request path needs it and the app has to be able to + run somewhere that has no `data/` directory. It is derived data: rebuilt + from `chunks` whenever they change, never authored. + """ + + __tablename__ = "search_indexes" + + name: Mapped[str] = mapped_column(String(50), primary_key=True) + payload: Mapped[bytes] = mapped_column(LargeBinary, nullable=False) + chunks: Mapped[int] = mapped_column(Integer, nullable=False) + built_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), server_default=func.now(), nullable=False + ) diff --git a/src/policypal/config.py b/src/policypal/config.py index 97f558a..22c9055 100644 --- a/src/policypal/config.py +++ b/src/policypal/config.py @@ -48,9 +48,8 @@ class Settings(BaseSettings): # exclusion that follows in the next sentence) at a hard chunk boundary. chunk_overlap: int = 50 - # Hybrid retrieval - bm25_index_path: str = "data/corpus/indices/bm25.pkl" - + # Hybrid retrieval. The BM25 index lives in the database, not on disk + # (ADR 0021), so there is no path to configure. sparse_top_k: int = 20 dense_top_k: int = 30 diff --git a/src/services/retrieval.py b/src/services/retrieval.py index 92404c8..31b6a6b 100644 --- a/src/services/retrieval.py +++ b/src/services/retrieval.py @@ -3,15 +3,16 @@ import re from dataclasses import dataclass from functools import lru_cache -from pathlib import Path from sqlalchemy import select from src.core.db import get_session from src.core.embedding import embed_query +from src.core.exceptions import MissingSearchIndexError from src.core.logging import get_logger from src.core.reranker import rerank from src.models.chunk import Chunk +from src.models.search_index import BM25_INDEX, SearchIndex from src.policypal.config import settings logger = get_logger(__name__) @@ -27,16 +28,23 @@ class RetrievedChunk: @lru_cache def _bm25_index() -> dict: - index_path = Path(settings.bm25_index_path) + """The stored BM25 index, read once per process (ADR 0021). - if not index_path.exists(): - raise FileNotFoundError(f"BM25 index not found at {index_path}. Run the chunk stage first.") - - with index_path.open("rb") as file: - # This file is only ever produced by our own ingestion pipeline - # (src/ingestion/chunk.py), never from user input or an external - # source, so there's no untrusted data to deserialize here. - return pkl.load(file) # nosec B301 + It comes from the database rather than a file so the app can run + somewhere that has no `data/` directory, and so it travels with the + chunks it indexes instead of beside them. + """ + with get_session() as session: + payload = session.scalar(select(SearchIndex.payload).where(SearchIndex.name == BM25_INDEX)) + + if payload is None: + raise MissingSearchIndexError( + "No BM25 index stored. Run `make build-index` (or `make ingest`) first." + ) + # Written only by our own ingestion (src/ingestion/chunking.py) into a + # table only this application writes to, never from user input or an + # external source, so there is no untrusted data to deserialize here. + return pkl.loads(payload) # nosec B301 def _sparse_search(query: str, top_k: int) -> list[str]: diff --git a/tests/test_ingestion_pipeline.py b/tests/test_ingestion_pipeline.py index f64835d..ffd1de7 100644 --- a/tests/test_ingestion_pipeline.py +++ b/tests/test_ingestion_pipeline.py @@ -1,15 +1,14 @@ """Tests for the source-agnostic ingestion orchestration and the Wikipedia source. The orchestrator's failure mode is silence: if it writes a malformed JSONL or -an index whose ids don't line up with its texts, nothing raises — retrieval -just quietly returns the wrong chunks. +hands the indexer ids that don't line up with its texts, nothing raises — +retrieval just quietly returns the wrong chunks. -The only pickle loaded below is the BM25 index these tests just built in a -tmp_path fixture — a build artifact of the code under test, never external -input. Same rationale as the justified load in src/services/retrieval.py. +What reaches the indexer is checked at the call, not by reading back a stored +index: the orchestrator's job is to hand over the right texts and ids, and +storing them is `build_and_store_index`'s (tested separately). """ import json -import pickle from unittest.mock import patch import pytest @@ -49,14 +48,15 @@ def _chunk(chunk_id, text="Some text about coverage.", doc_type="wikipedia"): def _run_execute(tmp_path, sources): - chunks_dir, index_dir = tmp_path / "chunks", tmp_path / "indices" + """Runs the orchestrator, returning where chunks were written and what the indexer got.""" + chunks_dir = tmp_path / "chunks" with patch.object(chunk_module, "SOURCES", sources), \ patch.object(chunk_module, "CHUNKS_DIR", chunks_dir), \ - patch("src.ingestion.chunking.INDEX_DIR", index_dir): + patch.object(chunk_module, "build_and_store_index") as indexer: chunk_module.execute() - return chunks_dir, index_dir + return chunks_dir, indexer # Orchestration @@ -85,27 +85,23 @@ def test_execute_writes_one_valid_json_object_per_line(tmp_path): def test_execute_indexes_the_contextualized_text_not_the_raw_text(tmp_path): """Retrieval depends on this: BM25 searches context, the DB stores raw.""" - _, index_dir = _run_execute(tmp_path, [_FakeSource("a", [_chunk("a_1", text="A deductible.")])]) + _, indexer = _run_execute(tmp_path, [_FakeSource("a", [_chunk("a_1", text="A deductible.")])]) - with (index_dir / "bm25.pkl").open("rb") as f: - index = pickle.load(f) - - assert index["texts"] == ["Context\nA deductible."] - assert index["chunk_ids"] == ["a_1"] + texts, chunk_ids = indexer.call_args.args + assert texts == ["Context\nA deductible."] + assert chunk_ids == ["a_1"] def test_execute_keeps_index_ids_aligned_with_index_texts(tmp_path): """A misalignment here returns the wrong chunk for every BM25 hit.""" chunks = [_chunk(f"a_{i}", text=f"Text {i}") for i in range(5)] - _, index_dir = _run_execute(tmp_path, [_FakeSource("a", chunks)]) - - with (index_dir / "bm25.pkl").open("rb") as f: - index = pickle.load(f) + _, indexer = _run_execute(tmp_path, [_FakeSource("a", chunks)]) - assert len(index["texts"]) == len(index["chunk_ids"]) == 5 - for i, chunk_id in enumerate(index["chunk_ids"]): - assert index["texts"][i].endswith(f"Text {i}") + texts, chunk_ids = indexer.call_args.args + assert len(texts) == len(chunk_ids) == 5 + for i, chunk_id in enumerate(chunk_ids): + assert texts[i].endswith(f"Text {i}") assert chunk_id == f"a_{i}" diff --git a/tests/test_search_index.py b/tests/test_search_index.py new file mode 100644 index 0000000..4f7b37f --- /dev/null +++ b/tests/test_search_index.py @@ -0,0 +1,103 @@ +"""The BM25 index, stored in the database rather than on local disk (ADR 0021). + +Real Postgres through the rolled-back `session` fixture. The only pickle +loaded here is the one these tests just built through the code under test. +""" +import pickle +from contextlib import contextmanager + +import pytest +from sqlalchemy import select + +from src.core.exceptions import MissingSearchIndexError +from src.ingestion import build_index, chunking +from src.models.chunk import Chunk +from src.models.search_index import BM25_INDEX, SearchIndex +from src.services import retrieval + +# BM25Okapi's idf is log((N - df + 0.5) / (df + 0.5)), which is exactly 0 for a +# term in one document of two — and `_sparse_search` keeps only scores above 0. +# Three documents is the smallest corpus where a hit actually scores. +_CORPUS = ["a deductible is what you pay first", "a copay is a flat fee", "a premium is monthly"] +_IDS = ["c1", "c2", "c3"] + + +@pytest.fixture +def stores(monkeypatch, session): + """Point everything that opens its own session at the rolled-back one.""" + @contextmanager + def same_session(): + yield session + session.flush() + + for module in (chunking, build_index, retrieval): + monkeypatch.setattr(module, "get_session", same_session) + # The real database has a built index and a loaded corpus; neither belongs + # in these assertions. Removed inside the transaction that is rolled back. + session.query(SearchIndex).delete() + session.query(Chunk).delete() + session.flush() + retrieval._bm25_index.cache_clear() + yield + retrieval._bm25_index.cache_clear() + + +def _stored(session): + return session.scalar(select(SearchIndex).where(SearchIndex.name == BM25_INDEX)) + + +def test_the_index_is_stored_with_the_chunk_count(stores, session): + chunking.build_and_store_index(["a deductible is what you pay first"], ["c1"]) + + row = _stored(session) + assert (row.name, row.chunks) == (BM25_INDEX, 1) + assert row.built_at is not None + + +def test_the_payload_carries_the_ids_but_not_the_texts(stores, session): + """Retrieval reads `bm25` and `chunk_ids` only; the text is a column away.""" + chunking.build_and_store_index(_CORPUS, _IDS) + + index = pickle.loads(_stored(session).payload) + + assert set(index) == {"bm25", "chunk_ids"} + assert index["chunk_ids"] == _IDS + assert index["bm25"].corpus_size == 3 + + +def test_rebuilding_replaces_the_one_row(stores, session): + chunking.build_and_store_index(["first"], ["c1"]) + chunking.build_and_store_index(_CORPUS, _IDS) + + assert session.scalars(select(SearchIndex.name)).all() == [BM25_INDEX] + assert _stored(session).chunks == 3 + + +def test_retrieval_reads_the_stored_index(stores, session): + chunking.build_and_store_index(_CORPUS, _IDS) + + assert retrieval._sparse_search("deductible", top_k=5) == ["c1"] + + +def test_retrieval_says_what_to_run_when_no_index_is_stored(stores): + with pytest.raises(MissingSearchIndexError, match="make build-index"): + retrieval._bm25_index() + + +def test_build_index_rebuilds_from_the_chunks_already_stored(stores, session): + """A database seeded from a dump has the chunks; the index is derived from them.""" + session.add_all([ + Chunk(content=content, source=f"{chunk_id}.md", chunk_id=chunk_id, embedding=[0.0] * 384) + for content, chunk_id in zip(_CORPUS, _IDS, strict=True) + ]) + session.flush() + + build_index.main() + + assert _stored(session).chunks == 3 + assert retrieval._sparse_search("copay", top_k=5) == ["c2"] + + +def test_build_index_says_what_to_run_when_there_are_no_chunks(stores): + with pytest.raises(SystemExit, match="make ingest"): + build_index.main() From 2fa1eef30b94c71942d9f1fd6b30c5aa1688a24a Mon Sep 17 00:00:00 2001 From: BADRI NARAYANAN Date: Tue, 22 Sep 2026 13:35:58 -0700 Subject: [PATCH 09/12] Put the API in a container, with the models in it and the documents out MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes the first of ADR 0002's two deferred items. Multi-stage, non-root, and verified by running rather than by reading. Two properties of this app shaped it. It carries a model runtime — torch and two transformers are in the request path — and it must never carry the documents, which are 983 MB of issuers' PDFs kept strictly local (ADR 0016). So torch comes from PyTorch's CPU index on linux, which dropped every nvidia package and triton from the lock, and both models are baked in at the revisions config.py pins, with HF_HUB_OFFLINE set at runtime: a cold start loads them from the image, and a HuggingFace outage cannot take the API down. CI builds the image, runs it, and fails if /app/data is not empty. A .dockerignore regression is otherwise invisible until an image is gigabytes too large, and this is the rule that must not break quietly. Trivy fails the build on fixable HIGH or CRITICAL findings. One gunicorn worker, eight threads. Flask-Limiter has no shared storage, so a second worker would keep its own counters and double the 5/min on register and 10/min on login. A deliberate ceiling, recorded as one. Production also logs JSON to stdout, because a container's filesystem is ephemeral and nobody reads it; development is unchanged so an ingestion run's lines do not interleave with its progress output. Also here because the image made them matter: .env.example, which did not exist though three settings have no default; `make test`, `lint` and `check` so passing locally means what CI means; and tiktoken and anyio dropped, both imported nowhere, tiktoken in particular sitting next to core/text.py's deliberate approximation looking like an unfinished intention. Measured: 2.48 GB, torch 2.14.0+cpu, zero nvidia packages, no dev tools, and a container with no data/ directory answering "What is a deductible?" from the Postgres-stored index at 0.999. --- .dockerignore | 28 ++ .env.example | 23 ++ .github/workflows/ci.yml | 38 ++ CHANGELOG.md | 20 + Dockerfile | 68 ++++ README.md | 29 +- docs/decisions/0022-the-api-as-a-container.md | 110 +++++ pyproject.toml | 14 +- src/core/logging.py | 14 +- src/core/text.py | 8 +- tests/test_api_logging.py | 27 ++ uv.lock | 384 +++--------------- 12 files changed, 439 insertions(+), 324 deletions(-) create mode 100644 .dockerignore create mode 100644 .env.example create mode 100644 Dockerfile create mode 100644 docs/decisions/0022-the-api-as-a-container.md diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..b45fd65 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,28 @@ +# Downloaded issuer PDFs and the corpus stay on the machine that built them +# (ADR 0016). Nothing in the image needs them, and the image must never carry +# them. +data/ + +.venv/ +logs/ +frontend/ +tests/ +scripts/ +docs/ +.playwright-mcp/ +.git/ +.github/ +.claude/ +.ruff_cache/ +.pytest_cache/ +.coverage +**/__pycache__/ +**/*.py[cod] +.env +.env.* +*.md +# pyproject.toml declares it as the package readme, so hatchling needs it to +# build the project itself. +!README.md +Makefile +docker-compose.yml diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..946f754 --- /dev/null +++ b/.env.example @@ -0,0 +1,23 @@ +# Copy to .env and fill in. Never commit .env. +# Every setting and its default is in src/policypal/config.py; only the three +# with no default are required. + +# --- Required --- +DATABASE_URL=postgresql+psycopg://policypal:password@localhost:5432/policypal +OPENAI_API_KEY= +# A long random string, e.g. `python -c "import secrets; print(secrets.token_urlsafe(48))"` +JWT_SECRET_KEY= + +# --- Optional --- +# Live plan prices come from the CMS Marketplace API. Without a key, plan +# search still works from the stored catalog; the prices are the stored +# age-27 ones rather than the user's age. +CMS_MARKETPLACE_API_KEY= + +# "production" switches logs to JSON on stdout as well as to a file. +ENVIRONMENT=development +LOG_LEVEL=INFO +LOG_DIR=logs/backend + +# Where the browser app is served from. CORS allows this origin and no other. +FRONTEND_ORIGIN=http://localhost:5173 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 389882b..a5f1a8e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -85,6 +85,44 @@ jobs: - name: Dependency audit run: npm audit --audit-level=high + # Closes the first of ADR 0002's two deferred items. The image is built and + # scanned on every PR; pushing it anywhere is the deploy workflow's job. + image: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1 + + - name: Build the API image + uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0 + with: + context: . + load: true + tags: policypal-api:${{ github.sha }} + cache-from: type=gha + cache-to: type=gha,mode=max + + # The PDFs and the corpus are local-only (ADR 0016) and .dockerignore + # excludes data/. A regression there would ship gigabytes of issuer + # documents, so it is checked rather than trusted. + - name: The image must carry no issuer documents + env: + IMAGE: policypal-api:${{ github.sha }} + run: | + set -euo pipefail + found=$(docker run --rm --entrypoint sh "$IMAGE" -c 'ls -A /app/data 2>/dev/null | wc -l') + test "$found" -eq 0 || { echo "data/ reached the image"; exit 1; } + + - name: Scan the image + uses: aquasecurity/trivy-action@dc5a429b52fcf669ce959baa2c2dd26090d2a6c4 # v0.32.0 + with: + image-ref: policypal-api:${{ github.sha }} + format: table + exit-code: "1" + ignore-unfixed: true + severity: HIGH,CRITICAL + secret-scan: runs-on: ubuntu-latest env: diff --git a/CHANGELOG.md b/CHANGELOG.md index eb45516..7fafe6a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,19 @@ ### Added +- A `Dockerfile` for the API, and a CI job that builds and scans the image + (ADR 0022) — the first of ADR 0002's two deferred items. Multi-stage, runs + as a non-root user, `torch` from PyTorch's CPU index on Linux so no CUDA + libraries ship, and both models baked in at their pinned revisions so a cold + start needs no HuggingFace call. CI runs the built image and fails if + `/app/data` is not empty: the issuer PDFs are local-only (ADR 0016) and a + `.dockerignore` regression would otherwise ship gigabytes of them unnoticed. + Trivy fails the build on fixable HIGH or CRITICAL findings. +- `.env.example`, which did not exist: the three settings with no default + (`DATABASE_URL`, `OPENAI_API_KEY`, `JWT_SECRET_KEY`) and the handful worth + setting, with no values in it. +- `make test`, `make lint` and `make check` run what CI runs, so passing + locally means the same thing. - Phase 4 live ingest: the catalog and SBCs for eighteen HealthCare.gov states — 3,276 plans from 137 issuers, every issuer's documents rather than the top parents'. 1,246 documents are stored and 1,940 plans (59.2%) have text behind @@ -201,6 +214,13 @@ ### Changed +- In production the logs also go to stdout as JSON, where a container platform + collects them; a container's own filesystem is ephemeral and unread. + Development is unchanged, so an ingestion run's log lines do not interleave + with its progress output. +- `tiktoken` and `anyio` are no longer dependencies. Neither was imported + anywhere; `tiktoken` in particular sat next to `core/text.py`'s deliberate + 1.35×-word-count approximation looking like an unfinished intention. - The BM25 index is stored in the database, in `search_indexes`, instead of `data/corpus/indices/bm25.pkl` (ADR 0021, new migration). A missing index was a `FileNotFoundError` nothing caught, so an app running anywhere but the diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..1dcc6e0 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,68 @@ +# The API only. Ingestion stays on the machine that holds data/ (ADR 0016, +# 0019); nothing in this image fetches or keeps a PDF. +FROM python:3.12-slim-bookworm AS builder + +COPY --from=ghcr.io/astral-sh/uv:0.11.23 /uv /bin/uv + +ENV UV_COMPILE_BYTECODE=1 \ + UV_LINK_MODE=copy \ + UV_PYTHON_DOWNLOADS=never + +WORKDIR /app + +# Dependencies resolve from the lock alone, so this layer is rebuilt only when +# the lock changes — not on every source edit. torch comes from PyTorch's CPU +# index on linux (see pyproject.toml), which leaves out the CUDA libraries. +COPY pyproject.toml uv.lock ./ +RUN --mount=type=cache,target=/root/.cache/uv \ + uv sync --locked --no-dev --no-install-project + +COPY src/ ./src/ +COPY migrations/ ./migrations/ +COPY alembic.ini README.md ./ +RUN --mount=type=cache,target=/root/.cache/uv \ + uv sync --locked --no-dev + +# Both models, at the revisions config.py pins, downloaded once here. A cold +# start then loads them from the image instead of reaching HuggingFace, so a +# HuggingFace outage cannot take the API down and no request waits on a +# 220 MB download. +# The venv's python directly, not `uv run`: `uv run` re-syncs the default +# groups, which pulls the dev dependencies back in after --no-dev removed them. +ENV HF_HOME=/opt/models +RUN JWT_SECRET_KEY=build OPENAI_API_KEY=build DATABASE_URL=postgresql+psycopg://build/build \ + /app/.venv/bin/python -c "\ +from sentence_transformers import CrossEncoder, SentenceTransformer; \ +from src.policypal.config import settings; \ +SentenceTransformer(settings.embedding_model, revision=settings.embedding_model_revision); \ +CrossEncoder(settings.reranker_model, revision=settings.reranker_model_revision)" + + +FROM python:3.12-slim-bookworm AS runtime + +# Nothing here runs as root, and nothing writes to the image. +RUN useradd --create-home --uid 10001 policypal + +ENV PATH="/app/.venv/bin:$PATH" \ + PYTHONUNBUFFERED=1 \ + PYTHONDONTWRITEBYTECODE=1 \ + HF_HOME=/opt/models \ + HF_HUB_OFFLINE=1 \ + LOG_DIR=/tmp/logs + +WORKDIR /app + +COPY --from=builder --chown=policypal:policypal /app/.venv /app/.venv +COPY --from=builder --chown=policypal:policypal /opt/models /opt/models +COPY --chown=policypal:policypal src/ ./src/ +COPY --chown=policypal:policypal migrations/ ./migrations/ +COPY --chown=policypal:policypal alembic.ini ./ + +USER policypal +EXPOSE 8000 + +# One worker: Flask-Limiter has no shared storage yet, so each worker would +# keep its own counters and the configured limits would multiply (ADR 0022). +# Threads carry the concurrency; the request path waits on OpenAI, not on CPU. +CMD ["gunicorn", "--bind", "0.0.0.0:8000", "--workers", "1", "--threads", "8", \ + "--timeout", "120", "--access-logfile", "-", "src.api.main:create_app()"] diff --git a/README.md b/README.md index 4ec97f5..6136d63 100644 --- a/README.md +++ b/README.md @@ -124,7 +124,34 @@ driven from the repo root: | `make ui-test` | Vitest | | `make ui-preview` | serves `frontend/dist` — run `make ui-build` first | -Copy the required variables below into a `.env` file at the repo root before running the API. +And what CI runs, in one command each: + +| Target | Runs | +| --- | --- | +| `make test` | pytest with the 85% coverage floor | +| `make lint` | ruff | +| `make check` | lint, tests, `alembic check`, bandit, pip-audit | + +Copy `.env.example` to `.env` at the repo root and fill in the three required values +before running the API. + +### Running the API in a container + +```bash +docker build -t policypal-api . +docker run --rm -p 8000:8000 --env-file .env -e ENVIRONMENT=production policypal-api +``` + +The image carries the API only — ingestion, refresh and the reports stay on the machine +that holds `data/` (ADR 0016, 0019), and `.dockerignore` keeps the issuer PDFs out of it. +It needs no `data/` directory at all: the BM25 index comes from the database (ADR 0021) +and both models are baked in at their pinned revisions, so a cold start makes no +HuggingFace call. `torch` comes from PyTorch's CPU index on Linux, so no CUDA libraries +ship (ADR 0022). + +It runs one gunicorn worker with eight threads. That is a deliberate ceiling, not a +default: Flask-Limiter has no shared storage yet, so a second worker would keep its own +counters and double the configured rate limits. > **macOS note:** AirPlay Receiver (Control Center) listens on `*:5000`. Flask still binds > `127.0.0.1:5000` alongside it, but `localhost` can resolve to `::1` and reach AirPlay instead — diff --git a/docs/decisions/0022-the-api-as-a-container.md b/docs/decisions/0022-the-api-as-a-container.md new file mode 100644 index 0000000..b4ac023 --- /dev/null +++ b/docs/decisions/0022-the-api-as-a-container.md @@ -0,0 +1,110 @@ +# 0022 — The API as a container + +## Status + +Accepted. Closes the first of ADR 0002's two deferred items ("Docker build + +image vulnerability scan. There is no Dockerfile for the app."). Depends on +ADR 0021, without which the image cannot serve a request. + +## Context + +The app has only ever run on the machine that built its corpus. Everything +that assumption hid had to be found before an image could work: ADR 0021 moved +the BM25 index off local disk, and the API had never configured logging at +all. + +Two properties of this app shape the image more than anything else: + +- **It carries a model runtime.** `torch`, `sentence-transformers` and two + models are in the request path, for the embedder and the cross-encoder. + `torch` alone is 553 MB of the 1.1 GB virtualenv. +- **It must never carry the documents.** `data/sbc/` is 983 MB of issuers' + PDFs, kept forever and strictly local (ADR 0016, ADR 0019). An image that + shipped them would break that, quietly, at a gigabyte a time. + +## Decision + +### The image runs the API, and nothing else + +Ingestion, refresh and the reports stay on the machine that holds `data/`. +Nothing in the image fetches a PDF, and `.dockerignore` excludes `data/`. + +**CI proves it rather than trusting it:** the built image is run and +`/app/data` must be empty. A `.dockerignore` regression is otherwise invisible +until an image is gigabytes too large. + +### torch comes from PyTorch's CPU index, on Linux only + +The default wheels bundle CUDA libraries that a CPU container cannot use. +`pyproject.toml` points `torch` at `https://download.pytorch.org/whl/cpu` +under `sys_platform == 'linux'`, so the image and CI get CPU wheels while +macOS development is untouched. Re-locking removed every `nvidia-*` package +and `triton`. + +### Both models are baked in at build time + +`SentenceTransformer` and `CrossEncoder` are instantiated during the build, at +the revisions `config.py` pins, into `HF_HOME=/opt/models`. The runtime sets +`HF_HUB_OFFLINE=1`. + +- A cold start loads them from the image instead of downloading ~220 MB. +- **A HuggingFace outage cannot take the API down**, which it could if the + first request fetched them. +- The pinned revisions are what ships, so the image is reproducible in the one + way ADR 0008 noted was weak. + +### One worker, several threads + +Flask-Limiter has no shared storage, so each worker would keep its own +counters and the configured limits would multiply by worker count — the two +that matter being 5/min on register and 10/min on login. Until there is a +shared backend, the image runs **one** gunicorn worker. + +Threads carry the concurrency instead, which suits this app: a request waits +on OpenAI and on Postgres, not on CPU. It also avoids paying for the model +runtime N times over in memory. + +This is a real ceiling, not a preference, and it is the first thing to revisit +when traffic justifies a Redis backend for the limiter. + +### Logs go to stdout in production + +`setup_logging()` wrote only to a rotating file. In a container that +filesystem is ephemeral and nobody reads it. In production the root logger +also writes JSON to stdout, where the platform collects it; in development it +does not, so an ingestion run's log lines do not interleave with its own +progress output. + +### It does not run as root, and does not write to itself + +A `policypal` user owns the virtualenv and the source. `LOG_DIR` points at +`/tmp`. The image has no shell-accessible secrets: every setting comes from +the environment, and `.env.example` documents which ones with no values in it. + +## Consequences + +- **The image is 2.48 GB** — large by ordinary standards, and mostly the model + runtime: CPU torch, and 217 MB of baked weights. Verified in the built + image: `torch 2.14.0+cpu`, **zero** `nvidia-*` packages, no `ruff` or + `pytest`, running as `policypal`, and `/app` holding only `src`, + `migrations` and `alembic.ini`. +- **Proved rather than assumed.** A container with no `data/` directory + (`Path('/app/data').exists()` is `False`) loads the BM25 index from Postgres + and answers "What is a deductible?" with the glossary entry at 0.999, + `HF_HUB_OFFLINE=1` throughout — so both models came from the image and no + request touched HuggingFace. Served over HTTP it returns the right counties + for a ZIP and 401 for an unauthenticated `/api/auth/me`, with access logs on + stdout. +- **Building it is slow on a poor connection.** The model-baking step took + about 7 minutes here, almost all of it downloading from HuggingFace + unauthenticated. CI caches the layer, so it is paid once per lock or config + change, not per build. +- **Cold starts load two models from disk.** Faster than downloading them, but + not free; scale-to-zero trades latency for cost, and the deployment ADR + decides that. +- **A model change is an image rebuild**, which is the point: the revision in + `config.py` and the weights in the image cannot drift apart. +- **The scan runs on every PR** and fails on fixable HIGH or CRITICAL findings. + A base-image CVE now breaks the build, which is the intended cost. +- **CD is still not built.** That is ADR 0002's second deferred item and the + next decision. diff --git a/pyproject.toml b/pyproject.toml index 312c98d..a5a48c6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,10 +6,10 @@ readme = "README.md" requires-python = ">=3.12" dependencies = [ "alembic>=1.18.5", - "anyio>=4.14.2", "bcrypt>=5.0.0", "email-validator>=2.3.0", "flask>=3.1.3", + "gunicorn>=23.0.0", "flask-cors>=6.0.5", "flask-jwt-extended>=4.7.4", "flask-limiter>=4.1.1", @@ -22,7 +22,6 @@ dependencies = [ "rank-bm25>=0.2.2", "sentence-transformers>=5.6.0", "sqlalchemy>=2.0.51", - "tiktoken>=0.13.0", "torch>=2.12.1", "tqdm>=4.68.3", "transformers>=5.12.1", @@ -50,3 +49,14 @@ dev = [ [tool.pytest.ini_options] pythonpath = ["."] + +# torch ships CUDA libraries by default, which are several hundred megabytes of +# dead weight in a CPU-only container. Linux (the image and CI) resolves from +# PyTorch's CPU index; macOS keeps the default wheels, which are already CPU. +[[tool.uv.index]] +name = "pytorch-cpu" +url = "https://download.pytorch.org/whl/cpu" +explicit = true + +[tool.uv.sources] +torch = [{ index = "pytorch-cpu", marker = "sys_platform == 'linux'" }] diff --git a/src/core/logging.py b/src/core/logging.py index 550bc52..b214f5b 100644 --- a/src/core/logging.py +++ b/src/core/logging.py @@ -31,6 +31,11 @@ def setup_logging() -> None: is_prod = settings.environment == "production" + # In a container the filesystem is ephemeral and nobody reads it; the + # platform collects stdout. Added only in production so a local ingestion + # run's log lines don't interleave with its own progress output. + handlers = ["file", "console"] if is_prod else ["file"] + dictConfig({ "version": 1, "disable_existing_loggers": False, @@ -51,16 +56,21 @@ def setup_logging() -> None: "backupCount": 5, # keep app.log + 5 rotated copies "encoding": "utf-8", "formatter": "json" if is_prod else "console" + }, + "console": { + "class": "logging.StreamHandler", + "stream": "ext://sys.stdout", + "formatter": "json" if is_prod else "console" } }, "root": { "level": settings.log_level, - "handlers": ["file"] + "handlers": handlers }, "loggers": { "sqlalchemy.engine": { "level": "WARNING", - "handlers": ["file"], + "handlers": handlers, "propagate": False }, # At DEBUG these log request lines and bodies: urllib3 the CMS diff --git a/src/core/text.py b/src/core/text.py index 9ade1a6..0b94276 100644 --- a/src/core/text.py +++ b/src/core/text.py @@ -1,3 +1,9 @@ def count_tokens(text: str) -> int: - """Approximate token count as 1.35x the word count.""" + """Approximate token count as 1.35x the word count. + + Deliberately an approximation, not a tokenizer. It decides chunk + boundaries, so switching to an exact count would re-chunk the corpus and + move every stored chunk_id — a re-ingest and a floor re-measure, to bound + a history budget and a chunk size that are both already generous. + """ return int(len(text.split()) * 1.35) diff --git a/tests/test_api_logging.py b/tests/test_api_logging.py index 9661502..30ee799 100644 --- a/tests/test_api_logging.py +++ b/tests/test_api_logging.py @@ -8,6 +8,10 @@ import logging from logging.handlers import RotatingFileHandler +import pytest + +from src.core import logging as app_logging + def test_creating_the_app_installs_the_file_handler(app): root = logging.getLogger() @@ -24,3 +28,26 @@ def test_the_libraries_that_would_log_secrets_stay_at_warning(app): """urllib3 logs the CMS URL with its ?apikey=; httpx and openai log the prompt.""" for name in ("urllib3", "httpx", "openai"): assert logging.getLogger(name).level == logging.WARNING + + +@pytest.fixture +def production(monkeypatch): + monkeypatch.setattr(app_logging.settings, "environment", "production") + yield + app_logging.setup_logging() # put the development configuration back + + +def _streams(root): + return [h for h in root.handlers if type(h) is logging.StreamHandler] + + +def test_production_also_logs_to_stdout(production): + """A container's filesystem is ephemeral and unread; the platform collects stdout.""" + app_logging.setup_logging() + + assert _streams(logging.getLogger()) + + +def test_development_logs_only_to_the_file(app): + """Otherwise an ingestion run's log lines interleave with its own progress output.""" + assert not _streams(logging.getLogger()) diff --git a/uv.lock b/uv.lock index 9b896dc..471e1bb 100644 --- a/uv.lock +++ b/uv.lock @@ -1,6 +1,10 @@ version = 1 revision = 3 requires-python = ">=3.12" +resolution-markers = [ + "sys_platform == 'linux'", + "sys_platform != 'linux'", +] [[package]] name = "alembic" @@ -500,81 +504,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/99/89/87ef49ffe383ef4e147d27b7bf2088fb0b54ea409dd87b5a89442e5828a5/cryptography-50.0.1-cp39-abi3-win_amd64.whl", hash = "sha256:55d16b1ef3ee0958d893a977b19777887e546c9954ea81b200c3301a864013f2", size = 3875429, upload-time = "2026-08-25T19:45:24.418Z" }, ] -[[package]] -name = "cuda-bindings" -version = "13.3.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "cuda-pathfinder" }, -] -wheels = [ - { url = "https://files.pythonhosted.org/packages/ce/67/5e7dba1ba576dd73da5dee894ca076ca5e959450dfff66d6d510a255d1f7/cuda_bindings-13.3.1-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c7855c4868aabc0cfae28abbe83d56734bdfbd08f08fc234ac1912a12858bf49", size = 6025351, upload-time = "2026-05-29T23:11:49.685Z" }, - { url = "https://files.pythonhosted.org/packages/39/2a/6d2e9047d1fb243dbaa364b01e0297534b9ed7fd27dba1c9f361519cf69b/cuda_bindings-13.3.1-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e32d08f71ebcdf00f0f41eab2eb37e8da94c8ed411cc9f7f7a019ce6b34abe3a", size = 6657965, upload-time = "2026-05-29T23:11:52.227Z" }, - { url = "https://files.pythonhosted.org/packages/cc/6e/2394f8163360f8391f8f1b7e72d300a82724edb81a7b7084c799fbd4c91f/cuda_bindings-13.3.1-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9efb21c1ee64981e184b9e0ba5eb3179e5ba3d4b51665a6cb52b8ef3d01a7cbf", size = 5920504, upload-time = "2026-05-29T23:11:56.883Z" }, - { url = "https://files.pythonhosted.org/packages/34/c2/ef9b6a63f7dc432712a462c816662e662e00d38caa9b861c8c2588195d03/cuda_bindings-13.3.1-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2732904099e0a4d4db774a5fc6d91ee95fae065b4d2ecabb4968c5fe2406c9d7", size = 6476660, upload-time = "2026-05-29T23:11:59.188Z" }, - { url = "https://files.pythonhosted.org/packages/b1/81/bff68ce829999c1e4209c761bbf903b1c06ec570416ddb25020864ad5907/cuda_bindings-13.3.1-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1ab2f74ed65bfef4163ba07a8db16f1085e0729291db12a2423aff84ee8278b8", size = 6013639, upload-time = "2026-05-29T23:12:03.509Z" }, - { url = "https://files.pythonhosted.org/packages/d4/e0/c8a1f0c8f9ffdea4f5fe6dbab89b326cef4d85caf489dad39e209da89416/cuda_bindings-13.3.1-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:efd4c814d311ec08c981f6dded1dbe7d4b371067ee4f6c14cccec4bde9590f80", size = 6534419, upload-time = "2026-05-29T23:12:05.633Z" }, - { url = "https://files.pythonhosted.org/packages/52/b8/83b1f563925b290f2d11a01a77a84013ba56052fe3653a5bef3ccfbb43d6/cuda_bindings-13.3.1-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c3c772dfff49681541d59630c90f858e173ac926b9c593a2b7123f2a1043cc76", size = 5809771, upload-time = "2026-05-29T23:12:10.422Z" }, - { url = "https://files.pythonhosted.org/packages/12/20/e79b4bfe98f075195afb6343d41c498f9dbd2d161d7021d4d28bceb83581/cuda_bindings-13.3.1-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:36febb7c1079d68a981dbbd8d5a67235b399802b82075c9388624719607e52b9", size = 6358584, upload-time = "2026-05-29T23:12:12.767Z" }, -] - -[[package]] -name = "cuda-pathfinder" -version = "1.5.5" -source = { registry = "https://pypi.org/simple" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/11/c8/26f2e4aae92f11522a96043892ba39a90eac610d5242523aa863212bc1c7/cuda_pathfinder-1.5.5-py3-none-any.whl", hash = "sha256:0228c023f95d1480f143ef5c8922d27a2ab052087a942e81dc289c9eb8f91689", size = 51671, upload-time = "2026-05-27T01:21:25.413Z" }, -] - -[[package]] -name = "cuda-toolkit" -version = "13.0.3.0" -source = { registry = "https://pypi.org/simple" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/d1/c7/a79086a62c98befcdb8349656c6f114e2db3b8b2422f6e25c97a7f2a9a3c/cuda_toolkit-13.0.3.0-py2.py3-none-any.whl", hash = "sha256:d693caaa261214ddd7dbb60d68e71cbed884e68c2be7509778f3051da0b91c3f", size = 2512, upload-time = "2026-04-14T00:50:08.173Z" }, -] - -[package.optional-dependencies] -cublas = [ - { name = "nvidia-cublas", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (platform_machine == 'AMD64' and sys_platform == 'win32')" }, - { name = "nvidia-cuda-nvrtc", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (platform_machine == 'AMD64' and sys_platform == 'win32')" }, -] -cudart = [ - { name = "nvidia-cuda-runtime", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (platform_machine == 'AMD64' and sys_platform == 'win32')" }, -] -cufft = [ - { name = "nvidia-cufft", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (platform_machine == 'AMD64' and sys_platform == 'win32')" }, - { name = "nvidia-nvjitlink", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (platform_machine == 'AMD64' and sys_platform == 'win32')" }, -] -cufile = [ - { name = "nvidia-cufile", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, -] -cupti = [ - { name = "nvidia-cuda-cupti", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (platform_machine == 'AMD64' and sys_platform == 'win32')" }, -] -curand = [ - { name = "nvidia-curand", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (platform_machine == 'AMD64' and sys_platform == 'win32')" }, -] -cusolver = [ - { name = "nvidia-cublas", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (platform_machine == 'AMD64' and sys_platform == 'win32')" }, - { name = "nvidia-cusolver", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (platform_machine == 'AMD64' and sys_platform == 'win32')" }, - { name = "nvidia-cusparse", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (platform_machine == 'AMD64' and sys_platform == 'win32')" }, - { name = "nvidia-nvjitlink", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (platform_machine == 'AMD64' and sys_platform == 'win32')" }, -] -cusparse = [ - { name = "nvidia-cusparse", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (platform_machine == 'AMD64' and sys_platform == 'win32')" }, - { name = "nvidia-nvjitlink", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (platform_machine == 'AMD64' and sys_platform == 'win32')" }, -] -nvjitlink = [ - { name = "nvidia-nvjitlink", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (platform_machine == 'AMD64' and sys_platform == 'win32')" }, -] -nvrtc = [ - { name = "nvidia-cuda-nvrtc", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (platform_machine == 'AMD64' and sys_platform == 'win32')" }, -] -nvtx = [ - { name = "nvidia-nvtx", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (platform_machine == 'AMD64' and sys_platform == 'win32')" }, -] - [[package]] name = "cyclonedx-python-lib" version = "11.12.0" @@ -814,6 +743,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b4/0d/ca7d15afbdc397e3401134c9e1800d51d12b829661786187a4ad08fe484f/greenlet-3.5.3-cp315-cp315t-win_arm64.whl", hash = "sha256:b7068bd09f761f3f5b4d214c2bed063186b2a86148c740b3873e3f56d79bac31", size = 242586, upload-time = "2026-06-26T18:23:37.93Z" }, ] +[[package]] +name = "gunicorn" +version = "26.2.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d9/8a/e4ef6ee11701b6cd64702848415ffb69eeff85cb388a3c6c7fe86f22f3f8/gunicorn-26.2.0.tar.gz", hash = "sha256:62b864895d9ebff0b2f9867ba04fe811c93121596540830c9c916d0769668447", size = 787921, upload-time = "2026-08-24T15:05:59.3Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fe/85/7522a52e5e2f42faf1a129113ab63e548c42e103e9af395b7bfe65e403e2/gunicorn-26.2.0-py3-none-any.whl", hash = "sha256:bd249d0b3f7972f7432f0a6b6ff3b3ee2d129f70cd1ff6c09a9dd9e29a2b88e3", size = 228389, upload-time = "2026-08-24T15:05:57.67Z" }, +] + [[package]] name = "h11" version = "0.16.0" @@ -1571,158 +1509,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/fd/6a/d3a169aaf8536cf228d56a09e04bcb713a2fe4410d4e2105b9419b5a9c89/numpy-2.5.0-cp314-cp314t-win_arm64.whl", hash = "sha256:016623417bb330d719d579daf2d6b9a01ddc52e41a9ed61a47f39fde46dcd865", size = 10686451, upload-time = "2026-06-21T20:57:49.313Z" }, ] -[[package]] -name = "nvidia-cublas" -version = "13.1.1.3" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "nvidia-cuda-nvrtc" }, -] -wheels = [ - { url = "https://files.pythonhosted.org/packages/a7/a1/0bd24ee8c8d03adac032fd2909426a00c88f8c57961b1277ded97f91119f/nvidia_cublas-13.1.1.3-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:b7a210458267ac818974c53038fbec2e969d5c99f305ab15c72522fa9f001dd5", size = 542848918, upload-time = "2026-04-08T18:46:22.985Z" }, - { url = "https://files.pythonhosted.org/packages/3b/cd/154ca20c38269e05eff77c1464e6c1da89f50a6390b565e9d82e06bc11e1/nvidia_cublas-13.1.1.3-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:37936a16db8fe4ac1f065c2139360608a543a09275cb1a1af612e08cfa065436", size = 423138758, upload-time = "2026-04-08T18:46:58.655Z" }, -] - -[[package]] -name = "nvidia-cuda-cupti" -version = "13.0.85" -source = { registry = "https://pypi.org/simple" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/2a/2a/80353b103fc20ce05ef51e928daed4b6015db4aaa9162ed0997090fe2250/nvidia_cuda_cupti-13.0.85-py3-none-manylinux_2_25_aarch64.whl", hash = "sha256:796bd679890ee55fb14a94629b698b6db54bcfd833d391d5e94017dd9d7d3151", size = 10310827, upload-time = "2025-09-04T08:26:42.012Z" }, - { url = "https://files.pythonhosted.org/packages/33/6d/737d164b4837a9bbd202f5ae3078975f0525a55730fe871d8ed4e3b952b0/nvidia_cuda_cupti-13.0.85-py3-none-manylinux_2_25_x86_64.whl", hash = "sha256:4eb01c08e859bf924d222250d2e8f8b8ff6d3db4721288cf35d14252a4d933c8", size = 10715597, upload-time = "2025-09-04T08:26:51.312Z" }, -] - -[[package]] -name = "nvidia-cuda-nvrtc" -version = "13.0.88" -source = { registry = "https://pypi.org/simple" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/c3/68/483a78f5e8f31b08fb1bb671559968c0ca3a065ac7acabfc7cee55214fd6/nvidia_cuda_nvrtc-13.0.88-py3-none-manylinux2010_x86_64.manylinux_2_12_x86_64.whl", hash = "sha256:ad9b6d2ead2435f11cbb6868809d2adeeee302e9bb94bcf0539c7a40d80e8575", size = 90215200, upload-time = "2025-09-04T08:28:44.204Z" }, - { url = "https://files.pythonhosted.org/packages/b7/dc/6bb80850e0b7edd6588d560758f17e0550893a1feaf436807d64d2da040f/nvidia_cuda_nvrtc-13.0.88-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:d27f20a0ca67a4bb34268a5e951033496c5b74870b868bacd046b1b8e0c3267b", size = 43015449, upload-time = "2025-09-04T08:28:20.239Z" }, -] - -[[package]] -name = "nvidia-cuda-runtime" -version = "13.0.96" -source = { registry = "https://pypi.org/simple" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/87/4f/17d7b9b8e285199c58ce28e31b5c5bbaa4d8271af06a89b6405258245de2/nvidia_cuda_runtime-13.0.96-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:ef9bcbe90493a2b9d810e43d249adb3d02e98dd30200d86607d8d02687c43f55", size = 2261060, upload-time = "2025-10-09T08:55:15.78Z" }, - { url = "https://files.pythonhosted.org/packages/2e/24/d1558f3b68b1d26e706813b1d10aa1d785e4698c425af8db8edc3dced472/nvidia_cuda_runtime-13.0.96-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:7f82250d7782aa23b6cfe765ecc7db554bd3c2870c43f3d1821f1d18aebf0548", size = 2243632, upload-time = "2025-10-09T08:55:36.117Z" }, -] - -[[package]] -name = "nvidia-cudnn-cu13" -version = "9.24.0.43" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "nvidia-cublas" }, -] -wheels = [ - { url = "https://files.pythonhosted.org/packages/ca/30/7c257e3d5cb4fecb147b93895c66e29c93f8e76d74b45bb418ff0587c4ec/nvidia_cudnn_cu13-9.24.0.43-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:a6812a554a1ff0413e9c52b84c26c050380649ab9615f9c16bded368ce9f421f", size = 650976863, upload-time = "2026-07-02T16:23:39.248Z" }, - { url = "https://files.pythonhosted.org/packages/5c/ba/791cffd048fe5b044e620df55267e3e95c0e6e07d50b41e377c03dfc910f/nvidia_cudnn_cu13-9.24.0.43-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:71f181cd810e90f9b6023b01186fe82d13d65f0ec098581ee201d39fad769e4b", size = 553099438, upload-time = "2026-07-02T16:27:42.58Z" }, -] - -[[package]] -name = "nvidia-cufft" -version = "12.0.0.61" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "nvidia-nvjitlink" }, -] -wheels = [ - { url = "https://files.pythonhosted.org/packages/8b/ae/f417a75c0259e85c1d2f83ca4e960289a5f814ed0cea74d18c353d3e989d/nvidia_cufft-12.0.0.61-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:2708c852ef8cd89d1d2068bdbece0aa188813a0c934db3779b9b1faa8442e5f5", size = 214053554, upload-time = "2025-09-04T08:31:38.196Z" }, - { url = "https://files.pythonhosted.org/packages/a8/2f/7b57e29836ea8714f81e9898409196f47d772d5ddedddf1592eadb8ab743/nvidia_cufft-12.0.0.61-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:6c44f692dce8fd5ffd3e3df134b6cdb9c2f72d99cf40b62c32dde45eea9ddad3", size = 214085489, upload-time = "2025-09-04T08:31:56.044Z" }, -] - -[[package]] -name = "nvidia-cufile" -version = "1.15.1.6" -source = { registry = "https://pypi.org/simple" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/3f/70/4f193de89a48b71714e74602ee14d04e4019ad36a5a9f20c425776e72cd6/nvidia_cufile-1.15.1.6-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:08a3ecefae5a01c7f5117351c64f17c7c62efa5fffdbe24fc7d298da19cd0b44", size = 1223672, upload-time = "2025-09-04T08:32:22.779Z" }, - { url = "https://files.pythonhosted.org/packages/ab/73/cc4a14c9813a8a0d509417cf5f4bdaba76e924d58beb9864f5a7baceefbf/nvidia_cufile-1.15.1.6-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:bdc0deedc61f548bddf7733bdc216456c2fdb101d020e1ab4b88d232d5e2f6d1", size = 1136992, upload-time = "2025-09-04T08:32:14.119Z" }, -] - -[[package]] -name = "nvidia-curand" -version = "10.4.0.35" -source = { registry = "https://pypi.org/simple" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/1e/72/7c2ae24fb6b63a32e6ae5d241cc65263ea18d08802aaae087d9f013335a2/nvidia_curand-10.4.0.35-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:133df5a7509c3e292aaa2b477afd0194f06ce4ea24d714d616ff36439cee349a", size = 61962106, upload-time = "2025-08-04T10:21:41.128Z" }, - { url = "https://files.pythonhosted.org/packages/a5/9f/be0a41ca4a4917abf5cb9ae0daff1a6060cc5de950aec0396de9f3b52bc5/nvidia_curand-10.4.0.35-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:1aee33a5da6e1db083fe2b90082def8915f30f3248d5896bcec36a579d941bfc", size = 59544258, upload-time = "2025-08-04T10:22:03.992Z" }, -] - -[[package]] -name = "nvidia-cusolver" -version = "12.0.4.66" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "nvidia-cublas" }, - { name = "nvidia-cusparse" }, - { name = "nvidia-nvjitlink" }, -] -wheels = [ - { url = "https://files.pythonhosted.org/packages/c8/c3/b30c9e935fc01e3da443ec0116ed1b2a009bb867f5324d3f2d7e533e776b/nvidia_cusolver-12.0.4.66-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:02c2457eaa9e39de20f880f4bd8820e6a1cfb9f9a34f820eb12a155aa5bc92d2", size = 223467760, upload-time = "2025-09-04T08:33:04.222Z" }, - { url = "https://files.pythonhosted.org/packages/5f/67/cba3777620cdacb99102da4042883709c41c709f4b6323c10781a9c3aa34/nvidia_cusolver-12.0.4.66-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:0a759da5dea5c0ea10fd307de75cdeb59e7ea4fcb8add0924859b944babf1112", size = 200941980, upload-time = "2025-09-04T08:33:22.767Z" }, -] - -[[package]] -name = "nvidia-cusparse" -version = "12.6.3.3" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "nvidia-nvjitlink" }, -] -wheels = [ - { url = "https://files.pythonhosted.org/packages/f8/94/5c26f33738ae35276672f12615a64bd008ed5be6d1ebcb23579285d960a9/nvidia_cusparse-12.6.3.3-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:80bcc4662f23f1054ee334a15c72b8940402975e0eab63178fc7e670aa59472c", size = 162155568, upload-time = "2025-09-04T08:33:42.864Z" }, - { url = "https://files.pythonhosted.org/packages/fa/18/623c77619c31d62efd55302939756966f3ecc8d724a14dab2b75f1508850/nvidia_cusparse-12.6.3.3-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:2b3c89c88d01ee0e477cb7f82ef60a11a4bcd57b6b87c33f789350b59759360b", size = 145942937, upload-time = "2025-09-04T08:33:58.029Z" }, -] - -[[package]] -name = "nvidia-cusparselt-cu13" -version = "0.8.1" -source = { registry = "https://pypi.org/simple" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/46/e1/cdc1797eadf82d3a9a575a19b33fdc871a97edbec42c00b5b5e914f4aff4/nvidia_cusparselt_cu13-0.8.1-py3-none-manylinux2014_aarch64.whl", hash = "sha256:4dca476c50bf4780d46cd0bfbd82e2bc10a08e4fef7950917ce8d7578d22a23f", size = 221051344, upload-time = "2025-09-05T18:49:51.289Z" }, - { url = "https://files.pythonhosted.org/packages/34/7d/2661f2fb3ac4302f3a246f5fc030213ac60c1fe0bce84f9783dbd831dbb7/nvidia_cusparselt_cu13-0.8.1-py3-none-manylinux2014_x86_64.whl", hash = "sha256:786ce87568c303fadb5afcc7102d454cd3040d75f6f8626f5db460d1871f4dd0", size = 170148586, upload-time = "2025-09-05T18:50:50.248Z" }, -] - -[[package]] -name = "nvidia-nccl-cu13" -version = "2.30.7" -source = { registry = "https://pypi.org/simple" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/d1/21/a73174c6157101bdf1ffc22b517f76ff0082613989dd9bc8f43e8034caac/nvidia_nccl_cu13-2.30.7-py3-none-manylinux_2_18_aarch64.whl", hash = "sha256:ca786ffa5a647c75d4d1f5cc72a6c4f537947e2ba8823d7c8aaf768e7a7b9f77", size = 215983881, upload-time = "2026-06-09T03:23:15.633Z" }, - { url = "https://files.pythonhosted.org/packages/3f/34/c500f90c7ae641b8e0f98965b36b8a7ac79cc8b296e8d251fe3eb592ee54/nvidia_nccl_cu13-2.30.7-py3-none-manylinux_2_18_x86_64.whl", hash = "sha256:cefa7fdb9710efd0f39c5f1be1d61ff6fc9a996c451265bd7fbdcf9455ed4b50", size = 215965170, upload-time = "2026-06-09T03:23:39.73Z" }, -] - -[[package]] -name = "nvidia-nvjitlink" -version = "13.0.88" -source = { registry = "https://pypi.org/simple" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/56/7a/123e033aaff487c77107195fa5a2b8686795ca537935a24efae476c41f05/nvidia_nvjitlink-13.0.88-py3-none-manylinux2010_x86_64.manylinux_2_12_x86_64.whl", hash = "sha256:13a74f429e23b921c1109976abefacc69835f2f433ebd323d3946e11d804e47b", size = 40713933, upload-time = "2025-09-04T08:35:43.553Z" }, - { url = "https://files.pythonhosted.org/packages/ab/2c/93c5250e64df4f894f1cbb397c6fd71f79813f9fd79d7cd61de3f97b3c2d/nvidia_nvjitlink-13.0.88-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:e931536ccc7d467a98ba1d8b89ff7fa7f1fa3b13f2b0069118cd7f47bff07d0c", size = 38768748, upload-time = "2025-09-04T08:35:20.008Z" }, -] - -[[package]] -name = "nvidia-nvshmem-cu13" -version = "3.4.5" -source = { registry = "https://pypi.org/simple" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/dc/0f/05cc9c720236dcd2db9c1ab97fff629e96821be2e63103569da0c9b72f19/nvidia_nvshmem_cu13-3.4.5-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:6dc2a197f38e5d0376ad52cd1a2a3617d3cdc150fd5966f4aee9bcebb1d68fe9", size = 60215947, upload-time = "2025-09-06T00:32:20.022Z" }, - { url = "https://files.pythonhosted.org/packages/3c/35/a9bf80a609e74e3b000fef598933235c908fcefcef9026042b8e6dfde2a9/nvidia_nvshmem_cu13-3.4.5-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:290f0a2ee94c9f3687a02502f3b9299a9f9fe826e6d0287ee18482e78d495b80", size = 60412546, upload-time = "2025-09-06T00:32:41.564Z" }, -] - -[[package]] -name = "nvidia-nvtx" -version = "13.0.85" -source = { registry = "https://pypi.org/simple" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/c2/f3/d86c845465a2723ad7e1e5c36dcd75ddb82898b3f53be47ebd429fb2fa5d/nvidia_nvtx-13.0.85-py3-none-manylinux1_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:4936d1d6780fbe68db454f5e72a42ff64d1fd6397df9f363ae786930fd5c1cd4", size = 148047, upload-time = "2025-09-04T08:29:01.761Z" }, - { url = "https://files.pythonhosted.org/packages/a8/64/3708a90d1ebe202ffdeb7185f878a3c84d15c2b2c31858da2ce0583e2def/nvidia_nvtx-13.0.85-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:cb7780edb6b14107373c835bf8b72e7a178bac7367e23da7acb108f973f157a6", size = 148878, upload-time = "2025-09-04T08:28:53.627Z" }, -] - [[package]] name = "openai" version = "3.13.0" @@ -2030,13 +1816,13 @@ version = "0.1.0" source = { editable = "." } dependencies = [ { name = "alembic" }, - { name = "anyio" }, { name = "bcrypt" }, { name = "email-validator" }, { name = "flask" }, { name = "flask-cors" }, { name = "flask-jwt-extended" }, { name = "flask-limiter" }, + { name = "gunicorn" }, { name = "langchain-text-splitters" }, { name = "openai" }, { name = "pdfplumber" }, @@ -2048,8 +1834,8 @@ dependencies = [ { name = "requests" }, { name = "sentence-transformers" }, { name = "sqlalchemy" }, - { name = "tiktoken" }, - { name = "torch" }, + { name = "torch", version = "2.14.0", source = { registry = "https://pypi.org/simple" }, marker = "sys_platform != 'linux'" }, + { name = "torch", version = "2.14.0+cpu", source = { registry = "https://download.pytorch.org/whl/cpu" }, marker = "sys_platform == 'linux'" }, { name = "tqdm" }, { name = "transformers" }, { name = "wikipedia-api" }, @@ -2068,13 +1854,13 @@ dev = [ [package.metadata] requires-dist = [ { name = "alembic", specifier = ">=1.18.5" }, - { name = "anyio", specifier = ">=4.14.2" }, { name = "bcrypt", specifier = ">=5.0.0" }, { name = "email-validator", specifier = ">=2.3.0" }, { name = "flask", specifier = ">=3.1.3" }, { name = "flask-cors", specifier = ">=6.0.5" }, { name = "flask-jwt-extended", specifier = ">=4.7.4" }, { name = "flask-limiter", specifier = ">=4.1.1" }, + { name = "gunicorn", specifier = ">=23.0.0" }, { name = "langchain-text-splitters", specifier = ">=1.1.2" }, { name = "openai", specifier = ">=3.13.0" }, { name = "pdfplumber", specifier = ">=0.11.10" }, @@ -2086,8 +1872,8 @@ requires-dist = [ { name = "requests", specifier = ">=2.32.0" }, { name = "sentence-transformers", specifier = ">=5.6.0" }, { name = "sqlalchemy", specifier = ">=2.0.51" }, - { name = "tiktoken", specifier = ">=0.13.0" }, - { name = "torch", specifier = ">=2.12.1" }, + { name = "torch", marker = "sys_platform != 'linux'", specifier = ">=2.12.1" }, + { name = "torch", marker = "sys_platform == 'linux'", specifier = ">=2.12.1", index = "https://download.pytorch.org/whl/cpu" }, { name = "tqdm", specifier = ">=4.68.3" }, { name = "transformers", specifier = ">=5.12.1" }, { name = "wikipedia-api", specifier = ">=0.15.0" }, @@ -2828,7 +2614,8 @@ dependencies = [ { name = "numpy" }, { name = "scikit-learn" }, { name = "scipy" }, - { name = "torch" }, + { name = "torch", version = "2.14.0", source = { registry = "https://pypi.org/simple" }, marker = "sys_platform != 'linux'" }, + { name = "torch", version = "2.14.0+cpu", source = { registry = "https://download.pytorch.org/whl/cpu" }, marker = "sys_platform == 'linux'" }, { name = "tqdm" }, { name = "transformers" }, { name = "typing-extensions" }, @@ -2977,53 +2764,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/32/d5/f9a850d79b0851d1d4ef6456097579a9005b31fea68726a4ae5f2d82ddd9/threadpoolctl-3.6.0-py3-none-any.whl", hash = "sha256:43a0b8fd5a2928500110039e43a5eed8480b918967083ea48dc3ab9f13c4a7fb", size = 18638, upload-time = "2025-03-13T13:49:21.846Z" }, ] -[[package]] -name = "tiktoken" -version = "0.13.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "regex" }, - { name = "requests" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/e4/e5/5f3cb2159769d0f4324c0e9e87f9de3c4b1cd45848a96b2eb3566ad5ca77/tiktoken-0.13.0.tar.gz", hash = "sha256:c9435714c3a84c2319499de9a300c0e604449dd0799ff246458b3bb6a7f433c1", size = 38986, upload-time = "2026-05-15T04:51:27.153Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/85/8e/144bde4e01df66b34bb865557c7cd754ed08b036217ebd79c9db5e9048a9/tiktoken-0.13.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:32ac870a806cfb260a02d0cb70426aef02e038297f8ad50df5040bb5af360791", size = 1034888, upload-time = "2026-05-15T04:50:31.579Z" }, - { url = "https://files.pythonhosted.org/packages/36/18/d4ac9d20956cdebca04841316660ed584c2fecdc2b81722a28bc7ad3b1e4/tiktoken-0.13.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:4d9980f11429ed2d737c463bb1fb78cf330caa026adf002f714aced7849a687b", size = 982970, upload-time = "2026-05-15T04:50:32.961Z" }, - { url = "https://files.pythonhosted.org/packages/74/ed/6bb8d05b9f731f749fee5c6f5ca63e981143c826a5985877330507bd13b7/tiktoken-0.13.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:3f277ebea5edd7b8bf03c6f9431e1d67d517530115572b2dc1d465326e8f88c7", size = 1115741, upload-time = "2026-05-15T04:50:34.475Z" }, - { url = "https://files.pythonhosted.org/packages/34/de/2ca96b07a82d972b74fe4b46de055b79c904e45c7eab699354a0bfa697dc/tiktoken-0.13.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:a116178fa7e1b4065bff05214360373a65cac22f965be7b3f73d00a0dbfe7649", size = 1136523, upload-time = "2026-05-15T04:50:35.782Z" }, - { url = "https://files.pythonhosted.org/packages/ee/dc/9dafec002c2d4424378563cf4cf5c7fb93631d2a55013c8b87554ee4012c/tiktoken-0.13.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:2c397ddda233208345b01bd30f2fca79ff730e55731d0108a603f9bc57f6af3b", size = 1181954, upload-time = "2026-05-15T04:50:36.99Z" }, - { url = "https://files.pythonhosted.org/packages/a1/d0/1f8578c45b2f24759b46f0b50d31878c63c73e6bf0f2227e10ec5c5408dc/tiktoken-0.13.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:95097e4f89b06403976e498abf61a0ee73a7497e73fb599cb211d8197a054d91", size = 1240069, upload-time = "2026-05-15T04:50:38.221Z" }, - { url = "https://files.pythonhosted.org/packages/aa/90/28d7f154888610aa9237e541986beb62b479df29d193a5a0617dbb1514d0/tiktoken-0.13.0-cp312-cp312-win_amd64.whl", hash = "sha256:8f2d16e7a7c783ad81f36e457d046d1f1c8af70b22aec8a13238efe531977c41", size = 874748, upload-time = "2026-05-15T04:50:39.587Z" }, - { url = "https://files.pythonhosted.org/packages/9c/83/b096c859c2a47c11731bf2f5885f4028b809dfe2396582883eed9cae372f/tiktoken-0.13.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:5df5d1507bd245f1ccad4a074698240021239e455eb0bb4ced4e3d7181872154", size = 1034228, upload-time = "2026-05-15T04:50:40.988Z" }, - { url = "https://files.pythonhosted.org/packages/53/61/c68e123b6d753e3fc2751e9b18e732c9d8bf1e1926762e736eee935d931c/tiktoken-0.13.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:8fe806a50664e83a6ffd56cbd1e4f5dcc6cd32a3e7538f70dc38b1a271384545", size = 982978, upload-time = "2026-05-15T04:50:42.195Z" }, - { url = "https://files.pythonhosted.org/packages/ef/8b/96cc178cc584e65d363134500f297790b06cd48cdeb1e8fcf7bbe60f4715/tiktoken-0.13.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:125bc05005e747f993a83dc67934249932d6e4209854452cd4c0b1d53fba3ba2", size = 1116355, upload-time = "2026-05-15T04:50:43.564Z" }, - { url = "https://files.pythonhosted.org/packages/86/f5/bab735d2c72ea55404b295d02d092644eb5f7cc6205e34d35eb9abfb9ab2/tiktoken-0.13.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:5e6358911cab4adee6712da27d65573496a4f68cf8a2b5fca6a4ad10fc5748cf", size = 1135772, upload-time = "2026-05-15T04:50:44.782Z" }, - { url = "https://files.pythonhosted.org/packages/4e/b9/6de04ebdf904edfaad87788011b3735087a0c9ea671b9027e1e4e965e8c8/tiktoken-0.13.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:975cbd78d085d75d26b59660e262736dcaed1e35f8f142cd6291025c01d25486", size = 1182415, upload-time = "2026-05-15T04:50:46.422Z" }, - { url = "https://files.pythonhosted.org/packages/0d/9c/470a05f3b1caf038f44880e334d47ab674e0c80d514c66b375d14d5afa10/tiktoken-0.13.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:75ab9bc99fa020a4c283424590ecd7f3afd70c1c281cb3fa3192a6c3af9f9615", size = 1239879, upload-time = "2026-05-15T04:50:48.052Z" }, - { url = "https://files.pythonhosted.org/packages/42/a6/c1936d16055436cb32e6c6128d68629622e00f4768562f55653752d34768/tiktoken-0.13.0-cp313-cp313-win_amd64.whl", hash = "sha256:6b1615f0ff71953d19729ceb18865429c185b0a23c5353f1bbca34a394bf60f7", size = 874829, upload-time = "2026-05-15T04:50:49.202Z" }, - { url = "https://files.pythonhosted.org/packages/d6/07/acb5992c3772b5a36284f742cfb7a5895aa4471d1848ac31464ad50d7fdf/tiktoken-0.13.0-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:6eb4a5bfbc6426938026b1a334e898ac53541360d62d8c689870160cc80abd67", size = 1033600, upload-time = "2026-05-15T04:50:50.4Z" }, - { url = "https://files.pythonhosted.org/packages/14/e9/742e9aec30f59b9f161f7ff7cd072e02ea836c9e1c0854a8076dfcd40d5c/tiktoken-0.13.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:43cee3e5400573b2046fbf092cc7a5bc30164f9e4c95ce20714da929df48737a", size = 982516, upload-time = "2026-05-15T04:50:52.03Z" }, - { url = "https://files.pythonhosted.org/packages/72/74/ca1541b053e7648254d2e4b42a253e1bb4359f2c91a0a8d49228c794e1a0/tiktoken-0.13.0-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:7de52e3f566d19b3b11bd37eea552c6c305ad74081f736882bd44d148ed4c48d", size = 1115518, upload-time = "2026-05-15T04:50:53.543Z" }, - { url = "https://files.pythonhosted.org/packages/46/e3/93825eaf5a4a504795b787e5d5dea07fbeb3dabf97aa7b450be8bde59c89/tiktoken-0.13.0-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:51384448aa508e4df84c0f7c1dc3211c7f7b8096325660ee5fc82f3e11b381ce", size = 1136867, upload-time = "2026-05-15T04:50:55.191Z" }, - { url = "https://files.pythonhosted.org/packages/8c/46/002b68de6827091d5ae90b048f326e8aad8d953520950e5ce1508879414f/tiktoken-0.13.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:e28157350f7ebf35008dd8e9e0fdb621f976e4230c881099c85e8cf07eaa50e2", size = 1181826, upload-time = "2026-05-15T04:50:56.296Z" }, - { url = "https://files.pythonhosted.org/packages/db/c6/d393e3185a276505182f7abd93fe714f3c444a2be9180798fa052347504e/tiktoken-0.13.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:165cf1820ea4a354985c2490a5205d4cc74661c934aca79dd0368232fff94e0f", size = 1239489, upload-time = "2026-05-15T04:50:57.918Z" }, - { url = "https://files.pythonhosted.org/packages/b7/4d/bc07d1f1635d4897a202acc0ae11c2886eaa7325c359ba4741b47bf8e225/tiktoken-0.13.0-cp313-cp313t-win_amd64.whl", hash = "sha256:6c43a675ca14f6f2749ba7f12075d37456015a24b859f2517b9beb4ef30807ec", size = 873820, upload-time = "2026-05-15T04:50:59.528Z" }, - { url = "https://files.pythonhosted.org/packages/8c/93/0dd6adca026a616c3a92974566b43381eea4b475ce1f36c062b8271a9ac5/tiktoken-0.13.0-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:eaaaef47c2406277181d2086484c317bf7fc433e2d5d03ff94f56b0dcec87471", size = 1034977, upload-time = "2026-05-15T04:51:00.957Z" }, - { url = "https://files.pythonhosted.org/packages/d9/77/5ec6e6bc5b30bed6d93f7f2162d8f6b32437b3ba27cb527cfe004f6109c9/tiktoken-0.13.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:ca8b310bd93b3772cb1b7922d915446864860f562bdfe4825c63a0aed3fb28cd", size = 983635, upload-time = "2026-05-15T04:51:02.629Z" }, - { url = "https://files.pythonhosted.org/packages/94/b0/c8ae9aff00d625c50659b4513e707a0462c4bf5d4d6cc1b802103225c02e/tiktoken-0.13.0-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:32e0c12305105002c047b3bb1070b0dd9a73b0cb3b2856a8972b810e7a4f5881", size = 1116036, upload-time = "2026-05-15T04:51:04.082Z" }, - { url = "https://files.pythonhosted.org/packages/1b/ac/6a5dddd1d0a6018ecb389bd0353e6b4a515eb4d2286611bd0ace1937b9e1/tiktoken-0.13.0-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:5ba5fd62507a932d1241346179e3b39bc7bf7408f03c272652d93b3bedf5db24", size = 1135544, upload-time = "2026-05-15T04:51:05.229Z" }, - { url = "https://files.pythonhosted.org/packages/f4/b8/585032b4384b2f7dcdaddcb52865c83a701a420d09e3c2b4a2be1c450c57/tiktoken-0.13.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:d108bc2d470fc53c8ecd24f2c0fd2b5f98c33e87cdb6aa2e9b8c5dced703d273", size = 1182217, upload-time = "2026-05-15T04:51:06.517Z" }, - { url = "https://files.pythonhosted.org/packages/cd/b6/993ff1ded3958215fd341a847b8e5ffeb5de473f435296870d314fc91ac4/tiktoken-0.13.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:cb99cb5127449f58d0a2d5f5ccfb390d8dbdfd919c221246caaee29d8725ed51", size = 1239404, upload-time = "2026-05-15T04:51:07.843Z" }, - { url = "https://files.pythonhosted.org/packages/dd/3d/fef7e06e3b33e7538db0ced734cf9fe23b6832d2ac4990c119c377aec55e/tiktoken-0.13.0-cp314-cp314-win_amd64.whl", hash = "sha256:115c4f26ffa11caac8b54eea35c2ad38c612c20a48d35dd15d70a02ac6f51f58", size = 918686, upload-time = "2026-05-15T04:51:08.925Z" }, - { url = "https://files.pythonhosted.org/packages/c1/82/a7fc44582bc32ab00de988a2299bf77c077f59068b233109e34b7d6ca7e6/tiktoken-0.13.0-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:472527e9132952f2fbf77cd290658bacf003d4d5a3fabc18e5fbd407cbae4d9b", size = 1034454, upload-time = "2026-05-15T04:51:10.035Z" }, - { url = "https://files.pythonhosted.org/packages/37/d0/24d8a890c14f432a05cea669c17bebeaa99f96a7c79523b590f564246411/tiktoken-0.13.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:4e2f67d27c9626cdd25fe33d9313c5cdb3d8d82da646b68d6eb8e7e9c20e6448", size = 982976, upload-time = "2026-05-15T04:51:11.23Z" }, - { url = "https://files.pythonhosted.org/packages/49/b7/2ab43f62788a9266187a9bfc1d3af99ad83e5eaa25fbef168a69cd5ad14f/tiktoken-0.13.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:2b920b35805cd64585a37c3dc7ce65fba4d2d36016be01e1d7942482ca29093a", size = 1115526, upload-time = "2026-05-15T04:51:12.608Z" }, - { url = "https://files.pythonhosted.org/packages/64/39/1494321ed323ce7a14d88e3cd6cb9058625977df1c6961ddc492bd10a9f3/tiktoken-0.13.0-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:493af3aa28a4aaf2e3d2600a2ee717252c9bf5ab38fff94eb5a02db5ab77e5ad", size = 1136466, upload-time = "2026-05-15T04:51:13.926Z" }, - { url = "https://files.pythonhosted.org/packages/96/d9/dfd086aa2d918c563a140720e0ce296cada1634efd2783d5cf51e05f984e/tiktoken-0.13.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:6644c9c2b5cf3916f5a3641d7d12fdb3f006a7b3d9ff6acdaec44e29ab1ff91e", size = 1181863, upload-time = "2026-05-15T04:51:15.025Z" }, - { url = "https://files.pythonhosted.org/packages/2f/68/a18b4f307086954fdae32714cb4f85562e34f9d34ab206e61f1816aa6018/tiktoken-0.13.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:5cb65b60b9408563676d874a3a4ee573370066f0dc4e29d84e82e989c6517424", size = 1239218, upload-time = "2026-05-15T04:51:16.103Z" }, - { url = "https://files.pythonhosted.org/packages/16/5b/f2aa703a4fc5d2dff73460a7d46cc2f3f44aa0f3dd8eeb20d2a0ecf68862/tiktoken-0.13.0-cp314-cp314t-win_amd64.whl", hash = "sha256:85b78cc3a2c3d48723ca751fa981f1fedccd54194ca0471b957364353a898b07", size = 918110, upload-time = "2026-05-15T04:51:17.237Z" }, -] - [[package]] name = "tokenizers" version = "0.22.2" @@ -3108,41 +2848,64 @@ wheels = [ name = "torch" version = "2.14.0" source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "sys_platform != 'linux'", +] dependencies = [ - { name = "cuda-bindings", marker = "python_full_version < '3.15' and sys_platform == 'linux'" }, - { name = "cuda-toolkit", extra = ["cublas", "cudart", "cufft", "cufile", "cupti", "curand", "cusolver", "cusparse", "nvjitlink", "nvrtc", "nvtx"], marker = "sys_platform == 'linux'" }, - { name = "filelock" }, - { name = "fsspec" }, - { name = "jinja2" }, - { name = "networkx" }, - { name = "nvidia-cudnn-cu13", marker = "sys_platform == 'linux'" }, - { name = "nvidia-cusparselt-cu13", marker = "sys_platform == 'linux'" }, - { name = "nvidia-nccl-cu13", marker = "sys_platform == 'linux'" }, - { name = "nvidia-nvshmem-cu13", marker = "sys_platform == 'linux'" }, - { name = "setuptools" }, - { name = "sympy" }, - { name = "triton", marker = "python_full_version < '3.15' and sys_platform == 'linux'" }, - { name = "typing-extensions" }, + { name = "filelock", marker = "sys_platform != 'linux'" }, + { name = "fsspec", marker = "sys_platform != 'linux'" }, + { name = "jinja2", marker = "sys_platform != 'linux'" }, + { name = "networkx", marker = "sys_platform != 'linux'" }, + { name = "setuptools", marker = "sys_platform != 'linux'" }, + { name = "sympy", marker = "sys_platform != 'linux'" }, + { name = "typing-extensions", marker = "sys_platform != 'linux'" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/17/76/bb4770f56cf6d8971671dbcbb7493e5a6a15ad2825f4e359b02c27c38297/torch-2.14.0-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:c1f844f1c750e87df4b68bc3afbc0e2b0c7ef19d7b8f666e48bdcf6a0c4f0056", size = 127303200, upload-time = "2026-09-02T13:43:20.311Z" }, - { url = "https://files.pythonhosted.org/packages/fe/7b/ec44bacf2c8886b85ba4ca2285e8b09f2dff5d9c99e6a031326954082cb5/torch-2.14.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:ada340e62591d06a2bcc2d68170f20f45f0b0665d372dc510a8ed7eb3b1d609a", size = 454010251, upload-time = "2026-09-02T13:44:24.927Z" }, - { url = "https://files.pythonhosted.org/packages/15/71/49399acd41f750a906c686bd23c08a2001ccb8dd25f2971003c2ed89c1dd/torch-2.14.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:fecffb58f51fd643d213acd68da21cc3fc19bea05a3bc64b4ee55128f47a4963", size = 554620488, upload-time = "2026-09-02T13:44:49.442Z" }, { url = "https://files.pythonhosted.org/packages/be/16/9489b137112040f9911d7527e452854f21cc4e499ce0da79864e6a7451a7/torch-2.14.0-cp312-cp312-win_amd64.whl", hash = "sha256:cad84f41bbdf3dcf333ce394aeeaf25237c4d94fd6b659ba5eb813c829978823", size = 124114011, upload-time = "2026-09-02T13:43:55.034Z" }, { url = "https://files.pythonhosted.org/packages/ca/40/0db773452c2a62b37761d3f418acf933d381f9e87077036fb57c2a386c37/torch-2.14.0-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:9d4b1022a5d9b71282ec67ad0d9e7235870096b8a246dc1c32d6ea1fc83dc998", size = 127311393, upload-time = "2026-09-02T13:43:59.607Z" }, - { url = "https://files.pythonhosted.org/packages/13/36/537fd9da2adad49e7b2bb20741398625bee548493158274e87369a8eed56/torch-2.14.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:731b9ebdea402b8b1996d4c2ae613b16660e559b19e47bdc45d970568bc91c53", size = 454010525, upload-time = "2026-09-02T13:45:03.719Z" }, - { url = "https://files.pythonhosted.org/packages/21/f1/39bd13b21f57d1982b7f3ddf663f01c7266e2957714880744eba9e8c8d11/torch-2.14.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:84bf384779a10c02fc3c6bdbab71a9cb66b0dd93c652d1ed5d6dfc0cb37e5962", size = 554619993, upload-time = "2026-09-02T13:45:28.209Z" }, { url = "https://files.pythonhosted.org/packages/89/a8/683d9c44737554b67ca76dd2db4f42258a0f014246cb511293e51e0154bd/torch-2.14.0-cp313-cp313-win_amd64.whl", hash = "sha256:0e7cf18cb0d8bd666b6120932e29c7aef3502b61a08b44da4839580c539a7cdb", size = 124113865, upload-time = "2026-09-02T13:44:33.705Z" }, { url = "https://files.pythonhosted.org/packages/ee/90/1241e7db5ccc2455f8735bd6b1becfad39916206ad18001c4c0014d139e2/torch-2.14.0-cp314-cp314-macosx_14_0_arm64.whl", hash = "sha256:860423e970f2ce02c4476e8e2d1350131b1c5c5a5e4912180e78b50b53241efa", size = 127321431, upload-time = "2026-09-02T13:44:38.236Z" }, - { url = "https://files.pythonhosted.org/packages/68/f0/bbeba65e3fbb4b8f61ea19cf4ebea9f4268cc6fe64e6f7d38bfb6cc152eb/torch-2.14.0-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:b985c7defeb8d28691b7513ebaecc64946c5f68ec770e9f4ddba39688864f46a", size = 454027631, upload-time = "2026-09-02T13:45:43.73Z" }, - { url = "https://files.pythonhosted.org/packages/50/75/8d2b9a7e724759470c209489b79260cac537f091cc9aac8001c8d2bc845c/torch-2.14.0-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:b2cd92bce63d40bf6fc2e5d840fc2f0063bd180a242daa761cb6088cc2f46e27", size = 554623549, upload-time = "2026-09-02T13:46:04.252Z" }, { url = "https://files.pythonhosted.org/packages/9a/1e/a5475c00b0555e686333e6b4036f2213e7cbea021a772ce6f9ced4dcbd2f/torch-2.14.0-cp314-cp314-win_amd64.whl", hash = "sha256:44b044b9f6f633d982839422a57433d6a1da520037fd88e0c8a47efde589b3b8", size = 124110863, upload-time = "2026-09-02T13:45:12.764Z" }, { url = "https://files.pythonhosted.org/packages/a5/49/bbaee76337742a42d2c5b0296ea62252567050e1d821bc2283b2e72ba6fb/torch-2.14.0-cp314-cp314t-macosx_14_0_arm64.whl", hash = "sha256:553aec938d37d77b783bcf801e638cad068870e7643c47eadac21eb180f551ed", size = 127653463, upload-time = "2026-09-02T13:45:17.1Z" }, - { url = "https://files.pythonhosted.org/packages/92/b5/6cc7a511bab384fbe8f4b8ddeecdd22e724b2162d6f15076f07a0a7ef15b/torch-2.14.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:cd8cd8f714d511ccdca907282d1da3d9be8322d4e1520b9c3bce39a5c1318a4b", size = 454009927, upload-time = "2026-09-02T13:46:19.637Z" }, - { url = "https://files.pythonhosted.org/packages/b6/da/0f04fe15fd05bf3f613a359b14acb44c35ae06fe69da0aefa8b5cdb4f9f9/torch-2.14.0-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:d2526f71e6638133b97b3cd2881ece3df521460a4727030e8ae2a72a7d3ae31d", size = 554580530, upload-time = "2026-09-02T13:46:34.818Z" }, { url = "https://files.pythonhosted.org/packages/f8/c3/72ae1f02747b1f012e1975743e48cd608f83095d7f9ce58de78b79248b35/torch-2.14.0-cp314-cp314t-win_amd64.whl", hash = "sha256:731784e3914843c6bcc7aba3987ff7610ac57dbbc816a5d6b9b62e04c240a641", size = 124400194, upload-time = "2026-09-02T13:45:53.555Z" }, ] +[[package]] +name = "torch" +version = "2.14.0+cpu" +source = { registry = "https://download.pytorch.org/whl/cpu" } +resolution-markers = [ + "sys_platform == 'linux'", +] +dependencies = [ + { name = "filelock", marker = "sys_platform == 'linux'" }, + { name = "fsspec", marker = "sys_platform == 'linux'" }, + { name = "jinja2", marker = "sys_platform == 'linux'" }, + { name = "networkx", marker = "sys_platform == 'linux'" }, + { name = "setuptools", marker = "sys_platform == 'linux'" }, + { name = "sympy", marker = "sys_platform == 'linux'" }, + { name = "typing-extensions", marker = "sys_platform == 'linux'" }, +] +wheels = [ + { url = "https://download-r2.pytorch.org/whl/cpu/torch-2.14.0%2Bcpu-cp312-cp312-linux_s390x.whl", hash = "sha256:07aefe1a437b6f1b41b0cba15e16583e2f43549ec9d062fcea8e7981a78e014a", upload-time = "2026-09-02T18:31:32Z" }, + { url = "https://download-r2.pytorch.org/whl/cpu/torch-2.14.0%2Bcpu-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:da8140c3d4a41500d29710e35bf8e66a4295ec31026487c69f54a3f583310f8d", upload-time = "2026-09-02T18:31:37Z" }, + { url = "https://download-r2.pytorch.org/whl/cpu/torch-2.14.0%2Bcpu-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:a09987c95ec4cffdb6df798d3d641558110a334cfccce22f2f046d83142bc260", upload-time = "2026-09-02T18:31:42Z" }, + { url = "https://download-r2.pytorch.org/whl/cpu/torch-2.14.0%2Bcpu-cp313-cp313-linux_s390x.whl", hash = "sha256:d87cfff3af33c937b88c9bc6c0dfa17f2156e2599f63447c696f2345928b518a", upload-time = "2026-09-02T18:31:55Z" }, + { url = "https://download-r2.pytorch.org/whl/cpu/torch-2.14.0%2Bcpu-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:092d5c12938850dfbd90a654b3c8dac34c33e300f88eb19ee6f4ef93992c6347", upload-time = "2026-09-02T18:31:59Z" }, + { url = "https://download-r2.pytorch.org/whl/cpu/torch-2.14.0%2Bcpu-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:160e1bc46aeded3111d2801f8ae10dc9a1b946843a7e126b4dbf5e19c5706e95", upload-time = "2026-09-02T18:32:05Z" }, + { url = "https://download-r2.pytorch.org/whl/cpu/torch-2.14.0%2Bcpu-cp314-cp314-linux_s390x.whl", hash = "sha256:266a253980047f3c037191f45d7e7ecfa838d13102141ca57b86fcc892663004", upload-time = "2026-09-02T18:32:17Z" }, + { url = "https://download-r2.pytorch.org/whl/cpu/torch-2.14.0%2Bcpu-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:27b34640fea7268862961b734c723873e15b1c2657b1eb84bcfac053e5841527", upload-time = "2026-09-02T18:32:23Z" }, + { url = "https://download-r2.pytorch.org/whl/cpu/torch-2.14.0%2Bcpu-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:f152f41dc5dc462afe0de780e451ebb47ea8b4451f8f919f9537aa8e2cbe1d7e", upload-time = "2026-09-02T18:32:29Z" }, + { url = "https://download-r2.pytorch.org/whl/cpu/torch-2.14.0%2Bcpu-cp314-cp314t-linux_s390x.whl", hash = "sha256:c1d81be03f715eca5b065fc767f73b2a26668a16d614e163a4378a3f38a5e471", upload-time = "2026-09-02T18:32:39Z" }, + { url = "https://download-r2.pytorch.org/whl/cpu/torch-2.14.0%2Bcpu-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:a6ff46c0c0c4f5b9b399dff5e4856a98ce18e61ca14ad74b21790174c7cbde08", upload-time = "2026-09-02T18:32:43Z" }, + { url = "https://download-r2.pytorch.org/whl/cpu/torch-2.14.0%2Bcpu-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:2baf9cbaef964d821465df0c53a91cc01b2d0ea29ebd199e2f9c7d27a7fc70a4", upload-time = "2026-09-02T18:32:48Z" }, + { url = "https://download-r2.pytorch.org/whl/cpu/torch-2.14.0%2Bcpu-cp315-cp315-manylinux_2_28_aarch64.whl", hash = "sha256:07d50e40232d2757be323693e38f531e04ee654d40537bc0b04e787c1e73a27a", upload-time = "2026-09-02T18:32:57Z" }, + { url = "https://download-r2.pytorch.org/whl/cpu/torch-2.14.0%2Bcpu-cp315-cp315-manylinux_2_28_x86_64.whl", hash = "sha256:a8b2fa9a03d81bc083f279737fce1cd8d1ed2e267da92215260cd644dcfdd3fe", upload-time = "2026-09-02T18:33:02Z" }, + { url = "https://download-r2.pytorch.org/whl/cpu/torch-2.14.0%2Bcpu-cp315-cp315t-manylinux_2_28_aarch64.whl", hash = "sha256:e22ddf8a9fff9934fe2dd2440d45c437bb0539f2d6d24b57b26981d6dca4157c", upload-time = "2026-09-02T18:33:11Z" }, + { url = "https://download-r2.pytorch.org/whl/cpu/torch-2.14.0%2Bcpu-cp315-cp315t-manylinux_2_28_x86_64.whl", hash = "sha256:18b417c953e9b4d8b79294ca5184586790f0a609b3929e39258cea854b945870", upload-time = "2026-09-02T18:33:16Z" }, +] + [[package]] name = "tornado" version = "6.5.8" @@ -3201,21 +2964,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/df/56/bbd60dd8668055803bf8ba55a81f9b8a8b31497f620109a9671d26a2076d/transformers-5.12.1-py3-none-any.whl", hash = "sha256:2a5e109d2021265df7098ffbb738295acaf5ad256f12cbc586db2ea4dcbb1a8a", size = 11150587, upload-time = "2026-06-15T17:27:46.679Z" }, ] -[[package]] -name = "triton" -version = "3.8.0" -source = { registry = "https://pypi.org/simple" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/87/07/0f8cd8e8db0472334253efdaaab3d0819fea27aa99bf0e7f1aeea4ceb5ae/triton-3.8.0-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a9c404c69ed4a39e8ec632eaf6b9fe058a060bf98979c177f6ef666f06bb8d50", size = 226474486, upload-time = "2026-08-28T16:08:18.29Z" }, - { url = "https://files.pythonhosted.org/packages/c1/09/b7012e5bfae67640f268aa584caa80fe1674f6b0da949046b679972c33e3/triton-3.8.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e91ffa46d095b252248297292dd22bcbacd53a125a0c2eefbbbf74925a320bc3", size = 247972921, upload-time = "2026-08-28T15:55:53.157Z" }, - { url = "https://files.pythonhosted.org/packages/87/4d/4c564374bcdadb166fccbf3e45aee0d4a473f88d341761bd2fefe3b8e8c1/triton-3.8.0-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b7004666652f500ed854a86988e4b3d69d247188b5d2092b5df1e44f4a954099", size = 226476793, upload-time = "2026-08-28T16:08:30.956Z" }, - { url = "https://files.pythonhosted.org/packages/b0/b6/3394d5548404c1cabd1dadadd28d0b3f9478db1dff8180da53bb3f0a1e19/triton-3.8.0-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1f0497218e26b7d79773ad9c2a3fa3b539ee69f587a13fac2e552b1d322a8015", size = 247975122, upload-time = "2026-08-28T15:56:04.112Z" }, - { url = "https://files.pythonhosted.org/packages/b8/59/bf0e9493118bb353ab59a5d6a65db3618d9b314417cc1459f0121e0ec5c9/triton-3.8.0-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1f6b48d0591929a3867973acac3dccd4e058585f91bfb41022de496c9ffab304", size = 226488654, upload-time = "2026-08-28T16:08:47.141Z" }, - { url = "https://files.pythonhosted.org/packages/93/d9/08c75f3459f19ad00425b564058e40efa4bcd79b816064cf27499303ea42/triton-3.8.0-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:387dae4cb0089a7b6ba1a428ae0782b65c4c58f57d94617cb22ca8593d8ccbca", size = 247972313, upload-time = "2026-08-28T15:56:14.007Z" }, - { url = "https://files.pythonhosted.org/packages/7c/34/429c5592181cfb7361a0a8e0bff218e7224b726709d75da2472b3e819f70/triton-3.8.0-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1b84e7d512490ba529111260fa6f7cad8b254a6bb5fbdf41d5ef9a5e57f52d0a", size = 226591133, upload-time = "2026-08-28T16:09:02.271Z" }, - { url = "https://files.pythonhosted.org/packages/fe/d1/aa8a3e935c37efee7945984fdb64d7e0851bf6d920afd97b2d21f9d23360/triton-3.8.0-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:74217bb56ed8692759227758e4c4b3bd2d608a209c1a7a081bf361fb4c2c1bf9", size = 248077577, upload-time = "2026-08-28T15:56:24.94Z" }, -] - [[package]] name = "truststore" version = "0.10.4" From 923761ceb3dc6d9a12eb679a1c20c9287ea53a97 Mon Sep 17 00:00:00 2001 From: BADRI NARAYANAN Date: Tue, 22 Sep 2026 13:42:01 -0700 Subject: [PATCH 10/12] Scan the image with a pinned Trivy, not its action The action's setup step runs an install script that exits 1 on the runner with nothing on stderr, so the scan never started and the job failed without saying why. Its published image is pinned instead, the same way gitleaks is installed by checksum rather than by an action. The image is scanned from a `docker save` tar with `--input`, so the scanner is not handed the docker socket. --- .github/workflows/ci.yml | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a5f1a8e..73298ab 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -114,14 +114,21 @@ jobs: found=$(docker run --rm --entrypoint sh "$IMAGE" -c 'ls -A /app/data 2>/dev/null | wc -l') test "$found" -eq 0 || { echo "data/ reached the image"; exit 1; } + # Trivy's own action installs its binary through a script that fails + # silently on the runner; its published image is pinned instead, the + # same way gitleaks is. `--input` on a saved tar means no docker socket + # has to be handed to the scanner. - name: Scan the image - uses: aquasecurity/trivy-action@dc5a429b52fcf669ce959baa2c2dd26090d2a6c4 # v0.32.0 - with: - image-ref: policypal-api:${{ github.sha }} - format: table - exit-code: "1" - ignore-unfixed: true - severity: HIGH,CRITICAL + env: + IMAGE: policypal-api:${{ github.sha }} + TRIVY_VERSION: "0.64.1" + run: | + set -euo pipefail + docker save "$IMAGE" -o "$RUNNER_TEMP/image.tar" + docker run --rm -v "$RUNNER_TEMP:/scan" "aquasec/trivy:${TRIVY_VERSION}" image \ + --input /scan/image.tar \ + --format table --exit-code 1 --ignore-unfixed --severity HIGH,CRITICAL + rm -f "$RUNNER_TEMP/image.tar" secret-scan: runs-on: ubuntu-latest From 3619745c40b87b5cd4b8c743381d2ba16848d47a Mon Sep 17 00:00:00 2001 From: BADRI NARAYANAN Date: Tue, 22 Sep 2026 13:59:20 -0700 Subject: [PATCH 11/12] Pin the base, uv and Trivy images by digest A tag can be moved to different bytes, so pinning by tag is not pinning. The repository already SHA-pins its GitHub Actions and installs gitleaks by checksum; images were the gap. python:3.12-slim-bookworm, ghcr.io/astral-sh/uv and aquasec/trivy now carry name:tag@sha256, the tag kept beside the digest so the version is still readable. --- .github/workflows/ci.yml | 5 +++-- Dockerfile | 9 ++++++--- docs/decisions/0022-the-api-as-a-container.md | 9 +++++++++ 3 files changed, 18 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 73298ab..3557345 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -121,11 +121,12 @@ jobs: - name: Scan the image env: IMAGE: policypal-api:${{ github.sha }} - TRIVY_VERSION: "0.64.1" + # Pinned by digest: a tag can be moved to different bytes. + TRIVY: "aquasec/trivy:0.64.1@sha256:a8ca29078522f30393bdb34225e4c0994d38f37083be81a42da3a2a7e1488e9e" run: | set -euo pipefail docker save "$IMAGE" -o "$RUNNER_TEMP/image.tar" - docker run --rm -v "$RUNNER_TEMP:/scan" "aquasec/trivy:${TRIVY_VERSION}" image \ + docker run --rm -v "$RUNNER_TEMP:/scan" "$TRIVY" image \ --input /scan/image.tar \ --format table --exit-code 1 --ignore-unfixed --severity HIGH,CRITICAL rm -f "$RUNNER_TEMP/image.tar" diff --git a/Dockerfile b/Dockerfile index 1dcc6e0..c21f0e2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,8 +1,11 @@ # The API only. Ingestion stays on the machine that holds data/ (ADR 0016, # 0019); nothing in this image fetches or keeps a PDF. -FROM python:3.12-slim-bookworm AS builder +FROM python:3.12-slim-bookworm@sha256:392307d22300de8b5986851a12d9176dfc0fc073e65bf6523ebd7dcbeb23564e AS builder -COPY --from=ghcr.io/astral-sh/uv:0.11.23 /uv /bin/uv +# Every base and tool image is pinned by digest, not by tag: a tag can be +# moved to different bytes, and the rest of this repo pins actions by SHA +# and gitleaks by checksum for the same reason. +COPY --from=ghcr.io/astral-sh/uv:0.11.23@sha256:d0a0a753ab981624b49c97abc98821c1c09f4ca69d1ef5cee69c501be3d88479 /uv /bin/uv ENV UV_COMPILE_BYTECODE=1 \ UV_LINK_MODE=copy \ @@ -38,7 +41,7 @@ SentenceTransformer(settings.embedding_model, revision=settings.embedding_model_ CrossEncoder(settings.reranker_model, revision=settings.reranker_model_revision)" -FROM python:3.12-slim-bookworm AS runtime +FROM python:3.12-slim-bookworm@sha256:392307d22300de8b5986851a12d9176dfc0fc073e65bf6523ebd7dcbeb23564e AS runtime # Nothing here runs as root, and nothing writes to the image. RUN useradd --create-home --uid 10001 policypal diff --git a/docs/decisions/0022-the-api-as-a-container.md b/docs/decisions/0022-the-api-as-a-container.md index b4ac023..c46638a 100644 --- a/docs/decisions/0022-the-api-as-a-container.md +++ b/docs/decisions/0022-the-api-as-a-container.md @@ -75,6 +75,15 @@ also writes JSON to stdout, where the platform collects it; in development it does not, so an ingestion run's log lines do not interleave with its own progress output. +### Every image it depends on is pinned by digest + +The base image, the `uv` image the build copies from, and the Trivy image that +scans the result are all pinned `name:tag@sha256:…`. A tag can be moved to +different bytes; a digest cannot. This is the same standard the repository +already applies to GitHub Actions (SHA-pinned) and to gitleaks (installed by +checksum), and the reason is the same: a build that is reproducible only until +someone else re-tags is not reproducible. + ### It does not run as root, and does not write to itself A `policypal` user owns the virtualenv and the source. `LOG_DIR` points at From ac841d80fab4380be3d3fa8544dc51948b4d9cfd Mon Sep 17 00:00:00 2001 From: BADRI NARAYANAN Date: Tue, 22 Sep 2026 13:40:05 -0700 Subject: [PATCH 12/12] Deploy to Azure, behind a green CI run and a human MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes the second of ADR 0002's two deferred items, which waited on a subscription that now exists. Container Apps for the API, Static Web Apps for the browser app, and a PostgreSQL Flexible Server. The choices that are not obvious: Migrations run as a Container Apps job on the same image digest that is about to serve traffic. Running Alembic from the GitHub runner instead would mean opening the database to GitHub's IP ranges for the sake of a deploy step; this keeps the server private and guarantees the schema is migrated by exactly the code that will then use it. The image is tagged with the commit and deployed by digest. A tag can be moved, a digest cannot, so what the reviewer approves is what runs. The trigger is `workflow_run` on a successful CI run, which makes green CI a precondition rather than a convention, and `environment: production` with a required reviewer is the approval. A post-deploy check polls the API until it answers 200, so a revision that starts but cannot serve fails the run. Sign-in is OIDC federated credentials, so no Azure secret is stored in GitHub at all. The commit a manual run asks for is checked against a 40-hex pattern before it reaches `ref:` or an image tag. One replica, deliberately: Flask-Limiter keeps its counters in process, so a second would double the 5/min on register and 10/min on login. Raising it and adding a shared store are the same piece of work. The database is seeded once by hand from a 75 MB dump, not by CI. Ingestion stays on the machine that holds data/ (ADR 0016, 0019), and nothing here uploads a PDF. docs/runbooks/deploy.md covers the one-time setup, the deploy path, rollback, and the failures worth expecting. What is still missing — staging, alerting, a shared rate-limit store — is named rather than left to be discovered. --- .github/workflows/cd.yml | 171 ++++++++++++++++++++ CHANGELOG.md | 9 ++ README.md | 19 ++- docs/decisions/0002-ci-scope.md | 29 ++-- docs/decisions/0023-deploying-to-azure.md | 111 +++++++++++++ docs/runbooks/deploy.md | 189 ++++++++++++++++++++++ 6 files changed, 512 insertions(+), 16 deletions(-) create mode 100644 .github/workflows/cd.yml create mode 100644 docs/decisions/0023-deploying-to-azure.md create mode 100644 docs/runbooks/deploy.md diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml new file mode 100644 index 0000000..2b6ee06 --- /dev/null +++ b/.github/workflows/cd.yml @@ -0,0 +1,171 @@ +name: CD + +# Only after CI has passed on main. `workflow_run` is what makes "green CI" +# a precondition rather than a hope: a push that fails CI never reaches here. +on: + workflow_run: + workflows: [CI] + types: [completed] + branches: [main] + workflow_dispatch: + inputs: + sha: + description: The commit to deploy. Defaults to the head of main. + required: false + +# Federated credentials, so no Azure secret is stored in GitHub (ADR 0023). +permissions: + contents: read + id-token: write + +env: + IMAGE_NAME: policypal-api + +jobs: + build: + # A failed CI run must not deploy. workflow_dispatch has no parent run. + if: >- + github.event_name == 'workflow_dispatch' || + github.event.workflow_run.conclusion == 'success' + runs-on: ubuntu-latest + outputs: + digest: ${{ steps.push.outputs.digest }} + sha: ${{ steps.commit.outputs.sha }} + steps: + # `inputs.sha` is typed by a human and ends up in `ref:` and in an image + # tag, so it is checked to be a full commit id and nothing else. + - id: commit + env: + DISPATCH_SHA: ${{ inputs.sha }} + RUN_SHA: ${{ github.event.workflow_run.head_sha }} + run: | + set -euo pipefail + sha="${DISPATCH_SHA:-${RUN_SHA:-$GITHUB_SHA}}" + if ! printf '%s' "$sha" | grep -Eq '^[0-9a-f]{40}$'; then + echo "not a commit id: refusing to deploy"; exit 1 + fi + echo "sha=$sha" >> "$GITHUB_OUTPUT" + + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ steps.commit.outputs.sha }} + + - uses: azure/login@a65d910e8af852a8061c627c456678983e180302 # v2.2.0 + with: + client-id: ${{ secrets.AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.AZURE_TENANT_ID }} + subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} + + - name: Sign in to the registry + env: + ACR_NAME: ${{ vars.ACR_NAME }} + run: az acr login --name "$ACR_NAME" + + - uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1 + + # Tagged with the commit, and deployed by digest: a tag can be moved, + # a digest cannot, so what is approved is exactly what runs. + - id: push + uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0 + with: + context: . + push: true + tags: ${{ vars.ACR_NAME }}.azurecr.io/policypal-api:${{ steps.commit.outputs.sha }} + cache-from: type=gha + cache-to: type=gha,mode=max + + deploy: + needs: build + runs-on: ubuntu-latest + # Production is a GitHub Environment with a required reviewer: this job + # waits for a human before anything reaches Azure. + environment: + name: production + url: ${{ vars.API_URL }} + steps: + - uses: azure/login@a65d910e8af852a8061c627c456678983e180302 # v2.2.0 + with: + client-id: ${{ secrets.AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.AZURE_TENANT_ID }} + subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} + + # Migrations run inside Azure, as a Container Apps job on the image being + # deployed, so the database needs no public firewall opening for a + # GitHub runner and the schema is migrated by exactly the code that will + # then serve it. + - name: Migrate the database + env: + RG: ${{ vars.AZURE_RESOURCE_GROUP }} + JOB: ${{ vars.MIGRATION_JOB_NAME }} + IMAGE: ${{ vars.ACR_NAME }}.azurecr.io/policypal-api@${{ needs.build.outputs.digest }} + run: | + set -euo pipefail + az containerapp job update --name "$JOB" --resource-group "$RG" --image "$IMAGE" + execution=$(az containerapp job start --name "$JOB" --resource-group "$RG" \ + --query name -o tsv) + echo "Waiting on $execution" + for _ in $(seq 1 60); do + status=$(az containerapp job execution show --name "$JOB" --resource-group "$RG" \ + --job-execution-name "$execution" --query properties.status -o tsv) + case "$status" in + Succeeded) echo "migrations applied"; exit 0 ;; + Failed|Degraded) echo "migration job $status"; exit 1 ;; + esac + sleep 10 + done + echo "migration job did not finish in 10 minutes"; exit 1 + + - name: Deploy the API + env: + RG: ${{ vars.AZURE_RESOURCE_GROUP }} + APP: ${{ vars.CONTAINERAPP_NAME }} + IMAGE: ${{ vars.ACR_NAME }}.azurecr.io/policypal-api@${{ needs.build.outputs.digest }} + run: | + set -euo pipefail + az containerapp update --name "$APP" --resource-group "$RG" --image "$IMAGE" + az containerapp ingress show --name "$APP" --resource-group "$RG" -o none + + - name: Check it answers + env: + API_URL: ${{ vars.API_URL }} + run: | + set -euo pipefail + for _ in $(seq 1 30); do + code=$(curl -s -o /dev/null -w '%{http_code}' "$API_URL/api/counties?zip=33101" || true) + if [ "$code" = "200" ]; then echo "API answering"; exit 0; fi + sleep 10 + done + echo "API did not answer 200 within five minutes"; exit 1 + + frontend: + needs: [build, deploy] + runs-on: ubuntu-latest + environment: + name: production + url: ${{ vars.FRONTEND_URL }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ needs.build.outputs.sha }} + + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: "24" + cache: npm + cache-dependency-path: frontend/package-lock.json + + - name: Build the browser app + working-directory: frontend + env: + # Baked in at build time: Vite has no runtime configuration. + VITE_API_BASE_URL: ${{ vars.API_URL }} + run: | + npm ci + npm run build + + - uses: Azure/static-web-apps-deploy@1a947af9992250f3bc2e68ad0754c0b0c11566c9 # v1.0.0 + with: + azure_static_web_apps_api_token: ${{ secrets.AZURE_STATIC_WEB_APPS_API_TOKEN }} + action: upload + app_location: frontend/dist + skip_app_build: true diff --git a/CHANGELOG.md b/CHANGELOG.md index 7fafe6a..ad001be 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,15 @@ `/app/data` is not empty: the issuer PDFs are local-only (ADR 0016) and a `.dockerignore` regression would otherwise ship gigabytes of them unnoticed. Trivy fails the build on fixable HIGH or CRITICAL findings. +- CD to Azure (ADR 0023), closing the second of ADR 0002's deferred items: + `.github/workflows/cd.yml` builds and pushes the API image to ACR, runs + `alembic upgrade head` as a Container Apps job on that exact digest, updates + the Container App, checks it answers, and deploys the browser app to Static + Web Apps. It runs only after a green CI run on `main`, waits on a required + reviewer, deploys by digest rather than tag, and signs in with OIDC so no + Azure credential is stored in GitHub. Rollback is reactivating the previous + revision. `docs/runbooks/deploy.md` has the one-time setup and what to do + when a deploy goes wrong. - `.env.example`, which did not exist: the three settings with no default (`DATABASE_URL`, `OPENAI_API_KEY`, `JWT_SECRET_KEY`) and the handful worth setting, with no values in it. diff --git a/README.md b/README.md index 6136d63..8b05a5d 100644 --- a/README.md +++ b/README.md @@ -352,8 +352,23 @@ npm run build CI (`.github/workflows/ci.yml`) also runs `ruff`, `alembic check` (schema drift), `bandit` (SAST), `pip-audit`/`npm audit` (dependency CVEs), a -gitleaks secret scan, and a backend coverage floor (`--cov-fail-under=85`) -on every PR and push to `main`. +gitleaks secret scan, a backend coverage floor (`--cov-fail-under=85`), and a +Docker build scanned by Trivy — which also runs the image and fails if any +issuer PDF reached it. Every pull request, and every push to `main`. + +### Deploying + +`.github/workflows/cd.yml` deploys to Azure after a green CI run on `main`: +the API to Container Apps, the browser app to Static Web Apps, against a +PostgreSQL Flexible Server. Migrations run as a Container Apps job on the +exact image digest being deployed, so the database needs no public opening. +It waits on a required reviewer, and rollback is reactivating the previous +revision. Sign-in is OIDC, so no Azure credential is stored in GitHub. + +Setup, the deploy path, and what to do when one goes wrong are in +[`docs/runbooks/deploy.md`](docs/runbooks/deploy.md); the reasoning is in +[ADR 0023](docs/decisions/0023-deploying-to-azure.md). Ingestion stays on the +machine that holds `data/` — the issuer PDFs are never uploaded (ADR 0016). ### Retrieval quality eval diff --git a/docs/decisions/0002-ci-scope.md b/docs/decisions/0002-ci-scope.md index 8572e1e..65998bd 100644 --- a/docs/decisions/0002-ci-scope.md +++ b/docs/decisions/0002-ci-scope.md @@ -2,7 +2,9 @@ ## Status -Accepted +Accepted. All three deferrals below have since been closed: coverage +thresholds in Phase 3, the Docker build and image scan by ADR 0022, and CD to +Azure by ADR 0023. ## Context @@ -26,19 +28,18 @@ and shouldn't fabricate. `npm audit --audit-level=high`. - **secret-scan**: `gitleaks detect` over full git history. -**Deliberately not built yet:** - -- **Docker build + image vulnerability scan.** There is no Dockerfile for - the app. Writing a production one for a Flask service that pulls in - PyTorch + transformers (a multi-GB dependency tree) is its own real task — - base image choice, layer caching, non-root user, multi-stage build to - keep the final image lean — not something to bolt on to get a CI checkbox - green. -- **CD to Azure (ACR push, migrations, deploy).** Requires an Azure - subscription, service principal, and GitHub Environment secrets that - don't exist in this project yet. Configuring cloud deployment - infrastructure isn't something to set up unprompted or with placeholder - credentials. +**Deferred when this was written; all three have since been built:** + +- ~~**Docker build + image vulnerability scan.**~~ **Resolved by ADR 0022.** + It was its own real task, as expected: CPU-only torch, both models baked in + at their pinned revisions, a non-root multi-stage build, and a CI step that + runs the image to prove no issuer PDF reached it. 2.48 GB, scanned by Trivy + on every pull request. +- ~~**CD to Azure (ACR push, migrations, deploy).**~~ **Resolved by ADR 0023**, + once a subscription existed. Container Apps, Static Web Apps and a Flexible + Server, deployed by digest behind a required reviewer, with migrations run + as a job inside the environment so the database needs no public opening. + Sign-in is OIDC, so no Azure credential is stored here after all. - ~~**Coverage thresholds.**~~ Resolved: `pytest --cov=src --cov-fail-under=85` is now the backend test step. It is a regression floor set just under the measured baseline, not an arbitrary target — infrastructure glue wrapping diff --git a/docs/decisions/0023-deploying-to-azure.md b/docs/decisions/0023-deploying-to-azure.md new file mode 100644 index 0000000..fb52481 --- /dev/null +++ b/docs/decisions/0023-deploying-to-azure.md @@ -0,0 +1,111 @@ +# 0023 — Deploying to Azure: Container Apps, Static Web Apps, Flexible Server + +## Status + +Accepted. Closes the second of ADR 0002's two deferred items ("CD to Azure +(ACR push, migrations, deploy)"). Depends on ADR 0022 for the image, and on +ADR 0021 without which a deployed container cannot answer. + +## Context + +ADR 0002 deferred CD because it "requires an Azure subscription, service +principal, and GitHub Environment secrets that don't exist in this project +yet". A subscription now exists. + +One constraint shapes everything: **the issuer PDFs stay on the machine that +downloaded them** (ADR 0016), and ADR 0019 already records that "nothing here +can run in CI or in the cloud". Ingestion, refresh and the reports are local +operations. What deploys is the API and the browser app; what travels is the +text already extracted, not the documents it came from. + +## Decision + +### Container Apps for the API + +- **Scale-to-zero** suits a project with no steady traffic, and the revision + model gives rollback for free. +- **Revisions are the rollback.** Reactivating the previous revision is one + command, which is what ADR 0002's "keep rollback a single re-run" asks for. +- **One replica, for now.** ADR 0022 explains why: Flask-Limiter has no shared + storage, so a second replica would double every rate limit. Raising the + replica count and adding a Redis backend for the limiter are the same piece + of work, and neither happens by accident. + +App Service was the alternative. It costs more at idle for the same thing. +AKS is far more operational surface than one Flask service justifies. + +### Static Web Apps for the browser app + +A Vite build is static files. Static Web Apps serves them on a CDN with SPA +fallback routing, on a free tier. The API stays a separate origin, which is +what `frontend_origin` and the single-origin CORS rule already assume. + +`VITE_API_BASE_URL` is baked in at build time, because Vite has no runtime +configuration. The deployed frontend is therefore tied to one API origin, and +changing it is a rebuild. + +### The database is seeded once, by hand, and never by CI + +Azure Database for PostgreSQL Flexible Server with `pgvector`. + +- **`pg_dump` → `pg_restore`, once.** 75 MB: the corpus, its 1,568 chunks, the + 31,359 SBC chunks, the plan catalog, and the BM25 index row (ADR 0021). +- **Not a migration and not a workflow step.** Seeding is an operator action + with the local database in front of them, because that is where the data + lives and where ingestion will keep running. +- **The PDFs are not uploaded.** Nothing in this decision moves + `data/sbc/` anywhere. + +### Migrations run in Azure, on the image being deployed + +`alembic upgrade head` runs as a **Container Apps job** using the same image +and digest that is about to serve traffic. + +The alternative — running Alembic from the GitHub runner — needs the database +to accept connections from GitHub's IP ranges. That is a public opening on the +database for the sake of a deploy step. Running it inside the environment +keeps the server private, and guarantees the schema is migrated by exactly the +code that will then use it. + +### Deployed by digest, gated by a human + +- **The image is pushed with the commit as its tag and deployed by digest.** + A tag can be moved; a digest cannot. What a reviewer approves is exactly + what runs. +- **`workflow_run` on a successful CI run is the trigger.** A push that fails + CI never reaches the deploy workflow — "green CI" is a precondition, not a + convention. +- **`environment: production` with a required reviewer** is the manual + approval. Nothing reaches Azure until a person says so. +- **A post-deploy check** requests `/api/counties?zip=` until it returns 200, + so a revision that starts but cannot serve fails the run rather than sitting + there. + +### No Azure credential is stored in GitHub + +`azure/login` with **OIDC federated credentials**: the workflow presents a +short-lived GitHub token, and Azure trusts it for this repository and +environment. There is no client secret to rotate or to leak. `id-token: +write` is granted at the workflow level and `contents: read` everywhere else. + +`workflow_dispatch` takes a commit to deploy, which a human types, so it is +checked against `^[0-9a-f]{40}$` before it reaches `ref:` or an image tag. + +## Consequences + +- **Cold starts are slow**, because scale-to-zero means loading two models + from the image on the first request after idling. The trade is cost; the + fix, if it stops being acceptable, is a minimum of one replica. +- **A deploy needs a human.** Deliberate. This application answers questions + about people's health coverage, and there is no staging environment to catch + a bad revision first. +- **There is no staging environment.** One environment, one reviewer. Adding + staging is worth doing before the first user who is not the author. +- **The deployed data is a snapshot.** Ingestion runs locally, so the cloud + database is as current as its last restore. The runbook says how to refresh + it, and `checked_at` in `sbc_documents` says how old it is. +- **Costs are real but small**: Container Apps scaled to zero, a free Static + Web App, and a Flexible Server that cannot scale to zero and is therefore + the monthly floor. +- **Still not done:** a shared rate-limit store, a staging environment, and + alerting. Named here rather than discovered later. diff --git a/docs/runbooks/deploy.md b/docs/runbooks/deploy.md new file mode 100644 index 0000000..0e50959 --- /dev/null +++ b/docs/runbooks/deploy.md @@ -0,0 +1,189 @@ +# Deploying PolicyPal to Azure + +What to create once, what runs on every merge, and what to do when a deploy +goes wrong. The decision behind it is [ADR 0023](../decisions/0023-deploying-to-azure.md). + +**The issuer PDFs never leave this machine** (ADR 0016). Nothing below uploads +`data/sbc/`, and ingestion stays local. + +## What gets deployed + +| Piece | Where | Built by | +| --- | --- | --- | +| API | Container Apps, one replica | `Dockerfile` (ADR 0022) | +| Browser app | Static Web Apps | `npm run build` in CI | +| Database | PostgreSQL Flexible Server + `pgvector` | seeded once, by hand | +| Migrations | Container Apps job, same image | `alembic upgrade head` | + +## One-time setup + +Names below are placeholders; use your own and set them as repository +variables. + +### 1. Resource group, registry, database + +```sh +az group create --name policypal-rg --location eastus + +az acr create --resource-group policypal-rg --name policypalacr --sku Basic + +az postgres flexible-server create \ + --resource-group policypal-rg --name policypal-db \ + --tier Burstable --sku-name Standard_B1ms --version 16 \ + --storage-size 32 --public-access None +az postgres flexible-server parameter set \ + --resource-group policypal-rg --server-name policypal-db \ + --name azure.extensions --value VECTOR +``` + +`--public-access None` keeps the server off the internet. Migrations run +inside the environment (ADR 0023), so nothing needs to reach it from outside. + +### 2. Container Apps environment, the app, and the migration job + +```sh +az containerapp env create --resource-group policypal-rg \ + --name policypal-env --location eastus + +az containerapp create --resource-group policypal-rg --name policypal-api \ + --environment policypal-env --target-port 8000 --ingress external \ + --min-replicas 0 --max-replicas 1 \ + --registry-server policypalacr.azurecr.io --system-assigned \ + --image policypalacr.azurecr.io/policypal-api:bootstrap + +az containerapp job create --resource-group policypal-rg \ + --name policypal-migrate --environment policypal-env \ + --trigger-type Manual --replica-timeout 600 \ + --registry-server policypalacr.azurecr.io --system-assigned \ + --image policypalacr.azurecr.io/policypal-api:bootstrap \ + --command "/app/.venv/bin/alembic" --args "upgrade" "head" +``` + +**`--max-replicas 1` is load-bearing.** Flask-Limiter keeps its counters in +process, so a second replica doubles every rate limit — including the 5/min on +register and 10/min on login (ADR 0022). Raise it only together with a shared +store for the limiter. + +### 3. Settings and secrets on the app + +```sh +az containerapp secret set --resource-group policypal-rg --name policypal-api \ + --secrets database-url="postgresql+psycopg://..." \ + openai-key="sk-..." jwt-secret="$(python -c 'import secrets;print(secrets.token_urlsafe(48))')" + +az containerapp update --resource-group policypal-rg --name policypal-api \ + --set-env-vars ENVIRONMENT=production LOG_DIR=/tmp/logs \ + FRONTEND_ORIGIN="https://.azurestaticapps.net" \ + DATABASE_URL=secretref:database-url \ + OPENAI_API_KEY=secretref:openai-key \ + JWT_SECRET_KEY=secretref:jwt-secret +``` + +The migration job needs `DATABASE_URL` and the two settings with no default +(`OPENAI_API_KEY`, `JWT_SECRET_KEY`) as well — `Settings` will not construct +without them, even though a migration uses neither. + +`.env.example` lists every setting. Never put one in the repo. + +### 4. Sign-in without a stored credential + +Create an app registration, give it `AcrPush` on the registry and +`Contributor` on the resource group, then add **federated credentials** for +this repository — one for `environment:production`, one for `ref:refs/heads/main`. +No client secret is created (ADR 0023). + +### 5. GitHub Environment and variables + +Create an Environment named `production` with **yourself as a required +reviewer** — that is the manual approval gate. + +| Secret | | +| --- | --- | +| `AZURE_CLIENT_ID`, `AZURE_TENANT_ID`, `AZURE_SUBSCRIPTION_ID` | from the app registration | +| `AZURE_STATIC_WEB_APPS_API_TOKEN` | from the Static Web App | + +| Variable | Example | +| --- | --- | +| `ACR_NAME` | `policypalacr` | +| `AZURE_RESOURCE_GROUP` | `policypal-rg` | +| `CONTAINERAPP_NAME` | `policypal-api` | +| `MIGRATION_JOB_NAME` | `policypal-migrate` | +| `API_URL` | `https://policypal-api..azurecontainerapps.io` | +| `FRONTEND_URL` | `https://.azurestaticapps.net` | + +### 6. Seed the database, once + +The schema comes from migrations; the contents come from here, because +ingestion runs here (ADR 0019). + +```sh +pg_dump --no-owner --no-privileges --format=custom \ + -h localhost -U policypal policypal > policypal.dump # ~75 MB + +pg_restore --no-owner --no-privileges --clean --if-exists \ + -d "postgresql://...@policypal-db.postgres.database.azure.com/policypal" policypal.dump +``` + +This carries the corpus, its chunks, the SBC text, the plan catalog and the +BM25 index row. It does **not** carry the PDFs, and must not. + +Afterwards, check the index arrived: + +```sql +select name, chunks, built_at from search_indexes; -- bm25 | 1568 | ... +``` + +If it is missing, run `make build-index` against the Azure database rather +than re-ingesting: the index is derived from the chunks (ADR 0021). + +## Every merge to main + +1. CI runs. A failed run never reaches CD. +2. **build** — the image is pushed to ACR, tagged with the commit. +3. **deploy** — waits for your approval, then runs migrations as a job on that + exact digest, updates the Container App, and polls + `/api/counties?zip=33101` until it returns 200. +4. **frontend** — builds with `VITE_API_BASE_URL` and uploads to Static Web Apps. + +Deploying a specific commit by hand: **Actions → CD → Run workflow**, with a +full 40-character commit id. + +## When it goes wrong + +**A bad revision is serving.** Roll back to the previous one: + +```sh +az containerapp revision list --resource-group policypal-rg --name policypal-api \ + --query "[].{name:name,active:properties.active,created:properties.createdTime}" -o table +az containerapp revision activate --resource-group policypal-rg \ + --name policypal-api --revision +``` + +**Migrations failed.** The deploy stops before the app is updated, so the old +revision keeps serving — against a database that may be half migrated. Read +the job's logs, fix forward if you can: + +```sh +az containerapp job execution list --resource-group policypal-rg \ + --name policypal-migrate -o table +``` + +Alembic downgrades exist but are a last resort: reverting a migration that has +already dropped something does not bring it back. + +**The API deployed but answers 500.** Almost always the database: either the +restore has not happened, or `search_indexes` is empty. Retrieval says which — +`MissingSearchIndexError` names `make build-index`. + +**Cold start feels broken.** The first request after idle loads two models from +the image. It is slow, not stuck. If that stops being acceptable, set +`--min-replicas 1` and pay for it. + +## What this does not do + +- **No staging environment.** One environment, one reviewer. +- **No alerting.** Failures are seen in the Actions run, not pushed anywhere. +- **No shared rate-limit store**, which is what pins the app at one replica. +- **No scheduled ingestion.** The deployed data is as current as the last + restore; `checked_at` in `sbc_documents` says how old that is, and + [sbc.md](sbc.md) is how it is refreshed — locally.