From b5eb6bde7ca55770dd21a8056286cd28e4c6c1b4 Mon Sep 17 00:00:00 2001 From: Laith Al-Saadoon Date: Mon, 12 Oct 2026 18:13:15 +0000 Subject: [PATCH 1/3] fix(site): name Codex CLI in the description every page and llms.txt carry The meta description of every synced page, the llms.txt summary and the JSON-LD site node said the site covers Claude Code trajectories only, while the converter also reads Codex CLI rollouts. The description now lives once in src/lib/repo.ts, in the words site/authored/index.md already uses, and astro.config.ts and Head.astro import it. --- site/astro.config.ts | 12 +++++++----- site/src/components/Head.astro | 5 ++--- site/src/lib/repo.ts | 9 +++++++++ 3 files changed, 18 insertions(+), 8 deletions(-) diff --git a/site/astro.config.ts b/site/astro.config.ts index 7f14ffd..f58f322 100644 --- a/site/astro.config.ts +++ b/site/astro.config.ts @@ -19,7 +19,13 @@ import { citationLinks } from "./src/lib/citation-links.js" import { COLLECTION_ROOT, sidebar, syncedTreePaths } from "./src/lib/content-tree.js" import { type MarkdownLinksOptions, markdownLinks, rawTwinLinks } from "./src/lib/markdown-links.js" import mermaid, { beautifulMermaid } from "./src/lib/mermaid-integration.js" -import { PERMALINK_COMMIT, REPO_URL, SITE_BASE, SITE_ORIGIN } from "./src/lib/repo.js" +import { + PERMALINK_COMMIT, + REPO_URL, + SITE_BASE, + SITE_DESCRIPTION, + SITE_ORIGIN +} from "./src/lib/repo.js" const projectRoot = fileURLToPath(new URL(".", import.meta.url)) /** The git repository the docs tree documents. `site/` is one directory inside it. */ @@ -45,10 +51,6 @@ const markdownLinkOptions: MarkdownLinksOptions = { siteBase: routePrefix } -const SITE_DESCRIPTION = - "ATIF-native analytics over Claude Code agent trajectories: sessions converted to ATIF, " + - "materialized as a corpus, queried through DuckDB views." - export default defineConfig({ site: SITE_ORIGIN, /* diff --git a/site/src/components/Head.astro b/site/src/components/Head.astro index 0261143..a08d8fc 100644 --- a/site/src/components/Head.astro +++ b/site/src/components/Head.astro @@ -29,6 +29,7 @@ import Default from "@astrojs/starlight/components/Head.astro" import { discoveryLinks } from "../lib/agent-surface.js" import { jsonLdGraph, jsonLdText, type SiteDescription } from "../lib/jsonld.js" +import { SITE_DESCRIPTION } from "../lib/repo.js" const { site } = Astro if (site === undefined) { @@ -53,9 +54,7 @@ const links = discoveryLinks(entry.id, context) */ const siteDescription: SiteDescription = { name: siteTitle, - description: - "ATIF-native analytics over Claude Code agent trajectories: sessions converted to ATIF, " + - "materialized as a corpus, queried through DuckDB views.", + description: SITE_DESCRIPTION, inLanguage: "en" } diff --git a/site/src/lib/repo.ts b/site/src/lib/repo.ts index 3cc0f61..889f56a 100644 --- a/site/src/lib/repo.ts +++ b/site/src/lib/repo.ts @@ -46,3 +46,12 @@ export const SITE_ORIGIN = "https://laithalsaadoon.github.io" * closes the twin half; every URL helper takes this value rather than concatenating it. */ export const SITE_BASE = "/atif-sql/" + +/** + * The site's one-line description: the llms.txt summary, the meta description of every synced page, + * and the JSON-LD site node. `authored/index.md` carries the same sentence in its frontmatter, since + * an authored page's description comes from its own file. + */ +export const SITE_DESCRIPTION = + "ATIF-native analytics over Claude Code and Codex CLI agent trajectories: convert sessions to " + + "ATIF, materialize a corpus, query it through DuckDB views." From a0fbfa30c441921d35bc4be970b44670830330e8 Mon Sep 17 00:00:00 2001 From: Laith Al-Saadoon Date: Mon, 12 Oct 2026 18:13:40 +0000 Subject: [PATCH 2/3] docs: point every citation at the code it names and correct the claims The docs cited code by `path:line`, and the site's citation pass checked only that each path exists. Of the 2,177 citations on main, 1,318 needed a change: 657 re-pointed by exact text match, 342 re-pointed after reading the code, 283 whose sentence was wrong about the code and is rewritten, and 36 removed with the claim they backed. 859 were already right. The rewritten pages add 324 citations, and the pages carry 2,466 after the second read. Claims corrected against the code include the dependency graph's package attribution (pydantic from atif-converter; pydantic-settings and pyarrow from atif-duck; sibling pins at ==0.3.0), the dead-code page's importlib sites, unused exports and dead definitions, the CLI reference's flags and exit codes, the vendored harbor version (0.24.0), the console script's manifest (the root pyproject.toml), source-removed sessions being kept rather than deleted, and the CONTRACT CLI heading. A second read of 431 citations (311 a heuristic flagged, 120 at random) found 8 still wrong; all are fixed. --- .vale-baseline.json | 8 +- AGENTS.md | 41 +- CONTRIBUTING.md | 73 ++- README.md | 6 +- RELEASING.md | 36 +- docs/CONTRACT-V2.md | 30 +- docs/CONTRACT.md | 40 +- docs/README.md | 16 +- docs/analysis/dead-code.md | 509 +++++++++++----- docs/analysis/risk-hotspots.md | 217 +++---- docs/architecture/data-flow.md | 89 ++- docs/architecture/module-map.md | 214 +++---- docs/architecture/system-overview.md | 84 +-- docs/behavior/processes.md | 304 +++++----- docs/behavior/state-machines.md | 100 ++-- docs/diagrams/architecture/components.md | 80 +-- docs/diagrams/behavioral/sequences.md | 84 +-- docs/diagrams/structural/dependency-graph.md | 60 +- docs/insights/business-logic.md | 212 +++---- docs/insights/contract-map.md | 585 +++++++++++-------- docs/insights/debugging-guide.md | 261 +++++---- docs/insights/impact-analysis.md | 263 +++++---- docs/insights/tech-debt.md | 180 +++--- docs/reference/cli.md | 208 +++---- docs/reference/public-api.md | 95 +-- packages/atif-analytics/README.md | 9 +- packages/atif-cli/README.md | 7 +- packages/atif-converter/README.md | 8 +- packages/atif-duck/README.md | 10 +- packages/atif-embed/README.md | 24 +- packages/atif-models/README.md | 3 +- site/authored/agents.md | 29 +- site/authored/index.md | 17 +- 33 files changed, 2200 insertions(+), 1702 deletions(-) diff --git a/.vale-baseline.json b/.vale-baseline.json index 90dafa1..b237fb7 100644 --- a/.vale-baseline.json +++ b/.vale-baseline.json @@ -2,11 +2,11 @@ "Google.Colons": 9, "Google.FirstPerson": 3, "Google.Headings": 63, - "Google.OxfordComma": 28, - "Google.Quotes": 17, + "Google.OxfordComma": 25, + "Google.Quotes": 15, "Google.Spelling": 1, - "Google.We": 23, + "Google.We": 22, "Google.Will": 5, - "Google.WordListCase": 119, + "Google.WordListCase": 113, "proselint.Typography": 1 } diff --git a/AGENTS.md b/AGENTS.md index de2ab30..eae65af 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -10,7 +10,8 @@ documents, and layers DuckDB views on top. One corpus holds one agent, and ## Workspace layout -uv WORKSPACE (virtual root, members under `packages/*`): +uv WORKSPACE (the root is the one published `atif-sql` distribution, whose +wheel bundles every member; members under `packages/*`): - `packages/atif-converter`: converts Claude Code and Codex transcripts to ATIF (ported from harbor's converters, on the ATIF models vendored in @@ -56,11 +57,13 @@ uv WORKSPACE (virtual root, members under `packages/*`): clients, one adapter per family (`openai_bedrock` for GPT-6, `anthropic_bedrock` for Claude 5.5, picked by `infrastructure.providers.build_structured_provider`). No other package - hardcodes a Bedrock model id. Layered: `infrastructure` > `domain`. + hardcodes a Bedrock LLM model id (atif-embed's settings carry the + embedding model's default). Layered: `infrastructure` > `domain`. - `packages/atif-analytics`: the LLM pipelines that spend money at Bedrock and are checkpointed per session (classify, conflicts, friction, perceived; see `PIPELINE_NAMES` in - `infrastructure/sqlite_state/checkpointer.py`). They read human turns only + `infrastructure/sqlite_state/checkpointer.py`, which spells friction + `user_friction`). They read human turns only (`domain.authorship`), and classify and conflicts skip automated review (`turn_audit`) and one-shot-job sessions. Session data comes through the `SessionSource` port (`domain.ports`): the default reads each session's @@ -136,11 +139,12 @@ Rules of the road: statements DuckDB won't prepare (`ATTACH`, the producer's session row) go through `sql_literal`. Helpers that build SQL take and return `SqlFragment` (`atif_duck.domain.sql_literal`). Every site that still interpolates - carries `# noqa: S608 # nosec B608 - ` (ruff reads + carries `# nosec B608 - `, after `# noqa: S608` + wherever ruff flags the line too (ruff reads the first marker, Bandit only the second; never blanket-skip B608 in `[tool.bandit]`), and `packages/atif-duck/tests/test_sql_text_boundaries.py` runs an AST audit over `registry.py`, `columnar.py`, `analytics.py`, - `authorship.py`, `lake.py` (both layers of each), `lake_steps.py` and atif-embed's + `authorship.py`, `lake.py`, `lake_sessions.py` (both layers of each), `lake_steps.py` and atif-embed's `corpus_text_rows.py` that fails on any placeholder that isn't a constant, a projection call, or `sql_literal(...)`. A statement built per table is a module-level constant (a dict comprehension over the table specs), which the @@ -325,8 +329,8 @@ copy of the queryable rows: force a full lake read; no usable lake falls back to the per-session reader. The watermark only moves after a run that read to the end (not to a `--limit`) and embedded every row. `embed --prune-orphans` counts stored - rows whose uuid is no lake step's (dry run unless `--no-dry-run`, refused - while sessions are pending a lake write). + rows whose uuid is no lake step's (dry run unless `--no-dry-run`; the real + run is refused while sessions are pending a lake write). - Analytics: `analyze` reads session data from the lake when the lake holds the corpus (`--no-lake` reads the files). A session is read from the lake only when its `session_meta` row carries the `materialized_at` and @@ -356,8 +360,9 @@ copy of the queryable rows: `--corpus-root`s given) beside the old one and swaps the directory in. `lake verify` compares every session's row count and content hash, table by table, between the lake and its artifacts and exits 65 on a difference (78 - with no usable lake). `lake status` (also folded into `atif-sql status`) - reports the state, snapshots, files and pending sessions. `lake compact` + with no usable lake). `lake status` reports the state, corpora, snapshots, + files, and last write; `atif-sql status` folds the same into its lake + section and adds the corpus's pending sessions. `lake compact` merges small files, rewrites delete-heavy ones, expires snapshots older than `--expire-older-than-days` (default 30) and removes unreferenced files once they're an hour old, so a reader on the previous catalog copy still finds @@ -397,7 +402,8 @@ For an LLM agent driving `atif-sql`, the discovery loop is this: `requires: vss` needs `atif-sql embed --all --no-dry-run` (a bare `embed` exits 64: a real run needs an explicit scope). For semantic search over TEXT, prefer `atif-sql search 'query'`, which embeds the text first, then runs - the same `semantic_search` kNN. + its own cosine kNN over `message_embeddings` (the ranking `semantic_search` + uses) and joins each hit to its step for a snippet. What the query sandbox will and won't do: `SELECT`, `EXPLAIN`, `SET TimeZone`, and in-memory DDL/DML run; `COPY`, `EXPORT`, `ATTACH`, @@ -417,13 +423,14 @@ the derived example must execute (or a documented `EXCLUSIONS` entry). ## Definition of done -`mise run check` fully green (lint + fmt + typecheck + lint:imports + lint:workflows + +`mise run check` fully green (lint + fmt + typecheck + typecheck:pyright + lint:imports + lint:workflows + lint:hooks + lint:pnpm-lock + security:vex:check + docs:prose + test). `docs:prose` is Vale over the published prose (docs/, site/authored/, README.md, AGENTS.md, CONTRIBUTING.md) at error level, with a file floor per glob so a shrunken scope is red, and a per-rule warning ratchet against `.vale-baseline.json`: reword the sentence it flags, or add a real term to `.vale/styles/config/vocabularies/atif-sql/accept.txt`. When a count falls, run `mise run docs:prose:baseline` and commit the lower file; never raise it. A spaced dash is an error (`Google.EmDash`), not a warning: write a colon, comma, period, or parentheses instead. `mise run security` is the report-only tier beside it: findings do not fail -it, a scanner that produced no usable SARIF does. +it, a scanner that produced no usable report does. A change under docs/ or site/ also needs `mise run docs:gate`: the site build, then `docs:links`, -which crawls site/dist as GitHub Pages serves it and fails on any internal 404. Link page to page by +which crawls site/dist as GitHub Pages serves it and fails on any internal 404, then `astro check` and +the site's dist probes (`pnpm run check && pnpm run test` in site/). Link page to page by relative `.md` path as GitHub renders it; the build turns each link into the page's route, or into the GitHub blob for a Markdown file the site does not publish. First-time setup: `mise trust && mise install && mise run install`. @@ -436,10 +443,10 @@ for a finding. ## Experiments -`experiments/` holds numbered experiment protocols (README per experiment). -Outputs go to `experiments/**/out/` (gitignored). No experiment is wired -into a gate: `mise run check` and pytest's `testpaths` both skip the -directory. +`experiments/` is where numbered experiment protocols go (README per +experiment); the tree holds none. Outputs go to `experiments/**/out/` +(gitignored). No experiment is wired into a gate: no `check` task runs one, +and pytest's `testpaths` skip the directory. Migration parity oracles are not part of this repo. Parity is not a standing gate: re-running one would need a reference implementation this workspace does diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index c2fbf26..cbd9e30 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -16,7 +16,8 @@ mise trust && mise install && mise run install ``` `mise install` provisions the toolchain pinned in `mise.toml` (`[tools]`: -Python 3.13 and the latest `uv`; `min_version = "2025.1.0"`). `mise run install` +Python 3.13, the latest `uv`, the security scanners, `node` and `pnpm` for the docs site, and Vale, +each resolved through `mise.lock`; `min_version = "2026.8.1"`). `mise run install` is `uv sync --all-packages` and depends on `hooks:install`, so it also installs the lefthook git hooks in the same step. @@ -33,21 +34,24 @@ that. ## Definition of done: `mise run check` -`[tasks.check]` in `mise.toml` depends on the gates below, and all of them must pass: +`[tasks.check]` in `mise.toml` depends on the eleven gates below, and all of them must pass: | Task | Command | What fails you | | --- | --- | --- | | `lint` | `uv run ruff check .` | the broad `select` list in `[tool.ruff.lint]` | | `fmt` | `uv run ruff format --check .` | formatting; `mise run fmt:write` applies it | | `typecheck` | `uv run ty check` | `[tool.ty.rules] all = "error"`, and `[tool.ty.terminal] error-on-warning = true` means a warning is also a failure | +| `typecheck:pyright` | `pyright` | an error under `[tool.pyright]`, which sets `typeCheckingMode = "strict"` over the same trees | | `lint:imports` | `uv run lint-imports` | the import contracts below | +| `lint:workflows` | `actionlint` | a workflow under `.github/workflows` that `actionlint` rejects (syntax, expressions, runner labels, `shellcheck` over each `run:` block) | +| `lint:hooks` | `uv run lefthook validate` | a key in `lefthook.yml` that lefthook's own schema rejects | | `lint:pnpm-lock` | `python3 scripts/verify_single_yaml_document.py site/pnpm-lock.yaml` | `site/pnpm-lock.yaml` holding more than one YAML document, which GitHub's dependency graph cannot read | | `security:vex:check` | `uv run python scripts/vex_to_osv_config.py ... --check` | either `osv-scanner.toml` (root, beside `uv.lock`, and `site/`, beside `pnpm-lock.yaml`, since osv-scanner reads only the config in a lockfile's own directory) or the `allow-ghsas:` line of `.github/workflows/dependency-review.yml` drifting from `security/atif-sql.openvex.json`, or a ledger PURL version that `uv.lock` or `site/pnpm-lock.yaml` no longer locks; `mise run security:vex` re-renders both | | `docs:prose` | `python3 scripts/vale_gate.py` | an error-level Vale alert in `docs/**/*.md`, `site/authored/**/*.md`, `README.md`, `AGENTS.md` or `CONTRIBUTING.md`, a run that checked fewer files than that scope holds, or a glob that matched fewer files than its floor in `scripts/vale_gate.py` (`docs/**/*.md` at least 15, `site/authored/**/*.md` at least 2, each root file at least 1), or a rule whose warnings rose past its count in `.vale-baseline.json` (or that warns and is not in it); a baseline that is missing, empty, or does not parse fails as well. `.vale.ini` names the rules and why each one below error sits there, and a new term goes in `.vale/styles/config/vocabularies/atif-sql/accept.txt` | -| `test` | `uv run pytest --no-header -q` | `[tool.pytest.ini_options] testpaths = ["packages/*/tests"]` | +| `test` | `uv run pytest --no-header -q` | `[tool.pytest.ini_options] testpaths = ["packages/*/tests", "fuzz"]` | Most gates declare `sources`, so mise skips one whose inputs have not moved by -mtime (`lint:pnpm-lock` and `docs:prose` run every time). Run `mise run --force check` when you want them all to execute anyway. +mtime (`typecheck:pyright`, `lint:pnpm-lock`, and `docs:prose` run every time). Run `mise run --force check` when you want them all to execute anyway. The Vale warnings are a ratchet, not a free pass. `.vale-baseline.json` holds each rule's warning count over the scope in the table, and `docs:prose` names any rule that rose. Reword the sentence that @@ -98,7 +102,8 @@ weekly and on pull requests that change the converter or `fuzz/`. A finding lands in `fuzz/out//artifacts/` (CI uploads it as `fuzz-findings`). Replay it with `uv run --group fuzz python fuzz/fuzz_.py `, fix the converter, and add -the input to `fuzz/corpus//` so `check` keeps it fixed. A new target is a function in +the input to `fuzz/regressions//` so `check` keeps it fixed (a seed in `fuzz/corpus/` +must convert, and a fixed crash is usually a rejection). A new target is a function in `TARGETS`, a `fuzz_.py` harness, a `fuzz/corpus//` seed directory and a `fuzz/.dict` dictionary; `fuzz` fails until all four exist. @@ -110,25 +115,28 @@ standards, not a fix to your branch. Raise it in the pull request instead. Commit messages must be [Conventional Commits](https://www.conventionalcommits.org): `type(scope): subject` with `type` one of `feat|fix|chore|docs|refactor|test|perf|ci`. -This is enforced, not advisory: `lefthook.yml`'s `commit-msg` hook runs -`uv run cz check --allow-abort --commit-msg-file {1}`, configured by +The format is enforced, not advisory: `lefthook.yml`'s `commit-msg` hook runs +`mise exec -- uv run cz check --allow-abort --commit-msg-file {1}`, configured by `[tool.commitizen]` in `pyproject.toml` (`name = "cz_conventional_commits"`, with `allowed_prefixes = ["Merge", "Revert", "Pull request", "fixup!", "squash!"]` -as the only exemptions). +as the only exemptions). `cz check` also accepts `build`, `bump`, `revert`, and `style`; +the shorter list is this repository's convention. The other hooks in `lefthook.yml` run a subset of the gate early, so a clean `mise run check` before you commit means the hooks have nothing to say: -- **pre-commit**, on staged `**/*.py`: `ruff check --fix` and `ruff format` - (both `stage_fixed: true`, so fixes are re-staged for you), `ty check`, and - `lint-imports`. Plus `uv lock --check` whenever a `pyproject.toml` or - `uv.lock` is staged. -- **pre-push**: `pytest --no-header -q`, plus `mise run docs:links` (a site build and the - crawl of `site/dist`) when the push touches `docs/` or `site/`. +- **pre-commit**: on staged `**/*.py`, `ruff format` and `ruff check --fix` + (both `stage_fixed: true`, so fixes are re-staged for you); `betterleaks git --staged` + over the staged diff, which blocks a staged secret and is the one hook `check` doesn't + cover; and `uv lock --check` whenever a `pyproject.toml` or `uv.lock` is staged. +- **pre-push**: `mise run` of `lint`, `typecheck` (ty only), `lint:imports`, `lint:workflows`, + `lint:hooks`, `lint:pnpm-lock`, `docs:prose`, and `test`, plus + `mise run docs:install && mise run docs:links` (a site build and the crawl of `site/dist`) + when the push touches `docs/` or `site/`. -`ty`, `lint-imports`, and the pre-push pytest are skipped during a merge or -rebase (`skip: [merge, rebase]`); `mise run check` is what covers those. +The pre-push `typecheck`, `lint:imports`, docs link crawl, and `test` jobs are skipped during a +merge or rebase (`skip: [merge, rebase]`); `mise run check` is what covers those. Work on a branch, and open a pull request against `main`. @@ -144,7 +152,7 @@ root packages. These shape the whole workspace: 2. **`atif_analytics` composes `atif_models` and nothing else of ours.** A `forbidden` contract pins its remaining edges shut: it may not import `atif_converter`, `atif_corpus`, `atif_duck`, `atif_embed`, or `atif_cli`. - That is why `atif_embed` reads the corpus through its own DuckDB adapter + The independence contract is why `atif_embed` reads the corpus through its own DuckDB adapter over the documented corpus layout rather than through `atif_duck`. The rest are `layers` contracts: @@ -174,9 +182,11 @@ failure. **A new package.** `packages/*` is globbed as a workspace member, but these places in the root `pyproject.toml` list packages explicitly and will not pick -it up on their own: `[tool.ruff] src`, `[tool.ruff.lint.isort] -known-first-party`, `[tool.ty.environment] root` plus `[tool.ty.src] include`, -and `[tool.importlinter] root_packages` (with its layers contract). +it up on their own: `[tool.hatch.build.targets.wheel] packages`, `[tool.ruff] src`, +`[tool.ruff.lint.isort] known-first-party`, `[tool.ty.environment] root` plus +`[tool.ty.src] include`, `[tool.pyright] include` plus `extraPaths`, +`[tool.coverage.run] source`, `[tool.bandit] exclude_dirs`, and +`[tool.importlinter] root_packages` (with its layers contract). The new member also needs the license wiring every existing member has: `license = "Apache-2.0"` with `license-files = ["LICENSE"]` in `[project]`, and @@ -188,9 +198,10 @@ it, and `uv build` fails outright if the file is missing. **A DuckDB view or macro.** The catalog is static and drift-tested. Per `AGENTS.md`, adding an object means adding a `DESCRIPTIONS` entry in -`atif_duck/domain/catalog.py`, an `ARG_EXEMPLARS` entry for any new parameter -name, `TABLE_MACRO_NAMES` membership if the DDL is `AS TABLE`, and a derived -example that actually executes (or a documented `EXCLUSIONS` entry). The tests +`atif_duck/domain/catalog.py`, an `ARG_EXEMPLARS` entry in `atif_duck/domain/examples.py` +for any new parameter name, `TABLE_MACRO_NAMES` membership if the DDL is `AS TABLE`, and a +derived example that actually executes (or a documented `EXCLUSIONS` entry, in the same +`examples.py`). The tests in `packages/atif-duck/tests/` fail until that is true. ## harbor and litellm are test oracles, not dependencies @@ -257,11 +268,11 @@ materialize, so read the diff as a pricing change. ## Never spend money in a test These paths call Amazon Bedrock and bill the caller: `atif-sql analyze` (the LLM -pipelines) and `atif-sql embed` (Cohere Embed v4). Each is guarded, and the -guards are part of the contract. The local provider (`ATIF_SQL_EMBED_PROVIDER=gemma`) -bills nothing but downloads 1.5 GB on first use, so its tests load a fake -SentenceTransformer and fake torch modules instead, and pass with or without the -`local` extra installed: +pipelines), `atif-sql embed` (Cohere Embed v4), and `atif-sql search`, which embeds its +query the same way. The two bulk paths are guarded, and the guards are part of the +contract. The local provider (`ATIF_SQL_EMBED_PROVIDER=gemma`) bills nothing but downloads +1.5 GB on first use, so its tests load a fake SentenceTransformer and fake torch modules +instead, and pass with or without the `local` extra installed: - `analyze` is dry-run by default; `--no-dry-run` is what makes it spend. - `embed` refuses a real run with no scope: no `--limit` and no `--all` exits 64. @@ -272,8 +283,8 @@ change. ## Experiments -`experiments/` holds numbered protocols with a README (and, once run, a REPORT) -per experiment. Nothing there is wired into a gate: pytest's `testpaths` only -collects `packages/*/tests`, and outputs are written to `experiments/**/out/`, +An experiment goes under `experiments/` as a numbered protocol with a README (and, once run, a +REPORT); the tree holds none. Nothing there is wired into a gate: pytest's `testpaths` only +collects `packages/*/tests` and `fuzz`, and outputs are written to `experiments/**/out/`, which `.gitignore` excludes. An experiment's recorded numbers are measurements; if you re-run one, add your measurement rather than editing the one already recorded. diff --git a/README.md b/README.md index 5817fc1..2a2e9c8 100644 --- a/README.md +++ b/README.md @@ -107,7 +107,7 @@ the only thing documented as installable is the `atif-sql` CLI above. | `atif-converter` | Claude Code and Codex CLI transcript → ATIF converters (ours, built on Harbor's ATIF models, vendored) + per-agent fidelity policy (loss accounting per session) | | `atif-corpus` | Corpus materialization: discovery, watermarks, quiescence, atomic artifact writes | | `atif-duck` | DuckDB views + macros over the materialized corpus (core surface plus the v2 analytics surface) | -| `atif-models` | Model alias registry + structured-output LLM clients for GPT-6 and Claude 5.5 on Bedrock; no other package hardcodes a model id | +| `atif-models` | Model alias registry + structured-output LLM clients for GPT-6 and Claude 5.5 on Bedrock; no other package hardcodes an LLM model id | | `atif-analytics` | the LLM pipelines `atif-sql analyze` runs: classify, conflicts, friction, and perceived | | `atif-embed` | Cohere Embed v4 on Bedrock or EmbeddingGemma 2 on this machine + LanceDB vector store + embedding backfill | | `atif-cli` | the composition root, and the source of the `atif-sql` command (see [What you can do](#what-you-can-do)) | @@ -152,7 +152,7 @@ matching `ATIF_SQL_*` env var still wins. ```bash atif-sql materialize --agent codex atif-sql status --agent codex -atif-sql query "SELECT agent, count(*) FROM sessions GROUP BY 1" +atif-sql query --agent codex "SELECT agent, count(*) FROM sessions GROUP BY 1" ``` One corpus holds one agent, so a Codex corpus and a Claude Code corpus stay @@ -227,7 +227,7 @@ vulnerability in a tool that reads local transcripts are in corpus is registered (override with `ATIF_SQL_QUERY_MEMORY_LIMIT` and `ATIF_SQL_QUERY_THREADS`), a private spill directory outside the corpus that's removed on exit, no extension installs at query time (`atif-sql embed --install-extension` is where the lance extension comes from), file -facing statements (`COPY`, `EXPORT`, `ATTACH`, `INSTALL`, `LOAD`, `PREPARE`, `EXECUTE`) refused +facing statements (`COPY`, `EXPORT`, `ATTACH`, `DETACH`, `INSTALL`, `LOAD`, `PREPARE`, `EXECUTE`) refused before they run, and a refusal to run as root unless `ATIF_SQL_ALLOW_ROOT=1` says so. The details and the one accepted disclosure (`duckdb_settings()` lists the granted paths) are in [docs/reference/cli.md](docs/reference/cli.md#query). diff --git a/RELEASING.md b/RELEASING.md index 6728e7f..66768e2 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -40,8 +40,8 @@ implementation that accepted the commit is the whole argument: CI computes, from the locked commitizen in `uv.lock`. A GitHub Action's release logic cannot be run locally at all, so its answer can only be observed after it has been committed. - **`version_files` rewrites arbitrary text in arbitrary files**, which is what a version living - in many places needs: one bump moves the published `[project] version`, `version = "0.2.0"` in - every member manifest, and `"atif-duck==0.2.0"` in the ones that carry dev pins. Keeping those + in many places needs: one bump moves the published `[project] version`, `version = "0.3.0"` in + every member manifest, and `"atif-duck==0.3.0"` in the ones that carry dev pins. Keeping those in step by hand is the failure this repository is most likely to ship. - **No new dependency.** commitizen is already in `[dependency-groups] dev`. @@ -91,7 +91,7 @@ entries with no `atif-*` among them. ## What the preparation preflight asserts -`publish.yml` preparation checks these properties before it attaches draft assets, and fails with a pointer to +`publish.yml` preparation checks these properties (`release.yml` checks the third) before it attaches draft assets, and fails with a pointer to this file when one is missing. Each is a defect `twine check` passes over — the metadata stays structurally valid and the damage is semantic — and each becomes permanent the moment a first release goes out under the wrong name or an unpinned requirement. @@ -131,15 +131,16 @@ is wrong are different failures. **3. `[tool.commitizen]` carries the version wiring.** commitizen's `version_provider` defaults to reading `version` out of `[tool.commitizen]` itself, which stays the source of truth; the root's `[project] version` is what a wheel carries and moves through `version_files`. Both hold -one string. `release.yml` asserts the commitizen key exists, because without it `cz bump` has -nothing to read and silently has nothing to do. +one string. `release.yml` asserts the commitizen key exists before it runs the gate, because +without it `cz bump` has nothing to read and fails only after `mise run check`, with +`[NO_VERSION_SPECIFIED]`. **4. The CLI reports the distribution's version.** cyclopts resolves a default `--version` by looking the calling module's name up in the installed metadata. The module is `atif_cli` and the distribution is `atif-sql`, so `importlib.metadata.version("atif_cli")` raises `PackageNotFoundError` and cyclopts answers **`0.0.0`**. `atif_cli.app` therefore names the distribution explicitly, through a callable so `importlib.metadata` stays off the import path -until `--version` is actually asked for (which the lean-import test pins). `publish.yml` +until `--version` is actually asked for. `publish.yml` compares the installed CLI's `--version` against the tag, so a regression here cannot reach PyPI quietly. @@ -154,7 +155,7 @@ governs. These properties of `[tool.commitizen]` are worth knowing before editin files are rewritten and before the commit, the only window where the re-resolved lockfile can land in the same commit as the versions it resolves. `git add` is separate because commitizen stages only the files it rewrote. -- `version_files` entries are per-file rather than globbed, because `--check-consistency` +- The pin entries in `version_files` are per-file rather than globbed, because `--check-consistency` requires a hit in every matched file and some members carry no intra-workspace pin. @@ -172,11 +173,10 @@ which is interactive and silently does nothing if you answer `n`. ```bash git status --short # must be empty; commit anything here FIRST uv run cz bump --version-files-only --check-consistency --yes -git diff --stat # expect only the lines listed above +git diff --stat # expect only the lines listed above, plus CHANGELOG.md -git checkout -- . # the version rewrites +git checkout -- . # the version rewrites and CHANGELOG.md git restore --staged --worktree uv.lock # the staged re-lock -rm -f CHANGELOG.md # written even in this mode git status --short # must be empty again ``` @@ -258,8 +258,8 @@ litellm`) prints nothing, and `test_dev_only_imports_guard.py` fails if `src/` i ## What a GitHub release carries, and how to verify it -Every release carries the same six assets, staged by `mise run release:assets` on the tag and -attached to the draft by `publish.yml` preparation: +Every release carries the same six assets, staged by `mise run release:assets` on the tag (the +attestation bundle by the `attest` job) and attached to the draft by `publish.yml` preparation: | Asset | What it is | | --- | --- | @@ -360,7 +360,7 @@ rehearsal that cannot rehearse the thing that matters. - What it would catch — invalid metadata, a malformed sdist or wheel — `publish.yml` already catches offline and deterministically, with `twine check` plus an actual - `uv tool install --find-links dist atif-sql` followed by running the installed command. + `uv tool install --reinstall --find-links dist atif-sql` followed by running the installed command. - What it cannot catch is a resolution against PyPI's index, because TestPyPI's index holds different packages. An install test there resolves a different graph than a user's. - The remaining failure a rehearsal would find is a misconfigured publisher — and TestPyPI @@ -384,7 +384,7 @@ the fat-wheel restructure above. Neither is on the table. | Ruleset `main review` on the default branch | Settings → Rules → Rulesets | Pull requests with two approvals, Code Owner review, the latest push approved, review threads resolved, stale approvals dismissed. Bypass actors: the deploy key, and the repository admin role for pull requests only. See below. | | Environment `release` with secret `RELEASE_DEPLOY_KEY`, deployments from `main` only | Settings → Environments | The private half of the write deploy key `release.yml` pushes the bump commit and tag with. See below. | | **Immutable releases** on | Settings → General → Releases | Preparation attaches all signed assets to the draft. Publishing locks those assets and the tag before the PyPI job downloads them. | -| Pages **Source = GitHub Actions** | Settings → Pages | `docs.yml` deploys through `actions/deploy-pages`, which needs the Actions source rather than a branch. `base: "/atif-sql/"` in `site/astro.config.ts` matches the resulting project-site path. | +| Pages **Source = GitHub Actions** | Settings → Pages | `docs.yml` deploys through `actions/deploy-pages`, which needs the Actions source rather than a branch. `base: SITE_BASE` in `site/astro.config.ts` (`"/atif-sql/"`, from `site/src/lib/repo.ts`) matches the resulting project-site path. | | The AI-crawler policy, at the ORIGIN root | The `laithalsaadoon.github.io` user-pages repository | `robots.txt` is per-origin (RFC 9309 §2.3), so a project site served from a path segment cannot own one: Astro emits this site's copy to `/atif-sql/robots.txt`, which no crawler fetches. `site/public/robots.txt` is the decided policy and the exact text to install at `https://laithalsaadoon.github.io/robots.txt`. Until it is installed there, these pages inherit whatever that origin already serves — and absent a file, that is fully permissive. Outside this repository's reach, so no gate here can assert it. | ### The release push and branch protection @@ -505,7 +505,7 @@ rather than spending another version: gh workflow run publish.yml -f tag=v0.2.0 ``` -The publish steps set `skip-existing: true`, so a re-run after a partial upload completes the +The publish step sets `skip-existing: true`, so a re-run after a partial upload completes the legs that failed instead of taking a 400 on the ones that succeeded. PyPI refuses to overwrite an existing filename either way, so this cannot replace bytes. @@ -514,8 +514,8 @@ an existing filename either way, so this cannot replace bytes. | Symptom | Cause | | --- | --- | | `[tool.commitizen] in pyproject.toml carries no 'version' key` | The `version` key was removed from `[tool.commitizen]`. Property 3 above. | -| `no wheel built for: atif-sql` / `unexpected distribution(s) built: atif-cli` | `packages/atif-cli/pyproject.toml` declares a `[project] name` other than `atif-sql`. Property 1 above. | -| `intra-workspace requirement 'atif-duck' is not pinned to ==` | A sibling dependency lost its `==` pin. Property 2 above. | +| `: Name is '', expected 'atif-sql'` | The root `pyproject.toml` declares a `[project] name` other than `atif-sql`. Property 1 above. | +| `: requires unpublished sibling(s): [...]` | An `atif-*` requirement reached the root `[project.dependencies]`. Property 2 above. | | `Version is '0.1.0', tag says '0.2.0'` | The tag and the manifests disagree — a `cz bump` that partially applied, or a hand-moved tag. | | `the installed CLI reports version '0.0.0'` | `cyclopts.App` has no explicit `version=`, so it looked up the module name instead of the distribution name. | | `EOFError` from `cz bump`, no other output | `--get-next` without `--yes` opens a prompt and dies on a non-TTY. | @@ -527,4 +527,4 @@ an existing filename either way, so this cannot replace bytes. | Preparation refuses a draft with existing assets | Rerun the failed upload job to reuse its saved artifacts. A fresh build cannot replace existing assets. | | Immutable release verification fails | The release is still a draft, immutability is disabled, or its attestation does not verify. Publication stops before PyPI. | | 403 from PyPI naming a publisher it cannot find | On a first release, the pending publisher does not exist yet. Afterwards, the publisher does not match the claims. | -| `expected one sdist and one wheel for , staged 0` | `uv build --all-packages` did not produce that project — usually a member removed from `[tool.uv.workspace] members`. | +| `expected exactly one wheel and one sdist, found and ` | `uv build --out-dir dist` built no root distribution, or built more than the root project (an `--all-packages` build adds the seven dev manifests). | diff --git a/docs/CONTRACT-V2.md b/docs/CONTRACT-V2.md index fafe0e4..aed9f18 100644 --- a/docs/CONTRACT-V2.md +++ b/docs/CONTRACT-V2.md @@ -12,13 +12,13 @@ pipelines (cluster, terms, community) were removed, along with their views, macros, and the structural cron lane. atif-analytics now runs the LLM pipelines classify, conflicts, friction, and perceived. classify labels only work category and goal (autonomy tier and success were dropped as -unreliable) and runs on small (luna). The cron lanes are materialize and -llm. The trajectory, autonomy-tier, and structural lines below describe the -original build. +unreliable) and runs on small (luna). The cron lanes are materialize, llm, +and compact. The trajectory, autonomy-tier, and structural lines below +describe the original build. ## New packages - atif-models: model alias registry + structured-output LLM client. NO other - package hardcodes a model id. + package hardcodes the id of an LLM it calls. - atif-analytics: the v2 pipelines (classify, trajectory, conflicts, friction, perceived, cluster, terms, community). Depends on atif-models ONLY; the `forbidden` contract bars atif-duck. atif-cli composes. @@ -74,12 +74,14 @@ friction=small (per-message enum), embed unchanged second adapter behind EmbeddingProvider). ## Ports & state: one shape per seam, declared in domain/ports.py -CachePort/CheckpointPort/RetryQueuePort/VectorStorePort, sqlite WAL state.db -per corpus, parquet shard dirs under /analytics//, -checkpoint keyed (session_id, pipeline) on (last_ts, last_mtime). +SessionSource (atif-analytics) and VectorStorePort (atif-embed), each in its +package's domain/ports.py; ParquetCache/SqliteCheckpoint/SqliteRetryQueue in +atif-analytics infrastructure over a sqlite WAL state.db per corpus, +parquet shard dirs under /analytics//, checkpoint keyed +(session_id, pipeline) on (last_ts, last_mtime). KEY PROPERTY: pipelines read the MATERIALIZED corpus -(atif-duck views: steps/messages/edges), not raw JSONL. Transcript text for -prompts = steps-based rendering under fixed caps (800K/session). Tool +(trajectory.json steps + edges.jsonl, or the lake), not raw JSONL. +Transcript text for prompts = steps-based rendering under fixed caps (800K/session). Tool calls render with their arguments (400 chars); each tool result renders as a size stub, so tool output stays on disk. ATIF_SQL_SESSION_TEXT_TOOL_RESULT_BODIES=true sends result bodies instead, @@ -101,11 +103,13 @@ primary uuid (first source uuid per step; document choice). ## Cron scripts/atif-sql-refresh.sh: lanes (materialize */10, structural :17, -llm nightly 10:20Z), flock per lane, AWS env hygiene + bearer-token read at -runtime, both corpora via ATIF_SQL_SOURCE_ROOT, log to -scripts/.run/, selftest script with the opt-out tripwire pattern. Installs via +llm nightly 10:20Z, compact nightly 03:40), flock per lane, AWS env hygiene + +bearer-token read at runtime, every corpus (the primary config root plus each +ATIF_SQL_EXTRA_CONFIG_DIRS entry) via CLAUDE_CONFIG_DIR + ATIF_SQL_SOURCE_ROOT, +log to scripts/.run/, selftest script with the opt-out tripwire pattern. Installs via `atif-sql cron install` printing the crontab block (no silent crontab writes). ## Cost guards analyze defaults dry_run=true; --no-dry-run only in the cron llm lane; -estimate_cost port; per-pipeline token accounting logged; embed --limit 500. +estimate_cost in the registry; per-pipeline token accounting logged; +embed --limit 500. diff --git a/docs/CONTRACT.md b/docs/CONTRACT.md index 242aa88..f1b9002 100644 --- a/docs/CONTRACT.md +++ b/docs/CONTRACT.md @@ -60,7 +60,8 @@ proofs are out of scope for the workspace. ``` - corpus-slug: a slug of the source root path; it IS the on-disk dir name. - One key is reserved and not hashed: `codex` names the Codex CLI corpus. + Two keys are reserved and not hashed: `default` names the `~/.claude` root + and `codex` names the Codex CLI corpus. - session_id boundary: the id comes from the transcript filename and is the one piece of outside text that becomes a corpus path. It must match `^[A-Za-z0-9][A-Za-z0-9._-]*$` and be at most 255 characters (every id @@ -95,8 +96,8 @@ proofs are out of scope for the workspace. deletes, moves or writes anything under a shared corpus directory such as blobs/ except to add a new blob, and the raw source archive lives only in sessions//source/. -- Quiescence: a session is (re)materialized when newest source mtime is older - than quiesce_seconds (default 300) AND either its sources moved since the +- Quiescence: a session is (re)materialized when newest source mtime is at + least quiesce_seconds old (default 300) AND either its sources moved since the watermark or its generation is stale (below). --force overrides the second half. "Now" is read after the scan, and a source mtime up to 1 s in the future is jitter, not a clock warning. @@ -147,7 +148,7 @@ proofs are out of scope for the workspace. - Columnar artifacts (old layout): an earlier materialize also wrote five parquet files per session, a cache of what the views compute from trajectory.json and session_events.jsonl, claimed by meta.columnar_schema. - materialize no longer writes them. A reader takes the columnar path for a + materialize doesn't write them. A reader takes the columnar path for a session only when meta.columnar_schema equals its own version AND all of those files are present and non-empty; otherwise it reads the session's trajectory. Whatever the path, every view and macro returns the same rows. @@ -166,7 +167,7 @@ proofs are out of scope for the workspace. that references it, so a published placeholder always resolves. The store is shared rather than per-session because a hash-named write is idempotent: a blob two sessions share is stored once, and re-converting a session - rewrites nothing. Ghost removal does not collect blobs; an unreferenced blob + rewrites nothing. Nothing collects blobs; an unreferenced blob costs space only. The hash and extension are validated before either becomes a path. @@ -249,7 +250,8 @@ proofs are out of scope for the workspace. edges, loss_report.json, meta.json + derived views above. Sessions carrying current columnar artifacts are read with read_parquet instead of read_json, per session, and the two sets are unioned; the returned RawSources says - which sessions took which path. The lake loader passes stage_columnar: + which sessions took which path. The lake loader calls register_raw with + stage_columnar: every session without current parquet of its own is decoded in Python and typed into parquet staged outside the corpus, and read from there (RawSources.staged_session_ids), which bounds the loader's memory. @@ -258,12 +260,14 @@ proofs are out of scope for the workspace. built through the connection's own API and registered as views (CREATE VIEW can't be prepared). Every remaining f-string placeholder in a SQL statement is a module constant, a catalog constant, a projection - expression, or `sql_literal(...)`, and an AST test over the SQL - modules fails on anything else. The statements DuckDB won't prepare - (ATTACH for the Lance store and the lake, the producer's one-row session - projection, the lake reader's corpus filter) + expression, or a `sql_literal(...)`, `int(...)`, or `float(...)` + coercion, and an AST test over the SQL modules fails on anything else. + The statements DuckDB won't prepare (ATTACH for the Lance store and the + lake, SET temp_directory on the lake connections, the producer's one-row + session projection, the corpus column the JSON path adds, the lake + reader's corpus filter, cost_estimate's pricing rows) are the only places `sql_literal` still escapes a value. -- ColumnarArtifactProducer(session_dir, session_id, trajectory, events_path?) +- ColumnarArtifactProducer.produce(session_dir, session_id, trajectory, events_path?) is a pure function of the trajectory (and of the session's events file, plain or compressed) that writes the parquet files with the views' own projection expressions, so the JSON columns are normalized exactly as @@ -275,7 +279,8 @@ proofs are out of scope for the workspace. cache_creation_input_tokens with the Codex spelling cache_write_input_tokens. - messages-parity view name: `steps` (one row per ATIF step) PLUS a `messages` compatibility view reconstructed from edges (uuid-keyed: uuid, parent_uuid, - session_id, ts, type, is_sidechain, is_compact_summary, role via join). + session_id, ts, type, is_sidechain, is_compact_summary, message_id, + source_file; `type` carries the record's role). - Static VIEW_SCHEMA dict + drift test, so a view rename fails a gate. - Macro signatures are pinned against the DDL by a drift test. - sessions view carries `corpus`, the corpus directory's name, on both read @@ -292,20 +297,20 @@ proofs are out of scope for the workspace. The lake is attached READ_ONLY from catalog.reader.duckdb before the sandbox locks the connection, and the grants name each live lake file. -## CLI (atif-cli composes; the only package importing the others) +## CLI (atif-cli composes the other packages) ```text -atif-sql convert [--agent claude-code|codex] +atif-sql convert [--agent claude-code|codex] # --agent defaults from ATIF_SQL_AGENT atif-sql materialize [--force] [--quiesce-seconds N] [--agent ...] [--workers N] [--no-lake] # sync corpus # report adds rejected / rejected_session_ids, # lake_synced / lake_pending / lake_error atif-sql status [--agent ...] # corpus freshness, counts, watermark age -atif-sql query 'SQL' [--format auto|json|csv] +atif-sql query 'SQL' [--format auto|table|json|csv] # sized to the host before registration # (ATIF_SQL_QUERY_MEMORY_LIMIT / _THREADS # override); private mkdtemp spill dir; # refuses uid 0 unless ATIF_SQL_ALLOW_ROOT=1; - # COPY/EXPORT/ATTACH/INSTALL/LOAD/PREPARE/EXECUTE + # COPY/EXPORT/ATTACH/DETACH/INSTALL/LOAD/PREPARE/EXECUTE # exit 70 sandbox_refused before execution; # never installs an extension; # --all-corpora (lake only, exit 78 without one), @@ -359,6 +364,7 @@ lake_load_batch_size=512 (sessions per lake transaction), lake_stage_workers (processes that stage a batch from its trajectories; default min(4, cpu_count)), lake_lock_timeout_seconds, lake_expire_days=30. _default_*() factories read env at call time. With agent=codex the two roots re-derive to $CODEX_HOME (default -~/.codex)/sessions and ~/.atif-sql/corpus/codex; an explicitly set +~/.codex)/sessions and the corpus root slugged from it +(~/.atif-sql/corpus/codex for the default); an explicitly set ATIF_SQL_SOURCE_ROOT or ATIF_SQL_CORPUS_ROOT always wins over that re-derivation. diff --git a/docs/README.md b/docs/README.md index 4f53645..c0e3d0f 100644 --- a/docs/README.md +++ b/docs/README.md @@ -6,9 +6,9 @@ this codebase quietly assumes. Every factual claim carries a `path:LOC` citation Prose is generated; structure is mechanical. Cross-references are deterministic. Citations come as full paths or as shorthands, and each one resolves into a -source file. A validator rejects any citation naming a file that does not exist, a line -outside that file's range, or a shorthand with no resolvable antecedent, and it exits non-zero on the -first one, so a citation here resolves or the tree does not ship. +source file. The docs site build turns each one into a link and fails on the first citation naming +a file that does not exist at the commit it links to, or a shorthand with no resolvable antecedent, +so a citation here resolves or the tree does not ship. ## Start here @@ -24,8 +24,8 @@ first one, so a citation here resolves or the tree does not ship. - [System overview](architecture/system-overview.md): the short version, the stack, and one diagram of the workspace members. -- [Module map](architecture/module-map.md): one section per member, ordered by source size, with the - files in each that carry the weight. +- [Module map](architecture/module-map.md): one section per member, with the files in each that + carry the weight. - [Data flow](architecture/data-flow.md): the flows (`materialize`, `query`, `analyze`) traced end to end, each as a sequence diagram. @@ -75,11 +75,11 @@ first one, so a citation here resolves or the tree does not ship. ## Documents this tree deliberately omits -- **`analysis/ownership.md`**: the history sits under a single bot identity, - with no human authors. A per-person ranked table and a bus-factor list would be noise dressed as +- **`analysis/ownership.md`**: every commit in the history names either the one maintainer or a + GitHub bot account. A per-person ranked table and a bus-factor list would be noise dressed as analysis. - **`reference/rpc-tools.md`**: there is no RPC, MCP, or HTTP surface. `fastapi`, `uvicorn`, - `starlette`, and the supabase client stack do appear in the installed dependency closure, arriving + `starlette`, and the supabase client stack do appear in the locked dev dependency closure, arriving transitively through harbor, and no first-party module imports any of them. An empty file in either slot would imply absence by omission rather than by silence, which is why diff --git a/docs/analysis/dead-code.md b/docs/analysis/dead-code.md index 995fd28..968568e 100644 --- a/docs/analysis/dead-code.md +++ b/docs/analysis/dead-code.md @@ -1,185 +1,389 @@ # atif-sql · Dead code -Measured 2026-08-28. - -Note (2026-09-27): the trajectory and structural pipelines were removed after this -measurement, and with them these rows below: `TRAJECTORY_DIRNAME`, -`CLUSTERS_FILENAME`, `CLUSTER_TERMS_FILENAME`, `COMMUNITIES_FILENAME`, and -`COMMUNITY_PROFILE_FILENAME` no longer exist in -`packages/atif-analytics/src/atif_analytics/domain/layout.py`. The rows and use sites in -`costs.py`, `transcript.py`, and the atif-models `registry.py` and `settings.py` were re-read on -2026-10-12; the other counts and line numbers haven't been re-measured. - -**Nothing in this workspace is deletable code.** Some public top-level -definitions under `packages/*/src` have no reference -outside their own file, and **none** has no reference anywhere. Every one of -those resolves to a call, construction, or annotation site inside its own module. -What the two tables below record is a narrower defect: an **export declaration -with no consumer**: a name in an `__all__` that nothing imports. +Measured 2026-10-12. + +**Seven public definitions are dead code.** Seven public top-level definitions +under `packages/*/src` are never used: no code in their own module or in any +other calls, reads, or imports them, and five of the seven appear only as a +string in their module's `__all__`. The first table below lists them. Every other public top-level +definition with no reference outside its own file resolves to a use inside its +own module (a call, a construction, an annotation, a default argument, or a +read). What the other two tables record is a narrower defect: an **export +declaration with no consumer**: a name in an `__all__` that nothing imports. **Technique, and what it cannot see.** No code index covers this repo (there is no `.codegraph/`, no LSP index, no AST symbol graph), and no dead-code analyzer is wired into the project: `grep -n "vulture|dead|unused|deptry|knip"` over `pyproject.toml` and `mise.toml` returns -only an unrelated hit, ty's `unused-ignore-comment` at `pyproject.toml:383`. The +only unrelated hits, ty's `unused-ignore-comment` rule at `pyproject.toml:460` +and a comment about it on line 465. The finding set is therefore derived, in these passes: 1. **AST enumeration.** `ast.parse` every file under `packages/*/src/**/*.py`, collecting every top-level `FunctionDef` / `AsyncFunctionDef` / `ClassDef` / - module-level assignment whose name does not begin with `_`, together with its - `__all__` membership, and whether its module declares an `__all__` at - all. + `type` alias / module-level assignment whose name does not begin with `_`, + together with its `__all__` membership, and whether its module declares an + `__all__` at all. 2. **Unanchored reference index.** For each name, the whole-word pattern - `(?/sessions//` tree and exposes it as the stable SQL surface, with raw readers built as `CREATE TEMP TABLE` over `read_json` under an explicit strict-projection `columns` filter -(`packages/atif-duck/src/atif_duck/infrastructure/registry.py:3-31`). Every registration function +(`packages/atif-duck/src/atif_duck/infrastructure/registry.py:3-20`). Every registration function follows a register-or-fail-loud contract: log through `logger.exception` and re-raise on any DDL -failure (`:31-32`, `:816-819`, `:1217-1220`). - -**Recent activity.** Commits in the 30-day window sit well above the median, so `↑ rising` under the -mechanical rule, which here means the file was written and rewritten during the short authoring -run, not that it is destabilizing. - -**Owners.** The bot identity at 100% (every commit touching the path, merges included), -a bot identity. - -**Findings.** Most of the workspace's `B608` sites live here, each carrying a per-line -`# noqa: S608` that names its interpolation: a module-constant table name (`:185`, `:237`), a glob -escaped through `sql_literal` (`:254`, `:277`, `:300`, `:319`), local regex literals (`:743`), an -`int()`-coerced width (`:962`), and the pricing rates (`:1099`). The file also holds one of the -type-checker defects: `_pricing_values_clause` escapes the model name through `sql_literal` and coerces -both rates with `float()` (`:975-1002`, specifically `:999`), because the `pricing` parameter is -reachable by an in-process embedding caller even though the CLI never passes one (`:992-998`). Its -uncovered units cluster in exactly the paths no test drives: `register_views` (`:333-819`), -`register_macros` (`:1005-1220`), `register_vss` (`:846-967`), -`_warn_incomplete_session_dirs` (`:175-193`), and `_pricing_values_clause` itself. The -`register_vss` gap is the load-bearing one: the ATTACH-failure arm that degrades a Lance directory -to an empty-store fallback and logs rather than raising (`:906-910`) is untested silent degradation. +failure (`:62-63`, `:955-958`, `:1527-1530`, `:2072-2075`). + +**Recent activity.** 12 commits in the 30-day window against a median of 1, so `↑ rising` under the +mechanical rule, which here measures how often the file was edited, not whether it is +destabilizing. + +**Owners.** The sole author at 100% (all 14 commits touching the path; the history has no merge +commits). + +**Findings.** More of the workspace's `B608` sites live here than in any other file (28 of 69), each +carrying a per-line `# nosec B608` that names its interpolation: a module-constant reader or table +with catalog columns (`:308`, `:361`), a glob or file list the statement binds as a parameter +(`:728`, `:830`), the corpus name through `sql_literal` (`:975`), local regex literals (`:1345`), +an `int()`-coerced width (`:1737`), an int-derived alias (`:1792`), and the pricing rates +(`:1955`). The file holds no type-checker defect; `_pricing_values_clause` escapes the model name +through `sql_literal` and coerces both rates with `float()` (`:1826-1853`, specifically `:1850`), +because the `pricing` parameter is reachable by an in-process embedding caller even though the CLI +never passes one, and a `str` rate there would get past the type checkers (`:1843-1848`). Its +uncovered units sit in `register_views` (`:984-1530`), `register_macros` (`:1856-2075`), +`register_vss` (`:1601-1742`), `_gate_session_dirs` (`:482-533`), and `_pricing_values_clause` +itself, although tests drive each of those paths: 14 test files call `register`, which runs +`register_views` and `register_macros`, seven call `register_vss`, and +`packages/atif-duck/tests/test_duck_views.py:663` passes a hostile model name through the pricing +clause. The `register_vss` gap is the one that matters most: the ATTACH-failure arm that degrades +a Lance directory to an empty-store fallback and logs rather than raising +(`packages/atif-duck/src/atif_duck/infrastructure/registry.py:1681-1685`) is untested silent +degradation: no test calls `register_vss` on a directory that fails to ATTACH. ### `atif_analytics.infrastructure.parquet_cache` **What's there.** The sharded parquet cache behind the analytics pipelines: `write_part` appends a -new shard, `read_all` unions every shard, and `replace_sessions` drops a session's prior rows so a -re-flush cannot duplicate them -(`packages/atif-analytics/src/atif_analytics/infrastructure/parquet_cache.py:90-111`, `:85-100`, -`:159-221`). `_shard_may_hold` prunes shards by parquet footer statistics and fails open on missing -or unreadable stats (`:116-156`), which is the safe direction. +new shard and `read_all` unions every whole shard +(`packages/atif-analytics/src/atif_analytics/infrastructure/parquet_cache.py:90-111`, +`:114-129`). `iter_part_files` is the one choke point both readers share, and it drops any shard +too small to carry parquet framing (`:71-87`). -**Recent activity.** Commits sit at the median, so `→ flat`. This file ranks second on defect -signals alone with no help from churn, which is the clearest demonstration that the two are -independent here. +**Recent activity.** One commit in the window, at the median, so `→ flat`. Churn adds nothing to +its score. -**Owners.** The bot identity at 100% (every commit touching the path), a bot identity. +**Owners.** The sole author at 100% (both commits touching the path). -**Findings.** No scanner findings, and several of the workspace's unguarded IO windows, all in the -same file. `write_part`'s sharded branch drops `part-.parquet` straight into the live -directory (`:73-74`) that `iter_part_files` globs (`:57`), with no tmp-then-rename; the code +**Findings.** No scanner findings, no complexity ratchet, and one unguarded IO window. +`write_part`'s sharded branch drops `part-.parquet` straight into the live +directory (`:102-103`) that `iter_part_files` globs (`:86`), with no tmp-then-rename; the code anticipates concurrency explicitly, since the nanosecond suffix exists to avoid collisions "when two -part-writes land in the same millisecond under concurrency" (`:70-72`). `read_all` then hands every -globbed part to `pl.read_parquet` with no size or completeness check (`:100`), while -`MIN_PARQUET_BYTES`' own docstring states the size check "is what keeps a torn artifact from failing -a pipeline that could skip it" (`:26-31`) and the legacy single-file branch does apply it (`:78`). -A reader can therefore pick up a shard polars is still writing, and the guard the module documents -protects only the path it is not on. `replace_sessions` compounds it by rewriting shards in place -(`:213`) and unlinking emptied ones (`:209`). The coverage gap lands on the same functions: -a large share of its uncovered units are in `replace_sessions` and `write_part`. It also owns the -`max-returns = 5` ratchet through `_shard_may_hold` (`pyproject.toml:342`, `:337`; the function at -`packages/atif-analytics/src/atif_analytics/infrastructure/parquet_cache.py:145`). +part-writes land in the same millisecond under concurrency" (`:99-101`). The read path does apply +the size guard: `MIN_PARQUET_BYTES`' own docstring states the size check "is what keeps a torn +artifact from failing a pipeline that could skip it" (`:27-32`), `iter_part_files` filters every +globbed part through it (`:87`), and the legacy single-file branch applies it too (`:107`). The +guard catches a stub of 16 bytes or fewer, not a half-written file, so a reader that globs while +polars is still writing a larger shard hands it to `pl.read_parquet` (`:129`) and fails there. ### `atif_cli.app` **What's there.** The cyclopts composition root and the only module importing atif-converter, atif-corpus, and atif-duck together, wiring every cross-package seam: the `ConverterPort` adapter, -the clock, version pins, the DuckDB connection (`packages/atif-cli/src/atif_cli/app.py:3-15`). Heavy +the clock, version pins, the DuckDB connection (`packages/atif-cli/src/atif_cli/app.py:3-9`). Heavy imports are deferred into the command bodies so the `schema` / `--help` fast path stays lean, a -property pinned by a fresh-interpreter test rather than by a lint (`:22-26`). - -**Recent activity.** The most commits of any file in the repo, so `↑ rising`, which is expected of a -composition root that gains a wiring line whenever any member changes. - -**Owners.** The bot identity at 100% (every commit touching the path, merges included), -a bot identity. - -**Findings.** A `B608` at the `search` kNN query, whose `# noqa` records that `dim` is -`len(vector)` while the session id, `k`, and the vector itself are `?`-bound (`:925-937`). It -carries the `max-args = 19` ratchet through `search`, whose parameters are the CLI flags -cyclopts binds (`pyproject.toml:340`, `:333-335`; the function at -`packages/atif-cli/src/atif_cli/app.py:860`). Its uncovered units (the largest single-file gap -in the workspace) concentrate in the commands that reach outward: `analyze` -(`:627-725`), `convert` (`:225-291`), and `status` (`:412-488`), with a few in `search` -(`:828-948`) and `main` (`:1093-1103`). `analyze`, `embed`, and `search` are the -billable commands that call Bedrock, so the least-covered command body in the tree is also the one -that spends money; the uncovered region inside `analyze` is the settings-override chain that -applies the budget ceilings a crontab line depends on (`:691-707`). +property pinned by a fresh-interpreter test rather than by a lint (`:50-53`). + +**Recent activity.** The most commits of any source file (19, 16 of them in the window), so +`↑ rising`, which is expected of a composition root that gains a wiring line whenever any member +changes. + +**Owners.** The sole author at 100% (all 19 commits touching the path). + +**Findings.** No `B608` site: the `search` kNN statement in `search_rows` binds the vector, the +session id, and `k` as parameters and formats only the width `len(query_vector)` and constant +fragments into `_SEARCH_SQL` (`:2201`, `:2218-2225`). It sets no complexity ratchet: `search` takes +seven parameters and the widest command, `analyze`, takes 13, while `max-args = 19` sits at +`atif_corpus.application.materialize.materialize` +(`packages/atif-corpus/src/atif_corpus/application/materialize.py:1497`). Its uncovered units +concentrate in the commands that reach outward: `analyze` +(`packages/atif-cli/src/atif_cli/app.py:1634-1735`), `convert` (`:675-757`), and `status` +(`:1147-1296`), with a few in `search` (`:2018-2176`) and `main` (`:2409-2442`). `analyze` always +calls Bedrock, and `embed` and `search` do with the default Cohere provider, so the least-covered +command body in the tree is also the one that spends money; the uncovered region inside `analyze` +is the settings-override chain that applies the budget ceilings a crontab line depends on +(`:1700-1714`). ### `atif_converter.domain.enrichment` @@ -149,12 +153,13 @@ exists because harbor 0.22.0 puts no record identity in `step.extra`: it reads ` message dict and `id` off the event dict, while real transcripts carry both on the event, so nothing lands in extra and there is nothing to join on (`:15-20`). -**Recent activity.** Commits well above the median, so `↑ rising`. +**Recent activity.** One commit in its history and none in the window, below the median, so +`→ flat`. -**Owners.** The bot identity at 100% (every commit touching the path), a bot identity. +**Owners.** The sole author at 100% (the one commit touching the path). -**Findings.** No scanner findings, no IO windows (this is a pure function), and both -complexity ratchets in the workspace that a single function sets: `max-branches = 39` and +**Findings.** No scanner findings, no IO windows (this is a pure function), and two of the +workspace's complexity ratchets set by one function: `max-branches = 39` and `max-complexity = 38` are both pinned at `enrich_trajectory` (`pyproject.toml:341`, `:336`, `:345-350`; the function at `packages/atif-converter/src/atif_converter/domain/enrichment.py:198`). Each branch is one named @@ -173,9 +178,9 @@ fast path at confidence 0.9, deterministic stamp rules, and an LLM tier for ever column is the row's provenance tier rather than the engine that computed it, and downstream atif-duck views bind to those literal values, so the vocabulary is a fixed contract (`:34-38`). -**Recent activity.** Commits well above the median, so `↑ rising`. +**Recent activity.** Two commits in the window, under the four the rule needs, so `→ flat`. -**Owners.** The bot identity at 100% (every commit touching the path), a bot identity. +**Owners.** The sole author at 100% (all three commits touching the path). **Findings.** No scanner findings, and the `max-statements = 119` ratchet, set at `_friction_async` (`pyproject.toml:343`, `:338`; the function at @@ -183,11 +188,11 @@ views bind to those literal values, so the vocabulary is a fixed contract (`:34- gap and the complexity outlier are the same region: most of the file's uncovered units sit inside `_friction_async` (`:210-498`), with the rest in `detect_user_friction` (`:501-606`) and `deterministic_stamps` (`:146-202`). This is the file where the two signals coincide most tightly: -the longest function in the workspace is also the least-exercised, and it is the function that -decides which candidate messages cross into the billable LLM tier. The budget-guard posture that -governs that decision assumes every candidate reaches the LLM even though roughly half survive the -fast tiers (`:554-556`), and the third-party disclosure is explicit: short user message bodies leave -the machine on tier 4 (`:40-41`). +the function with the most statements in the workspace is also the least-exercised, and it is the +function that decides which candidate messages cross into the billable LLM tier. The budget-guard +posture that governs that decision assumes every candidate reaches the LLM even though roughly half +survive the fast tiers (`:554-556`), and the third-party disclosure is explicit: short user message +bodies leave the machine on tier 4 (`:40-41`). ## See also diff --git a/docs/architecture/data-flow.md b/docs/architecture/data-flow.md index 83047ad..37b1d2b 100644 --- a/docs/architecture/data-flow.md +++ b/docs/architecture/data-flow.md @@ -1,18 +1,19 @@ # atif-sql · Data flow The distribution declares exactly one entry point, the console script -`atif-sql = "atif_cli.app:main"` (`packages/atif-cli/pyproject.toml:42`), and the `main` it names -installs a WARNING-and-up loguru sink and hands control to cyclopts (`packages/atif-cli/src/atif_cli/app.py:1125`). +`atif-sql = "atif_cli.app:main"` (`pyproject.toml:75`), and the `main` it names +installs a WARNING-and-up loguru sink and hands control to cyclopts (`packages/atif-cli/src/atif_cli/app.py:2409`). Every process below therefore begins as a CLI invocation; there is no HTTP, RPC, or queue surface to enter through. The flows below either produce the corpus or consume it: `materialize` fills -`/sessions/` (`:339`), `analyze` fills `/analytics/` (`:627`), and -`query` binds both and runs caller SQL over them (`:497`). The other commands are subsets of -these flows: `convert` (`:225`) is one iteration of flow 1's inner loop, `search` (`:828`) -re-enters flow 2's registration at `:886` and adds one kNN statement, `embed` (`:734`) writes the -vector store flows 2 and 3 read, and `schema` (`:1055`), `examples` (`:963`), and `status` -(`:412`) answer from static data or `stat` calls with no downstream participant. +`/sessions/` (`:863`), `analyze` fills `/analytics/` (`:1634`), and +`query` binds both and runs caller SQL over them (`:1305`). The other commands are subsets of +these flows: `convert` (`:675`) is one iteration of flow 1's inner loop, `search` (`:2018`) +re-enters flow 2's registration at `:2116` and adds one kNN statement, `embed` (`:1801`) writes the +vector store that flow 2 and `search` read, `schema` (`:2335`) and `examples` (`:2243`) answer +from static catalog data, and `status` (`:1147`) replays materialize's plan and reads the lake's +vitals without writing anything. `materialize` also keeps the DuckLake beside the corpora current (flow 1, step 10), and `query` reads it when it can (flow 2, step 4). `lake rebuild` loads it from every corpus's artifacts and @@ -25,38 +26,38 @@ catalog copy. 1. The `materialize` command resolves `CorpusSettings` (pydantic-settings, env prefix `ATIF_SQL_`), then injects what the pure use case will not own: the `ConverterPort` adapter, the wall-clock instant, and the harbor / converter version pins - stamped into every `meta.json` (`packages/atif-cli/src/atif_cli/app.py:352`). + stamped into every `meta.json` (`packages/atif-cli/src/atif_cli/app.py:958-971`). 2. One pass through the corpus use case runs scan, plan, convert, and write in that order and - returns a `MaterializationReport` (`packages/atif-corpus/src/atif_corpus/application/materialize.py:476`). + returns a `MaterializationReport` (`packages/atif-corpus/src/atif_corpus/application/materialize.py:1497`). 3. The scanner discovers every session under the raw transcript root and separates unreadable - sessions from absent ones, so a `stat` failure is never mistaken for a deletion (`packages/atif-corpus/src/atif_corpus/infrastructure/scanner.py:143`). It walks whichever layout the + sessions from absent ones, so a `stat` failure is never mistaken for a deletion (`packages/atif-corpus/src/atif_corpus/infrastructure/scanner.py:206`). It walks whichever layout the agent selects, descending exactly `transcript_depth` directories and reporting an unlistable one at - any level (`packages/atif-corpus/src/atif_corpus/infrastructure/scanner.py:154`). + any level (`packages/atif-corpus/src/atif_corpus/infrastructure/scanner.py:176`). 4. The pure planner partitions the scan into to-materialize, up-to-date, and skipped-live using the previous watermark and the quiescence policy; `force` overrides staleness but never - liveness (`packages/atif-corpus/src/atif_corpus/domain/sessions.py:159`). + liveness (`packages/atif-corpus/src/atif_corpus/domain/sessions.py:169`). 5. For each planned session the use case calls `converter.convert` across the port; one session's - exception is caught and recorded so a single bad transcript cannot abort the sync (`packages/atif-corpus/src/atif_corpus/application/materialize.py:310`). + exception is caught and recorded so a single bad transcript cannot abort the sync (`packages/atif-corpus/src/atif_corpus/application/materialize.py:793`). By default the sessions are spread over a process pool (`--workers`, default `min(8, cpu_count)`) - whose workers each hold a pickled copy of the adapter; the pool changes no output byte (`:385`). + whose workers each hold a pickled copy of the adapter; the pool changes no output byte (`:931`). 6. The adapter that satisfies `ConverterPort` lives in atif-cli because the independence contract forbids atif-corpus from importing atif-converter; it raises rather than returning an invalid - trajectory (`packages/atif-cli/src/atif_cli/converter_adapter.py:51`). + trajectory (`packages/atif-cli/src/atif_cli/converter_adapter.py:129-130`). 7. Under `--agent codex` the same step runs `convert_codex_and_audit`, which reads the one rollout it - is given and converts it with our ported Codex converter (`packages/atif-converter/src/atif_converter/application/convert_codex.py:128`). + is given and converts it with our ported Codex converter (`packages/atif-converter/src/atif_converter/application/convert_codex.py:144`). 8. `convert_and_audit` reads the source files once (fingerprinted and parsed in the same pass, `harbor_adapter.read_session`), converts those records with our ported converter through the seam at `packages/atif-converter/src/atif_converter/infrastructure/harbor_adapter.py` - (`convert_loaded_session`, validated by harbor's public `TrajectoryValidator`), then builds + (`convert_loaded_session`, validated by the `TrajectoryValidator` vendored from harbor), then builds the loss report and edges from the same records, enriches the trajectory, and refuses the result - if any source moved since the read (`packages/atif-converter/src/atif_converter/application/convert_and_audit.py:105`). -9. The artifacts are written under `.staging/`: `trajectory.json.zst`, `edges.jsonl.zst` and - `session_events.jsonl.zst` (zstd, each decompressing to the plain file an earlier version - wrote), then `loss_report.json`. No per-session parquet is written; the CLI wires no + if any source moved since the read (`packages/atif-converter/src/atif_converter/application/convert_and_audit.py:220`). +9. The artifacts are written under `.staging/`: `trajectory.json.zst`, `loss_report.json`, + `edges.jsonl.zst` and `session_events.jsonl.zst` (the three `.zst` files are zstd, each + decompressing to the plain file an earlier version wrote). No per-session parquet is written; the CLI wires no `ArtifactProducer`. `meta.json` is written last and the whole directory swaps into `sessions//`, so a reader sees one complete generation or the other - (`packages/atif-corpus/src/atif_corpus/application/materialize.py:240`); the watermark advances - only for sessions that succeeded (`:608`). + (`packages/atif-corpus/src/atif_corpus/application/materialize.py:706-733`); the watermark advances + only for sessions that succeeded (`:1778`). 10. Before the first swap, the pass records every session it may publish or mark in `/sink_pending.json`. After the swaps, still in the parent process, it hands the `SessionSink` (atif-duck's `DuckLakeSessionSink`, plugged in by the CLI unless `--no-lake`) every @@ -78,7 +79,6 @@ sequenceDiagram participant CLI as atif-cli participant Corpus as atif-corpus participant Conv as atif-converter - participant Harbor as harbor participant Producer as atif-duck producer participant Disk as corpus disk participant Sink as atif-duck lake sink @@ -90,8 +90,7 @@ sequenceDiagram Corpus->>Corpus: build_plan -> stale / current / live loop each planned session Corpus->>Conv: ConverterPort.convert(session_jsonl) - Conv->>Harbor: pinned private converter(session_dir) - Harbor-->>Conv: ATIF trajectory + Conv->>Conv: read_session, convert_loaded_session, vendored TrajectoryValidator Conv-->>Corpus: ConversionOutput Corpus->>Disk: stage trajectory.json.zst, loss report, edges.jsonl.zst, events Corpus->>Disk: meta.json last, swap dir @@ -112,12 +111,12 @@ sequenceDiagram 1. The `query` command resolves the corpus root and reads the active embedder's `(model_id, dim)` without constructing the embedder, so the vector store's stamped identity can be checked before - anything binds over it (`packages/atif-cli/src/atif_cli/app.py:529`). + anything binds over it (`packages/atif-cli/src/atif_cli/app.py:1463`). 2. A DuckDB connection is opened with no path or URI, so the engine runs in-process against - memory (`:591`). + memory (`:1476`). 3. Registration builds the whole catalog on that connection in a fixed order (raw TEMP tables, base views, VSS, macros, the authorship macro and views, analytics views, analytics macros), - because each later stage binds against the earlier one at `CREATE` time (`packages/atif-duck/src/atif_duck/infrastructure/registry.py:1679`). + because each later stage binds against the earlier one at `CREATE` time (`packages/atif-duck/src/atif_duck/infrastructure/registry.py:2172-2193`). 4. When the DuckLake at `ATIF_SQL_LAKE_ROOT` exists, its recorded schema matches the running code, and it holds this corpus at this root, the raw relations are views over the lake tables instead, filtered to this corpus (`--all-corpora` drops the filter): the command loads the ducklake @@ -135,15 +134,15 @@ sequenceDiagram (DuckDB's JSON reader decompresses a `.zst` file itself) into a TEMP TABLE over an explicit path list; the two sets are unioned into one raw view per surface. The parse cost of a `query` invocation is therefore O(sessions - without artifacts) per connection (`packages/atif-duck/src/atif_duck/infrastructure/registry.py:402`). -6. Trajectory, edges, and loss readers are semi-joined against the meta table, so a session - directory missing its `meta.json` contributes nothing rather than a partial artifact set. This - is the read-side half of flow 1's meta-last write ordering (`packages/atif-duck/src/atif_duck/infrastructure/registry.py:221`). -7. The VSS step `ATTACH`es the LanceDB store through the lance extension (`packages/atif-duck/src/atif_duck/infrastructure/registry.py:887`) and reads the - store's stamped `model` and `dim` back out (`packages/atif-duck/src/atif_duck/infrastructure/registry.py:921`) before creating any view over it; a store + without artifacts) per connection (`packages/atif-duck/src/atif_duck/infrastructure/registry.py:597`). +6. The edges and loss readers are semi-joined against the meta table, and the trajectory path list + is built from it, so a session directory missing its `meta.json` contributes nothing rather than + a partial artifact set. This is the read-side half of flow 1's meta-last write ordering (`packages/atif-duck/src/atif_duck/infrastructure/registry.py:709`). +7. The VSS step `ATTACH`es the LanceDB store through the lance extension (`packages/atif-duck/src/atif_duck/infrastructure/registry.py:1678`) and reads the + store's stamped `model` and `dim` back out (`packages/atif-duck/src/atif_duck/infrastructure/registry.py:1712`) before creating any view over it; a store written by a different provider or width raises instead of binding, because vectors from different models live in incompatible spaces and would return numerically valid but meaningless - cosine scores (`packages/atif-duck/src/atif_duck/domain/embedding_guard.py:46`). + cosine scores (`packages/atif-duck/src/atif_duck/domain/embedding_guard.py:78`). 8. Before any of that, the connection was sized to the host (`_configure_query_resources`: a memory cap from available RAM, a thread count from that cap, a private `mkdtemp` spill directory, and extension auto-install and auto-load off), because registration is what needs the cap. The @@ -157,7 +156,7 @@ sequenceDiagram `EXPORT`, `ATTACH`, `DETACH`, `INSTALL`, `LOAD`, `PREPARE` and `EXECUTE` exit 70 (`sandbox_refused`) before anything runs. Then the caller's statement executes and the cursor drains in batches to stdout: a JSON array of row objects on a pipe, a width-aligned table on a - TTY (`packages/atif-cli/src/atif_cli/output.py:154`). The spill directory is removed when the + TTY (`packages/atif-cli/src/atif_cli/output.py:158`). The spill directory is removed when the process exits. ```mermaid @@ -192,26 +191,26 @@ sequenceDiagram 1. The `analyze` command loads `AnalyticsSettings`, reuses the same corpus-root resolution the other commands share, and stamps explicit `--max-sessions` / `--max-cost-usd` ceilings over the - env defaults so a crontab line carries its spend cap visibly (`packages/atif-cli/src/atif_cli/app.py:1240`). + env defaults so a crontab line carries its spend cap visibly (`packages/atif-cli/src/atif_cli/app.py:1709-1714`). 2. The command opens the lake's session source when the lake holds the corpus - (`packages/atif-cli/src/atif_cli/app.py:1624`, `packages/atif-cli/src/atif_cli/lake_sessions.py:83`), + (`packages/atif-cli/src/atif_cli/app.py:1716`, `packages/atif-cli/src/atif_cli/lake_sessions.py:111`), and the pipeline runner builds one shared `CorpusReader` over it (or over the file source) for every stage and loops the LLM stages (classify, conflicts, friction, perceived); the `skip_*` flags subtract stages (`packages/atif-analytics/src/atif_analytics/application/analyze.py:62`). 3. On the lake, a session whose lake rows match its current `meta.json` is read with batched - statements over the step-level tables (`packages/atif-duck/src/atif_duck/infrastructure/lake_sessions.py:71`), + statements over the step-level tables (`packages/atif-duck/src/atif_duck/infrastructure/lake_sessions.py:80`), reading ahead in the newest-first walk order, and every other session from its files. Without a lake, rows are read with stdlib `json` over `/sessions//trajectory.json.zst` (or the plain file on an old-layout session); - atif-analytics never imports DuckDB or atif-duck, which the `forbidden` import contract puts - out of its reach (`packages/atif-analytics/src/atif_analytics/infrastructure/corpus_reader.py:238`). + atif-analytics declares no DuckDB dependency, and the `forbidden` import contract puts atif-duck + out of its reach (`packages/atif-analytics/src/atif_analytics/infrastructure/corpus_reader.py:238`, `pyproject.toml:611`). Either way the steps sit behind the reader's bounded memos (`packages/atif-analytics/src/atif_analytics/infrastructure/corpus_reader.py:317`), and the reader labels each session's kind with the same rule `session_outcomes.kind` applies, so classify and conflicts can skip `turn_audit` and `one_shot_job` sessions. 4. Before the first stage starts, one `RunBudget` is constructed from the per-run dollar ceiling - (`packages/atif-analytics/src/atif_analytics/application/analyze.py:83`); every stage then goes + (`packages/atif-analytics/src/atif_analytics/application/analyze.py:89`); every stage then goes through one call site, each receiving the shared reader and that budget, and a stage that finds - the budget exhausted is skipped with nothing stamped (`:123`). + the budget exhausted is skipped with nothing stamped (`:112`). 5. A stage drops the sessions whose checkpoint row still matches their mtime and last step timestamp, so a re-run costs nothing for unchanged work (`packages/atif-analytics/src/atif_analytics/infrastructure/sqlite_state/checkpointer.py:137`). 6. Each call's user message is the session transcript (friction sends one short message instead) diff --git a/docs/architecture/module-map.md b/docs/architecture/module-map.md index 628ce7f..4cf11af 100644 --- a/docs/architecture/module-map.md +++ b/docs/architecture/module-map.md @@ -5,37 +5,39 @@ both the workspace and the one published distribution: its `[project]` table bui `atif-sql` wheel carrying all seven module trees, its dependency list is the union of the members' third-party requirements, and it also holds the shared dev dependency-group and the shared tool config (`pyproject.toml:1`, `:16`, `:34`). The internal import graph is a -star: atif-cli declares its siblings as `==`-pinned dependencies -(`packages/atif-cli/pyproject.toml:32`), plus one further edge from atif-analytics to atif-models that -a `forbidden` import-linter contract leaves open (`pyproject.toml:608`), with every other pair closed -by an `independence` contract (`pyproject.toml:603`). Every cross-package import sits inside a -function body or a `TYPE_CHECKING` block so the CLI's fast path pulls in no duckdb, harbor, lancedb, -boto3, or polars, which is why `PLC0415` is ignored workspace-wide (`pyproject.toml:227`). Modules -below are ordered by total source size, descending. +star: atif-cli declares five of its six siblings as `==`-pinned dependencies +(`packages/atif-cli/pyproject.toml:32-36`) and reaches atif-models only through atif-analytics, +plus one further edge from atif-analytics to atif-models that a `forbidden` import-linter contract +leaves open while it closes every other atif-analytics edge (`pyproject.toml:608`), with every pair +among the other five members closed by an `independence` contract (`pyproject.toml:603`). Outside +atif-cli's adapter modules, which `atif_cli.app` imports only inside command bodies, every +cross-package import sits inside a function body or a `TYPE_CHECKING` block, so the CLI's fast path +pulls in no duckdb, harbor, lancedb, boto3, or polars, which is why `PLC0415` is ignored +workspace-wide (`pyproject.toml:227`). ## atif-analytics `run_analyze` composes the LLM pipelines in a fixed stage order: classify, conflicts, friction, -perceived (`packages/atif-analytics/src/atif_analytics/application/analyze.py:36`). Every stage calls +perceived (`packages/atif-analytics/src/atif_analytics/application/analyze.py:37`). Every stage calls a model and honors `dry_run`, which defaults to True as a cost guard so the stages return plan dicts instead of spending (`:13`); each plan turns a measured prompt length into tokens at the resolved model's own `chars_per_token` (`packages/atif-analytics/src/atif_analytics/domain/costs.py:28`). Every classifier system prompt lives in one module, each assembled by joining a task body to shared sections -(`packages/atif-analytics/src/atif_analytics/application/prompts.py:714`), and the same text goes to +(`packages/atif-analytics/src/atif_analytics/application/prompts.py:714-723`), and the same text goes to both model families. The user message comes from the same module: the transcript, or friction's one message, fenced in a tag pair whose id is a digest of the text, then the one-line request (`:742`, `:753`). The pydantic v2 response schemas they bind against are pure domain models whose field descriptions are themselves part of the prompt surface (`packages/atif-analytics/src/atif_analytics/domain/models.py:3`). Who wrote a user step comes from `atif_analytics.domain.authorship`, a pinned twin of atif-duck's rule -table (`packages/atif-analytics/src/atif_analytics/domain/authorship.py`). This is the one member -permitted to import a sibling: atif-models and nothing else (`pyproject.toml:608`). +table (`packages/atif-analytics/src/atif_analytics/domain/authorship.py`). Apart from atif-cli, this +is the one member permitted to import a sibling: atif-models and nothing else (`pyproject.toml:608`). - `packages/atif-analytics/src/atif_analytics/application/prompts.py`: the task-framing system prompts, public constants assembled at - `packages/atif-analytics/src/atif_analytics/application/prompts.py:714`, the request lines - (`:726`), and the two user-message builders, `transcript_prompt` (`:742`) and `message_prompt` + `packages/atif-analytics/src/atif_analytics/application/prompts.py:714-723`, the request lines + (`:726-729`), and the two user-message builders, `transcript_prompt` (`:742`) and `message_prompt` (`:753`). - `packages/atif-analytics/src/atif_analytics/application/use_cases/friction.py`: the friction tiers behind a human-turn and message-length pre-filter: regex fast path, deterministic @@ -56,7 +58,7 @@ permitted to import a sibling: atif-models and nothing else (`pyproject.toml:608 length, so what a tool returned stays on disk and never reaches Bedrock (`packages/atif-analytics/src/atif_analytics/domain/transcript.py:247`). `ATIF_SQL_SESSION_TEXT_TOOL_RESULT_BODIES=true` puts the bodies back, each clipped at 50,000 - characters (`packages/atif-analytics/src/atif_analytics/domain/config.py:21`). + characters (`packages/atif-analytics/src/atif_analytics/infrastructure/settings.py:94`, `:90`). - `packages/atif-analytics/src/atif_analytics/domain/models.py`: the response schemas, from `SessionClassification` (`packages/atif-analytics/src/atif_analytics/domain/models.py:22`) to `PerceivedErrorsResult` (`:355`). @@ -65,19 +67,19 @@ permitted to import a sibling: atif-models and nothing else (`pyproject.toml:608 `register(con, corpus_root)` binds a DuckDB connection to a contract-shaped corpus tree and exposes the core views and macros (`packages/atif-duck/src/atif_duck/domain/catalog.py:30`, -`packages/atif-duck/src/atif_duck/infrastructure/registry.py:1679`). Those names are not introspected +`packages/atif-duck/src/atif_duck/infrastructure/registry.py:2095`). Those names are not introspected at runtime: a static catalog answers `atif-sql schema` in under 50 ms with no DuckDB bind, and two drift tests assert it column-for-column against `DESCRIBE` and signature-for-signature against the DDL (`packages/atif-duck/src/atif_duck/domain/catalog.py:3`). The analytics views and macros register separately, each only when its backing parquet is populated, because a corpus with no `atif-sql analyze` run is the default state -(`packages/atif-duck/src/atif_duck/infrastructure/analytics.py:119`). It is the one member with -`domain/` and `infrastructure/` but no `application/`, a deliberate shape that is why it carries no -layers contract among the members (`pyproject.toml:550`). +(`packages/atif-duck/src/atif_duck/infrastructure/analytics.py:119`). It has `domain/` and +`infrastructure/` but no `application/`, and unlike atif-models, the other member with that shape, +it carries no layers contract (`pyproject.toml:550`). -- `packages/atif-duck/src/atif_duck/infrastructure/registry.py`: the raw readers, most - of the core views and macros, and the Lance attach path - (`packages/atif-duck/src/atif_duck/infrastructure/registry.py:767`). +- `packages/atif-duck/src/atif_duck/infrastructure/registry.py`: the raw readers + (`packages/atif-duck/src/atif_duck/infrastructure/registry.py:597`), most of the core views + (`:984`) and macros (`:1856`), and the Lance attach path (`:1601`). - `packages/atif-duck/src/atif_duck/domain/raw_readers.py`: the raw relation names and column shapes the registry binds and the lake stores, so neither declares them twice. - `packages/atif-duck/src/atif_duck/domain/lake.py`: the lake's tables, partition specs and @@ -96,10 +98,11 @@ layers contract among the members (`pyproject.toml:550`). the uuids `ducklake_table_changes` names after a snapshot, and the corpus's lineage and snapshot range. - `packages/atif-duck/src/atif_duck/infrastructure/lake_settings.py`: `LakeSettings`, the - `ATIF_SQL_`-prefixed lake root, corpus base, batch sizes, lock timeout and snapshot expiry. + `ATIF_SQL_`-prefixed lake root, corpus base, batch sizes, stage worker count, lock timeout and + snapshot expiry. - `packages/atif-duck/src/atif_duck/domain/catalog.py`: `VIEW_NAMES` - (`packages/atif-duck/src/atif_duck/domain/catalog.py:30`), `MACRO_SIGNATURES` (`:287`), the - analytics catalogs (`:314`), and one `DESCRIPTIONS` entry per object (`:431`). + (`packages/atif-duck/src/atif_duck/domain/catalog.py:30`), `MACRO_SIGNATURES` (`:343`), the + analytics catalogs (`:370-462`), and one `DESCRIPTIONS` entry per object (`:487`). - `packages/atif-duck/src/atif_duck/infrastructure/analytics.py`: the v2 views and macros over the analytics parquet outputs, gated on file presence (`packages/atif-duck/src/atif_duck/infrastructure/analytics.py:119`). @@ -108,7 +111,7 @@ layers contract among the members (`pyproject.toml:550`). (`packages/atif-duck/src/atif_duck/domain/examples.py:191`). - `packages/atif-duck/src/atif_duck/domain/authorship.py`: the `step_author` rule table: prefix matches after stripping leading whitespace, first rule wins, twinned in atif-analytics - (`packages/atif-duck/src/atif_duck/domain/authorship.py:50`). + (`packages/atif-duck/src/atif_duck/domain/authorship.py:51`). - `packages/atif-duck/src/atif_duck/infrastructure/authorship.py`: the `step_author` macro and the `user_steps`, `human_turns`, and `session_outcomes` views, registered after the core macros and before the analytics surface @@ -117,30 +120,31 @@ layers contract among the members (`pyproject.toml:550`). dimension guard, a deliberate twin of atif-embed's copy because the independence contract forbids sharing it (`packages/atif-duck/src/atif_duck/domain/embedding_guard.py:5`). - `packages/atif-duck/src/atif_duck/__init__.py`: re-exports the catalog constants and the - `register*` entry points (`packages/atif-duck/src/atif_duck/__init__.py:30`). + `register*` entry points (`packages/atif-duck/src/atif_duck/__init__.py:17-30`). - `packages/atif-duck/src/atif_duck/domain/__init__.py`: the pure catalog layer, no duckdb import (`packages/atif-duck/src/atif_duck/domain/__init__.py:5`). -- `packages/atif-duck/src/atif_duck/infrastructure/__init__.py`: the registry layer's - re-export surface (`packages/atif-duck/src/atif_duck/infrastructure/__init__.py:5`). +- `packages/atif-duck/src/atif_duck/infrastructure/__init__.py`: the infrastructure layer's + re-export surface, the registry entry points plus the columnar producer + (`packages/atif-duck/src/atif_duck/infrastructure/__init__.py:10-22`). ## atif-cli -The composition root: it declares its siblings as `==`-pinned dependencies -(`packages/atif-cli/pyproject.toml:32`) and wires every cross-package seam: the `ConverterPort` -adapter, the clock, version pins, the DuckDB connection (`packages/atif-cli/src/atif_cli/app.py:5`). +The composition root: it declares five of its six siblings as `==`-pinned dependencies +(`packages/atif-cli/pyproject.toml:32-36`) and wires every cross-package seam: the `ConverterPort` +adapter, the clock, version pins, the DuckDB connection (`packages/atif-cli/src/atif_cli/app.py:7-9`). The commands hang off one cyclopts `App`: `@app.command` functions from `convert` -(`packages/atif-cli/src/atif_cli/app.py:226`) to `schema` (`:1055`), plus the `cron` sub-app -registered at `:65`, with `main` (`:1093`) exposed as the single console script named `atif-sql` -(`packages/atif-cli/pyproject.toml:42`). Heavy imports (duckdb, harbor through atif-converter, -pydantic through atif-corpus) are deferred into the command bodies that use them so `schema`, -`--help`, and `--version` stay on a lean import graph that a fresh-interpreter test pins -(`packages/atif-cli/src/atif_cli/app.py:22`). Failures resolve to stable exit codes, 64 for parse, 65 -for catalog, 70 for runtime, split between a pure taxonomy module -(`packages/atif-cli/src/atif_cli/errors.py:25`) and a driver-dependent classifier -(`packages/atif-cli/src/atif_cli/duck_errors.py:34`) so the lean path never imports duckdb. +(`packages/atif-cli/src/atif_cli/app.py:675`) to `schema` (`:2335`), plus the `cron`, `lake`, and +`corpus` sub-apps registered at `:104-109`, with `main` (`:2409`) exposed as the single console +script named `atif-sql` (`pyproject.toml:75`). Heavy imports (duckdb, the ATIF models through +atif-converter, pydantic through atif-corpus) are deferred into the command bodies that use them so +`schema`, `--help`, and `--version` stay on a lean import graph that a fresh-interpreter test pins +(`packages/atif-cli/src/atif_cli/app.py:50-53`). Failures resolve to stable exit codes, 64 for parse, +65 for catalog, 70 for runtime, split between a pure taxonomy module +(`packages/atif-cli/src/atif_cli/errors.py:30`) and a driver-dependent classifier +(`packages/atif-cli/src/atif_cli/duck_errors.py:35`) so the lean path never imports duckdb. - `packages/atif-cli/src/atif_cli/app.py`: the commands and every wiring decision - between them (`packages/atif-cli/src/atif_cli/app.py:52`). + between them (`packages/atif-cli/src/atif_cli/app.py:3`). - `packages/atif-cli/src/atif_cli/lake.py`: the `lake` sub-app: `rebuild`, `verify`, `status` and `compact`, each deferring atif-duck's imports into the command body. - `packages/atif-cli/src/atif_cli/corpus.py`: the `corpus` sub-app: `slim` converts an @@ -151,16 +155,16 @@ for catalog, 70 for runtime, split between a pure taxonomy module writes one; `cron status` reports per-lane lock and last-run state from injected probes (`packages/atif-cli/src/atif_cli/cron.py:7`). - `packages/atif-cli/src/atif_cli/output.py`: `--format auto` resolves to a table on a TTY - and JSON on a pipe (`packages/atif-cli/src/atif_cli/output.py:74`); `emit_cursor` streams - `fetchmany` batches so the client holds one batch (`:154`). + and JSON on a pipe (`packages/atif-cli/src/atif_cli/output.py:84`); `emit_cursor` streams + `fetchmany` batches so the client holds one batch (`:158`). - `packages/atif-cli/src/atif_cli/duck_errors.py`: the duckdb exception classifier, plus a widened caught set for the registration path since `EmbeddingProviderMismatch` is not a - `duckdb.Error` (`packages/atif-cli/src/atif_cli/duck_errors.py:31`). + `duckdb.Error` (`packages/atif-cli/src/atif_cli/duck_errors.py:32`). - `packages/atif-cli/src/atif_cli/converter_adapter.py`: `RealConverter` adapts - atif-converter's use case to atif-corpus's port, the one place allowed to import both - (`packages/atif-cli/src/atif_cli/converter_adapter.py:44`). + atif-converter's two use cases, one per agent, to atif-corpus's port, the one place allowed to + import both (`packages/atif-cli/src/atif_cli/converter_adapter.py:80`). - `packages/atif-cli/src/atif_cli/errors.py`: `EXIT_CODES` and the `ClassifiedError` shape, - pure and free of any `atif_*` import (`packages/atif-cli/src/atif_cli/errors.py:52`). + pure and free of any `atif_*` import (`packages/atif-cli/src/atif_cli/errors.py:30`, `:70`). - `packages/atif-cli/src/atif_cli/__main__.py`: `python -m atif_cli` reaching the same `main` (`packages/atif-cli/src/atif_cli/__main__.py:5`). - `packages/atif-cli/src/atif_cli/__init__.py`: package docstring only @@ -168,9 +172,10 @@ for catalog, 70 for runtime, split between a pure taxonomy module ## atif-embed -The vector-search write path: Cohere Embed v4 on Bedrock behind `EmbeddingProvider`, a local LanceDB -store behind `VectorStorePort`, and a corpus reader behind `TextRowsPort` -(`packages/atif-embed/src/atif_embed/domain/ports.py:31`). `run_backfill` anti-joins step text against +The vector-search write path: an embedder behind `EmbeddingProvider` (Cohere Embed v4 on Bedrock by +default, or EmbeddingGemma 2 on this machine), a local LanceDB store behind `VectorStorePort`, and a +corpus reader behind `TextRowsPort` +(`packages/atif-embed/src/atif_embed/domain/ports.py:37`). `run_backfill` anti-joins step text against the store's uuid-to-text-hash map, embeds the misses, and bounds loss in layers: chunked discovery, mid-run checkpoints, and per-batch isolation inside a chunk (`packages/atif-embed/src/atif_embed/application/embed.py:61`). Stamps keep the vector space @@ -179,8 +184,9 @@ provider switch fails loud instead of corrupting kNN (`packages/atif-embed/src/atif_embed/domain/embedding_guard.py:3`), and `text_hash` of the exact text a row was built from, so a re-conversion under a stable uuid reads as stale (`packages/atif-embed/src/atif_embed/domain/text_stamp.py:3`). Additive schema columns migrate online -through a metadata-only `add_columns`, keyed on a `SCHEMA_VERSION` sidecar file -(`packages/atif-embed/src/atif_embed/infrastructure/lance_store.py:61`). Discovery reads the lake's +through a metadata-only `add_columns` when a column is missing +(`packages/atif-embed/src/atif_embed/infrastructure/lance_store.py:166`), and the store records its +`SCHEMA_VERSION` in a sidecar file (`:67`). Discovery reads the lake's `steps` through `LakeStepsPort` when a lake holds the corpus, only the rows changed since a watermark kept in the store directory (`packages/atif-embed/src/atif_embed/infrastructure/lake_text_rows.py`), and falls back to the @@ -191,19 +197,19 @@ per-session reader otherwise. (`packages/atif-embed/src/atif_embed/infrastructure/lance_store.py:195`). - `packages/atif-embed/src/atif_embed/infrastructure/cohere_bedrock.py`: the `invoke_model` adapter under a tenacity retry with botocore retries off, and the document-`int8` / query-`float` - asymmetry (`packages/atif-embed/src/atif_embed/infrastructure/cohere_bedrock.py:284`). + asymmetry (`packages/atif-embed/src/atif_embed/infrastructure/cohere_bedrock.py:285`). - `packages/atif-embed/src/atif_embed/application/embed.py`: `discover_unembedded` (`packages/atif-embed/src/atif_embed/application/embed.py:43`), `run_backfill` (`:61`), and the sync - `embed_query` the search command calls (`:265`). + `embed_query` the search command calls (`:280`). - `packages/atif-embed/src/atif_embed/infrastructure/corpus_text_rows.py`: reads the contract corpus layout with its own DuckDB connection, since importing atif-duck is forbidden - (`packages/atif-embed/src/atif_embed/infrastructure/corpus_text_rows.py:155`). -- `packages/atif-embed/src/atif_embed/domain/ports.py`: the Protocols: - `EmbeddingProvider` (`packages/atif-embed/src/atif_embed/domain/ports.py:31`), `VectorStorePort` - (`:59`), `TextRowsPort` (`:87`). + (`packages/atif-embed/src/atif_embed/infrastructure/corpus_text_rows.py:199`). +- `packages/atif-embed/src/atif_embed/domain/ports.py`: the port Protocols: + `EmbeddingProvider` (`packages/atif-embed/src/atif_embed/domain/ports.py:37`), `VectorStorePort` + (`:65`), `TextRowsPort` (`:93`), and `LakeStepsPort` (`:202`). - `packages/atif-embed/src/atif_embed/domain/embedding_guard.py`: `ensure_store_matches`, the fail-loud provider and dimension rule - (`packages/atif-embed/src/atif_embed/domain/embedding_guard.py:46`). + (`packages/atif-embed/src/atif_embed/domain/embedding_guard.py:38`). - `packages/atif-embed/src/atif_embed/domain/errors.py`: the `DomainError` taxonomy the CLI catches, with `terminal` separating operator-needed states from retryable ones (`packages/atif-embed/src/atif_embed/domain/errors.py:14`). @@ -214,37 +220,38 @@ per-session reader otherwise. ## atif-corpus One materialization pass is sweep, scan, plan, convert, write, advance watermark, and `materialize` is -that pass (`packages/atif-corpus/src/atif_corpus/application/materialize.py:476`). The decision half +that pass (`packages/atif-corpus/src/atif_corpus/application/materialize.py:1497`). The decision half is pure: the domain never stats a file and never reads a clock, so `build_plan` partitions scanned sessions into to-materialize, up-to-date, and skipped-live deterministically from mtimes in epoch -nanoseconds plus an injected now (`packages/atif-corpus/src/atif_corpus/domain/sessions.py:159`). The +nanoseconds plus an injected now (`packages/atif-corpus/src/atif_corpus/domain/sessions.py:169`). The write half is atomic by construction: artifacts land in a temp session directory that is renamed into place, and every writer fsyncs the tmp file before the rename and the directory after, because atif-duck reads the corpus with no locks and no journal (`packages/atif-corpus/src/atif_corpus/infrastructure/atomic.py:5`). Conversion arrives through the `ConverterPort` Protocol, typed to the contract's artifact shapes rather than converter internals since this member may never import atif-converter -(`packages/atif-corpus/src/atif_corpus/domain/ports.py:41`). The `SessionSink` Protocol in the same +(`packages/atif-corpus/src/atif_corpus/domain/ports.py:101`). The `SessionSink` Protocol in the same module is the other way out: after the swaps, the pass hands it every session it published or marked, and records the ones it hasn't taken yet in `sink_pending.json` so a failed or killed hand-over is retried by the next pass. What counts as a transcript, and how deep under the source root it sits, is one value object per agent (depth 1 for Claude Code's `/.jsonl`, depth 3 for Codex's `//
` nesting; -`packages/atif-corpus/src/atif_corpus/domain/source_layout.py:119`), so the scanner walks either -layout without branching on the agent, and `layout_for` refuses an agent that has none (`:131`). +`packages/atif-corpus/src/atif_corpus/domain/source_layout.py:111-124`), so the scanner walks either +layout without branching on the agent, and `layout_for` refuses an agent that has none (`:141-143`). The agent enum itself is an AST-pinned twin of the converter's, since the independence contract forbids the import (`packages/atif-corpus/src/atif_corpus/domain/agents.py:25`). - `packages/atif-corpus/src/atif_corpus/application/materialize.py`: the pass itself - (`packages/atif-corpus/src/atif_corpus/application/materialize.py:476`), the report value object - (`:118`), and the public `read_watermark` its consumers call (`:160`). + (`packages/atif-corpus/src/atif_corpus/application/materialize.py:1497`), the report value object + (`:220`), and the public `read_watermark` its consumers call (`:424`). - `packages/atif-corpus/src/atif_corpus/infrastructure/scanner.py`: the only place this - member stats the source corpus, keeping a genuinely absent transcript separate from a failed `stat` - so a transient error never deletes live artifacts - (`packages/atif-corpus/src/atif_corpus/infrastructure/scanner.py:15`). + member stats the source corpus + (`packages/atif-corpus/src/atif_corpus/infrastructure/scanner.py:5`); `SourceScan` keeps a + genuinely absent transcript separate from a failed `stat`, so a transient error never marks a live + session as source-removed (`:67`). - `packages/atif-corpus/src/atif_corpus/domain/sessions.py`: `SessionSource` (`packages/atif-corpus/src/atif_corpus/domain/sessions.py:42`), `QuiescencePolicy` (`:71`), - `MaterializationPlan` (`:108`), and `build_plan` (`:159`). + `MaterializationPlan` (`:118`), and `build_plan` (`:169`). - `packages/atif-corpus/src/atif_corpus/infrastructure/atomic.py`: tmp-sibling write, fsync, rename, for files (plain and zstd-compressed) and whole directories. - `packages/atif-corpus/src/atif_corpus/infrastructure/compress_artifacts.py`: compresses an @@ -252,41 +259,41 @@ forbids the import (`packages/atif-corpus/src/atif_corpus/domain/agents.py:25`). before the plain file is removed and keeping its mtime. - `packages/atif-corpus/src/atif_corpus/domain/watermark.py`: `diff_source_mtimes` partitions two mtime maps into added, modified, and removed - (`packages/atif-corpus/src/atif_corpus/domain/watermark.py:58`). + (`packages/atif-corpus/src/atif_corpus/domain/watermark.py:57`). - `packages/atif-corpus/src/atif_corpus/domain/layout.py`: `CorpusLayout` is the single writer-side computation of every contract path - (`packages/atif-corpus/src/atif_corpus/domain/layout.py:26`). + (`packages/atif-corpus/src/atif_corpus/domain/layout.py:65`). - `packages/atif-corpus/src/atif_corpus/infrastructure/fake_converter.py`: a scriptable `ConverterPort` fake shipped in `infrastructure` so the type checker holds it to the port on every run (`packages/atif-corpus/src/atif_corpus/infrastructure/fake_converter.py:3`). - `packages/atif-corpus/src/atif_corpus/infrastructure/settings.py`: `ATIF_SQL_`-prefixed settings whose default factories read env at call time, not import time - (`packages/atif-corpus/src/atif_corpus/infrastructure/settings.py:44`). + (`packages/atif-corpus/src/atif_corpus/infrastructure/settings.py:7-8`). ## atif-converter `convert_and_audit` returns a conversion result paired with a loss report: the trajectory plus an accounting of what upstream dropped -(`packages/atif-converter/src/atif_converter/application/convert_and_audit.py:105`). The conversion is -ours: `convert_claude_code_records`, a port of harbor 0.23.0's Claude Code converter built on the -public ATIF data classes (`packages/atif-converter/src/atif_converter/domain/claude_code_conversion.py:75`), -reached through the file-reading seam at -`packages/atif-converter/src/atif_converter/infrastructure/claude_code_converter.py:73` and validated -with harbor's public `TrajectoryValidator` -(`packages/atif-converter/src/atif_converter/infrastructure/harbor_adapter.py:68`). The Codex path is +(`packages/atif-converter/src/atif_converter/application/convert_and_audit.py:220`). The conversion is +ours: `convert_claude_code_records`, a port of harbor 0.24.0's Claude Code converter built on the +public ATIF data classes (`packages/atif-converter/src/atif_converter/domain/claude_code_conversion.py:981`), +reached through `convert_loaded_claude_code_session`, which takes the records the use case already +read (`packages/atif-converter/src/atif_converter/infrastructure/claude_code_converter.py:72`), and +validated with the vendored copy of harbor's `TrajectoryValidator` +(`packages/atif-converter/src/atif_converter/infrastructure/harbor_adapter.py:76`). The Codex path is the same shape one module over: `convert_codex_and_audit` -(`packages/atif-converter/src/atif_converter/application/convert_codex.py:128`) over -`convert_codex_records` (`packages/atif-converter/src/atif_converter/domain/codex_conversion.py:781`), +(`packages/atif-converter/src/atif_converter/application/convert_codex.py:144`) over +`convert_codex_records` (`packages/atif-converter/src/atif_converter/domain/codex_conversion.py:924`), one rollout in and one trajectory out by construction. harbor's own private converters are the -parity ORACLE, reached from the tests only (`packages/atif-converter/tests/harbor_oracle.py:94`, `:111`), +parity ORACLE, reached from the tests only (`packages/atif-converter/tests/harbor_oracle.py:118`, `:135`), frozen to goldens and diffed against the live corpus. harbor is a dev dependency only: the ATIF -data classes and validator the converters build on are vendored from harbor 0.23.0 in +data classes and validator the converters build on are vendored from harbor 0.24.0 in `packages/atif-converter/src/atif_converter/domain/atif/` and held to upstream by `packages/atif-converter/tests/test_vendored_atif.py`. The known conversion gaps are types rather than prose: `FidelityGap` enumerates them -(`packages/atif-converter/src/atif_converter/domain/fidelity.py:44`) and a pure enrichment pass repairs +(`packages/atif-converter/src/atif_converter/domain/fidelity.py:55`) and a pure enrichment pass repairs some of them by re-running harbor's deterministic normalization order over the raw records (`packages/atif-converter/src/atif_converter/domain/enrichment.py:198`). Codex has its own set -(`packages/atif-converter/src/atif_converter/domain/codex_fidelity.py:64`), all values namespaced +(`packages/atif-converter/src/atif_converter/domain/codex_fidelity.py:66`), all values namespaced `codex_*` so one `gaps_observed` array can carry either agent's, and its own enrichment pass, which attributes agent steps by a re-derived `api_call_id` rather than by message text because harbor drops empty text parts and an empty assistant message can never be placed by matching @@ -296,10 +303,11 @@ list of records; the fingerprints are re-checked once the artifacts are built, s resumes writing mid-conversion fails instead of publishing artifacts for bytes it no longer holds (`packages/atif-converter/src/atif_converter/infrastructure/raw_records.py`, `load_session` and `mutated_files`). Per-step cost -estimates come from `pricing.cost_per_token` -(`packages/atif-converter/src/atif_converter/domain/pricing.py`), which reads the vendored +estimates come from `pricing.priced_cost_per_token` on the Claude Code path and +`pricing.cost_per_token` on the Codex path +(`packages/atif-converter/src/atif_converter/domain/pricing.py`), which read the vendored `model_prices.json` (a filtered copy of litellm's public price data, regenerated by -`scripts/update_prices.py`) and reproduces `litellm.cost_per_token`'s floats exactly; a model the +`scripts/update_prices.py`) and reproduce `litellm.cost_per_token`'s floats exactly; a model the table doesn't price is NULL. litellm is a dev dependency, for the identity test only. - `packages/atif-converter/src/atif_converter/domain/pricing.py`: `fast_cost_per_token` prices the @@ -309,18 +317,20 @@ table doesn't price is NULL. litellm is a dev dependency, for the identity test - `packages/atif-converter/src/atif_converter/domain/enrichment.py`: `enrich_trajectory` restores source uuids, the compact-summary flag, and the cache-creation total (`packages/atif-converter/src/atif_converter/domain/enrichment.py:198`). -- `packages/atif-converter/src/atif_converter/infrastructure/harbor_adapter.py`: the only - module importing harbor, which ships no `py.typed`, so the untyped surface stays contained - (`packages/atif-converter/src/atif_converter/infrastructure/harbor_adapter.py:11`). +- `packages/atif-converter/src/atif_converter/infrastructure/harbor_adapter.py`: the Claude + Code conversion seam, which keeps its historical name: `read_session` reads a session once, + `convert_loaded_session` converts those records, and `validate_trajectory` runs the vendored + `TrajectoryValidator`; harbor itself isn't a runtime dependency, and no module under `src/` + imports it (`packages/atif-converter/src/atif_converter/infrastructure/harbor_adapter.py:5-7`). - `packages/atif-converter/src/atif_converter/infrastructure/raw_records.py`: the one raw reader the census, edges emitter, and enrichment share, with `mutated_files` as the race check - (`packages/atif-converter/src/atif_converter/infrastructure/raw_records.py:146`). + (`packages/atif-converter/src/atif_converter/infrastructure/raw_records.py:320`, `:382`). - `packages/atif-converter/src/atif_converter/application/convert_and_audit.py`: the use - case (`packages/atif-converter/src/atif_converter/application/convert_and_audit.py:105`) and its - refusal on any mid-window mutation (`:93`). + case (`packages/atif-converter/src/atif_converter/application/convert_and_audit.py:220`) and its + refusal on any mid-window mutation (`:199`). - `packages/atif-converter/src/atif_converter/domain/fidelity.py`: `RecordType` - (`packages/atif-converter/src/atif_converter/domain/fidelity.py:18`), `FidelityGap` - (`:44`), and the `LossReport` value object (`:83`). + (`packages/atif-converter/src/atif_converter/domain/fidelity.py:26`), `FidelityGap` + (`:55`), and the `LossReport` value object (`:114`). - `packages/atif-converter/src/atif_converter/domain/edges.py`: one `edges.jsonl` line per raw record, key order fixed by the contract (`packages/atif-converter/src/atif_converter/domain/edges.py:22`). @@ -360,12 +370,12 @@ which one pure transform enforces before the schema reaches the wire (`packages/atif-models/src/atif_models/infrastructure/anthropic_bedrock.py:130`), which joins the `text` blocks that follow any `thinking` block, folds a string enum value that differs from the schema only in letter case onto the declared one (`fold_enum_case`, `:102`), raises - `RefusalError` on `stop_reason: refusal`, and retries a `max_tokens` stop once at one effort step - lower (`:185`, `:239`); usage extraction (`:81`). + `RefusalError` on `stop_reason: refusal` (`:253-257`), and retries a `max_tokens` stop once at one + effort step lower (`:200-219`); usage extraction (`:81`). - `packages/atif-models/src/atif_models/infrastructure/openai_bedrock.py`: `OpenAiBedrockProvider` (`packages/atif-models/src/atif_models/infrastructure/openai_bedrock.py:81`), which retries a - `length` finish once at one effort step lower (`:149`), and usage extraction (`:63`). + `length` finish once at one effort step lower (`:165-177`), and usage extraction (`:63`). - `packages/atif-models/src/atif_models/domain/registry.py`: `ModelSpec` (`packages/atif-models/src/atif_models/domain/registry.py:50`), whose long-context threshold and rates price a request over the threshold at the long rates and whose `chars_per_token` is the @@ -374,7 +384,7 @@ which one pure transform enforces before the schema reaches the wire long-context share of the tokens at the long rates (`:180`). - `packages/atif-models/src/atif_models/domain/ports.py`: the `LlmStructuredProvider` Protocol (`packages/atif-models/src/atif_models/domain/ports.py:131`), the - terminal-versus-retryable error split (`:43`), and lock-guarded usage accumulation that keeps the + terminal-versus-retryable error split (`:43-58`), and lock-guarded usage accumulation that keeps the long-context share apart (`:81`). - `packages/atif-models/src/atif_models/domain/schema.py`: `to_openai_strict` keeps `$defs` and `$ref`, drops `default` and `title` @@ -398,7 +408,7 @@ which one pure transform enforces before the schema reaches the wire env, no clock (`packages/atif-models/src/atif_models/domain/__init__.py:7`). - `packages/atif-models/src/atif_models/infrastructure/__init__.py`: the adapters, the factory, and env settings, importing downward into `domain` only - (`packages/atif-models/src/atif_models/infrastructure/__init__.py:7`). + (`packages/atif-models/src/atif_models/infrastructure/__init__.py:5-8`). ## See also diff --git a/docs/architecture/system-overview.md b/docs/architecture/system-overview.md index d30e8c1..d5cb719 100644 --- a/docs/architecture/system-overview.md +++ b/docs/architecture/system-overview.md @@ -6,59 +6,62 @@ session transcripts Claude Code leaves at `~/.claude/projects/**/*.jsonl` and the rollouts Codex CLI leaves at `~/.codex/sessions///
/rollout-*.jsonl`, converts each one to ATIF (Harbor's Agent Trajectory Interchange Format), materializes the results as an on-disk -corpus, and answers SQL against that corpus through DuckDB (`README.md:12`). The design bet is +corpus, and answers SQL against that corpus through DuckDB (`README.md:13-17`). The design bet is stated in the README itself: converting once at the boundary, with an explicit and tested fidelity policy for what the upstream converter drops, beats re-deriving trajectory semantics inside every -SQL view (`README.md:15`). Which agent a command works on is one flag, `--agent claude-code|codex`, and it +SQL view (`README.md:17-19`). Which agent a command works on is one flag, `--agent claude-code|codex`, and it moves the default source and corpus roots with it (`packages/atif-corpus/src/atif_corpus/infrastructure/settings.py`); one corpus holds one agent, and -the `sessions` view reports which (`packages/atif-duck/src/atif_duck/infrastructure/registry.py:428`). +the `sessions` view reports which (`packages/atif-duck/src/atif_duck/infrastructure/registry.py:1040`). The reader it serves is an engineer or an agent asking how sessions actually went: which tools ran, where tokens went, where a session turned into friction. Users get one installable distribution and one console script, -`atif-sql = "atif_cli.app:main"` (`packages/atif-cli/pyproject.toml:42`), installed with -`uv tool install atif-sql` (`README.md:26`). The command surface is the commands registered -with `@app.command` in `packages/atif-cli/src/atif_cli/app.py` plus the `cron` sub-app -attached at `:65`. Nothing here is a server; DuckDB is embedded +`atif-sql = "atif_cli.app:main"` (`pyproject.toml:75`), installed with +`uv tool install atif-sql` (`README.md:29`). The command surface is the commands registered +with `@app.command` in `packages/atif-cli/src/atif_cli/app.py` plus the `cron`, `lake`, and +`corpus` sub-apps attached at `:104-109`. Nothing here is a server; DuckDB is embedded (`packages/atif-duck/pyproject.toml:20`), as are the SQLite analytics state file and the LanceDB vector store. The commands `analyze` / `embed` / `search` call Amazon Bedrock and spend money -per invocation, and each is dry-run by default (`README.md:37`). +per invocation (`embed` and `search` run on this machine instead under +`ATIF_SQL_EMBED_PROVIDER=gemma`): `analyze` is a dry run by default, `embed` refuses to run without +`--limit` or `--all`, and `search` embeds one query per call (`README.md:50-54`). ## How the pieces fit The directories under `packages/` are internal module boundaries, not separate installs -(`README.md:43`); they are uv workspace members (`pyproject.toml:100`). `atif-converter` owns the -conversion itself: its converters ported from Harbor 0.23.0 under Apache-2.0 +(`README.md:101`); they are uv workspace members (`pyproject.toml:111`). `atif-converter` owns the +conversion itself: its converters ported from Harbor under Apache-2.0 and held to parity with Harbor +0.24.0 (`packages/atif-converter/src/atif_converter/domain/claude_code_conversion.py`, `packages/atif-converter/src/atif_converter/domain/codex_conversion.py`), built on Harbor's ATIF data classes and validator, vendored in `packages/atif-converter/src/atif_converter/domain/atif/`. harbor itself is a dev dependency: its private converters survive only as the parity oracle in the converter's tests -(`packages/atif-converter/tests/harbor_oracle.py:94`), and `packages/atif-converter/tests/test_vendored_atif.py` holds the vendored +(`packages/atif-converter/tests/harbor_oracle.py:3`), and `packages/atif-converter/tests/test_vendored_atif.py` holds the vendored copy to upstream. It also owns the fidelity policy as types: `FidelityGap` enumerates the known upstream conversion gaps -(`packages/atif-converter/src/atif_converter/domain/fidelity.py:44`), and `CodexFidelityGap` +(`packages/atif-converter/src/atif_converter/domain/fidelity.py:55`), and `CodexFidelityGap` enumerates the ones that are Codex's -(`packages/atif-converter/src/atif_converter/domain/codex_fidelity.py:64`). `atif-corpus` +(`packages/atif-converter/src/atif_converter/domain/codex_fidelity.py:66`). `atif-corpus` drives materialization, writing per-session artifacts plus a corpus watermark -(`docs/CONTRACT.md:21`). Its `ConverterPort` Protocol lets an implementation raise anything: the +(`docs/CONTRACT.md:22`). Its `ConverterPort` Protocol lets an implementation raise anything: the materialize use case records the failure against that session and continues, so one broken -transcript never aborts a sync (`packages/atif-corpus/src/atif_corpus/domain/ports.py:41`). +transcript never aborts a sync (`packages/atif-corpus/src/atif_corpus/domain/ports.py:104-106`). `atif-duck` reads the corpus root and never imports the packages that wrote it -(`docs/CONTRACT.md:52`). Its one public `register` registers raw readers, views, vector search, +(`docs/CONTRACT.md:247`). Its `register` entry point registers raw readers, views, vector search, macros, the authorship views, and the analytics surface in that order, because views bind against the raw TEMP tables at CREATE time -(`packages/atif-duck/src/atif_duck/infrastructure/registry.py:1740`); the raw readers are -`CREATE TEMP TABLE` over `read_json` (`:13`), and every view and macro name is declared in a static -catalog (`packages/atif-duck/src/atif_duck/domain/catalog.py:30`). `atif-analytics` is the -largest module by source files; it writes parquet plus one SQLite WAL `state.db` under +(`packages/atif-duck/src/atif_duck/infrastructure/registry.py:2095`); the per-session raw readers +are `CREATE TEMP TABLE` over `read_json` (`:15`), and every view and macro name is declared in a +static catalog (`packages/atif-duck/src/atif_duck/domain/catalog.py:30`). `atif-analytics` writes +parquet plus one SQLite WAL `state.db` under `/analytics/` (`packages/atif-analytics/src/atif_analytics/domain/layout.py:6`). -`atif-embed` backfills Cohere Embed v4 vectors into LanceDB -(`packages/atif-embed/pyproject.toml:4`), which `atif-duck` reads back through DuckDB's lance +`atif-embed` backfills Cohere Embed v4 vectors, or EmbeddingGemma 2 vectors computed on this +machine, into LanceDB (`packages/atif-embed/pyproject.toml:4`), which `atif-duck` reads back through DuckDB's lance extension rather than by importing lancedb -(`packages/atif-duck/src/atif_duck/infrastructure/registry.py:830`). `atif-models` is the single +(`packages/atif-duck/src/atif_duck/infrastructure/registry.py:1621-1624`). `atif-models` is the single owner of model ids, so no other package hardcodes one (`packages/atif-models/pyproject.toml:4`). Every corpus is also loaded into one DuckLake, kept at `ATIF_SQL_LAKE_ROOT` beside the corpora. @@ -78,13 +81,14 @@ views over the lake, falling back to the per-session artifacts when the lake is doesn't hold the corpus (`packages/atif-duck/src/atif_duck/infrastructure/lake.py`). The direction of those edges is enforced rather than conventional. `[tool.importlinter]` -(`pyproject.toml:462`) declares a layer contract per member, an independence contract forbidding -converter / corpus / duck / models / embed from importing each other (`:417`), and a forbidden -contract limiting `atif-analytics` to `atif-models` alone among the members (`:422`). `atif-cli` is +(`pyproject.toml:550`) declares a layers contract for every member except `atif-cli` and +`atif-duck`, an independence contract forbidding converter / corpus / duck / models / embed from +importing each other (`:603`), and a forbidden contract limiting `atif-analytics` to `atif-models` +alone among the members (`:608`). `atif-cli` is the sole composition root, and it imports its siblings lazily inside command bodies -(`packages/atif-cli/src/atif_cli/app.py:388`, `:583`, `:688`) so that `atif-sql schema` pays for +(`packages/atif-cli/src/atif_cli/app.py:712`, `:1697`, `:1903`) so that `atif-sql schema` pays for none of the analytics or vector stack. Failures surface as typed exceptions mapped to stable -process exit codes at the CLI edge (`:38`). +process exit codes at the CLI edge (`packages/atif-cli/src/atif_cli/errors.py:30`). ## Stack @@ -92,28 +96,28 @@ process exit codes at the CLI edge (`:38`). | --- | --- | --- | | Language | Python, `requires-python = ">=3.13"` | `packages/atif-cli/pyproject.toml:14` | | Toolchain pin | mise, `python = "3.13"` | `mise.toml:32` | -| Packaging | uv workspace, `members = ["packages/*"]` | `pyproject.toml:100` | -| Build backend | `uv_build>=0.11.14,<0.12` | `packages/atif-cli/pyproject.toml:48` | -| CLI framework | `cyclopts>=4.10.2` | `packages/atif-cli/pyproject.toml:37` | +| Packaging | uv workspace, `members = ["packages/*"]` | `pyproject.toml:111` | +| Build backend | `hatchling>=1.28.0` for the published wheel, `uv_build>=0.11.14,<0.13` for the member manifests | `pyproject.toml:90`, `packages/atif-cli/pyproject.toml:53` | +| CLI framework | `cyclopts>=5.1.1` | `packages/atif-cli/pyproject.toml:37` | | Query engine | `duckdb>=1.5.2,<2` | `packages/atif-duck/pyproject.toml:20` | | Lake | DuckLake, through DuckDB's `ducklake` extension, with a DuckDB-file catalog | `packages/atif-duck/src/atif_duck/infrastructure/lake.py` | -| Trajectory models | vendored from harbor 0.23.0 (`pydantic>=2.13.2`) | `packages/atif-converter/src/atif_converter/domain/atif/__init__.py` | +| Trajectory models | vendored from harbor 0.24.0 (`pydantic>=2.13.2`) | `packages/atif-converter/src/atif_converter/domain/atif/__init__.py` | | Vector store | `lancedb>=0.30,<0.40` | `packages/atif-embed/pyproject.toml:22` | -| Model access | `boto3>=1.42.91` for Bedrock | `packages/atif-models/pyproject.toml:24` | +| Model access | `boto3>=1.42.91` for Bedrock | `packages/atif-models/pyproject.toml:28` | | Dataframes | `polars>=1.40.0` | `packages/atif-embed/pyproject.toml:24` | -| Validation | `pydantic>=2.13.2` | `packages/atif-converter/pyproject.toml:25` | +| Validation | `pydantic>=2.13.2` | `packages/atif-converter/pyproject.toml:23` | | Logging | `loguru>=0.7.3` | `packages/atif-corpus/pyproject.toml:19` | -| Lint and format | ruff, `select = ["ALL"]` | `pyproject.toml:146` | -| Architecture gate | import-linter contracts | `pyproject.toml:462` | -| Tests | pytest, `testpaths = ["packages/*/tests"]` | `pyproject.toml:389` | -| Definition of done | `mise run check` | `mise.toml:197` | +| Lint and format | ruff, `select = ["ALL"]` | `pyproject.toml:209` | +| Architecture gate | import-linter contracts | `pyproject.toml:550` | +| Tests | pytest, `testpaths = ["packages/*/tests", "fuzz"]` | `pyproject.toml:477` | +| Definition of done | `mise run check` | `mise.toml:243` | ## Module map Nodes are the uv workspace members. Every edge is an import confirmed at an import site. There is no `atif-corpus` to `atif-converter` edge: the independence contract forbids it, and -`RealConverter` in `packages/atif-cli/src/atif_cli/converter_adapter.py:44` satisfies -`ConverterPort` by importing both from the composition root (`:36`, `:38`). `atif-duck`, +`RealConverter` in `packages/atif-cli/src/atif_cli/converter_adapter.py:80` satisfies +`ConverterPort` by importing both from the composition root (`:64`, `:69`). `atif-duck`, `atif-analytics`, and `atif-embed` exchange data through corpus files on disk, never through an import. diff --git a/docs/behavior/processes.md b/docs/behavior/processes.md index b07155b..197f1a3 100644 --- a/docs/behavior/processes.md +++ b/docs/behavior/processes.md @@ -2,77 +2,86 @@ Every process in this system starts as a CLI invocation. There is one console script, `atif-sql = "atif_cli.app:main"` -(`packages/atif-cli/pyproject.toml:42`), and its commands are the complete -initiator set: the `@app.command` sites plus the `cron` sub-App registered at -`packages/atif-cli/src/atif_cli/app.py:66`. No HTTP route, RPC tool, message +(`pyproject.toml:75`), and its commands are the complete +initiator set: the nine `@app.command` sites plus the `cron`, `lake`, and `corpus` sub-apps registered at +`packages/atif-cli/src/atif_cli/app.py:104-109`. No HTTP route, RPC tool, message handler, or job queue exists to initiate anything else; the only scheduled -initiator is a crontab line into `scripts/atif-sql-refresh.sh`. Long-running +initiators are the crontab lines, one per lane, into `scripts/atif-sql-refresh.sh`. Long-running work is in-process async under `anyio` and `asyncio.run`, not a worker. ## materialize: sync the materialized corpus -Entry point: `packages/atif-cli/src/atif_cli/app.py:352` +Entry point: `packages/atif-cli/src/atif_cli/app.py:863` 1. Resolve `CorpusSettings` and wire the pass: `--source-root` given without `--corpus-root` re-derives the corpus root from the overridden source's slug so re-pointing the source cannot overwrite another corpus, and the CLI hands in the wall clock plus the harbor and converter version pins alongside - `RealConverter` behind `ConverterPort` (`:190`, - `packages/atif-cli/src/atif_cli/converter_adapter.py:44`). + `RealConverter` behind `ConverterPort` (`:624-625`, `:958-974`, + `packages/atif-cli/src/atif_cli/converter_adapter.py:80`). 2. Sweep `.staging/`: an entry whose `tmp-` owner is a dead pid is crash debris and is removed, while a live pid belongs to a concurrent pass and is left alone - (`packages/atif-corpus/src/atif_corpus/application/materialize.py:274`). + (`packages/atif-corpus/src/atif_corpus/application/materialize.py:971`). 3. Read `watermark.json` as a path-to-mtime map; an unreadable or wrongly shaped file degrades to empty, costing one full re-materialization rather - than refusing to sync: `packages/atif-corpus/src/atif_corpus/application/materialize.py:160`. + than refusing to sync: `packages/atif-corpus/src/atif_corpus/application/materialize.py:424`. 4. Scan the source root for main transcripts and every side-file, then fold sessions living under a directory that would not list into the unreadable set using the watermark as the only record of what lived there: - `packages/atif-corpus/src/atif_corpus/infrastructure/scanner.py:143`, - `packages/atif-corpus/src/atif_corpus/application/materialize.py:342`. -5. Guard, then retain sourceless sessions: zero scanned sessions over a + `packages/atif-corpus/src/atif_corpus/infrastructure/scanner.py:206`, + `packages/atif-corpus/src/atif_corpus/application/materialize.py:1092`. +5. Guard, then retain sourceless sessions: a corpus whose `meta.json` names + another agent raises `CorpusAgentMismatchError` before anything is touched, + and zero scanned sessions over a non-empty corpus raises `SuspiciousEmptyScanError` instead of marking everything source-removed. Otherwise every corpus session with no source in the scan is KEPT and marked `source_present: false` once; marking is skipped entirely when any source directory failed to list, because absence is then not evidence of deletion (`_retain_sourceless_sessions`). -6. Build the pure `MaterializationPlan` from quiescence and the watermark, - force-replanning sessions the watermark calls current whose artifact +6. Build the pure `MaterializationPlan` from quiescence, the watermark, and + each session's recorded generation, force-replanning sessions the watermark calls current whose artifact directory is missing (the one state reachable by a kill inside the swap - window): `packages/atif-corpus/src/atif_corpus/domain/sessions.py:159`, - `packages/atif-corpus/src/atif_corpus/application/materialize.py:313`. -7. Per planned session: convert, write trajectory, loss report, edges, then - run the `ArtifactProducer` (the typed columnar parquet files, unless - `--no-columnar`), and `meta.json` last into a staging directory, then rename the whole directory + window): `packages/atif-corpus/src/atif_corpus/domain/sessions.py:169`, + `packages/atif-corpus/src/atif_corpus/application/materialize.py:1017`. +7. Per planned session: convert (the converter writes the raw source archive + while it verifies its input), store the attachments in the shared blob + store, write trajectory, loss report, edges, and session events, and `meta.json` last into a staging directory, then rename the whole directory into `sessions//` so a reader observes only a complete generation; a - session that raises is recorded and the pass continues (`packages/atif-corpus/src/atif_corpus/application/materialize.py:228`, - `packages/atif-corpus/src/atif_corpus/infrastructure/atomic.py:98`). + session that raises is recorded and the pass continues (`packages/atif-corpus/src/atif_corpus/application/materialize.py:612`, + `packages/atif-corpus/src/atif_corpus/infrastructure/atomic.py:197`). + A retained session whose recorded generation is stale re-converts from its + restored source archive in the same stage. With `--workers` above 1 this step runs on a spawn-context process pool; each worker runs the same per-session function under its own pid, outcomes are folded back in plan order, and a single planned session runs inline: - `packages/atif-corpus/src/atif_corpus/application/materialize.py:385`. -8. Advance the watermark for succeeded sessions only, retaining every entry of + `packages/atif-corpus/src/atif_corpus/application/materialize.py:874`. +8. Hand every session published or marked this pass, plus any an earlier pass + left in `sink_pending.json`, to the DuckLake session sink in batches + (`--no-lake` skips it); the first batch that raises, and every batch after + it, stays pending for the next pass: + `packages/atif-corpus/src/atif_corpus/application/materialize.py:384`. +9. Advance the watermark for succeeded sessions only, retaining every entry of a failed, unplanned, or unreadable session so staleness still signals a retry, and write it atomically before emitting the report: - `packages/atif-corpus/src/atif_corpus/application/materialize.py:428`, - `packages/atif-corpus/src/atif_corpus/infrastructure/atomic.py:64`. + `packages/atif-corpus/src/atif_corpus/application/materialize.py:1310`, + `packages/atif-corpus/src/atif_corpus/infrastructure/atomic.py:72`. ### Related -- `packages/atif-corpus/src/atif_corpus/domain/ports.py:41` -- `packages/atif-corpus/src/atif_corpus/domain/layout.py:26` -- `packages/atif-corpus/src/atif_corpus/infrastructure/settings.py:44` -- `packages/atif-corpus/src/atif_corpus/infrastructure/atomic.py:43` -- `packages/atif-cli/src/atif_cli/app.py:312` +- `packages/atif-corpus/src/atif_corpus/domain/ports.py:101` +- `packages/atif-corpus/src/atif_corpus/domain/layout.py:65` +- `packages/atif-corpus/src/atif_corpus/infrastructure/settings.py:88` +- `packages/atif-corpus/src/atif_corpus/infrastructure/atomic.py:51` +- `packages/atif-cli/src/atif_cli/app.py:765` ## convert: one session to ATIF plus a loss audit -Entry point: `packages/atif-cli/src/atif_cli/app.py:226` +Entry point: `packages/atif-cli/src/atif_cli/app.py:675` -1. Reject a path that is not an existing `.jsonl` file before any harbor work, - which the CLI maps to exit 64: - `packages/atif-converter/src/atif_converter/infrastructure/harbor_adapter.py:163`. +1. Reject a path that is not an existing `.jsonl` file before the fingerprint + snapshot reads anything, which the CLI maps to exit 64: + `packages/atif-converter/src/atif_converter/infrastructure/harbor_adapter.py:71-73`. 2. Read every source file once: the main transcript and each discovered side-file under `/`, workflow-nested ones included. Each file is stat'ed, then streamed through a sha256 digest into the JSON parser, so the @@ -80,33 +89,36 @@ Entry point: `packages/atif-cli/src/atif_cli/app.py:226` `packages/atif-converter/src/atif_converter/infrastructure/raw_records.py` (`load_session`), reached through `harbor_adapter.read_session`, which classifies a non-UTF-8 file as `ConversionError`. -3. Convert those records with our ported converter (a parity port of harbor - 0.22.0's, built on the public ATIF models) and dump the result to a +3. Replace every inline base64 attachment with a placeholder and collect its + bytes, then convert the records with our ported converter (a port of harbor + 0.24.0's, built on the ATIF models vendored from harbor, + `packages/atif-converter/src/atif_converter/domain/claude_code_conversion.py:2`) and dump the result to a JSON-mode dict; a `None` return means no convertible events and raises `EmptySessionError`: `packages/atif-converter/src/atif_converter/infrastructure/harbor_adapter.py` (`convert_loaded_session`), `packages/atif-converter/src/atif_converter/infrastructure/claude_code_converter.py` (`convert_loaded_claude_code_session`). -4. Build the loss report from the same records: some `FidelityGap` members - are structural for every harbor 0.22.0 conversion, and the rest are added - from what the census found: - `packages/atif-converter/src/atif_converter/infrastructure/census.py:57`, - `packages/atif-converter/src/atif_converter/application/convert_and_audit.py:62`. -5. Derive the `edges.jsonl` lines from the same records, then enrich the +4. Count the same records into the census and derive the `edges.jsonl` lines + from them, then enrich the trajectory in place with `source_uuids`, `is_compact_summary`, and `cache_creation_total` (the harbor trajectory has no other consumer, so no - deep copy is taken): `packages/atif-converter/src/atif_converter/domain/edges.py:99`, + deep copy is taken): `packages/atif-converter/src/atif_converter/domain/edges.py:111`, `packages/atif-converter/src/atif_converter/domain/enrichment.py:198`. +5. Build the loss report: one `FidelityGap` member, `PARENT_CHAIN_FLATTENED`, + is structural for every conversion, and the rest are added from what the + census found and from what enrichment didn't repair: + `packages/atif-converter/src/atif_converter/infrastructure/census.py:57`, + `packages/atif-converter/src/atif_converter/application/convert_and_audit.py:137`. 6. Re-check the fingerprints, stat and digest both. Any movement since the read raises `SourceMutatedDuringConversion` rather than publishing artifacts that describe bytes the session no longer holds: - `packages/atif-converter/src/atif_converter/application/convert_and_audit.py:93`, + `packages/atif-converter/src/atif_converter/application/convert_and_audit.py:199`, `packages/atif-converter/src/atif_converter/infrastructure/raw_records.py` (`mutated_files`). 7. Re-validate the enriched trajectory, and write it plus `edges.jsonl` beside it or stream the trajectory to stdout; a validation error exits 65: - `packages/atif-cli/src/atif_cli/app.py:272`. + `packages/atif-cli/src/atif_cli/app.py:737-757`. Per file that is two opens, one parse and two hash passes; the converter and the audit never read the session separately. @@ -114,22 +126,22 @@ the audit never read the session separately. ### Related - `packages/atif-converter/src/atif_converter/infrastructure/census.py:57` -- `packages/atif-converter/src/atif_converter/infrastructure/harbor_adapter.py:97` -- `packages/atif-converter/src/atif_converter/domain/errors.py:57` -- `packages/atif-converter/src/atif_converter/domain/fidelity.py:44` -- `packages/atif-cli/src/atif_cli/errors.py:25` +- `packages/atif-converter/src/atif_converter/infrastructure/harbor_adapter.py:76` +- `packages/atif-converter/src/atif_converter/domain/errors.py:47` +- `packages/atif-converter/src/atif_converter/domain/fidelity.py:55` +- `packages/atif-cli/src/atif_cli/errors.py:30` ## analyze: orchestrate the analytics stages -Entry point: `packages/atif-cli/src/atif_cli/app.py:1240` +Entry point: `packages/atif-cli/src/atif_cli/app.py:1634` 1. Resolve `AnalyticsSettings`, reusing the corpus-root resolution the other commands share, then let `--max-sessions` and `--max-cost-usd` override the - env ceilings so a crontab line carries its spend cap visibly (`:1307`). + env ceilings so a crontab line carries its spend cap visibly (`:1711-1714`). `--llm-only` is accepted and dropped, since every stage is an LLM stage. 2. Open the lake's session source when the lake holds the corpus, else warn once and read the files (`--no-lake` forces the files): - `packages/atif-cli/src/atif_cli/app.py:1624`. Build one `CorpusReader` over + `packages/atif-cli/src/atif_cli/app.py:1738`. Build one `CorpusReader` over it, shared by every stage: the steps memo is the expensive part and every stage walks the same sessions (`packages/atif-analytics/src/atif_analytics/application/analyze.py:62`, @@ -141,12 +153,12 @@ Entry point: `packages/atif-cli/src/atif_cli/app.py:1240` declaration order. A stage entered with the budget already exhausted is skipped with nothing stamped, its consecutive-skip streak persisted, and the log escalates to ERROR at three consecutive runs: - `packages/atif-analytics/src/atif_analytics/application/analyze.py:97`, + `packages/atif-analytics/src/atif_analytics/application/analyze.py:109-128`, `packages/atif-analytics/src/atif_analytics/infrastructure/sqlite_state/checkpointer.py:218`. 5. Emit the per-stage summary carrying `budget_exhausted` and, on a real run, `llm_spent_usd`: - `packages/atif-analytics/src/atif_analytics/application/analyze.py:144`, - `packages/atif-cli/src/atif_cli/app.py:1322`. + `packages/atif-analytics/src/atif_analytics/application/analyze.py:150-152`, + `packages/atif-cli/src/atif_cli/app.py:1735`. ### Related @@ -178,22 +190,22 @@ turns, the main-chain user steps whose author is `human`. `packages/atif-analytics/src/atif_analytics/application/use_cases/classify.py:311`. 3. Anti-join the cache for session ids already written, then drop sessions whose last-step timestamp and mtime bounds are unchanged since the last run (`:115`, - `packages/atif-analytics/src/atif_analytics/infrastructure/sqlite_state/checkpointer.py:136`). + `packages/atif-analytics/src/atif_analytics/infrastructure/sqlite_state/checkpointer.py:137`). 4. Drain the retry queue back into the admission set, then subtract units with a live entry (inside backoff or past the attempt cap) because the retry queue is the single re-admission gate and the checkpoint path would re-bill them: `packages/atif-analytics/src/atif_analytics/infrastructure/sqlite_state/retry_queue.py:145`, - `:159`. + `:176`. 5. Walk newest-first and stop admitting at `llm_max_sessions_per_run` so a deferred session is never rendered; a session that renders to nothing, or whose kind isn't `interactive`, is checkpointed at current bounds instead of re-rendered every tick: - `packages/atif-analytics/src/atif_analytics/application/use_cases/classify.py:147`. + `packages/atif-analytics/src/atif_analytics/application/use_cases/classify.py:147-178`. The rendered transcript carries each tool result as a size stub, `[tool_result ] (N chars omitted)`, unless `ATIF_SQL_SESSION_TEXT_TOOL_RESULT_BODIES` turns the bodies on: - `packages/atif-analytics/src/atif_analytics/domain/transcript.py:306`. + `packages/atif-analytics/src/atif_analytics/domain/transcript.py:308-311`. 6. Dispatch each write chunk in budget-checked sub-batches of `BUDGET_CHECK_BATCH`, advancing the cursor by what was actually sent so a mid-chunk budget stop leaves the remainder unstamped rather than silently @@ -210,8 +222,8 @@ turns, the main-chain user steps whose author is `human`. (`packages/atif-analytics/src/atif_analytics/application/use_cases/classify.py:247`). 8. Write the chunk as a parquet part, checkpoint the completed sessions at current bounds, and clear them from the retry queue: - `packages/atif-analytics/src/atif_analytics/infrastructure/parquet_cache.py:268`, - `packages/atif-analytics/src/atif_analytics/application/use_cases/classify.py:288`. + `packages/atif-analytics/src/atif_analytics/infrastructure/parquet_cache.py:160`, + `packages/atif-analytics/src/atif_analytics/application/use_cases/classify.py:288-294`. ### Related @@ -223,14 +235,15 @@ turns, the main-chain user steps whose author is `human`. ## embed: backfill step embeddings into LanceDB -Entry point: `packages/atif-cli/src/atif_cli/app.py:766` +Entry point: `packages/atif-cli/src/atif_cli/app.py:1801` 1. Refuse a real run with no scope: without `--limit`, `--all`, or `--dry-run` the command exits 64, so a mistyped invocation cannot start a - full backfill (`:778`). + full backfill (`:1908`). 2. Resolve the Lance URI from `EmbedSettings`, defaulting to - `/embeddings_lance` - (`packages/atif-embed/src/atif_embed/infrastructure/settings.py:51`). + `/embeddings_lance` (`/embeddings_lance_gemma` + under the `gemma` provider) + (`packages/atif-embed/src/atif_embed/infrastructure/settings.py:133`). 3. Read the store's uuid-to-text-hash map once, then stream candidates whose hash is absent or stale through `TextRowsPort`; the staleness comparison happens before the limit cap so `--limit N` always makes N rows of progress @@ -243,23 +256,24 @@ Entry point: `packages/atif-cli/src/atif_cli/app.py:766` read (`packages/atif-embed/src/atif_embed/infrastructure/corpus_text_rows.py`). 4. Under `--dry-run`, count candidates and return the plan dict, returning before boto3 is ever imported: - `packages/atif-embed/src/atif_embed/application/embed.py:120`. -5. Build the Cohere-on-Bedrock provider once, then refuse to append when the + `packages/atif-embed/src/atif_embed/application/embed.py:124`. +5. Build the provider `ATIF_SQL_EMBED_PROVIDER` selects once (Cohere on + Bedrock by default, or EmbeddingGemma 2 on this machine), then refuse to append when the store's stamped model and dimension differ from the live embedder's, because - mixing vector spaces corrupts kNN silently (`:157`, + mixing vector spaces corrupts kNN silently (`:169-183`, `packages/atif-embed/src/atif_embed/domain/embedding_guard.py:38`). 6. Per chunk, a multiple of the batch size so a checkpoint boundary never - splits a Bedrock batch: embed the texts and drop rows the provider failed, + splits a provider batch: embed the texts and drop rows the provider failed, leaving them pending for the next run - (`packages/atif-embed/src/atif_embed/application/embed.py:147`, - `packages/atif-embed/src/atif_embed/infrastructure/cohere_bedrock.py:309`). + (`packages/atif-embed/src/atif_embed/application/embed.py:153`, + `packages/atif-embed/src/atif_embed/infrastructure/cohere_bedrock.py:310`). 7. Delete stale rows under the same uuid before appending the fixed-size float32 array frame, so text that changed cannot leave two rows fanning the - kNN join out: `packages/atif-embed/src/atif_embed/application/embed.py:207`, + kNN join out: `packages/atif-embed/src/atif_embed/application/embed.py:213-218`, `packages/atif-embed/src/atif_embed/infrastructure/lance_store.py:402`. 8. Compact every eight chunks to bound fragment count, then optimize and ensure the HNSW index on the way out so `search` pays no brute-force scan: - `packages/atif-embed/src/atif_embed/application/embed.py:243`, + `packages/atif-embed/src/atif_embed/application/embed.py:252`, `packages/atif-embed/src/atif_embed/infrastructure/lance_store.py:414`. 9. When no row failed, hand the store's row count to `TextRowsPort.commit`: the lake reader moves its watermark to the snapshot it read, unless the @@ -271,32 +285,38 @@ deletes) the ones no step names (`packages/atif-embed/src/atif_embed/application ### Related -- `packages/atif-embed/src/atif_embed/domain/ports.py:31` -- `packages/atif-embed/src/atif_embed/domain/ports.py:59` -- `packages/atif-embed/src/atif_embed/domain/ports.py:87` +- `packages/atif-embed/src/atif_embed/domain/ports.py:37` +- `packages/atif-embed/src/atif_embed/domain/ports.py:65` +- `packages/atif-embed/src/atif_embed/domain/ports.py:93` - `packages/atif-embed/src/atif_embed/domain/errors.py:29` -- `packages/atif-cli/src/atif_cli/app.py:833` +- `packages/atif-cli/src/atif_cli/app.py:1945` ## query: one SQL statement over the catalog -Entry point: `packages/atif-cli/src/atif_cli/app.py:529` +Entry point: `packages/atif-cli/src/atif_cli/app.py:1305` 1. Short-circuit `--examples` to the examples listing without opening DuckDB; - a missing statement emits a classified parse error and exits 64 (`:564`). + a missing statement emits a classified parse error and exits 64 (`:1426-1440`). 2. Resolve the corpus root and the expected embedder identity, then open an - in-memory DuckDB connection (`:586`). -3. Register the raw readers over the contract layout (trajectory rows come + in-memory DuckDB connection (`:1461-1476`). +3. Attach the lake when it exists, its schema is current, and it holds the + corpus: every raw relation is then a view over the lake's tables scoped to + that corpus, and nothing loads before the statement runs + (`packages/atif-cli/src/atif_cli/app.py:1580`, + `packages/atif-duck/src/atif_duck/infrastructure/lake.py:1037`). Otherwise + warn once and register the raw readers over the contract layout (trajectory rows come from `read_parquet` for sessions with current columnar artifacts and from `read_json` for the rest, unioned per surface), gating trajectory, edges, and loss on `meta.json` presence so a torn session directory contributes nothing to any view: - `packages/atif-duck/src/atif_duck/infrastructure/registry.py:180`. + `packages/atif-duck/src/atif_duck/infrastructure/registry.py:597`. 4. Create the core views, then bind `message_embeddings` over the Lance store guard-before-bind: a store written by another provider raises rather than - returning numerically valid garbage cosine scores (`:333`, `:846`). -5. Create the macros, then the analytics views and analytics macros over the + returning numerically valid garbage cosine scores (`:984`, `:1601`). +5. Create the macros and the authorship views (`user_steps`, `human_turns`, + `session_outcomes`), then the analytics views and analytics macros over the analytics parquets, which bind against both the parquets and the base views - (`:1005`, `packages/atif-duck/src/atif_duck/infrastructure/analytics.py:124`). + (`:1856`, `packages/atif-duck/src/atif_duck/infrastructure/analytics.py:119`). 6. Harden the connection in a fixed order (the memory cap, thread count and private spill directory were set before step 3, since registration is what needs them): a directory allowlist holding only the spill area, a file @@ -307,95 +327,101 @@ Entry point: `packages/atif-cli/src/atif_cli/app.py:529` `EXPORT`, `ATTACH`, `DETACH`, `INSTALL`, `LOAD`, `PREPARE` and `EXECUTE` before anything runs (exit 70, `sandbox_refused`); then execute the caller's statement and stream the cursor: a plain table on a TTY, a JSON - array of row objects on a pipe (`packages/atif-cli/src/atif_cli/output.py:154`). + array of row objects on a pipe (`packages/atif-cli/src/atif_cli/output.py:158`). The spill directory is removed on exit. 8. Classify any DuckDB failure into parse, catalog, or runtime (or an embedding-provider mismatch) and exit 64, 65, or 70 with a JSON error - envelope: `packages/atif-cli/src/atif_cli/duck_errors.py:34`, `:66`. + envelope: `packages/atif-cli/src/atif_cli/duck_errors.py:35`, `:67`. ### Related -- `packages/atif-duck/src/atif_duck/infrastructure/registry.py:1212` -- `packages/atif-duck/src/atif_duck/domain/catalog.py:51` -- `packages/atif-duck/src/atif_duck/domain/embedding_guard.py:46` -- `packages/atif-cli/src/atif_cli/output.py:202` -- `packages/atif-cli/src/atif_cli/errors.py:25` +- `packages/atif-duck/src/atif_duck/infrastructure/registry.py:2095` +- `packages/atif-duck/src/atif_duck/domain/catalog.py:59` +- `packages/atif-duck/src/atif_duck/domain/embedding_guard.py:51` +- `packages/atif-cli/src/atif_cli/output.py:206` +- `packages/atif-cli/src/atif_cli/errors.py:30` ## search: semantic top-k over step embeddings -Entry point: `packages/atif-cli/src/atif_cli/app.py:860` +Entry point: `packages/atif-cli/src/atif_cli/app.py:2018` 1. Resolve the corpus root, the Lance URI, and the expected model and - dimension from `EmbedSettings` (`:878`, - `packages/atif-embed/src/atif_embed/infrastructure/settings.py:57`). + dimension from `EmbedSettings` (`:2100-2103`, + `packages/atif-embed/src/atif_embed/infrastructure/settings.py:139`). 2. Attach the lake as `query` does (or fall back to the per-session artifacts with a warning), then register the full catalog on a fresh in-memory connection; a registration failure or a provider mismatch leaves - through the classified-error path with exit 65: - `packages/atif-duck/src/atif_duck/infrastructure/registry.py:1212`, - `packages/atif-cli/src/atif_cli/duck_errors.py:66`. `--all-corpora` scopes + through the classified-error path (a provider mismatch exits 65): + `packages/atif-duck/src/atif_duck/infrastructure/registry.py:2095`, + `packages/atif-cli/src/atif_cli/duck_errors.py:67`. `--all-corpora` scopes the steps to every corpus the lake holds and binds `message_embeddings` over each corpus's store (`register_vss_stores`). -3. Count `message_embeddings` first: an empty or absent store exits 2 with the +3. Count `message_embeddings` before embedding anything: an empty or absent store exits 2 with the backfill hint instead of returning an empty result that reads like "no - matches" (`packages/atif-cli/src/atif_cli/app.py:930`). -4. Embed the query text in Cohere `search_query` float mode (Cohere forces - float for queries even when documents were stored int8): - `packages/atif-embed/src/atif_embed/application/embed.py:265`, - `packages/atif-embed/src/atif_embed/infrastructure/cohere_bedrock.py:400`. + matches" (`packages/atif-cli/src/atif_cli/app.py:2145`). A store on disk + that no installed lance extension can read exits 78 before the count + (`:2130-2143`). +4. Embed the query text with the selected provider: Cohere `search_query` + float mode by default (Cohere forces float for queries even when documents + were stored int8), or EmbeddingGemma 2's `SearchQuery` prompt: + `packages/atif-embed/src/atif_embed/application/embed.py:280`, + `packages/atif-embed/src/atif_embed/infrastructure/cohere_bedrock.py:401`. 5. Bind the query vector, the optional session filter, and `k` as parameters - rather than interpolating them: `packages/atif-cli/src/atif_cli/app.py:945`. + rather than interpolating them: `packages/atif-cli/src/atif_cli/app.py:2214-2217`. 6. Order by `array_cosine_distance` ascending: that is what triggers the cosine HNSW index lookup, and cosine is the only magnitude-invariant choice against - int8-cast document vectors whose magnitudes run into the thousands (`:919`). + int8-cast document vectors whose magnitudes run into the thousands (`:2196`). 7. Join back to `steps` on the first entry of `source_uuids` for a 200-character snippet, selecting cosine similarity as the reported score - (`:933`). -8. Emit uuid, session id, similarity, and snippet; a DuckDB error classifies to - its own exit code (`:946`, - `packages/atif-cli/src/atif_cli/output.py:129`). + (`:2188-2194`). +8. Emit uuid, session id, snippet, and similarity (plus `corpus` under + `--all-corpora`); a DuckDB error classifies to + its own exit code (`:2170-2174`, + `packages/atif-cli/src/atif_cli/output.py:132`). ### Related -- `packages/atif-duck/src/atif_duck/infrastructure/registry.py:830` -- `packages/atif-duck/src/atif_duck/domain/embedding_guard.py:46` +- `packages/atif-duck/src/atif_duck/infrastructure/registry.py:1601` +- `packages/atif-duck/src/atif_duck/domain/embedding_guard.py:51` - `packages/atif-embed/src/atif_embed/infrastructure/lance_store.py:414` -- `packages/atif-cli/src/atif_cli/errors.py:25` +- `packages/atif-cli/src/atif_cli/errors.py:30` ## refresh lane: the scheduled driver -Entry point: `scripts/atif-sql-refresh.sh:109` +Entry point: `scripts/atif-sql-refresh.sh:160` 1. Pin the identity trio once from the invoking environment and unset `AWS_PROFILE` and `AWS_DEFAULT_PROFILE`, because an inherited profile name with no matching config entry makes botocore raise at credential resolution - before Bedrock is reached (`:84`, `:96`). + before Bedrock is reached (`:135-140`, `:147`). 2. Normalize the lane argument before it names a lock file, so two spellings of one lane cannot take two locks; an unknown mode exits 64 rather than defaulting to a plane. The removed `structural` lane (and `struct`) still - parses: it logs one "lane removed" line and exits 0 (`:138`, `:143`). + parses: it logs one "lane removed" line and exits 0 (`:160-168`, `:164-167`). 3. Read the rotated Bedrock bearer token at run time and export it only when non-empty, because botocore treats an empty value as a real broken - credential and skips the default chain (`:122`). + credential and skips the default chain (`:238-241`). 4. Resolve the corpus list (the primary config directory first, then each `ATIF_SQL_EXTRA_CONFIG_DIRS` entry) and export both `CLAUDE_CONFIG_DIR` and `ATIF_SQL_SOURCE_ROOT` per corpus so the source root and the corpus slug - stay coherent (`:134`, `:312`). + stay coherent (`:250-260`, `:497-498`). 5. Resolve the CLI through the override, then a user install, then the workspace venv script, and take a nonblocking per-lane `flock`: a busy lane - skips this tick rather than queueing behind it (`:151`, `:166`). + skips this tick rather than queueing behind it (`:267-273`, `:279-285`). 6. On the `llm` lane, probe `atif-sql --help` for `analyze` and exit 0 when it is absent, so an armed crontab line against an older CLI is a no-op instead - of a nightly error (`:260`). + of a nightly error (`:324-329`). 7. Run the lane per corpus: `materialize` plus a bounded `embed --limit 500` piggyback, or `analyze --no-dry-run --llm-only --max-sessions 1000000 --max-cost-usd 1000000`, the only spending line in the file, with both caps set out of reach on - purpose (`:334`, `:371`). + purpose (`:394`, `:446`). The `materialize` lane then runs one extra + `materialize --agent codex` pass over the Codex transcripts, with no embed + piggyback (`:453`). 8. On embed exit 78, write a terminal marker keyed on the store path and its mtime and suppress further embeds for that corpus until the mtime changes; - every other nonzero exit stays transient and retries next tick (`:217`, - `:244`). + every other nonzero exit stays transient and retries next tick (`:398-403`, + `:380-390`). 9. The `compact` lane runs `atif-sql lake compact --memory-limit 2GiB` once (not per corpus; `ATIF_SQL_REFRESH_COMPACT_MEMORY_LIMIT` changes the budget) under a 4G scope, after taking the materialize lane's lock too (waiting up to @@ -404,35 +430,35 @@ Entry point: `scripts/atif-sql-refresh.sh:109` ### Related -- `scripts/atif-sql-refresh-selftest.sh:97` -- `scripts/atif-sql-refresh-selftest.sh:110` +- `scripts/atif-sql-refresh-selftest.sh:121` +- `scripts/atif-sql-refresh-selftest.sh:134` - `packages/atif-cli/src/atif_cli/cron.py:47` -- `packages/atif-cli/src/atif_cli/app.py:66` +- `packages/atif-cli/src/atif_cli/app.py:104` ## Minor flows -- status: entry at `packages/atif-cli/src/atif_cli/app.py:444`. Read-only +- status: entry at `packages/atif-cli/src/atif_cli/app.py:1147`. Read-only freshness report: scans source mtimes and replays the same pure plan - `materialize` would build without converting anything, so its stale, + `materialize` would build, generation staleness included, without converting anything, so its stale, up-to-date, and live counts are exactly what a pass would do - (`packages/atif-corpus/src/atif_corpus/domain/sessions.py:159`). -- examples: entry at `packages/atif-cli/src/atif_cli/app.py:995`. Derives one + (`packages/atif-corpus/src/atif_corpus/application/materialize.py:1433`). +- examples: entry at `packages/atif-cli/src/atif_cli/app.py:2243`. Derives one runnable example per view and macro from the static catalog and filters by `--category` and `--requires`; an unknown value exits 64 (`packages/atif-duck/src/atif_duck/domain/examples.py:191`). -- schema: entry at `packages/atif-cli/src/atif_cli/app.py:1727`. Dumps the +- schema: entry at `packages/atif-cli/src/atif_cli/app.py:2335`. Dumps the core and analytics view schemas and macro signatures, each with its `requires` value, from the static catalog with no DuckDB import and no - connection (`packages/atif-duck/src/atif_duck/domain/catalog.py:56`). -- cron install: entry at `packages/atif-cli/src/atif_cli/cron.py:185`. Renders + connection (`packages/atif-duck/src/atif_duck/domain/catalog.py:59`). +- cron install: entry at `packages/atif-cli/src/atif_cli/cron.py:186`. Renders the crontab block, plus one comment per removed lane, for a human to paste and never writes the - crontab itself (`:79`). -- cron status: entry at `packages/atif-cli/src/atif_cli/cron.py:198`. Probes + crontab itself (`:85`). +- cron status: entry at `packages/atif-cli/src/atif_cli/cron.py:207`. Probes each lane's flock nonblocking, so the probe perturbs nothing, and parses the - last completion and last skip per lane out of the refresh log (`:91`, `:109`). -- main: entry at `packages/atif-cli/src/atif_cli/app.py:1125`. Replaces - loguru's default DEBUG sink with WARNING-and-up so routine reads keep stderr - quiet, then hands control to cyclopts. + last completion and last skip per lane out of the refresh log (`:133`, `:97`). +- main: entry at `packages/atif-cli/src/atif_cli/app.py:2409`. Replaces + loguru's default DEBUG sink with WARNING-and-up (`ATIF_SQL_LOG_LEVEL` widens + it) so routine reads keep stderr quiet, then hands control to cyclopts. ## See also diff --git a/docs/behavior/state-machines.md b/docs/behavior/state-machines.md index 327c143..6b071c0 100644 --- a/docs/behavior/state-machines.md +++ b/docs/behavior/state-machines.md @@ -1,11 +1,11 @@ # atif-sql · State machines No entity in this workspace carries a `status` enum with declared transitions. The enums the workspace -declares aren't lifecycles either. `RecordType` (`packages/atif-converter/src/atif_converter/domain/fidelity.py:18`) -and `FidelityGap` (`:44`) are closed classification vocabularies whose members never move into one -another. `OutputFormat` (`packages/atif-cli/src/atif_cli/output.py:58`) has one resolution: -`resolve_format` maps `AUTO` to `TABLE` on a TTY and `JSON` otherwise (`:79-81`), but that is an -idempotent pure function evaluated per emit, a no-op for every explicit format (`:74-80`), whose result +declares aren't lifecycles either. `RecordType` (`packages/atif-converter/src/atif_converter/domain/fidelity.py:26`) +and `FidelityGap` (`:55`) are closed classification vocabularies whose members never move into one +another. `OutputFormat` (`packages/atif-cli/src/atif_cli/output.py:61`) has one resolution: +`resolve_format` maps `AUTO` to `TABLE` on a TTY and `JSON` otherwise (`:84`), but that is an +idempotent function evaluated per emit, a no-op for every explicit format (`:82-83`), whose result is never stored and never advances again. One irreversible resolution with no persisted state is not a lifecycle. @@ -16,55 +16,75 @@ names and transition labels are verbatim source text from each machine's `Define ## Corpus session materialization Every pass over the raw transcript corpus reclassifies each discovered session from scratch, so the -state is not a stored field; it is the pair "what the watermark records" and "whether the session's -artifact directory exists". The classification is the `if / elif / else` at -`packages/atif-corpus/src/atif_corpus/domain/sessions.py:200-206`, and the outcome states are the +state is not a stored field; it is what the watermark records, whether the session's artifact +directory exists, and what its `meta.json` (or the empty-session record) says about the generation +and the source. The classification is the `if / elif / else` at +`packages/atif-corpus/src/atif_corpus/domain/sessions.py:218-223`, and the outcome states are the counters and id tuples of `MaterializationReport` -(`packages/atif-corpus/src/atif_corpus/application/materialize.py:117-157`). +(`packages/atif-corpus/src/atif_corpus/application/materialize.py:220-287`). ```mermaid stateDiagram-v2 [*] --> skipped_live: scan_sources + [*] --> to_materialize: scan_sources [*] --> unreadable_session_ids: unreadable skipped_live --> to_materialize: QuiescencePolicy to_materialize --> up_to_date: _write_session + to_materialize --> empty_session_ids: EmptySourceError to_materialize --> failures: MaterializationFailure failures --> to_materialize: _advance_watermark + empty_session_ids --> up_to_date: _advance_watermark up_to_date --> skipped_live: newest_mtime_ns up_to_date --> to_materialize: _is_stale up_to_date --> to_materialize: _unmaterialized_session_ids + up_to_date --> to_materialize: _generation_stale_session_ids up_to_date --> removed_session_ids: _retain_sourceless_sessions (kept, marked) unreadable_session_ids --> skipped_live: scan_sources - removed_session_ids --> [*] + removed_session_ids --> retained_count: _retain_sourceless_sessions + retained_count --> archive_session_ids: _archive_jobs + archive_session_ids --> retained_count: _write_session + retained_count --> to_materialize: _is_stale ``` -- `skipped_live` is the entry state, not an error state. A transcript is appended to many times per +- `skipped_live` is the entry state for a session still being written, not an error state; a session + first scanned after it went quiet enters `to_materialize` directly. A transcript is appended to many times per turn, and a session qualifies to convert only once its newest source mtime is at least - `quiesce_seconds` old, 300 by default (`packages/atif-corpus/src/atif_corpus/domain/sessions.py:81`). + `quiesce_seconds` old, 300 by default (`packages/atif-corpus/src/atif_corpus/domain/sessions.py:84`). `force` overrides staleness but never quiescence, because converting a half-written transcript - produces a wrong artifact rather than a stale one (`:170-174`). + produces a wrong artifact rather than a stale one (`:183-185`). - `failures --> to_materialize` is the retry, and it works only because `_advance_watermark` retains the failed session's entries. Staleness is "recorded set differs from scanned set", so the retained entry *is* the retry signal; dropping it would classify the session `up_to_date` forever - (`packages/atif-corpus/src/atif_corpus/application/materialize.py:435-448`). + (`packages/atif-corpus/src/atif_corpus/application/materialize.py:1324-1331`). - `up_to_date --> to_materialize: _unmaterialized_session_ids` is the crash-recovery edge. It is reachable exactly one way: a pass killed inside the directory swap, after the previous generation was renamed aside and before the new one landed. The watermark records source mtimes only and cannot - express a missing artifact directory, so the directory check is what force-replans it (`:302-328`). -- `unreadable_session_ids` is a parking state, and the only state a session can starve in. A transient + express a missing artifact directory, so the directory check is what force-replans it (`:1025-1031`). +- `empty_session_ids` isn't a failure state. A converter that raises `EmptySourceError` found a + well-formed transcript with nothing to convert, so its watermark entries advance and + `empty_sessions.json` records the generation it was checked under + (`packages/atif-corpus/src/atif_corpus/application/materialize.py:1742-1745`, `:1360-1384`). The + next pass counts it `up_to_date` until its source or the converter schema moves. +- `unreadable_session_ids` is a parking state, and a session can starve in it. A transient stat failure clears next pass; a permanent one (a side-file left at mode 000) leaves the session never materialized, never `up_to_date`, never `skipped_live`, and deliberately never marked source-removed, which - is why the report carries the ids at all (`:135-142`). -- `removed_session_ids` is the sole terminal state: `shutil.rmtree` deletes the corpus session - directory (`:411`) and `_advance_watermark` drops its entries, because the raw source is - authoritative and there are no tombstones (`:386-392`). A session whose sources merely could not be - read is never marked source-removed (`:404-410`), and a scan that found zero sessions over a non-empty corpus - raises `SuspiciousEmptyScanError` instead of removing everything (`:542-548`). -- Transition sites: `scan_sources` at `:522`; `QuiescencePolicy` at `:575`; `_write_session` at `:588`; - `MaterializationFailure` at `:599`; `_retain_sourceless_sessions` (it keeps and marks, never deletes); `_advance_watermark` at - `:610`. - -Defined at: `packages/atif-corpus/src/atif_corpus/application/materialize.py:476` + is why the report carries the ids at all (`:253-261`). +- No state here deletes a session. `removed_session_ids` lists the sessions whose source vanished this + pass: `_retain_sourceless_sessions` keeps the corpus session directory and rewrites its `meta.json` + once with `source_present: false` and `source_removed_at` + (`packages/atif-corpus/src/atif_corpus/application/materialize.py:1255-1260`), because the agent expires + old transcripts and the corpus holds the only copy left (`:46-49`). Later passes count the session in + `retained_count`. `_advance_watermark` drops its entries, so a source that comes back reads as stale + and converts again from the live file (`:1333-1336`). A retained session whose recorded generation + is stale re-converts from its raw source archive, when it has one, and stays marked (`:1270-1307`). + A session whose sources merely could not be read is never marked source-removed (`:1202-1208`), and + a scan that found zero sessions over a non-empty corpus raises `SuspiciousEmptyScanError` instead of + marking every session (`:1661-1667`). +- Transition sites: `scan_sources` at `:1622`; `QuiescencePolicy` at `:1699`; `_write_session` at `:777`; + `MaterializationFailure` at `:1740`; `_retain_sourceless_sessions` (it keeps and marks, never deletes); `_advance_watermark` at + `:1778`. + +Defined at: `packages/atif-corpus/src/atif_corpus/application/materialize.py:1497` ## Embed store schema version @@ -72,7 +92,7 @@ The vector store's state is a schema generation recorded in a `schema_version.js the Lance table, written on create and checked on open. A sidecar rather than a table column, because the version describes the schema and reading it must not require the schema to be readable (`packages/atif-embed/src/atif_embed/infrastructure/lance_store.py:63-67`). `SCHEMA_VERSION` is `2` -(`:59`). +(`:61`). ```mermaid stateDiagram-v2 @@ -85,15 +105,15 @@ stateDiagram-v2 - `v1` and `unrecorded` have no incoming edge on purpose. This code never writes either state; it discriminates a store it finds on disk. `v1` is the five-column shape lacking `text_hash` and `truncated`; `unrecorded` is a store whose columns are already current but whose sidecar is absent, - which `read_schema_version` reports as `None` (`:141-154`). + which `read_schema_version` reports as `None` (`:143-156`). - `v1 --> v2` is online and metadata-only. `migrate_pre_stamp_table` adds the two columns through Lance schema evolution with SQL default expressions: no rows dropped, no vectors re-embedded, and - readers keep working throughout (`:164-190`). `text_hash` backfills to `_PRE_STAMP_SENTINEL`, the - literal `` (`:71`), whose angle brackets make it impossible to equal a real blake2b + readers keep working throughout (`:166-192`). `text_hash` backfills to `_PRE_STAMP_SENTINEL`, the + literal `` (`:73`), whose angle brackets make it impossible to equal a real blake2b digest; every migrated row therefore mismatches its corpus hash in the discovery anti-join and - re-embeds incrementally through the ordinary staleness path (`:67-70`). + re-embeds incrementally through the ordinary staleness path (`:69-72`). - `v2 --> v2: _has_table` is idempotent reopening. `migrate_pre_stamp_table` returns early when - nothing is missing (`:180-181`) and the sidecar stamp is left alone. + nothing is missing (`:182-183`) and the sidecar stamp is left alone. - The machine has no terminal state in source. `v2` is absorbing; the store is never dropped here. A breaking change (a provider or dimension switch) is refused rather than migrated, and stays fail-loud through `embedding_guard` (`:25-27`). @@ -119,16 +139,16 @@ stateDiagram-v2 ``` - The first `enqueue` lands in `backoff`, never `due`: it stamps `next_attempt_at` at now plus - `_backoff_delta(1)`, which is two minutes (`:78-81`, `:109`). Backoff is exponential in minutes - (2, 4, 8, 16, 32), and `_BACKOFF_CAP_MIN` clamps it at 60 (`:43`, `:79`). That clamp is defensive + `_backoff_delta(1)`, which is two minutes (`:78-81`, `:123`). Backoff is exponential in minutes + (2, 4, 8, 16, 32), and `_BACKOFF_CAP_MIN` clamps it at 60 (`:43`, `:80`). That clamp is defensive rather than reachable at the shipped default: `drain` admits a unit only while - `attempts < max_attempts` (`:145`), so with `MAX_ATTEMPTS_DEFAULT` of 5 (`:42`) the largest value + `attempts < max_attempts` (`:162`), so with `MAX_ATTEMPTS_DEFAULT` of 5 (`:42`) the largest value `_backoff_delta` receives is 5 and the longest real wait is 32 minutes. - `backoff` and `exhausted` together are exactly what `blocked_units` returns, and `due` is what - `drain` returns; the two are complementary partitions of the live rows (`:159-190`, `:128-156`). + `drain` returns; the two are complementary partitions of the live rows (`:176-207`, `:145-173`). Together they make the queue the single re-admission gate for a failed unit: a failed unit is not checkpointed, so without the `blocked_units` subtraction the checkpoint path would re-admit and - re-bill it on every run (`:166-176`). + re-bill it on every run (`:188-192`). - `exhausted` is terminal in effect and has no outgoing edge. `drain` never returns it, so the pipeline never dispatches it, so it never fails again and is never marked done. The guard is `keep -= blocked` at @@ -140,13 +160,13 @@ stateDiagram-v2 - `completed --> backoff` is real, and the attempt counter survives it. `enqueue` clears `completed_at` back to NULL on conflict (`packages/atif-analytics/src/atif_analytics/infrastructure/sqlite_state/retry_queue.py:128-133`) - while reading the prior `attempts` with no filter on `completed_at` (`:103-108`), so a unit that + while reading the prior `attempts` with no filter on `completed_at` (`:115-118`), so a unit that failed four times, succeeded, then failed again re-enters at `attempts = 5` and is `exhausted` immediately under the default `MAX_ATTEMPTS_DEFAULT` of 5 (`:42`). Every state name here is a row predicate, not a stored string; the table has no status column (`:45-56`). Every pipeline named by `PIPELINE_NAMES` -(`packages/atif-analytics/src/atif_analytics/infrastructure/sqlite_state/checkpointer.py:37-42`) fire +(`packages/atif-analytics/src/atif_analytics/infrastructure/sqlite_state/checkpointer.py:37-42`) fires the identical call sequence (`drain`, then `blocked_units`, then `enqueue` on failure and `mark_done` on success): diff --git a/docs/diagrams/architecture/components.md b/docs/diagrams/architecture/components.md index 781b25b..805006e 100644 --- a/docs/diagrams/architecture/components.md +++ b/docs/diagrams/architecture/components.md @@ -61,37 +61,39 @@ classDiagram Each node is one uv workspace member, the unit the import-linter contracts at `pyproject.toml:550-611` constrain. Every use case and registration entry point -is a module-level function, so the `+` entries are functions rather than methods. +is a module-level function, so the `+` entries are functions rather than methods, apart from two +in `ModelRegistry`: `spec_for` is a method of `LlmSettings`, and `classify_structured` is a Protocol +method. `AtifCliApp`'s entries each carry an `@app.command` decorator on the line -above; `app` is the `cyclopts.App` at `packages/atif-cli/src/atif_cli/app.py:52` -and `main` at `packages/atif-cli/src/atif_cli/app.py:1125` is the `atif-sql` -console script declared at `packages/atif-cli/pyproject.toml:42`. +above; `app` is the `cyclopts.App` at `packages/atif-cli/src/atif_cli/app.py:90` +and `main` at `packages/atif-cli/src/atif_cli/app.py:2409` is the `atif-sql` +console script declared at `pyproject.toml:75`. | Class | Workspace member | Method entry | Declared at | | --- | --- | --- | --- | -| `AtifCliApp` | atif-cli | `materialize` | `packages/atif-cli/src/atif_cli/app.py:352` | -| `AtifCliApp` | atif-cli | `query` | `packages/atif-cli/src/atif_cli/app.py:529` | -| `AtifCliApp` | atif-cli | `analyze` | `packages/atif-cli/src/atif_cli/app.py:1240` | -| `AtifCliApp` | atif-cli | `embed` | `packages/atif-cli/src/atif_cli/app.py:766` | -| `AtifCliApp` | atif-cli | `search` | `packages/atif-cli/src/atif_cli/app.py:860` | -| `AtifConverter` | atif-converter | `convert_and_audit` | `packages/atif-converter/src/atif_converter/application/convert_and_audit.py:105` | -| `AtifConverter` | atif-converter | `convert_session` | `packages/atif-converter/src/atif_converter/infrastructure/harbor_adapter.py:142` | +| `AtifCliApp` | atif-cli | `materialize` | `packages/atif-cli/src/atif_cli/app.py:863` | +| `AtifCliApp` | atif-cli | `query` | `packages/atif-cli/src/atif_cli/app.py:1305` | +| `AtifCliApp` | atif-cli | `analyze` | `packages/atif-cli/src/atif_cli/app.py:1634` | +| `AtifCliApp` | atif-cli | `embed` | `packages/atif-cli/src/atif_cli/app.py:1801` | +| `AtifCliApp` | atif-cli | `search` | `packages/atif-cli/src/atif_cli/app.py:2018` | +| `AtifConverter` | atif-converter | `convert_and_audit` | `packages/atif-converter/src/atif_converter/application/convert_and_audit.py:220` | +| `AtifConverter` | atif-converter | `convert_session` | `packages/atif-converter/src/atif_converter/infrastructure/harbor_adapter.py:157` | | `AtifConverter` | atif-converter | `enrich_trajectory` | `packages/atif-converter/src/atif_converter/domain/enrichment.py:198` | -| `AtifConverter` | atif-converter | `validate_trajectory` | `packages/atif-converter/src/atif_converter/infrastructure/harbor_adapter.py:97` | -| `CorpusMaterializer` | atif-corpus | `materialize` | `packages/atif-corpus/src/atif_corpus/application/materialize.py:476` | -| `CorpusMaterializer` | atif-corpus | `read_watermark` | `packages/atif-corpus/src/atif_corpus/application/materialize.py:160` | -| `CorpusMaterializer` | atif-corpus | `scan_source_root` | `packages/atif-corpus/src/atif_corpus/infrastructure/scanner.py:211` | -| `CorpusMaterializer` | atif-corpus | `build_plan` | `packages/atif-corpus/src/atif_corpus/domain/sessions.py:159` | -| `DuckRegistry` | atif-duck | `register` | `packages/atif-duck/src/atif_duck/infrastructure/registry.py:1679` | -| `DuckRegistry` | atif-duck | `register_views` | `packages/atif-duck/src/atif_duck/infrastructure/registry.py:767` | -| `DuckRegistry` | atif-duck | `register_vss` | `packages/atif-duck/src/atif_duck/infrastructure/registry.py:1273` | -| `DuckRegistry` | atif-duck | `register_macros` | `packages/atif-duck/src/atif_duck/infrastructure/registry.py:1452` | +| `AtifConverter` | atif-converter | `validate_trajectory` | `packages/atif-converter/src/atif_converter/infrastructure/harbor_adapter.py:76` | +| `CorpusMaterializer` | atif-corpus | `materialize` | `packages/atif-corpus/src/atif_corpus/application/materialize.py:1497` | +| `CorpusMaterializer` | atif-corpus | `read_watermark` | `packages/atif-corpus/src/atif_corpus/application/materialize.py:424` | +| `CorpusMaterializer` | atif-corpus | `scan_source_root` | `packages/atif-corpus/src/atif_corpus/infrastructure/scanner.py:302` | +| `CorpusMaterializer` | atif-corpus | `build_plan` | `packages/atif-corpus/src/atif_corpus/domain/sessions.py:169` | +| `DuckRegistry` | atif-duck | `register` | `packages/atif-duck/src/atif_duck/infrastructure/registry.py:2095` | +| `DuckRegistry` | atif-duck | `register_views` | `packages/atif-duck/src/atif_duck/infrastructure/registry.py:984` | +| `DuckRegistry` | atif-duck | `register_vss` | `packages/atif-duck/src/atif_duck/infrastructure/registry.py:1601` | +| `DuckRegistry` | atif-duck | `register_macros` | `packages/atif-duck/src/atif_duck/infrastructure/registry.py:1856` | | `DuckRegistry` | atif-duck | `register_authorship` | `packages/atif-duck/src/atif_duck/infrastructure/authorship.py:38` | | `DuckRegistry` | atif-duck | `build_examples` | `packages/atif-duck/src/atif_duck/domain/examples.py:191` | | `EmbedBackfill` | atif-embed | `run_backfill` | `packages/atif-embed/src/atif_embed/application/embed.py:61` | -| `EmbedBackfill` | atif-embed | `embed_query` | `packages/atif-embed/src/atif_embed/application/embed.py:265` | +| `EmbedBackfill` | atif-embed | `embed_query` | `packages/atif-embed/src/atif_embed/application/embed.py:280` | | `EmbedBackfill` | atif-embed | `discover_unembedded` | `packages/atif-embed/src/atif_embed/application/embed.py:43` | -| `AnalyticsPipelines` | atif-analytics | `run_analyze` | `packages/atif-analytics/src/atif_analytics/application/analyze.py:36` | +| `AnalyticsPipelines` | atif-analytics | `run_analyze` | `packages/atif-analytics/src/atif_analytics/application/analyze.py:37` | | `AnalyticsPipelines` | atif-analytics | `classify_sessions` | `packages/atif-analytics/src/atif_analytics/application/use_cases/classify.py:374` | | `AnalyticsPipelines` | atif-analytics | `detect_conflicts` | `packages/atif-analytics/src/atif_analytics/application/use_cases/conflicts.py:354` | | `AnalyticsPipelines` | atif-analytics | `build_provider` | `packages/atif-analytics/src/atif_analytics/application/use_cases/_shared.py:36` | @@ -103,20 +105,22 @@ console script declared at `packages/atif-cli/pyproject.toml:42`. `register` calls the other registrars, then the analytics registrars, in binding order at -`packages/atif-duck/src/atif_duck/infrastructure/registry.py:1740-1752`; +`packages/atif-duck/src/atif_duck/infrastructure/registry.py:2172-2193`; `run_analyze` dispatches the LLM stages from a table at -`packages/atif-analytics/src/atif_analytics/application/analyze.py:97-102`. +`packages/atif-analytics/src/atif_analytics/application/analyze.py:103-108`. ### Relationships -Every edge is one import plus one call. **Every cross-package import in this -workspace is indented into the function body that needs it**, never at module top. +Every edge is one import plus one call. **Every cross-package import in `atif_cli.app` +and in atif-analytics is indented into the function body that needs it**, never at module top. A fresh-interpreter test asserts that a bare `import atif_cli.app` pulls in none of `duckdb`, `harbor`, `lancedb`, `boto3`, or `polars` -(`_FORBIDDEN_EAGER_IMPORTS` at `packages/atif-cli/tests/test_lean_import.py:17-31`), -and `packages/atif-cli/src/atif_cli/app.py:22-25` records that as the reason the +(`_FORBIDDEN_EAGER_IMPORTS` at `packages/atif-cli/tests/test_lean_import.py:17-38`), +and `packages/atif-cli/src/atif_cli/app.py:50-53` records that as the reason the heavy imports sit in the command bodies. A line-anchored grep for `^from atif_` -therefore finds none of these edges. +therefore finds none of these edges. It does hit the atif-cli helper modules that `atif_cli.app` +imports inside its function bodies (`converter_adapter`, `duck_errors`, `lake_sessions`, +`embed_lake`), which import their siblings at module top. Most edges originate at `AtifCliApp`, which `packages/atif-cli/src/atif_cli/app.py:3-7` names the workspace's composition @@ -127,11 +131,11 @@ root. Those edges are the whole of its outbound surface: an unanchored grep for | Edge | Verb | Import site | Call site | | --- | --- | --- | --- | -| `AtifCliApp -> AtifConverter` | converts | `packages/atif-cli/src/atif_cli/app.py:253` | `packages/atif-cli/src/atif_cli/app.py:261` | -| `AtifCliApp -> CorpusMaterializer` | materializes | `packages/atif-cli/src/atif_cli/app.py:388` | `packages/atif-cli/src/atif_cli/app.py:396` | -| `AtifCliApp -> DuckRegistry` | registers | `packages/atif-cli/src/atif_cli/app.py:615`, `packages/atif-cli/src/atif_cli/app.py:906` | `packages/atif-cli/src/atif_cli/app.py:626`, `packages/atif-cli/src/atif_cli/app.py:918` | -| `AtifCliApp -> AnalyticsPipelines` | dispatches | `packages/atif-cli/src/atif_cli/app.py:720` | `packages/atif-cli/src/atif_cli/app.py:739` | -| `AtifCliApp -> EmbedBackfill` | embeds | `packages/atif-cli/src/atif_cli/app.py:806` | `packages/atif-cli/src/atif_cli/app.py:826` | +| `AtifCliApp -> AtifConverter` | converts | `packages/atif-cli/src/atif_cli/app.py:712` | `packages/atif-cli/src/atif_cli/app.py:726` | +| `AtifCliApp -> CorpusMaterializer` | materializes | `packages/atif-cli/src/atif_cli/app.py:931` | `packages/atif-cli/src/atif_cli/app.py:958` | +| `AtifCliApp -> DuckRegistry` | registers | `packages/atif-cli/src/atif_cli/app.py:1459`, `packages/atif-cli/src/atif_cli/app.py:2086` | `packages/atif-cli/src/atif_cli/app.py:1486`, `packages/atif-cli/src/atif_cli/app.py:2116` | +| `AtifCliApp -> AnalyticsPipelines` | dispatches | `packages/atif-cli/src/atif_cli/app.py:1697` | `packages/atif-cli/src/atif_cli/app.py:1718` | +| `AtifCliApp -> EmbedBackfill` | embeds | `packages/atif-cli/src/atif_cli/app.py:1903` | `packages/atif-cli/src/atif_cli/app.py:1937` | | `AnalyticsPipelines -> ModelRegistry` | resolves | `packages/atif-analytics/src/atif_analytics/application/use_cases/_shared.py:46` | `packages/atif-analytics/src/atif_analytics/application/use_cases/_shared.py:49`, `packages/atif-analytics/src/atif_analytics/application/use_cases/_shared.py:50` | ### Relationships the import contracts forbid @@ -141,7 +145,7 @@ root. Those edges are the whole of its outbound surface: an unanchored grep for no edge may connect any two of them. `pyproject.toml:607-611` declares a `forbidden` contract admitting exactly one analytics edge, to `atif_models`. Both are checked by `lint:imports`, one of the gates `mise run check` -depends on (`mise.toml:204`, in the list at `mise.toml:199-211`). +depends on (`mise.toml:250`, in the list at `mise.toml:245-265`). A Protocol declared in one member and implemented in another is **not** an edge between them: the implementation satisfies it structurally, and the composition @@ -151,9 +155,9 @@ atif-models, a reach atif-cli doesn't take. | Absent edge | How the same work reaches across the boundary | | --- | --- | -| `CorpusMaterializer -> AtifConverter` | `materialize` takes a `ConverterPort` parameter (`packages/atif-corpus/src/atif_corpus/application/materialize.py:480`), the Protocol at `packages/atif-corpus/src/atif_corpus/domain/ports.py:41`. `AtifCliApp` constructs the `RealConverter` adapter (`packages/atif-cli/src/atif_cli/converter_adapter.py:44`) and injects it at `packages/atif-cli/src/atif_cli/app.py:399`; that adapter's docstring names the independence contract as its reason to live in atif-cli (`packages/atif-cli/src/atif_cli/converter_adapter.py:5-9`). | -| `EmbedBackfill -> DuckRegistry` | `EmbedBackfill` reads the corpus through its own DuckDB `TextRowsPort` adapter, `DuckDbTextRows` (`packages/atif-embed/src/atif_embed/infrastructure/corpus_text_rows.py:155`) against the Protocol at `packages/atif-embed/src/atif_embed/domain/ports.py:87`, over the `docs/CONTRACT.md` corpus layout (stated at `pyproject.toml:546-548`). | -| `AnalyticsPipelines -> CorpusMaterializer` | `AnalyticsPipelines` reads the materialized corpus directly through its own `CorpusReader` (`packages/atif-analytics/src/atif_analytics/infrastructure/corpus_reader.py:314`), constructed once per run at `packages/atif-analytics/src/atif_analytics/application/analyze.py:57`. | +| `CorpusMaterializer -> AtifConverter` | `materialize` takes a `ConverterPort` parameter (`packages/atif-corpus/src/atif_corpus/application/materialize.py:1501`), the Protocol at `packages/atif-corpus/src/atif_corpus/domain/ports.py:101`. `AtifCliApp` constructs the `RealConverter` adapter (`packages/atif-cli/src/atif_cli/converter_adapter.py:80`) and injects it at `packages/atif-cli/src/atif_cli/app.py:961`; that adapter's docstring names the independence contract as its reason to live in atif-cli (`packages/atif-cli/src/atif_cli/converter_adapter.py:5-9`). | +| `EmbedBackfill -> DuckRegistry` | `EmbedBackfill` reads the corpus through its own DuckDB `TextRowsPort` adapter, `DuckDbTextRows` (`packages/atif-embed/src/atif_embed/infrastructure/corpus_text_rows.py:199`) against the Protocol at `packages/atif-embed/src/atif_embed/domain/ports.py:93`, over the `docs/CONTRACT.md` corpus layout (stated at `pyproject.toml:546-548`). With the lake on, the default, `AtifCliApp` wraps that adapter in `LakeTextRows` (`packages/atif-embed/src/atif_embed/infrastructure/lake_text_rows.py:89`), which reads the lake through the `LakeStepsPort` Protocol (`packages/atif-embed/src/atif_embed/domain/ports.py:202`) that atif-cli implements over atif-duck (`packages/atif-cli/src/atif_cli/embed_lake.py:70`). | +| `AnalyticsPipelines -> CorpusMaterializer` | `AnalyticsPipelines` reads the materialized corpus directly through its own `CorpusReader` (`packages/atif-analytics/src/atif_analytics/infrastructure/corpus_reader.py:314`), constructed once per run at `packages/atif-analytics/src/atif_analytics/application/analyze.py:62`. With the lake on, the default, `AtifCliApp` hands that reader a `SessionSource` over the lake, `LakeSessionSource` (`packages/atif-cli/src/atif_cli/lake_sessions.py:92`), which atif-cli adapts from atif-duck's lake reader. | | `AtifCliApp -> ModelRegistry` | `AtifCliApp` never reaches atif-models. Model selection happens inside `AnalyticsPipelines`, whose `build_provider` resolves a spec through `spec_for` (`packages/atif-analytics/src/atif_analytics/application/use_cases/_shared.py:49`) and hands it to `build_structured_provider` (`packages/atif-analytics/src/atif_analytics/application/use_cases/_shared.py:50`), so no model id is written down outside atif-models. | diff --git a/docs/diagrams/behavioral/sequences.md b/docs/diagrams/behavioral/sequences.md index 44586c4..6555af4 100644 --- a/docs/diagrams/behavioral/sequences.md +++ b/docs/diagrams/behavioral/sequences.md @@ -9,7 +9,7 @@ sequenceDiagram participant SCAN as scanner participant PLAN as build_plan participant ADP as RealConverter - participant HB as harbor ClaudeCode + participant CONV as converter participant FS as corpus artifacts CLI->>UC: materialize() UC->>FS: read watermark @@ -18,8 +18,8 @@ sequenceDiagram UC->>PLAN: quiesce + wmark PLAN-->>UC: to_materialize UC->>ADP: convert(jsonl) - ADP->>HB: convert events - HB-->>ADP: trajectory + ADP->>CONV: convert events + CONV-->>ADP: trajectory ADP-->>UC: conv output UC->>FS: write + swap UC->>FS: advance wmark @@ -28,23 +28,25 @@ sequenceDiagram Sources, in dispatch order: -- CLI materialize: `packages/atif-cli/src/atif_cli/app.py:352`; report rendered at `:299`. -- materialize UC: `packages/atif-corpus/src/atif_corpus/application/materialize.py:476`; - `read_watermark` at `:160` called from `:521`, `scan_sources` called at `:522`, `build_plan` called - at `:572`, `_write_session` at `:179` called from `:588`, watermark advanced at `:610`, report - built at `:613`. -- scanner: `packages/atif-corpus/src/atif_corpus/infrastructure/scanner.py:143`. -- build_plan: `packages/atif-corpus/src/atif_corpus/domain/sessions.py:159`. -- RealConverter: `packages/atif-cli/src/atif_cli/converter_adapter.py:51`, calling - `packages/atif-converter/src/atif_converter/application/convert_and_audit.py:105`. +- CLI materialize: `packages/atif-cli/src/atif_cli/app.py:863`; report rendered at `:1027`. +- materialize UC: `packages/atif-corpus/src/atif_corpus/application/materialize.py:1497`; + `read_watermark` at `:424` called from `:1620`, `scan_sources` called at `:1622`, `build_plan` called + at `:1696`, `_write_session` at `:612` called from `:777`, watermark advanced at `:1776-1779`, report + built at `:1791`. +- scanner: `packages/atif-corpus/src/atif_corpus/infrastructure/scanner.py:206`. +- build_plan: `packages/atif-corpus/src/atif_corpus/domain/sessions.py:169`. +- RealConverter: `packages/atif-cli/src/atif_cli/converter_adapter.py:80`, calling + `packages/atif-converter/src/atif_converter/application/convert_and_audit.py:220`. - converter: our ported Claude Code converter, - `packages/atif-converter/src/atif_converter/domain/claude_code_conversion.py:75`, reached from the - seam at `packages/atif-converter/src/atif_converter/infrastructure/harbor_adapter.py:83`; harbor - supplies only the data classes and the validator (`:68`). + `packages/atif-converter/src/atif_converter/domain/claude_code_conversion.py:981`, reached from the + seam at `packages/atif-converter/src/atif_converter/infrastructure/harbor_adapter.py:128`; the ATIF + data classes and the validator the seam checks with (`:76`) are harbor's, vendored under + `atif_converter.domain.atif`. - corpus artifacts: path arithmetic in - `packages/atif-corpus/src/atif_corpus/domain/layout.py:26`; the artifacts written and the - directory swapped at `packages/atif-corpus/src/atif_corpus/application/materialize.py:222-240` - through `packages/atif-corpus/src/atif_corpus/infrastructure/atomic.py:64` and `:98`. + `packages/atif-corpus/src/atif_corpus/domain/layout.py:65`; the artifacts written and the + directory swapped at `packages/atif-corpus/src/atif_corpus/application/materialize.py:691-733` + through `write_json_atomic` (`packages/atif-corpus/src/atif_corpus/infrastructure/atomic.py:72`), + its zstd siblings (`:142`, `:159`), and `replace_dir_atomic` (`:165`). ## query @@ -65,6 +67,7 @@ sequenceDiagram DB->>LN: read model,dim LN-->>DB: stored id+dim REG->>DB: register_macros + REG->>DB: register_authorship REG->>DB: analytics DDL CLI->>DB: harden + lock CLI->>DB: execute(sql) @@ -74,24 +77,27 @@ sequenceDiagram Sources, in dispatch order: -- CLI query: `packages/atif-cli/src/atif_cli/app.py:529`; `duckdb.connect()` at `:591`, `register` - called at `:594`, `_harden_query_connection` at `:137` called from `:601`, caller SQL executed at - `:606`, emit at `:612`. -- atif_duck register: `packages/atif-duck/src/atif_duck/infrastructure/registry.py:1212`, whose - documented order is fixed at `:1283-1294`: `register_raw` `:196`, `register_views` `:333`, - `register_vss` `:846`, `register_macros` `:1005`, then the analytics pair. +- CLI query: `packages/atif-cli/src/atif_cli/app.py:1305`; `duckdb.connect()` at `:1476`, the lake + attached at `:1482` when it holds the corpus, `register` called at `:1486`, + `_harden_query_connection` at `:522` called from `:1495`, caller SQL executed at `:1514`, emit at + `:1520`. +- atif_duck register: `packages/atif-duck/src/atif_duck/infrastructure/registry.py:2095`, whose + documented order is fixed at `:2116-2124`: `register_raw` `:597` (`register_lake_raw` instead when + the lake is attached), `register_views` `:984`, `register_vss` `:1601`, `register_macros` `:1856`, + then the authorship surface and the analytics pair. - DuckDB in-process: the connection opened by the CLI; the sandbox settings and - `lock_configuration` are issued against it at `packages/atif-cli/src/atif_cli/app.py:180-188`. -- corpus artifacts: session globs read through `read_json` at - `packages/atif-duck/src/atif_duck/infrastructure/registry.py:210-213`; the analytics parquets bind - through `packages/atif-duck/src/atif_duck/infrastructure/analytics.py:124`, with the analytics - macros at `:225`. + `lock_configuration` are issued against it at `packages/atif-cli/src/atif_cli/app.py:582-591`. +- corpus artifacts: on the per-session path, session globs read through `read_json` at + `packages/atif-duck/src/atif_duck/infrastructure/registry.py:715-730`; the analytics parquets bind + through `packages/atif-duck/src/atif_duck/infrastructure/analytics.py:119`, with the analytics + macros at `:236`. - LanceDB store: reached through DuckDB's lance extension; - `packages/atif-duck/src/atif_duck/infrastructure/registry.py:878-887` installs, loads, and - ATTACHes it; the stamped `(model, dim)` identity is read at `:937` and checked by - `packages/atif-duck/src/atif_duck/domain/embedding_guard.py:46`. -- stdout emitter: `packages/atif-cli/src/atif_cli/output.py:154`; a registration failure routes - through `packages/atif-cli/src/atif_cli/duck_errors.py:66` instead. + `packages/atif-duck/src/atif_duck/infrastructure/registry.py:1663-1680` loads it and ATTACHes the + store (installing it is `install_lance_extension` at `:1585`, behind `atif-sql embed`); the + stamped `(model, dim)` identity is read at `:1712` and checked by + `packages/atif-duck/src/atif_duck/domain/embedding_guard.py:51`. +- stdout emitter: `packages/atif-cli/src/atif_cli/output.py:158`; a registration failure routes + through `packages/atif-cli/src/atif_cli/duck_errors.py:67` instead. ## analyze @@ -119,13 +125,13 @@ sequenceDiagram Sources, in dispatch order: -- CLI analyze: `packages/atif-cli/src/atif_cli/app.py:1240`; settings resolved at `:1296`, - `run_analyze` called at `:1312`. -- run_analyze: `packages/atif-analytics/src/atif_analytics/application/analyze.py:36`; the - `stages` list at `:97-102`, each stage invoked at `:123`. +- CLI analyze: `packages/atif-cli/src/atif_cli/app.py:1634`; settings resolved at `:1700-1707`, + `run_analyze` called at `:1718`. +- run_analyze: `packages/atif-analytics/src/atif_analytics/application/analyze.py:37`; the + `stages` list at `:103-108`, each stage invoked at `:129`. - CorpusReader: `packages/atif-analytics/src/atif_analytics/infrastructure/corpus_reader.py:314`, constructed once per run at - `packages/atif-analytics/src/atif_analytics/application/analyze.py:57`; `session_text` at + `packages/atif-analytics/src/atif_analytics/application/analyze.py:62`; `session_text` at `packages/atif-analytics/src/atif_analytics/infrastructure/corpus_reader.py:484`. - LLM stages: the classify exemplar at `packages/atif-analytics/src/atif_analytics/application/use_cases/classify.py:374`; structured diff --git a/docs/diagrams/structural/dependency-graph.md b/docs/diagrams/structural/dependency-graph.md index 1c67ed4..fdb98e8 100644 --- a/docs/diagrams/structural/dependency-graph.md +++ b/docs/diagrams/structural/dependency-graph.md @@ -7,7 +7,7 @@ the uv workspace members declared by `[tool.uv.workspace] members = ["packages/* The internal direction is enforced rather than conventional. `pyproject.toml:550` declares `[tool.importlinter]` over every root package (`:551`), and `mise run lint:imports` -(`mise.toml:159-162`) is one of the gates `mise run check` depends on (`mise.toml:197-211`). Some of +(`mise.toml:172`) is one of the gates `mise run check` depends on (`mise.toml:245-265`). Some of those contracts fix the shape drawn here: the `independence` contract at `pyproject.toml:602-605` forbids any import edge among atif_converter, atif_corpus, atif_duck, atif_models, and atif_embed, and the `forbidden` contract at `:607-611` allows atif_analytics to import atif_models and nothing else @@ -48,11 +48,10 @@ flowchart LR duck --> duckdb converter -->|all members| loguru analytics --> polars - analytics --> pyarrow + duck --> pyarrow embed --> lancedb - embed --> pydsettings + duck --> pydsettings embed --> boto3 - models --> pydantic models --> anyio models --> tenacity models --> anthropic @@ -67,7 +66,9 @@ grepping every `*.py` under `packages/*/src` for a `from X` or `import X` line a | elided node | importing module | declared at | import site | | --- | --- | --- | --- | -| pytz | none | `packages/atif-duck/pyproject.toml:29` | no import site; DuckDB's own client imports it to materialize TIMESTAMPTZ, per the comment at `:26-28` | +| pytz | none | `packages/atif-duck/pyproject.toml:33` | no import site; DuckDB's own client imports it to materialize TIMESTAMPTZ, per the comment at `:30-32` | +| `zstandard` | atif-corpus most often; atif-analytics, atif-converter, atif-duck, and atif-embed too | `packages/atif-corpus/pyproject.toml:22` | `packages/atif-corpus/src/atif_corpus/infrastructure/source_archive.py:38` | +| `imagesize` | atif-converter | `packages/atif-converter/pyproject.toml:30` | `packages/atif-converter/src/atif_converter/domain/blobs.py:69` | ## Internal edges @@ -81,17 +82,17 @@ from a file under the source member's `src/`. | atif-cli to atif-corpus | as above; declared `packages/atif-cli/pyproject.toml:34` | | atif-cli to atif-duck | as above; declared `packages/atif-cli/pyproject.toml:35` | | atif-cli to atif-embed | as above; declared `packages/atif-cli/pyproject.toml:36` | -| atif-analytics to atif-models | the `forbidden` contract's single permitted edge (`pyproject.toml:607-611`); declared `packages/atif-analytics/pyproject.toml:24` | +| atif-analytics to atif-models | the `forbidden` contract's single permitted edge (`pyproject.toml:607-611`); declared `packages/atif-analytics/pyproject.toml:23` | Absences carry meaning too. atif-cli imports atif_models in no source file even though it composes everything else; it reaches the model registry through atif-analytics, and its -`[project.dependencies]` list omits atif-models accordingly (`packages/atif-cli/pyproject.toml:32-39`). +`[project.dependencies]` list omits atif-models accordingly (`packages/atif-cli/pyproject.toml:32-43`). And no edge exists in either direction between atif-converter, atif-corpus, atif-duck, atif-embed, or atif-models: they communicate only by writing and reading the corpus on disk. Adding any such edge fails the `lint:imports` gate of `mise run check`. -The inter-member dependencies are `==0.1.0`-pinned rather than bare -(`packages/atif-cli/pyproject.toml:32-36`) because `[tool.uv.sources]` at `:59-64` is a local source an +The inter-member dependencies are `==0.3.0`-pinned rather than bare +(`packages/atif-cli/pyproject.toml:32-36`) because `[tool.uv.sources]` at `:60-65` is a local source an external installer never sees; a bare name would ship as a `Requires-Dist` resolved from the public PyPI namespace. @@ -102,51 +103,54 @@ on import-site count, then on the member's own src line count, descending. | external node | edge drawn from | declared at | import site | | --- | --- | --- | --- | -| loguru | atif-converter | `packages/atif-converter/pyproject.toml:31` | `packages/atif-converter/src/atif_converter/domain/codex_conversion.py:75` | +| loguru | atif-converter | `packages/atif-converter/pyproject.toml:24` | `packages/atif-converter/src/atif_converter/domain/codex_conversion.py:70` | | polars | atif-analytics | `packages/atif-analytics/pyproject.toml:25` | `packages/atif-analytics/src/atif_analytics/application/use_cases/classify.py:41` | -| pydantic | atif-models | `packages/atif-models/pyproject.toml:30` | `packages/atif-models/src/atif_models/infrastructure/openai_bedrock.py:33` | -| duckdb | atif-duck | `packages/atif-duck/pyproject.toml:20` | `packages/atif-duck/src/atif_duck/infrastructure/registry.py:60` | -| pydantic-settings | atif-embed | `packages/atif-embed/pyproject.toml:27` | `packages/atif-embed/src/atif_embed/infrastructure/settings.py:16` | +| pydantic | atif-converter | `packages/atif-converter/pyproject.toml:23` | `packages/atif-converter/src/atif_converter/domain/atif/trajectory.py:10` | +| duckdb | atif-duck | `packages/atif-duck/pyproject.toml:20` | `packages/atif-duck/src/atif_duck/infrastructure/registry.py:132` | +| pydantic-settings | atif-duck | `packages/atif-duck/pyproject.toml:25` | `packages/atif-duck/src/atif_duck/infrastructure/lake_settings.py:15` | | tenacity | atif-models | `packages/atif-models/pyproject.toml:32` | `packages/atif-models/src/atif_models/infrastructure/openai_bedrock.py:34` | -| pyarrow | atif-analytics | `packages/atif-analytics/pyproject.toml:26` | `packages/atif-analytics/src/atif_analytics/infrastructure/parquet_cache.py:137` | +| pyarrow | atif-duck | `packages/atif-duck/pyproject.toml:29` | `packages/atif-duck/src/atif_duck/infrastructure/columnar.py:287` | | lancedb | atif-embed | `packages/atif-embed/pyproject.toml:22` | `packages/atif-embed/src/atif_embed/infrastructure/lance_store.py:42` | | anyio | atif-models | `packages/atif-models/pyproject.toml:23` | `packages/atif-models/src/atif_models/infrastructure/openai_bedrock.py:30` | -| cyclopts | atif-cli | `packages/atif-cli/pyproject.toml:37` | `packages/atif-cli/src/atif_cli/app.py:35` | +| cyclopts | atif-cli | `packages/atif-cli/pyproject.toml:37` | `packages/atif-cli/src/atif_cli/app.py:69` | | boto3 | atif-embed | `packages/atif-embed/pyproject.toml:20` | `packages/atif-embed/src/atif_embed/infrastructure/cohere_bedrock.py:136` | | anthropic | atif-models | `packages/atif-models/pyproject.toml:22` | `packages/atif-models/src/atif_models/infrastructure/anthropic_bedrock.py:51` | What the drawn edge deliberately compresses: - **loguru is universal.** Every member declares it (`packages/atif-analytics/pyproject.toml:24`, - `packages/atif-cli/pyproject.toml:43`, `packages/atif-converter/pyproject.toml:31`, + `packages/atif-cli/pyproject.toml:43`, `packages/atif-converter/pyproject.toml:24`, `packages/atif-corpus/pyproject.toml:19`, `packages/atif-duck/pyproject.toml:21`, `packages/atif-embed/pyproject.toml:23`, `packages/atif-models/pyproject.toml:29`), and the edge is drawn from atif-converter only because more of the importing files are its than any other member's. The edge label states the real fan-out. - **atif-corpus has no drawn external edge.** It declares its externals: loguru - (`packages/atif-corpus/pyproject.toml:19`), pydantic (`:20`), pydantic-settings (`:21`). Each is - imported more often, or in more sites, by another member, so the attribution rule sources all of them - elsewhere. Its own import sites are real: `packages/atif-corpus/src/atif_corpus/infrastructure/settings.py:19` + (`packages/atif-corpus/pyproject.toml:19`), pydantic (`:20`), pydantic-settings (`:21`), and + `zstandard` (`:22`). Another member imports loguru and pydantic in more files, and atif-duck wins the + pydantic-settings tie on src line count, so the attribution rule sources those three elsewhere. atif-corpus + imports `zstandard` in more files than any other member, and `zstandard` is one of the nodes the legend + leaves off the diagram. Its own import sites are real: `packages/atif-corpus/src/atif_corpus/infrastructure/settings.py:26` and `packages/atif-corpus/src/atif_corpus/domain/sessions.py:29`. -- **harbor is a dev dependency now.** Production code builds on harbor 0.23.0's ATIF data classes +- **harbor is a dev dependency.** Production code builds on harbor 0.24.0's ATIF data classes and validator, vendored in `packages/atif-converter/src/atif_converter/domain/atif/`, and imports nothing from harbor itself. - The conversion itself is ours, ported from 0.22.0 - (`packages/atif-converter/src/atif_converter/domain/claude_code_conversion.py:75`, - `packages/atif-converter/src/atif_converter/domain/codex_conversion.py:781`). An `ast` guard keeps + The conversions are ours: the Claude Code one is ported from harbor 0.24.0 + (`packages/atif-converter/src/atif_converter/domain/claude_code_conversion.py:2`), and the Codex one + from 0.22.0, brought to 0.24.0 + (`packages/atif-converter/src/atif_converter/domain/codex_conversion.py:2-3`). An `ast` guard keeps `harbor` and `litellm` out of every `src/` tree (`packages/atif-converter/tests/test_dev_only_imports_guard.py`), and harbor's private converters are reachable from the parity oracle in the tests only - (`packages/atif-converter/tests/harbor_oracle.py:94`). + (`packages/atif-converter/tests/harbor_oracle.py:43`). ## Declared dependencies are not the installed closure The diagram's external nodes are what a member asks for, and that is a strictly smaller set than what -`uv sync` installs. The gap used to include a web stack this system never runs: harbor's own dependency block -lists `fastapi`, `supabase`, and `uvicorn`, and harbor was a runtime dependency until it moved to the -dev group. No member declares any of them, and +`uv sync` installs. The gap includes a web stack this system never runs: harbor, a dev-group +dependency (`pyproject.toml:160`), lists `fastapi`, `supabase`, and `uvicorn` in its own dependency +block, and `uv sync` installs the dev group by default. No member declares any of them, and `grep -rnE "^[[:space:]]*(from|import) +(fastapi|uvicorn|starlette|supabase)(\.| |$)" --include='*.py' packages/` returns no matches across every source and test file. atif-sql exposes no HTTP surface; it is one -console script, `atif-sql = "atif_cli.app:main"` (`packages/atif-cli/pyproject.toml:42`), over a set of +console script, `atif-sql = "atif_cli.app:main"` (`pyproject.toml:75`), over a set of libraries. Every storage engine in the graph is embedded, so no server node belongs here either: duckdb runs diff --git a/docs/insights/business-logic.md b/docs/insights/business-logic.md index 2594ebc..375c43e 100644 --- a/docs/insights/business-logic.md +++ b/docs/insights/business-logic.md @@ -27,25 +27,25 @@ per run, per pipeline) is stated per row. | Rule | Domain | Citation | Failure mode | | --- | --- | --- | --- | -| A session path must end in `.jsonl` AND be an existing file | Conversion | `packages/atif-converter/src/atif_converter/infrastructure/harbor_adapter.py:50` | raise `InvalidSessionInput`; CLI exits 64 | -| Production code imports nothing from harbor or litellm (the ATIF data classes and the validator are vendored in `packages/atif-converter/src/atif_converter/domain/atif/`) | Conversion | `packages/atif-converter/tests/test_dev_only_imports_guard.py`; the conversion itself is ours at `packages/atif-converter/src/atif_converter/domain/claude_code_conversion.py:75` and `packages/atif-converter/src/atif_converter/domain/codex_conversion.py:781` | the `ast` guard fails the suite on any `harbor` or `litellm` import in any member's `src/` | -| The converter must return a trajectory for the session | Conversion | `packages/atif-converter/src/atif_converter/infrastructure/harbor_adapter.py:83` | raise `EmptySessionError`; CLI exits 2 | -| The trajectory must pass harbor's `TrajectoryValidator` AFTER enrichment mutates `extra` | Conversion | `packages/atif-converter/src/atif_converter/infrastructure/harbor_adapter.py:68`, `packages/atif-converter/src/atif_converter/application/convert_and_audit.py:107` | `validation_errors` is non-empty; the materialize adapter raises `TrajectoryValidationError` | -| No source file may change, vanish, or appear between the one read that parses and fingerprints it and the end of the audit; the converter and the audit consume that one read's records, so each file is opened twice (read, then re-hash), parsed once, hashed twice | Conversion | `packages/atif-converter/src/atif_converter/application/convert_and_audit.py:93-102`, checked once after enrichment at `packages/atif-converter/src/atif_converter/application/convert_and_audit.py`; the read is `packages/atif-converter/src/atif_converter/infrastructure/raw_records.py` (`load_session`); tests `packages/atif-converter/tests/test_snapshot_and_drift.py` (`TestConvertRefusesOnMidConversionWrite`, `TestSinglePass`) | raise `SourceMutatedDuringConversion`; the materialize pass records the session as failed and continues | -| A JSONL line that is not valid JSON, or parses to a non-dict, is not a record | Conversion | `packages/atif-converter/src/atif_converter/infrastructure/raw_records.py:136-142` | silent drop, DEBUG log (same policy as harbor) | +| A session path must end in `.jsonl` AND be an existing file | Conversion | `packages/atif-converter/src/atif_converter/infrastructure/harbor_adapter.py:71-73` | raise `InvalidSessionInput`; CLI exits 64 | +| Production code imports nothing from harbor or litellm (the ATIF data classes and the validator are vendored in `packages/atif-converter/src/atif_converter/domain/atif/`) | Conversion | `packages/atif-converter/tests/test_dev_only_imports_guard.py`; the conversion itself is ours at `packages/atif-converter/src/atif_converter/domain/claude_code_conversion.py:981` and `packages/atif-converter/src/atif_converter/domain/codex_conversion.py:924` | the `ast` guard fails the suite on any `harbor` or `litellm` import in any member's `src/` | +| The converter must return a trajectory for the session | Conversion | `packages/atif-converter/src/atif_converter/infrastructure/harbor_adapter.py:115-117` | raise `EmptySessionError`; CLI exits 2 | +| The trajectory must pass the vendored ATIF `TrajectoryValidator` AFTER enrichment mutates `extra` | Conversion | `packages/atif-converter/src/atif_converter/infrastructure/harbor_adapter.py:76`, `packages/atif-converter/src/atif_converter/application/convert_and_audit.py:289` | `validation_errors` is non-empty; the materialize adapter raises `TrajectoryValidationError` | +| No source file may change, vanish, or appear between the one read that parses and fingerprints it and the end of the audit; the converter and the audit consume that one read's records, so each file is opened twice (read, then re-hash), parsed once, hashed twice | Conversion | `packages/atif-converter/src/atif_converter/application/convert_and_audit.py:11-24`, checked once after enrichment at `packages/atif-converter/src/atif_converter/application/convert_and_audit.py:285`; the read is `packages/atif-converter/src/atif_converter/infrastructure/raw_records.py` (`load_session`); tests `packages/atif-converter/tests/test_snapshot_and_drift.py` (`TestConvertRefusesOnMidConversionWrite`, `TestSinglePass`) | raise `SourceMutatedDuringConversion`; the materialize pass records the session as failed and continues | +| A JSONL line that is not valid JSON is not a record; a line that parses to a non-object is kept for the converter, as harbor keeps it, and only the audit's record pairs drop it | Conversion | `packages/atif-converter/src/atif_converter/infrastructure/raw_records.py:199-202`, `packages/atif-converter/src/atif_converter/infrastructure/raw_records.py:316` | invalid JSON: silent drop, DEBUG log (same policy as harbor) | | An agent step and an assistant-record group pair up only when their tool ids intersect, or when BOTH sides carry no tool ids | Conversion | `packages/atif-converter/src/atif_converter/domain/enrichment.py:304-318`; test `packages/atif-converter/tests/test_enrichment.py:331` | refuse: attribution stops at that step, `enrichment_truncated_at_step` recorded, WARNING logged | | A user step and a user text record pair up only while the record's harbor-derived text equals the step's message | Conversion | `packages/atif-converter/src/atif_converter/domain/enrichment.py:347-360`; test `packages/atif-converter/tests/test_enrichment.py:372` | refuse: same truncation marker, WARNING logged | -| A session materializes only once its newest source mtime is at least `quiesce_seconds` old (default 300 seconds of source silence, per session) | Corpus | `packages/atif-corpus/src/atif_corpus/domain/sessions.py:83-104` and `packages/atif-corpus/src/atif_corpus/domain/sessions.py:201-202`; tests `packages/atif-corpus/tests/test_domain.py:40` (inclusive boundary) and `packages/atif-corpus/tests/test_domain.py:98` | deferred into `skipped_live`; revisited next pass | -| A source mtime in the FUTURE never satisfies the quiescence threshold | Corpus | `packages/atif-corpus/src/atif_corpus/domain/sessions.py:96-104`; test `packages/atif-corpus/tests/test_domain.py:45` | stays skipped, plus a WARNING every pass so the starvation is visible | -| `watermark.json` must parse as a mapping of path to int | Corpus | `packages/atif-corpus/src/atif_corpus/application/materialize.py:160-176` | degrade to empty (one full re-materialization pass), WARNING logged, never a refusal to sync | -| A scan finding zero sessions while the corpus holds materialized ones is not a deletion | Corpus | `packages/atif-corpus/src/atif_corpus/application/materialize.py:97-104` and `packages/atif-corpus/src/atif_corpus/application/materialize.py:553-559`; test `packages/atif-corpus/tests/test_materialize.py:292` | raise `SuspiciousEmptyScanError`; nothing is removed | -| Only a genuine `FileNotFoundError` proves a source vanished; any other `OSError` means unreadable | Corpus | `packages/atif-corpus/src/atif_corpus/infrastructure/scanner.py:83-96` and `packages/atif-corpus/src/atif_corpus/infrastructure/scanner.py:176-186`; test `packages/atif-corpus/tests/test_materialize.py:819` | session lands in `SourceScan.unreadable`, is never ghosted, and its watermark entries are retained | -| A retry-queue or checkpoint `pipeline` must be one of the contract names in `PIPELINE_NAMES` | Analytics | `packages/atif-analytics/src/atif_analytics/infrastructure/sqlite_state/retry_queue.py:96-98`, `packages/atif-analytics/src/atif_analytics/infrastructure/sqlite_state/checkpointer.py:37-42`; tests `packages/atif-analytics/tests/test_state.py:145` and `packages/atif-analytics/tests/test_state.py:150` | raise `ValueError` | +| A session materializes only once its newest source mtime is at least `quiesce_seconds` old (default 300 seconds of source silence, per session) | Corpus | `packages/atif-corpus/src/atif_corpus/domain/sessions.py:84` and `packages/atif-corpus/src/atif_corpus/domain/sessions.py:218-219`; tests `packages/atif-corpus/tests/test_domain.py:40` (inclusive boundary) and `packages/atif-corpus/tests/test_domain.py:98` | deferred into `skipped_live`; revisited next pass | +| A source mtime in the FUTURE never satisfies the quiescence threshold | Corpus | `packages/atif-corpus/src/atif_corpus/domain/sessions.py:106-114`; test `packages/atif-corpus/tests/test_domain.py:45` | stays skipped, plus a WARNING every pass once the mtime is more than `future_warn_seconds` (1 second) ahead, so the starvation is visible | +| `watermark.json` must parse as a mapping of path to int | Corpus | `packages/atif-corpus/src/atif_corpus/application/materialize.py:424-452` | degrade to empty (one full re-materialization pass), WARNING logged, never a refusal to sync | +| A scan finding zero sessions while the corpus holds materialized ones (with every source directory listed) is not a deletion | Corpus | `packages/atif-corpus/src/atif_corpus/application/materialize.py:179` and `packages/atif-corpus/src/atif_corpus/application/materialize.py:1661-1667`; test `packages/atif-corpus/tests/test_materialize.py:380` | raise `SuspiciousEmptyScanError`; nothing is marked source-removed | +| Only a genuine `FileNotFoundError` proves a source vanished; any other `OSError` means unreadable | Corpus | `packages/atif-corpus/src/atif_corpus/infrastructure/scanner.py:113-125` and `packages/atif-corpus/src/atif_corpus/infrastructure/scanner.py:264-273`; test `packages/atif-corpus/tests/test_materialize.py:909` | session lands in `SourceScan.unreadable`, is never ghosted, and its watermark entries are retained | +| A `pipeline` added to the retry queue must be one of the contract names in `PIPELINE_NAMES` (the tuple the checkpoint module defines) | Analytics | `packages/atif-analytics/src/atif_analytics/infrastructure/sqlite_state/retry_queue.py:96-98`, `packages/atif-analytics/src/atif_analytics/infrastructure/sqlite_state/checkpointer.py:37-42`; tests `packages/atif-analytics/tests/test_state.py:145` and `packages/atif-analytics/tests/test_state.py:150` | raise `ValueError` | | Every LLM-output `confidence` is bounded to the closed interval 0.0 to 1.0 (dimensionless, per emitted row) | Analytics | `packages/atif-analytics/src/atif_analytics/domain/models.py:66-69`, `packages/atif-analytics/src/atif_analytics/domain/models.py:146-149`, `packages/atif-analytics/src/atif_analytics/domain/models.py:238-241`, `packages/atif-analytics/src/atif_analytics/domain/models.py:342-345` | pydantic `ValidationError` at `model_validate`, translated to `ProviderUnavailable` by either adapter (`packages/atif-models/src/atif_models/infrastructure/openai_bedrock.py:242-246`, `packages/atif-models/src/atif_models/infrastructure/anthropic_bedrock.py:284-288`); the unit goes to the retry queue | | Every categorical LLM-output field is a closed `Literal` union, and free text is length-capped (goal 280 chars, rationale 200 chars, evidence 280 chars, summary 280 chars, a turn uuid 64 chars) | Analytics | `packages/atif-analytics/src/atif_analytics/domain/models.py:35`, `packages/atif-analytics/src/atif_analytics/domain/models.py:56-59`, `packages/atif-analytics/src/atif_analytics/domain/models.py:82-85`, `packages/atif-analytics/src/atif_analytics/domain/models.py:228-231`, `packages/atif-analytics/src/atif_analytics/domain/models.py:320-323`, `packages/atif-analytics/src/atif_analytics/domain/models.py:331-334` | same pydantic rejection path | -| A returned `turn_uuid` must be in the session's USER-role main-chain text-step header uuid set | Analytics | `packages/atif-analytics/src/atif_analytics/application/use_cases/perceived.py:130-138` and `packages/atif-analytics/src/atif_analytics/application/use_cases/perceived.py:299-316`; tests `packages/atif-analytics/tests/test_perceived.py:167`, `packages/atif-analytics/tests/test_resource_guards.py:617` | row dropped with a WARNING; an EMPTY universe drops every row (fails closed) | -| A returned conflict pair's two uuids must BOTH be real `edges.jsonl` uuids for that session | Analytics | `packages/atif-analytics/src/atif_analytics/application/use_cases/conflicts.py:283-310`; test `packages/atif-analytics/tests/test_resource_guards.py:590` | pair dropped; an unreadable universe drops every pair (fails closed) | -| An untrusted step body may not present itself as a turn header | Analytics | `packages/atif-analytics/src/atif_analytics/domain/transcript.py:70-91`; tests `packages/atif-analytics/tests/test_resource_guards.py:561` and `packages/atif-analytics/tests/test_resource_guards.py:576` | coerce: `[uuid=` is rewritten to `(uuid=` case-insensitively, length preserved, before any header is built | +| A returned `turn_uuid` must be in the session's human-turn header uuid set (main-chain user steps a human wrote) | Analytics | `packages/atif-analytics/src/atif_analytics/application/use_cases/perceived.py:130-138` and `packages/atif-analytics/src/atif_analytics/application/use_cases/perceived.py:299-316`; tests `packages/atif-analytics/tests/test_perceived.py:167`, `packages/atif-analytics/tests/test_resource_guards.py:498` | row dropped with a WARNING; an EMPTY universe drops every row (fails closed) | +| A returned conflict pair's two uuids must BOTH be real `edges.jsonl` uuids for that session | Analytics | `packages/atif-analytics/src/atif_analytics/application/use_cases/conflicts.py:283-310`; test `packages/atif-analytics/tests/test_resource_guards.py:471` | pair dropped; an unreadable universe drops every pair (fails closed) | +| An untrusted step body may not present itself as a turn header | Analytics | `packages/atif-analytics/src/atif_analytics/domain/transcript.py:70-91`; tests `packages/atif-analytics/tests/test_resource_guards.py:442` and `packages/atif-analytics/tests/test_resource_guards.py:457` | coerce: `[uuid=` is rewritten to `(uuid=` case-insensitively, length preserved, before any header is built | | A friction candidate is a human turn (a main-chain user step whose author is `human`) with a uuid, 1 to `friction_max_chars` characters (default 300, per message); hook feedback, task notifications, retry nudges and image metadata never qualify | Analytics | `packages/atif-analytics/src/atif_analytics/application/use_cases/friction.py:125-143`, `packages/atif-analytics/src/atif_analytics/domain/transcript.py:228-230`; test `packages/atif-analytics/tests/test_friction_tiers.py:98` | filtered out at the candidate boundary; no LLM call is ever made for it | | `ATIF_SQL_LLM_FAMILY` must name a family with a wired provider adapter; both registry families (`openai`, `anthropic`) have one, so the validator guards a family that reaches the registry before its adapter does | Models | `packages/atif-models/src/atif_models/infrastructure/settings.py:25` and `packages/atif-models/src/atif_models/infrastructure/settings.py:41-54`; test `packages/atif-models/tests/test_settings.py:68` (narrows `RUNNABLE_FAMILIES` to prove the refusal) | raise `ValueError` at settings load: a startup refusal instead of a Bedrock 400 on every call | | `size_for` accepts only a pipeline with an `llm_size_` field | Models | `packages/atif-models/src/atif_models/infrastructure/settings.py:68-78`; test `packages/atif-models/tests/test_settings.py:95` | raise `KeyError` | @@ -54,17 +54,17 @@ per run, per pipeline) is stated per row. | `stop_reason` of `refusal` is a decline (anthropic adapter) | Models | `packages/atif-models/src/atif_models/infrastructure/anthropic_bedrock.py:252-257`; test `packages/atif-models/tests/test_anthropic_provider.py:150` | raise `RefusalError` naming the `stop_details.category` (terminal, never retried) | | `finish_reason` of `length` gets exactly ONE retry, at `reasoning_effort` degraded one step (openai adapter) | Models | `packages/atif-models/src/atif_models/infrastructure/bedrock_common.py:50-55`, `packages/atif-models/src/atif_models/infrastructure/openai_bedrock.py:149-181`, `packages/atif-models/src/atif_models/infrastructure/openai_bedrock.py:226-232`; tests `packages/atif-models/tests/test_openai_provider.py:129`, `packages/atif-models/tests/test_openai_provider.py:144`, `packages/atif-models/tests/test_openai_provider.py:163` | raise `ProviderUnavailable` when still truncated or already at the floor; the unit goes to the retry queue | | `stop_reason` of `max_tokens` gets exactly ONE retry, at `output_config.effort` degraded one step on the same ladder (anthropic adapter) | Models | `packages/atif-models/src/atif_models/infrastructure/anthropic_bedrock.py:185-219`, `packages/atif-models/src/atif_models/infrastructure/anthropic_bedrock.py:258-264`; tests `packages/atif-models/tests/test_anthropic_provider.py:158` and `packages/atif-models/tests/test_anthropic_provider.py:171` | raise `ProviderUnavailable` when still truncated or already at the floor; the unit goes to the retry queue | -| Response content must be a non-empty string that parses as JSON and validates against the schema (openai adapter) | Models | `packages/atif-models/src/atif_models/infrastructure/openai_bedrock.py:233-246`; tests `packages/atif-models/tests/test_openai_provider.py:198`, `packages/atif-models/tests/test_openai_provider.py:206`, `packages/atif-models/tests/test_openai_provider.py:212` | raise `ProviderUnavailable` | +| A response must carry a choice, and its content must be a non-empty string that parses as JSON and validates against the schema (openai adapter) | Models | `packages/atif-models/src/atif_models/infrastructure/openai_bedrock.py:213-216`, `packages/atif-models/src/atif_models/infrastructure/openai_bedrock.py:233-246`; tests `packages/atif-models/tests/test_openai_provider.py:198`, `packages/atif-models/tests/test_openai_provider.py:206`, `packages/atif-models/tests/test_openai_provider.py:212` | raise `ProviderUnavailable` | | An anthropic response must stop at `end_turn`, and its `text` blocks (any `thinking` block is skipped) must join into JSON that validates against the schema | Models | `packages/atif-models/src/atif_models/infrastructure/anthropic_bedrock.py:265-288`; tests `packages/atif-models/tests/test_anthropic_provider.py:139`, `packages/atif-models/tests/test_anthropic_provider.py:185`, `packages/atif-models/tests/test_anthropic_provider.py:191`, `packages/atif-models/tests/test_anthropic_provider.py:199` | raise `ProviderUnavailable` | | An anthropic string enum value that differs from one declared member only in letter case is that member | Models | `packages/atif-models/src/atif_models/infrastructure/anthropic_bedrock.py:102-127`, `packages/atif-models/src/atif_models/infrastructure/anthropic_bedrock.py:282-283`; tests `packages/atif-models/tests/test_anthropic_provider.py:218`, `packages/atif-models/tests/test_anthropic_provider.py:225`, `packages/atif-models/tests/test_anthropic_provider.py:232`, `packages/atif-models/tests/test_anthropic_provider.py:237` | coerce: folded onto the declared spelling before `model_validate`; a value matching no member, or more than one, passes through for pydantic to reject | -| A store's stamped `(model, dim)` must match the active embedder before any vector is read or appended | Embeddings | `packages/atif-embed/src/atif_embed/domain/embedding_guard.py:38-80`, `packages/atif-embed/src/atif_embed/application/embed.py:166-174`; tests `packages/atif-embed/tests/test_embed_use_case.py:153`, `packages/atif-embed/tests/test_lance_store.py:321` and `packages/atif-embed/tests/test_lance_store.py:330` | raise `EmbeddingProviderMismatch` (`terminal = True`); `query`/`search` exit 65, `embed` exits 78 | +| A store's stamped `(model, dim)` must match the active embedder before any vector is read or appended | Embeddings | `packages/atif-embed/src/atif_embed/domain/embedding_guard.py:38-80`, `packages/atif-embed/src/atif_embed/application/embed.py:173-183`; tests `packages/atif-embed/tests/test_embed_use_case.py:153`, `packages/atif-embed/tests/test_lance_store.py:321` and `packages/atif-embed/tests/test_lance_store.py:330` | raise `EmbeddingProviderMismatch` (`terminal = True`); `query`/`search` exit 65, `embed` exits 78 | | `ensure_index` accepts only `cosine`, `l2`, or `dot` | Embeddings | `packages/atif-embed/src/atif_embed/infrastructure/lance_store.py:414-419`; test `packages/atif-embed/tests/test_lance_store.py:110` | raise `ValueError` | | A text over 50,000 characters (per embeddable text) cannot be embedded whole | Embeddings | `packages/atif-embed/src/atif_embed/domain/text_stamp.py:35`, `packages/atif-embed/src/atif_embed/domain/text_stamp.py:43-47`, `packages/atif-embed/src/atif_embed/domain/text_stamp.py:65-68`; tests `packages/atif-embed/tests/test_embed_use_case.py:369` and `packages/atif-embed/tests/test_embed_use_case.py:386` | coerce: sent head-only, and the row stamps `truncated = true` so the search miss is attributable | -| Every macro parameter name must have an `ARG_EXEMPLARS` entry | SQL surface | `packages/atif-duck/src/atif_duck/domain/examples.py:58-75`, `packages/atif-duck/src/atif_duck/domain/examples.py:148-158`; test `packages/atif-duck/tests/test_examples.py:188` | `build_examples()` raises; the drift test fails with it | -| Every catalog object must yield a runnable example or carry a documented `EXCLUSIONS` entry | SQL surface | `packages/atif-duck/src/atif_duck/domain/examples.py:94-99`; test `packages/atif-duck/tests/test_examples.py:160` | drift test fails | -| A materialized session dir with no `meta.json` is incomplete | SQL surface | `packages/atif-duck/src/atif_duck/infrastructure/registry.py:160-177`, `packages/atif-duck/src/atif_duck/infrastructure/registry.py:188-192`, `packages/atif-duck/src/atif_duck/infrastructure/registry.py:213-221`; test `packages/atif-duck/tests/test_duck_views.py:630` | excluded from every view, plus one WARNING per skipped dir so the exclusion is observable | -| A real `embed` run needs an explicit scope: `--limit N` or `--all` | CLI | `packages/atif-cli/src/atif_cli/app.py:810-820` | exit 64 with the hint naming both flags and `--dry-run` | -| `search` against an empty embeddings store is not a zero-result query | CLI | `packages/atif-cli/src/atif_cli/app.py:930-941` | exit 2 with the hint `atif-sql embed --all --no-dry-run` | +| Every macro parameter name must have an `ARG_EXEMPLARS` entry | SQL surface | `packages/atif-duck/src/atif_duck/domain/examples.py:59-76`, `packages/atif-duck/src/atif_duck/domain/examples.py:161-171`; test `packages/atif-duck/tests/test_examples.py:218` | `build_examples()` raises; the drift test fails with it | +| Every catalog object must yield a runnable example or carry a documented `EXCLUSIONS` entry | SQL surface | `packages/atif-duck/src/atif_duck/domain/examples.py:95-100`; test `packages/atif-duck/tests/test_examples.py:190` | drift test fails | +| A materialized session dir with no `meta.json` is incomplete | SQL surface | `packages/atif-duck/src/atif_duck/infrastructure/registry.py:639-644`, `packages/atif-duck/src/atif_duck/infrastructure/registry.py:703-709`, `packages/atif-duck/src/atif_duck/infrastructure/registry.py:527-532`; test `packages/atif-duck/tests/test_duck_views.py:690` | excluded from every view, plus one WARNING per skipped dir so the exclusion is observable | +| A real `embed` run needs an explicit scope: `--limit N` or `--all` | CLI | `packages/atif-cli/src/atif_cli/app.py:1908-1918` | exit 64 with the hint naming both flags and `--dry-run` | +| `search` against an empty embeddings store is not a zero-result query | CLI | `packages/atif-cli/src/atif_cli/app.py:2145-2156` | exit 2 with the hint `atif-sql embed --all --no-dry-run` | Notes on two rows above. The strict-JSON transform in `packages/atif-models/src/atif_models/domain/schema.py:6-11` is a *rewriting* validation @@ -87,34 +87,33 @@ pinned by `packages/atif-models/tests/test_anthropic_provider.py:112` and | Invariant | Where enforced | Citation | | --- | --- | --- | -| The `FidelityGap` members are the complete, typed statement of what harbor 0.22.0's conversion loses; some are structural and observed on every session, the rest are session-conditional | Application code | `packages/atif-converter/src/atif_converter/domain/fidelity.py:44-79`; `packages/atif-converter/src/atif_converter/application/convert_and_audit.py:52-59` (structural set) and `packages/atif-converter/src/atif_converter/application/convert_and_audit.py:70-76` (conditional set); test `packages/atif-converter/tests/test_convert_and_audit.py:163` | -| Only `RecordType.USER` and `RecordType.ASSISTANT` are convertible; every other record type is dropped upstream | Application code | `packages/atif-converter/src/atif_converter/domain/fidelity.py:41`; test `packages/atif-converter/tests/test_convert_and_audit.py:121` | -| `LossReport.records_total` equals the `edges.jsonl` line count for any session, because the census and the edges emitter read one snapshot | Application code | `packages/atif-converter/src/atif_converter/domain/fidelity.py:91-98`, `packages/atif-converter/src/atif_converter/infrastructure/census.py:57-67`, `packages/atif-converter/src/atif_converter/application/convert_and_audit.py:137-139`; test `packages/atif-converter/tests/test_snapshot_and_drift.py:122` | -| `records_converted` counts raw user/assistant RECORDS, not ATIF steps; harbor bundles several assistant events sharing one `message.id` into one agent step, so the two numbers differ by design | Application code | `packages/atif-converter/src/atif_converter/application/convert_and_audit.py:122-126`, `packages/atif-converter/src/atif_converter/application/convert_and_audit.py:62-68` | -| `subagent_files_convertible` equals `subagent_files_found`: our converter discovers every side file itself, workflow-nested ones included, so nothing found is unconvertible; the retired `workflow_subagents_missed` value is never emitted | Application code | `packages/atif-converter/src/atif_converter/application/convert_and_audit.py:64`; test `packages/atif-converter/tests/test_convert_and_audit.py:127` | -| A truncation marker or a non-zero leftover count means enrichment attribution was REFUSED, never guessed | Application code | `packages/atif-converter/src/atif_converter/domain/enrichment.py:45-62`, `packages/atif-converter/src/atif_converter/domain/enrichment.py:379-399`; tests `packages/atif-converter/tests/test_enrichment.py:272`, `packages/atif-converter/tests/test_enrichment.py:287`, `packages/atif-converter/tests/test_enrichment.py:302` | -| Staged side-files are per-FILE symlinks under real directories, flattened with `__` joins, because Python 3.13's `rglob` does not descend a symlinked directory and a flat namespace can collide | Application code | `packages/atif-converter/src/atif_converter/infrastructure/harbor_adapter.py:71-92` | -| `edges.jsonl` line order is `(ts, uuid)` with missing values sorting first as empty strings, and the key order is contract-fixed | Application code | `packages/atif-converter/src/atif_converter/domain/edges.py:22-32`, `packages/atif-converter/src/atif_converter/domain/edges.py:82-90`; tests `packages/atif-converter/tests/test_edges.py:43` and `packages/atif-converter/tests/test_edges.py:47` | -| A renamed harbor private method and a bad transcript must not look alike in a materialization log | Application code | `packages/atif-converter/src/atif_converter/domain/errors.py:47-54`; test `packages/atif-converter/tests/test_snapshot_and_drift.py:331` | +| The `FidelityGap` members are the typed list of what the conversion loses, inherited from harbor 0.22.0's converter that this one ports for parity; `parent_chain_flattened` is structural and observed on every session, the rest are session-conditional | Application code | `packages/atif-converter/src/atif_converter/domain/fidelity.py:55-99`; `packages/atif-converter/src/atif_converter/application/convert_and_audit.py:79` (structural set) and `packages/atif-converter/src/atif_converter/application/convert_and_audit.py:151-155` (conditional set); test `packages/atif-converter/tests/test_convert_and_audit.py:157` | +| Only `RecordType.USER` and `RecordType.ASSISTANT` are convertible; of the other record types, the ones `session_events` keeps are captured and the rest are dropped | Application code | `packages/atif-converter/src/atif_converter/domain/fidelity.py:52`; test `packages/atif-converter/tests/test_convert_and_audit.py:123` | +| `LossReport.records_total` equals the `edges.jsonl` line count for any session, because the census and the edges emitter read one snapshot | Application code | `packages/atif-converter/src/atif_converter/domain/fidelity.py:123-127`, `packages/atif-converter/src/atif_converter/infrastructure/census.py:57-67`, `packages/atif-converter/src/atif_converter/application/convert_and_audit.py:266-268`; test `packages/atif-converter/tests/test_snapshot_and_drift.py:133` | +| `records_converted` counts raw user/assistant RECORDS, not ATIF steps; harbor bundles several assistant events sharing one `message.id` into one agent step, so the two numbers differ by design | Application code | `packages/atif-converter/src/atif_converter/application/convert_and_audit.py:238-241`, `packages/atif-converter/src/atif_converter/application/convert_and_audit.py:143-147` | +| `subagent_files_convertible` equals `subagent_files_found`: our converter discovers every side file itself, workflow-nested ones included, so nothing found is unconvertible; the retired `workflow_subagents_missed` value is never emitted | Application code | `packages/atif-converter/src/atif_converter/application/convert_and_audit.py:167-170`; test `packages/atif-converter/tests/test_convert_and_audit.py:129` | +| A truncation marker or a non-zero leftover count means enrichment attribution was REFUSED, never guessed | Application code | `packages/atif-converter/src/atif_converter/domain/enrichment.py:46-62`, `packages/atif-converter/src/atif_converter/domain/enrichment.py:379-399`; tests `packages/atif-converter/tests/test_enrichment.py:272`, `packages/atif-converter/tests/test_enrichment.py:287`, `packages/atif-converter/tests/test_enrichment.py:302` | +| Side-files are read in place, each keyed by its path under the side directory joined with `__` (a leading `subagents` part dropped), the flat name harbor's staging gave it; when two flatten to one name, the first in sorted `rglob` order wins | Application code | `packages/atif-converter/src/atif_converter/infrastructure/claude_code_converter.py:46-69` | +| `edges.jsonl` line order is `(ts, uuid)` with missing values sorting first as empty strings, and the key order is contract-fixed | Application code | `packages/atif-converter/src/atif_converter/domain/edges.py:22-32`, `packages/atif-converter/src/atif_converter/domain/edges.py:82-97`; tests `packages/atif-converter/tests/test_edges.py:47` and `packages/atif-converter/tests/test_edges.py:81` | ### Corpus | Invariant | Where enforced | Citation | | --- | --- | --- | -| A reader never observes a torn artifact SET: all the artifacts are written into `/.staging/` and the whole directory is swapped in by rename | Application code | `packages/atif-corpus/src/atif_corpus/infrastructure/atomic.py:98-137`, `packages/atif-corpus/src/atif_corpus/application/materialize.py:190-244`; tests `packages/atif-corpus/tests/test_atomic.py:85`, `packages/atif-corpus/tests/test_materialize.py:346` | -| `meta.json` is written LAST inside the staging directory, and it is the marker `atif-duck` gates every reader on | Application code, both sides | `packages/atif-corpus/src/atif_corpus/application/materialize.py:207-212`, `packages/atif-duck/src/atif_duck/infrastructure/registry.py:188-192`; test `packages/atif-duck/tests/test_duck_views.py:630` | -| Persistence order matches write order: each artifact is fsynced before the rename that publishes it, and the parent directory after | Application code | `packages/atif-corpus/src/atif_corpus/infrastructure/atomic.py:64-95`, `packages/atif-corpus/src/atif_corpus/infrastructure/atomic.py:140-158`; tests `packages/atif-corpus/tests/test_atomic.py:131`, `packages/atif-corpus/tests/test_atomic.py:149`, `packages/atif-corpus/tests/test_atomic.py:194` | -| A kill inside the swap window leaves the session dir MISSING rather than torn, and the next pass force-replans it | Application code | `packages/atif-corpus/src/atif_corpus/infrastructure/atomic.py:113-123`, `packages/atif-corpus/src/atif_corpus/application/materialize.py:313-339`; tests `packages/atif-corpus/tests/test_materialize.py:791`, `packages/atif-corpus/tests/test_domain.py:216` | -| Only sessions that materialized successfully advance their watermark entries, so a failed session stays stale and is retried | Application code | `packages/atif-corpus/src/atif_corpus/application/materialize.py:428-473`, `packages/atif-corpus/src/atif_corpus/application/materialize.py:615-622`; test `packages/atif-corpus/tests/test_materialize.py:333` | -| A VANISHED source path keeps its watermark entry unless the owning session succeeded this pass or left the scan entirely; the retained entry IS the staleness signal | Application code | `packages/atif-corpus/src/atif_corpus/application/materialize.py:455-473`; tests `packages/atif-corpus/tests/test_materialize.py:191` and `packages/atif-corpus/tests/test_materialize.py:227` | -| `owns_path` is the single definition of a session's watermark scope, so staleness and retention can never disagree about which entries belong to a session | Application code | `packages/atif-corpus/src/atif_corpus/domain/sessions.py:129-140`; tests `packages/atif-corpus/tests/test_domain.py:191`, `packages/atif-corpus/tests/test_domain.py:202`, `packages/atif-corpus/tests/test_domain.py:207` | +| A reader never observes a torn artifact SET: all the artifacts are written into `/.staging/` and the whole directory is swapped in by rename | Application code | `packages/atif-corpus/src/atif_corpus/infrastructure/atomic.py:165-204`, `packages/atif-corpus/src/atif_corpus/application/materialize.py:627-632`; tests `packages/atif-corpus/tests/test_atomic.py:85`, `packages/atif-corpus/tests/test_materialize.py:435` | +| `meta.json` is written LAST inside the staging directory, and it is the marker `atif-duck` gates every reader on | Application code, both sides | `packages/atif-corpus/src/atif_corpus/application/materialize.py:706-731`, `packages/atif-duck/src/atif_duck/infrastructure/registry.py:703-709`; test `packages/atif-duck/tests/test_duck_views.py:690` | +| Persistence order matches write order: each artifact is fsynced before the rename that publishes it, and the parent directory after | Application code | `packages/atif-corpus/src/atif_corpus/infrastructure/atomic.py:72-103`, `packages/atif-corpus/src/atif_corpus/infrastructure/atomic.py:51-69`; tests `packages/atif-corpus/tests/test_atomic.py:131`, `packages/atif-corpus/tests/test_atomic.py:149`, `packages/atif-corpus/tests/test_atomic.py:194` | +| A kill inside the swap window leaves the session dir MISSING rather than torn, and the next pass force-replans it | Application code | `packages/atif-corpus/src/atif_corpus/infrastructure/atomic.py:180-189`, `packages/atif-corpus/src/atif_corpus/application/materialize.py:1017-1052`; tests `packages/atif-corpus/tests/test_materialize.py:881`, `packages/atif-corpus/tests/test_domain.py:216` | +| Only sessions that materialized successfully (or converted to nothing) advance their watermark entries, so a failed session stays stale and is retried | Application code | `packages/atif-corpus/src/atif_corpus/application/materialize.py:1310-1357`, `packages/atif-corpus/src/atif_corpus/application/materialize.py:1776-1779`; test `packages/atif-corpus/tests/test_materialize.py:422` | +| A VANISHED source path keeps its watermark entry unless the owning session succeeded this pass or left the scan entirely; the retained entry IS the staleness signal | Application code | `packages/atif-corpus/src/atif_corpus/application/materialize.py:1341-1352`; tests `packages/atif-corpus/tests/test_materialize.py:218` and `packages/atif-corpus/tests/test_materialize.py:254` | +| `owns_path` is the single definition of a session's watermark scope, so staleness and retention can never disagree about which entries belong to a session | Application code | `packages/atif-corpus/src/atif_corpus/domain/sessions.py:139`; tests `packages/atif-corpus/tests/test_domain.py:191`, `packages/atif-corpus/tests/test_domain.py:202`, `packages/atif-corpus/tests/test_domain.py:207` | | Only an exactly equal `mtime_ns` counts as unchanged; an mtime that moved BACKWARDS is `modified` | Application code | `packages/atif-corpus/src/atif_corpus/domain/watermark.py:61-76`; test `packages/atif-corpus/tests/test_domain.py:86` | -| A staging entry is removed only when EVERY pid its name carries is proven gone; only `ProcessLookupError` proves that | Application code | `packages/atif-corpus/src/atif_corpus/application/materialize.py:256-310`; tests `packages/atif-corpus/tests/test_materialize.py:429`, `packages/atif-corpus/tests/test_materialize.py:457`, `packages/atif-corpus/tests/test_materialize.py:481` | -| One broken transcript never aborts a corpus sync: the port may raise anything, and the pass records the failure and continues | Application code | `packages/atif-corpus/src/atif_corpus/domain/ports.py:41-47`, `packages/atif-corpus/src/atif_corpus/application/materialize.py:597-616`; test `packages/atif-corpus/tests/test_materialize.py:316` | -| The domain reads no clock: `now_ns` is always passed in, so identical inputs always yield an identical plan | Application code | `packages/atif-corpus/src/atif_corpus/domain/sessions.py:18-21`, `packages/atif-corpus/src/atif_corpus/application/materialize.py:48-52`, `packages/atif-corpus/src/atif_corpus/application/materialize.py:528`; test `packages/atif-corpus/tests/test_domain.py:145` | -| Every plan partition is sorted by session id, so the same scan yields the same work order, log output, and failure ordering | Application code | `packages/atif-corpus/src/atif_corpus/domain/sessions.py:110-121`, `packages/atif-corpus/src/atif_corpus/domain/sessions.py:195`; test `packages/atif-corpus/tests/test_domain.py:145` | -| The corpus root `~/.claude` maps to the reserved slug `default`; every other root is `-<8 hex of sha256 of the resolved path>` | Application code | `packages/atif-corpus/src/atif_corpus/domain/slug.py:23`, `packages/atif-corpus/src/atif_corpus/domain/slug.py:45-50`; tests `packages/atif-corpus/tests/test_slug.py:19`, `packages/atif-corpus/tests/test_slug.py:24`, `packages/atif-corpus/tests/test_slug.py:32` | -| The per-session artifact filenames and the watermark filename are fixed by the contract | Application code | `packages/atif-corpus/src/atif_corpus/domain/layout.py:17-22`; test `packages/atif-corpus/tests/test_domain.py:245` | +| A staging entry is removed only when EVERY pid its name carries is proven gone; only `ProcessLookupError` proves that | Application code | `packages/atif-corpus/src/atif_corpus/application/materialize.py:944-1014`; tests `packages/atif-corpus/tests/test_materialize.py:518`, `packages/atif-corpus/tests/test_materialize.py:546`, `packages/atif-corpus/tests/test_materialize.py:570` | +| One broken transcript never aborts a corpus sync: the port may raise anything, and the pass records the failure and continues | Application code | `packages/atif-corpus/src/atif_corpus/domain/ports.py:101-107`, `packages/atif-corpus/src/atif_corpus/application/materialize.py:793-798`; test `packages/atif-corpus/tests/test_materialize.py:405` | +| The domain reads no clock: `now_ns` is always passed in, so identical inputs always yield an identical plan | Application code | `packages/atif-corpus/src/atif_corpus/domain/sessions.py:18-21`, `packages/atif-corpus/src/atif_corpus/application/materialize.py:104-110`, `packages/atif-corpus/src/atif_corpus/application/materialize.py:1625`; test `packages/atif-corpus/tests/test_domain.py:145` | +| Every plan partition is sorted by session id, so the same scan yields the same work order, log output, and failure ordering | Application code | `packages/atif-corpus/src/atif_corpus/domain/sessions.py:121-123`, `packages/atif-corpus/src/atif_corpus/domain/sessions.py:212`; test `packages/atif-corpus/tests/test_domain.py:145` | +| The corpus root `~/.claude` maps to the reserved slug `default` and `~/.codex/sessions` to `codex`; every other root is `-<8 hex of sha256 of the resolved path>` | Application code | `packages/atif-corpus/src/atif_corpus/domain/slug.py:32`, `packages/atif-corpus/src/atif_corpus/domain/slug.py:40`, `packages/atif-corpus/src/atif_corpus/domain/slug.py:65-72`; tests `packages/atif-corpus/tests/test_slug.py:25`, `packages/atif-corpus/tests/test_slug.py:30`, `packages/atif-corpus/tests/test_slug.py:44`, `packages/atif-corpus/tests/test_slug.py:52` | +| The per-session artifact filenames and the watermark filename are fixed by the contract | Application code | `packages/atif-corpus/src/atif_corpus/domain/layout.py:18-25`; test `packages/atif-corpus/tests/test_domain.py:245` | ### Analytics @@ -123,12 +122,12 @@ pinned by `packages/atif-models/tests/test_anthropic_provider.py:112` and | The retry queue is the SINGLE re-admission gate for a failed unit: a live entry blocks dispatch until `drain` says it is due, because a failed unit is never checkpointed and would otherwise be re-admitted fresh every run forever | Application code | `packages/atif-analytics/src/atif_analytics/infrastructure/sqlite_state/retry_queue.py:176-193`, `packages/atif-analytics/src/atif_analytics/application/use_cases/perceived.py:169-177`; tests `packages/atif-analytics/tests/test_state.py:124`, `packages/atif-analytics/tests/test_perceived.py:237` | | A FAILED unit is not checkpointed; a REFUSED unit is (terminal), and its refusal is durable: an in-band sentinel row for session-keyed `classify`, a `analytics/refusals` sidecar row for the uuid-keyed pipelines | Application code | `packages/atif-analytics/src/atif_analytics/application/use_cases/classify.py:247-270`, `packages/atif-analytics/src/atif_analytics/application/use_cases/_shared.py:196-234`, `packages/atif-analytics/src/atif_analytics/application/use_cases/perceived.py:281-292`; test `packages/atif-analytics/tests/test_perceived.py:258` | | Nothing is stamped (no checkpoint, no cache row, no retry entry) for a unit that was never dispatched | Application code | `packages/atif-analytics/src/atif_analytics/application/use_cases/_shared.py:160-188`, `packages/atif-analytics/src/atif_analytics/application/use_cases/perceived.py:250-270`; test `packages/atif-analytics/tests/test_perceived.py:286` | -| Budget overshoot is bounded by `BUDGET_CHECK_BATCH` units (8, per dispatch batch) on the priciest watched model, and does not scale with the session ceiling or the write chunk size | Application code | `packages/atif-analytics/src/atif_analytics/application/use_cases/_shared.py:108-115`, `packages/atif-analytics/src/atif_analytics/application/use_cases/_shared.py:151-157`; test `packages/atif-analytics/tests/test_resource_guards.py:362` (over both dispatch shapes) | -| A checkpoint skip requires BOTH `last_ts` and `last_mtime` to be non-advancing; either bound moving forward re-admits the session | Application code | `packages/atif-analytics/src/atif_analytics/infrastructure/sqlite_state/checkpointer.py:136-176`; tests `packages/atif-analytics/tests/test_state.py:48` and `packages/atif-analytics/tests/test_state.py:57` | -| `state.db` is corpus-scoped at `/analytics/state.db`, so one corpus's completions can never skip another's re-scoring | Application code | `packages/atif-analytics/src/atif_analytics/infrastructure/sqlite_state/checkpointer.py:22-24` | +| Budget overshoot is bounded by `BUDGET_CHECK_BATCH` units (8, per dispatch batch) on the priciest watched model, and does not scale with the session ceiling or the write chunk size | Application code | `packages/atif-analytics/src/atif_analytics/application/use_cases/_shared.py:108-115`, `packages/atif-analytics/src/atif_analytics/application/use_cases/_shared.py:151-157`; test `packages/atif-analytics/tests/test_resource_guards.py:265` (parametrized over classify and conflicts) | +| A checkpoint skip requires BOTH `last_ts` and `last_mtime` to be non-advancing; either bound moving forward re-admits the session | Application code | `packages/atif-analytics/src/atif_analytics/infrastructure/sqlite_state/checkpointer.py:137-177`; tests `packages/atif-analytics/tests/test_state.py:48` and `packages/atif-analytics/tests/test_state.py:57` | +| `state.db` is corpus-scoped at `/analytics/state.db`, so one corpus's completions can never skip another's re-scoring | Application code | `packages/atif-analytics/src/atif_analytics/infrastructure/sqlite_state/checkpointer.py:23-24` | | Sidechain steps are excluded from every rendered transcript, because the prompts are calibrated on transcripts with no subagent content and harbor inlines sidechains | Application code | `packages/atif-analytics/src/atif_analytics/domain/transcript.py:31-33`, `packages/atif-analytics/src/atif_analytics/domain/transcript.py:242-244`, `packages/atif-analytics/src/atif_analytics/domain/transcript.py:282`; test `packages/atif-analytics/tests/test_transcript.py:91` | | A compact-summary step is synthetic text: its author is `harness`, so it never counts toward a human-AI pair and renders under the `harness` label | Application code | `packages/atif-analytics/src/atif_analytics/domain/transcript.py:146-149`, `packages/atif-analytics/src/atif_analytics/domain/transcript.py:295-297`; test `packages/atif-analytics/tests/test_transcript.py:137` | -| Only `render_session_text` writes a real `[uuid=` header; every body is escaped first | Application code | `packages/atif-analytics/src/atif_analytics/domain/transcript.py:43-48`, `packages/atif-analytics/src/atif_analytics/domain/transcript.py:275-277`, `packages/atif-analytics/src/atif_analytics/domain/transcript.py:298-302`; test `packages/atif-analytics/tests/test_resource_guards.py:561` | +| Only `render_session_text` writes a real `[uuid=` header; every body is escaped first | Application code | `packages/atif-analytics/src/atif_analytics/domain/transcript.py:43-48`, `packages/atif-analytics/src/atif_analytics/domain/transcript.py:275-277`, `packages/atif-analytics/src/atif_analytics/domain/transcript.py:298-302`; test `packages/atif-analytics/tests/test_resource_guards.py:442` | | Every LLM prompt fences its transcript or message in a `` or `` tag pair whose id is the first 8 hex digits of the content's sha256, with the one-line request after the fence, so a body cannot close its own fence and a rerun sends the same bytes | Application code | `packages/atif-analytics/src/atif_analytics/application/prompts.py:732-756`; tests `packages/atif-analytics/tests/test_prompts.py:96`, `packages/atif-analytics/tests/test_prompts.py:103`, `packages/atif-analytics/tests/test_prompts.py:108`, `packages/atif-analytics/tests/test_prompts.py:117` | | Who wrote a user step is a prefix match over `AUTHOR_PREFIX_RULES` after stripping leading space, tab, CR and LF, first rule wins, with a compaction summary always `harness`; the SQL `step_author` macro and the Python twin in atif-analytics must agree | Application code | `packages/atif-duck/src/atif_duck/domain/authorship.py:51`, `packages/atif-analytics/src/atif_analytics/domain/authorship.py:108-122`; tests `packages/atif-duck/tests/test_authorship_twin_pin.py:64`, `packages/atif-cli/tests/test_authorship_parity.py:53` | | classify and conflicts admit only `interactive` sessions: a `turn_audit` (opens on an audit prompt) or `one_shot_job` (at most one human turn) is checkpointed as skipped without a call | Application code | `packages/atif-analytics/src/atif_analytics/application/use_cases/classify.py:160-184`, `packages/atif-analytics/src/atif_analytics/domain/authorship.py:130` | @@ -143,25 +142,25 @@ pinned by `packages/atif-models/tests/test_anthropic_provider.py:112` and | `estimate_cost` returns `None`, never `0.0`, when either rate is unknown; callers must render "pricing unavailable" | Application code | `packages/atif-models/src/atif_models/domain/registry.py:196-203`; tests `packages/atif-models/tests/test_registry.py:135` and `packages/atif-models/tests/test_registry.py:145` | | Usage is accumulated BEFORE any `finish_reason` or `stop_reason` gate, because a truncated or refused call still billed tokens | Application code | `packages/atif-models/src/atif_models/infrastructure/openai_bedrock.py:209-212`, `packages/atif-models/src/atif_models/infrastructure/anthropic_bedrock.py:248-251`; tests `packages/atif-models/tests/test_openai_provider.py:251` and `packages/atif-models/tests/test_anthropic_provider.py:171` | | `RETRY_CODES` (the retryable Bedrock codes) is a deliberate TWIN of atif-embed's set, pinned against the same literal in each package's own suite because the two may not import each other | Application code | `packages/atif-models/src/atif_models/infrastructure/bedrock_common.py:27-42` (shared by both adapters); tests `packages/atif-models/tests/test_openai_provider.py:344` and `packages/atif-models/tests/test_openai_provider.py:348` | -| Every embeddings row stamps `model`, `dim`, `text_hash`, and `truncated`, so a re-conversion that changes a step's text is detectable under the same uuid | Application code | `packages/atif-embed/src/atif_embed/domain/text_stamp.py:6-18`, `packages/atif-embed/src/atif_embed/application/embed.py:216-235`; test `packages/atif-embed/tests/test_embed_use_case.py:89` | -| A uuid never fans out to two vectors: a stale row under the same uuid is DELETED before the new vector is appended | Application code | `packages/atif-embed/src/atif_embed/application/embed.py:203-210`, `packages/atif-embed/src/atif_embed/domain/text_stamp.py:50-63`; tests `packages/atif-embed/tests/test_embed_use_case.py:211`, `packages/atif-embed/tests/test_lance_store.py:129` | +| Every embeddings row stamps `model`, `dim`, `text_hash`, and `truncated`, so a re-conversion that changes a step's text is detectable under the same uuid | Application code | `packages/atif-embed/src/atif_embed/domain/text_stamp.py:7-18`, `packages/atif-embed/src/atif_embed/application/embed.py:225-234`; test `packages/atif-embed/tests/test_embed_use_case.py:89` | +| A uuid never fans out to two vectors: a stale row under the same uuid is DELETED before the new vector is appended | Application code | `packages/atif-embed/src/atif_embed/application/embed.py:213-219`, `packages/atif-embed/src/atif_embed/domain/text_stamp.py:50-63`; tests `packages/atif-embed/tests/test_embed_use_case.py:211`, `packages/atif-embed/tests/test_lance_store.py:129` | | `_PRE_STAMP_SENTINEL` (``) can never equal a real blake2b hex digest, so every pre-stamp row mismatches its corpus hash and re-embeds through the ordinary staleness path | Application code | `packages/atif-embed/src/atif_embed/infrastructure/lance_store.py:69-73`; test `packages/atif-embed/tests/test_lance_store.py:192` | -| ADDITIVE schema drift migrates ONLINE (metadata-only `add_columns`, no re-embedding, search stays up); a BREAKING change (provider or dimension switch) stays fail-loud | Application code | `packages/atif-embed/src/atif_embed/infrastructure/lance_store.py:16-32`, `packages/atif-embed/src/atif_embed/infrastructure/lance_store.py:166-192`; tests `packages/atif-embed/tests/test_embed_use_case.py:293` and `packages/atif-embed/tests/test_embed_use_case.py:314` | -| The write chunk is a multiple of `batch_size`, so a checkpoint boundary never splits a Bedrock batch | Application code | `packages/atif-embed/src/atif_embed/application/embed.py:145-147` | -| The provider/dimension guard exists as two copies (atif-embed and atif-duck) because the independence contract forbids the import; a behavioral change in either must be ported | Application code | `packages/atif-embed/src/atif_embed/domain/embedding_guard.py:15-18`, `packages/atif-duck/src/atif_duck/domain/embedding_guard.py:46`; tests `packages/atif-embed/tests/test_guard_twin_pin.py:76`, `packages/atif-embed/tests/test_guard_twin_pin.py:79`, `packages/atif-embed/tests/test_guard_twin_pin.py:85` | +| ADDITIVE schema drift migrates ONLINE (metadata-only `add_columns`, no re-embedding, search stays up); a BREAKING change (provider or dimension switch) stays fail-loud | Application code | `packages/atif-embed/src/atif_embed/infrastructure/lance_store.py:18-27`, `packages/atif-embed/src/atif_embed/infrastructure/lance_store.py:166-192`; tests `packages/atif-embed/tests/test_embed_use_case.py:293` and `packages/atif-embed/tests/test_embed_use_case.py:314` | +| The write chunk, `max(batch_size * 4, 256)`, is a multiple of `batch_size` at both default batch sizes (96 for Cohere, 32 for Gemma) and at any batch size of 64 or more, so at those sizes a checkpoint boundary never splits a provider batch | Application code | `packages/atif-embed/src/atif_embed/application/embed.py:153-155` | +| The provider/dimension guard exists as two copies (atif-embed and atif-duck) because the independence contract forbids the import; a behavioral change in either must be ported | Application code | `packages/atif-embed/src/atif_embed/domain/embedding_guard.py:15-18`, `packages/atif-duck/src/atif_duck/domain/embedding_guard.py:5-11`; tests `packages/atif-embed/tests/test_guard_twin_pin.py:76`, `packages/atif-embed/tests/test_guard_twin_pin.py:79`, `packages/atif-embed/tests/test_guard_twin_pin.py:85` | | The recovery hint names the real store locations (`ATIF_SQL_LANCE_URI` or `/embeddings_lance`), never a fixed home-directory path | Application code | `packages/atif-embed/src/atif_embed/domain/embedding_guard.py:25-35`; tests `packages/atif-embed/tests/test_guard_twin_pin.py:97` and `packages/atif-embed/tests/test_guard_twin_pin.py:103` | ### SQL surface and CLI | Invariant | Where enforced | Citation | | --- | --- | --- | -| Registration order is raw TEMP tables, then views, then VSS, then macros, then the authorship macro and views, then the v2 analytics views and macros; each layer binds against the previous at CREATE time | Application code | `packages/atif-duck/src/atif_duck/infrastructure/registry.py:1740-1752`; test `packages/atif-duck/tests/test_duck_views.py:615` | -| `DESCRIPTIONS` covers the catalog exactly | Application code | `packages/atif-duck/src/atif_duck/domain/catalog.py:431`; test `packages/atif-duck/tests/test_examples.py:178` | -| `cost_estimate`'s `est_cost_usd` covers PRICED steps only and is meaningful only when `unpriced_steps = 0`; an inner join would return a partial number indistinguishable from a complete one | Application code | `packages/atif-duck/src/atif_duck/infrastructure/registry.py:1053-1058`, `packages/atif-duck/src/atif_duck/infrastructure/registry.py:1066-1077`; tests `packages/atif-duck/tests/test_duck_views.py:407` and `packages/atif-duck/tests/test_duck_views.py:482` | -| Both `cost_estimate` counters filter on `model_name IS NOT NULL`, because user steps carry no model and would otherwise make every session look like a pricing gap | Application code | `packages/atif-duck/src/atif_duck/infrastructure/registry.py:1060-1063`; test `packages/atif-duck/tests/test_duck_views.py:533` | -| An absent or unattachable Lance store degrades to an empty `message_embeddings` TABLE with the right schema, so `semantic_search` always binds | Application code | `packages/atif-duck/src/atif_duck/infrastructure/registry.py:884-905`; tests `packages/atif-duck/tests/test_vss.py:112` and `packages/atif-duck/tests/test_vss.py:248` | -| Exit codes are a stable wire contract shared with `atif-converter`'s taxonomy on every common number (64, 65, 70, 127) | Application code | `packages/atif-cli/src/atif_cli/errors.py:5-11`, `packages/atif-cli/src/atif_cli/errors.py:23-39` | -| `terminal` versus transient decides the `embed` exit code, so an unattended lane can stop retrying instead of burning identical ticks | Application code | `packages/atif-embed/src/atif_embed/domain/errors.py:17-26`, `packages/atif-cli/src/atif_cli/app.py:833-847` | +| Registration order is raw TEMP tables, then views, then VSS, then macros, then the authorship macro and views, then the v2 analytics views and macros; each layer binds against the previous at CREATE time | Application code | `packages/atif-duck/src/atif_duck/infrastructure/registry.py:2172-2193`; test `packages/atif-duck/tests/test_duck_views.py:675` | +| `DESCRIPTIONS` covers the catalog exactly | Application code | `packages/atif-duck/src/atif_duck/domain/catalog.py:487`; test `packages/atif-duck/tests/test_examples.py:208` | +| `cost_estimate`'s `est_cost_usd` covers PRICED steps only and is meaningful only when `unpriced_steps = 0`; an inner join would return a partial number indistinguishable from a complete one | Application code | `packages/atif-duck/src/atif_duck/infrastructure/registry.py:1925-1930`, `packages/atif-duck/src/atif_duck/infrastructure/registry.py:1938-1954`; tests `packages/atif-duck/tests/test_duck_views.py:464` and `packages/atif-duck/tests/test_duck_views.py:542` | +| Both `cost_estimate` counters filter on `model_name IS NOT NULL`, because user steps carry no model and would otherwise make every session look like a pricing gap | Application code | `packages/atif-duck/src/atif_duck/infrastructure/registry.py:1944-1949`; test `packages/atif-duck/tests/test_duck_views.py:425` | +| An absent or unattachable Lance store degrades to an empty `message_embeddings` TABLE with the right schema, so `semantic_search` always binds | Application code | `packages/atif-duck/src/atif_duck/infrastructure/registry.py:1687-1705`; tests `packages/atif-duck/tests/test_vss.py:112` and `packages/atif-duck/tests/test_vss.py:248` | +| Exit codes are a stable wire contract: `EXIT_CODES` is the workspace's one table, atif-converter carries no codes of its own (`convert` translates its typed errors), and no number is ever reassigned, so the retired `harbor_missing` keeps 127 | Application code | `packages/atif-cli/src/atif_cli/errors.py:5-17`, `packages/atif-cli/src/atif_cli/errors.py:29-66` | +| `terminal` versus transient decides the `embed` exit code, so an unattended lane can stop retrying instead of burning identical ticks | Application code | `packages/atif-embed/src/atif_embed/domain/errors.py:17-26`, `packages/atif-cli/src/atif_cli/app.py:1945-1959` | ## Calculations @@ -173,21 +172,21 @@ pinned by `packages/atif-models/tests/test_anthropic_provider.py:112` and | Long-context marking for one call | the call's input tokens and the spec's `long_context_threshold` (272,000 for every GPT-6 size, 100,000 for Haiku 5.5, none for Sonnet and Opus 5.5) | `long_context = True` when input tokens exceed the threshold; the accumulator then adds the call's input and output to `long_input_tokens` and `long_output_tokens`, per pipeline | `packages/atif-models/src/atif_models/domain/registry.py:175-177`, `packages/atif-models/src/atif_models/domain/ports.py:99-109`; tests `packages/atif-models/tests/test_registry.py:118`, `packages/atif-models/tests/test_openai_provider.py:282`, `packages/atif-models/tests/test_openai_provider.py:293`, `packages/atif-models/tests/test_anthropic_provider.py:251`, `packages/atif-models/tests/test_anthropic_provider.py:264` | | Accumulated-usage dollar estimate for one `ModelSpec` | `pricing_in`, `pricing_out`, `long_pricing_in`, `long_pricing_out` (USD per 1M tokens), accumulated `input_tokens` and `output_tokens`, and their `long_input_tokens` and `long_output_tokens` shares | USD: the long share at the long rates (the standard rates when the spec has none) plus the rest at the standard rates, or `None` when a standard rate is unknown | `packages/atif-models/src/atif_models/domain/registry.py:180-210`; tests `packages/atif-models/tests/test_registry.py:78`, `packages/atif-models/tests/test_registry.py:84`, `packages/atif-models/tests/test_registry.py:98` | | Running actual spend for a run | every watched provider's `UsageAccumulator` summary, long-context shares included, plus its `ModelSpec` | total USD spent this run, across all LLM pipelines | `packages/atif-analytics/src/atif_analytics/application/use_cases/_shared.py:126-144` | -| `cost_estimate(sid)` SQL macro | a session's `steps` rows (`prompt_tokens`, `cached_tokens`, `completion_tokens`, `model_name`) LEFT JOINed against `DEFAULT_PRICING` | `(est_cost_usd, priced_steps, unpriced_steps)` | `packages/atif-duck/src/atif_duck/infrastructure/registry.py:1066-1083`; test `packages/atif-duck/tests/test_duck_views.py:368` | +| `cost_estimate(sid)` SQL macro | a session's `steps` rows (`prompt_tokens`, `cached_tokens`, `completion_tokens`, `model_name`) LEFT JOINed against `DEFAULT_PRICING` | `(est_cost_usd, priced_steps, unpriced_steps)` | `packages/atif-duck/src/atif_duck/infrastructure/registry.py:1938-1954`; test `packages/atif-duck/tests/test_duck_views.py:425` | | Retry backoff delay | attempt counter | `min(2 ** attempts, 60)` MINUTES, per `(pipeline, unit_id)`: 2, 4, 8, 16, 32, capped at 60 | `packages/atif-analytics/src/atif_analytics/infrastructure/sqlite_state/retry_queue.py:78-81`; test `packages/atif-analytics/tests/test_state.py:102` | | Bedrock call retry schedule | the raised exception | up to 10 attempts, exponential wait with multiplier 2, minimum 2 SECONDS and maximum 60 SECONDS, only for `RETRY_CODES` and network errors | `packages/atif-models/src/atif_models/infrastructure/openai_bedrock.py:183-189`, `packages/atif-models/src/atif_models/infrastructure/anthropic_bedrock.py:221-227`, `packages/atif-models/src/atif_models/infrastructure/bedrock_common.py:66-73`; tests `packages/atif-models/tests/test_openai_provider.py:310` and `packages/atif-models/tests/test_anthropic_provider.py:275` | | Parquet write chunk size | `batch_size` (default 96 units) | `max(batch_size * 4, 256)` ROWS per part (bounds crash loss, not spend) | `packages/atif-analytics/src/atif_analytics/application/use_cases/_shared.py:247-255` | -| Quiescence age test | newest source `mtime_ns`, `now_ns`, `quiesce_seconds` | boolean: `now_ns - newest_mtime_ns >= quiesce_seconds * 1_000_000_000` | `packages/atif-corpus/src/atif_corpus/domain/sessions.py:96-104`; test `packages/atif-corpus/tests/test_domain.py:40` | +| Quiescence age test | newest source `mtime_ns`, `now_ns`, `quiesce_seconds` | boolean: `now_ns - newest_mtime_ns >= quiesce_seconds * 1_000_000_000` | `packages/atif-corpus/src/atif_corpus/domain/sessions.py:106-114`; test `packages/atif-corpus/tests/test_domain.py:40` | | Source delta between two scans | two `path` to `mtime_ns` maps | added / modified / removed sorted tuples; `touched` is added then modified | `packages/atif-corpus/src/atif_corpus/domain/watermark.py:57-76`; test `packages/atif-corpus/tests/test_domain.py:76` | -| Corpus slug | a corpus root path | `default` for `~/.claude`, else `-<8 hex of sha256>` | `packages/atif-corpus/src/atif_corpus/domain/slug.py:45-50`; test `packages/atif-corpus/tests/test_slug.py:24` | -| Loss accounting for one session | raw record counts by `RecordType`, side-file classification | `records_converted` (user plus assistant records), `records_dropped` (total minus converted), `gaps_observed` | `packages/atif-converter/src/atif_converter/application/convert_and_audit.py:62-90`, `packages/atif-converter/src/atif_converter/domain/fidelity.py:109-112` | +| Corpus slug | a corpus root path | `default` for `~/.claude`, `codex` for `~/.codex/sessions`, else `-<8 hex of sha256>` | `packages/atif-corpus/src/atif_corpus/domain/slug.py:65-72`; test `packages/atif-corpus/tests/test_slug.py:44` | +| Loss accounting for one session | raw record counts by `RecordType`, side-file classification, and the count of records kept as `session_events` rows | `records_converted` (user plus assistant records), `records_captured`, `records_dropped` (total minus converted minus captured), `gaps_observed` | `packages/atif-converter/src/atif_converter/application/convert_and_audit.py:137-172`, `packages/atif-converter/src/atif_converter/domain/fidelity.py:142-145` | | Completed human-to-AI exchange count | a session's `StepEvent` list | integer pair count, the perceived-error eligibility input | `packages/atif-analytics/src/atif_analytics/domain/transcript.py:328-352`; test `packages/atif-analytics/tests/test_perceived.py:63` | | Content stamp for an embeddable text | the exact text sent to the embedder | blake2b digest, 16 bytes hex-encoded (128 bits) | `packages/atif-embed/src/atif_embed/domain/text_stamp.py:26-40` | -| `friction_rate(since_days)` | `user_friction` label counts per session; `human_turns` rows as denominator | per-session `rate` plus one counter per friction label | `packages/atif-duck/src/atif_duck/infrastructure/analytics.py:276-319` | -| `session_outcomes` | a session's main-chain `steps`, each labeled by `step_author` | `kind` from the opening user step's author and the human-turn count (`turn_audit`, `one_shot_job` at one human turn or fewer, else `interactive`); `outcome` is the first that holds of an audit verdict `pass` or `block` as the last agent message, any Stop hook block (`reviewer_blocked`), any interrupt (`interrupted`), else `clean_end` | `packages/atif-duck/src/atif_duck/infrastructure/authorship.py:84-132` | -| `semantic_search(query_vec, k)` | a unit-norm query vector and `k` | top-`k` `(uuid, sim, distance)` ordered by cosine distance | `packages/atif-duck/src/atif_duck/infrastructure/registry.py:1145-1152`; test `packages/atif-duck/tests/test_vss.py:195` | -| `todo_velocity(sid)` | `todo_state_current` rows for one session | completed count divided by distinct subject count, `NULL` when there are no subjects | `packages/atif-duck/src/atif_duck/infrastructure/registry.py:1101-1107`; test `packages/atif-duck/tests/test_duck_views.py:566` | -| `subagent_fanout(sid)` | `subagent_spawns` rows for one session | count of Task/Agent launch INTENTS, not side-transcript files | `packages/atif-duck/src/atif_duck/infrastructure/registry.py:1116-1124`; test `packages/atif-duck/tests/test_duck_views.py:572` | +| `friction_rate(since_days)` | `user_friction` label counts per session; `human_turns` rows as denominator | per-session `rate` plus one counter per friction label | `packages/atif-duck/src/atif_duck/infrastructure/analytics.py:292-325` | +| `session_outcomes` | a session's main-chain `steps`, each labeled by `step_author` | `kind` from the opening user step's author and the human-turn count (`turn_audit`, `one_shot_job` at one human turn or fewer, else `interactive`); `outcome` is the first that holds of an audit verdict `pass` or `block` as the last agent message, any Stop hook block (`reviewer_blocked`), any interrupt (`interrupted`), else `clean_end` | `packages/atif-duck/src/atif_duck/infrastructure/authorship.py:91-131` | +| `semantic_search(query_vec, k)` | a unit-norm query vector and `k` | top-`k` `(uuid, sim, distance)` ordered by cosine distance | `packages/atif-duck/src/atif_duck/infrastructure/registry.py:2016-2023`; test `packages/atif-duck/tests/test_vss.py:195` | +| `todo_velocity(sid)` | `todo_state_current` rows for one session | completed count divided by distinct subject count, `NULL` when there are no subjects | `packages/atif-duck/src/atif_duck/infrastructure/registry.py:1973-1978`; test `packages/atif-duck/tests/test_duck_views.py:626` | +| `subagent_fanout(sid)` | `subagent_spawns` rows for one session | count of Task/Agent launch INTENTS, not side-transcript files | `packages/atif-duck/src/atif_duck/infrastructure/registry.py:1982-1994`; test `packages/atif-duck/tests/test_duck_views.py:632` | Some of these need the formula spelled out. @@ -201,9 +200,9 @@ strips a dated model suffix (`claude-haiku-4-5-20251001` matches `claude-haiku-4 `DEFAULT_PRICING`, entries of `(in_rate, out_rate)` in **USD per 1,000,000 tokens**, base rates only: the prompt-cache write and read multipliers (1.25x, 2x, 0.1x of base input) are deliberately not modeled, matching the macro -(`packages/atif-duck/src/atif_duck/domain/catalog.py:395-417`, pinned by -`packages/atif-duck/tests/test_duck_views.py:459` against published list rates and by -`packages/atif-duck/tests/test_duck_views.py:470` for sanity bounds). +(`packages/atif-duck/src/atif_duck/domain/catalog.py:602-626`, pinned by +`packages/atif-duck/tests/test_duck_views.py:519` against published list rates and by +`packages/atif-duck/tests/test_duck_views.py:530` for sanity bounds). **`human_ai_pair_count`.** Walk the step list in materialized order. Skip any step that is sidechain, compact-summary, or has empty text. A human turn arms a pending pair; a @@ -220,30 +219,34 @@ perceived pipeline admits a session only at 2 or more pairs ### Cost and spend -- **Dry-run by default:** every LLM analytics stage and the embed backfill treat - `dry_run=True` as the default, returning a plan dict instead of spending; a real run is - an explicit opt-out. `packages/atif-analytics/src/atif_analytics/application/analyze.py:41`, - `packages/atif-analytics/src/atif_analytics/application/use_cases/perceived.py:366`. +- **Dry-run by default:** every LLM analytics stage treats `dry_run=True` as the default, + returning a plan dict instead of spending, and `analyze` spends only under `--no-dry-run`. + The embed backfill is the exception: `run_backfill` defaults to `dry_run=False`, so the + explicit-scope gate under Embeddings and the SQL surface is what keeps a mistyped `embed` + from spending. `packages/atif-analytics/src/atif_analytics/application/analyze.py:42`, + `packages/atif-analytics/src/atif_analytics/application/use_cases/perceived.py:366`, + `packages/atif-cli/src/atif_cli/app.py:1640`, + `packages/atif-embed/src/atif_embed/application/embed.py:69`. - **Session ceiling:** each LLM pipeline admits at most `llm_max_sessions_per_run` sessions per run (default 50, newest-first so fresh sessions win), enforced DURING the admission walk so a deferred session is never rendered or eligibility-probed. `packages/atif-analytics/src/atif_analytics/infrastructure/settings.py:76-80`, `packages/atif-analytics/src/atif_analytics/application/use_cases/perceived.py:179-199`; - tests `packages/atif-analytics/tests/test_resource_guards.py:84` and `packages/atif-analytics/tests/test_resource_guards.py:113`. + tests `packages/atif-analytics/tests/test_resource_guards.py:79` and `packages/atif-analytics/tests/test_resource_guards.py:108`. - **Cost ceiling:** one `RunBudget` of `llm_max_cost_usd_per_run` (default 25.0 USD per `analyze` run, shared across every LLM pipeline) is priced from running actuals and checked at every dispatch batch boundary; when crossed, remaining LLM work stops and nothing is stamped for unstarted units. `packages/atif-analytics/src/atif_analytics/infrastructure/settings.py:81-85`, - `packages/atif-analytics/src/atif_analytics/application/analyze.py:65-87`, + `packages/atif-analytics/src/atif_analytics/application/analyze.py:71-89`, `packages/atif-analytics/src/atif_analytics/application/use_cases/_shared.py:97-148`; - test `packages/atif-analytics/tests/test_analyze.py:75`. + test `packages/atif-analytics/tests/test_analyze.py:78`. - **Budget-skip starvation escalation:** a stage skipped for budget records a durable `budget_skips` row, and the log escalates from WARNING to ERROR once the SAME stage has - been starved 3 consecutive runs; a stage that actually runs clears its rows, so the row - count IS the streak. `packages/atif-analytics/src/atif_analytics/application/analyze.py:103-140`, - `packages/atif-analytics/src/atif_analytics/infrastructure/sqlite_state/checkpointer.py:54-62`, - `packages/atif-analytics/src/atif_analytics/infrastructure/sqlite_state/checkpointer.py:217-246`; test `packages/atif-analytics/tests/test_analyze.py:99`. + been starved 3 consecutive runs; only a stage that runs and finishes inside the budget + clears its rows, so the row count IS the streak. `packages/atif-analytics/src/atif_analytics/application/analyze.py:109-146`, + `packages/atif-analytics/src/atif_analytics/infrastructure/sqlite_state/checkpointer.py:55-63`, + `packages/atif-analytics/src/atif_analytics/infrastructure/sqlite_state/checkpointer.py:218-247`; tests `packages/atif-analytics/tests/test_analyze.py:100` and `packages/atif-analytics/tests/test_analyze.py:142`. - **Retry attempt cap:** a `(pipeline, unit_id)` stops being drained at 5 attempts, which is what closes the uncapped-retry money leak. `packages/atif-analytics/src/atif_analytics/infrastructure/sqlite_state/retry_queue.py:42`, @@ -291,25 +294,25 @@ perceived pipeline admits a session only at 2 or more pairs - **Force overrides staleness only, never quiescence:** a live session is not converted even under `--force`, because converting a half-written transcript produces a WRONG artifact rather than a stale one. - `packages/atif-corpus/src/atif_corpus/domain/sessions.py:170-174`, `packages/atif-corpus/src/atif_corpus/domain/sessions.py:200-206`; tests + `packages/atif-corpus/src/atif_corpus/domain/sessions.py:183-185`, `packages/atif-corpus/src/atif_corpus/domain/sessions.py:218-220`; tests `packages/atif-corpus/tests/test_domain.py:117`, - `packages/atif-corpus/tests/test_materialize.py:148`. + `packages/atif-corpus/tests/test_materialize.py:160`. - **Nothing is deleted, and marking is all-or-nothing per pass:** a session whose source vanished keeps its artifacts and is marked `source_present: false`; when ANY source directory could not be listed, marking is skipped entirely for the pass, because absence is not evidence of deletion without a complete picture of what exists. - `packages/atif-corpus/src/atif_corpus/application/materialize.py:560-577`; tests - `packages/atif-corpus/tests/test_materialize.py:609` and `packages/atif-corpus/tests/test_materialize.py:647`. + `packages/atif-corpus/src/atif_corpus/application/materialize.py:1668-1690`; tests + `packages/atif-corpus/tests/test_materialize.py:698` and `packages/atif-corpus/tests/test_materialize.py:736`. - **A `--sessions` filter never widens marking:** source-removal marking keys off the FULL scan, so an unplanned session is never marked just because it was not planned. - `packages/atif-corpus/src/atif_corpus/application/materialize.py:511-519`, `packages/atif-corpus/src/atif_corpus/application/materialize.py:573-577`; test `packages/atif-corpus/tests/test_materialize.py:280`. + `packages/atif-corpus/src/atif_corpus/application/materialize.py:1563-1571`, `packages/atif-corpus/src/atif_corpus/application/materialize.py:1679-1690`; test `packages/atif-corpus/tests/test_materialize.py:368`. - **Staging sweep errs toward keeping:** a signal delivered, a `PermissionError`, or any unexpected `OSError` all answer "do not delete", and a recycled pid only means the sweep - skips debris a later pass collects. `packages/atif-corpus/src/atif_corpus/application/materialize.py:256-271`; test - `packages/atif-corpus/tests/test_materialize.py:481`. + skips debris a later pass collects. `packages/atif-corpus/src/atif_corpus/application/materialize.py:953-968`; test + `packages/atif-corpus/tests/test_materialize.py:570`. - **Corrupt state degrades, never refuses:** an unreadable or wrong-shaped `watermark.json` is treated as unmaterialized, costing one full pass, which is always - safe. `packages/atif-corpus/src/atif_corpus/application/materialize.py:160-176`. + safe. `packages/atif-corpus/src/atif_corpus/application/materialize.py:424-452`. ### Embeddings and the SQL surface @@ -318,31 +321,32 @@ perceived pipeline admits a session only at 2 or more pairs so an unattended lane exits 78 and suppresses retries instead of burning identical ticks. `packages/atif-embed/src/atif_embed/domain/embedding_guard.py:3-18`, `packages/atif-embed/src/atif_embed/domain/errors.py:29-41`, - `packages/atif-cli/src/atif_cli/app.py:833-847`. -- **Embed requires an explicit scope:** a bare `atif-sql embed --no-dry-run` exits 64 so a - full backfill cannot happen from a mistyped command; `--dry-run` needs no scope because - it spends nothing. `packages/atif-cli/src/atif_cli/app.py:778-783`, `packages/atif-cli/src/atif_cli/app.py:810-820`. + `packages/atif-cli/src/atif_cli/app.py:1945-1959`. +- **Embed requires an explicit scope:** a bare `atif-sql embed` is a real run and exits 64, + so a full backfill cannot happen from a mistyped command; `--dry-run` needs no scope + because it spends nothing. `packages/atif-cli/src/atif_cli/app.py:1864-1868`, `packages/atif-cli/src/atif_cli/app.py:1908-1918`. - **`skip_vss` escape hatch:** registration can skip both the `message_embeddings` view and the `semantic_search` macro, because the backfill writes the store the view reads - and binding it first would be circular. `packages/atif-duck/src/atif_duck/infrastructure/registry.py:1244-1250`; + and binding it first would be circular. `packages/atif-duck/src/atif_duck/infrastructure/registry.py:2134-2140`; test `packages/atif-duck/tests/test_vss.py:258`. - **Register-or-fail-loud:** any DuckDB error during view or macro registration is logged with `logger.exception` and re-raised: an empty or absent corpus fails registration - rather than yielding empty views. `packages/atif-duck/src/atif_duck/infrastructure/registry.py:201-207`, `packages/atif-duck/src/atif_duck/infrastructure/registry.py:1201-1204`; test - `packages/atif-duck/tests/test_duck_views.py:623`. + rather than yielding empty views. `packages/atif-duck/src/atif_duck/infrastructure/registry.py:955-958`, `packages/atif-duck/src/atif_duck/infrastructure/registry.py:1527-1530`; test + `packages/atif-duck/tests/test_duck_views.py:683`. - **A new view or macro cannot land silently:** the drift tests require a `DESCRIPTIONS` entry, an `ARG_EXEMPLARS` entry for any new parameter name, `TABLE_MACRO_NAMES` membership when the DDL is `AS TABLE`, and an example that EXECUTES, or a documented - `EXCLUSIONS` entry. `packages/atif-duck/src/atif_duck/domain/examples.py:94-99`; tests - `packages/atif-duck/tests/test_examples.py:160`, `packages/atif-duck/tests/test_examples.py:178`, `packages/atif-duck/tests/test_examples.py:188`, `packages/atif-duck/tests/test_examples.py:197`. + `EXCLUSIONS` entry. `packages/atif-duck/src/atif_duck/domain/examples.py:95-100`; tests + `packages/atif-duck/tests/test_examples.py:190`, `packages/atif-duck/tests/test_examples.py:208`, `packages/atif-duck/tests/test_examples.py:218`, `packages/atif-duck/tests/test_examples.py:227`. ### Scheduling -- **Cron lanes, split by cost:** `materialize` every 10 minutes (cheap incremental) - and `llm` once daily at 10:20 (the lane that spends). - `packages/atif-cli/src/atif_cli/cron.py:43-50`. +- **Cron lanes, split by cost:** `materialize` every 10 minutes (cheap incremental), + `llm` once daily at 10:20 (the lane that spends), and `compact` nightly at 03:40 (lake + maintenance). + `packages/atif-cli/src/atif_cli/cron.py:43-51`. - **`cron install` never writes a crontab:** it prints the block for a human to paste, - after checking `crontab -l`. `packages/atif-cli/src/atif_cli/cron.py:184-202`. + after checking `crontab -l`. `packages/atif-cli/src/atif_cli/cron.py:186-203`. ## See also diff --git a/docs/insights/contract-map.md b/docs/insights/contract-map.md index 0d6a345..0c729b3 100644 --- a/docs/insights/contract-map.md +++ b/docs/insights/contract-map.md @@ -8,144 +8,167 @@ JSON key order, a column projection, a `typing.Protocol` typed to a document rat implementation. So a contract in this file is any of: 1. a `typing.Protocol` declared in one package's `domain/ports.py` and satisfied by an adapter - elsewhere (each is listed below); + elsewhere (each has a section below or a bullet under "Other contracts"); 2. a shape declared **twice or more** in packages that cannot import each other, where the two declarations must agree or a query silently returns wrong rows; 3. a build-enforced dependency rule (import-linter), which is a stronger fact than a convention; 4. an upstream API this workspace depends on and pins; -5. a version constraint that ships in a wheel and binds an external installer. +5. a version pin between workspace members, which the release tooling rewrites and checks on + every bump. **Every Protocol here is satisfied STRUCTURALLY, with no import in either direction.** That is the single most important structural fact for reading this file. `RealConverter` -(`packages/atif-cli/src/atif_cli/converter_adapter.py:44`) implements `ConverterPort` -(`packages/atif-corpus/src/atif_corpus/domain/ports.py:41`) by shape alone; it never names the -Protocol, and it cannot, because atif-corpus may never import atif-converter. The Protocol name -appears at only these import sites in the whole workspace: -`packages/atif-corpus/src/atif_corpus/application/materialize.py:92` (the module that CALLS it) and -`packages/atif-cli/tests/test_converter_adapter.py:17` (the test that binds the two together in an -annotation). So each Protocol row below has a producer of the shape and an implementer the type -system never links to it, and the answer to "who finds out if the shape drifts" is a named test or -nobody. Do not read a Protocol row as a dependency edge: `pyproject.toml:602-605` forbids the edge +(`packages/atif-cli/src/atif_cli/converter_adapter.py:80`) implements `ConverterPort` +(`packages/atif-corpus/src/atif_corpus/domain/ports.py:101`) by shape alone; it names the +Protocol only in its docstring, and the Protocol can't name it back, because atif-corpus may never +import atif-converter. The Protocol name appears at only these import sites in the whole workspace: +`packages/atif-corpus/src/atif_corpus/application/materialize.py:172` (the module that CALLS it), +`packages/atif-cli/tests/test_converter_adapter.py:17`, and +`packages/atif-cli/tests/test_codex_cli.py:26` (the tests that bind the two together in an +annotation). So each Protocol row below has a producer of the shape and an implementer that never +imports it; the type system meets the two only at a typed site (an annotated test binding, a +factory's annotated return, a call into a port-typed parameter), and the answer to "who finds out +if the shape drifts" is one of those sites or nobody. Do not read a Protocol row as a dependency edge: `pyproject.toml:602-605` forbids the edge it would imply. Every contract below names its producer, its consumers, the verbatim shape, the assumptions consumers make beyond the shape, and the drift risk. **Consumer counts are grep-derived and confirmed at each import, annotation, or call site**: this repo has no code index -(`.gitignore:24` lists `.codegraph/`, and no index exists on disk), so no count here comes from a +(`.gitignore:25` lists `.codegraph/`, and no index exists on disk), so no count here comes from a symbol graph. These grep hazards shape every count: names collide (`DomainError` and `EmbeddingProviderMismatch` across packages, `cached_tokens` across coordinate spaces), so every -attribution here is by module path and never by bare name; and every cross-package import is -indented inside a function body or a `TYPE_CHECKING` block, so a line-anchored grep finds nothing. +attribution here is by module path and never by bare name; and outside four atif-cli modules +(listed under the import-linter contracts) every cross-package import under `src/` is indented +inside a function body or a `TYPE_CHECKING` block, so a line-anchored grep of `src/` finds almost +nothing. -Contracts are ordered by confirmed consumer count, descending. +The sections below aren't ranked; "Other contracts" at the end collects the smaller ones. ## The materialized corpus artifact layout -**Producer:** `packages/atif-corpus/src/atif_corpus/domain/layout.py:18-22` +**Producer:** `packages/atif-corpus/src/atif_corpus/domain/layout.py:18-24` **Consumers:** -- `packages/atif-duck/src/atif_duck/infrastructure/registry.py:210-213`: inlines the - filenames as SQL glob literals for its `read_json` readers. -- `packages/atif-embed/src/atif_embed/infrastructure/corpus_text_rows.py:120-121`: opens - `trajectory.json` and gates on `meta.json` with its own DuckDB connection. +- `packages/atif-duck/src/atif_duck/infrastructure/registry.py:697-701`: builds the `meta.json` + glob (and, further down, each session's stored-file list) and binds it as the one parameter of + its `read_json` readers, spelling the names from its own constants at + `packages/atif-duck/src/atif_duck/domain/artifacts.py:28-36`. +- `packages/atif-embed/src/atif_embed/infrastructure/corpus_text_rows.py:155-173`: gates on + `meta.json` and opens each stored trajectory with its own DuckDB connection. - `packages/atif-analytics/src/atif_analytics/infrastructure/corpus_reader.py:72-74`: declares `TRAJECTORY_FILENAME` / `EDGES_FILENAME` / `META_FILENAME` again as its own constants. -- `packages/atif-corpus/src/atif_corpus/application/materialize.py:222-238`: the writer side, the +- `packages/atif-corpus/src/atif_corpus/application/materialize.py:691-731`: the writer side, the only place the artifacts are produced. -- `docs/CONTRACT.md:21-39`: the hand-written specification all of them agree to. +- `packages/atif-cli/tests/test_artifact_names_twin_pin.py:29`: pins the writer's names against + every reader's copy, and `:19` pins the `.zst` suffix. +- `docs/CONTRACT.md:22-60`: the hand-written specification all of them agree to. **Shape:** ```text / # default: ~/.atif-sql/corpus// sessions// - trajectory.json # compact JSON (separators=(',',':')), ATIF-v1.7 + trajectory.json.zst # compact JSON (separators=(',',':')), ATIF-v1.7, one zstd frame loss_report.json # atif_converter LossReport.to_json() - edges.jsonl # one line per RAW record: {uuid, parent_uuid, + edges.jsonl.zst # one line per RAW record: {uuid, parent_uuid, # message_id, type, ts, is_sidechain, # is_compact_summary, source_file, tool_use_ids: [..]} + session_events.jsonl.zst # one line per KEPT non-message record; may be empty meta.json # {session_id, source_mtime_ns, source_files: [...], - # harbor_version, converter_version, materialized_at, - # agent, columnar_schema} - session.parquet # optional typed columnar artifacts: the views' rows - steps.parquet # for this session, written 0444 by atif-duck's - tool_calls.parquet # ColumnarArtifactProducer at materialize time and - tool_results.parquet # claimed by meta.columnar_schema (currently 1) + # harbor_version, converter_version, converter_schema, + # materialized_at, agent, source_present, + # source_removed_at?, source_archive?, + # columnar_schema? (old layout)} + source/.zst # raw source archive: one zstd file per source file + blobs/sha256//. # inline attachments, content-addressed, written 0444 watermark.json # {path: mtime_ns} across source corpus + empty_sessions.json # sessions whose transcript held nothing to convert + sink_pending.json # sessions the SessionSink hasn't taken yet ``` -The parquet files are a cache of what the views compute from `trajectory.json`, never a source of -truth: atif-duck reads them only when `meta.columnar_schema` matches its own version and all of them -are present and non-empty (`packages/atif-duck/src/atif_duck/infrastructure/columnar.py`), and -falls back to `trajectory.json` per session otherwise. The names and the schema version live in -`packages/atif-duck/src/atif_duck/domain/columnar.py`; atif-corpus never learns them, it runs the -producer through the `ArtifactProducer` port and merges the returned keys into `meta.json`. +A corpus written by an older build keeps the plain `trajectory.json`, `edges.jsonl`, and +`session_events.jsonl`, plus five per-session parquet files, until `atif-sql corpus slim` converts +it, and every reader accepts either spelling. The parquet files are a cache of what the views +compute from the trajectory, never a source of truth, and materialize doesn't write them: the CLI +passes no artifact producer (`packages/atif-cli/src/atif_cli/app.py:975-977`), and the lake loader +runs atif-duck's `ColumnarArtifactProducer` to stage each session it loads +(`packages/atif-duck/src/atif_duck/infrastructure/lake.py:561`). atif-duck reads a session's own +parquet files only when `meta.columnar_schema` matches its version and all five are present and +non-empty (`packages/atif-duck/src/atif_duck/infrastructure/columnar.py:116`), and falls back to +the trajectory per session otherwise. The names and the schema version live in +`packages/atif-duck/src/atif_duck/domain/columnar.py`; atif-corpus never learns them, and it merges +whatever keys an `ArtifactProducer` returns into `meta.json`. **Assumptions consumers make:** - **`meta.json` present means the session dir is complete.** Every reader implements the same - torn-set gate independently: `packages/atif-duck/src/atif_duck/infrastructure/registry.py:188-192` restricts the trajectory/edges/loss readers to - dirs where `v_raw_meta` has a row; `packages/atif-embed/src/atif_embed/infrastructure/corpus_text_rows.py:121-124` skips a dir with no `meta.json`; + torn-set gate independently: `packages/atif-duck/src/atif_duck/infrastructure/registry.py:703-709` restricts the edges and loss readers to + dirs where `v_raw_meta` has a row (the trajectory list is built from `v_raw_meta` too); `packages/atif-embed/src/atif_embed/infrastructure/corpus_text_rows.py:164-169` skips a dir with no `meta.json`; `packages/atif-analytics/src/atif_analytics/infrastructure/corpus_reader.py:178-191` does the same and logs a warning. Nothing in the file layout expresses - this: it is a write-ORDER promise made at - `packages/atif-corpus/src/atif_corpus/application/materialize.py:209-212`. -- **No reader ever sees a partially written directory**, because the writer stages all the - artifacts under `/.staging/` and swaps the whole dir with `os.replace` - (`packages/atif-corpus/src/atif_corpus/application/materialize.py:201-212`), and `.staging` is deliberately outside `sessions/` so a DuckDB glob - cannot reach it (`packages/atif-corpus/src/atif_corpus/domain/layout.py:50-54`). + this: it's a write-ORDER promise made at + `packages/atif-corpus/src/atif_corpus/application/materialize.py:635-638`. +- **No reader ever sees a partially written directory**, because the writer stages every + per-session artifact under `/.staging/` and swaps the whole dir into place + (`packages/atif-corpus/src/atif_corpus/application/materialize.py:656` and `:733`), and `.staging` is deliberately outside `sessions/` so a DuckDB glob + can't reach it (`packages/atif-corpus/src/atif_corpus/domain/layout.py:89-94`). A `SIGKILL` or power loss inside the swap leaves the session missing rather than torn. - **`trajectory.json` is ONE JSON document, `edges.jsonl` is newline-delimited.** atif-duck encodes - that split in its reader choice at `packages/atif-duck/src/atif_duck/infrastructure/registry.py:23-25`; atif-embed re-derives it at - `packages/atif-embed/src/atif_embed/infrastructure/corpus_text_rows.py:7-8`. -- **A single trajectory document can be enormous.** Both DuckDB readers set a 1 GiB - `maximum_object_size`, and their measured ceilings disagree: `packages/atif-duck/src/atif_duck/infrastructure/registry.py:74-80` cites 436 MB - observed, `packages/atif-embed/src/atif_embed/infrastructure/corpus_text_rows.py:47-49` cites 85 MB. Same constant, two independent + that split in its reader choice at `packages/atif-duck/src/atif_duck/infrastructure/registry.py:25-28`; atif-embed re-derives it at + `packages/atif-embed/src/atif_embed/infrastructure/corpus_text_rows.py:88`. +- **A single trajectory document can be enormous.** Both DuckDB readers allow a 1 GiB + `maximum_object_size`, and their measured ceilings disagree: atif-duck caps a bound it sizes from the largest file at 1 GiB, citing 436 MB + observed (`packages/atif-duck/src/atif_duck/infrastructure/registry.py:217-220`), while atif-embed sets 1 GiB flat, citing 85 MB (`packages/atif-embed/src/atif_embed/infrastructure/corpus_text_rows.py:56-58`). Same ceiling, two independent justifications. - **`source_mtime_ns` and every `watermark.json` value are epoch NANOSECONDS** from `os.stat().st_mtime_ns` (unit stated in the identifier and again at - `packages/atif-corpus/src/atif_corpus/domain/watermark.py:12-13`). atif-duck types the column - `BIGINT` at `packages/atif-duck/src/atif_duck/infrastructure/registry.py:136`, which preserves the magnitude but drops the unit from the schema. + `packages/atif-corpus/src/atif_corpus/domain/watermark.py:11-12`). atif-duck types the column + `BIGINT` at `packages/atif-duck/src/atif_duck/domain/raw_readers.py:101`, which preserves the magnitude but drops the unit from the schema. - **`materialized_at` is an ISO-8601 UTC string, not a timestamp.** atif-duck projects it as - `VARCHAR` (`packages/atif-duck/src/atif_duck/infrastructure/registry.py:140`); the CLI supplies it (`packages/atif-cli/src/atif_cli/app.py:400`). - -**Where the hand-written contract disagrees with the code, the code wins.** Its CLI heading names -atif-cli the only package importing the others, and the manifest agrees: it declares the sibling -dependencies -(`packages/atif-cli/pyproject.toml:32-36`) and the CLI imports every one of them. `docs/CONTRACT.md:16-17` -lists VSS/`semantic_search` and the v2 LLM-analytics pipelines as out of scope and then reverses -itself at `docs/CONTRACT.md:17-19`; both are shipped commands -(`packages/atif-cli/pyproject.toml:42` plus the `embed` / `search` / `analyze` commands). Read -`docs/CONTRACT.md:21-39` as authoritative for the layout (that is the part the packages actually -implement) and the manifest as authoritative for who imports whom. + `VARCHAR` (`packages/atif-duck/src/atif_duck/domain/raw_readers.py:105`); the CLI supplies it (`packages/atif-cli/src/atif_cli/app.py:963`). + +**The hand-written contract and the manifests split the job.** Its CLI heading +(`docs/CONTRACT.md:300`) names atif-cli the package that composes the others. The atif-cli +manifest declares five siblings (`packages/atif-cli/pyproject.toml:32-36`) and the CLI imports +every one of them; atif-models reaches it through atif-analytics, which imports atif-models +(`packages/atif-analytics/pyproject.toml:23`) as the import-linter contracts permit. +`docs/CONTRACT.md:17-19` leaves VSS/`semantic_search` and the analytics pipelines to +`docs/CONTRACT-V2.md` rather than ruling them out; both are shipped commands (the `atif-analytics` +and `atif-embed` pins at `packages/atif-cli/pyproject.toml:32` and `:36`, plus the `analyze`, +`embed`, and `search` commands at `packages/atif-cli/src/atif_cli/app.py:1634`, `:1801`, and +`:2018`). Read `docs/CONTRACT.md:22-60` as authoritative for the layout (that's the part the +packages actually implement) and the manifests as authoritative for who imports whom. **Drift risk:** a new artifact, a renamed file, or a new `meta.json` key must be applied in every -place above, which no test links, and most of them are reader-side, so an addition silently reaches -nobody. Mitigation: the writer-side constants at -`packages/atif-corpus/src/atif_corpus/domain/layout.py:18-22` are the single source of truth on the +place above, and most of them are reader-side, so an addition silently reaches nobody. The file +names and the `.zst` suffix are pinned across all four packages by +`packages/atif-cli/tests/test_artifact_names_twin_pin.py:29` and `:19`; a new `meta.json` key has no such +pin. Mitigation: the writer-side constants at +`packages/atif-corpus/src/atif_corpus/domain/layout.py:18-24` are the single source of truth on the write side. Any layout change starts there and then greps the reader modules named above. ## The static SQL catalog **Producer:** `packages/atif-duck/src/atif_duck/domain/catalog.py:30` (`VIEW_NAMES`), with -`VIEW_SCHEMA:56`, `MACRO_NAMES:268`, `MACRO_SIGNATURES:287`, `ANALYTICS_VIEW_NAMES:314`, -`ANALYTICS_VIEW_SCHEMA:328`, `ANALYTICS_MACRO_SIGNATURES:397`, `TABLE_MACRO_NAMES:413`, -`DESCRIPTIONS:431` +`VIEW_SCHEMA:59`, `MACRO_NAMES:324`, `MACRO_SIGNATURES:343`, `ANALYTICS_VIEW_NAMES:370`, +`ANALYTICS_VIEW_SCHEMA:384`, `ANALYTICS_MACRO_SIGNATURES:453`, `TABLE_MACRO_NAMES:469`, +`DESCRIPTIONS:487` **Consumers:** -- `packages/atif-cli/src/atif_cli/app.py:1745-1778`: the `schema` command reads `VIEW_SCHEMA`, +- `packages/atif-cli/src/atif_cli/app.py:2335-2387`: the `schema` command reads `VIEW_SCHEMA`, `MACRO_SIGNATURES`, `ANALYTICS_VIEW_SCHEMA`, and `ANALYTICS_MACRO_SIGNATURES`, tags each object with its `requires` value, and answers with no DuckDB connection. -- `packages/atif-duck/src/atif_duck/domain/examples.py:36-42`: the examples generator imports every +- `packages/atif-duck/src/atif_duck/domain/examples.py:35-44`: the examples generator imports every catalog and derives one runnable query per object. -- `packages/atif-cli/src/atif_cli/app.py:1025-1046`: the `examples` command calls +- `packages/atif-cli/src/atif_cli/app.py:2294`: the `examples` command calls `build_examples()`. -- `packages/atif-duck/tests/test_duck_views.py:31-33`: the `DESCRIBE`-vs-`VIEW_SCHEMA` and +- `packages/atif-duck/tests/test_duck_views.py:51` and `:67`: the `DESCRIBE`-vs-`VIEW_SCHEMA` and DDL-vs-`MACRO_SIGNATURES` drift tests. -- `packages/atif-duck/tests/test_examples.py:27-35`: `DESCRIPTIONS` coverage, `ARG_EXEMPLARS` - coverage, and the `AS TABLE` set drift test. -- `packages/atif-duck/tests/test_analytics_views.py:20-21`: the analytics-side drift test. -- `packages/atif-cli/tests/test_app.py:78-84`: asserts the CLI's JSON payload keys equal the +- `packages/atif-duck/tests/test_examples.py:208`, `:218`, and `:227`: `DESCRIPTIONS` coverage, + `ARG_EXEMPLARS` coverage, and the `AS TABLE` set drift test. +- `packages/atif-duck/tests/test_analytics_views.py:260` and `:242`: the analytics-side drift + tests. +- `packages/atif-cli/tests/test_app.py:135`: asserts the CLI's JSON payload keys equal the catalog keys. **Shape:** @@ -161,7 +184,7 @@ VIEW_NAMES: tuple[str, ...] = ( VIEW_SCHEMA: dict[str, tuple[tuple[str, str], ...]] = { "sessions": ( ("session_id", "VARCHAR"), - ("cwd", "VARCHAR"), + ("agent", "VARCHAR"), ... ), ... @@ -177,27 +200,27 @@ MACRO_SIGNATURES: dict[str, tuple[str, ...]] = { **Assumptions consumers make:** - **Column ORDER is part of the contract, not just the column set.** The drift test asserts tuple - equality against `DESCRIBE` output, stated at `packages/atif-duck/src/atif_duck/domain/catalog.py:47-50`; reordering a view's `SELECT` + equality against `DESCRIBE` output, stated at `packages/atif-duck/src/atif_duck/domain/catalog.py:56-58`; reordering a view's `SELECT` list fails CI even though every column still exists. -- **`DESCRIPTIONS` covers every object in every catalog, exactly.** `packages/atif-duck/tests/test_examples.py:181-185` +- **`DESCRIPTIONS` covers every object in every catalog, exactly.** `packages/atif-duck/tests/test_examples.py:208` fails on a missing entry AND on a stale entry keyed to an object that no longer exists, so the dict is a bijection with the union of the name catalogs. - **Every macro parameter name has an `ARG_EXEMPLARS` literal**, or example generation raises - rather than emitting a broken query: `packages/atif-duck/src/atif_duck/domain/examples.py:151-157` fails loud, and - `packages/atif-duck/tests/test_examples.py:189-194` pre-empts it. -- **`TABLE_MACRO_NAMES` decides the SQL call shape.** `packages/atif-duck/src/atif_duck/domain/examples.py:159-162` emits + rather than emitting a broken query: `packages/atif-duck/src/atif_duck/domain/examples.py:165-171` fails loud, and + `packages/atif-duck/tests/test_examples.py:218` pre-empts it. +- **`TABLE_MACRO_NAMES` decides the SQL call shape.** `packages/atif-duck/src/atif_duck/domain/examples.py:172-175` emits `SELECT * FROM name(args)` for a member and `SELECT name(args)` otherwise, so a macro whose DDL gains or loses `AS TABLE` breaks every derived example, pinned by the regex test at - `packages/atif-duck/tests/test_examples.py:198-210`. -- **The catalog is answerable without a corpus.** `packages/atif-cli/src/atif_cli/app.py:1094-1098` states the sub-50 ms, + `packages/atif-duck/tests/test_examples.py:227`. +- **The catalog is answerable without a corpus.** `packages/atif-cli/src/atif_cli/app.py:2344-2346` states the sub-50 ms, no-DuckDB-bind guarantee the `schema` command rests on; a runtime `DESCRIBE` would violate it. -- **The raw readers are deliberately NOT in `VIEW_NAMES`** (`packages/atif-duck/src/atif_duck/infrastructure/registry.py:65-68`), so a +- **The raw readers are deliberately NOT in `VIEW_NAMES`** (`packages/atif-duck/src/atif_duck/infrastructure/registry.py:141-142`), so a consumer enumerating `VIEW_NAMES` does not see `v_raw_trajectories` and friends. **Drift risk:** adding a view or macro without a `DESCRIPTIONS` entry, an `ARG_EXEMPLARS` entry for each new parameter name, or `TABLE_MACRO_NAMES` membership fails CI loudly: the risk is inverted here, and the real exposure is a *type* that only the fixture corpus produces. Mitigation: -`VIEW_SCHEMA["message_embeddings"]` at `packages/atif-duck/src/atif_duck/domain/catalog.py:218-224` hardcodes `FLOAT[1024]`, so run the +`VIEW_SCHEMA["message_embeddings"]` at `packages/atif-duck/src/atif_duck/domain/catalog.py:281-287` hardcodes `FLOAT[1024]`, so run the drift test against a store built at a non-default `output_dimension` before changing that setting. ## The import-linter architecture contracts @@ -206,19 +229,19 @@ drift test against a store built at a non-default `output_dimension` before chan **Consumers:** -- `mise.toml:197`: `lint:imports` is one of the `mise run check` gates, so the build is +- `mise.toml:250`: `lint:imports` is one of the `mise run check` gates, so the build is the consumer. -- `packages/atif-corpus/src/atif_corpus/domain/ports.py:5-8`: the `ConverterPort` docstring cites - the independence contract as the reason the Protocol exists at all. +- `packages/atif-corpus/src/atif_corpus/domain/ports.py:5-7`: the ports module docstring cites + the independence contract as the reason its Protocols are typed to the contract's shapes. - `packages/atif-embed/src/atif_embed/domain/ports.py:11-15`: cites it as the reason `TextRowsPort` exists. - `packages/atif-embed/src/atif_embed/infrastructure/corpus_text_rows.py:5-10`: cites it as the reason atif-embed carries its own corpus reader. - `packages/atif-duck/src/atif_duck/domain/embedding_guard.py:5-9`: cites it as the reason the guard is duplicated. -- `packages/atif-duck/src/atif_duck/infrastructure/analytics.py:13-15`: cites it as the reason the +- `packages/atif-duck/src/atif_duck/infrastructure/analytics.py:16-18`: cites it as the reason the analytics artifact names are pinned by hand. -- `packages/atif-analytics/src/atif_analytics/domain/layout.py:8-10`: cites it as the reason its +- `packages/atif-analytics/src/atif_analytics/domain/layout.py:11-13`: cites it as the reason its layout is computed in one place. **Shape:** @@ -243,15 +266,18 @@ forbidden_modules = ["atif_converter", "atif_corpus", "atif_duck", "atif_embed", - **atif-cli is the only composition root**, and it is absent from both the `independence` module list (`pyproject.toml:605`) and the `forbidden` source list (`pyproject.toml:610`). The one module that imports - two independent packages is `packages/atif-cli/src/atif_cli/converter_adapter.py:36-38`, and its + two independent packages at module scope is `packages/atif-cli/src/atif_cli/converter_adapter.py:64-74` + (`app.py` imports them inside command bodies), and its docstring names that privilege explicitly at `packages/atif-cli/src/atif_cli/converter_adapter.py:5-9`. - **atif-analytics is absent from the independence list on purpose** so it can compose atif-models, with the `forbidden` contract pinning its other edges shut; the reasoning is inline at - `pyproject.toml:597-601`. Verified: `grep -rn 'atif_duck' packages/atif-analytics/` returns - nothing, so the CONTRACT-V2-era design of reading atif-duck's views is not what the code does. -- **atif-duck declares no `layers` contract.** Most members do (`pyproject.toml:553-595`); atif-duck - has `domain/` and `infrastructure/` but no `application/`, so there is no third layer to order. -- **Every layered member's `domain/` is the innermost layer**, which is why all the Protocols live + `pyproject.toml:597-601`. Verified: `grep -rn 'atif_duck' packages/atif-analytics/` finds two + prose mentions of the authorship twin and no import, so the CONTRACT-V2-era design of reading atif-duck's views is not what the code does. +- **atif-duck declares no `layers` contract.** Five members do (`pyproject.toml:553-595`), atif-models + among them with just `infrastructure > domain`; atif-duck has the same two layers, `domain/` and + `infrastructure/`, and no contract orders them, so its `domain/` staying free of infrastructure + imports is a convention. +- **Every layered member's `domain/` is the innermost layer**, which is why every port Protocol lives in `domain/ports.py` and none in an `application/ports.py`: no such file exists in this workspace. - **The comment at `pyproject.toml:597-601` grants a permission that atif-cli doesn't exercise.** @@ -261,14 +287,17 @@ forbidden_modules = ["atif_converter", "atif_corpus", "atif_duck", "atif_embed", (`packages/atif-cli/tests/test_analyze_lake.py:44`). The live atif-models edge is atif-analytics' alone, its import sites led by `packages/atif-analytics/src/atif_analytics/application/use_cases/_shared.py:32-33`. -- **Every cross-package import in this workspace is INDENTED** (inside a function body or a +- **Every cross-package import under `src/` is INDENTED** (inside a function body or a `TYPE_CHECKING` block) because `PLC0415` is ignored workspace-wide to satisfy the lean-import test (`pyproject.toml:227`). The exceptions are module-scope imports in atif-cli modules that are themselves only ever imported inside a command body: - `packages/atif-cli/src/atif_cli/converter_adapter.py:36-38` and - `packages/atif-cli/src/atif_cli/duck_errors.py:26`. A line-anchored grep for `^from atif_` finds - no cross-package consumers, which is why every count in this file comes from an unanchored grep - confirmed at the site. + `packages/atif-cli/src/atif_cli/converter_adapter.py:64-74`, + `packages/atif-cli/src/atif_cli/duck_errors.py:27`, + `packages/atif-cli/src/atif_cli/lake_sessions.py:43-51`, and + `packages/atif-cli/src/atif_cli/embed_lake.py:19`. A line-anchored grep of `src/` for another + package's `^from atif_` finds only those four; the atif-cli and atif-analytics test suites do + import siblings at module scope. That's why every count in this file comes from an unanchored + grep confirmed at the site. **Drift risk:** a new workspace member that is not added to `root_packages` (`pyproject.toml:551`) is silently unchecked: import-linter reports every contract kept while the new package imports whatever it @@ -291,7 +320,7 @@ either a `layers` contract or a place in the `independence` list in the same com `packages/atif-analytics/src/atif_analytics/application/use_cases/perceived.py:368`) - `packages/atif-analytics/src/atif_analytics/application/use_cases/_shared.py:32`: the provider factory and the usage/budget plumbing. -- `packages/atif-models/src/atif_models/infrastructure/providers.py:16`: `build_structured_provider`, +- `packages/atif-models/src/atif_models/infrastructure/providers.py:20`: `build_structured_provider`, which picks the adapter by `spec.family` and returns it typed as the port (`:20-34`). - `packages/atif-models/src/atif_models/infrastructure/openai_bedrock.py:81`: `OpenAiBedrockProvider`, the openai-family adapter (the default family). @@ -396,14 +425,14 @@ a mismatched adapter fails the typecheck gate. **Consumers:** -- `packages/atif-duck/src/atif_duck/infrastructure/registry.py:107-117`: `_EDGE_COLUMNS`, the - same keys in the same order with DuckDB types. +- `packages/atif-duck/src/atif_duck/domain/raw_readers.py:62-72`: `EDGE_COLUMNS` (the registry's + `_EDGE_COLUMNS`), the same keys in the same order with DuckDB types. - `packages/atif-analytics/src/atif_analytics/infrastructure/corpus_reader.py:249-268`: reads the `uuid` field for the conflicts pipeline's returned-uuid validity guard. -- `packages/atif-corpus/src/atif_corpus/application/materialize.py:224-227`: writes the lines and +- `packages/atif-corpus/src/atif_corpus/application/materialize.py:695-698`: writes the lines and owns line termination. -- `packages/atif-duck/src/atif_duck/domain/catalog.py:330`: the `messages` view is defined as - exactly this surface. +- `packages/atif-duck/src/atif_duck/domain/catalog.py:94`: the `messages` view publishes this + surface, every key but `tool_use_ids`, plus `session_id`. **Shape:** @@ -426,41 +455,45 @@ EDGE_FIELDS: tuple[str, ...] = ( - **The producer emits lines WITHOUT trailing newlines and the writer adds them.** Stated on the port's `edges_lines` field at - `packages/atif-corpus/src/atif_corpus/domain/ports.py:30-33` and honoured at - `packages/atif-corpus/src/atif_corpus/application/materialize.py:224-227`; a producer that terminated its own lines would double them. + `packages/atif-corpus/src/atif_corpus/domain/ports.py:74-77` and honoured at + `packages/atif-corpus/src/atif_corpus/application/materialize.py:695-698`; a producer that terminated its own lines would double them. - **`parent_uuid` is a nullable string, and atif-duck declares it `VARCHAR` outright** rather than - letting JSON union inference decide; the reason is recorded at `packages/atif-duck/src/atif_duck/infrastructure/registry.py:103-106`. + letting JSON union inference decide; the reason is recorded at `packages/atif-duck/src/atif_duck/domain/raw_readers.py:57-61`. - **`tool_use_ids` means two different things by record type**: tool_use block ids for assistant records, the `tool_use_id` of tool_result blocks for user records (`packages/atif-converter/src/atif_converter/domain/edges.py:11-13`). One field, two semantics, discriminated by the sibling `type` column. - **`edges.jsonl` is the only source of raw-record identity.** The trajectory cannot supply uuids - (`packages/atif-duck/src/atif_duck/infrastructure/registry.py:38-40`, fidelity gap 7 `UUID_NOT_PRESERVED`), which is why the `messages` view is + (`packages/atif-duck/src/atif_duck/infrastructure/registry.py:70-72`, fidelity gap 7 `UUID_NOT_PRESERVED`), which is why the `messages` view is reconstructed from edges rather than from steps. -- **`records_total` in the loss report equals this file's line count.** Asserted at - `packages/atif-converter/src/atif_converter/domain/fidelity.py:91-95`. Both derive from the same +- **`records_total` in the loss report equals this file's line count.** Stated at + `packages/atif-converter/src/atif_converter/domain/fidelity.py:123-127` and asserted by + `packages/atif-converter/tests/test_snapshot_and_drift.py:133`. Both derive from the same raw census, so a consumer may cross-check one against the other. **Drift risk:** the keys are declared twice in packages that cannot import each other, and a -new key added on the producer side is simply invisible to `_EDGE_COLUMNS`: a projection reader +new key added on the producer side is simply invisible to `EDGE_COLUMNS`: a projection reader drops unknown keys without erroring. Mitigation: `EDGE_FIELDS` is a single tuple; changing it means -editing `packages/atif-duck/src/atif_duck/infrastructure/registry.py:107-117` in the same commit. +editing `packages/atif-duck/src/atif_duck/domain/raw_readers.py:62-72` in the same commit. ## `EmbeddingProvider`, `VectorStorePort`, `TextRowsPort` -**Producer:** `packages/atif-embed/src/atif_embed/domain/ports.py:31`, `packages/atif-embed/src/atif_embed/domain/ports.py:59`, `packages/atif-embed/src/atif_embed/domain/ports.py:87` +**Producer:** `packages/atif-embed/src/atif_embed/domain/ports.py:37`, `packages/atif-embed/src/atif_embed/domain/ports.py:65`, `packages/atif-embed/src/atif_embed/domain/ports.py:93` **Consumers:** - `packages/atif-embed/src/atif_embed/application/embed.py:35`: imports every one of them; annotated at `packages/atif-embed/src/atif_embed/application/embed.py:46-47` and `packages/atif-embed/src/atif_embed/application/embed.py:65-67`. -- `packages/atif-embed/src/atif_embed/infrastructure/cohere_bedrock.py:285`: the - `EmbeddingProvider` adapter. -- `packages/atif-embed/src/atif_embed/infrastructure/lance_store.py:377`: the `VectorStorePort` +- `packages/atif-embed/src/atif_embed/infrastructure/cohere_bedrock.py:285`: the Cohere + `EmbeddingProvider` adapter (the default); `packages/atif-embed/src/atif_embed/infrastructure/gemma_local.py:193` + is the local EmbeddingGemma one, and `packages/atif-embed/src/atif_embed/infrastructure/providers.py:19` + picks between them. +- `packages/atif-embed/src/atif_embed/infrastructure/lance_store.py:376`: the `VectorStorePort` adapter. -- `packages/atif-embed/src/atif_embed/infrastructure/corpus_text_rows.py:156`: the `TextRowsPort` - adapter. -- `packages/atif-embed/tests/embed_fixtures.py:171`: the deterministic `EmbeddingProvider` double. -- `packages/atif-embed/tests/test_embed_use_case.py:458`: a recording `TextRowsPort` double. +- `packages/atif-embed/src/atif_embed/infrastructure/corpus_text_rows.py:199`: the `TextRowsPort` + adapter over the corpus files; `packages/atif-embed/src/atif_embed/infrastructure/lake_text_rows.py:89` + wraps it with a lake-backed reader and a watermark. +- `packages/atif-embed/tests/embed_fixtures.py:170`: the deterministic `EmbeddingProvider` double. +- `packages/atif-embed/tests/test_embed_use_case.py:457`: a recording `TextRowsPort` double. **Shape:** @@ -491,65 +524,77 @@ class TextRowsPort(Protocol): embedded: dict[str, str] | None = None, limit: int | None = None, ) -> Iterator[PendingText]: ... + @property + def discovery(self) -> str: ... + def commit(self, *, stored_rows: int) -> None: ... ``` **Assumptions consumers make:** - **`embed_documents` returns one slot per input text in input order, and the caller enforces it.** - `packages/atif-embed/src/atif_embed/application/embed.py:191` zips with `strict=True`, so a provider returning a different length raises + `packages/atif-embed/src/atif_embed/application/embed.py:200` zips with `strict=True`, so a provider returning a different length raises `ValueError` instead of misattributing vectors to texts. - **A `None` slot means "not embedded this run", not "failed the run".** The port documents bounded - loss at `packages/atif-embed/src/atif_embed/domain/ports.py:47-51`; `packages/atif-embed/src/atif_embed/application/embed.py:192-198` filters the `None`s, counts them as skipped, and + loss at `packages/atif-embed/src/atif_embed/domain/ports.py:53-56`; `packages/atif-embed/src/atif_embed/application/embed.py:198-204` filters the `None`s, counts them as skipped, and leaves those uuids for the next pass. -- **The store's `(model, dim)` stamp is checked BEFORE any append**, `packages/atif-embed/src/atif_embed/application/embed.py:166-175`, using the +- **The store's `(model, dim)` stamp is checked BEFORE any append**, `packages/atif-embed/src/atif_embed/application/embed.py:175-183`, using the identity `table_identity()` returns; `None` there means empty and any provider may claim it. - **`add_chunk` accepts a polars frame with a fixed-size `pl.Array`, not a `pl.List`.** - The schema is built at `packages/atif-embed/src/atif_embed/application/embed.py:216-238` and the reason is stated at `packages/atif-embed/src/atif_embed/application/embed.py:212-215`: a variable-size + The schema is built at `packages/atif-embed/src/atif_embed/application/embed.py:225-244` and the reason is stated at `packages/atif-embed/src/atif_embed/application/embed.py:221-224`: a variable-size list is rejected by Lance for indexing. - **`iter_unembedded`'s laziness bounds only the CALLER's residency.** The port says so explicitly - at `packages/atif-embed/src/atif_embed/domain/ports.py:111-114`: peak resident text inside an implementation is each adapter's own - problem, and `packages/atif-embed/src/atif_embed/infrastructure/corpus_text_rows.py:54-60` establishes it by batching on BYTES (4 MiB) rather than + at `packages/atif-embed/src/atif_embed/domain/ports.py:117-120`: peak resident text inside an implementation is each adapter's own + problem, and `packages/atif-embed/src/atif_embed/infrastructure/corpus_text_rows.py:71-77` establishes it by batching on BYTES (4 MiB) rather than on file count. - **Staleness is decided by hash, not by uuid presence.** A uuid under a different `text_hash` is - yielded with `replaces_existing=True` (`packages/atif-embed/src/atif_embed/domain/ports.py:105-109`), and the caller must delete before + yielded with `replaces_existing=True` (`packages/atif-embed/src/atif_embed/domain/ports.py:112-115`), and the caller must delete before appending or the uuid fans out to two vectors - (`packages/atif-embed/src/atif_embed/application/embed.py:203-209`). -- **The CLI does not inject these ports.** `packages/atif-cli/src/atif_cli/app.py:826-831` calls - `run_backfill` with only `corpus_root`, `settings`, `limit`, and `dry_run`; the use case - constructs each default adapter itself under a deferred import (`packages/atif-embed/src/atif_embed/application/embed.py:103-110`, `packages/atif-embed/src/atif_embed/application/embed.py:157-159`) - so a dry run never loads boto3. - -- **No test binds any of these adapters to its Protocol.** Unlike `ConverterPort`, they - have no conformance assertion anywhere in `packages/atif-embed/tests/`; the only static link is - the defaulting assignment inside `run_backfill`, where each concrete class is assigned to a - parameter already annotated with the port - (`packages/atif-embed/src/atif_embed/application/embed.py:103-110` and `packages/atif-embed/src/atif_embed/application/embed.py:157-159` against the - annotations at `packages/atif-embed/src/atif_embed/application/embed.py:65-67`). ty and pyright - check those assignments; if the defaulting branch were ever refactored to construct the - adapters elsewhere, nothing would check conformance at all. - -**Drift risk:** `dimension` is read once per run and stamped on every row (`packages/atif-embed/src/atif_embed/application/embed.py:161-162`), so + (`packages/atif-embed/src/atif_embed/application/embed.py:212-219`). +- **The CLI injects only the text-rows port.** `packages/atif-cli/src/atif_cli/app.py:1937-1943` calls + `run_backfill` with `corpus_root`, `settings`, `text_rows` (`DuckDbTextRows`, wrapped in + `LakeTextRows` unless `--no-lake` is passed), `limit`, and `dry_run`; the use case constructs the + default store and embedder itself under deferred imports (`packages/atif-embed/src/atif_embed/application/embed.py:114-117`, `packages/atif-embed/src/atif_embed/application/embed.py:166-169`), + and the embedder only after the dry-run return, so a dry run never loads boto3 or torch. + +- **Each file-path adapter is bound to its Protocol by a test**, as `RealConverter` is to + `ConverterPort`: `packages/atif-embed/tests/test_embed_ports.py:49`, `:55`, and `:61` assign + `CohereBedrockEmbedder`, `LanceVectorStore`, and `DuckDbTextRows` to port-annotated names and + sweep the Protocol's members, and `packages/atif-embed/tests/test_gemma_local.py:146` does the + same for the Gemma embedder. Inside `run_backfill` each default is assigned to a parameter + already annotated with the port + (`packages/atif-embed/src/atif_embed/application/embed.py:114-117` against the + annotations at `packages/atif-embed/src/atif_embed/application/embed.py:65-67`), and + `build_embedder` returns both adapters under an `EmbeddingProvider` annotation; ty and pyright + check those too. `LakeTextRows` has no member sweep: its one static link is a test passing it as + `run_backfill`'s `text_rows` argument + (`packages/atif-embed/tests/test_lake_text_rows.py:265`), which pyright checks and ty doesn't, + because ty ignores argument types under `packages/*/tests/` (`pyproject.toml:450-457`). The + CLI types its own `text_rows` as `Any`. + +**Drift risk:** `dimension` is read once per run and stamped on every row (`packages/atif-embed/src/atif_embed/application/embed.py:171`), so a provider whose width depends on the input rather than on configuration would write a store whose rows disagree with their own `dim` column. Mitigation: `dimension` is a property fixed at adapter -construction from settings: `packages/atif-embed/src/atif_embed/infrastructure/cohere_bedrock.py:305-307` reads it from -`packages/atif-embed/src/atif_embed/infrastructure/settings.py:31`, never from a response. +construction from settings: `packages/atif-embed/src/atif_embed/infrastructure/cohere_bedrock.py:305-308` reads it from +`packages/atif-embed/src/atif_embed/infrastructure/settings.py:110-114`, never from a response. ## `ConverterPort` and `ConversionOutput` -**Producer:** `packages/atif-corpus/src/atif_corpus/domain/ports.py:41` (`ConverterPort`) and `packages/atif-corpus/src/atif_corpus/domain/ports.py:21` +**Producer:** `packages/atif-corpus/src/atif_corpus/domain/ports.py:101` (`ConverterPort`) and `packages/atif-corpus/src/atif_corpus/domain/ports.py:66` (`ConversionOutput`) **Consumers:** -- `packages/atif-corpus/src/atif_corpus/application/materialize.py:92`: imports the port; - annotated at `packages/atif-corpus/src/atif_corpus/application/materialize.py:193` and `packages/atif-corpus/src/atif_corpus/application/materialize.py:480`. -- `packages/atif-cli/src/atif_cli/converter_adapter.py:38-69`: `RealConverter`, the production - adapter and the only module importing both atif-converter and atif-corpus. -- `packages/atif-corpus/src/atif_corpus/infrastructure/fake_converter.py:16-65`: the test adapter +- `packages/atif-corpus/src/atif_corpus/application/materialize.py:172`: imports the port; + annotated at `packages/atif-corpus/src/atif_corpus/application/materialize.py:615` and `packages/atif-corpus/src/atif_corpus/application/materialize.py:1501`. +- `packages/atif-cli/src/atif_cli/converter_adapter.py:80-148`: `RealConverter`, the production + adapter and the only module importing both atif-converter and atif-corpus at module scope. +- `packages/atif-corpus/src/atif_corpus/infrastructure/fake_converter.py:72`: the test adapter that ships in `src/`, not in `tests/`. - `packages/atif-cli/tests/test_converter_adapter.py:17`: asserts the mapping and binds - `RealConverter` to the port annotation at `packages/atif-cli/tests/test_converter_adapter.py:91`. -- `packages/atif-cli/src/atif_cli/app.py:399`: wires `RealConverter()` into the use case. + `RealConverter` to the port annotation at `packages/atif-cli/tests/test_converter_adapter.py:95`; + `packages/atif-cli/tests/test_codex_cli.py:156` binds the Codex-configured adapter the same way. +- `packages/atif-cli/src/atif_cli/app.py:961`: wires `RealConverter(agent=settings.agent)` into the + use case. **Shape:** @@ -559,6 +604,9 @@ class ConversionOutput: trajectory_dict: dict[str, Any] loss_report_dict: dict[str, Any] edges_lines: list[str] + events_lines: list[str] = field(default_factory=list) + source_archive: tuple[ArchivedSource, ...] = () + blobs: tuple[BlobOutput, ...] = () class ConverterPort(Protocol): @@ -566,65 +614,70 @@ class ConverterPort(Protocol): Implementations may raise any exception: the materialize use case records the failure against the session and continues — one broken - transcript must never abort a corpus sync. + transcript must never abort a corpus sync. :class:`EmptySourceError` is + the one exception with its own meaning: the session is empty, not broken. """ - def convert(self, session_jsonl: Path) -> ConversionOutput: + def convert(self, session_jsonl: Path, *, archive_dir: Path | None = None) -> ConversionOutput: """Convert one session (main JSONL + its side-files) to artifacts.""" ... ``` **Assumptions consumers make:** -- **`RealConverter` never names the Protocol it implements.** It imports only the return-value - dataclass (`packages/atif-cli/src/atif_cli/converter_adapter.py:38`), so nothing in either package - links the class to the port. The single static link in the workspace is the annotated assignment +- **`RealConverter` never imports the Protocol it implements.** It names it only in its docstring + and imports the return-value data classes and the empty-source error + (`packages/atif-cli/src/atif_cli/converter_adapter.py:69-74`), so nothing in either package + links the class to the port. The static links are the annotated assignments `converter: ConverterPort = RealConverter()` at - `packages/atif-cli/tests/test_converter_adapter.py:91`, whose own comment - (`packages/atif-cli/tests/test_converter_adapter.py:89-90`) states that the assignment is what ty - verifies. Delete that test and a signature change on either side becomes a runtime - `AttributeError` at materialize time. + `packages/atif-cli/tests/test_converter_adapter.py:95`, whose own comment + (`packages/atif-cli/tests/test_converter_adapter.py:93-94`) states that the assignment is what ty + verifies, and its Codex twin at `packages/atif-cli/tests/test_codex_cli.py:156`, plus the CLI's + call at `packages/atif-cli/src/atif_cli/app.py:961`, which hands the adapter to materialize's + `converter: ConverterPort` parameter. A signature change on either side fails the typecheck gate + at those sites, not at materialize time. - **The port is typed to `docs/CONTRACT.md`'s artifact shapes, not to converter internals**: loosely typed `dict[str, Any]` / `list[str]` fields instead of the converter's own `ConversionResult`. The docstring at `packages/atif-corpus/src/atif_corpus/domain/ports.py:3-10` names the independence contract as the reason, so the weak typing is the contract, not an omission. -- **Exceptions cross this port by design.** `packages/atif-corpus/src/atif_corpus/domain/ports.py:44-47` says an implementation may raise - anything and the use case records the failure and continues. `RealConverter` relies on that: it +- **Exceptions cross this port by design.** `packages/atif-corpus/src/atif_corpus/domain/ports.py:104-107` says an implementation may raise + anything and the use case records the failure and continues; `EmptySourceError` alone means + the session is empty, not broken. `RealConverter` relies on that: it raises `TrajectoryValidationError` for an invalid trajectory rather than materializing it - (`packages/atif-cli/src/atif_cli/converter_adapter.py:63-64`, reasoning at `packages/atif-cli/src/atif_cli/converter_adapter.py:21-25`). + (`packages/atif-cli/src/atif_cli/converter_adapter.py:129-130`, reasoning at `packages/atif-cli/src/atif_cli/converter_adapter.py:49-53`). - **`trajectory_dict` is the ENRICHED trajectory and is NOT yet compact-serialized.** The writer - owns `separators=(",", ":")`; the mapping decision is stated at `packages/atif-cli/src/atif_cli/converter_adapter.py:13-14` and - the writer applies it at `packages/atif-corpus/src/atif_corpus/application/materialize.py:222`. + owns `separators=(",", ":")`; the mapping decision is stated at `packages/atif-cli/src/atif_cli/converter_adapter.py:30-31` and + the writer applies it at `packages/atif-corpus/src/atif_corpus/application/materialize.py:691-693`. - **`loss_report_dict` is `LossReport.to_json()`-shaped with enum members flattened to strings** - (`packages/atif-cli/src/atif_cli/converter_adapter.py:15-16`), which is what makes it readable by atif-duck's - `_LOSS_REPORT_COLUMNS` projection at `packages/atif-duck/src/atif_duck/infrastructure/registry.py:122-131`. -- **Importing the adapter drags harbor.** `packages/atif-cli/src/atif_cli/converter_adapter.py:27-29` forbids importing it at - `atif_cli.app` module scope, and `packages/atif-cli/src/atif_cli/app.py:387` obeys by importing inside the command body, + (`packages/atif-cli/src/atif_cli/converter_adapter.py:32-33`), which is what makes it readable by atif-duck's + `LOSS_REPORT_COLUMNS` projection at `packages/atif-duck/src/atif_duck/domain/raw_readers.py:77-90`. +- **Importing the adapter loads atif-converter and its ATIF pydantic models.** `packages/atif-cli/src/atif_cli/converter_adapter.py:55-57` forbids importing it at + `atif_cli.app` module scope, and `packages/atif-cli/src/atif_cli/app.py:925` obeys by importing inside the command body, enforced by the fresh-interpreter lean-import test named at `pyproject.toml:227`. **Drift risk:** the `dict[str, Any]` fields mean a converter that renames a trajectory key type-checks perfectly and produces artifacts atif-duck's explicit projections silently null out; and because the implementer never names the port, a `convert` signature change on either side is -caught by exactly one assertion. Mitigation: -`packages/atif-cli/tests/test_converter_adapter.py:88-92` is that assertion (treat it as part of +caught only where a typed site meets the adapter. Mitigation: +`packages/atif-cli/tests/test_converter_adapter.py:92-96` is the dedicated assertion (treat it as part of the contract, not as a redundant smoke test), and -`packages/atif-cli/tests/test_converter_adapter.py:17-60` pins the field mapping while atif-duck's +`packages/atif-cli/tests/test_converter_adapter.py:44` pins the field mapping while atif-duck's `DESCRIBE` drift test catches a projection that stops matching. ## Token semantics and the pricing table -**Producer:** `packages/atif-duck/src/atif_duck/infrastructure/registry.py:41-44` (the semantics) -and `packages/atif-duck/src/atif_duck/domain/catalog.py:405` (`DEFAULT_PRICING`) +**Producer:** `packages/atif-duck/src/atif_duck/infrastructure/registry.py:73-76` (the semantics) +and `packages/atif-duck/src/atif_duck/domain/catalog.py:612` (`DEFAULT_PRICING`) **Consumers:** -- `packages/atif-duck/src/atif_duck/infrastructure/registry.py:1066-1083`: the `cost_estimate` +- `packages/atif-duck/src/atif_duck/infrastructure/registry.py:1936-1956`: the `cost_estimate` macro, which consumes both. -- `packages/atif-duck/src/atif_duck/infrastructure/registry.py:1020`: `register_macros` resolves +- `packages/atif-duck/src/atif_duck/infrastructure/registry.py:1892`: `register_macros` resolves the pricing override against `DEFAULT_PRICING`. -- `packages/atif-duck/src/atif_duck/domain/catalog.py:72-75`: the `steps` view schema publishes +- `packages/atif-duck/src/atif_duck/domain/catalog.py:85-88`: the `steps` view schema publishes `prompt_tokens`, `completion_tokens`, `cached_tokens`, `cache_creation` to every SQL consumer. -- `packages/atif-duck/src/atif_duck/domain/catalog.py:347-350`: the agent-facing `cost_estimate` +- `packages/atif-duck/src/atif_duck/domain/catalog.py:557-563`: the agent-facing `cost_estimate` description carries the trust condition. **Shape:** @@ -634,42 +687,44 @@ and `packages/atif-duck/src/atif_duck/domain/catalog.py:405` (`DEFAULT_PRICING`) # Anthropic's published pricing page. DEFAULT_PRICING: dict[str, tuple[float, float]] = { "claude-fable-5": (10.0, 50.0), + ... "claude-opus-5": (5.0, 25.0), + ... "claude-sonnet-5": (2.0, 10.0), - "claude-haiku-4-5": (1.0, 5.0), ... + "claude-haiku-4-5": (1.0, 5.0), } ``` **Assumptions consumers make:** - **`prompt_tokens` is a CUMULATIVE TOTAL, per step: non-cached + cache_read + cache_creation.** - Stated at `packages/atif-duck/src/atif_duck/infrastructure/registry.py:41-44` and read from `metrics.prompt_tokens` at `packages/atif-duck/src/atif_duck/infrastructure/registry.py:376-377`. The + Stated at `packages/atif-duck/src/atif_duck/infrastructure/registry.py:73-76` and read from `metrics.prompt_tokens` at `packages/atif-duck/src/atif_duck/infrastructure/projections.py:130`. The identifier does not say so, so a consumer summing `prompt_tokens + cached_tokens` double-counts every cache read. - **`cached_tokens` is the cache-READ subset of `prompt_tokens`**, not an additional quantity - (`packages/atif-duck/src/atif_duck/infrastructure/registry.py:380-381`). `cost_estimate` therefore prices `prompt_tokens - cached_tokens` and - documents that identity at `packages/atif-duck/src/atif_duck/infrastructure/registry.py:1043-1049`; cache reads are charged nothing. + (`packages/atif-duck/src/atif_duck/infrastructure/projections.py:96-97`). `cost_estimate` therefore prices `prompt_tokens - cached_tokens` and + documents that identity at `packages/atif-duck/src/atif_duck/infrastructure/registry.py:1915-1918`; cache reads are charged nothing. - **`in_rate` / `out_rate` are USD per 1,000,000 tokens.** The unit appears in neither name: only - the comment at `packages/atif-duck/src/atif_duck/domain/catalog.py:395` and the `/ 1e6` at `packages/atif-duck/src/atif_duck/infrastructure/registry.py:1071` carry it. + the comment at `packages/atif-duck/src/atif_duck/domain/catalog.py:602` and the `/ 1e6` at `packages/atif-duck/src/atif_duck/infrastructure/registry.py:1943` carry it. - **`est_cost_usd` is USD and covers the PRICED steps only.** It is meaningful only when - `unpriced_steps = 0`, stated in each of these places: `packages/atif-duck/src/atif_duck/infrastructure/registry.py:1053-1058`, `packages/atif-duck/src/atif_duck/domain/catalog.py:347-350`, and the - `LEFT JOIN` shape itself at `packages/atif-duck/src/atif_duck/infrastructure/registry.py:1079`. + `unpriced_steps = 0`, stated in each of these places: `packages/atif-duck/src/atif_duck/infrastructure/registry.py:1925-1930`, `packages/atif-duck/src/atif_duck/domain/catalog.py:557-563`, and the + `LEFT JOIN` shape itself at `packages/atif-duck/src/atif_duck/infrastructure/registry.py:1951`. - **`unpriced_steps` counts only steps with a non-NULL `model_name`**, because user steps carry no model and cost nothing - (`packages/atif-duck/src/atif_duck/infrastructure/registry.py:1060-1063`). Counting them would put every conversation + (`packages/atif-duck/src/atif_duck/infrastructure/registry.py:1932-1935`). Counting them would put every conversation above zero and mask real pricing gaps. -- **Cache write and read multipliers are NOT modelled.** `packages/atif-duck/src/atif_duck/domain/catalog.py:402-404` states that the 1.25x +- **Cache write and read multipliers are NOT modelled.** `packages/atif-duck/src/atif_duck/domain/catalog.py:609-611` states that the 1.25x / 2x / 0.1x tiers are out of scope, so `est_cost_usd` under-reports a cache-heavy session. -- **A step's `model_name` matches a pricing row by dated-suffix-stripping prefix** - (`packages/atif-duck/src/atif_duck/infrastructure/registry.py:1080`), so `claude-haiku-4-5-20251001` prices as `claude-haiku-4-5`. +- **A step's `model_name` matches a pricing row by exact name once a trailing `-YYYYMMDD` is stripped** + (`packages/atif-duck/src/atif_duck/infrastructure/registry.py:1952`), so `claude-haiku-4-5-20251001` prices as `claude-haiku-4-5`. **Drift risk:** `cached_tokens` exists in two coordinate spaces under one name: atif-duck's -per-step cache-read count (`packages/atif-duck/src/atif_duck/infrastructure/registry.py:380`) and atif-models' per-call +per-step cache-read count (`packages/atif-duck/src/atif_duck/infrastructure/projections.py:132`) and atif-models' per-call `CallUsage.cached_tokens` (`packages/atif-models/src/atif_models/domain/ports.py:77`). Neither name states its scope, so a cross-plane join or a copied formula silently mixes them. Mitigation: attribute the field by module path, and read the producer docstring -(`packages/atif-duck/src/atif_duck/infrastructure/registry.py:41-44` or +(`packages/atif-duck/src/atif_duck/infrastructure/registry.py:73-76` or `packages/atif-models/src/atif_models/domain/ports.py:65-71`) before using either in arithmetic. ## The `EmbeddingProviderMismatch` guard, declared twice @@ -677,17 +732,17 @@ attribute the field by module path, and read the producer docstring **Producer:** two coordinate declarations, coupled by contract rather than by import: `packages/atif-embed/src/atif_embed/domain/errors.py:29` with the rule at `packages/atif-embed/src/atif_embed/domain/embedding_guard.py:38`, and -`packages/atif-duck/src/atif_duck/domain/embedding_guard.py:33` with the rule at `packages/atif-duck/src/atif_duck/domain/embedding_guard.py:46` +`packages/atif-duck/src/atif_duck/domain/embedding_guard.py:33` with the rule at `packages/atif-duck/src/atif_duck/domain/embedding_guard.py:51` **Consumers:** -- `packages/atif-embed/src/atif_embed/application/embed.py:166-175`: the WRITE path calls +- `packages/atif-embed/src/atif_embed/application/embed.py:175-183`: the WRITE path calls `ensure_store_matches` before appending. -- `packages/atif-duck/src/atif_duck/infrastructure/registry.py:830-861`: the READ/bind path calls +- `packages/atif-duck/src/atif_duck/infrastructure/registry.py:1707-1722`: the READ/bind path calls it before binding `message_embeddings` (guard-before-bind). -- `packages/atif-cli/src/atif_cli/duck_errors.py:26-31`: puts atif-duck's copy in - `REGISTRATION_ERRORS` and maps it to exit 65 at `packages/atif-cli/src/atif_cli/duck_errors.py:74-81`. -- `packages/atif-embed/tests/test_guard_twin_pin.py:93`: reads both twin modules as source text +- `packages/atif-cli/src/atif_cli/duck_errors.py:27-32`: puts atif-duck's copy in + `REGISTRATION_ERRORS` and maps it to exit 65 at `packages/atif-cli/src/atif_cli/duck_errors.py:75-82`. +- `packages/atif-embed/tests/test_guard_twin_pin.py:85`: reads both twin modules as source text and requires the recovery hint to appear after the `raise` keyword. **Shape:** @@ -717,44 +772,45 @@ def ensure_store_matches( makes it more than a comment. - **The message text must be constructed INSIDE the `raise`.** Hoisting it to a local satisfies ruff's `EM102` / `TRY003` and defeats the twin pin, so both rules are suppressed on purpose at - `packages/atif-duck/src/atif_duck/domain/embedding_guard.py:67-74`. + `packages/atif-duck/src/atif_duck/domain/embedding_guard.py:72-79`. - **A `None` on either stored value means "empty store, any provider may claim it"**, not - "unknown, be careful" (`packages/atif-duck/src/atif_duck/domain/embedding_guard.py:61-62`). + "unknown, be careful" (`packages/atif-duck/src/atif_duck/domain/embedding_guard.py:60-61`). - **`expected_dim=None` trusts `model_id` alone**, and dim is checked only because Cohere's single model id can emit different Matryoshka widths - (`packages/atif-duck/src/atif_duck/domain/embedding_guard.py:57-59`). + (`packages/atif-duck/src/atif_duck/domain/embedding_guard.py:62-64`). - **The two classes have different base classes and that matters at the CLI.** atif-duck's derives from `Exception` (`packages/atif-duck/src/atif_duck/domain/embedding_guard.py:33`) while atif-embed's derives from that package's `DomainError` (`packages/atif-embed/src/atif_embed/domain/errors.py:29`), so a bare `except duckdb.Error` on the registration path would let atif-duck's escape as exit 1 (the reason `REGISTRATION_ERRORS` widens the caught set, stated at - `packages/atif-cli/src/atif_cli/duck_errors.py:14-18`). + `packages/atif-cli/src/atif_cli/duck_errors.py:15-19`). - **The recovery hint must not name a fixed home directory.** The store lives at `/embeddings_lance` by default, and both copies record that naming the wrong path makes the operator delete nothing (`packages/atif-duck/src/atif_duck/domain/embedding_guard.py:21-24` and `packages/atif-embed/src/atif_embed/domain/embedding_guard.py:25-29`). -**Drift risk:** the twin pin checks that the hint constant reaches the `raise`; it does not check -that the two hint STRINGS are equal, so the copies could diverge in wording while both tests pass. -Mitigation: treat the two `RECOVERY_HINT` literals as one value. Change both in the same commit, -which is what both docstrings instruct. +**Drift risk:** the twin pin holds both `RECOVERY_HINT` literals to one expected string +(`packages/atif-embed/tests/test_guard_twin_pin.py:76` and `:79`) and checks that each reaches its +`raise`; nothing compares the two comparison rules or the rest of the message, so the logic could +diverge while every test passes. Mitigation: treat the two functions as one. Change both in +the same commit, which is what both docstrings instruct. ## harbor's ATIF models, vendored and consumed by our ported converters **Producer:** upstream `harbor.models.trajectories` (RFC 0001 data classes) and -`harbor.utils.trajectory_validator` at harbor 0.22.0, vendored as `packages/atif-converter/src/atif_converter/domain/atif/` +`harbor.utils.trajectory_validator` at harbor 0.24.0, vendored as `packages/atif-converter/src/atif_converter/domain/atif/` (`UPSTREAM_VERSION` in its `__init__`); harbor itself is a dev dependency **Consumers:** -- `packages/atif-converter/src/atif_converter/domain/claude_code_conversion.py:75`: `convert_claude_code_records`, our Claude Code converter, a parity port of harbor - 0.22.0's, building `Trajectory` / `Step` / `ToolCall` / `Metrics` from the public models. -- `packages/atif-converter/src/atif_converter/domain/codex_conversion.py:781`: `convert_codex_records`, the Codex counterpart. +- `packages/atif-converter/src/atif_converter/domain/claude_code_conversion.py:981`: `convert_claude_code_records`, our Claude Code converter, a parity port of harbor + 0.24.0's, building `Trajectory` / `Step` / `ToolCall` / `Metrics` from the public models. +- `packages/atif-converter/src/atif_converter/domain/codex_conversion.py:924`: `convert_codex_records`, the Codex counterpart. - `packages/atif-converter/src/atif_converter/infrastructure/harbor_adapter.py`: `validate_trajectory`, the only call into the vendored validator. - `packages/atif-converter/tests/test_dev_only_imports_guard.py`: the `ast` guard that keeps `harbor` and `litellm` out of every `src/` tree. - `packages/atif-converter/tests/test_vendored_atif.py`: the conformance test (the vendored files equal the installed harbor's, and both agree on the goldens). -- `packages/atif-converter/tests/harbor_oracle.py:94` and `:111`: the parity ORACLE (harbor's private converters, reached from the tests +- `packages/atif-converter/tests/harbor_oracle.py:118` and `:135`: the parity ORACLE (harbor's private converters, reached from the tests only), with frozen goldens and a live-corpus diff. **Shape:** @@ -767,19 +823,19 @@ DEV_ONLY: frozenset[str] = frozenset({"harbor", "litellm"}) **Assumptions consumers make:** - **The data classes are the contract, and they are versioned by ATIF.** Our converters write - `schema_version` explicitly (`packages/atif-converter/src/atif_converter/domain/claude_code_conversion.py:72`, `packages/atif-converter/src/atif_converter/domain/codex_conversion.py:74`), so a harbor release that adds a newer + `schema_version` explicitly (`packages/atif-converter/src/atif_converter/domain/claude_code_conversion.py:1067`, `packages/atif-converter/src/atif_converter/domain/codex_conversion.py:1023`), so a harbor release that adds a newer default still validates what we emit; a release that changes a field fails the drift tests. - **Parity is measured, not assumed.** `test_harbor_oracle.py` asserts the live oracle equals the frozen goldens, and `test_parity_*` asserts our output equals both. A harbor bump that changes - conversion behavior surfaces as a named JSON-path diff (`packages/atif-converter/tests/harbor_oracle.py:142`), which is a decision to + conversion behavior surfaces as a named JSON-path diff (`packages/atif-converter/tests/harbor_oracle.py:166`), which is a decision to record in the fidelity policy before any re-freeze. - **Side-file discovery is ours.** Every `*.jsonl` under a session's side directory is read, workflow-nested ones included, with nested path parts joined by `__` - (`packages/atif-converter/src/atif_converter/infrastructure/claude_code_converter.py:57`); harbor's + (`packages/atif-converter/src/atif_converter/infrastructure/claude_code_converter.py:56`); harbor's own discovery cannot see those files, which is why the oracle stages them flat the same way. - **harbor ships no `py.typed`**, so every import from it carries an `import-untyped` ignore. -## The `atif-sql` distribution's `==0.2.0` sibling pins +## The workspace members' `==0.3.0` sibling pins **Producer:** `packages/atif-cli/pyproject.toml:32-36` and `packages/atif-analytics/pyproject.toml:23` @@ -790,7 +846,7 @@ DEV_ONLY: frozenset[str] = frozenset({"harbor", "litellm"}) - `packages/atif-cli/pyproject.toml:60-65`: `[tool.uv.sources]`, the local-workspace resolution that these pins deliberately do NOT express. - `packages/atif-analytics/pyproject.toml:47-48`: the same pairing for its single sibling dependency. -- `pyproject.toml:491-494`: `[tool.commitizen] version = "0.2.0"`, the one version this repo +- `pyproject.toml:491-494`: `[tool.commitizen] version = "0.3.0"`, the one version this repo publishes. - `pyproject.toml:16-17`: the distribution is named `atif-sql` at the workspace root, while the console script's module stays `atif_cli` in a member. @@ -799,11 +855,11 @@ DEV_ONLY: frozenset[str] = frozenset({"harbor", "litellm"}) ```toml dependencies = [ - "atif-analytics==0.2.0", - "atif-converter==0.2.0", - "atif-corpus==0.2.0", - "atif-duck==0.2.0", - "atif-embed==0.2.0", + "atif-analytics==0.3.0", + "atif-converter==0.3.0", + "atif-corpus==0.3.0", + "atif-duck==0.3.0", + "atif-embed==0.3.0", "cyclopts>=5.1.1", "duckdb>=1.5.2,<2", "loguru>=0.7.3", @@ -819,24 +875,29 @@ version_files = [ ] ``` +The other five members (atif-converter, atif-corpus, atif-duck, atif-embed, atif-models) declare +no sibling, so these six pins are every edge between members. + **Assumptions consumers make:** -- **`[tool.uv.sources]` is invisible to an external installer.** uv's build backend does not - translate a workspace source into a version constraint, so a bare name would ship as - `Requires-Dist: atif-duck` and resolve from PyPI to whatever the newest release of that name is, - owned by whoever owns it; the reasoning is inline at `packages/atif-cli/pyproject.toml:24-31`. +- **No external installer sees these pins.** The member manifests are dev manifests that the + release never builds (`pyproject.toml:10-14`); the one published wheel is the root's, its + `dependencies` hold no `atif-*` entry (`pyproject.toml:34-40`), and + `packages/atif-cli/tests/test_distribution.py:143` fails if one reaches it. The comment at + `packages/atif-cli/pyproject.toml:24-31` reasons as if the manifest shipped: the pins keep + the workspace coherent, and `[tool.uv.sources]` is what resolves them. - **One version covers the whole repository.** commitizen reads its own `version` key rather than the PEP 621 field (`pyproject.toml:491-494`), and `version_files` propagates it to the published `[project] version` and to every member manifest; a separate number per member for one artifact is the shape being refused. -- **The `version_files` entries are per-file rather than globbed on purpose**, because - `--check-consistency` requires a hit in every matched file and most members carry no - dev pin between members (`pyproject.toml:525-527`). +- **The pin entries in `version_files` are per-file rather than globbed on purpose**, because + `--check-consistency` requires a hit in every matched file and five of the seven members + pin no sibling (`pyproject.toml:525-527`). - **The lockfile must land in the same commit as the versions it resolves.** `pre_bump_hooks` run `uv lock` and `git add uv.lock` after the rewrite and before the commit (`pyproject.toml:509-519`), because `uv.lock` records every member's version and `mise run lock:check` would otherwise fail on the release commit itself. -- **A breaking change moves 0.2.0 to 0.3.0, not 1.0.0**, because `major_version_zero = true` +- **A breaking change moves 0.3.0 to 0.4.0, not 1.0.0**, because `major_version_zero = true` (`pyproject.toml:500-502`); reaching 1.0.0 is a decision, not a side effect of a `!` in a subject line. @@ -848,20 +909,21 @@ rather than skipping a file that no longer contains the current version; the gua ## Other contracts - **`LossReport.to_json()`**: producer - `packages/atif-converter/src/atif_converter/domain/fidelity.py:114`, whose docstring calls the - keys wire contract; consumed by `packages/atif-duck/src/atif_duck/infrastructure/registry.py:122-131` - as `_LOSS_REPORT_COLUMNS` and by `packages/atif-cli/src/atif_cli/converter_adapter.py:67`. + `packages/atif-converter/src/atif_converter/domain/fidelity.py:147`, whose docstring calls the + keys wire contract; consumed by `packages/atif-duck/src/atif_duck/domain/raw_readers.py:77-90` + as `LOSS_REPORT_COLUMNS` and by `packages/atif-cli/src/atif_cli/converter_adapter.py:133`. `record_counts` and `gaps_observed` stay `JSON` because they are an enum-keyed dict and a sorted list of enum values. - **The `meta.json` provenance record**: written at - `packages/atif-corpus/src/atif_corpus/application/materialize.py:230-237`, projected at - `packages/atif-duck/src/atif_duck/infrastructure/registry.py:134-141`. `harbor_version` and - `converter_version` are supplied by the CLI (`packages/atif-cli/src/atif_cli/app.py:401-402`), so - the corpus records which converter built it without atif-corpus importing either. + `packages/atif-corpus/src/atif_corpus/application/materialize.py:706-731`, projected at + `packages/atif-duck/src/atif_duck/domain/raw_readers.py:99-121`. `harbor_version` (the vendored + models' upstream release) and `converter_version` (the `atif-sql` release) are supplied by the + CLI (`packages/atif-cli/src/atif_cli/app.py:966` and `:971`), so the corpus records which + converter built it without atif-corpus importing either. - **The embeddings-store row shape**: the Arrow fields written at `packages/atif-embed/src/atif_embed/infrastructure/lance_store.py:115-121`, of which the `message_embeddings` view exposes only some - (`packages/atif-duck/src/atif_duck/domain/catalog.py:218-224`). `text_hash` and `truncated` are + (`packages/atif-duck/src/atif_duck/domain/catalog.py:281-287`). `text_hash` and `truncated` are unreachable from SQL, so a query cannot distinguish a row embedded from its head alone from a complete one. - **The Lance schema version sidecar**: `SCHEMA_VERSION = 2` and `schema_version.json` at `packages/atif-embed/src/atif_embed/infrastructure/lance_store.py:61-67`, kept as a sidecar rather @@ -878,15 +940,16 @@ rather than skipping a file that no longer contains the current version; the gua - **The analytics parquet layout**: the artifact names at `packages/atif-analytics/src/atif_analytics/domain/layout.py:25-41`, of which atif-duck pins the pipeline directories as `_ANALYTICS_SOURCES` (`packages/atif-duck/src/atif_duck/infrastructure/analytics.py:61-66`). - `REFUSALS_DIRNAME` (`packages/atif-analytics/src/atif_analytics/domain/layout.py:39`) has no view, so the refusal audit its docstring calls - queryable is not reachable from SQL. + `REFUSALS_DIRNAME` (`packages/atif-analytics/src/atif_analytics/domain/layout.py:39`) has no view, as its own comment says + (`packages/atif-analytics/src/atif_analytics/domain/layout.py:35-38`), so the refusal audit has no name in `atif-sql schema` and a + reader has to `read_parquet` its shard files directly. - **The CLI exit-code taxonomy**: `EXIT_CODES` at - `packages/atif-cli/src/atif_cli/errors.py:25-39`, consumed at many sites in + `packages/atif-cli/src/atif_cli/errors.py:30-66`, consumed at many sites in `packages/atif-cli/src/atif_cli/app.py` and mapped from DuckDB exceptions at - `packages/atif-cli/src/atif_cli/duck_errors.py:34-63`. Code 78 is the load-bearing one: it means + `packages/atif-cli/src/atif_cli/duck_errors.py:35-64`. Code 78 matters most: it means an operator must act, and unattended lanes suppress retries on it - (`packages/atif-cli/src/atif_cli/errors.py:35-37`). -- **`PendingText` and the text stamp**: `packages/atif-embed/src/atif_embed/domain/text_stamp.py:50`, + (`packages/atif-cli/src/atif_cli/errors.py:49-51`). +- **`PendingText` and the text stamp**: `packages/atif-embed/src/atif_embed/domain/text_stamp.py:51`, with `MAX_EMBEDDABLE_CHARS = 50_000` at `packages/atif-embed/src/atif_embed/domain/text_stamp.py:35` in CHARACTERS per text; the same constant governs both the row's `truncated` flag and the adapter's wire-level clip so the two can never disagree (`packages/atif-embed/src/atif_embed/domain/text_stamp.py:31-34`). @@ -895,23 +958,37 @@ rather than skipping a file that no longer contains the current version; the gua atif-duck's `steps` rules by hand (message-union flattening, `extra.source_uuids[0]` as the step key, `agent` → `assistant`, error recovery from the typed `extra.is_error` or from `extra.tool_result_metadata.is_error`), and - `packages/atif-embed/src/atif_embed/infrastructure/corpus_text_rows.py:12-25` reproduces some of + `packages/atif-embed/src/atif_embed/infrastructure/corpus_text_rows.py:13-26` reproduces some of them again. - **The retryable Bedrock error codes, declared twice**: `RETRY_CODES` at `packages/atif-models/src/atif_models/infrastructure/bedrock_common.py:27-42`, which both LLM adapters retry on through `is_retryable` (`:66-73`), and its twin at `packages/atif-embed/src/atif_embed/infrastructure/cohere_bedrock.py:56-72`. The two packages can't import each other, so each suite pins its copy against the same spelled-out set - (`packages/atif-models/tests/test_openai_provider.py:345` and - `packages/atif-embed/tests/test_cohere_bedrock.py:445`), and each copy's comment names the other + (`packages/atif-models/tests/test_openai_provider.py:344` and + `packages/atif-embed/tests/test_cohere_bedrock.py:444`), and each copy's comment names the other (`packages/atif-embed/src/atif_embed/infrastructure/cohere_bedrock.py:56-60`). - **The `ATIF_SQL_` settings prefix**: the one env namespace across every member; - `packages/atif-embed/src/atif_embed/infrastructure/settings.py:31` pins - `output_dimension: Literal[256, 512, 1024, 1536] = 1024`, which is the value the static catalog - hardcodes as `FLOAT[1024]`. + `packages/atif-embed/src/atif_embed/infrastructure/settings.py:64` declares + `output_dimension: int | None = None`, and an unset width takes the provider's default from + `packages/atif-embed/src/atif_embed/infrastructure/settings.py:36` (1024 for Cohere, 768 for + Gemma). Cohere's 1024 is the value the static catalog hardcodes as `FLOAT[1024]`. - **The lean-import contract**: `pyproject.toml:227` and `pyproject.toml:242-245` record that `atif_cli.app` must not import duckdb, harbor, lancedb, boto3, or polars at module scope, asserted in a fresh interpreter by `packages/atif-cli/tests/test_lean_import.py`; the deferred-import sites exist because of it. +- **The other ports, satisfied the same way**: atif-corpus's `ArtifactProducer` + (`packages/atif-corpus/src/atif_corpus/domain/ports.py:123`), which atif-duck's + `ColumnarArtifactProducer` fits (`packages/atif-duck/src/atif_duck/infrastructure/columnar.py:450`) + and which the CLI's `materialize` doesn't inject, and `SessionSink` + (`packages/atif-corpus/src/atif_corpus/domain/ports.py:155`), which atif-duck's + `DuckLakeSessionSink` fits (`packages/atif-duck/src/atif_duck/infrastructure/lake.py:868`) and + the CLI wires in at `packages/atif-cli/src/atif_cli/app.py:986`; atif-embed's `LakeStepsPort` and + `LakeStepSnapshot` (`packages/atif-embed/src/atif_embed/domain/ports.py:202` and `:154`), which + atif-cli's `DuckLakeSteps` fits (`packages/atif-cli/src/atif_cli/embed_lake.py:70`); and + atif-analytics' `SessionSource` (`packages/atif-analytics/src/atif_analytics/domain/ports.py:40`), + which `TrajectoryFileSource` + (`packages/atif-analytics/src/atif_analytics/infrastructure/corpus_reader.py:271`) and atif-cli's + `LakeSessionSource` (`packages/atif-cli/src/atif_cli/lake_sessions.py:92`) fit. ## See also diff --git a/docs/insights/debugging-guide.md b/docs/insights/debugging-guide.md index 3ea5fe5..3a5e56e 100644 --- a/docs/insights/debugging-guide.md +++ b/docs/insights/debugging-guide.md @@ -7,14 +7,14 @@ observability platform. That narrows the search surface to a few places, and this guide is organized around them: 1. **The process exit code.** `EXIT_CODES` is one dict - (`packages/atif-cli/src/atif_cli/errors.py:25-39`) and it is the primary + (`packages/atif-cli/src/atif_cli/errors.py:30-66`) and it is the primary diagnostic. Not every key is reachable. 2. **The classified error envelope on stderr**: one readable line on a TTY, - one JSON object on a pipe (`packages/atif-cli/src/atif_cli/output.py:274-281`). -3. **The loguru sink.** Exactly one, added in `main()`, stderr, WARNING and up - (`packages/atif-cli/src/atif_cli/app.py:1133-1134`). + one JSON object on a pipe (`packages/atif-cli/src/atif_cli/output.py:278-285`). +3. **The loguru sink.** Exactly one, added in `main()`, stderr, WARNING and up unless + `ATIF_SQL_LOG_LEVEL` sets another level (`packages/atif-cli/src/atif_cli/app.py:2427-2430`). 4. **The cron refresh log and its marker files**, for anything that failed - unattended (`scripts/atif-sql-refresh.sh:102-103`). + unattended (`scripts/atif-sql-refresh.sh:151-153`). The exit-code contract, read from source and confirmed by running the CLI: @@ -22,24 +22,25 @@ The exit-code contract, read from source and confirmed by running the CLI: | --- | --- | --- | | 0 | `ok` | success | | 2 | `empty_session`, `no_embeddings` | nothing to work on; not a fault | -| 64 | `invalid_input`, `parse_error` | the caller's input is malformed | -| 65 | `catalog_error`, `validation_error`, `embedding_mismatch` | the named object cannot be bound | -| 70 | `runtime_error` | everything else the adapter raises | -| 78 | `terminal_state`, `suspicious_scan` | an operator has to act; retrying cannot succeed | -| 127 | `harbor_missing` | RETIRED: kept so the table never renumbers; no code path raises it since the conversion became ours | +| 64 | `invalid_input`, `parse_error` | the caller's input is malformed; a flag cyclopts refuses exits 64 too | +| 65 | `catalog_error`, `validation_error`, `embedding_mismatch`, `lake_mismatch` | the named object cannot be bound, or the lake disagrees with the artifacts | +| 70 | `runtime_error`, `sandbox_refused` | everything else the adapter raises, or a statement kind the query sandbox never runs | +| 77 | `root_refused` | `query`, `search`, or `analyze` running as uid 0 without `ATIF_SQL_ALLOW_ROOT=1` | +| 78 | `terminal_state`, `suspicious_scan`, `extension_missing`, `lake_unavailable` | an operator has to act; retrying cannot succeed | +| 127 | `harbor_missing` | RETIRED: kept so the table never renumbers; no code path raises it | | 1 | *(none)* | an unhandled exception, outside the taxonomy | One of these codes exists to be told apart from a crash rather than from another code. **`suspicious_scan: 78`** is raised only by -`materialize` (`packages/atif-cli/src/atif_cli/app.py:429-443`), when the source scan finds +`materialize` (`packages/atif-cli/src/atif_cli/app.py:1012-1026`), when the source scan finds zero sessions over a non-empty corpus and marking sessions source-removed is refused; it shares 78 with `terminal_state` because both mean an operator has to act, and carries its own `kind` string because the remedy is a path rather than a store. **Exit 1 is still a reachable outcome**, and it means an unhandled exception: `analyze` maps -nothing into the taxonomy (`packages/atif-cli/src/atif_cli/app.py:755-773` has no `except`), -and neither does a registration error outside `REGISTRATION_ERRORS` -(`packages/atif-cli/src/atif_cli/duck_errors.py:14-18`). Read a 1 as a bug report, never as a +no analytics error into the taxonomy (`packages/atif-cli/src/atif_cli/app.py:1717-1731` has a +`finally` and no `except`), and neither does a registration error outside `REGISTRATION_ERRORS` +(`packages/atif-cli/src/atif_cli/duck_errors.py:32`). Read a 1 as a bug report, never as a diagnosis. On a pipe the envelope is the last line on stderr, in this shape: @@ -52,36 +53,36 @@ On a pipe the envelope is the last line on stderr, in this shape: | Symptom | Likely surface | First check | Citation | | --- | --- | --- | --- | -| `query` exits 64, kind `parse_error` | `duckdb.ParserException`, or no SQL argument at all | Run `atif-sql schema --format json` for the real view and column names; a bare `atif-sql query` also exits 64 | `packages/atif-cli/src/atif_cli/duck_errors.py:37-43`, `packages/atif-cli/src/atif_cli/app.py:599-610` | -| `query` exits 65, kind `catalog_error` | `duckdb.CatalogException`: the view, macro, or column does not exist | `atif-sql schema` answers from the static catalog with no DuckDB bind, so it works even on an unmaterialized corpus | `packages/atif-cli/src/atif_cli/duck_errors.py:44-50`, `packages/atif-cli/src/atif_cli/app.py:1086-1122` | -| `query` exits 70 with `IO Error: No files found that match the pattern .../sessions/*/meta.json` | The corpus is not materialized; `register_raw`'s glob matches nothing | `atif-sql status`; a `watermark: never materialized` line means run `atif-sql materialize` | `packages/atif-cli/src/atif_cli/duck_errors.py:51-57`, `packages/atif-duck/src/atif_duck/infrastructure/registry.py:306-309` | -| `query` or `search` exits 65, kind `embedding_mismatch` | The Lance store's stamped `(model, dim)` differs from the active embedder | The message names both sides; either point `ATIF_SQL_EMBED_MODEL_ID` at the model that wrote the store, or delete the store | `packages/atif-duck/src/atif_duck/domain/embedding_guard.py:61-80`, `packages/atif-cli/src/atif_cli/duck_errors.py:74-81` | -| `search` exits 2, kind `no_embeddings` | The store is empty or absent, so `register_vss` bound an empty fallback table | `atif-sql query 'SELECT count(*) FROM message_embeddings'`; zero rows confirms it | `packages/atif-cli/src/atif_cli/app.py:930-941`, `packages/atif-duck/src/atif_duck/infrastructure/registry.py:896-914` | -| A bare `atif-sql embed` exits 64 | The scope guard, which runs before any Bedrock client is built | Pass `--limit N`, `--all`, or `--dry-run`; this is the accidental-full-backfill guard, not a fault | `packages/atif-cli/src/atif_cli/app.py:810-820` | -| `embed` exits 78 and the cron lane stops trying | A `DomainError` with `terminal = True`; the store or its config needs an operator | Read the terminal marker file the refresh lane drops: three lines, store path, mtime, reason | `packages/atif-embed/src/atif_embed/domain/errors.py:14-26`, `scripts/atif-sql-refresh.sh:244-249` | -| A session directory exists under `sessions/` but no view returns its rows | Torn artifact set: the dir has no `meta.json`, so the meta gate excludes it from every reader | Grep the WARNING; `meta.json` is written last and completes the set | `packages/atif-duck/src/atif_duck/infrastructure/registry.py:173-177`, `docs/CONTRACT.md:22-31` | -| `atif-sql status` and `SELECT count(*) FROM sessions` disagree | `status` counts directories with no meta check; the view applies the meta gate | The difference is exactly the number of incomplete dirs; use it as the torn-set count | `packages/atif-cli/src/atif_cli/app.py:482-486`, `packages/atif-duck/src/atif_duck/infrastructure/registry.py:221` | -| `materialize` prints all zeroes and reads as an idle, complete corpus | Every session was unreadable; an unreadable session lands in no other counter | Read the `unreadable` field, not just `materialized` | `packages/atif-cli/src/atif_cli/app.py:312-346`, `packages/atif-corpus/src/atif_corpus/application/materialize.py:136-142` | -| `materialize` aborts with a traceback naming `SuspiciousEmptyScanError`, exit 1 | The scan found zero sessions over a non-empty corpus, almost always a wrong `source_root` | Compare the `source_root` line in `atif-sql status` against where the transcripts actually are | `packages/atif-corpus/src/atif_corpus/application/materialize.py:97-104`, raise at `packages/atif-corpus/src/atif_corpus/application/materialize.py:553-559` | -| A changed session never materializes | Quiescence: the newest source mtime is younger than `quiesce_seconds` (default 300) | `atif-sql status` reports it under `live`; `--force` overrides staleness but never quiescence | `packages/atif-corpus/src/atif_corpus/domain/sessions.py:81-104`, `packages/atif-corpus/src/atif_corpus/domain/sessions.py:172-173`, `packages/atif-corpus/src/atif_corpus/domain/sessions.py:201-203` | -| A session stays `live` forever and warns about a FUTURE mtime | The writing host's clock is ahead, so the age is negative and never meets the threshold | Fix the clock on the host writing the transcript | `packages/atif-corpus/src/atif_corpus/domain/sessions.py:96-103` | -| `materialize` or `status` aborts inside `read_watermark` with `ValueError` or `TypeError` | A watermark whose values are not numbers; the `int(value)` coercion sits outside the try | Delete `watermark.json`; a missing watermark costs one full re-materialization pass and is always safe | `packages/atif-corpus/src/atif_corpus/application/materialize.py:160-176` | -| Every session fails conversion at once, on the same `ConversionError` | The converter is a port of harbor 0.23.0's, so a systematic failure is a bug in the port or a transcript shape it never saw | Run the parity oracle: `ATIF_PARITY_LIMIT=0 uv run pytest packages/atif-converter/tests/test_parity_live.py`, and read the diff paths | `packages/atif-converter/tests/harbor_oracle.py:142` | -| One session fails with `N source file(s) changed while converting …` | The session is still being written; the snapshot re-check refused to publish inconsistent artifacts | Retry once the session goes quiet; this is self-clearing | `packages/atif-converter/src/atif_converter/application/convert_and_audit.py:93-102`, `packages/atif-converter/src/atif_converter/domain/errors.py:57-70` | -| `convert` prints a trajectory and then exits 65 | The enriched trajectory failed harbor's `TrajectoryValidator` | Read `validation_errors` in the JSON it already printed | `packages/atif-cli/src/atif_cli/app.py:283-304`, `packages/atif-cli/src/atif_cli/converter_adapter.py:62-64` | -| `loss_report.json` shows `records_dropped > 0` | Fidelity gap 2: harbor 0.22.0 converts only user and assistant records | Check `gaps_observed` against the named gaps; these are documented losses, not defects | `packages/atif-converter/src/atif_converter/domain/fidelity.py:44-79` | -| `analyze` summary carries `budget_exhausted: true` and a stage `{"skipped": "budget_exhausted"}` | The dollar ceiling was crossed against running actual usage | Read `llm_spent_usd` in the same summary; raise `--max-cost-usd` or wait for the next run | `packages/atif-analytics/src/atif_analytics/application/analyze.py:164-179`, `packages/atif-analytics/src/atif_analytics/application/analyze.py:202-204` | -| An LLM stage skips the same sessions every run, forever | The retry queue is exhausted at 5 attempts; `attempts >= 5` with `completed_at IS NULL` is permanent and nothing resets it | `sqlite3 /analytics/state.db 'SELECT * FROM retry_queue'` (note the pipeline is `user_friction`, never `friction`) | `packages/atif-analytics/src/atif_analytics/infrastructure/sqlite_state/retry_queue.py:176-193`, `packages/atif-analytics/src/atif_analytics/infrastructure/sqlite_state/checkpointer.py:35-41` | +| `query` exits 64, kind `parse_error` | `duckdb.ParserException`, or no SQL argument at all | Run `atif-sql schema --format json` for the real view and column names; a bare `atif-sql query` also exits 64 | `packages/atif-cli/src/atif_cli/duck_errors.py:38-44`, `packages/atif-cli/src/atif_cli/app.py:1429-1440` | +| `query` exits 65, kind `catalog_error` | `duckdb.CatalogException`: the view, macro, or column does not exist | `atif-sql schema` answers from the static catalog with no DuckDB bind, so it works even on an unmaterialized corpus | `packages/atif-cli/src/atif_cli/duck_errors.py:45-51`, `packages/atif-cli/src/atif_cli/app.py:2335` | +| `query` exits 70 with `IO Error: No files found that match the pattern .../sessions/*/meta.json` | The corpus is not materialized; `register_raw`'s glob matches nothing | `atif-sql status`; a `watermark: never materialized` line means run `atif-sql materialize` | `packages/atif-cli/src/atif_cli/duck_errors.py:52-58`, `packages/atif-duck/src/atif_duck/infrastructure/registry.py:698` | +| `query` or `search` exits 65, kind `embedding_mismatch` | The Lance store's stamped `(model, dim)` differs from the active embedder | The message names both sides; either point `ATIF_SQL_EMBED_MODEL_ID` at the model that wrote the store, or delete the store | `packages/atif-duck/src/atif_duck/domain/embedding_guard.py:78-85`, `packages/atif-cli/src/atif_cli/duck_errors.py:75-82` | +| `search` exits 2, kind `no_embeddings` | The store is empty or absent, so `register_vss` bound an empty fallback table | `atif-sql query 'SELECT count(*) FROM message_embeddings'`; a count of 0 confirms it | `packages/atif-cli/src/atif_cli/app.py:2145-2156`, `packages/atif-duck/src/atif_duck/infrastructure/registry.py:1687-1705` | +| A bare `atif-sql embed` exits 64 | The scope guard, which runs before any Bedrock client is built | Pass `--limit N`, `--all`, or `--dry-run`; this is the accidental-full-backfill guard, not a fault | `packages/atif-cli/src/atif_cli/app.py:1908-1918` | +| `embed` exits 78 and the cron lane stops trying | A `DomainError` with `terminal = True`; the store or its config needs an operator | Read the terminal marker file the refresh lane drops: three lines, store path, mtime, reason | `packages/atif-embed/src/atif_embed/domain/errors.py:14-26`, `scripts/atif-sql-refresh.sh:398-403` | +| A session directory exists under `sessions/` but no view returns its rows | Torn artifact set: the dir has no `meta.json`, so the meta gate excludes it from every reader | Grep the WARNING; `meta.json` is written last and completes the set | `packages/atif-duck/src/atif_duck/infrastructure/registry.py:528-532`, `packages/atif-corpus/src/atif_corpus/application/materialize.py:706-707` | +| `atif-sql status` and `SELECT count(*) FROM sessions` disagree | `status` counts directories with no meta check; on the per-session path the view applies the meta gate and the session id boundary, and skips a dir with no readable trajectory | Run the count with `--no-lake`: the difference is the number of excluded dirs, and each one logs a WARNING | `packages/atif-cli/src/atif_cli/app.py:1224-1228`, `packages/atif-duck/src/atif_duck/infrastructure/registry.py:709` | +| `materialize` prints all zeroes and reads as an idle, complete corpus | Every session was unreadable; an unreadable session lands in no other counter | Read the `unreadable` field, not just `materialized` | `packages/atif-cli/src/atif_cli/app.py:765`, `packages/atif-corpus/src/atif_corpus/application/materialize.py:253-261` | +| `materialize` exits 78, kind `suspicious_scan` | The scan found zero sessions over a non-empty corpus while every source directory listed, almost always a wrong `source_root` | Compare the `source_root` line in `atif-sql status` against where the transcripts actually are; nothing was marked | `packages/atif-cli/src/atif_cli/app.py:1012-1026`, raise at `packages/atif-corpus/src/atif_corpus/application/materialize.py:1661-1667` | +| A changed session never materializes | Quiescence: the newest source mtime is younger than `quiesce_seconds` (default 300) | `atif-sql status` reports it under `live`; `--force` overrides staleness but never quiescence | `packages/atif-corpus/src/atif_corpus/domain/sessions.py:84`, `packages/atif-corpus/src/atif_corpus/domain/sessions.py:218-219`, `packages/atif-corpus/src/atif_corpus/domain/sessions.py:220-221` | +| A session stays `live` forever and warns about a FUTURE mtime | The writing host's clock is ahead, so the age is negative and never meets the threshold | Fix the clock on the host writing the transcript | `packages/atif-corpus/src/atif_corpus/domain/sessions.py:107-113` | +| `materialize` re-converts every quiescent session after `watermark.json` is damaged | An unreadable, wrong-shaped, or non-numeric watermark degrades to empty with a WARNING, so every session reads as stale | Grep `treating corpus as unmaterialized`; the pass writes a fresh watermark, and the cost is one full re-materialization | `packages/atif-corpus/src/atif_corpus/application/materialize.py:424-451` | +| Every session fails conversion at once, on the same `ConversionError` | The converter is a port of harbor 0.24.0's, so a systematic failure is a bug in the port or a transcript shape it never saw | Run the parity oracle: `ATIF_PARITY_LIMIT=0 uv run pytest packages/atif-converter/tests/test_parity_live.py`, and read the diff paths | `packages/atif-converter/tests/harbor_oracle.py:358` | +| One session fails with `N source file(s) changed while converting …` | The session is still being written; the snapshot re-check refused to publish inconsistent artifacts | Retry once the session goes quiet; this is self-clearing | `packages/atif-converter/src/atif_converter/application/convert_and_audit.py:199-218`, `packages/atif-converter/src/atif_converter/domain/errors.py:57` | +| `convert` prints a trajectory and then exits 65 | The enriched trajectory failed the `TrajectoryValidator` vendored from harbor | Read `validation_errors` in the JSON it already printed | `packages/atif-cli/src/atif_cli/app.py:745-757`, `packages/atif-converter/src/atif_converter/infrastructure/harbor_adapter.py:76` | +| `loss_report.json` shows `records_dropped > 0` | Fidelity gap 2: only user and assistant records become steps, and a record `session_events.jsonl` doesn't keep is dropped | Check `gaps_observed` against the named gaps; these are documented losses, not defects | `packages/atif-converter/src/atif_converter/domain/fidelity.py:71-77` | +| `analyze` summary carries `budget_exhausted: true` and a stage `{"skipped": "budget_exhausted", "consecutive_skips": N}` | The dollar ceiling was crossed against running actual usage | Read `llm_spent_usd` in the same summary; raise `--max-cost-usd` or wait for the next run | `packages/atif-analytics/src/atif_analytics/application/analyze.py:112-128`, `packages/atif-analytics/src/atif_analytics/application/analyze.py:150-152` | +| An LLM stage skips the same sessions every run, forever | The retry queue is exhausted at 5 attempts; `attempts >= 5` with `completed_at IS NULL` is permanent and nothing resets it | `sqlite3 /analytics/state.db 'SELECT * FROM retry_queue'` (note the pipeline is `user_friction`, never `friction`) | `packages/atif-analytics/src/atif_analytics/infrastructure/sqlite_state/retry_queue.py:199-204`, `packages/atif-analytics/src/atif_analytics/infrastructure/sqlite_state/checkpointer.py:37-42` | | An LLM run stalls for minutes with no output | Bedrock throttling under tenacity, in either adapter: 10 attempts, exponential 2 s to 60 s | The per-backoff WARNING names the attempt number and the sleep | `packages/atif-models/src/atif_models/infrastructure/openai_bedrock.py:183-189`, `packages/atif-models/src/atif_models/infrastructure/anthropic_bedrock.py:221-227`, `packages/atif-models/src/atif_models/infrastructure/bedrock_common.py:76-85` | | A classify row reads `[refused]`, or a refusals sidecar row appears | `RefusalError`: a content filter or a refusal `finish_reason` (openai), or `stop_reason: refusal` (anthropic, with its `stop_details.category` in the reason) | Nothing to fix here: a refusal is terminal, checkpointed, and never retried | `packages/atif-models/src/atif_models/domain/ports.py:43-48`, `packages/atif-models/src/atif_models/infrastructure/openai_bedrock.py:219-225`, `packages/atif-models/src/atif_models/infrastructure/anthropic_bedrock.py:252-257` | | An LLM unit lands in the retry queue with a truncation error | `finish_reason=length` (openai) or `stop_reason=max_tokens` (anthropic) twice, or once already at the effort floor | Raise `max_completion_tokens` or shrink the prompt; the automatic ladder has only one reachable rung | `packages/atif-models/src/atif_models/infrastructure/openai_bedrock.py:226-232`, `packages/atif-models/src/atif_models/infrastructure/openai_bedrock.py:160-181`, `packages/atif-models/src/atif_models/infrastructure/anthropic_bedrock.py:258-264`, `packages/atif-models/src/atif_models/infrastructure/anthropic_bedrock.py:195-219` | | An anthropic unit lands in the retry queue naming `unexpected stop_reason` or `no text content` | The response stopped somewhere other than `end_turn`, or carried only a `thinking` block | Read the reason string; it names the stop reason or the block types the response held | `packages/atif-models/src/atif_models/infrastructure/anthropic_bedrock.py:265-276` | | `llm_spent_usd` runs well past the dry-run plan's estimate | The plan prices at the standard rates only; a call over the spec's long-context threshold (272,000 input tokens on GPT-6, 100,000 on Haiku 5.5) bills the whole request at the long rates, and running actuals price that | Divide the plan's `estimated_input_tokens` by its `llm_calls`: an average near the threshold means some calls bill at the long rates. The long-context split isn't in the `analyze` summary | `packages/atif-analytics/src/atif_analytics/domain/costs.py:35-49`, `packages/atif-models/src/atif_models/domain/registry.py:175-210` | -| A classify, conflicts, or perceived verdict ignores what a tool returned | The transcript renders each tool result as a size stub, `(N chars omitted)`, so the model never saw the content | Set `ATIF_SQL_SESSION_TEXT_TOOL_RESULT_BODIES=true` to send bodies clipped at 50,000 characters, and expect the input tokens to grow with them | `packages/atif-analytics/src/atif_analytics/domain/transcript.py:306-312`, `packages/atif-analytics/src/atif_analytics/infrastructure/settings.py:91-94` | -| `analyze` crashes with a raw traceback and exit 1 | The `analyze` command body has no `except`, so nothing maps analytics errors into `EXIT_CODES` | Read the traceback's innermost frame; the exit code carries no information here | `packages/atif-cli/src/atif_cli/app.py:739-757` | -| `atif-sql examples` raises a bare `KeyError` naming `ARG_EXEMPLARS` or `DESCRIPTIONS` | A catalog drift tripwire firing at example-build time, not query time | Add the missing exemplar or description entry named in the message | `packages/atif-duck/src/atif_duck/domain/examples.py:150-158`, `packages/atif-duck/src/atif_duck/domain/examples.py:165-175` | -| A cron lane logs `analytics not yet installed, skipping` and exits 0 | The resolved CLI predates the `analyze` subcommand; `~/.local/bin/atif-sql` wins over the workspace venv | Check which binary the lane resolved, in the CLI-resolution order the script documents | `scripts/atif-sql-refresh.sh:180-185`, `scripts/atif-sql-refresh.sh:150-158` | -| The refresh log shows `[structural] lane removed 2026-09-27 ...` once an hour | A crontab line still calls the `structural` lane, which the script accepts and exits 0 on | Delete the hourly `structural` crontab line; `atif-sql cron install` prints a reminder comment for it | `scripts/atif-sql-refresh.sh:141-144`, `packages/atif-cli/src/atif_cli/cron.py:56` | +| A classify, conflicts, or perceived verdict ignores what a tool returned | The transcript renders each tool result as a size stub, `(N chars omitted)`, so the model never saw the content | Set `ATIF_SQL_SESSION_TEXT_TOOL_RESULT_BODIES=true` to send bodies clipped at 50,000 characters, and expect the input tokens to grow with them | `packages/atif-analytics/src/atif_analytics/domain/transcript.py:306-312`, `packages/atif-analytics/src/atif_analytics/infrastructure/settings.py:90-94` | +| `analyze` crashes with a raw traceback and exit 1 | The `analyze` command body has no `except`, so nothing maps analytics errors into `EXIT_CODES` | Read the traceback's innermost frame; the exit code carries no information here | `packages/atif-cli/src/atif_cli/app.py:1717-1731` | +| `atif-sql examples` raises a bare `KeyError` naming `ARG_EXEMPLARS` or `DESCRIPTIONS` | A catalog drift tripwire firing at example-build time, not query time | Add the missing exemplar or description entry named in the message | `packages/atif-duck/src/atif_duck/domain/examples.py:164-171`, `packages/atif-duck/src/atif_duck/domain/examples.py:180-188` | +| A cron lane logs `analytics not yet installed, skipping` and exits 0 | The resolved CLI predates the `analyze` subcommand; `~/.local/bin/atif-sql` wins over the workspace venv | Check which binary the lane resolved, in the CLI-resolution order the script documents | `scripts/atif-sql-refresh.sh:324-328`, `scripts/atif-sql-refresh.sh:266-273` | +| The refresh log shows `[structural] lane removed 2026-09-27 ...` once an hour | A crontab line still calls the `structural` lane, which the script accepts and exits 0 on | Delete the hourly `structural` crontab line; `atif-sql cron install` prints a reminder comment for it | `scripts/atif-sql-refresh.sh:164-167`, `packages/atif-cli/src/atif_cli/cron.py:201-202` | ## Log and error surfaces @@ -91,49 +92,50 @@ shell-side refresh log. | Surface | Where it emits | What to grep for | Citation | | --- | --- | --- | --- | -| The one loguru sink | stderr, WARNING and above; `logger.remove()` deletes loguru's default handler first | `WARNING` / `ERROR`; nothing at INFO or DEBUG is emitted at all | `packages/atif-cli/src/atif_cli/app.py:1125-1135` | -| Classified error envelope, pipe form | stderr, one JSON object | `"error"`, `"kind"`, `"hint"` | `packages/atif-cli/src/atif_cli/errors.py:60-68` | -| Classified error, TTY form | stderr, one line plus an optional hint line | `[]` at line start, then `hint:` | `packages/atif-cli/src/atif_cli/output.py:274-281` | -| Process exit code | the shell | the reachable `EXIT_CODES` values | `packages/atif-cli/src/atif_cli/errors.py:25-39` | -| Per-session materialize failures | stderr, table format only | `FAILED :` and `UNREADABLE ` | `packages/atif-cli/src/atif_cli/app.py:343-346` | -| Materialize report | stdout, JSON on a pipe | `unreadable`, `failures`, `removed_session_ids` | `packages/atif-cli/src/atif_cli/app.py:320-332` | -| Corpus freshness | `atif-sql status` stdout | `watermark_age_seconds`, `staleness`, `source_root` | `packages/atif-cli/src/atif_cli/app.py:508-520` | -| DDL failure narration | stderr via `logger.exception`, with loguru's decorated traceback, then re-raised | `Failed to register raw readers over`, `Failed to register derived views`, `Failed to register macros` | `packages/atif-duck/src/atif_duck/infrastructure/registry.py:308`, `packages/atif-duck/src/atif_duck/infrastructure/registry.py:802`, `packages/atif-duck/src/atif_duck/infrastructure/registry.py:1203` | -| Torn-session exclusion | stderr WARNING, one line per excluded dir | `Skipping incomplete session dir` | `packages/atif-duck/src/atif_duck/infrastructure/registry.py:173-177` | -| Empty-store fallback | stderr WARNING at VSS bind | `No Lance embeddings table at` | `packages/atif-duck/src/atif_duck/infrastructure/registry.py:897-902` | -| Unreadable-source warnings | stderr WARNING during the scan | `NOT treated as deleted`, `NOT treating it as deleted` | `packages/atif-corpus/src/atif_corpus/infrastructure/scanner.py:133-138`, `packages/atif-corpus/src/atif_corpus/infrastructure/scanner.py:179-184` | -| Source-removal marking suppression | stderr WARNING | `not marking any session source-removed`, `keeping session` | `packages/atif-corpus/src/atif_corpus/application/materialize.py:566-570`, `packages/atif-corpus/src/atif_corpus/application/materialize.py:416-420` | -| Watermark degradation | stderr WARNING | `treating corpus as unmaterialized` | `packages/atif-corpus/src/atif_corpus/application/materialize.py:171`, `packages/atif-corpus/src/atif_corpus/application/materialize.py:174` | +| The one loguru sink | stderr, WARNING and above unless `ATIF_SQL_LOG_LEVEL` sets another level; `logger.remove()` deletes loguru's default handler first | `WARNING` / `ERROR`; INFO and DEBUG lines appear only under `ATIF_SQL_LOG_LEVEL=INFO` or `DEBUG` | `packages/atif-cli/src/atif_cli/app.py:2427-2435` | +| Classified error envelope, pipe form | stderr, one JSON object | `"error"`, `"kind"`, `"hint"` | `packages/atif-cli/src/atif_cli/errors.py:78-86` | +| Classified error, TTY form | stderr, one line plus an optional hint line | `[]` at line start, then `hint:` | `packages/atif-cli/src/atif_cli/output.py:279-282` | +| Process exit code | the shell | the reachable `EXIT_CODES` values | `packages/atif-cli/src/atif_cli/errors.py:30-66` | +| Per-session materialize failures | stderr, table format only | `FAILED :`, `UNREADABLE `, `REJECTED ''`, and `LAKE PENDING` | `packages/atif-cli/src/atif_cli/app.py:828-839` | +| Materialize report | stdout, JSON on a pipe | `unreadable`, `failures`, `removed_session_ids` | `packages/atif-cli/src/atif_cli/app.py:783-809` | +| Corpus freshness | `atif-sql status` stdout | `watermark_age_seconds`, `staleness`, `source_root` | `packages/atif-cli/src/atif_cli/app.py:1271-1296` | +| DDL failure narration | stderr via `logger.exception`, with loguru's decorated traceback, then re-raised | `Failed to register raw readers over`, `Failed to register derived views`, `Failed to register macros` | `packages/atif-duck/src/atif_duck/infrastructure/registry.py:957`, `packages/atif-duck/src/atif_duck/infrastructure/registry.py:1529`, `packages/atif-duck/src/atif_duck/infrastructure/registry.py:2074` | +| Torn-session exclusion | stderr WARNING, one line per excluded dir | `Skipping incomplete session dir` | `packages/atif-duck/src/atif_duck/infrastructure/registry.py:528-532` | +| Empty-store fallback | stderr WARNING at VSS bind | `No Lance embeddings table at` | `packages/atif-duck/src/atif_duck/infrastructure/registry.py:1688-1693` | +| Unreadable-source warnings | stderr WARNING during the scan | `NOT treated as deleted`, `NOT treating it as deleted` | `packages/atif-corpus/src/atif_corpus/infrastructure/scanner.py:169`, `packages/atif-corpus/src/atif_corpus/infrastructure/scanner.py:269` | +| Source-removal marking suppression | stderr WARNING | `not marking any session source-removed`, `keeping session` | `packages/atif-corpus/src/atif_corpus/application/materialize.py:1675`, `packages/atif-corpus/src/atif_corpus/application/materialize.py:1204` | +| Watermark degradation | stderr WARNING | `treating corpus as unmaterialized` | `packages/atif-corpus/src/atif_corpus/application/materialize.py:435`, `packages/atif-corpus/src/atif_corpus/application/materialize.py:438`, `packages/atif-corpus/src/atif_corpus/application/materialize.py:449` | | Bedrock retry narration | stderr WARNING, one per backoff | `bedrock invoke retry` (LLM, both adapters), `Retrying` … `seconds as it raised` (embeddings) | `packages/atif-models/src/atif_models/infrastructure/bedrock_common.py:76-85`, `packages/atif-embed/src/atif_embed/infrastructure/cohere_bedrock.py:94-101` | | Degraded-effort retry | stderr WARNING | `finish_reason=length at effort=` (openai), `stop_reason=max_tokens at effort=` (anthropic) | `packages/atif-models/src/atif_models/infrastructure/openai_bedrock.py:169-175`, `packages/atif-models/src/atif_models/infrastructure/anthropic_bedrock.py:205-210` | -| Cost-ceiling events | stderr, WARNING or ERROR at 3 consecutive skips | `cost ceiling hit` | `packages/atif-analytics/src/atif_analytics/application/analyze.py:110-121`, `packages/atif-analytics/src/atif_analytics/application/use_cases/classify.py:230` | -| Terminally failed embed batch | stderr ERROR | `failed terminally`, `the next run re-picks these rows` | `packages/atif-embed/src/atif_embed/infrastructure/cohere_bedrock.py:377-384` | -| Clipped embedding input | stderr WARNING | `Clipping text at position`, `content past the cap will not match a search` | `packages/atif-embed/src/atif_embed/infrastructure/cohere_bedrock.py:332-338` | -| Refresh-script log | a file appended under the script's `.run/` sibling, one `date -Is`-stamped line per event; the directory is gitignored, so the path is named in prose only | `refresh complete (mode=`, `skip[`, `TERMINAL:`, `FATAL:` | `scripts/atif-sql-refresh.sh:100-103`, `scripts/atif-sql-refresh.sh:337` | -| Refresh log, machine-read subset | `atif-sql cron status` stdout | only the completion and skip line shapes are parsed | `packages/atif-cli/src/atif_cli/cron.py:55-56` | -| Lane lock state | `atif-sql cron status`, from a nonblocking flock probe plus the pidfile | `RUNNING (pid …)` versus `idle` | `packages/atif-cli/src/atif_cli/cron.py:109-124`, `packages/atif-cli/src/atif_cli/cron.py:127-144` | -| Embed terminal marker | one file per corpus beside the refresh log; three lines: store path, store mtime, reason | the reason string, parsed out of the exit-78 envelope | `scripts/atif-sql-refresh.sh:238-249` | -| Analytics durable state | `/analytics/state.db`, sqlite in WAL mode, no read command | `retry_queue`, `budget_skips`, `session_checkpoint` | `packages/atif-analytics/src/atif_analytics/domain/layout.py:81-83`, `packages/atif-analytics/src/atif_analytics/infrastructure/sqlite_state/checkpointer.py:43-62` | -| Analyze summary | stdout JSON | `budget_exhausted`, `llm_spent_usd`, `consecutive_skips` | `packages/atif-analytics/src/atif_analytics/application/analyze.py:121`, `packages/atif-analytics/src/atif_analytics/application/analyze.py:144-146` | +| Cost-ceiling events | stderr, WARNING or ERROR at 3 consecutive skips | `cost ceiling hit` | `packages/atif-analytics/src/atif_analytics/application/analyze.py:116-126`, `packages/atif-analytics/src/atif_analytics/application/use_cases/classify.py:230` | +| Terminally failed embed batch | stderr ERROR | `failed terminally`, `the next run re-picks these rows` | `packages/atif-embed/src/atif_embed/infrastructure/cohere_bedrock.py:378-385` | +| Clipped embedding input | stderr WARNING | `Clipping text at position`, `content past the cap will not match a search` | `packages/atif-embed/src/atif_embed/infrastructure/cohere_bedrock.py:333-339` | +| Refresh-script log | a file appended under the script's `.run/` sibling (or `ATIF_SQL_REFRESH_RUN_DIR`), one `date -Is`-stamped line per event; the directory is gitignored, so the path is named in prose only | `refresh complete (mode=`, `skip[`, `TERMINAL:`, `FATAL:` | `scripts/atif-sql-refresh.sh:151-154`, `scripts/atif-sql-refresh.sh:540` | +| Refresh log, machine-read subset | `atif-sql cron status` stdout | only the completion and skip line shapes are parsed | `packages/atif-cli/src/atif_cli/cron.py:61-62` | +| Lane lock state | `atif-sql cron status`, from a nonblocking flock probe plus the pidfile | `RUNNING (pid …)` versus `idle` | `packages/atif-cli/src/atif_cli/cron.py:115-130`, `packages/atif-cli/src/atif_cli/cron.py:133-150` | +| Embed terminal marker | one file per corpus beside the refresh log; three lines: store path, store mtime, reason | the reason string, parsed out of the exit-78 envelope | `scripts/atif-sql-refresh.sh:398-403` | +| Analytics durable state | `/analytics/state.db`, sqlite in WAL mode, no read command | `retry_queue`, `budget_skips`, `session_checkpoint` | `packages/atif-analytics/src/atif_analytics/domain/layout.py:81-83`, `packages/atif-analytics/src/atif_analytics/infrastructure/sqlite_state/checkpointer.py:44-62` | +| Analyze summary | stdout JSON | `budget_exhausted`, `llm_spent_usd`, `consecutive_skips` | `packages/atif-analytics/src/atif_analytics/application/analyze.py:127`, `packages/atif-analytics/src/atif_analytics/application/analyze.py:150-152` | | Per-session loss accounting | `loss_report.json` inside each corpus session dir | `gaps_observed`, `records_dropped`, `records_total` | `packages/atif-converter/src/atif_converter/domain/fidelity.py:114-137` | These properties of this surface change how you read it. -**Everything below WARNING is unreachable through the CLI.** `main()` hardcodes -the level and there is no `--verbose`, no `--log-level`, and no `LOGURU_LEVEL` -support: `logger.remove()` deletes the default handler that would honor the -env var (`packages/atif-cli/src/atif_cli/app.py:1133-1134`). The INFO lines -that would narrate a materialize pass -(`packages/atif-corpus/src/atif_corpus/application/materialize.py:634-644`) or +**Everything below WARNING is hidden by default.** `main()` takes the level +from `ATIF_SQL_LOG_LEVEL` (default `WARNING`; an unknown level falls back to it +with a warning). There is no `--verbose` and no `--log-level`, and `LOGURU_LEVEL` +has no effect: `logger.remove()` deletes the default handler that would honor +it (`packages/atif-cli/src/atif_cli/app.py:2427-2435`). The INFO lines that +narrate a materialize pass +(`packages/atif-corpus/src/atif_corpus/application/materialize.py:1811-1825`) or an embed backfill -(`packages/atif-embed/src/atif_embed/application/embed.py:255-261`) never -appear. Reaching them means importing `atif_cli.app` and calling `app()` -directly after adding your own sink, instead of going through `main()`. +(`packages/atif-embed/src/atif_embed/application/embed.py:270-276`) appear only +under `ATIF_SQL_LOG_LEVEL=INFO`, and each materialize pool worker reads the same +variable (`packages/atif-cli/src/atif_cli/app.py:844`). **A `logger.exception` traceback on stderr is not a crash.** The registration path logs and re-raises, so the decorated traceback is followed by the JSON envelope and a classified exit code -(`packages/atif-duck/src/atif_duck/infrastructure/registry.py:306-309`). Read +(`packages/atif-duck/src/atif_duck/infrastructure/registry.py:955-958`). Read the last stderr line, not the first. ## First-checks ladder @@ -142,61 +144,63 @@ Cheapest first. Steps 1 through 6 are free and read-only; step 10 spends money. 1. **Read the exit code.** It partitions the problem before you read any text: 64 means your input, 65 means the object cannot be bound, 70 means the - adapter, 78 means an operator is required, and 1 means the taxonomy did not - cover it. `packages/atif-cli/src/atif_cli/errors.py:25-39` + adapter or the query sandbox, 77 means the process runs as root, 78 means + an operator is required, and 1 means the taxonomy did not cover it. + `packages/atif-cli/src/atif_cli/errors.py:30-66` 2. **Read the last line on stderr.** On a pipe it is the JSON envelope with `kind`, `message`, and `hint`; on a TTY it is `[kind] message` plus a `hint:` line. The hint names the recovery command in most cases. - `packages/atif-cli/src/atif_cli/output.py:267-281` + `packages/atif-cli/src/atif_cli/output.py:271-285` 3. **Run `atif-sql status`.** It replays the exact planning decision `materialize` would make, without converting anything, so its `stale / up-to-date / live` split answers "would a pass do anything right now?", and it prints the resolved `source_root` and `corpus_root`, which is how a wrong-root problem becomes visible. - `packages/atif-cli/src/atif_cli/app.py:475-481`, - `packages/atif-cli/src/atif_cli/app.py:495-496` + `packages/atif-cli/src/atif_cli/app.py:1209-1216`, + `packages/atif-cli/src/atif_cli/app.py:1244-1245` 4. **Run `atif-sql schema` or `atif-sql examples`.** Both answer from the static catalog with no DuckDB connection, so they work on an unmaterialized corpus and separate "my SQL is wrong" from "the corpus is not there". Every example is executed by the test suite against a fixture corpus, so a verbatim copy is known to parse. - `packages/atif-cli/src/atif_cli/app.py:1022-1026`, - `packages/atif-cli/src/atif_cli/app.py:1099`, - `packages/atif-cli/src/atif_cli/app.py:987-991` + `packages/atif-cli/src/atif_cli/app.py:2341-2345`, + `packages/atif-cli/src/atif_cli/app.py:2254-2255`, + `packages/atif-cli/src/atif_cli/app.py:2251-2253` 5. **Cross-check `status`'s session count against the view.** `atif-sql query - 'SELECT count(*) FROM sessions'` applies the meta gate; `status` counts - directories. A gap is the torn-dir count, and each torn dir also emits a + --no-lake 'SELECT count(*) FROM sessions'` applies the meta gate; `status` + counts directories. A gap is the number of dirs the gates excluded (a torn + dir, a rejected name, no readable trajectory), and each one also emits a WARNING naming itself. - `packages/atif-duck/src/atif_duck/infrastructure/registry.py:173-177` + `packages/atif-duck/src/atif_duck/infrastructure/registry.py:528-532` 6. **Run `atif-sql cron status`.** For anything that failed unattended: per lane it reports whether the flock is held and by which pid, the last completion with its exit code, the last skip, and a log tail. The lock probe acquires and releases nonblocking, so it perturbs no running lane. - `packages/atif-cli/src/atif_cli/cron.py:205-208`, - `packages/atif-cli/src/atif_cli/cron.py:226-245` + `packages/atif-cli/src/atif_cli/cron.py:207`, + `packages/atif-cli/src/atif_cli/cron.py:228-253` 7. **Look for a terminal marker file** beside the refresh log. Its presence is why an embed stopped being attempted; it holds the store path, the mtime it was keyed on, and the reason. Any operator touch that changes the store directory's mtime clears it on the next tick. - `scripts/atif-sql-refresh.sh:217-236` + `scripts/atif-sql-refresh.sh:380-390` 8. **Query `state.db` directly** at `/analytics/state.db`. There is no CLI or SQL surface over it. `retry_queue` rows with `attempts >= 5` and `completed_at IS NULL` are permanently blocked and need manual clearing; `budget_skips` rows are the consecutive-skip streak. The pipeline name is `user_friction`. `packages/atif-analytics/src/atif_analytics/infrastructure/sqlite_state/retry_queue.py:42-56` -9. **Re-run the failing library call in-process with your own loguru sink** if - the WARNING lines were not enough. This is the only way to see the DEBUG - narration of the staging sweep, the mid-scan races, the atomic-write tmp - cleanups, and every view and macro registration. - `packages/atif-cli/src/atif_cli/app.py:1133-1134` +9. **Raise the log level** if the WARNING lines were not enough. + `ATIF_SQL_LOG_LEVEL=DEBUG` shows the DEBUG narration of the staging sweep, + the mid-scan races, the atomic-write tmp cleanups, and every view and macro + registration. + `packages/atif-cli/src/atif_cli/app.py:2396` 10. **Only now re-run a billable command**, and bound it: `analyze` defaults to a dry run and needs `--no-dry-run` to spend, and `embed` refuses to run - without `--limit N` or `--all`. `atif-sql search` always calls Bedrock, - once, to embed the query. - `packages/atif-cli/src/atif_cli/app.py:743`, - `packages/atif-cli/src/atif_cli/app.py:810`, - `packages/atif-cli/src/atif_cli/app.py:943` + without `--limit N` or `--all`. `atif-sql search` embeds the query once per + run, on Bedrock under the default Cohere provider. + `packages/atif-cli/src/atif_cli/app.py:1640`, + `packages/atif-cli/src/atif_cli/app.py:1908`, + `packages/atif-cli/src/atif_cli/app.py:2159` ## Known incident patterns @@ -207,7 +211,7 @@ Cheapest first. Steps 1 through 6 are free and read-only; step 10 spends money. flag on the error, exit 78 as its own code, and a marker keyed on the store path plus its mtime that suppresses retries until an operator touch changes it. `packages/atif-embed/src/atif_embed/domain/errors.py:14-26`, - `scripts/atif-sql-refresh.sh:187-194` + `scripts/atif-sql-refresh.sh:331-338` - **Destroy-and-rebuild on a missing column:** raising a rebuild-demanding error for a store that merely predates the `text_hash` stamp rebuilt both production corpora at the cost of a full Cohere re-embed. Signal: a full @@ -223,13 +227,14 @@ Cheapest first. Steps 1 through 6 are free and read-only; step 10 spends money. (verified live 2026-08-23). Signal: two config roots, one corpus directory, and sessions from both interleaved. Mitigation: every tick exports both `CLAUDE_CONFIG_DIR` and `ATIF_SQL_SOURCE_ROOT` per corpus. - `scripts/atif-sql-refresh.sh:52-59`, `scripts/atif-sql-refresh.sh:312-313` + `scripts/atif-sql-refresh.sh:91-98`, `scripts/atif-sql-refresh.sh:497-498` - **Divergence from harbor's conversion:** the converters are parity ports of harbor - 0.23.0's private ones, and harbor keeps changing those files upstream. Signal: + 0.24.0's private ones, and harbor keeps changing those files upstream. Signal: `test_harbor_oracle.py` reports a golden diff after a harbor bump, or the live parity test names diverging JSON paths. Mitigation: each diff is a decision to - follow upstream or not, recorded in the fidelity policy; re-freeze the goldens - only after it (`packages/atif-converter/tests/harbor_oracle.py:135`). `packages/atif-converter/tests/harbor_oracle.py:59` + follow upstream or not, and a decision not to follow is named in the oracle's + `DELIBERATE_DIVERGENCES`; re-freeze the goldens with `ATIF_FREEZE_GOLDENS=1` + only after it (`packages/atif-converter/tests/test_harbor_oracle.py:31`). `packages/atif-converter/tests/harbor_oracle.py:53` - **`SourceMutatedDuringConversion`:** a session that resumes writing mid-conversion would yield a census, a trajectory, and an `edges.jsonl` each describing different bytes. Signal: one session failing with a @@ -237,17 +242,17 @@ Cheapest first. Steps 1 through 6 are free and read-only; step 10 spends money. files are read once, fingerprinted in the same pass that parses them, and the fingerprints are re-checked (stat and digest) once the artifacts are built, refusing rather than publishing; retrying once the session goes quiet succeeds. - `packages/atif-converter/src/atif_converter/application/convert_and_audit.py:11-22`, - `packages/atif-converter/src/atif_converter/application/convert_and_audit.py:93-102` + `packages/atif-converter/src/atif_converter/application/convert_and_audit.py:11-24`, + `packages/atif-converter/src/atif_converter/application/convert_and_audit.py:199-218` - **`SuspiciousEmptyScanError`:** a wrong `source_root` (a typo, an unmounted disk, a stale env var) makes the scan empty, and the pass would then - mark the entire corpus source-removed. Signal: a traceback naming the error, with the - scanned root and the materialized session count in the message, and nothing - removed. Mitigation: the guard runs before any session is marked and refuses the + mark the entire corpus source-removed. Signal: exit 78 with kind `suspicious_scan`, + the scanned root and the materialized session count in the message, and nothing + marked. Mitigation: the guard runs before any session is marked and refuses the pass; an operator who genuinely emptied the source tree deletes the corpus directory explicitly. - `packages/atif-corpus/src/atif_corpus/application/materialize.py:97-104`, - `packages/atif-corpus/src/atif_corpus/application/materialize.py:553-559` + `packages/atif-corpus/src/atif_corpus/application/materialize.py:179-189`, + `packages/atif-corpus/src/atif_corpus/application/materialize.py:1661-1667` - **Absence mistaken for deletion:** `Path.glob` swallows `PermissionError` and yields nothing, which would make every session under an unlistable project directory look deleted. Signal: a WARNING naming the directory count, and a pass @@ -255,8 +260,8 @@ Cheapest first. Steps 1 through 6 are free and read-only; step 10 spends money. as gone; an unlistable directory disables source-removal marking for the whole pass, and the affected session ids are resolved from the watermark so their entries are retained. - `packages/atif-corpus/src/atif_corpus/infrastructure/scanner.py:83-95`, - `packages/atif-corpus/src/atif_corpus/infrastructure/scanner.py:119-140`, `packages/atif-corpus/src/atif_corpus/application/materialize.py:560-571` + `packages/atif-corpus/src/atif_corpus/infrastructure/scanner.py:113-125`, + `packages/atif-corpus/src/atif_corpus/infrastructure/scanner.py:153-173`, `packages/atif-corpus/src/atif_corpus/application/materialize.py:1668-1677` - **Lane-lock inheritance through fd 9:** the refresh lane's flock is held by an open descriptor, not by a pid, so a descendant that inherits it keeps the lane locked for as long as it lives: a lane that stops doing work while @@ -264,7 +269,7 @@ Cheapest first. Steps 1 through 6 are free and read-only; step 10 spends money. reporting a held lock with no corresponding work in the log. Mitigation: every `atif-sql` invocation in the script closes fd 9 with `9>&-`, while the parent keeps its own. - `scripts/atif-sql-refresh.sh:160-176` + `scripts/atif-sql-refresh.sh:288-292` - **`AWS_PROFILE` reaching a Bedrock call:** an inherited profile name with no matching `~/.aws/config` entry makes botocore raise `FileNotFoundError` at credential resolution, killing the lane before it reaches Bedrock at all. @@ -272,27 +277,29 @@ Cheapest first. Steps 1 through 6 are free and read-only; step 10 spends money. rather than a Bedrock one. Mitigation: the script unsets `AWS_PROFILE` and `AWS_DEFAULT_PROFILE`, and authenticates from a run-time-read bearer token or the default credential chain. - `scripts/atif-sql-refresh.sh:91-96`, `scripts/atif-sql-refresh.sh:116-128` + `scripts/atif-sql-refresh.sh:142-147`, `scripts/atif-sql-refresh.sh:232-244` - **The `FidelityGap` members:** these are documented, verified upstream - losses, not defects: workflow-nested subagents missed by harbor's own - discovery, non-message records dropped, the parent chain flattened by - timestamp sort, subagents inlined rather than embedded, compaction summaries - unhandled, the cache split only partially preserved, and the event uuid not - preserved. Signal: `records_dropped > 0` or a non-empty `gaps_observed` in a + losses, not defects: non-message records dropped, the parent chain flattened + by timestamp sort, subagents inlined rather than embedded, compaction + summaries unhandled, the cache split only partially preserved, and the event + uuid not preserved. Gap 1 (workflow-nested subagents missed by harbor's own + discovery) is retired, because the converter discovers every side file itself. Signal: `records_dropped > 0` or a non-empty `gaps_observed` in a `loss_report.json`. Mitigation: none is intended; the converter's job is honest loss accounting, and unit tests pin each gap so a harbor bump that changes behavior trips the suite. - `packages/atif-converter/src/atif_converter/domain/fidelity.py:44-79` -- **Budget overshoot and the streak that never escalates:** the cost ceiling - stops dispatch at an 8-unit batch boundary rather than capping spend exactly, - and the consecutive-skip streak is cleared whenever a stage ran at all, - including a run that aborted mid-chunk on the ceiling. Signal: repeated - `cost ceiling hit` WARNINGs that never become the ERROR line. Mitigation: - read `llm_spent_usd` in the summary rather than trusting the streak, and - treat a mid-stage abort as leaving nothing durable behind: no checkpoint, no - cache row, no retry entry for the unstarted units. + `packages/atif-converter/src/atif_converter/domain/fidelity.py:55-99` +- **Budget overshoot and the starved stage:** the cost ceiling stops dispatch + at an 8-unit batch boundary rather than capping spend exactly, and a stage + the ceiling keeps cutting off starves run after run. Signal: + `cost ceiling hit` WARNINGs naming the same stage on consecutive runs, which + become the ERROR line at 3 consecutive budget skips. Mitigation: read `llm_spent_usd` in + the summary rather than treating the ceiling as a hard cap. The streak is + cleared only when a stage finishes inside its budget, so a stage that aborts + mid-chunk on the ceiling holds its streak, and a mid-stage abort leaves + nothing durable behind: no checkpoint, no cache row, no retry entry for the + unstarted units. `packages/atif-analytics/src/atif_analytics/application/use_cases/_shared.py:146-157`, - `packages/atif-analytics/src/atif_analytics/application/analyze.py:103-142` + `packages/atif-analytics/src/atif_analytics/application/analyze.py:112-146` - **Deterministic truncation and the one-rung ladder:** resending a truncated request (`finish_reason=length` on openai, `stop_reason=max_tokens` on anthropic) with identical parameters truncates identically and doubles the bill, so the @@ -314,7 +321,7 @@ Cheapest first. Steps 1 through 6 are free and read-only; step 10 spends money. exit 65. Mitigation: `REGISTRATION_ERRORS` widens the caught tuple to cover both, and a test reads both twin modules as source text to require the shared recovery hint appear after the `raise` keyword in each. - `packages/atif-cli/src/atif_cli/duck_errors.py:14-31`, + `packages/atif-cli/src/atif_cli/duck_errors.py:15-32`, `packages/atif-duck/src/atif_duck/domain/embedding_guard.py:5-16` ## See also diff --git a/docs/insights/impact-analysis.md b/docs/insights/impact-analysis.md index 0228d77..1f48b10 100644 --- a/docs/insights/impact-analysis.md +++ b/docs/insights/impact-analysis.md @@ -21,18 +21,22 @@ statements for the symbol and then **reading every hit's import line** to confir name resolved to. These properties of this codebase make that confirmation step load-bearing rather than pedantic: -- **Every cross-package import is indented**: inside a function body or an `if TYPE_CHECKING:` block, - because the lean-import contract defers them (`packages/atif-cli/src/atif_cli/app.py:22-26`). A - line-anchored `^from atif_` grep returns no cross-package consumers. The census behind this file - allowed leading whitespace and then read each enclosing scope. +- **Nearly every cross-package import is indented**: inside a function body or an `if TYPE_CHECKING:` + block, because the lean-import contract defers them (`packages/atif-cli/src/atif_cli/app.py:50-53`). + The exceptions are four atif-cli helper modules (`atif_cli.converter_adapter`, `atif_cli.duck_errors`, + `atif_cli.embed_lake`, and `atif_cli.lake_sessions`) that import siblings at module scope, and + `atif_cli.app` imports each of them only inside a function body. A line-anchored `^from atif_` grep + finds only those four. The census behind this file allowed leading whitespace and then read each + enclosing scope. - **Names collide across packages.** `DomainError` is declared independently in several packages and `EmbeddingProviderMismatch` in more than one: `packages/atif-duck/src/atif_duck/domain/embedding_guard.py:33` derives from bare `Exception`, `packages/atif-embed/src/atif_embed/domain/errors.py:29` from atif-embed's own `DomainError`. Near-names compound it: `packages/atif-embed/src/atif_embed/infrastructure/cohere_bedrock.py:44` imports `EmbeddingProviderUnavailable`, which is not the `EmbeddingProvider` Protocol. -- **A Protocol implementation imports nothing.** Structural satisfaction leaves no edge in either - direction, so the adapters are invisible to any import-graph query. See the Protocols section. +- **A Protocol implementation usually imports nothing from its Protocol.** Structural satisfaction + leaves no edge in either direction, so most adapters are invisible to any import-graph query. See + the Protocols section. The internal graph is a star: `atif-cli` to its declared siblings, plus exactly one `atif-analytics → atif-models` edge. atif-cli doesn't declare atif-models @@ -50,66 +54,66 @@ The catalogs (`VIEW_NAMES`, `MACRO_NAMES`, `ANALYTICS_VIEW_NAMES`, | Downstream | Type | Touch on change | Citation | | --- | --- | --- | --- | -| `atif_duck.domain.examples`: derives every example from every catalog | direct import | yes | `packages/atif-duck/src/atif_duck/domain/examples.py:35-43` | -| `DESCRIPTIONS`: one entry per object; `_description()` raises `KeyError` without it | direct import | yes | `packages/atif-duck/src/atif_duck/domain/catalog.py:431` and `packages/atif-duck/src/atif_duck/domain/examples.py:166-175` | -| `ARG_EXEMPLARS`: needed only when a macro introduces a NEW parameter name; `_macro_sql()` raises `KeyError` without it | direct import | yes | `packages/atif-duck/src/atif_duck/domain/examples.py:75-92` and `packages/atif-duck/src/atif_duck/domain/examples.py:148-158` | -| `TABLE_MACRO_NAMES`: membership required when the DDL says `AS TABLE`, or the derived SQL uses the wrong call shape | direct import | yes | `packages/atif-duck/src/atif_duck/domain/catalog.py:413-422` | -| `atif_duck.infrastructure.registry`: owns the real DDL and imports `DEFAULT_PRICING` | direct import | yes | `packages/atif-duck/src/atif_duck/infrastructure/registry.py:53` | -| `atif_duck.infrastructure.analytics`: `register_analytics` and `register_analytics_macros` own the DDL the analytics catalogs describe | indirect | yes | `packages/atif-duck/src/atif_duck/infrastructure/analytics.py:119` and `packages/atif-duck/src/atif_duck/infrastructure/analytics.py:226` | +| `atif_duck.domain.examples`: derives every example from every catalog | direct import | yes | `packages/atif-duck/src/atif_duck/domain/examples.py:35-44` | +| `DESCRIPTIONS`: one entry per object; `_description()` raises `KeyError` without it | direct import | yes | `packages/atif-duck/src/atif_duck/domain/catalog.py:487` and `packages/atif-duck/src/atif_duck/domain/examples.py:182-188` | +| `ARG_EXEMPLARS`: needed only when a macro introduces a NEW parameter name; `_macro_sql()` raises `KeyError` without it | direct import | yes | `packages/atif-duck/src/atif_duck/domain/examples.py:76-93` and `packages/atif-duck/src/atif_duck/domain/examples.py:163-171` | +| `TABLE_MACRO_NAMES`: membership required when the DDL says `AS TABLE`, or the derived SQL uses the wrong call shape | direct import | yes | `packages/atif-duck/src/atif_duck/domain/catalog.py:469` | +| `atif_duck.infrastructure.registry`: owns the real DDL and imports `DEFAULT_PRICING` | direct import | yes | `packages/atif-duck/src/atif_duck/infrastructure/registry.py:87` | +| `atif_duck.infrastructure.analytics`: `register_analytics` and `register_analytics_macros` own the DDL the analytics catalogs describe | indirect | yes | `packages/atif-duck/src/atif_duck/infrastructure/analytics.py:119` and `packages/atif-duck/src/atif_duck/infrastructure/analytics.py:236` | | `atif_duck.infrastructure.authorship`: `register_authorship` owns the DDL for `step_author`, `user_steps`, `human_turns`, and `session_outcomes` | indirect | yes | `packages/atif-duck/src/atif_duck/infrastructure/authorship.py:38` | -| `atif_cli.app.schema`: deferred import of `VIEW_SCHEMA`, `MACRO_SIGNATURES`, `ANALYTICS_VIEW_SCHEMA`, and `ANALYTICS_MACRO_SIGNATURES` | direct import | no | `packages/atif-cli/src/atif_cli/app.py:1745-1750` | -| `atif_cli.app.examples`: deferred import of `build_examples` + the two value tuples | direct import | no | `packages/atif-cli/src/atif_cli/app.py:1022-1026` | -| `test_examples.py::test_every_example_executes`: parametrized over every derived example, `fetchall()` forces materialization | test | yes | `packages/atif-duck/tests/test_examples.py:73-79` | -| `test_examples.py` drift catchers: description coverage, exemplar coverage, `AS TABLE` set equality, example-or-exclusion coverage | test | yes | `packages/atif-duck/tests/test_examples.py:160-211` | -| `test_duck_views.py::test_view_schema_matches_describe`: `DESCRIBE` output equals `VIEW_SCHEMA` column-for-column | test | yes | `packages/atif-duck/tests/test_duck_views.py:50-59` | -| `test_duck_views.py::test_macro_signatures_match_ddl`: regex-parses the DDL and asserts equality | test | yes | `packages/atif-duck/tests/test_duck_views.py:66-87` | -| `test_analytics_views.py::test_analytics_macro_signatures_match_ddl`: same for the analytics half | test | yes | `packages/atif-duck/tests/test_analytics_views.py:313-328` | -| `test_duck_views.py::test_default_pricing_matches_published_list_rates_exactly`: oracle table, key-for-key | test | yes | `packages/atif-duck/tests/test_duck_views.py:459-467` | -| `test_app.py::TestSchema` / `TestExamples`: assert the CLI payload against the catalog, not literals | test | no | `packages/atif-cli/tests/test_app.py:134-165` and `packages/atif-cli/tests/test_app.py:94-111` | +| `atif_cli.app.schema`: deferred import of `VIEW_SCHEMA`, `MACRO_SIGNATURES`, `ANALYTICS_VIEW_SCHEMA`, and `ANALYTICS_MACRO_SIGNATURES` | direct import | no | `packages/atif-cli/src/atif_cli/app.py:2352-2357` | +| `atif_cli.app.examples`: deferred import of `build_examples` + the two value tuples | direct import | no | `packages/atif-cli/src/atif_cli/app.py:2270-2274` | +| `test_examples.py::test_every_example_executes`: parametrized over every derived example, `fetchall()` forces materialization | test | yes | `packages/atif-duck/tests/test_examples.py:102-109` | +| `test_examples.py` drift catchers: description coverage, exemplar coverage, `AS TABLE` set equality, example-or-exclusion coverage | test | yes | `packages/atif-duck/tests/test_examples.py:190-241` | +| `test_duck_views.py::test_view_schema_matches_describe`: `DESCRIBE` output equals `VIEW_SCHEMA` column-for-column | test | yes | `packages/atif-duck/tests/test_duck_views.py:51-59` | +| `test_duck_views.py::test_macro_signatures_match_ddl`: regex-parses the DDL and asserts equality | test | yes | `packages/atif-duck/tests/test_duck_views.py:67-89` | +| `test_analytics_views.py::test_analytics_macro_signatures_match_ddl`: same for the analytics half | test | yes | `packages/atif-duck/tests/test_analytics_views.py:242-257` | +| `test_duck_views.py::test_default_pricing_matches_published_list_rates_exactly`: oracle table, key-for-key | test | yes | `packages/atif-duck/tests/test_duck_views.py:519-527` | +| `test_app.py::TestSchema` / `TestExamples`: the JSON payload tests assert against the catalog and `build_examples()`, not literals | test | no | `packages/atif-cli/tests/test_app.py:135-149` and `packages/atif-cli/tests/test_app.py:183-202` | ### Blast-radius notes - **`EXCLUSIONS` is the only sanctioned way to add a catalog object without an example, and it is policed in both directions.** The dict is currently empty - (`packages/atif-duck/src/atif_duck/domain/examples.py:99`), and a stale key (or a key that still + (`packages/atif-duck/src/atif_duck/domain/examples.py:100`), and a stale key (or a key that still emits an example) fails `test_exclusions_reference_real_catalog_objects` - (`packages/atif-duck/tests/test_examples.py:170-175`). + (`packages/atif-duck/tests/test_examples.py:200`). - **`atif-sql schema` prints every catalog, each object tagged with its `requires` value.** It - reads the core and analytics schema and signature dicts (`packages/atif-cli/src/atif_cli/app.py:1745-1750`) + reads the core and analytics schema and signature dicts (`packages/atif-cli/src/atif_cli/app.py:2352-2357`) and labels every name through `object_requires`, so a new view or macro lands in both `atif-sql schema` and `atif-sql examples` output. `packages/atif-cli/tests/test_app.py:145` asserts set equality against the union of `VIEW_SCHEMA` and `ANALYTICS_VIEW_SCHEMA`, and - `packages/atif-cli/tests/test_app.py:151-165` pins the `requires` values. + `packages/atif-cli/tests/test_app.py:151-169` pins the `requires` values. - **Column ORDER in `VIEW_SCHEMA` is load-bearing.** The drift test asserts tuple equality against `DESCRIBE`, so reordering a `SELECT` list in the DDL without reordering the catalog entry fails CI - (`packages/atif-duck/src/atif_duck/domain/catalog.py:47-50`). + (`packages/atif-duck/src/atif_duck/domain/catalog.py:55-58`). ## The vendored ATIF models, and the parity oracle behind the ported converters -Defined at: `packages/atif-converter/src/atif_converter/domain/claude_code_conversion.py:75` (Claude Code) and `packages/atif-converter/src/atif_converter/domain/codex_conversion.py:781` (Codex) +Defined at: `packages/atif-converter/src/atif_converter/domain/claude_code_conversion.py:981` (Claude Code) and `packages/atif-converter/src/atif_converter/domain/codex_conversion.py:924` (Codex) -Production code builds on harbor 0.23.0's ATIF data classes (`harbor.models.trajectories`) and +Production code builds on harbor 0.24.0's ATIF data classes (`harbor.models.trajectories`) and `harbor.utils.trajectory_validator`, vendored in `packages/atif-converter/src/atif_converter/domain/atif/`; harbor itself is a dev -dependency. `packages/atif-converter/tests/test_vendored_atif.py` compares the copy to the installed harbor. The conversion is a parity port of harbor 0.23.0's +dependency. `packages/atif-converter/tests/test_vendored_atif.py` compares the copy to the installed harbor. The conversion is a parity port of harbor 0.24.0's private converters, and those private methods are reachable from the tests only, as the oracle -(`packages/atif-converter/tests/harbor_oracle.py:94`, `packages/atif-converter/tests/harbor_oracle.py:111`), frozen to goldens under `packages/atif-converter/tests/goldens/`. +(`packages/atif-converter/tests/harbor_oracle.py:118`, `packages/atif-converter/tests/harbor_oracle.py:135`), frozen to goldens under `packages/atif-converter/tests/goldens/`. | Downstream | Type | Touch on change | Citation | | --- | --- | --- | --- | | `test_dev_only_imports_guard.py`: `ast` guard: no `harbor` or `litellm` import in any `src/` | test | yes | `packages/atif-converter/tests/test_dev_only_imports_guard.py` | | `test_vendored_atif.py`: vendored files equal upstream's; both models and validators agree on the goldens | test | yes | `packages/atif-converter/tests/test_vendored_atif.py` | -| `harbor_adapter.convert_session()` / `codex_adapter.convert_codex_session()`: the seams (converter in, validated dict out) | direct import | yes | `packages/atif-converter/src/atif_converter/infrastructure/harbor_adapter.py:83`, `packages/atif-converter/src/atif_converter/infrastructure/codex_adapter.py:63` | -| `FidelityGap` / `CodexFidelityGap`: the losses the port inherited on purpose; fixing one is a decision to diverge from the oracle | indirect | yes | `packages/atif-converter/src/atif_converter/domain/fidelity.py:52`, `packages/atif-converter/src/atif_converter/domain/codex_fidelity.py:66` | -| `test_harbor_oracle.py`: live oracle equals frozen goldens; a harbor bump that changes conversion fails here first | test | yes | `packages/atif-converter/tests/test_harbor_oracle.py:1` | -| `test_parity_live.py`: newest N local sessions of each agent, both converters, every diverging path | test | yes | `packages/atif-converter/tests/test_parity_live.py:1` | -| `atif_converter.domain.enrichment` / `codex_enrichment`: re-derive step identity by replicating the converter's normalization order | indirect | yes | `packages/atif-converter/src/atif_converter/domain/enrichment.py:1`, `packages/atif-converter/src/atif_converter/domain/codex_enrichment.py:1` | -| `EXIT_CODES["harbor_missing"] = 127`: retired, kept so the table never renumbers | config | no | `packages/atif-cli/src/atif_cli/errors.py:47` | -| `atif_cli.app.materialize`: stamps `_version_of("harbor")` into every `meta.json` | direct import | likely | `packages/atif-cli/src/atif_cli/app.py:426` | +| `harbor_adapter.convert_session()` / `codex_adapter.convert_codex_session()`: the seams (converter in, validated dict out) | direct import | yes | `packages/atif-converter/src/atif_converter/infrastructure/harbor_adapter.py:157`, `packages/atif-converter/src/atif_converter/infrastructure/codex_adapter.py:67` | +| `FidelityGap` / `CodexFidelityGap`: the losses the port inherited on purpose; fixing one is a decision to diverge from the oracle | indirect | yes | `packages/atif-converter/src/atif_converter/domain/fidelity.py:55`, `packages/atif-converter/src/atif_converter/domain/codex_fidelity.py:66` | +| `test_harbor_oracle.py`: live oracle equals frozen goldens; a harbor bump that changes conversion fails here first | test | yes | `packages/atif-converter/tests/test_harbor_oracle.py:67` | +| `test_parity_live.py`: newest N local sessions of each agent, both converters, every diverging path | test | yes | `packages/atif-converter/tests/test_parity_live.py:102` | +| `atif_converter.domain.enrichment` / `codex_enrichment`: re-derive step identity by replicating the converter's normalization order | indirect | yes | `packages/atif-converter/src/atif_converter/domain/enrichment.py:15-19`, `packages/atif-converter/src/atif_converter/domain/codex_enrichment.py:154` | +| `EXIT_CODES["harbor_missing"] = 127`: retired, kept so the table never renumbers | config | no | `packages/atif-cli/src/atif_cli/errors.py:62` | +| `atif_cli.app.materialize`: stamps the vendored models' `UPSTREAM_VERSION` into every `meta.json` as `harbor_version` | direct import | likely | `packages/atif-cli/src/atif_cli/app.py:926` and `packages/atif-cli/src/atif_cli/app.py:966` | ## The materialized corpus artifact layout -Defined at: `packages/atif-corpus/src/atif_corpus/domain/layout.py:18-22`, specified by -`docs/CONTRACT.md:21-31` +Defined at: `packages/atif-corpus/src/atif_corpus/domain/layout.py:18-32`, specified by +`docs/CONTRACT.md:22-60` atif-corpus writes it; atif-duck, atif-embed, and atif-analytics each read it **independently**, because the independence contract forbids them a shared reader. So the layout strings exist in several @@ -117,40 +121,40 @@ places with no import edge between them. | Downstream | Type | Touch on change | Citation | | --- | --- | --- | --- | -| `atif_corpus.application.materialize`: the writer, routed through `CorpusLayout` | direct import | yes | `packages/atif-corpus/src/atif_corpus/application/materialize.py:75-76` and `packages/atif-corpus/src/atif_corpus/application/materialize.py:530` | -| `atif_duck.infrastructure.registry`: builds globs under `corpus_root / "sessions"` | indirect | yes | `packages/atif-duck/src/atif_duck/infrastructure/registry.py:209-213` | -| `atif_embed.infrastructure.corpus_text_rows`: its own `sessions/` walk, gated on `meta.json` | indirect | yes | `packages/atif-embed/src/atif_embed/infrastructure/corpus_text_rows.py:112-123` | +| `atif_corpus.application.materialize`: the writer, routed through `CorpusLayout` | direct import | yes | `packages/atif-corpus/src/atif_corpus/application/materialize.py:135-144` and `packages/atif-corpus/src/atif_corpus/application/materialize.py:1617` | +| `atif_duck.infrastructure.registry`: builds globs under `corpus_root / "sessions"` | indirect | yes | `packages/atif-duck/src/atif_duck/infrastructure/registry.py:691-701` | +| `atif_embed.infrastructure.corpus_text_rows`: its own `sessions/` walk, gated on `meta.json` | indirect | yes | `packages/atif-embed/src/atif_embed/infrastructure/corpus_text_rows.py:155-173` | | `atif_analytics.infrastructure.corpus_reader`: re-declares the filename constants | indirect | yes | `packages/atif-analytics/src/atif_analytics/infrastructure/corpus_reader.py:72-74` and `packages/atif-analytics/src/atif_analytics/infrastructure/corpus_reader.py:178-191` | -| `atif_cli.app.convert`: writes `edges.jsonl` beside a `--out` trajectory on the one-shot path | indirect | likely | `packages/atif-cli/src/atif_cli/app.py:276` | -| `atif_cli.app.status`: reads the watermark through the producer's public `read_watermark` | direct import | likely | `packages/atif-cli/src/atif_cli/app.py:460-470` | -| `docs/CONTRACT.md:21-31`: the specification; the file declares itself orchestrator-owned and changeable only via itself | config | yes | `docs/CONTRACT.md:3` | -| `test_domain.py::TestCorpusLayout.test_contract_paths`: pins every contract string | test | yes | `packages/atif-corpus/tests/test_domain.py:245-249` | -| Independent fixture-corpus builders, one per consuming package | test | yes | `packages/atif-duck/tests/duck_fixtures.py:482`, `packages/atif-embed/tests/embed_fixtures.py:120`, `packages/atif-analytics/tests/analytics_fixtures.py:218`, `packages/atif-corpus/tests/corpus_fixtures.py:77` | +| `atif_cli.app.convert`: writes `edges.jsonl` beside a `--trajectory-out` trajectory on the one-shot path | indirect | likely | `packages/atif-cli/src/atif_cli/app.py:740-741` | +| `atif_cli.app.status`: reads the watermark through the producer's public `read_watermark` | direct import | likely | `packages/atif-cli/src/atif_cli/app.py:1197` | +| `docs/CONTRACT.md:22-60`: the specification; the file declares itself orchestrator-owned and changeable only via itself | config | yes | `docs/CONTRACT.md:3` | +| `test_domain.py::TestCorpusLayout.test_contract_paths`: pins every contract string | test | yes | `packages/atif-corpus/tests/test_domain.py:245-258` | +| Independent fixture-corpus builders, one per consuming package | test | yes | `packages/atif-duck/tests/duck_fixtures.py:593`, `packages/atif-embed/tests/embed_fixtures.py:120`, `packages/atif-analytics/tests/analytics_fixtures.py:282` | ### Blast-radius notes - **Nothing mechanical catches a layout change that updates one reader and misses the others.** No import edge connects them, so the only signal is each package's own fixture corpus going out of - agreement with the writer. The `meta.json`-written-last torn-set rule is duplicated verbatim in every + agreement with the writer. The `meta.json`-written-last torn-set rule is re-implemented in every reader: evidence of the duplication cost, and one place per reader to edit if the completion marker - ever moves (`packages/atif-duck/src/atif_duck/infrastructure/registry.py:188`, - `packages/atif-embed/src/atif_embed/infrastructure/corpus_text_rows.py:28`, + ever moves (`packages/atif-duck/src/atif_duck/infrastructure/registry.py:639-644`, + `packages/atif-embed/src/atif_embed/infrastructure/corpus_text_rows.py:28-29`, `packages/atif-analytics/src/atif_analytics/infrastructure/corpus_reader.py:13-15`). - **`.staging` sits outside `sessions/` for a reader-visibility reason, not tidiness.** DuckDB's `read_json` glob matches dot-dirs, so moving staging under `sessions/` exposes half-written session dirs to every reader and to the source-removal walk - (`packages/atif-corpus/src/atif_corpus/domain/layout.py:47-55`). + (`packages/atif-corpus/src/atif_corpus/domain/layout.py:85-94`). - **`LossReport.to_json()`'s keys are wire contract, not an internal shape.** atif-duck reads them back through `_LOSS_REPORT_COLUMNS`, so renaming a key breaks the `loss_reports` view with no type - error anywhere (`packages/atif-converter/src/atif_converter/domain/fidelity.py:114-137`, - `packages/atif-duck/src/atif_duck/infrastructure/registry.py:122`). + error anywhere (`packages/atif-converter/src/atif_converter/domain/fidelity.py:147-171`, + `packages/atif-duck/src/atif_duck/infrastructure/registry.py:287`). ## The domain Protocols -Defined at: `packages/atif-corpus/src/atif_corpus/domain/ports.py:41` (`ConverterPort`), -`packages/atif-embed/src/atif_embed/domain/ports.py:31`, -`packages/atif-embed/src/atif_embed/domain/ports.py:59`, -`packages/atif-embed/src/atif_embed/domain/ports.py:87` (`EmbeddingProvider`, +Defined at: `packages/atif-corpus/src/atif_corpus/domain/ports.py:101` (`ConverterPort`), +`packages/atif-embed/src/atif_embed/domain/ports.py:37`, +`packages/atif-embed/src/atif_embed/domain/ports.py:65`, +`packages/atif-embed/src/atif_embed/domain/ports.py:93` (`EmbeddingProvider`, `VectorStorePort`, `TextRowsPort`), `packages/atif-models/src/atif_models/domain/ports.py:131` (`LlmStructuredProvider`) @@ -160,8 +164,13 @@ dependency: `ConverterPort` (atif-corpus may not import atif-converter) and `Tex may not import atif-duck). The others are ordinary seams: atif-analytics is permitted to import atif-models, and the atif-embed adapters live in the same package as their ports. -**Every adapter satisfies its Protocol without importing it.** Each row below marked `indirect` names a -class whose only textual reference to the Protocol is a docstring, so no import-graph query reaches it. +**Nearly every adapter satisfies its Protocol without importing it.** The exceptions take a +Protocol-typed argument themselves: `LakeTextRows` imports `TextRowsPort` to type the fallback reader +it wraps (`packages/atif-embed/src/atif_embed/infrastructure/lake_text_rows.py:52`), and the +analytics corpus reader imports `SessionSource` to type `CorpusReader`'s source +(`packages/atif-analytics/src/atif_analytics/infrastructure/corpus_reader.py:70`). Each row below +marked `indirect` names a class whose only textual reference to the Protocol is a docstring, so no +import-graph query reaches it. | Downstream | Type | Touch on change | Citation | | --- | --- | --- | --- | @@ -170,24 +179,25 @@ class whose only textual reference to the Protocol is a docstring, so no import- | `OpenAiBedrockProvider` / `AnthropicBedrockProvider`: the two real adapters, one per family; each imports `CallUsage` / `ProviderUnavailable` / `RefusalError` / `SchemaT` / `UsageAccumulator` from the port module and NOT the Protocol | indirect | yes | `packages/atif-models/src/atif_models/infrastructure/openai_bedrock.py:41-47` and `packages/atif-models/src/atif_models/infrastructure/openai_bedrock.py:81`, `packages/atif-models/src/atif_models/infrastructure/anthropic_bedrock.py:61-67` and `packages/atif-models/src/atif_models/infrastructure/anthropic_bedrock.py:130` | | `test_models_ports.py`: binds `OpenAiBedrockProvider` to the port by annotation and `isinstance`; the Anthropic adapter has no such test | test | yes | `packages/atif-models/tests/test_models_ports.py:29-38` | | `FakeProvider`: the deterministic double, named in prose only | test | yes | `packages/atif-analytics/tests/analytics_fixtures.py:300` | -| `ConverterPort` ← `materialize` use case, in several signature positions | direct import | yes | `packages/atif-corpus/src/atif_corpus/application/materialize.py:92`, `packages/atif-corpus/src/atif_corpus/application/materialize.py:193`, `packages/atif-corpus/src/atif_corpus/application/materialize.py:480` | -| `RealConverter`: the production adapter; imports `ConversionOutput` only, names `ConverterPort` in its docstring | indirect | yes | `packages/atif-cli/src/atif_cli/converter_adapter.py:38` and `packages/atif-cli/src/atif_cli/converter_adapter.py:44-45` | -| `FakeConverter`: shipped in `src/` rather than `tests/`; same shape, same non-import | indirect | yes | `packages/atif-corpus/src/atif_corpus/infrastructure/fake_converter.py:16` and `packages/atif-corpus/src/atif_corpus/infrastructure/fake_converter.py:37` | -| `ConversionOutput`: the frozen dataclass that DOES cross the boundary as an import | direct import | yes | `packages/atif-corpus/src/atif_corpus/domain/ports.py:21-38` | +| `ConverterPort` ← `materialize` use case, in several signature positions | direct import | yes | `packages/atif-corpus/src/atif_corpus/application/materialize.py:615`, `packages/atif-corpus/src/atif_corpus/application/materialize.py:877`, `packages/atif-corpus/src/atif_corpus/application/materialize.py:1501` | +| `RealConverter`: the production adapter; imports `ConversionOutput` and three other types from the port module, never `ConverterPort`, which it names only in docstrings | indirect | yes | `packages/atif-cli/src/atif_cli/converter_adapter.py:69-74` and `packages/atif-cli/src/atif_cli/converter_adapter.py:80-81` | +| `FakeConverter`: shipped in `src/` rather than `tests/`; same shape, same non-import | indirect | yes | `packages/atif-corpus/src/atif_corpus/infrastructure/fake_converter.py:21` and `packages/atif-corpus/src/atif_corpus/infrastructure/fake_converter.py:72` | +| `ConversionOutput`: the frozen dataclass that DOES cross the boundary as an import | direct import | yes | `packages/atif-corpus/src/atif_corpus/domain/ports.py:65-66` | | `EmbeddingProvider` / `VectorStorePort` / `TextRowsPort` ← the one embed use case, under `if TYPE_CHECKING:` | direct import | yes | `packages/atif-embed/src/atif_embed/application/embed.py:35`, `packages/atif-embed/src/atif_embed/application/embed.py:46-47`, `packages/atif-embed/src/atif_embed/application/embed.py:65-67` | -| The production adapters, one per embed port, none importing its Protocol | indirect | yes | `packages/atif-embed/src/atif_embed/infrastructure/cohere_bedrock.py:285`, `packages/atif-embed/src/atif_embed/infrastructure/lance_store.py:377`, `packages/atif-embed/src/atif_embed/infrastructure/corpus_text_rows.py:156` | -| `test_converter_adapter.py`: the one place a Protocol name is imported and bound to an implementation | test | yes | `packages/atif-cli/tests/test_converter_adapter.py:17` and `packages/atif-cli/tests/test_converter_adapter.py:91` | +| The production adapters (two embedding providers, the Lance store, and the corpus text-rows reader), none importing its Protocol | indirect | yes | `packages/atif-embed/src/atif_embed/infrastructure/cohere_bedrock.py:285`, `packages/atif-embed/src/atif_embed/infrastructure/gemma_local.py:193`, `packages/atif-embed/src/atif_embed/infrastructure/lance_store.py:376`, `packages/atif-embed/src/atif_embed/infrastructure/corpus_text_rows.py:199` | +| `test_converter_adapter.py`: imports `ConverterPort` and binds `RealConverter` to it by annotation; `test_embed_ports.py` and `test_models_ports.py` do the same for the embed and model ports | test | yes | `packages/atif-cli/tests/test_converter_adapter.py:17` and `packages/atif-cli/tests/test_converter_adapter.py:92`, `packages/atif-embed/tests/test_embed_ports.py:27` | | `[tool.coverage.report] exclude_also`: a bare `...` line is excluded because a Protocol body is a signature | config | no | `pyproject.toml:649-652` | ### Blast-radius notes -- **Adding a method to a Protocol breaks implementations that name it nowhere.** Because no adapter - imports its Protocol, the only mechanical signal is a `pyright`/`ty` error where the implementation is - passed into a port-typed parameter; for `RealConverter` that call site is in a different package - from both the Protocol and the class (`packages/atif-cli/src/atif_cli/converter_adapter.py:44`, - `packages/atif-corpus/src/atif_corpus/application/materialize.py:193`). `FakeConverter` lives in +- **Adding a method to a Protocol breaks implementations that name it nowhere.** Because the adapters + don't import their Protocols, the mechanical signal is a `pyright`/`ty` error where an implementation + is bound to a port type: a port-typed parameter, or an annotated name in a binding test. For + `RealConverter` that call site is the `materialize` command in `atif_cli.app`, a different module + from the class and a different package from the Protocol (`packages/atif-cli/src/atif_cli/app.py:961`, + `packages/atif-corpus/src/atif_corpus/application/materialize.py:1501`). `FakeConverter` lives in `src/`, so a missed update fails the type gate rather than only a test - (`packages/atif-corpus/src/atif_corpus/infrastructure/fake_converter.py:37`). For + (`packages/atif-corpus/src/atif_corpus/infrastructure/fake_converter.py:5-8`). For `LlmStructuredProvider` the call sites are the factory's two return statements, one per adapter (`packages/atif-models/src/atif_models/infrastructure/providers.py:31` and `packages/atif-models/src/atif_models/infrastructure/providers.py:34`). @@ -195,11 +205,11 @@ class whose only textual reference to the Protocol is a docstring, so no import- implementations may raise anything and the materialize use case records the failure against the session and continues, because one broken transcript must never abort a corpus sync. Tightening this into a typed-error contract changes materialize's control flow, not only a signature - (`packages/atif-corpus/src/atif_corpus/domain/ports.py:41-47`). + (`packages/atif-corpus/src/atif_corpus/domain/ports.py:104-107`). - **`EmbeddingProvider.embed_documents` returns one slot per input, `None` for a failure, rather than raising.** That is what bounds loss: a terminally failing batch must not discard sibling batches whose embeddings were already billed - (`packages/atif-embed/src/atif_embed/domain/ports.py:44-52`). + (`packages/atif-embed/src/atif_embed/domain/ports.py:50-57`). ## The import-linter contracts @@ -211,22 +221,22 @@ surfaces above exist as duplicated strings and Protocols instead of imports. | Downstream | Type | Touch on change | Citation | | --- | --- | --- | --- | -| `mise run lint:imports`: `uv run lint-imports`, an entry in `[tasks.check].depends` | config | yes | `mise.toml:162` and `mise.toml:204` | +| `mise run lint:imports`: `uv run lint-imports`, an entry in `[tasks.check].depends` | config | yes | `mise.toml:175` and `mise.toml:250` | | `root_packages`: every import package must be listed, or a new member goes unchecked | config | yes | `pyproject.toml:551` | | `atif_cli`: the composition root, absent from `independence.modules` and listed in `forbidden_modules`; it declares its siblings and NOT atif-models, so the comment granting it that permission describes an unused allowance | indirect | likely | `pyproject.toml:605`, `pyproject.toml:611`, `packages/atif-cli/pyproject.toml:32-36` | | `ConverterPort`: exists because atif-corpus may not import atif-converter | indirect | yes | `packages/atif-corpus/src/atif_corpus/domain/ports.py:5-9` | | `TextRowsPort`: exists because atif-embed may not import atif-duck | indirect | yes | `packages/atif-embed/src/atif_embed/domain/ports.py:11-15` | -| The `embedding_guard` twins: one rule, copied into atif-duck and atif-embed | direct import | yes | `packages/atif-duck/src/atif_duck/domain/embedding_guard.py:33` and `packages/atif-embed/src/atif_embed/domain/embedding_guard.py:38` | +| The `embedding_guard` twins: one rule, copied into atif-duck and atif-embed | direct import | yes | `packages/atif-duck/src/atif_duck/domain/embedding_guard.py:51` and `packages/atif-embed/src/atif_embed/domain/embedding_guard.py:38` | | `test_guard_twin_pin.py`: reads the atif-duck copy as SOURCE TEXT via `ast`, never importing it | test | yes | `packages/atif-embed/tests/test_guard_twin_pin.py:41` and `packages/atif-embed/tests/test_guard_twin_pin.py:55-72` | -| The independent corpus readers | indirect | yes | `packages/atif-duck/src/atif_duck/infrastructure/registry.py:209`, `packages/atif-embed/src/atif_embed/infrastructure/corpus_text_rows.py:113`, `packages/atif-analytics/src/atif_analytics/infrastructure/corpus_reader.py:72` | +| The independent corpus readers | indirect | yes | `packages/atif-duck/src/atif_duck/infrastructure/registry.py:597`, `packages/atif-embed/src/atif_embed/infrastructure/corpus_text_rows.py:155`, `packages/atif-analytics/src/atif_analytics/infrastructure/corpus_reader.py:178` | | `[tool.coverage.run] source`: the same import packages, named rather than pathed | config | likely | `pyproject.toml:625-633` | ### Blast-radius notes - **A new cross-package import fails the `lint:imports` gate and nothing else.** `lint:imports` runs - `uv run lint-imports` over `packages/*/src/**/*.py` plus `pyproject.toml` - (`mise.toml:161-162`). The type checkers and the test suite stay green, so the contract violation - surfaces only when `mise run check` reaches that gate. + `uv run lint-imports`, and its `sources` list names `packages/*/src/**/*.py`, the root + `pyproject.toml`, and the member manifests (`mise.toml:174-175`). The type checkers and the test + suite stay green, so the contract violation surfaces only when `mise run check` reaches that gate. - **atif-duck declares no `layers` contract.** It has `domain/` and `infrastructure/` and no `application/`, so its layer direction is convention rather than enforcement; only the `independence` contract constrains it (`pyproject.toml:605`). @@ -237,41 +247,42 @@ surfaces above exist as duplicated strings and Protocols instead of imports. ## `EXIT_CODES`: the CLI wire contract -Defined at: `packages/atif-cli/src/atif_cli/errors.py:25-39` +Defined at: `packages/atif-cli/src/atif_cli/errors.py:30-66` -Its keys map to the distinct codes 0, 2, 64, 65, 70, 78, and 127. This is the contract an agent +Its keys map to the distinct codes 0, 2, 64, 65, 70, 77, 78, and 127. This is the contract an agent driving the CLI reads, so the dict is the surface, not the individual numbers, and not the messages. | Downstream | Type | Touch on change | Citation | | --- | --- | --- | --- | -| `atif_cli.app`: the module-scope import; the dict is on the lean path | direct import | yes | `packages/atif-cli/src/atif_cli/app.py:38` | -| `atif_cli.app.convert`: exits `invalid_input`, `empty_session`, `runtime_error`, `validation_error` | direct import | yes | `packages/atif-cli/src/atif_cli/app.py:264-304` | -| `atif_cli.app.query`: `parse_error`, raised after the classified envelope is emitted | direct import | yes | `packages/atif-cli/src/atif_cli/app.py:603-610` | -| `atif_cli.app.embed`: `invalid_input` plus a `EXIT_CODES[kind]` dispatch over the classifier's verdict | direct import | yes | `packages/atif-cli/src/atif_cli/app.py:814-847` | -| `atif_cli.app.search`: `no_embeddings`, the exit that tells an operator to run `embed` | direct import | yes | `packages/atif-cli/src/atif_cli/app.py:935-941` | -| `atif_cli.app.examples`: `invalid_input` on an unknown `--category` / `--requires` value | direct import | yes | `packages/atif-cli/src/atif_cli/app.py:1036-1042` | -| `atif_cli.duck_errors`: classifies `duckdb.Error` into parse / catalog / runtime plus `embedding_mismatch` | direct import | yes | `packages/atif-cli/src/atif_cli/duck_errors.py:25` and `packages/atif-cli/src/atif_cli/duck_errors.py:40-86` | -| `atif_cli.cron`: the unattended lane, which suppresses retries on code 78 | direct import | likely | `packages/atif-cli/src/atif_cli/cron.py:32` and `packages/atif-cli/src/atif_cli/cron.py:175` | -| `ClassifiedError.to_payload()`: the JSON error envelope on non-TTY stderr | indirect | likely | `packages/atif-cli/src/atif_cli/errors.py:60-68` | -| `test_app.py`: asserts against the dict, never against a literal, except where the number itself is the claim | test | yes | `packages/atif-cli/tests/test_app.py:17` and `packages/atif-cli/tests/test_app.py:744` | -| `test_cron.py`, `test_integration.py`, `test_vss_commands.py`: more suites keyed off the dict | test | yes | `packages/atif-cli/tests/test_cron.py:27`, `packages/atif-cli/tests/test_integration.py:99`, `packages/atif-cli/tests/test_vss_commands.py:20` | -| `docs/CONTRACT.md:62-66` §CLI: the documented command surface these codes are raised from | config | likely | `docs/CONTRACT.md:62-66` | +| `atif_cli.app`: the module-scope import; the dict is on the lean path | direct import | yes | `packages/atif-cli/src/atif_cli/app.py:73` | +| `atif_cli.app.convert`: exits `invalid_input`, `empty_session`, `runtime_error`, `validation_error` | direct import | yes | `packages/atif-cli/src/atif_cli/app.py:727-757` | +| `atif_cli.app.query`: `parse_error`, raised after the classified envelope is emitted | direct import | yes | `packages/atif-cli/src/atif_cli/app.py:1429-1440` | +| `atif_cli.app.embed`: `invalid_input` plus a `EXIT_CODES[kind]` dispatch over the classifier's verdict | direct import | yes | `packages/atif-cli/src/atif_cli/app.py:1908-1959` | +| `atif_cli.app.search`: `no_embeddings`, the exit that tells an operator to run `embed` | direct import | yes | `packages/atif-cli/src/atif_cli/app.py:2146-2156` | +| `atif_cli.app.examples`: `invalid_input` on an unknown `--category` / `--requires` value | direct import | yes | `packages/atif-cli/src/atif_cli/app.py:2280-2290` | +| `atif_cli.duck_errors`: classifies `duckdb.Error` into parse / catalog / runtime plus `embedding_mismatch` | direct import | yes | `packages/atif-cli/src/atif_cli/duck_errors.py:26` and `packages/atif-cli/src/atif_cli/duck_errors.py:35-90` | +| `atif_cli.cron`: the `install` / `status` companion to the cron lanes, which exits `invalid_input` when it can't find the refresh script; the lane itself, `scripts/atif-sql-refresh.sh`, suppresses retries on code 78 | direct import | likely | `packages/atif-cli/src/atif_cli/cron.py:32` and `packages/atif-cli/src/atif_cli/cron.py:181`, `scripts/atif-sql-refresh.sh:398` | +| `ClassifiedError.to_payload()`: the JSON error envelope on non-TTY stderr | indirect | likely | `packages/atif-cli/src/atif_cli/errors.py:78-86` | +| `test_app.py`: asserts against the dict, never against a literal, except where the number itself is the claim | test | yes | `packages/atif-cli/tests/test_app.py:41` and `packages/atif-cli/tests/test_app.py:1511` | +| `test_cron.py`, `test_integration.py`, `test_vss_commands.py`: more suites keyed off the dict | test | yes | `packages/atif-cli/tests/test_cron.py:28`, `packages/atif-cli/tests/test_integration.py:147`, `packages/atif-cli/tests/test_vss_commands.py:21` | +| `docs/CONTRACT.md:300-336` §CLI: the documented command surface these codes are raised from | config | likely | `docs/CONTRACT.md:300-336` | ### Blast-radius notes -- **Several keys share code 2 or 65 on purpose.** `empty_session` and `no_embeddings` both - exit 2; `catalog_error`, `validation_error`, and `embedding_mismatch` all exit 65. Renaming a key is - safe for callers reading numbers and breaks every internal `EXIT_CODES["..."]` lookup, which is the - opposite of the usual polarity (`packages/atif-cli/src/atif_cli/errors.py:26-38`). +- **Several keys share a code on purpose.** `empty_session` and `no_embeddings` both exit 2; + `invalid_input` and `parse_error` exit 64; `catalog_error`, `validation_error`, `embedding_mismatch`, + and `lake_mismatch` exit 65; `runtime_error` and `sandbox_refused` exit 70; four keys exit 78. + Renaming a key is safe for callers reading numbers and breaks every internal `EXIT_CODES["..."]` + lookup, which is the opposite of the usual polarity (`packages/atif-cli/src/atif_cli/errors.py:30-66`). - **Exit 78 is the only code with retry semantics attached.** `terminal_state` means an operator must act, so an unattended lane suppresses retries on it rather than burning identical ticks; a test pins - both the mapping and the literal 78 (`packages/atif-cli/src/atif_cli/errors.py:35-37`, - `packages/atif-cli/tests/test_app.py:744`). + both the mapping and the literal 78 (`packages/atif-cli/src/atif_cli/errors.py:49-51`, + `packages/atif-cli/tests/test_app.py:1511`). - **A code outside the dict is the failure mode this surface exists to prevent.** An unhandled - traceback exits 1, which appears nowhere in `EXIT_CODES`, and tests assert that a lazily bound - Lance scan tripping mid-stream still classifies instead of exiting 1 - (`packages/atif-cli/src/atif_cli/duck_errors.py:15-17`, - `packages/atif-cli/tests/test_app.py:657` and `packages/atif-cli/tests/test_app.py:1039`). + traceback exits 1, which appears nowhere in `EXIT_CODES`, and tests assert that a store mismatch + still classifies instead of exiting 1, both at registration and when a lazily bound Lance scan + trips mid-stream (`packages/atif-cli/src/atif_cli/duck_errors.py:15-19`, + `packages/atif-cli/tests/test_app.py:1418` and `packages/atif-cli/tests/test_app.py:1800`). ## The published `atif-sql` distribution @@ -288,14 +299,14 @@ purpose: the distribution is `atif-sql`, the console script's module is `atif_cl | `[project.dependencies]`: the union of every member's third-party requirements, and the ONLY thing an installer sees | config | yes | `pyproject.toml:41-57` | | `[tool.hatch.build.targets.wheel] packages`: the module trees the wheel carries; an omission ships a wheel missing a module | config | yes | `pyproject.toml:93-102` | | `test_distribution.py`: asserts the union, the absence of any `atif-*` requirement, and the module list with its `py.typed` markers | test | yes | `packages/atif-cli/tests/test_distribution.py:3-22` | -| Dev pins in the CLI and analytics manifests, resolved locally through `[tool.uv.sources]` and never published | config | likely | `packages/atif-cli/pyproject.toml:32-36`, `packages/atif-analytics/pyproject.toml:24` | +| Dev pins in the CLI and analytics manifests, resolved locally through `[tool.uv.sources]` and never published | config | likely | `packages/atif-cli/pyproject.toml:32-36`, `packages/atif-analytics/pyproject.toml:23` | | `[tool.commitizen] version_files`: glob:regex entries rewriting the published version, every member manifest, and the dev pin blocks | config | yes | `pyproject.toml:534-539` | | `[tool.commitizen] version`: the single source of truth, because `version_provider` defaults to reading it | config | yes | `pyproject.toml:491-494` | -| `cz bump --check-consistency` in `release.yml`: fails when the current version is absent from any listed file | config | yes | `.github/workflows/release.yml:193` | +| `cz bump --check-consistency` in `release.yml`: fails when the current version is absent from any listed file | config | yes | `.github/workflows/release.yml:222` | | `pre_bump_hooks`: `uv lock` then `git add uv.lock`, so the re-resolved lockfile lands in the bump commit | config | yes | `pyproject.toml:516-519` | -| `uv lock --check`: enforced as a lefthook pre-commit step and as a CI step | config | likely | `mise.toml:84`, `lefthook.yml:95`, `.github/workflows/check.yml:51` | -| The console script `atif-sql = "atif_cli.app:main"` | config | likely | `packages/atif-cli/pyproject.toml:42` | -| `tag_format = "v$version"`: `publish.yml` strips the leading `v` and `gh release create --verify-tag` fails if the formats diverge | config | likely | `pyproject.toml:499` | +| `uv lock --check`: enforced as a lefthook pre-commit step and as a CI step | config | likely | `mise.toml:96`, `lefthook.yml:95`, `.github/workflows/check.yml:57` | +| The console script `atif-sql = "atif_cli.app:main"` | config | likely | `pyproject.toml:75` | +| `tag_format = "v$version"`: `publish.yml` strips the leading `v`, and `release.yml`'s `gh release create --verify-tag` fails if the formats diverge | config | likely | `pyproject.toml:499` | ### Blast-radius notes @@ -308,12 +319,12 @@ purpose: the distribution is `atif-sql`, the console script's module is `atif_cl produces a wheel that installs cleanly and raises `ModuleNotFoundError` at run time. The union assertion in `test_distribution.py` is what turns that into a failing gate (`packages/atif-cli/tests/test_distribution.py:92`). -- **`major_version_zero = true`, so a breaking change moves 0.2.0 to 0.3.0.** Reaching 1.0.0 is a +- **`major_version_zero = true`, so a breaking change moves 0.3.0 to 0.4.0.** Reaching 1.0.0 is a decision, not a side effect of a `!` in a commit subject (`pyproject.toml:500-502`). ## The lean import path of `atif_cli.app` -Defined at: `packages/atif-cli/src/atif_cli/app.py:22-26` +Defined at: `packages/atif-cli/src/atif_cli/app.py:50-53` A bare `import atif_cli.app` must not pull duckdb, harbor, lancedb, boto3, or polars. The fast path (`schema`, `examples`, `--help`, `--version`) needs none of them, and each costs hundreds of @@ -322,14 +333,14 @@ it. | Downstream | Type | Touch on change | Citation | | --- | --- | --- | --- | -| `test_lean_import.py`: a forbidden list checked in a FRESH interpreter via `subprocess` | test | yes | `packages/atif-cli/tests/test_lean_import.py:17-32` and `packages/atif-cli/tests/test_lean_import.py:43-49` | +| `test_lean_import.py`: a forbidden list checked in a FRESH interpreter via `subprocess` | test | yes | `packages/atif-cli/tests/test_lean_import.py:17-38` and `packages/atif-cli/tests/test_lean_import.py:41-55` | | ruff `PLC0415` (import-outside-top-level) ignored workspace-wide, 182 measured sites | config | yes | `pyproject.toml:227` | -| `atif_cli.errors`: kept `atif_*`-free and duckdb-free so it stays on the lean path | direct import | yes | `packages/atif-cli/src/atif_cli/errors.py:13-15` | -| `atif_cli.duck_errors`: the concrete `duckdb.Error` classifier, split out for exactly that reason | direct import | yes | `packages/atif-cli/src/atif_cli/duck_errors.py:26` and `packages/atif-cli/src/atif_cli/duck_errors.py:31` | +| `atif_cli.errors`: kept `atif_*`-free and duckdb-free so it stays on the lean path | direct import | yes | `packages/atif-cli/src/atif_cli/errors.py:19-21` | +| `atif_cli.duck_errors`: the concrete `duckdb.Error` classifier, split out for exactly that reason | direct import | yes | `packages/atif-cli/src/atif_cli/duck_errors.py:26` and `packages/atif-cli/src/atif_cli/duck_errors.py:35` | | `atif_duck.domain.examples`: a pure domain module with no duckdb import, safe on the lean path | direct import | yes | `packages/atif-duck/src/atif_duck/domain/examples.py:26-27` | -| Deferred imports inside command bodies (`convert`, `materialize`, `status`, `query`, `analyze`, `embed`, `search`, `examples`, `schema`) | runtime dispatch | likely | `packages/atif-cli/src/atif_cli/app.py:615`, `packages/atif-cli/src/atif_cli/app.py:906`, `packages/atif-cli/src/atif_cli/app.py:1099` | -| `atif_cli.cron`: imported at module scope, so it must itself stay lean | direct import | yes | `packages/atif-cli/src/atif_cli/app.py:37` | -| `if TYPE_CHECKING:` blocks: the only way to name a heavy type in an annotation on this path | direct import | likely | `packages/atif-cli/src/atif_cli/app.py:33` | +| Deferred imports inside command bodies (`convert`, `materialize`, `status`, `query`, `analyze`, `embed`, `search`, `examples`, `schema`) | runtime dispatch | likely | `packages/atif-cli/src/atif_cli/app.py:712`, `packages/atif-cli/src/atif_cli/app.py:1456`, `packages/atif-cli/src/atif_cli/app.py:2352` | +| `atif_cli.cron`: imported at module scope, so it must itself stay lean | direct import | yes | `packages/atif-cli/src/atif_cli/app.py:72` | +| `if TYPE_CHECKING:` blocks: the only way to name a heavy type in an annotation on this path | direct import | likely | `packages/atif-cli/src/atif_cli/app.py:84` | ### Blast-radius notes @@ -341,15 +352,15 @@ it. pays for duckdb only when it runs (`packages/atif-cli/tests/test_lean_import.py:21`). - **A new `atif_*` package imported at `app.py` module scope needs a forbidden-list entry too.** The test enumerates modules, so a member absent from `_FORBIDDEN_EAGER_IMPORTS` can regress the fast path - without failing anything (`packages/atif-cli/tests/test_lean_import.py:17-32`). + without failing anything (`packages/atif-cli/tests/test_lean_import.py:17-38`). ## Other notable surfaces -- **`DEFAULT_PRICING`** (`packages/atif-duck/src/atif_duck/domain/catalog.py:405`): models with +- **`DEFAULT_PRICING`** (`packages/atif-duck/src/atif_duck/domain/catalog.py:612`): models with published list rates. A model absent here lands in `cost_estimate`'s `unpriced_steps` rather than being dropped, and tests pin the table: an exact oracle - (`packages/atif-duck/tests/test_duck_views.py:459`) and a corpus-coverage gate - (`packages/atif-duck/tests/test_duck_views.py:533`). + (`packages/atif-duck/tests/test_duck_views.py:519`) and a corpus-coverage gate + (`packages/atif-duck/tests/test_duck_views.py:593`). - **The model alias registry** (`packages/atif-models/src/atif_models/domain/registry.py:95`): the `(family, size)` entries resolved through `resolve()` at `packages/atif-models/src/atif_models/domain/registry.py:162`. `ModelSpec` reaches every @@ -366,18 +377,18 @@ it. branch in `build_structured_provider`, which sends every family other than `anthropic` to the OpenAI adapter (`packages/atif-models/src/atif_models/infrastructure/providers.py:28-34`), and a `RUNNABLE_FAMILIES` entry (`packages/atif-models/src/atif_models/infrastructure/settings.py:25`). -- **`RECOVERY_HINT`** (`packages/atif-embed/src/atif_embed/domain/embedding_guard.py:23` and its +- **`RECOVERY_HINT`** (`packages/atif-embed/src/atif_embed/domain/embedding_guard.py:30` and its atif-duck twin): one operator instruction, pinned by assertions including one that forbids naming a home directory and one that forbids inlining the path at the raise site - (`packages/atif-embed/tests/test_guard_twin_pin.py:97-106`). + (`packages/atif-embed/tests/test_guard_twin_pin.py:85-100`). - **The `# noqa: S608` sites**: `S608` is enforced, and each suppression carries its static-catalog reason inline; some of them are in the examples generator, where the interpolated name comes from the catalog rather than from user input - (`packages/atif-duck/src/atif_duck/domain/examples.py:144`, - `packages/atif-duck/src/atif_duck/domain/examples.py:161`, `pyproject.toml:212-215`). -- **Watermark and quiescence** (`packages/atif-corpus/src/atif_corpus/domain/watermark.py:28`): the + (`packages/atif-duck/src/atif_duck/domain/examples.py:157`, + `packages/atif-duck/src/atif_duck/domain/examples.py:174`, `pyproject.toml:212-215`). +- **Watermark and quiescence** (`packages/atif-corpus/src/atif_corpus/domain/watermark.py:57`): the freshness rule that decides what `materialize` re-converts, surfaced to `status` through the - producer's public `read_watermark` (`packages/atif-cli/src/atif_cli/app.py:460`) rather than a + producer's public `read_watermark` (`packages/atif-cli/src/atif_cli/app.py:1207`) rather than a private reader. ## See also diff --git a/docs/insights/tech-debt.md b/docs/insights/tech-debt.md index 8de7eee..1a69dea 100644 --- a/docs/insights/tech-debt.md +++ b/docs/insights/tech-debt.md @@ -1,8 +1,9 @@ # atif-sql · Tech debt -**There are no `TODO`, `FIXME`, `HACK`, or `XXX` markers in this repository.** These -scans confirm it: over `packages/*/src`, over `packages/` plus `scripts/`, and over every -file `git ls-files` names. That means the marker channel carries no signal here. It does +**There are no `TODO`, `FIXME`, `HACK`, or `XXX` markers in this repository's code.** Scans +over `packages/*/src` and over `packages/` plus `scripts/` return nothing, and a scan over +every file `git ls-files` names finds the words only in Vale style lists and in docs pages +that discuss them. That means the marker channel carries no signal here. It does not mean the codebase is debt-free, and this page exists because the debt is real and lives elsewhere. @@ -10,24 +11,23 @@ So the register below is assembled from these sources rather than from comments. citation on this page carries its full path, so no reference resolves through a nearby antecedent. -1. **The suppression config, read as the debt register someone already wrote.** `select = - ["ALL"]` with a short list of ignores (`pyproject.toml:209-247`), some pyright rules switched +1. **The suppression config, read as the debt register someone already wrote.** `select = ["ALL"]` + with a short list of ignores (`pyproject.toml:209-247`), some pyright rules switched off (`pyproject.toml:728-771`), ty at `all = "error"` with no rule turned off over `src/` (`pyproject.toml:447-448`; the overrides below it relax a few rules for the tests trees and the fuzz entrypoints only), and per-site `# noqa` / `# pyright: ignore` counts taken by grep. Almost every config entry carries its own measured count and its cost-of-removal analysis inline, which is a better debt - record than a comment marker and is quoted here rather than re-derived. + record than a comment marker. It's quoted here, with a measurement on this tree beside + any count the tree doesn't match. 2. **Version ceilings and the reason attached to each**, read at the pin in the member manifests, not inferred from the lockfile. -3. **Install weight and platform coverage**, computed independently from `uv.lock` by - walking the `atif-sql` dependency graph twice (once whole, once with `harbor` blocked), - following extras as well as direct dependencies. +3. **Platform coverage**, read from the wheel and source-distribution entries `uv.lock` records for each + native package in the runtime closure and checked against the release notes in `RELEASING.md`. 4. **Asymmetric enforcement between sibling surfaces**: a coverage floor enforced in CI but not in the declared definition of done, so a local green run does not measure it. -5. **Coverage and test-shape gaps**, per module. Coverage figures are measured from the - generated report, which is gitignored, so every figure is cited to the source module it - describes. +5. **Test-shape gaps**, read from the test modules themselves: which tests reach real + transcripts and which run against a double. Category and cost vocabularies are closed. Categories: `marker`, `wrong abstraction`, `error handling`, `dead code adjacent`, `deprecated pattern`, `version pin`, `duplicated @@ -40,52 +40,56 @@ rises. | Rank | Debt item | Category | Cost to fix | Citation | | --- | --- | --- | --- | --- | -| 1 | RETIRED 2026-09-11. The conversion path used to run through one PRIVATE upstream method per agent; both are now ported into this package on harbor's public ATIF models (`packages/atif-converter/src/atif_converter/domain/claude_code_conversion.py:75`, `packages/atif-converter/src/atif_converter/domain/codex_conversion.py:781`), held to parity by an oracle in the tests (`packages/atif-converter/tests/harbor_oracle.py:94`), and the pin widened to `harbor>=0.22.0,<1`. The install weight in row 3 is retired too. | Coupling | Done | `packages/atif-converter/pyproject.toml:26` | -| 2 | Seven upstream conversion losses are catalogued as `FidelityGap` enum members rather than fixed. Gap 3 flattens the `parentUuid` tree by timestamp sort, losing branch and rewind structure; gap 7 drops the event `uuid`, making step-to-raw-record identity unrecoverable from the trajectory (recovered out of band through a parallel `edges.jsonl` sidecar the corpus layout mandates). | `wrong abstraction` | L | `packages/atif-converter/src/atif_converter/domain/fidelity.py:44-79`, `docs/CONTRACT.md:26-28` | -| 3 | RETIRED 2026-09-27. `harbor` used to pull a web server, the whole `supabase` client stack, and `litellm` with `openai`, `tiktoken`, `tokenizers` and `huggingface-hub` into every install, for the ATIF data classes, the validator, and one price lookup per step. The data classes and the validator are vendored in `atif_converter.domain.atif`, prices in `atif_converter/domain/model_prices.json`, and harbor and litellm are dev dependencies only, guarded by `packages/atif-converter/tests/test_dev_only_imports_guard.py`. | `version pin` | Done | `RELEASING.md`, "harbor and litellm are not in the closure" | -| 4 | 415 pyright findings sit behind three disabled Unknown-propagation rules, and the config names the fix it has not built: a `TypedDict` model of the `~/.claude` JSONL record and the ATIF trajectory would close 144 of them. Until that model exists, every JSON-shaped value in the hottest modules is `dict[str, Any]` narrowed by `isinstance`, with the type checker's opinion switched off. The two heaviest concentrations are the converter's enrichment module at 59 findings and the analytics corpus reader at 33. | `wrong abstraction` | L | `pyproject.toml:728-762` | -| 5 | Some SQL statements are built by string interpolation through a hand-rolled quote-doubling escape, because DuckDB rejects prepared parameters as table-function arguments. ruff's `# noqa: S608` silences ruff only; bandit reports the same 15 as B608 and those findings upload to code scanning, so the suppression is not portable across the two scanners that both implement the rule. | `deprecated pattern` | L | `pyproject.toml:210-215`, `pyproject.toml:836-841`, `packages/atif-duck/src/atif_duck/infrastructure/registry.py:172`, `packages/atif-embed/src/atif_embed/infrastructure/corpus_text_rows.py:108` | -| 6 | No test reads a real `~/.claude` corpus. The whole integration tier is a few tests over small synthetic sessions, and parity oracles against the predecessor implementation are explicitly not part of this repo. The seven fidelity gaps are policy about real-world JSONL shapes that nothing real-world exercises. | `missing tests` | L | `packages/atif-cli/tests/test_integration.py:3-9`, `packages/atif-cli/tests/test_integration.py:24-34`, `AGENTS.md:92-94` | -| 7 | RETIRED IN PART 2026-09-27. `hdbscan 0.8.44` never published an aarch64 wheel, so a first install on Graviton, an ARM CI runner, or a `linux/arm64` container used to compile Cython extensions and need a C toolchain; hdbscan left the lock with the structural pipelines, and every native package in the closure now ships an aarch64 wheel. The musl half stands: musl is not merely slow but impossible, because `lancedb 0.39.0` publishes no sdist at all, so there is nothing to build from. | `version pin` | L | `RELEASING.md:204-213` | -| 8 | The four size ratchets and the complexity ratchet are set at the measured worst function in the tree: `max-statements = 119` for `_friction_async`, `max-complexity = 38` for `enrich_trajectory`, `max-args = 19` for the `search` command. The ratchet stops growth and permanently blesses the current outliers; nothing in the config plans their reduction. | `wrong abstraction` | L | `pyproject.toml:327-350`, `packages/atif-analytics/src/atif_analytics/application/use_cases/friction.py:210`, `packages/atif-converter/src/atif_converter/domain/enrichment.py:198` | -| 9 | `TC001`/`TC002`/`TC003` are ignored on the strength of a manual probe: moving 58 imports under `TYPE_CHECKING` broke `typing.get_type_hints` on 159 first-party functions and classes. The 182-site `PLC0415` ignore beside it has a regression test proving its property in a fresh interpreter; the `get_type_hints` property has none, so the argument that justifies the ignore is not locked. | `missing tests` | M | `pyproject.toml:227-246`, `packages/atif-cli/tests/test_lean_import.py:35` | -| 10 | Every security scanner has its findings exit code swallowed, by a stated contract: findings do not fail the job, only a scanner that produced no usable report does. Gating is delegated entirely to GitHub code-scanning alerts and branch protection, so nothing inside the repo fails on a new vulnerability. | `error handling` | M | `mise.toml:217-223`, `mise.toml:366`, `mise.toml:482` | -| 11 | The provider/dimension guard exists as two deliberate 83-line twins because the independence contract forbids either package importing the other. The drift pin reads the atif-duck twin as SOURCE TEXT, parses it with `ast`, and asserts substrings, including a raw `source.index("raise EmbeddingProviderMismatch")` that silently changes meaning if the raise site is renamed or a second raise appears. The two docstrings have already diverged; the pin covers the recovery string and the interpolation, not the rule. | `duplicated logic` | M | `packages/atif-embed/tests/test_guard_twin_pin.py:3-13`, `packages/atif-embed/tests/test_guard_twin_pin.py:93-95`, `packages/atif-duck/src/atif_duck/domain/embedding_guard.py:5-16` | -| 12 | RETIRED 2026-09-27. The clustering path was the least-tested and least portable code in the tree. It was deleted with the cluster, terms, and community pipelines. | `missing tests` | Done | `packages/atif-analytics/src/atif_analytics/application/analyze.py:17-20` | -| 13 | Both binding contract documents disclaim describing the code. CONTRACT-V2 states outright that when it disagrees with the code, the code wins and the file is design intent "not a description of current behavior"; CONTRACT v1 calls its own scope section historical. A binding document that pre-emptively surrenders authority cannot be used to detect drift, which is the only thing a contract document is for. *judgment-call*: the disclaimers are honest, and honesty about staleness is still staleness. | `dead code adjacent` | M | `docs/CONTRACT-V2.md:3-8`, `docs/CONTRACT.md:5-8` | -| 14 | RETIRED 2026-09-27. `lancedb>=0.30,<0.38` used to be declared by both atif-analytics and atif-embed, with no comment on either side. atif-analytics dropped the dependency with its Lance reader, so only atif-embed declares it. | `version pin` | Done | `packages/atif-embed/pyproject.toml:22` | -| 15 | `FakeConverter`, a scriptable test double, ships inside the installable `atif-corpus` wheel under `infrastructure/`, and it is the converter the end-to-end materialize suite runs against. A fake that both mutates and reads corpus state stands in for the real harbor adapter in exactly the tests that would catch a state-transition bug. | `wrong abstraction` | S | `packages/atif-corpus/src/atif_corpus/infrastructure/fake_converter.py:3-9`, `packages/atif-corpus/tests/test_materialize.py:25` | -| 16 | Independently declared `DomainError(Exception)` bases with the same name and no shared ancestor, so a caller composing members cannot write one `except` clause and any symbol-name search cross-attributes them all. | `duplicated logic` | S | `packages/atif-converter/src/atif_converter/domain/errors.py:14`, `packages/atif-embed/src/atif_embed/domain/errors.py:14`, `packages/atif-models/src/atif_models/domain/ports.py:39` | -| 17 | The 89% coverage floor is not part of the declared definition of done. `mise run check` depends on its gates and `test:cov` is not among them, so a local green run can drop coverage below the floor and only the CI job notices. | `missing tests` | S | `mise.toml:197-212`, `pyproject.toml:648`, `.github/workflows/check.yml:78` | +| 1 | Six conversion losses inherited from harbor's Claude Code converter are catalogued as `FidelityGap` members rather than fixed, because the converter is a parity port and fixing one means diverging from the oracle on purpose. Gap 3 flattens the `parentUuid` tree by timestamp sort, losing branch and rewind structure, and every conversion reports it. Gap 7 is the converter never writing the event `uuid` into `Step.extra`: enrichment restores step-to-raw-record identity as `step.extra["source_uuids"]`, the corpus layout keeps a parallel `edges.jsonl` sidecar with every raw record's uuid and parent, and the gap is reported only where enrichment refuses to attribute a step. | `wrong abstraction` | L | `packages/atif-converter/src/atif_converter/domain/fidelity.py:55-99`, `packages/atif-converter/src/atif_converter/application/convert_and_audit.py:79`, `packages/atif-converter/src/atif_converter/application/convert_and_audit.py:124`, `docs/CONTRACT.md:29-31` | +| 2 | Three Unknown-propagation rules are off, and the config names the fix it hasn't built: its record traces 415 findings to their origins and says a `TypedDict` model of the `~/.claude` JSONL record and the ATIF trajectory would close 144 of them. No such model exists, so every JSON-shaped value in the hottest modules is `dict[str, Any]` narrowed by `isinstance`, with the type checker's opinion switched off. Turned back on over this tree, the three rules report 1,106 findings outside `fuzz/`, 818 of them under `packages/*/src`. The heaviest files are the converter's Claude Code conversion module at 101, its result-signal module at 92, its Codex conversion module at 77, and its enrichment module at 61. | `wrong abstraction` | L | `pyproject.toml:728-762` | +| 3 | SQL statements are assembled as f-strings at 70 sites, each carrying a per-line `# noqa: S608` and `# nosec B608` that names what it interpolates. Paths from outside reach `read_json` as bound parameters, and the hand-rolled quote-doubling escape `sql_literal` remains for the statement kinds DuckDB won't prepare, such as `ATTACH` and the producer's one-row session projection. Both scanners that implement the rule are silenced at every site, so neither flags a new unsafe interpolation. The guard is an AST audit test over the eleven modules that build SQL text, which hold 67 of the 70 sites (the other three, in `atif_duck.domain.examples`, interpolate catalog view and macro names). | `deprecated pattern` | L | `pyproject.toml:212-215`, `packages/atif-embed/src/atif_embed/infrastructure/corpus_text_rows.py:83-85`, `packages/atif-duck/src/atif_duck/domain/sql_literal.py:15-18`, `packages/atif-duck/src/atif_duck/infrastructure/registry.py:1678`, `packages/atif-duck/src/atif_duck/infrastructure/columnar.py:536`, `packages/atif-duck/tests/test_sql_text_boundaries.py:457` | +| 4 | No test CI runs reads a real `~/.claude` corpus. The live parity tests read local transcripts and skip wherever there are none, CI included; the integration tier is a few tests over two small synthetic sessions; and the migration parity oracles `AGENTS.md` mentions aren't part of this repo. The six fidelity gaps are policy about real-world JSONL shapes that nothing in CI exercises. | `missing tests` | L | `packages/atif-converter/tests/test_parity_live.py:3-9`, `packages/atif-converter/tests/test_parity_live.py:107`, `packages/atif-cli/tests/test_integration.py:3-9`, `packages/atif-cli/tests/test_integration.py:24-34`, `AGENTS.md:451-453` | +| 5 | Alpine and other musl targets can't install the package, and not merely slowly: `duckdb` and `lancedb` publish no `musllinux` wheels, and `lancedb 0.39.0` publishes no sdist at all, so there is nothing to build from. The `manylinux`, macOS, and Windows targets have a prebuilt wheel for every native package in the closure. | `version pin` | L | `RELEASING.md:217-219`, `RELEASING.md:211-213` | +| 6 | The four size ratchets and the complexity ratchet are set at the measured worst function in the tree: `max-statements = 119` for `_friction_async`, `max-branches = 39` and `max-complexity = 38` for `enrich_trajectory`, and `max-args = 19` for `materialize` (the config's comment names the `search` command, which takes seven). The ratchet stops growth and permanently blesses the current outliers; nothing in the config plans their reduction. | `wrong abstraction` | L | `pyproject.toml:327-350`, `packages/atif-analytics/src/atif_analytics/application/use_cases/friction.py:210`, `packages/atif-converter/src/atif_converter/domain/enrichment.py:198`, `packages/atif-corpus/src/atif_corpus/application/materialize.py:1497` | +| 7 | `TC001`/`TC002`/`TC003` are ignored on the strength of a manual probe the config records: moving 58 imports under `TYPE_CHECKING` broke `typing.get_type_hints` on 159 first-party functions and classes (the three rules flag 33 imports under `packages/*/src` on this tree). The `PLC0415` ignore beside it (182 sites by the config's count, 178 under `packages/*/src` on this tree) has a regression test proving its property in a fresh interpreter; the `get_type_hints` property has none, so the argument that justifies the ignore is not locked. | `missing tests` | M | `pyproject.toml:227-246`, `packages/atif-cli/tests/test_lean_import.py:41` | +| 8 | Every scanner in the `security` tier has its findings exit code swallowed, by a stated contract: findings do not fail the job, only a scanner that produced no usable report does. Gating on those findings is delegated to GitHub code-scanning alerts and branch protection. Inside the repo, only dependency review fails on a vulnerability, and only on a pull request whose dependency changes carry a high or critical advisory; a new advisory against a locked dependency, a new secret, or a new SAST finding fails nothing. | `error handling` | M | `mise.toml:270-276`, `mise.toml:443`, `mise.toml:561`, `.github/workflows/dependency-review.yml:49` | +| 9 | The provider/dimension guard exists as two deliberate twins, 88 lines in atif-duck and 83 in atif-embed, because the independence contract forbids either package importing the other. The drift pin reads the atif-duck twin as SOURCE TEXT, parses it with `ast`, and asserts substrings, including a raw `source.index("raise EmbeddingProviderMismatch")` that silently changes meaning if the raise site is renamed or a second raise appears. The two docstrings differ; the pin covers the recovery string and the interpolation, not the rule. | `duplicated logic` | M | `packages/atif-embed/tests/test_guard_twin_pin.py:3-13`, `packages/atif-embed/tests/test_guard_twin_pin.py:93-95`, `packages/atif-duck/src/atif_duck/domain/embedding_guard.py:5-16` | +| 10 | CONTRACT-V2, the binding document for the analytics pipelines, VSS, the model registry, and the cron lanes, disclaims describing the code: when it disagrees with the code, the code is authoritative, and the reader is told to use it "for design intent, not as a description of current behavior" instead. CONTRACT v1 binds the corpus layout, the enrichment pass, the transcript-derived surface, and the CLI shapes, and hands everything on top of them to V2. So the surfaces V2 covers have no binding document that can detect drift, which is the only thing a contract document is for. *judgment-call*: the disclaimer warns the reader, and the drift it warns about stays. | `dead code adjacent` | M | `docs/CONTRACT-V2.md:3-8`, `docs/CONTRACT.md:5-8` | +| 11 | `FakeConverter`, a scriptable test double, ships inside the installable `atif-sql` wheel under `atif_corpus/infrastructure/`, and it's the converter the atif-corpus end-to-end materialize suite runs against. It fabricates contract-shaped output from the session id without converting the transcript, and it stands in for the real converter adapter in exactly the tests that would catch a state-transition bug. | `wrong abstraction` | S | `packages/atif-corpus/src/atif_corpus/infrastructure/fake_converter.py:3-9`, `pyproject.toml:98`, `packages/atif-corpus/tests/test_materialize.py:3`, `packages/atif-cli/src/atif_cli/converter_adapter.py:80` | +| 12 | Independently declared `DomainError(Exception)` bases with the same name and no shared ancestor, so a caller composing members cannot write one `except` clause and any symbol-name search cross-attributes them all. | `duplicated logic` | S | `packages/atif-converter/src/atif_converter/domain/errors.py:14`, `packages/atif-embed/src/atif_embed/domain/errors.py:14`, `packages/atif-models/src/atif_models/domain/ports.py:39` | +| 13 | The 89% coverage floor is not part of the declared definition of done. `mise run check` depends on its gates and `test:cov` is not among them, so a local green run can drop coverage below the floor and only the CI job notices. | `missing tests` | S | `mise.toml:245-265`, `pyproject.toml:648`, `.github/workflows/check.yml:90` | ## Explicit markers -**None.** This section is empty because the repository contains no comment markers, not +**None in code.** This section is empty because the code contains no comment markers, not because the search was narrow. - Scanned `packages/*/src` for `\bTODO\b`, `\bFIXME\b`, `\bHACK\b`, `\bXXX\b`: no matches. - Scanned `packages/` plus `scripts/` with the same pattern: no matches. -- Scanned every file `git ls-files` reports: no matches. +- Scanned every file `git ls-files` reports: the only matches are the Vale style lists that + name the markers (`.vale/styles/proselint/Annotations.yml:6-8`, + `.vale/styles/Google/Acronyms.yml:56`) and docs pages that discuss them, this one included. - No `@deprecated` decorator and no `# DEPRECATED` banner sits on any first-party symbol. - Every `deprecat*` hit in the tree describes an UPSTREAM deprecation: `db.table_names()` at + Every `deprecat*` hit in code and config names an UPSTREAM deprecation or is data: + `db.table_names()` at `packages/atif-embed/src/atif_embed/infrastructure/lance_store.py:90`, a lancedb kwarg family at - `packages/atif-embed/src/atif_embed/infrastructure/lance_store.py:274` and an osv-scanner - flag at `mise.toml:362`. -- The closest thing to a marker is `RELEASING.md:215`, which names the harbor dependency - subtree "the standing follow-up" in prose: a declined-scope note in the release record - rather than a comment in code. It is register row 3. + `packages/atif-embed/src/atif_embed/infrastructure/lance_store.py:273`, an osv-scanner + flag at `mise.toml:431-432`, short SPDX ids at `.github/workflows/dependency-review.yml:67`, + model retirement dates in the bundled price table, and example text in an analytics prompt + at `packages/atif-analytics/src/atif_analytics/application/prompts.py:260`. +- The closest thing to a marker is a comment at `scripts/atif-sql-refresh.sh:86-89`, which + calls leaving embedding out of the Codex pass a deliberate deferral and semantic search over + that corpus a follow-up. It names a scope choice rather than a defect, so it has no register + row. The absence is enforced, not incidental. `select = ["ALL"]` at `pyproject.toml:209` leaves flake8-fixme and flake8-todos on, and neither appears among the ignores at `pyproject.toml:216-247`. -Probed 2026-08-28 by dropping one `# TODO: ...` comment into `packages/atif-duck/src/`: -`ruff check` exits 2 with `TD002` (missing author), `TD003` (missing issue link), and -`FIX002` (line contains `TODO`). A marker cannot reach `main` past `mise run lint`, so the +Probed 2026-10-12 by piping a module with one `# TODO: ...` comment to `ruff check` under a +`packages/atif-duck/src/` filename: it exits 1 with `TD002` (missing author), `TD003` (missing +issue link), and `FIX002` (line contains `TODO`). A marker cannot reach `main` past `mise run lint`, so the debt lives in the channels the register draws on instead of in markers. One more channel exists and contributes no row: a comment explaining the code by reference to an implementation no installer of this package can obtain. `grep -riE 'claude-sql|predecessor' -packages/*/src` returns nothing. No lint can police that one, so it is empty by review +packages/*/src` returns one line, +`packages/atif-converter/src/atif_converter/domain/codex_edges.py:30`, where the word names a +record's previous record in a rollout log. No lint can police that one, so it is empty by review rather than by gate, which is why it is worth naming here. ## Pattern-level smells @@ -115,24 +119,38 @@ Shows up in: Cost: L. The analysis is done and written; the work it defers is a `TypedDict` model of the JSONL record, stub distributions, and refactors of named functions. -### One private upstream method carried the whole product (retired) +### The converter is a parity port, so it keeps upstream's losses on purpose -`atif-converter` used to exist to call `ClaudeCode._convert_events_to_trajectory` and its Codex -sibling, both private harbor methods. The port of 2026-09-11 moved the conversion into this -package (`packages/atif-converter/src/atif_converter/domain/claude_code_conversion.py:75`, `packages/atif-converter/src/atif_converter/domain/codex_conversion.py:781`), built on the public ATIF data classes, with harbor's private -converters kept as a test-only parity oracle (`packages/atif-converter/tests/harbor_oracle.py:94`) frozen to goldens and diffed over the -live corpus. Most of the consequences are gone with it: the minor-version ceiling, the -staging layer that reconstructed harbor's directory shape, and the "gaps" that only described -harbor's file discovery. What stays is by choice: the data-loss gaps are still typed as enum -members because the port is a PARITY port, and fixing one now means deciding to diverge from the -oracle. The remaining consequence, the dependency arriving with many other distributions, stands -until harbor's models and validator are available without the rest of it. +`atif-converter` converts Claude Code and Codex transcripts itself, on the public ATIF data +classes vendored under `atif_converter.domain.atif`. harbor's private converters survive only in +the tests, as the parity oracle the port is measured against, live while harbor still ships +them and frozen to goldens after that. The data-loss gaps stay typed as enum members because +the port is a PARITY port: fixing one means deciding to diverge from the oracle. The live +oracle also rests on private upstream methods, so the day harbor drops them, every test that +needs one skips and the frozen goldens are the only reference left. + +Shows up in: + +- `packages/atif-converter/src/atif_converter/domain/claude_code_conversion.py:981` and + `packages/atif-converter/src/atif_converter/domain/codex_conversion.py:924`: the two + conversion entry points, one per agent. +- `packages/atif-converter/tests/harbor_oracle.py:118` and + `packages/atif-converter/tests/harbor_oracle.py:358`: the live oracle call and the diff the + port is held to. +- `packages/atif-converter/tests/harbor_oracle.py:83`: the skip that takes over when the + private method is gone. +- `packages/atif-converter/src/atif_converter/domain/fidelity.py:55-62`: the enum docstring + that names the gaps as kept by choice. + +Cost: L. It's the work of register row 1: each gap closed is a deliberate divergence from the +oracle that has to be decided and pinned on its own. ### The independence contract is paid for in copies, and the copies are pinned by text The independent packages may never import each other, so shared logic is duplicated instead of -extracted. Byte-identical SQL-literal escapers exist, one per package that inlines -SQL, and the same pure provider/dimension guard exists as twins. The escapers +extracted. SQL-literal escapers with identical bodies exist in atif-duck and atif-embed, the +composition root carries a third, and the same pure provider/dimension guard exists as twins. +The escapers in the two packages are the contract's price paid honestly: each package owns one copy, in `domain/`, and no import would be legal. The guard is the price paid dangerously. The mechanism keeping the twins honest is a test that reads the other package's file as text, parses it with `ast`, @@ -142,52 +160,55 @@ disagree on their logic, and it breaks on a rename that changes nothing. Shows up in: -- `packages/atif-duck/src/atif_duck/domain/sql_literal.py:18` and - `packages/atif-embed/src/atif_embed/domain/sql_literal.py:24`: the domain-layer escapers, +- `packages/atif-duck/src/atif_duck/domain/sql_literal.py:31` and + `packages/atif-embed/src/atif_embed/domain/sql_literal.py:30`: the domain-layer escapers, each in its own package's `domain/`, each reached by that package's adapters. -- `packages/atif-cli/src/atif_cli/app.py:106`: the composition root's copy, which +- `packages/atif-cli/src/atif_cli/app.py:134`: the composition root's copy, which could legally import either of those above and does not. - `packages/atif-embed/tests/test_guard_twin_pin.py:93-95`: the substring assertions that stand in for a shared import. - `packages/atif-duck/src/atif_duck/domain/embedding_guard.py:5-16`: the twin whose - docstring has already drifted from its counterpart's. + docstring differs from its counterpart's. Cost: M. Collapsing the guard twins needs a decision about a shared-kernel package the contract forbids. The escapers need no decision: a shared kernel would absorb them, and short of one, one copy per package is the contract working rather than failing. -### Findings never fail anything +### The scanner tier never fails on a finding -Every security scanner that runs has its findings exit code swallowed (most +Every scanner in the `security` tier has its findings exit code swallowed (most sites use a trailing `|| true`, the rest an equivalent flag) under an explicit contract: a finding does not fail the task, a missing report does. The reasoning is sound, because a -non-zero exit kills the CI step before the SARIF upload. The consequence is that no gate -inside this repository fails on a new -vulnerability, a new secret, or a new SAST finding, and enforcement lives entirely in -GitHub code-scanning alerts and branch protection. The same asymmetry appears in the +non-zero exit kills the CI step before the SARIF upload. The consequence is that a new secret, +a new SAST finding, or a new advisory against a locked dependency fails no gate inside this +repository, and enforcement for those lives in GitHub code-scanning alerts and branch +protection. Two gates do fail: dependency review fails a pull request whose dependency changes +carry a high or critical advisory, and `security:vex:check` fails `mise run check` when the +OpenVEX suppression ledger drifts from the lockfile. The same asymmetry appears in the coverage floor: `fail_under = 89` exists, and the task that reads it is not among the gates `mise run check` depends on, so the declared definition of done does not measure coverage. -The suppression ledger this tier generates is currently empty, which reads identically to -"nothing needed suppressing" and to "nobody has looked". +bandit's `B608` findings never reach code scanning at all, because every SQL site carries +`# nosec B608`, so the SQL review lives in per-line comments rather than in any scanner's +output. Shows up in: -- `mise.toml:217-223`: the contract, stated plainly. -- `mise.toml:366` and `mise.toml:482`: swallow sites, one of them - bandit's, whose B608 findings therefore reach code scanning permanently. -- `mise.toml:197-212`: the gates, without `test:cov`. -- `pyproject.toml:648`: the floor that only `.github/workflows/check.yml:78` enforces. +- `mise.toml:270-276`: the contract, stated plainly. +- `mise.toml:443` and `mise.toml:561`: swallow sites, the second one bandit's. +- `.github/workflows/dependency-review.yml:49`: the severity at which a pull request fails. +- `mise.toml:245-265`: the gates, with `security:vex:check` and without `test:cov`. +- `pyproject.toml:648`: the floor that only `.github/workflows/check.yml:90` enforces. Cost: M. Splitting each scanner into "run and upload" plus "assert the finding delta against a baseline" is mechanical; agreeing on the baseline is the work. -### The code the type checkers guard hardest is the code the tests reach least (clustering half retired) +### The end-to-end materialize suite runs against a double -Coverage misses used to concentrate in the numeric clustering path: the clustering module and -the Lance reader. That path was deleted on 2026-09-27 with the cluster, terms, and community -pipelines. The rest of the finding stands: the end-to-end -materialize suite runs against a fake converter that ships in the wheel, so the state -transitions that matter most are exercised against a double. +The atif-corpus end-to-end materialize suite runs against a fake converter that ships in the +wheel and fabricates its output from the session id, so the state transitions that matter +most are exercised against a double. The tests that push transcripts through the real +converter are the CLI integration tier, over two small synthetic sessions, and the opt-in live +parity tests, which read local transcripts and skip wherever there are none, CI included. Shows up in: @@ -195,9 +216,12 @@ Shows up in: the materialize suite uses instead of the real adapter. - `packages/atif-cli/tests/test_integration.py:3-9`: the whole integration tier: a few small synthetic sessions. +- `packages/atif-converter/tests/test_parity_live.py:3-9`: the live parity tests, and where + they skip. Cost: M. A corpus fixture built from real, scrubbed `~/.claude` records is the whole of -it, and it is what would exercise the fidelity gaps nothing real-world currently touches. +it, and it's what would put the fidelity gaps in front of CI, where no real transcript +reaches them. ## See also diff --git a/docs/reference/cli.md b/docs/reference/cli.md index 6fc49fb..73df439 100644 --- a/docs/reference/cli.md +++ b/docs/reference/cli.md @@ -1,12 +1,12 @@ # atif-sql · CLI -The `atif-sql` console script (`packages/atif-cli/pyproject.toml:42`) is the repo's public contract: one cyclopts router (`packages/atif-cli/src/atif_cli/app.py:52`) dispatches the subcommands below, and a root-level `--version` resolves the installed `atif-sql` distribution (`:60`). +The `atif-sql` console script (`pyproject.toml:75`) is the repo's public contract: one cyclopts router (`packages/atif-cli/src/atif_cli/app.py:90`) dispatches the subcommands below, and a root-level `--version` resolves the installed `atif-sql` distribution (`:99`). -These conventions hold across every subcommand. `--format` takes `auto`, `table`, `json`, or `csv` from the `OutputFormat` `StrEnum` at `packages/atif-cli/src/atif_cli/output.py:58`, and `auto` resolves to `table` when stdout is a TTY and `json` otherwise (`packages/atif-cli/src/atif_cli/output.py:74`). Exit codes come from one table, `EXIT_CODES` at `packages/atif-cli/src/atif_cli/errors.py:28`: `0` ok, `2` empty-session / no-embeddings, `64` invalid input or SQL parse error, `65` catalog error / validation error / embedding mismatch, `70` runtime error, `78` terminal state or suspicious scan, `127` harbor missing. A number in that table is a wire contract: keys may be added, never renumbered. A usage error cyclopts refuses (an unknown option or command, a missing or uncoercible value) prints cyclopts' error panel and exits `64`, never cyclopts' own `2`. And stderr carries WARNING and above only, so a piped read emits data on stdout and nothing on stderr unless something is wrong; `ATIF_SQL_LOG_LEVEL` widens it (`INFO`, `DEBUG`) and is the only way to reach the INFO surface, because the entry point replaces loguru's default handler and with it the `LOGURU_LEVEL` that would have parameterized it (`packages/atif-cli/src/atif_cli/app.py:1141-1148`). +These conventions hold across every subcommand. `--format` takes `auto`, `table`, `json`, or `csv` from the `OutputFormat` `StrEnum` at `packages/atif-cli/src/atif_cli/output.py:61`, and `auto` resolves to `table` when stdout is a TTY and `json` otherwise (`packages/atif-cli/src/atif_cli/output.py:84`). Exit codes come from one table, `EXIT_CODES` at `packages/atif-cli/src/atif_cli/errors.py:30`: `0` ok, `2` empty-session / no-embeddings, `64` invalid input or SQL parse error, `65` catalog error / validation error / embedding mismatch / lake mismatch, `70` runtime error / sandbox refusal, `77` root refusal, `78` terminal state / suspicious scan / extension missing / lake unavailable, `127` harbor missing (kept so the table never renumbers; no code path raises it). A number in that table is a wire contract: keys may be added, never renumbered. A usage error cyclopts refuses (an unknown option or command, a missing or uncoercible value) prints cyclopts' error panel and exits `64`, never cyclopts' own `2`. And stderr carries WARNING and above only, so a piped read emits data on stdout and nothing on stderr unless something is wrong; `ATIF_SQL_LOG_LEVEL` widens it (`INFO`, `DEBUG`) and is the only way to reach the INFO surface, because the entry point replaces loguru's default handler and with it the `LOGURU_LEVEL` that would have parameterized it (`packages/atif-cli/src/atif_cli/app.py:2390-2397`). -`analyze`, `embed`, and `search` call Amazon Bedrock and spend money; the others are offline. +`analyze` calls Amazon Bedrock and spends money, and so do `embed` and `search` under the default `cohere` embedding provider (the `gemma` provider runs on this machine); the others make no model calls. -Some subcommands take `--agent`, which selects the transcript format and, with it, the default source and corpus roots: `convert`, `materialize`, `status`, and `query`. The accepted spellings are `claude-code` (the default) and `codex`, and an unknown one exits `64` naming both (`packages/atif-cli/src/atif_cli/app.py:257`). `codex` moves the source root to `$CODEX_HOME` (default `~/.codex`) `/sessions` and the corpus root to `~/.atif-sql/corpus/codex`; an explicit flag or `ATIF_SQL_*` env var still wins (`packages/atif-corpus/src/atif_corpus/infrastructure/settings.py`). +Some subcommands take `--agent`, which selects the transcript format and, with it, the default source and corpus roots: `convert`, `materialize`, `status`, and `query`. The accepted spellings are `claude-code` (the default) and `codex`, and an unknown one exits `64` naming both (`packages/atif-cli/src/atif_cli/app.py:663-666`). `codex` moves the source root to `$CODEX_HOME` (default `~/.codex`) `/sessions` and the corpus root to `~/.atif-sql/corpus/codex`; an explicit flag or `ATIF_SQL_*` env var still wins (`packages/atif-corpus/src/atif_corpus/infrastructure/settings.py`). ## convert @@ -15,16 +15,16 @@ atif-sql convert [OPTIONS] SESSION-JSONL ``` Convert one Claude Code session JSONL or one Codex CLI rollout JSONL to ATIF plus a loss report and edges. -`packages/atif-cli/src/atif_cli/app.py:282` +`packages/atif-cli/src/atif_cli/app.py:675` Flags: -- `SESSION-JSONL` / `--session-jsonl`: required path to the transcript, `~/.claude/projects//.jsonl`, or `~/.codex/sessions///
/rollout--.jsonl` under `--agent codex`. `:283` -- `--agent`: `claude-code` or `codex`; picks the converter and the fidelity policy. Defaults from `ATIF_SQL_AGENT`, then to `claude-code`. `:284` -- `--include-subagents` / `--no-subagents`: stage `/subagents/**.jsonl` side-files alongside the main chain; default `True`, with the negative form named explicitly. `:228` -- `--trajectory-out`: write the trajectory JSON here and `edges.jsonl` beside it, instead of stdout. `:229` +- `SESSION-JSONL` / `--session-jsonl`: required path to the transcript, `~/.claude/projects//.jsonl`, or `~/.codex/sessions///
/rollout--.jsonl` under `--agent codex`. `:676` +- `--agent`: `claude-code` or `codex`; picks the converter and the fidelity policy. Defaults from `ATIF_SQL_AGENT`, then to `claude-code`. `:678` +- `--include-subagents` / `--no-subagents`: stage `/subagents/**.jsonl` side-files alongside the main chain; default `True`, with the negative form named explicitly. `:679` +- `--trajectory-out`: write the trajectory JSON here and `edges.jsonl` beside it, instead of stdout. `:680` -Exit codes: `0` ok, `2` empty session, `64` invalid input, `65` validation, `70` conversion. `:250` +Exit codes: `0` ok, `2` empty session, `64` invalid input, `65` validation, `70` conversion. `:709-710` ## materialize @@ -33,31 +33,29 @@ atif-sql materialize [OPTIONS] ``` Sync the materialized corpus with the raw transcript corpus in one scan-plan-convert-write pass. -`packages/atif-cli/src/atif_cli/app.py:448` +`packages/atif-cli/src/atif_cli/app.py:863` A transcript whose session id fails the boundary in `packages/atif-corpus/src/atif_corpus/domain/session_id.py` (`^[A-Za-z0-9][A-Za-z0-9._-]*$`, at most 255 characters) is skipped with a logged reason and counted: the report carries `rejected` and `rejected_session_ids` beside `unreadable` and `unreadable_session_ids`, and the table form prints one `REJECTED` line per name on stderr. Nothing from such a session is written, and a corpus directory an older version wrote under that name is kept and never marked. -Nothing is ever deleted. A session whose source transcript vanished keeps its artifacts, and its `meta.json` gets `source_present: false` and `source_removed_at` once; the report's `removed` / `removed_session_ids` are the sessions newly marked this pass and `retained` counts every session kept without a source. Every live conversion also writes a zstd archive of the session's raw source files to `sessions//source/` (listed in `meta.source_archive`), from the bytes the converter parsed. A session is stale when its sources moved OR when its recorded `converter_schema` (the converter's `CONVERTER_SCHEMA_VERSION`; and, on a columnar pass, `columnar_schema`) differs from the running one, and `meta.converter_version` is now the real `atif-sql` release instead of `"unknown"`; a stale source-removed session with an archive is re-converted from it and counted in `from_archive` / `from_archive_session_ids`. A transcript with nothing to convert is counted in `empty` / `empty_session_ids`, recorded in `empty_sessions.json`, and not retried until it's written to or the converter schema changes, so `failed` only counts real failures. See `docs/CONTRACT.md`. +Nothing is ever deleted. A session whose source transcript vanished keeps its artifacts, and its `meta.json` gets `source_present: false` and `source_removed_at` once; the report's `removed` / `removed_session_ids` are the sessions newly marked this pass and `retained` counts every session kept without a source. Every live conversion also writes a zstd archive of the session's raw source files to `sessions//source/` (listed in `meta.source_archive`), from the bytes the converter parsed. A session is stale when its sources moved OR when its recorded `converter_schema` (the converter's `CONVERTER_SCHEMA_VERSION`) differs from the running one, and `meta.converter_version` records the `atif-sql` release as provenance only; a stale source-removed session with an archive is re-converted from it and counted in `from_archive` / `from_archive_session_ids`. A transcript with nothing to convert is counted in `empty` / `empty_session_ids`, recorded in `empty_sessions.json`, and not retried until it's written to or the converter schema changes, so `failed` only counts real failures. See `docs/CONTRACT.md`. -The convert-write stage runs across a process pool by default. Each worker builds its own converter once and writes through the same per-session staging directory and atomic swap the single-process path uses, so the artifacts are byte-identical either way and a crash still costs at most the session in flight. `--workers 1` is the single-process reference path. The report's `convert_seconds` is the per-session sum, so with several workers it can exceed `total_seconds`, which stays the wall clock; `workers` in the report is the pool size the pass actually used, which is never more than the number of sessions planned (`packages/atif-corpus/src/atif_corpus/application/materialize.py:385`). +The convert-write stage runs across a process pool by default. Each worker builds its own converter once and writes through the same per-session staging directory and atomic swap the single-process path uses, so the artifacts are byte-identical either way and a crash still costs at most the session in flight. `--workers 1` is the single-process reference path. The report's `convert_seconds` is the per-session sum, so with several workers it can exceed `total_seconds`, which stays the wall clock; `workers` in the report is the pool size the pass actually used, which is never more than the number of sessions planned (`packages/atif-corpus/src/atif_corpus/application/materialize.py:244-247`). Flags: -- `--agent`: `claude-code` (default) or `codex`; picks the discovery layout and the default roots. `:426` -- `--force` / `--no-force`: re-materialize every quiescent session regardless of the watermark; default `False`. `:341` -- `--quiesce-seconds`: source-silence threshold; defaults from settings (contract: 300). `:342` -- `--source-root`: override the raw transcript root, otherwise `ATIF_SQL_SOURCE_ROOT` or `/projects`. `:343` -- `--corpus-root`: override the materialized corpus root, otherwise env or `~/.atif-sql/corpus/`. `:344` -- `--sessions`: comma-separated session-id filter; only these sessions are planned this pass. `:345` -- `--workers`: processes for the convert-write stage; default `ATIF_SQL_MATERIALIZE_WORKERS`, else `min(8, cpu_count)`. `1` is the single-process path. `:455` -- `--format`: report format. `:346` - -Exit codes: `0` ok, `64` `--workers` below `1`; `78` the corpus at this root holds the other agent's sessions, refused with nothing touched; `78` suspicious scan: the source scan found zero sessions while the corpus holds materialized ones, so the pass refused to mark them source-removed and touched nothing. Check `--source-root`; a retry over the same root cannot succeed. `packages/atif-cli/src/atif_cli/app.py:429-443` -- `--format`: report format. `:346` +- `--agent`: `claude-code` (default) or `codex`; picks the discovery layout and the default roots. `packages/atif-cli/src/atif_cli/app.py:865` +- `--force` / `--no-force`: re-materialize every quiescent session regardless of the watermark; default `False`. `:866` +- `--quiesce-seconds`: source-silence threshold; defaults from settings (contract: 300). `:867` +- `--source-root`: override the raw transcript root, otherwise `ATIF_SQL_SOURCE_ROOT` or `/projects` (`/sessions` under `--agent codex`). `:868` +- `--corpus-root`: override the materialized corpus root, otherwise env or `~/.atif-sql/corpus/`. `:869` +- `--sessions`: comma-separated session-id filter; only these sessions are planned this pass. `:870` +- `--workers`: processes for the convert-write stage; default `ATIF_SQL_MATERIALIZE_WORKERS`, else `min(8, cpu_count)`. `1` is the single-process path. `:871` +- `--lake` / `--no-lake`: after the sessions are swapped into place, replace their rows in the lake at `ATIF_SQL_LAKE_ROOT` (default), or skip the lake. It does nothing until `atif-sql lake rebuild` has created the lake, and a failed lake write leaves the sessions in `sink_pending.json` for the next pass to retry. `packages/atif-cli/src/atif_cli/app.py:872` +- `--format`: report format. `:873` Each session is written as `trajectory.json.zst`, `edges.jsonl.zst` and `session_events.jsonl.zst` (zstd; decompressed, each is the plain file an earlier version wrote), a plain `loss_report.json` and `meta.json`, and the raw source archive. No per-session parquet is written, so the report's `artifact_seconds` is `0.0`. A corpus written by an earlier version keeps its plain files and parquet until `atif-sql corpus slim` converts it (see below). -Exit codes: `0` ok, `78` the corpus at this root holds the other agent's sessions, refused with nothing touched; `78` suspicious scan: the source scan found zero sessions while the corpus holds materialized ones, so the pass refused to mark them source-removed and touched nothing. Check `--source-root`; a retry over the same root cannot succeed. `packages/atif-cli/src/atif_cli/app.py:429-443` +Exit codes: `0` ok, `64` `--workers` below `1` (`packages/atif-cli/src/atif_cli/app.py:946-956`); `78` the corpus at this root holds the other agent's sessions, refused with nothing touched; `78` suspicious scan: the source scan found zero sessions while the corpus holds materialized ones, so the pass refused to mark them source-removed and touched nothing. Check `--source-root`; a retry over the same root cannot succeed. `:998-1026` ## status @@ -66,21 +64,21 @@ atif-sql status [OPTIONS] ``` Report corpus freshness: watermark age, counts, bytes, staleness. -`packages/atif-cli/src/atif_cli/app.py:444` +`packages/atif-cli/src/atif_cli/app.py:1147` -It replays the planning a default `materialize` would do, so `staleness.stale` includes sessions a converter change made stale (`staleness.generation_stale` counts those alone) and `staleness.from_archive` counts source-removed sessions the next pass would re-convert from their archive. `retained_sessions` counts sessions kept without a source, `empty_sessions` the transcripts recorded as empty, and `converter_schema` / `columnar_schema` are the running versions a current session must carry. +It replays the planning a default `materialize` would do, so `staleness.stale` includes sessions a converter change made stale (`staleness.generation_stale` counts those alone) and `staleness.from_archive` counts source-removed sessions the next pass would re-convert from their archive. `retained_sessions` counts sessions kept without a source and `empty_sessions` the transcripts recorded as empty. `converter_schema` is the converter version a current session must carry, and `columnar_schema` is the version a session's per-session parquet must carry for `query` to read it on the columnar path. Flags: -- `--agent`: `claude-code` (default) or `codex`; the scan and the reported `agent` field follow it. `:529` -- `--source-root`: override the raw transcript root. `:414` -- `--corpus-root`: override the materialized corpus root. `:415` -- `--quiesce-seconds`: source-silence threshold used to replay `materialize`'s planning decision. `:416` -- `--format`: report format. `:417` +- `--agent`: `claude-code` (default) or `codex`; the scan and the reported `agent` field follow it. `:1149` +- `--source-root`: override the raw transcript root. `:1150` +- `--corpus-root`: override the materialized corpus root. `:1151` +- `--quiesce-seconds`: source-silence threshold used to replay `materialize`'s planning decision. `:1152` +- `--format`: report format. `:1153` -Both roots resolve through `_corpus_settings` (`:433`), so a `--source-root` given without `--corpus-root` re-derives the corpus root from the overridden source's slug unless `ATIF_SQL_CORPUS_ROOT` is set (`:206`). +Both roots resolve through `_corpus_settings` (`:594`), so a `--source-root` given without `--corpus-root` re-derives the corpus root from the overridden source's slug unless `ATIF_SQL_CORPUS_ROOT` is set (`:624`). -The report also says how the corpus is stored and how `query` reads it without the lake. `layout` counts the complete sessions still in the old layout (plain JSON files or per-session parquet) and their bytes; the table form prints one line, and `atif-sql corpus slim` converts them. `query path` is `columnar` when every complete session still carries current parquet, `json` when none does (every session written or slimmed since materialize stopped writing parquet), and `mixed` in between. `status` applies the same per-session predicate the registry does (`meta.columnar_schema` current and all five files present and non-empty). `packages/atif-cli/src/atif_cli/app.py:639` +The report also says how the corpus is stored and how `query` reads it without the lake. `layout` counts the complete sessions still in the old layout (plain JSON files or per-session parquet) and their bytes; the table form prints one line, and `atif-sql corpus slim` converts them. `query path` is `columnar` when every complete session still carries current parquet, `json` when none does (a corpus that materialize wrote or `corpus slim` converted), `mixed` in between, and `empty` when there's no complete session. `status` applies the same per-session predicate the registry does (`meta.columnar_schema` current and all five files present and longer than a parquet header). `packages/atif-cli/src/atif_cli/app.py:1238-1239` ## query @@ -89,21 +87,21 @@ atif-sql query [OPTIONS] [ARGS] ``` Run one SQL statement against the atif-duck catalog and emit results. -`packages/atif-cli/src/atif_cli/app.py:529` +`packages/atif-cli/src/atif_cli/app.py:1305` Flags: -- `SQL`: positional-only statement; omitting it without `--examples` is a parse error. `:498` -- `--examples`: short-circuit to the `examples` listing, honoring `--category` and `--requires`, without opening DuckDB. `:501` -- `--category`: forwarded to the `examples` listing. `:502` -- `--requires`: forwarded to the `examples` listing. `:503` -- `--agent`: `claude-code` (default) or `codex`; selects which corpus the statement reads. `:628` -- `--corpus-root`: override the materialized corpus root. `:504` -- `--all-corpora`: scope every view to every corpus the lake holds; `sessions.corpus` says which one a row came from. It has no per-session fallback, so it exits `78` without a usable lake and `64` together with `--no-lake`. -- `--lake` / `--no-lake`: read the lake when it holds the corpus (default), or read the per-session files. -- `--format`: `table` on a TTY, a JSON array of row objects on a pipe. `:505` +- `SQL`: positional-only statement; omitting it without `--examples` is a parse error. `:1306` +- `--examples`: short-circuit to the `examples` listing, honoring `--category` and `--requires`, without opening DuckDB. `:1309` +- `--category`: forwarded to the `examples` listing. `:1310` +- `--requires`: forwarded to the `examples` listing. `:1311` +- `--agent`: `claude-code` (default) or `codex`; selects which corpus the statement reads. `:1312` +- `--corpus-root`: override the materialized corpus root. `:1313` +- `--all-corpora`: scope every view to every corpus the lake holds; `sessions.corpus` says which one a row came from. It has no per-session fallback, so it exits `78` without a usable lake and `64` together with `--no-lake`. `:1314` +- `--lake` / `--no-lake`: read the lake when it holds the corpus (default), or read the per-session files. `:1315` +- `--format`: `table` on a TTY, a JSON array of row objects on a pipe. `:1316` -The statement runs against a hardened connection: reads reach the registered views and nothing else, and nothing under the corpus root is writable. Before registration the connection is sized to the host: a memory cap derived from available RAM (half of physical RAM or 8 GiB, whichever is larger, never above 80% of what's available) and a thread count of one per 2 GiB of that cap, capped at the CPUs the process may use. `ATIF_SQL_QUERY_MEMORY_LIMIT` (a DuckDB size such as `6GB`) and `ATIF_SQL_QUERY_THREADS` override both; a malformed value exits 64. The spill directory is a private `mkdtemp` (mode 0700) under the system temp dir, the only directory the sandbox grants, and it's removed when the process exits. Extension auto-install and auto-load are off, and the lance extension is loaded only when it's already installed, so registration never reaches the network (`packages/atif-cli/src/atif_cli/app.py`, `_configure_query_resources`). +The statement runs against a hardened connection: reads reach the registered views and nothing else, and nothing under the corpus root is writable. Before registration the connection is sized to the host: a memory cap derived from available RAM (half of physical RAM or 8 GiB, whichever is larger, never above 80% of physical or available RAM, and never below 512 MiB) and a thread count of one per 2 GiB of that cap, capped at the CPUs the process may use. `ATIF_SQL_QUERY_MEMORY_LIMIT` (a DuckDB size such as `6GB`) and `ATIF_SQL_QUERY_THREADS` override both; a malformed value exits 64. The spill directory is a private `mkdtemp` (mode 0700) under the system temp dir, the only directory the sandbox grants, and it's removed when the process exits. Extension auto-install and auto-load are off, and the lance extension is loaded only when it's already installed, so registration never reaches the network (`packages/atif-cli/src/atif_cli/app.py`, `_configure_query_resources`). Layered guards keep caller SQL from writing the corpus. DuckDB's file grants are read-write and it has no read-only grant, so `COPY ... TO (USE_TMP_FILE false)` would overwrite one; the CLI therefore refuses every statement kind that names a file before executing anything, using DuckDB's own parser: `COPY`, `EXPORT`, `ATTACH`, `DETACH`, `INSTALL`, `LOAD`, `PREPARE` and `EXECUTE` exit 70 with kind `sandbox_refused`, for any uid, and a batch containing one of them runs nothing. And because a `0444` file mode doesn't bind root, `query` refuses to run as uid 0 (exit 77, kind `root_refused`) unless `ATIF_SQL_ALLOW_ROOT=1` is set, which logs a warning. `search` and `analyze` refuse root the same way. @@ -111,9 +109,9 @@ What caller SQL can still see: `duckdb_settings()` and `current_setting(...)` re The views themselves carry no corpus path as statement text. The registry hands its globs and file lists to `read_json(?)` as bound parameters and builds the parquet readers through DuckDB's relation API, so a corpus root such as `o'brien ?; --$1` and transcript content carrying SQL text both register as data (`packages/atif-duck/src/atif_duck/infrastructure/registry.py`). A session directory whose name fails the session id boundary (`packages/atif-duck/src/atif_duck/domain/session_id.py`) registers nothing and is logged once. -Without the lake, an old-layout session that still carries current parquet is served from it, and every other session is parsed from its stored trajectory (`trajectory.json.zst`, decompressed by DuckDB's JSON reader) on each run. That makes the fallback slow on a large corpus, and it holds each document it reads in memory, so a full scan of `tool_calls` or `tool_results` over a large corpus can run out of query memory. That error's hint says to run `atif-sql lake rebuild` (or raise `ATIF_SQL_QUERY_MEMORY_LIMIT`), because the lake answers the same query in bounded memory. The fallback stays for a corpus the lake doesn't hold yet, a lake whose schema is stale, and small corpora. The per-session parquet files the registry bound are granted to the sandbox the same way the analytics parquets are (as individual `allowed_paths` entries, `packages/atif-cli/src/atif_cli/app.py:221`), and they're written read-only (`0444`), so a `COPY ... TO` at one of them fails at the filesystem even though DuckDB's grant is read-write. `atif-sql status` says which path a corpus takes. +Without the lake, an old-layout session that still carries current parquet is served from it, and every other session is parsed from its stored trajectory (`trajectory.json.zst`, decompressed by DuckDB's JSON reader) on each run. That makes the fallback slow on a large corpus, and it holds each document it reads in memory, so a full scan of `tool_calls` or `tool_results` over a large corpus can run out of query memory. That error's hint says to run `atif-sql lake rebuild` (or raise `ATIF_SQL_QUERY_MEMORY_LIMIT`), because the lake answers the same query in bounded memory. The fallback stays for a corpus the lake doesn't hold yet, a lake whose schema is stale, and small corpora. The per-session parquet files the registry bound are granted to the sandbox the same way the analytics parquets are (as individual `allowed_paths` entries, `packages/atif-cli/src/atif_cli/app.py:583-587`), and they're written read-only (`0444`), so a `COPY ... TO` at one of them fails at the filesystem even though DuckDB's grant is read-write. `atif-sql status` says which path a corpus takes. -Exit codes: `64` parse error or a malformed `ATIF_SQL_QUERY_*` override, `65` catalog error, `65` embedding mismatch, `70` runtime error, `70` `sandbox_refused` (a statement kind the sandbox never runs), `77` `root_refused`. `:550` +Exit codes: `64` parse error, a malformed `ATIF_SQL_QUERY_*` override, or `--all-corpora` with `--no-lake`, `65` catalog error, `65` embedding mismatch, `70` runtime error, `70` `sandbox_refused` (a statement kind the sandbox never runs), `77` `root_refused`, `78` `lake_unavailable` (`--all-corpora` without a usable lake). `:1400-1411` ## analyze @@ -122,7 +120,7 @@ atif-sql analyze [OPTIONS] ``` Run the LLM analytics pipelines: classify, conflicts, friction, and perceived. -`packages/atif-cli/src/atif_cli/app.py:1240` +`packages/atif-cli/src/atif_cli/app.py:1634` The stages run on the model family `ATIF_SQL_LLM_FAMILY` names: `openai` (GPT-6, the default) or `anthropic` (Claude 5.5). Transcripts carry tool results as size stubs unless @@ -130,21 +128,21 @@ The stages run on the model family `ATIF_SQL_LLM_FAMILY` names: `openai` (GPT-6, Flags: -- `--since-days`: restrict the stages to sessions whose last step is within N days; default `30`. `:1242` -- `--limit`: cap the number of sessions, newest-first, per stage. `:1243` -- `--max-sessions`: hard per-run session ceiling per LLM pipeline; overrides `ATIF_SQL_LLM_MAX_SESSIONS_PER_RUN`, default 50. `:1244` -- `--max-cost-usd`: hard per-run dollar ceiling across all LLM pipelines, checked against running actual usage; overrides `ATIF_SQL_LLM_MAX_COST_USD_PER_RUN`, default 25.0. `:1245` -- `--no-dry-run`: execute the LLM stages for real, which costs money; the default is a dry run emitting plan dicts and cost estimates. `:1246` -- `--llm-only`: accepted and ignored. Every stage is an LLM stage now, and the flag stays so old crontab lines keep parsing. `:1247` -- `--skip-classify`: opt out of the classify stage. `:1248` -- `--skip-conflicts`: opt out of the conflicts stage. `:1249` -- `--skip-friction`: opt out of the friction stage. `:1250` -- `--skip-perceived`: opt out of the perceived stage. `:1251` -- `--corpus-root`: override the materialized corpus root. `:1252` -- `--no-lake`: read session data from the per-session files rather than the lake. `:1534` -- `--format`: summary format. `:1253` - -Session data comes from the lake when it holds the corpus; a session whose lake rows aren't its current artifacts is read from its files, and with no usable lake the command warns once and reads the files. The summary's `session_source` says which source ran (`lake` or `files`), and on the lake `sessions_read_from_files` counts the sessions the last stage read from files. If a lake read fails partway (a `lake rebuild` swapped the lake out from under the run), the command warns once, reads the files for the rest of the run, and reports `lake_read_failed: true`. The plans, the rendered transcripts and the checkpoint bounds are the same whichever source runs. Outputs land under the corpus's `analytics/` directory either way. `:1624` +- `--since-days`: restrict the stages to sessions whose last step is within N days; default `30`. `:1636` +- `--limit`: cap the number of sessions, newest-first, per stage. `:1637` +- `--max-sessions`: hard per-run session ceiling per LLM pipeline; overrides `ATIF_SQL_LLM_MAX_SESSIONS_PER_RUN`, default 50. `:1638` +- `--max-cost-usd`: hard per-run dollar ceiling across all LLM pipelines, checked against running actual usage; overrides `ATIF_SQL_LLM_MAX_COST_USD_PER_RUN`, default 25.0. `:1639` +- `--no-dry-run`: execute the LLM stages for real, which costs money; the default is a dry run emitting plan dicts and cost estimates. `:1640` +- `--llm-only`: accepted and ignored. Every stage is an LLM stage, and the flag exists so existing crontab lines keep parsing. `:1641` +- `--skip-classify`: opt out of the classify stage. `:1642` +- `--skip-conflicts`: opt out of the conflicts stage. `:1643` +- `--skip-friction`: opt out of the friction stage. `:1644` +- `--skip-perceived`: opt out of the perceived stage. `:1645` +- `--corpus-root`: override the materialized corpus root. `:1646` +- `--lake` / `--no-lake`: read session data from the lake when it holds the corpus (default), or from the per-session files. `:1647` +- `--format`: summary format. `:1648` + +Session data comes from the lake when it holds the corpus; a session whose lake rows aren't its current artifacts is read from its files, and with no usable lake the command warns once and reads the files. The summary's `session_source` says which source ran (`lake` or `files`), and on the lake `sessions_read_from_files` counts the sessions the last stage read from files. If a lake read fails partway (a `lake rebuild` swapped the lake out from under the run), the command warns once, reads the files for the rest of the run, and reports `lake_read_failed: true`. The plans, the rendered transcripts and the checkpoint bounds are the same whichever source runs. Outputs land under the corpus's `analytics/` directory either way. `:1716-1734` classify and conflicts skip non-interactive sessions (`session_outcomes.kind` of `turn_audit` or `one_shot_job`), and friction and perceived read human turns only. The deterministic authorship views (`user_steps`, `human_turns`, `session_outcomes`) are plain views, so they need no `analyze` run. @@ -155,7 +153,7 @@ atif-sql embed [OPTIONS] ``` Embed unembedded corpus steps and append them to LanceDB, with Cohere Embed v4 on Bedrock by default or with EmbeddingGemma 2 on this machine. -`packages/atif-cli/src/atif_cli/app.py:766` +`packages/atif-cli/src/atif_cli/app.py:1801` `ATIF_SQL_EMBED_PROVIDER` picks the embedder: @@ -166,15 +164,18 @@ Each store is stamped with the model and width that wrote it, so a store written Flags: -- `--limit`: cap the number of steps embedded this run. `:736` -- `--all`: explicitly embed every unembedded step, a full backfill. `:737` -- `--dry-run`: preview only; emit the plan JSON with keys `pipeline, discovery, candidates, batches, batch_size, concurrency, provider, model, dim, store, limit, dry_run` and make no embedding calls. `:738` -- `--corpus-root`: override the materialized corpus root. `:739` -- `--format`: output format. `:740` +- `--limit`: cap the number of steps embedded this run. `packages/atif-cli/src/atif_cli/app.py:1803` +- `--all`: explicitly embed every unembedded step, a full backfill. `:1804` +- `--dry-run`: preview only; emit the plan JSON with keys `pipeline, discovery, candidates, batches, batch_size, concurrency, provider, model, dim, store, limit, dry_run` and make no embedding calls. `:1805` +- `--install-extension`: install the lance DuckDB extension and exit. It touches neither Bedrock nor the store. `:1806` +- `--prune-orphans`: count the stored rows whose uuid no lake step carries, and with `--no-dry-run` delete them; it never calls Bedrock. `:1807` +- `--lake` / `--no-lake`: discover the steps to embed from the lake when it holds the corpus (default), or from the per-session files. `:1808` +- `--corpus-root`: override the materialized corpus root. `:1809` +- `--format`: output format. `:1810` -A real run requires an explicit scope: a bare `atif-sql embed` exits `64` with a hint rather than starting an unbounded backfill. `:778` +A real run requires an explicit scope: a bare `atif-sql embed` exits `64` with a hint rather than starting an unbounded backfill. `:1908-1918` -Exit codes: `0` success, `64` missing `--limit` or `--all`, or an `ATIF_SQL_OUTPUT_DIMENSION` the provider can't emit, `70` runtime (Bedrock, local model, DuckDB, or Lance failure: transient, safe to retry), `78` terminal state, where the store or its config needs operator action and unattended lanes suppress retries. The `gemma` provider without the `local` extra exits `78` with the install command in the message. `:767` +Exit codes: `0` success, `64` missing `--limit` or `--all`, or an `ATIF_SQL_OUTPUT_DIMENSION` the provider can't emit, `70` runtime (Bedrock, local model, DuckDB, or Lance failure: transient, safe to retry), `78` terminal state, where the store or its config needs operator action and unattended lanes suppress retries. The `gemma` provider without the `local` extra exits `78` with the install command in the message. `--prune-orphans` exits `78` (`lake_unavailable`) without a usable lake, and refuses to delete while a session of the corpus is pending a lake write. `:1887-1892` ## search @@ -183,21 +184,21 @@ atif-sql search [OPTIONS] QUERY_TEXT ``` Semantic top-k nearest-neighbor search over step embeddings. -`packages/atif-cli/src/atif_cli/app.py:860` +`packages/atif-cli/src/atif_cli/app.py:2018` Flags: -- `QUERY_TEXT`: required positional-only text, embedded by the provider `ATIF_SQL_EMBED_PROVIDER` selects: Cohere Embed v4 in `search_query` mode, or EmbeddingGemma 2 on this machine under its `SearchQuery` prompt. The search reads that provider's store. `:829` -- `-k` / `--k`: top-k; default `10`. This is the CLI's only short flag. `:832` -- `--session-id`: confine the kNN to one session. `:833` -- `--corpus-root`: override the materialized corpus root. `:834` -- `--all-corpora`: search every corpus's embedding store and add a `corpus` column; exits `78` without a usable lake and `64` together with `--no-lake`. -- `--lake` / `--no-lake`: read the lake when it holds the corpus (default), or read the per-session files. -- `--format`: output format. `:835` +- `QUERY_TEXT`: required positional-only text, embedded by the provider `ATIF_SQL_EMBED_PROVIDER` selects: Cohere Embed v4 in `search_query` mode, or EmbeddingGemma 2 on this machine under its `SearchQuery` prompt. The search reads that provider's store. `:2019` +- `-k` / `--k`: top-k; default `10`. Apart from the `-h` help flag cyclopts adds, it's the CLI's only short flag. `:2022` +- `--session-id`: confine the kNN to one session. `:2023` +- `--corpus-root`: override the materialized corpus root. `:2024` +- `--all-corpora`: search every corpus's embedding store and add a `corpus` column; exits `78` without a usable lake and `64` together with `--no-lake`. `:2025` +- `--lake` / `--no-lake`: read the lake when it holds the corpus (default), or read the per-session files. `:2026` +- `--format`: output format. `:2027` -Output columns are `uuid`, `session_id`, `snippet`, and `sim` (`:927`), ranked by cosine distance ascending so the highest similarity comes first (`:935`). +Output columns are `uuid`, `session_id`, `snippet`, and `sim` (`:2188-2191`), plus `corpus` under `--all-corpora`, ranked by cosine distance ascending so the highest similarity comes first (`:2196`). -Exit codes: `0` success, `2` no embeddings yet, `65` embedding mismatch when the store was written by another provider, `70` runtime, including a failed query embedding, `78` the `gemma` provider without the `local` extra. `:864` +Exit codes: `0` success, `2` no embeddings yet, `64` `--all-corpora` together with `--no-lake`, `65` embedding mismatch when the store was written by another provider, `70` runtime, including a failed query embedding, `77` `root_refused`, `78` a store whose lance DuckDB extension isn't installed, `--all-corpora` without a usable lake, or the `gemma` provider without the `local` extra. `:2069-2075` ## examples @@ -206,17 +207,17 @@ atif-sql examples [OPTIONS] ``` List tested example queries for every view and macro, derived from the static atif-duck catalog. -`packages/atif-cli/src/atif_cli/app.py:995` +`packages/atif-cli/src/atif_cli/app.py:2243` Flags: -- `--category`: filter to one of `view`, `table-macro`, `scalar-macro`, validated against `CATEGORY_VALUES`. `:965` -- `--requires`: filter to one of `core`, `analytics`, `vss`, validated against `REQUIRES_VALUES`. `:966` -- `--format`: a TTY table grouped by `requires`, or a JSON object carrying `note` and `examples`. `:967` +- `--category`: filter to one of `view`, `table-macro`, `scalar-macro`, validated against `CATEGORY_VALUES`. `:2245` +- `--requires`: filter to one of `core`, `analytics`, `vss`, validated against `REQUIRES_VALUES`. `:2246` +- `--format`: a TTY table grouped by `requires`, or a JSON object carrying `note` and `examples`. `:2247` -Both value sets are `Literal` aliases in the producer package: `Requires` at `packages/atif-duck/src/atif_duck/domain/examples.py:48` and `Category` at `:53`, exported as tuples at `:55` and `:56`. +Both value sets are `Literal` aliases in the producer package: `Requires` at `packages/atif-duck/src/atif_duck/domain/examples.py:49` and `Category` at `:54`, exported as tuples at `:56` and `:57`. -Exit codes: `0` ok, `64` unknown `--category` or `--requires` value. `packages/atif-cli/src/atif_cli/app.py:1020` +Exit codes: `0` ok, `64` unknown `--category` or `--requires` value. `packages/atif-cli/src/atif_cli/app.py:2280-2290` ## schema @@ -225,13 +226,13 @@ atif-sql schema [OPTIONS] ``` List every core and analytics view with its columns and every macro signature, each with its `requires` value. -`packages/atif-cli/src/atif_cli/app.py:1727` +`packages/atif-cli/src/atif_cli/app.py:2335` Flags: -- `--format`: a TTY listing, or a JSON object carrying `views`, `view_requires` (view name to `requires`), `macros` (each with `name`, `params`, and `requires`), and `examples_hint`. `:1729` +- `--format`: a TTY listing, or a JSON object carrying `views`, `view_requires` (view name to `requires`), `macros` (each with `name`, `params`, and `requires`), and `examples_hint`. `:2337` -The answer comes from the static `VIEW_SCHEMA`, `MACRO_SIGNATURES`, `ANALYTICS_VIEW_SCHEMA`, and `ANALYTICS_MACRO_SIGNATURES` dicts with no DuckDB import and no view registration. `requires` is `core` for an object that binds on any corpus, `analytics` for one that binds once `analyze` has written its parquet, and `vss` for one that needs the embedding store. `:1744` +The answer comes from the static `VIEW_SCHEMA`, `MACRO_SIGNATURES`, `ANALYTICS_VIEW_SCHEMA`, and `ANALYTICS_MACRO_SIGNATURES` dicts with no DuckDB import and no view registration. `requires` is `core` for an object that binds on any corpus, `analytics` for one that binds once `analyze` has written its parquet, and `vss` for one that needs the embedding store. `:2352-2361` ## `lake` @@ -251,7 +252,7 @@ atif-sql lake rebuild [OPTIONS] ``` Load every corpus's per-session artifacts into a fresh lake, then swap it into place. -`packages/atif-cli/src/atif_cli/lake.py:79` +`packages/atif-cli/src/atif_cli/lake.py:80` Flags: @@ -270,7 +271,7 @@ atif-sql lake verify [OPTIONS] ``` Compare every session's lake rows with its per-session artifacts. -`packages/atif-cli/src/atif_cli/lake.py:150` +`packages/atif-cli/src/atif_cli/lake.py:151` Flags: @@ -290,7 +291,7 @@ atif-sql lake status [OPTIONS] ``` Report the lake: present, schema current, corpora, snapshots, files, last write. -`packages/atif-cli/src/atif_cli/lake.py:240` +`packages/atif-cli/src/atif_cli/lake.py:241` Flags: @@ -306,7 +307,7 @@ atif-sql lake compact [OPTIONS] ``` Merge small files, expire old snapshots, and remove the files nothing references. -`packages/atif-cli/src/atif_cli/lake.py:283` +`packages/atif-cli/src/atif_cli/lake.py:284` Flags: @@ -328,7 +329,7 @@ atif-sql cron COMMAND Inspect and manually install the `atif-sql` refresh cron lanes. `packages/atif-cli/src/atif_cli/cron.py:38` -The group is attached to the root router by `app.command(cron_app)` at `packages/atif-cli/src/atif_cli/app.py:98`, and its lanes (`materialize` on `*/10 * * * *` and `llm` on `20 10 * * *`) are declared once in `LANES` at `packages/atif-cli/src/atif_cli/cron.py:47`. The `structural` lane was removed on 2026-09-27 and is listed in `REMOVED_LANES` (`:56`). `scripts/atif-sql-refresh.sh` still accepts `structural` (and `struct`) and exits 0 after logging `[structural] lane removed 2026-09-27 ...`, so a crontab line nobody has deleted yet stays quiet. +The group is attached to the root router by `app.command(cron_app)` at `packages/atif-cli/src/atif_cli/app.py:104`, and its lanes (`materialize` on `*/10 * * * *`, `llm` on `20 10 * * *`, and `compact` on `40 3 * * *`) are declared once in `LANES` at `packages/atif-cli/src/atif_cli/cron.py:47`. The `structural` lane was removed on 2026-09-27 and is listed in `REMOVED_LANES` (`:57`). `scripts/atif-sql-refresh.sh` still accepts `structural` (and `struct`) and exits 0 after logging `[structural] lane removed 2026-09-27 ...`, so a crontab line nobody has deleted yet stays quiet. ### cron install @@ -337,13 +338,13 @@ atif-sql cron install [OPTIONS] ``` Print the crontab block for the refresh lanes and never write it. The block includes one comment per removed lane asking you to delete its line. -`packages/atif-cli/src/atif_cli/cron.py:185` +`packages/atif-cli/src/atif_cli/cron.py:186` Flags: -- `--script`: path to `atif-sql-refresh.sh`. `packages/atif-cli/src/atif_cli/cron.py:185` +- `--script`: path to `atif-sql-refresh.sh`. `packages/atif-cli/src/atif_cli/cron.py:186` -Without the flag the script is located by walking up from this module (`:147`); a tree where `scripts/atif-sql-refresh.sh` is unreachable exits `64` demanding `--script` (`:175`). +Without the flag the script is located by walking up from this module (`:161-164`); a tree where `scripts/atif-sql-refresh.sh` is unreachable exits `64` demanding `--script` (`packages/atif-cli/src/atif_cli/cron.py:176-181`). ### cron status @@ -352,15 +353,15 @@ atif-sql cron status [OPTIONS] ``` Report each lane's lock holder plus the last run and skip parsed from the refresh log. -`packages/atif-cli/src/atif_cli/cron.py:199` +`packages/atif-cli/src/atif_cli/cron.py:207` Flags: -- `--script`: path to `atif-sql-refresh.sh`, whose `.run/` sibling holds the locks and the log. `packages/atif-cli/src/atif_cli/cron.py:201` -- `--tail`: how many trailing log lines to include; `0` disables, default `10`. `:202` -- `--format`: human lines on a TTY, JSON on a pipe. `:203` +- `--script`: path to `atif-sql-refresh.sh`, whose `.run/` sibling holds the locks and the log. `packages/atif-cli/src/atif_cli/cron.py:209` +- `--tail`: how many trailing log lines to include; `0` disables, default `10`. `:210` +- `--format`: human lines on a TTY, JSON on a pipe. `:211` -The lock probe acquires and releases nonblocking, so the command perturbs no running lane. `:127` +The lock probe acquires and releases nonblocking, so the command perturbs no running lane. `:146-149` ## corpus slim @@ -383,6 +384,7 @@ Flags: - `--corpus-root`: a corpus to slim; repeat for more. Default: every corpus the lake holds, plus every directory under `ATIF_SQL_CORPUS_BASE` that holds `sessions/`. - `--lake-root`: the lake that verifies the corpora (default `ATIF_SQL_LAKE_ROOT`). - `--no-dry-run`: act. +- `--format`: report format. The report lists, per corpus, the sessions in the old layout, the plain and compressed bytes, the parquet files and what happened to them (`deleted`, `would_delete`, `kept`, or `none`), and `bytes_before`, `bytes_after` and `bytes_freed`. diff --git a/docs/reference/public-api.md b/docs/reference/public-api.md index 0a12831..d70939f 100644 --- a/docs/reference/public-api.md +++ b/docs/reference/public-api.md @@ -2,7 +2,7 @@ **The supported entry point is the `atif-sql` command, not an import.** This repository ships exactly one installable distribution, `atif-sql`, whose only console script is -`atif-sql = "atif_cli.app:main"` (`pyproject.toml:64`). Every workspace member ships inside that +`atif-sql = "atif_cli.app:main"` (`pyproject.toml:75`). Every workspace member ships inside that one wheel rather than as install targets, and the published name is deliberately split from the `atif_cli` module that provides the entry point (`pyproject.toml:16-18`). Install with `uvx atif-sql` or `uv tool install atif-sql`; `docs/reference/cli.md` is the primary reference for the surface a user @@ -10,19 +10,19 @@ actually calls. What follows documents the **internal seam**: the highest-traffic symbols that one workspace member imports from another. That seam is enforced rather than conventional: the root `pyproject.toml` -declares `[tool.importlinter]` over every root package at `pyproject.toml:462-463`, with an -`independence` contract at `pyproject.toml:514-517` forbidding atif-converter, atif-corpus, atif-duck, +declares `[tool.importlinter]` over every root package at `pyproject.toml:550-551`, with an +`independence` contract at `pyproject.toml:602-605` forbidding atif-converter, atif-corpus, atif-duck, atif-models, and atif-embed from importing each other at all, and a `forbidden` contract at -`pyproject.toml:519-523` limiting atif-analytics to atif-models alone. Every symbol below is a real +`pyproject.toml:607-611` limiting atif-analytics to atif-models alone. Every symbol below is a real declaration read at the cited line; none of them is a supported import path for code outside this workspace, and no compatibility promise attaches to any of them. The entries also cover the atif-analytics prompt builders that every LLM stage shares inside that member. These facts about the seam shape are worth carrying into every entry below. First, the boundary abstractions are `typing.Protocol` classes, and all of them sit under a `domain/` package: -`ConverterPort` at `packages/atif-corpus/src/atif_corpus/domain/ports.py:41`, `EmbeddingProvider`, +`ConverterPort` at `packages/atif-corpus/src/atif_corpus/domain/ports.py:101`, `EmbeddingProvider`, `VectorStorePort`, and `TextRowsPort` at -`packages/atif-embed/src/atif_embed/domain/ports.py:31`, `:59`, and `:87`, and +`packages/atif-embed/src/atif_embed/domain/ports.py:37`, `:65`, and `:93`, and `LlmStructuredProvider` at `packages/atif-models/src/atif_models/domain/ports.py:131`. No `abstractmethod` exists anywhere in the workspace, so adapters satisfy a port structurally. Second, the name `DomainError` is declared independently more than once (one base class per erroring package, at @@ -89,13 +89,16 @@ def build_plan( now_ns: int, force: bool = False, unmaterialized_session_ids: Collection[str] = (), + generation_stale_session_ids: Collection[str] = (), ) -> MaterializationPlan: ``` Partitions scanned sessions into the plan buckets: the pure decision at the center of -materialization, taking the watermark and the quiescence policy as data. +materialization, taking the watermark and the quiescence policy as data. A session named in +`generation_stale_session_ids` (its recorded converter or columnar schema differs from this pass's) +is stale even though no source byte moved. -`packages/atif-corpus/src/atif_corpus/domain/sessions.py:159-211` +`packages/atif-corpus/src/atif_corpus/domain/sessions.py:169-228` ### build_structured_provider @@ -123,10 +126,12 @@ class ConversionOutput: ``` Everything one conversion yields that the corpus writes to disk: the ATIF trajectory dict, the -loss-report dict, and one already-serialized `edges.jsonl` line per raw record without trailing -newlines, because the writer owns line termination. +loss-report dict, one already-serialized `edges.jsonl` line per raw record without trailing +newlines (the writer owns line termination), the same for each kept non-message record of +`session_events.jsonl`, the raw source files the converter archived, and the inline attachments +it replaced with placeholders. -`packages/atif-corpus/src/atif_corpus/domain/ports.py:21-38` +`packages/atif-corpus/src/atif_corpus/domain/ports.py:65-98` ### convert_and_audit @@ -135,14 +140,16 @@ def convert_and_audit( session_jsonl: Path, *, include_subagents: bool = True, + archive: SourceArchiveWriter | None = None, ) -> tuple[ConversionResult, LossReport]: ``` Converts one Claude Code session JSONL to ATIF and produces its loss accounting in the same call, returning the enriched trajectory and a `LossReport` whose `records_dropped` is an upper bound on what -the materialized trajectory is missing. +the materialized trajectory is missing. With `archive`, the same verifying re-read also streams +every source file into that writer. -`packages/atif-converter/src/atif_converter/application/convert_and_audit.py:105-155` +`packages/atif-converter/src/atif_converter/application/convert_and_audit.py:220-294` ### CorpusLayout @@ -154,7 +161,7 @@ class CorpusLayout: Computes every contract path under one `corpus_root` as pure path arithmetic that never touches the filesystem, so the layout can be asserted against the contract without a tmpdir. -`packages/atif-corpus/src/atif_corpus/domain/layout.py:25-75` +`packages/atif-corpus/src/atif_corpus/domain/layout.py:64-168` ### corpus_slug @@ -164,7 +171,7 @@ def corpus_slug(corpus_root: Path | str) -> str: Maps one corpus root to a stable, human-legible directory key. -`packages/atif-corpus/src/atif_corpus/domain/slug.py:32-50` +`packages/atif-corpus/src/atif_corpus/domain/slug.py:49-72` ### CorpusSettings @@ -174,7 +181,7 @@ class CorpusSettings(BaseSettings): Env-driven settings for corpus materialization, read under the `ATIF_SQL_` prefix. -`packages/atif-corpus/src/atif_corpus/infrastructure/settings.py:44-58` +`packages/atif-corpus/src/atif_corpus/infrastructure/settings.py:88-130` ### DomainError @@ -193,9 +200,10 @@ it in the same module. class EmbedSettings(BaseSettings): ``` -Env-driven settings for the embedding pipeline: model, batch size, concurrency, and the LanceDB URI. +Env-driven settings for the embedding pipeline: the provider (`cohere` or `gemma`), each provider's +model, the output width, batch size, concurrency, and the LanceDB URI. -`packages/atif-embed/src/atif_embed/infrastructure/settings.py:19-64` +`packages/atif-embed/src/atif_embed/infrastructure/settings.py:46-146` ### EmbeddingProviderMismatch @@ -207,7 +215,7 @@ Raised on a stamped-versus-active `(model, dim)` mismatch, and terminal by desig embedding models occupy incompatible spaces, so the store must be dropped and re-embedded rather than queried across the switch. -`packages/atif-duck/src/atif_duck/domain/embedding_guard.py:33-43` +`packages/atif-duck/src/atif_duck/domain/embedding_guard.py:33-48` ### EmptySessionError @@ -306,7 +314,7 @@ MACRO_SIGNATURES: dict[str, tuple[str, ...]] = { Hand-maintained parameter names for every core macro, kept static because DuckDB's `duckdb_functions()` returns NULL `parameters` for table macros and cannot recover them at runtime. -`packages/atif-duck/src/atif_duck/domain/catalog.py:287-298` +`packages/atif-duck/src/atif_duck/domain/catalog.py:343-354` ### main @@ -315,9 +323,10 @@ def main() -> None: ``` The console-script entry point behind the `atif-sql` command; it replaces loguru's default DEBUG sink -with WARNING-and-up so routine reads keep stderr quiet, then hands off to the cyclopts app. +with WARNING-and-up (`ATIF_SQL_LOG_LEVEL` widens it) so routine reads keep stderr quiet, then hands +off to the cyclopts app and turns a cyclopts usage error into exit `64`. -`packages/atif-cli/src/atif_cli/app.py:1125-1135` +`packages/atif-cli/src/atif_cli/app.py:2409-2442` ### materialize @@ -337,16 +346,23 @@ def materialize( source_layout: SourceLayout = CLAUDE_CODE_LAYOUT, workers: int = 1, worker_setup: Callable[[], None] | None = None, + artifact_producer: ArtifactProducer | None = None, + expected_meta: Mapping[str, object] | None = None, + converter_schema: int | None = None, + session_sink: SessionSink | None = None, + sink_batch_size: int = DEFAULT_SINK_BATCH_SIZE, + convert_clock: Callable[[], float] = time.perf_counter, ) -> MaterializationReport: ``` Runs one materialization pass over the raw transcript corpus, taking conversion as an injected -`ConverterPort` so tests substitute a fake and atif-cli injects the harbor-backed adapter. +`ConverterPort` so tests substitute a fake and atif-cli injects `RealConverter`, its adapter over +atif-converter. `workers` above `1` runs the convert+write stage on a spawn-context process pool that pickles `converter` into each worker; `worker_setup` is the composition root's per-process hook (atif-cli uses it to install its stderr log sink). The default `1` is the inline path. -`packages/atif-corpus/src/atif_corpus/application/materialize.py:707-917` +`packages/atif-corpus/src/atif_corpus/application/materialize.py:1497-1826` ### MaterializationReport @@ -359,7 +375,7 @@ What one materialization pass did, for logs and the CLI status line. `convert_se per-session sum and `workers` the pool size the pass used, so with several workers `convert_seconds` can exceed `total_seconds`, the wall clock. -`packages/atif-corpus/src/atif_corpus/application/materialize.py:148-194` +`packages/atif-corpus/src/atif_corpus/application/materialize.py:219-317` ### message_prompt @@ -423,7 +439,7 @@ The rule for when a session is settled enough to convert: quiescent once its new at least `quiesce_seconds` in the past, an observed rather than announced signal that the writer finished. -`packages/atif-corpus/src/atif_corpus/domain/sessions.py:70-104` +`packages/atif-corpus/src/atif_corpus/domain/sessions.py:70-114` ### RefusalError @@ -448,15 +464,19 @@ def register( lance_uri: Path | None = None, expected_model: str | None = None, expected_dim: int | None = None, + lake: LakeReader | None = None, + lance_uris: Sequence[Path] | None = None, ) -> RawSources: ``` Registers raw readers, views, VSS, core macros, the authorship surface (`step_author`, `user_steps`, `human_turns`, `session_outcomes`), and the analytics views and macros over `corpus_root` in -dependency order on one DuckDB connection; every call re-scans the whole corpus into TEMP tables, so the cost is O(corpus) per -connection and a caller should reuse one connection per process. +dependency order on one DuckDB connection. Without `lake`, every call re-scans the corpus and parses +each session that has no current per-session parquet into TEMP tables, so the cost is O(JSON sessions) +per connection and a caller should reuse one connection per process. With `lake`, the raw relations +are views over the lake's tables and nothing loads eagerly. -`packages/atif-duck/src/atif_duck/infrastructure/registry.py:1679-1753` +`packages/atif-duck/src/atif_duck/infrastructure/registry.py:2095-2194` ### run_analyze @@ -471,14 +491,16 @@ def run_analyze( skip_conflicts: bool = False, skip_friction: bool = False, skip_perceived: bool = False, + source: SessionSource | None = None, ) -> dict[str, Any]: ``` Runs the LLM stages in order (classify, conflicts, friction, perceived) over one shared corpus reader and one run budget, returning a per-stage summary dict; the `skip_*` flags subtract individual -stages. +stages, and `source` says where session data comes from (atif-cli passes the lake's when the lake +holds the corpus; the default reads the per-session files). -`packages/atif-analytics/src/atif_analytics/application/analyze.py:36-149` +`packages/atif-analytics/src/atif_analytics/application/analyze.py:37-155` ### run_backfill @@ -498,18 +520,21 @@ async def run_backfill( Discovers unembedded steps, embeds them, and appends to the Lance store; all its ports default to None and are constructed lazily so a dry run never loads boto3. -`packages/atif-embed/src/atif_embed/application/embed.py:61-262` +`packages/atif-embed/src/atif_embed/application/embed.py:61-277` ### scan_source_root ```py -def scan_source_root(source_root: Path) -> tuple[SessionSource, ...]: +def scan_source_root( + source_root: Path, + layout: SourceLayout = CLAUDE_CODE_LAYOUT, +) -> tuple[SessionSource, ...]: ``` The sessions found under `source_root`, discarding the per-scan diagnostics that the fuller `scan_sources` returns. -`packages/atif-corpus/src/atif_corpus/infrastructure/scanner.py:211-222` +`packages/atif-corpus/src/atif_corpus/infrastructure/scanner.py:302-316` ### TrajectoryValidationError @@ -546,7 +571,7 @@ The hand-maintained column schema for every core view, where column order is loa test asserts tuple equality against DuckDB `DESCRIBE` output, so editing view DDL without updating this dict fails CI instead of surfacing as a runtime mystery. -`packages/atif-duck/src/atif_duck/domain/catalog.py:56-260` +`packages/atif-duck/src/atif_duck/domain/catalog.py:56-316` ## See also diff --git a/packages/atif-analytics/README.md b/packages/atif-analytics/README.md index b3b09e3..8ba5518 100644 --- a/packages/atif-analytics/README.md +++ b/packages/atif-analytics/README.md @@ -6,8 +6,8 @@ The v2 analytics pipelines over the materialized ATIF corpus: read the materialized corpus (`trajectory.json` steps + `edges.jsonl`, each stored compressed or plain), render session transcripts under the documented caps, and classify through - `atif_models.LlmStructuredProvider` (GPT-6 or Claude 5.5 structured - outputs on bedrock-runtime; sizes classify=small, conflicts=large, + `atif_models.domain.ports.LlmStructuredProvider` (GPT-6 or Claude 5.5 + structured outputs on bedrock-runtime; sizes classify=small, conflicts=large, friction=small, perceived=medium). `classify` labels `work_category` and `goal` only. `classify` and `conflicts` skip non-interactive sessions (`turn_audit`, `one_shot_job`), and `friction` and `perceived` read human turns only. @@ -25,8 +25,9 @@ The v2 analytics pipelines over the materialized ATIF corpus: sharded parquet caches under `/analytics//`. Composition: atif-analytics imports **only** atif-models among the workspace -packages (import-linter independence contract); atif-cli composes it with -atif-duck's registered views via the `atif-sql analyze` command. +packages (import-linter forbidden contract); atif-cli composes it via the +`atif-sql analyze` command, handing it a session source over atif-duck's lake +(`atif_cli.lake_sessions`). All pipelines default to `dry_run=True` and return plan dicts with `estimate_cost_tokens` projections (cost guard per CONTRACT-V2). diff --git a/packages/atif-cli/README.md b/packages/atif-cli/README.md index 71994dc..9125cc5 100644 --- a/packages/atif-cli/README.md +++ b/packages/atif-cli/README.md @@ -16,7 +16,7 @@ atif-sql materialize [--force] [--quiesce-seconds N] [--sessions id,id] # contract-compatible extension: # plan only the named sessions atif-sql status # corpus freshness, read-only -atif-sql query 'SQL' [--format auto|json|csv] +atif-sql query 'SQL' [--format auto|table|json|csv] atif-sql schema # static catalog, no duckdb bind, <50ms atif-sql examples # tested example queries per view/macro # (alias: atif-sql query --examples) @@ -27,6 +27,11 @@ atif-sql embed --all --no-dry-run # embed corpus steps into LanceDB atif-sql search '' # embed the text, then semantic kNN atif-sql cron install | status # print the refresh crontab block, or # report lane locks + last runs +atif-sql lake rebuild | verify | status | compact + # build, check, report on, and compact + # the DuckLake every query reads +atif-sql corpus slim [--no-dry-run] # convert a corpus to the compressed + # layout; a dry run by default ``` Conventions: `--format auto` resolves to a table on a TTY and JSON on a pipe; diff --git a/packages/atif-converter/README.md b/packages/atif-converter/README.md index ef8236a..c5d8fba 100644 --- a/packages/atif-converter/README.md +++ b/packages/atif-converter/README.md @@ -2,8 +2,10 @@ Converts Claude Code session JSONL and Codex rollouts into ATIF trajectories (converters ported from harbor and held to parity with harbor 0.24.0, on harbor's ATIF models vendored in -`atif_converter.domain.atif`), and owns the FIDELITY POLICY: the seven known upstream -conversion gaps are encoded as `atif_converter.domain.fidelity.FidelityGap`, +`atif_converter.domain.atif`), and owns the FIDELITY POLICY: the known upstream +conversion gaps are encoded as `atif_converter.domain.fidelity.FidelityGap` (six +for Claude Code, plus one retired number) and +`atif_converter.domain.codex_fidelity.CodexFidelityGap` (six for Codex), and every conversion is audited into a `LossReport` (raw-side census vs converted output). harbor is a dev dependency only; the tests pin harbor 0.24.0 behavior and hold the vendored models to it, so they are the drift alarm for a @@ -14,7 +16,7 @@ harbor bump. `LossReport.record_counts` (and therefore `records_total` / `records_dropped`) counts every record of every `*.jsonl` discovered under the session's side directory, at any depth and regardless of whether a `subagents/` part appears -in the path. That is the same file set the adapter stages into harbor and the +in the path. That is the same file set the converter reads and the same set `edges.jsonl` is built from, so `records_total` always equals the edges line count. A session with a side-file outside `subagents/` reports a higher `records_total` than a census scoped to `subagents/` alone would. diff --git a/packages/atif-duck/README.md b/packages/atif-duck/README.md index 629de9c..1b042e0 100644 --- a/packages/atif-duck/README.md +++ b/packages/atif-duck/README.md @@ -69,12 +69,12 @@ it writes. - **Views**: `sessions`, `steps` (one row per ATIF step — the messages-parity view), `messages` (uuid-keyed COMPAT view from `edges.jsonl`), `tool_calls`, `tool_results`, `todo_events`, - `todo_state_current`, `subagent_spawns`, `subagent_steps`, + `todo_state_current`, `subagent_spawns`, `subagent_steps`, `subagents`, `task_creations`, `task_updates`, `tasks_state_current`, - `skill_invocations`, `skill_usage`, `loss_reports`, - `message_embeddings` (the VSS view; empty until `atif-sql embed --all - --no-dry-run` runs), and the three authorship views `user_steps`, - `human_turns`, `session_outcomes` (below). + `skill_invocations`, `skill_usage`, `images`, `loss_reports`, + `session_events`, `message_embeddings` (the VSS view; empty until + `atif-sql embed --all --no-dry-run` runs), and the three authorship views + `user_steps`, `human_turns`, `session_outcomes` (below). - **Macros**: `ago`, `model_used`, `cost_estimate`, `tool_rank`, `todo_velocity`, `subagent_fanout`, `skill_rank`, `skill_source_mix`, `semantic_search(query_vec, k)` (kNN over `message_embeddings`; skipped diff --git a/packages/atif-embed/README.md b/packages/atif-embed/README.md index 847e7a5..78315ff 100644 --- a/packages/atif-embed/README.md +++ b/packages/atif-embed/README.md @@ -39,6 +39,7 @@ atif_embed/ discovery.py PendingSelection — first-wins, hash skip, stale, limit embedding_guard.py ensure_store_matches — fail-loud (model, dim) guard errors.py DomainError taxonomy + sql_literal.py SqlFragment + sql_literal (the Lance delete predicate's quoting) text_stamp.py text_hash + the 50K cap + PendingText infrastructure/ cohere_bedrock.py Cohere Embed v4 adapter (batch 96, semaphore 8, @@ -59,13 +60,13 @@ atif_embed/ ## Independence: why TextRowsPort exists atif-embed is a workspace-independent package (import-linter independence -contract: converter / corpus / duck / embed may never import each other; +contract: converter / corpus / duck / models / embed may never import each other; only atif-cli composes). But the backfill needs the corpus's embeddable text rows — a surface atif-duck's `steps` view already computes. Resolution (the ConverterPort precedent from atif-corpus): the use case depends on a domain port, `TextRowsPort.iter_unembedded(corpus_root, -exclude, limit)`, and atif-embed ships its OWN DuckDB implementation +embedded, limit)`, and atif-embed ships its OWN DuckDB implementation (`DuckDbTextRows`) that reads the CONTRACT corpus layout directly — `read_json` over each session's stored trajectory (`trajectory.json.zst`, or the plain `trajectory.json` of an older corpus, batched by decompressed size) with the @@ -160,10 +161,10 @@ An `embed --all` over a large corpus is a multi-hour, billed run, so nothing already paid for is thrown away: - discovery splits the trajectory files into batches of at most 4 MB of total - file bytes and runs ONE DuckDB statement per batch. DuckDB materializes a - result set fully inside `execute()`, so paging a single all-paths query off - `fetchmany` bounds nothing — the split is what bounds it. Bytes, rather - than a file count, is the axis because residency tracks the bytes one + decompressed trajectory bytes and runs ONE DuckDB statement per batch. + DuckDB materializes a result set fully inside `execute()`, so paging a + single all-paths query off `fetchmany` bounds nothing; the split is what + bounds it. Bytes, rather than a file count, is the axis because residency tracks the bytes one statement reads while wall time tracks the statement COUNT; batching by bytes bounds both. Peak resident text is therefore `max(4 MB budget, largest single session)`, so a corpus holding one huge @@ -221,12 +222,13 @@ already paid for is thrown away: deliberate twin of atif-models' copy and is pinned by a test on both sides. Every adapter failure surfaces as a `DomainError` subclass -(`EmbeddingProviderUnavailable` / `EmbeddingResponseInvalid`), which is what -the CLI catches to emit a classified `{kind, exit_code, hint}` envelope; a raw +(`EmbeddingProviderUnavailable` / `EmbeddingResponseInvalid` / +`EmbeddingProviderNotInstalled`), which is what the CLI catches to emit a +classified `{kind, message, hint}` envelope and exit with its code; a raw botocore traceback would escape that. -Documents are stored int8-encoded then float-widened; queries always embed -as float. Cosine ranking is therefore on mixed-magnitude vectors — cosine -similarity/distance is magnitude-invariant, but do NOT swap in L2 +With Cohere, documents are stored int8-encoded then float-widened; queries +always embed as float. Cosine ranking is therefore on mixed-magnitude +vectors; cosine similarity/distance is magnitude-invariant, but do NOT swap in L2 (`array_distance`) downstream: raw int8-derived document vectors have magnitudes in the thousands while query vectors are unit-normalized. diff --git a/packages/atif-models/README.md b/packages/atif-models/README.md index f1703ed..cbc14a3 100644 --- a/packages/atif-models/README.md +++ b/packages/atif-models/README.md @@ -4,7 +4,8 @@ Model alias registry + structured-output LLM client for the atif-sql workspace (CONTRACT-V2 §Model registry / §Client contract). - `atif_models.domain.registry` — frozen `ModelSpec` + size-alias registry. - NO other package hardcodes a Bedrock model id. + NO other package hardcodes a Bedrock LLM id (the embedding model ids are + atif-embed settings). - `atif_models.domain.ports` — `LlmStructuredProvider` protocol, error taxonomy (`RefusalError` / `ProviderUnavailable`), `CallUsage` + `UsageAccumulator` token accounting. diff --git a/site/authored/agents.md b/site/authored/agents.md index 4542c92..7333807 100644 --- a/site/authored/agents.md +++ b/site/authored/agents.md @@ -37,14 +37,16 @@ adds what a schema cannot state: what a silence means, and which surface you are - **A session still being written is skipped on purpose.** Materialization is gated on quiescence: a session is converted when its newest source file has been silent for longer than the quiesce - threshold and is newer than the recorded watermark. So the most interesting session (the one open + threshold and either is newer than the recorded watermark or was converted under a different converter + generation. So the most interesting session (the one open in another terminal) is the one deliberately not in the corpus yet. `--force` overrides the watermark; nothing overrides physics, so wait for the write to settle. - **`requires` on an example is a precondition, not a hint.** An example tagged `analytics` needs - `atif-sql analyze` to have run; one tagged `vss` needs an embedding run. Query one before its - pipeline has run and you get a catalog miss with exit 65, which is a different problem from a query - that legitimately matched nothing. + `atif-sql analyze` to have run; one tagged `vss` needs an embedding run. Query an `analytics` one + before its pipeline has run and you get a catalog miss with exit 65, which is a different problem + from a query that legitimately matched nothing. A `vss` one before any embedding run returns zero + rows instead: `message_embeddings` binds as an empty table so `semantic_search` still binds. - **A bare `atif-sql embed` exits 64 rather than doing the obvious thing.** A real embedding run calls Bedrock once per unembedded step, so it requires an explicit scope: `--limit N` or `--all`. The @@ -57,8 +59,9 @@ adds what a schema cannot state: what a silence means, and which surface you are evidence that it was absent from the transcript. Read the loss report before concluding that. - **Branch on the exit code, never on the message text.** The codes are a stable wire contract: `64` - for malformed input or malformed SQL, `65` for a catalog miss or a store written by another embedding - provider, `70` for anything the database or an adapter raises, `78` for a state only an operator can + for malformed input or malformed SQL, `65` for a catalog miss, a trajectory that fails validation, lake + rows that differ from their corpus, or a store written by another embedding provider, `70` for anything the + database or an adapter raises, `77` for a refusal to run as root, `78` for a state only an operator can clear, `127` retired (kept so the table never renumbers), and `2` for a parsed-but-empty result an automated caller should treat as "nothing to do". The message beside the code is prose a maintainer rewrites freely. @@ -76,7 +79,7 @@ never a call that has to fail first. The CLI and the library are the same implementation: `atif-cli` is a composition root and holds no query logic of its own, so a fact learned on one surface transfers to the other. The layering is enforced rather than documented: import-linter fails the build when a leaf package imports another -leaf package. +leaf package (the one allowed edge is `atif-analytics` to `atif-models`). :::agent @@ -102,8 +105,9 @@ is a subquery over `sessions`, so a pasted example runs against any corpus rathe you do not have yet. Errors arrive on stderr as a JSON envelope when the channel is a pipe, carrying a kind, a message and -a hint. Exit 0 is success. An exit in the sixties is fixed by changing the call; an exit of 70 or 78 is -fixed by changing the environment. +a hint. Exit 0 is success. An exit of 64 is fixed by changing the call, a 65 needs its kind read (a +catalog miss wants a different query or a pipeline run, a mismatch wants a rebuilt lake or store), and +an exit of 70, 77, or 78 is fixed by changing the environment. ## 5. What to avoid @@ -115,10 +119,11 @@ fixed by changing the environment. is that a view over raw transcript JSON re-implements the adapter, badly, once per view. If a fact about a trajectory is missing, it belongs in the converter or in the catalog, not in your query. -- **Reading an analytics or vector view before its pipeline has run.** Point back at section 2: that is - a catalog miss, and the fix is a command rather than a different query. +- **Reading an analytics or vector view before its pipeline has run.** Point back at section 2: an + analytics view is a catalog miss and a vector one is empty, and either way the fix is a command + rather than a different query. -- **Naming a Bedrock model id anywhere outside `atif-models`.** The alias registry exists so a model +- **Naming a Bedrock LLM id anywhere outside `atif-models`.** The alias registry exists so a model swap is one edit. A hardcoded id elsewhere is a second source of truth that nothing reconciles. - **Adding a view or macro without its catalog entries.** The drift tests require a description entry, diff --git a/site/authored/index.md b/site/authored/index.md index 6ed7df8..4e6bcec 100644 --- a/site/authored/index.md +++ b/site/authored/index.md @@ -17,9 +17,9 @@ It is a uv Python workspace, Apache-2.0, and it replaces `claude-sql`. ```mermaid The conversion pipeline: a Claude Code session JSONL or a Codex CLI rollout JSONL is converted once to an ATIF corpus on disk, and every DuckDB view, analytics pipeline and embedding run reads that corpus. flowchart LR - jsonl[Claude Code session JSONL] --> convert[atif-sql convert] - rollout[Codex CLI rollout JSONL] --> convert - convert --> corpus[ATIF corpus on disk] + jsonl[Claude Code session JSONL] --> materialize[atif-sql materialize] + rollout[Codex CLI rollout JSONL] --> materialize + materialize --> corpus[ATIF corpus on disk] corpus --> duck[DuckDB views and macros] duck --> query[atif-sql query] corpus --> analyze[atif-sql analyze] @@ -32,17 +32,18 @@ flowchart LR | Package | What it owns | | --- | --- | -| `atif-converter` | The Harbor `ClaudeCode` and `Codex` adapter wrappers, and the per-agent fidelity policy that accounts for every known upstream conversion gap per session | +| `atif-converter` | The Claude Code and Codex converters, ported from Harbor's adapters onto its vendored ATIF models, and the per-agent fidelity policy that accounts for every known upstream conversion gap per session | | `atif-corpus` | Corpus materialization: source discovery, watermarks, quiescence, atomic artifact writes | | `atif-duck` | The DuckDB views and macros over the materialized corpus, declared in a drift-tested static catalog | -| `atif-models` | The model alias registry and the structured-output LLM clients for GPT-6 and Claude 5.5. No other package names a Bedrock model id | +| `atif-models` | The model alias registry and the structured-output LLM clients for GPT-6 and Claude 5.5. No other package names a Bedrock LLM id (the embedding model ids are `atif-embed` settings) | | `atif-analytics` | The LLM pipelines: classify, conflicts, friction, perceived | | `atif-embed` | Cohere Embed v4 on Bedrock or EmbeddingGemma 2 on this machine, a LanceDB vector store, and the embedding backfill | | `atif-cli` | The cyclopts CLI that composes the rest | -Each package is layered, and the independence contract is enforced by import-linter rather than by -convention: converter, corpus, duck, models and embed may never import one another, and `atif-cli` is -the only composition root. +Five of the packages carry an import-linter `layers` contract, and the independence contract is +enforced the same way rather than by convention: converter, corpus, duck, models and embed may never +import one another, `atif-analytics` imports only `atif-models`, and `atif-cli` is the only +composition root. ## The discovery loop From 292ab871797188ff5dca3d88e6a14779d7c9f9ed Mon Sep 17 00:00:00 2001 From: Laith Al-Saadoon Date: Mon, 12 Oct 2026 18:13:53 +0000 Subject: [PATCH 3/3] feat(gates): fail on a docs citation whose lines moved or don't exist scripts/docs_citations.py (`mise run docs:citations`, inside `docs:gate`, on every pre-push, and in docs.yml against the pull request's base) runs two checks over docs/ and site/authored/. The static check fails on a citation naming a file git does not track, a line past the end of its file, or a shorthand with no antecedent. On main it finds 15 past-the-end citations; on this tree it finds none. The drift check maps each citation on a doc line the change leaves alone through its cited file's diff from the base. Lines that moved fail with the new span, and `mise run docs:citations:fix` rewrites them. Lines that were removed, or rewritten and shifted, fail for a reader to re-point. An edit in place prints a note. Replayed over the 29 commits before this change, it fails 19 of them on 991 citations and names the new lines for 934. On the 5c906c5..4fccb1e range, all 21 citations it fails were among those this audit had to fix. A check that the cited line holds an identifier from its sentence was measured and not built: on the corrected docs it can judge 1,203 of 2,465 citations, and 304 of the 311 it flags are correct (a 25% false-positive rate), because a test, behavior or config citation rarely repeats the sentence's identifiers. --- .github/workflows/docs.yml | 4 + AGENTS.md | 4 +- CONTRIBUTING.md | 9 +- docs/README.md | 6 +- lefthook.yml | 7 + mise.toml | 24 +- .../atif-cli/tests/test_docs_citations.py | 212 ++++++++ scripts/docs_citations.py | 488 ++++++++++++++++++ 8 files changed, 747 insertions(+), 7 deletions(-) create mode 100644 packages/atif-cli/tests/test_docs_citations.py create mode 100755 scripts/docs_citations.py diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index a39738a..1f0631a 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -66,9 +66,13 @@ jobs: - run: mise run docs:install + # ATIF_SQL_CITATIONS_BASE is what `docs:citations` maps each citation forward from: the + # pull request's base, or on a push the commit main held before it. Empty on a manual run, + # where the gate falls back to the merge base with origin/main. - name: gate the site env: ATIF_SQL_DOCS_COMMIT: ${{ github.event.pull_request.head.sha || github.sha }} + ATIF_SQL_CITATIONS_BASE: ${{ github.event.pull_request.base.sha || github.event.before }} run: mise run docs:gate - uses: actions/upload-pages-artifact@fc324d3547104276b827a68afc52ff2a11cc49c9 # v5.0.0 diff --git a/AGENTS.md b/AGENTS.md index eae65af..fa616c3 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -428,7 +428,9 @@ lint:hooks + lint:pnpm-lock + security:vex:check + docs:prose + test). `docs:pro the published prose (docs/, site/authored/, README.md, AGENTS.md, CONTRIBUTING.md) at error level, with a file floor per glob so a shrunken scope is red, and a per-rule warning ratchet against `.vale-baseline.json`: reword the sentence it flags, or add a real term to `.vale/styles/config/vocabularies/atif-sql/accept.txt`. When a count falls, run `mise run docs:prose:baseline` and commit the lower file; never raise it. A spaced dash is an error (`Google.EmDash`), not a warning: write a colon, comma, period, or parentheses instead. `mise run security` is the report-only tier beside it: findings do not fail it, a scanner that produced no usable report does. -A change under docs/ or site/ also needs `mise run docs:gate`: the site build, then `docs:links`, +A change that moves cited code needs `mise run docs:citations`, which fails on a docs citation whose +lines moved, were removed, or don't exist; `mise run docs:citations:fix` re-points the moved ones. +A change under docs/ or site/ also needs `mise run docs:gate`: `docs:citations`, the site build, then `docs:links`, which crawls site/dist as GitHub Pages serves it and fails on any internal 404, then `astro check` and the site's dist probes (`pnpm run check && pnpm run test` in site/). Link page to page by relative `.md` path as GitHub renders it; the build turns each link into the page's route, or into diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index cbd9e30..01d4f1e 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -68,13 +68,18 @@ whichever keeps the sentence's sense; `.vale.ini` says why the rule sits at erro A change under `docs/` or `site/` also needs `mise run docs:install && mise run docs:gate`, which `check` leaves out because it needs node and a site build. CI runs it in `.github/workflows/docs.yml`, -and the pre-push hook runs its link crawl when a push touches either tree. +and the pre-push hook runs its link crawl when a push touches either tree. A code change can break +a page too: a citation such as `` `packages/atif-cli/src/atif_cli/app.py:675` `` names lines, and +code that moves leaves the citation pointing at whatever took its place. `docs:citations` runs on +every push and in `docs:gate`, and `mise run docs:citations:fix` re-points every citation it +reports as moved. | Task | Command | What fails you | | --- | --- | --- | | `docs:build` | `pnpm run build` in `site/` | a relative link the build left unresolved (`starlight-links-validator` with `errorOnRelativeLinks`), a fragment naming no heading, or a citation to a path absent at the pinned commit | | `docs:links` | `python3 scripts/docs_links.py dist` | any internal `href` or `src` in `site/dist` (pages, head, raw `.md` twins, the `llms.txt` bundles) that GitHub Pages would answer with a 404, hidden paths included, or a fragment naming no `id`; also 0 pages, 0 internal links, or a page of the content collection with no route or twin in the build | -| `docs:gate` | `pnpm run check && pnpm run test` in `site/`, after `docs:build` and `docs:links` | `astro check` and the `vitest` probes over `site/dist` | +| `docs:citations` | `python3 scripts/docs_citations.py` | a citation naming a file git does not track, a line past the end of its file, or a shorthand with no antecedent; or, against the base commit, a citation on a doc line you left alone whose cited lines moved (fixable with `docs:citations:fix`) or were removed or rewritten | +| `docs:gate` | `pnpm run check && pnpm run test` in `site/`, after `docs:citations`, `docs:build`, and `docs:links` | `astro check` and the `vitest` probes over `site/dist` | `mise run docs:links:live` runs the same crawl over the deployed site, from its sitemap. The `live-links` job in `docs.yml` runs it after every deploy and every Monday, and fails on any diff --git a/docs/README.md b/docs/README.md index c0e3d0f..d116025 100644 --- a/docs/README.md +++ b/docs/README.md @@ -7,8 +7,10 @@ Prose is generated; structure is mechanical. Cross-references are deterministic. Citations come as full paths or as shorthands, and each one resolves into a source file. The docs site build turns each one into a link and fails on the first citation naming -a file that does not exist at the commit it links to, or a shorthand with no resolvable antecedent, -so a citation here resolves or the tree does not ship. +a file that does not exist at the commit it links to, or a shorthand with no resolvable antecedent. +`mise run docs:citations` also fails on a line outside its file and, against the base commit, on a +citation whose cited lines a change moved or removed without editing the citation, so a citation +here resolves to the code it names or the tree does not ship. ## Start here diff --git a/lefthook.yml b/lefthook.yml index c25cc2f..c0c7b0b 100644 --- a/lefthook.yml +++ b/lefthook.yml @@ -172,6 +172,13 @@ pre-push: # edit that trips an error-level rule is cheaper to reword here than after a CI round trip. run: mise run docs:prose + - name: docs-citations + # A code change moves the lines a docs citation names without touching the docs, so this + # runs on every push, not only on docs changes: about 2 s, against the merge base with + # origin/main. + run: mise run docs:citations + fail_text: "A docs citation points at lines that moved or are gone: run `mise run docs:citations:fix`, then fix what it still names." + - name: docs-links # The site build plus `docs:links`, the offline crawl of site/dist that fails on any # internal 404 GitHub Pages would serve. Only when the push touches the docs tree or the diff --git a/mise.toml b/mise.toml index 33417f4..195f8b3 100644 --- a/mise.toml +++ b/mise.toml @@ -816,9 +816,29 @@ description = "Crawl the deployed docs site from its sitemap: fail on any intern # the rule scripts/vale_gate.py states for its docs/ floor. run = "python3 scripts/docs_links.py live --min-pages 15" +[tasks."docs:citations"] +description = "Fail on a docs citation whose lines don't exist, or whose cited code moved since the base commit" +# Two checks over docs/ and site/authored/, the Markdown the site's citation pass links. Static: +# every `path:LINE` names a tracked file and lines inside it, and every `:LINE` shorthand has an +# antecedent. Drift: each citation on a doc line this change leaves alone is mapped through its +# cited file's diff from the base (ATIF_SQL_CITATIONS_BASE, which docs.yml sets to the pull +# request's base, else the merge base with origin/main); a moved block fails with the new span, +# removed or rewritten lines fail for a reader to re-point, an edit in place prints a note. +# citation-links.ts checks only that a path exists, which is how 1,318 of main's 2,177 citations +# came to need a re-point, a rewrite, or removal by 2026-10-12. `python3`: stdlib only, like the other +# gate helpers. Its planted-defect tests (packages/atif-cli/tests/test_docs_citations.py) run +# under `test`, and one of them runs the static check over the real tree, so `check` holds it. +run = "python3 scripts/docs_citations.py" + +[tasks."docs:citations:fix"] +description = "Rewrite every citation docs:citations reports as moved to the lines its code moved to" +# Only `moved` findings: the same text at new line numbers, so the rewrite is mechanical. A +# `changed` finding needs a reader, and the task prints it again after the rewrite. +run = "python3 scripts/docs_citations.py --fix" + [tasks."docs:gate"] -description = "Docs site definition of done: build + built-site link crawl + astro check + the dist probes" -depends = ["docs:build", "docs:links"] +description = "Docs site definition of done: citations + build + built-site link crawl + astro check + the dist probes" +depends = ["docs:citations", "docs:build", "docs:links"] dir = "site" # The probes read site/dist rather than the inputs, and derive every count from the build — # a hardcoded page or diagram count would pass a build that emitted neither. diff --git a/packages/atif-cli/tests/test_docs_citations.py b/packages/atif-cli/tests/test_docs_citations.py new file mode 100644 index 0000000..7ea4a5b --- /dev/null +++ b/packages/atif-cli/tests/test_docs_citations.py @@ -0,0 +1,212 @@ +# SPDX-License-Identifier: Apache-2.0 + +"""`scripts/docs_citations.py` (`mise run docs:citations`) fails on planted citation drift. + +Each test builds a two-commit repository in `tmp_path`: a base where `docs/page.md` cites +`src/a.py` correctly, then a working-tree change. A clean tree passes; each way a citation goes +wrong without its doc line changing turns the gate red on its own: lines inserted above the +cited code (moved, and `--fix` re-points it), the cited code deleted (changed), a range past +the end of the file, a shorthand under a fresh heading, a path git does not track, and an empty +scope. An edit in place passes with a note, and a citation whose doc line the change edits is +left to whoever edited it. The real tree's own citations pass the static check. +""" + +from __future__ import annotations + +import importlib.util +import subprocess +import sys +from pathlib import Path +from types import ModuleType + +import pytest + +#: Repository root: `packages/atif-cli/tests/` -> three parents up. +ROOT = Path(__file__).resolve().parents[3] +SCRIPT = ROOT / "scripts" / "docs_citations.py" + +CODE = "".join(f"line {n}\n" for n in range(1, 10)).replace("line 3\n", "def target():\n") +PAGE = ( + "# Page\n\n" + "The target is `src/a.py:3`, and the block after it is `:4-5`.\n\n" + "| Name | Where |\n| --- | --- |\n| target | `src/a.py:3` |\n" +) + + +def _load() -> ModuleType: + spec = importlib.util.spec_from_file_location("docs_citations", SCRIPT) + assert spec is not None + assert spec.loader is not None + module = importlib.util.module_from_spec(spec) + sys.modules["docs_citations"] = module + spec.loader.exec_module(module) + return module + + +gate = _load() + + +def _git(root: Path, *args: str) -> str: + return subprocess.run( # noqa: S603 - git from PATH with fixed flags inside tmp_path + [ # noqa: S607 + "git", + "-C", + str(root), + "-c", + "user.name=t", + "-c", + "user.email=t@example.invalid", + "-c", + "commit.gpgsign=false", + *args, + ], + capture_output=True, + text=True, + check=True, + ).stdout.strip() + + +@pytest.fixture +def repo(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path: + """A repository whose base commit cites `src/a.py` correctly from `docs/page.md`.""" + for name in ("GIT_DIR", "GIT_WORK_TREE", "GIT_INDEX_FILE", "ATIF_SQL_CITATIONS_BASE"): + monkeypatch.delenv(name, raising=False) + (tmp_path / "src").mkdir() + (tmp_path / "docs").mkdir() + (tmp_path / "src" / "a.py").write_text(CODE) + (tmp_path / "docs" / "page.md").write_text(PAGE) + _git(tmp_path, "init", "-q", "-b", "main") + _git(tmp_path, "add", "-A") + _git(tmp_path, "commit", "-q", "-m", "base") + return tmp_path + + +def _run(repo: Path, *extra: str) -> tuple[int, str]: + base = _git(repo, "rev-parse", "HEAD") + result = subprocess.run( # noqa: S603 - this interpreter and the repository's own script + [sys.executable, str(SCRIPT), "--root", str(repo), "--base", base, *extra], + capture_output=True, + text=True, + check=False, + ) + return result.returncode, result.stdout + result.stderr + + +def test_clean_tree_passes(repo: Path) -> None: + code, out = _run(repo) + assert code == 0, out + assert "3 citations on 1 pages, 0 problems" in out + + +def test_lines_inserted_above_the_cited_code_fail_as_moved(repo: Path) -> None: + (repo / "src" / "a.py").write_text("new 1\nnew 2\n" + CODE) + code, out = _run(repo) + assert code == 1 + assert "moved docs/page.md:3: `src/a.py:3` now sits at `src/a.py:5`" in out + assert "moved docs/page.md:3: `:4-5` now sits at `:6-7`" in out + assert "moved docs/page.md:7: `src/a.py:3` now sits at `src/a.py:5`" in out + + +def test_fix_rewrites_moved_citations_and_then_passes(repo: Path) -> None: + (repo / "src" / "a.py").write_text("new 1\nnew 2\n" + CODE) + code, out = _run(repo, "--fix") + assert code == 0, out + assert "rewrote 3 moved citations" in out + page = (repo / "docs" / "page.md").read_text() + assert "The target is `src/a.py:5`, and the block after it is `:6-7`." in page + assert "| target | `src/a.py:5` |" in page + assert (repo / "src" / "a.py").read_text().splitlines()[4] == "def target():" + + +def test_a_block_moved_past_a_rewrite_is_found_by_its_text(repo: Path) -> None: + lines = CODE.splitlines(keepends=True) + # The target moves to the end while the lines around its old place are rewritten. + (repo / "src" / "a.py").write_text("".join([*lines[:2], "x\n", "y\n", *lines[5:], lines[2]])) + code, out = _run(repo) + assert code == 1 + assert "`src/a.py:3` now sits at `src/a.py:9`" in out + + +def test_deleted_cited_code_fails_as_changed(repo: Path) -> None: + (repo / "src" / "a.py").write_text(CODE.replace("def target():\n", "")) + code, out = _run(repo) + assert code == 1 + assert "changed docs/page.md:3: `src/a.py:3`: the cited lines in src/a.py were removed" in out + + +def test_an_edit_in_place_passes_with_a_note(repo: Path) -> None: + (repo / "src" / "a.py").write_text(CODE.replace("def target():", "def target(x):")) + code, out = _run(repo) + assert code == 0, out + assert "edited docs/page.md:3: `src/a.py:3`" in out + + +def test_a_doc_line_the_change_edits_is_left_to_its_editor(repo: Path) -> None: + (repo / "src" / "a.py").write_text("new 1\nnew 2\n" + CODE) + page = repo / "docs" / "page.md" + page.write_text(page.read_text().replace("The target is", "The function is")) + code, out = _run(repo) + assert code == 1 + assert "docs/page.md:3" not in out + assert "docs/page.md:7" in out + + +def test_a_range_past_the_end_of_the_file_fails(repo: Path) -> None: + page = repo / "docs" / "page.md" + page.write_text(page.read_text() + "\nThe tail is `src/a.py:8-12`.\n") + code, out = _run(repo) + assert code == 1 + assert ( + "range docs/page.md:9: `src/a.py:8-12` names lines 8-12 of src/a.py, which has 9" in out + ) + + +def test_a_shorthand_under_a_fresh_heading_fails(repo: Path) -> None: + page = repo / "docs" / "page.md" + page.write_text(page.read_text() + "\n## Next\n\nAnd `:4`.\n") + code, out = _run(repo) + assert code == 1 + assert "orphan docs/page.md:11: `:4` is a shorthand with no antecedent" in out + + +def test_a_shorthand_in_a_table_row_does_not_inherit_the_row_above(repo: Path) -> None: + page = repo / "docs" / "page.md" + page.write_text(page.read_text() + "| again | `:4` |\n") + code, out = _run(repo) + assert code == 1 + assert "orphan docs/page.md:8: `:4`" in out + + +def test_an_untracked_path_fails(repo: Path) -> None: + (repo / "src" / "b.py").write_text("x\n") + page = repo / "docs" / "page.md" + page.write_text(page.read_text() + "\nSee `src/b.py:1`.\n") + code, out = _run(repo) + assert code == 1 + assert "missing docs/page.md:9: `src/b.py:1` names src/b.py, which git does not track" in out + + +def test_a_scope_with_no_citation_fails(repo: Path) -> None: + (repo / "docs" / "page.md").write_text("# Page\n\nNo citations here.\n") + code, out = _run(repo) + assert code == 1 + assert "found 0 citations: the gate checked nothing" in out + + +def test_a_base_that_does_not_resolve_is_a_usage_error(repo: Path) -> None: + result = subprocess.run( # noqa: S603 - this interpreter and the repository's own script + [sys.executable, str(SCRIPT), "--root", str(repo), "--base", "no-such-rev"], + capture_output=True, + text=True, + check=False, + ) + assert result.returncode == 2 + assert "does not name a commit" in result.stderr + + +def test_the_real_tree_passes_the_static_check() -> None: + report = gate.check(ROOT, None) + assert report.problems == [] + # Never a vacuous pass: the docs tree carries thousands of citations. + assert report.citations > 1000 + assert report.pages >= 15 diff --git a/scripts/docs_citations.py b/scripts/docs_citations.py new file mode 100755 index 0000000..64818d8 --- /dev/null +++ b/scripts/docs_citations.py @@ -0,0 +1,488 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: Apache-2.0 +"""Fail when a docs citation names a line that isn't there, or code moved out from under it. + + scripts/docs_citations.py [--base REV] [--fix] [--label LABEL] [--root DIR] + +The docs cite code as `` `path:LINE` `` or `` `path:START-END` ``, and `:LINE` shorthands resolve +against the last full path in the same section (`site/src/lib/citation-links.ts` turns each one +into a permalink, with the same rules: the antecedent resets at every heading and every table +row). That build pass checks only that the path exists, so a line that moved, or a range past the +end of the file, ships as a permalink to the wrong code. This gate runs two checks over +`docs/**/*.md` and `site/authored/**/*.md` in the working tree: + +- static every citation names a tracked file, its lines exist in it (1 <= START <= END <= + the file's line count), and every shorthand has an antecedent +- drift against a base commit, every citation whose doc line this change leaves untouched + still names the lines it named at the base. The cited file's own diff (`git diff + -U0 BASE`) maps each cited line forward: + + ok every line unchanged at the same number + edited changed in place at the same number; reported, not failed, since the + line most likely still holds the construct the sentence names + moved the same text now sits at other numbers: fails, and `--fix` rewrites the + citation to them + changed the cited lines were removed, or rewritten and shifted: fails, because only + a reader can say where the named code went + +A citation on a doc line the change edits is out of the drift check: whoever edited the line +re-read it. That is also how a `changed` failure clears: re-point the citation, which edits its +line. The base is `--base`, else `ATIF_SQL_CITATIONS_BASE` (CI sets the pull request's base +commit), else `git merge-base HEAD origin/main`; a base that does not resolve is a usage error, +never a silent pass. + +Why this exists: on 2026-10-12 main held 2,177 citations, and an audit against that tree found +1,318 of them needing a re-point, a rewrite, or removal. Replayed over the 29 commits before the +audit, the drift check fails 19 of them, on 991 citations, and names the new lines for 934. + +stdlib only, 3.9-compatible, run with `python3` like the other gate helpers here. + +Exit 0 when every citation resolves and nothing drifted, 1 when one did not or the scope held no +citation, 2 on a usage error or a base that does not resolve. +""" + +from __future__ import annotations + +import os +import re +import subprocess +import sys +from dataclasses import dataclass, field +from pathlib import Path +from typing import TYPE_CHECKING, NoReturn + +if TYPE_CHECKING: + from collections.abc import Callable, Iterator, Sequence + +ROOT = Path(__file__).resolve().parent.parent + +#: The trees whose Markdown the site publishes and the citation pass links. +SCOPE = ("docs", "site/authored") + +#: The extensions a citation path may carry: `EXT` in `site/src/lib/citation-links.ts`, verbatim. +EXT = ( + "rs|py|pyi|toml|json|jsonc|json5|md|mdx|yml|yaml|mjs|cjs|js|jsx|ts|tsx|tf|tfvars|sh|bash|" + "zsh|lock|txt|typed|go|java|kt|kts|rb|php|cs|c|h|cc|cpp|hpp|swift|scala|sql|proto|graphql|" + "gradle|cfg|ini|mk|css|scss|html|vue|svelte|ipynb|xml|csv|snap" +) + +#: `CITATION` in `citation-links.ts`: an optional path, an optional `:START[-END]`, a label. +_CITATION = re.compile( + r"^(?:(?P[A-Za-z0-9_.\-]+):)??" + rf"(?P[A-Za-z0-9_./\-]+\.(?:{EXT}))?" + r"(?::(?P\d+)(?:-(?P\d+))?)?" + r"(?P