chore: real token usage, Retry-After, mypy in CI, ruff 0.16

.github/workflows/ci.yml

@@ -15,8 +15,11 @@ jobs:
       - uses: actions/setup-python@v5
         with:
           python-version: "3.12"
-      - run: pip install ruff==0.12.2  # bump deliberately: new releases add default rules
+          cache: "pip"
+      # Versions pinned in the dev extra; mypy needs the runtime deps' types.
+      - run: pip install -e ".[dev,review]"
       - run: ruff check src tests eval scripts examples
+      - run: mypy
 
   test:
     runs-on: ubuntu-latest

CHANGELOG.md

@@ -31,6 +31,13 @@ exported from `docket`, the `docket` / `docket-api` commands, the HTTP API in
 
 ### Added
 
+- Real LLM usage per document: `ProcessingMetrics.llm_input_tokens` /
+  `llm_output_tokens` from what the provider reports (Ollama
+  `prompt_eval_count`/`eval_count`, OpenAI `usage`), `llm_unreported_calls`
+  for calls that reported nothing, and `llm_models` for the models used.
+  `BatchMetrics` sums them. The chars/4 `llm_estimated_tokens` stays.
+- CI type-checks `src/docket` with mypy (pinned, like ruff, in the `dev`
+  extra); the codebase was brought to zero errors.
 - `corroborate(result, purchase_order=..., transactions=...)`: a purchase
   order total, or a bank payment of the total in the same currency, confirms
   that field; a payment whose text carries the invoice number confirms the
@@ -49,6 +56,11 @@ exported from `docket`, the `docket` / `docket-api` commands, the HTTP API in
 
 ### Changed
 
+- LLM retries honour `Retry-After` (seconds or an HTTP date, capped at
+  60 s) and otherwise back off exponentially with jitter, instead of a fixed
+  linear delay.
+- ruff moved to 0.16.8 and its wider default rule set; the codebase is clean
+  under it.
 - The low-OCR-confidence review rule looks only at the words holding the
   value's own digits, not the label beside it or another value on the same
   line; names are no longer gated; subtotal and total that reconcile across

docs/openapi.json

@@ -4292,9 +4292,36 @@
             "title": "Llm Calls",
             "default": 0
           },
+          "llm_models": {
+            "items": {
+              "type": "string"
+            },
+            "type": "array",
+            "title": "Llm Models",
+            "description": "Models called, in first-use order."
+          },
+          "llm_input_tokens": {
+            "type": "integer",
+            "title": "Llm Input Tokens",
+            "description": "Prompt tokens the provider reported.",
+            "default": 0
+          },
+          "llm_output_tokens": {
+            "type": "integer",
+            "title": "Llm Output Tokens",
+            "description": "Completion tokens the provider reported.",
+            "default": 0
+          },
+          "llm_unreported_calls": {
+            "type": "integer",
+            "title": "Llm Unreported Calls",
+            "description": "Calls whose provider reported no token counts; not in the two sums above.",
+            "default": 0
+          },
           "llm_estimated_tokens": {
             "type": "integer",
             "title": "Llm Estimated Tokens",
+            "description": "Characters / 4 over every call, reported or not.",
             "default": 0
           },
           "extract_attempts": {

eval/benchmark_competitors.py

@@ -49,7 +49,7 @@
 ROOT = EVAL.parent
 sys.path.insert(0, str(EVAL))
 
-from metrics import field_accuracy, source_of  # noqa: E402
+from metrics import field_accuracy, source_of
 
 DATASETS = [ROOT / "eval" / "golden_dataset", ROOT / "eval" / "real_samples"]
 SCANS = {".jpg", ".jpeg", ".png", ".pdf"}
@@ -149,7 +149,9 @@ def fail(path, expected, field_map, exc, started) -> dict:
 
 def run_invoice2data(docs, env):
     from invoice2data.api import extract_data
-    from invoice2data.input import paddleocr as paddle_reader  # its tesseract reader needs imagemagick; paddle is in the venv anyway
+    from invoice2data.input import (
+        paddleocr as paddle_reader,  # its tesseract reader needs imagemagick; paddle is in the venv anyway
+    )
 
     for path, expected in docs:
         if expected.get("doc_type") != "invoice" or path.suffix.lower() == ".pdf":
@@ -182,7 +184,7 @@ def run_docpick(docs, env):
 
     pipeline = DocpickPipeline()
     for path, expected in docs:
-        doc_type, schema, field_map = {
+        _doc_type, schema, field_map = {
             "invoice": ("invoice", InvoiceSchema, DOCPICK_INVOICE),
             "receipt": ("receipt", ReceiptSchema, DOCPICK_RECEIPT),
         }.get(expected.get("doc_type"), (None, None, None))

eval/benchmark_methods.py

@@ -20,11 +20,11 @@
 
 sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "src"))
 
-from docket.classify import classify_llm, classify_rules  # noqa: E402
-from docket.classify_tfidf import classify_tfidf  # noqa: E402
-from docket.llm_client import LLMError, vision_transcribe  # noqa: E402
-from docket.ocr import DocumentSource  # noqa: E402
-from docket.ocr.tesseract import TesseractBackend  # noqa: E402
+from docket.classify import classify_llm, classify_rules
+from docket.classify_tfidf import classify_tfidf
+from docket.llm_client import LLMError, vision_transcribe
+from docket.ocr import DocumentSource
+from docket.ocr.tesseract import TesseractBackend
 
 ROOT = Path(__file__).parent
 DIRS = [ROOT / "golden_dataset", ROOT / "real_samples"]

eval/benchmark_ocr.py

@@ -42,7 +42,7 @@
 sys.path.insert(0, str(ROOT / "src"))
 sys.path.insert(0, str(Path(__file__).resolve().parent))
 
-from metrics import (  # noqa: E402
+from metrics import (
     citation_coverage,
     citation_coverage_summary,
     expected_cells,
@@ -55,11 +55,11 @@
     word_scores,
 )
 
-from docket import __version__, catalog, config  # noqa: E402
-from docket.ocr import AcquisitionOptions, OcrSettings, acquire  # noqa: E402
-from docket.options import OcrOptions, ProcessOptions, ReviewOptions  # noqa: E402
-from docket.pipeline import process_document  # noqa: E402
-from docket.review_reasons import key_field_confidence  # noqa: E402
+from docket import __version__, catalog, config
+from docket.ocr import AcquisitionOptions, OcrSettings, acquire
+from docket.options import OcrOptions, ProcessOptions, ReviewOptions
+from docket.pipeline import process_document
+from docket.review_reasons import key_field_confidence
 
 DATASETS = [ROOT / "eval" / "golden_dataset", ROOT / "eval" / "real_samples"]
 IMAGE_SUFFIXES = {".png", ".jpg", ".jpeg", ".tif", ".tiff", ".bmp", ".webp"}
@@ -332,11 +332,11 @@ def version(module: str) -> str | None:
             return None
 
     try:
-        tesseract = subprocess.run(["tesseract", "--version"], capture_output=True, text=True).stdout.split("\n")[0]
+        tesseract = subprocess.run(["tesseract", "--version"], capture_output=True, text=True, check=False).stdout.split("\n")[0]
     except OSError:
         tesseract = None
     try:
-        commit = subprocess.run(["git", "rev-parse", "--short", "HEAD"], capture_output=True, text=True,
+        commit = subprocess.run(["git", "rev-parse", "--short", "HEAD"], capture_output=True, text=True, check=False,
                                 cwd=ROOT).stdout.strip()
     except OSError:
         commit = None

eval/benchmark_variance.py

@@ -25,7 +25,7 @@
 ROOT = Path(__file__).resolve().parent.parent
 sys.path.insert(0, str(ROOT / "src"))
 
-from docket import __version__, process_document  # noqa: E402
+from docket import __version__, process_document
 
 DEFAULT_DOCUMENT = ROOT / "eval" / "golden_dataset" / "receipt_taxed.txt"
 DEFAULT_OUT = ROOT / "eval" / "results" / "llm_variance.json"

eval/build_golden.py

@@ -405,22 +405,22 @@ class Doc:
         pages=[[
             ("title", "SOFTWARE MAINTENANCE AGREEMENT"),
             ("gap", 30),
-            ("para", "This Software Maintenance Agreement (the \"Agreement\") is entered into as of "
+            ("para", ("This Software Maintenance Agreement (the \"Agreement\") is entered into as of "
                      "January 1, 2026 (the \"Effective Date\") by and between Helix Data Systems Ltd, a company "
                      "registered in Ireland (the \"Provider\"), and Greenfield Municipal Utilities, a public "
-                     "utility (the \"Customer\")."),
+                     "utility (the \"Customer\").")),
             ("gap", 20),
-            ("para", "1. Services. The Provider shall maintain and support the Customer's billing platform, "
-                     "including corrective releases, security patches and a service desk available on business days."),
-            ("para", "2. Term. The Agreement runs for twenty-four months from the Effective Date and ends on "
+            ("para", ("1. Services. The Provider shall maintain and support the Customer's billing platform, "
+                     "including corrective releases, security patches and a service desk available on business days.")),
+            ("para", ("2. Term. The Agreement runs for twenty-four months from the Effective Date and ends on "
                      "December 31, 2027. It renews automatically for successive twelve-month periods unless either "
-                     "party gives at least ninety (90) days' written notice before the end of the current term."),
-            ("para", "3. Fees. The Customer shall pay a total fee of EUR 96,000.00, invoiced in eight equal "
-                     "quarterly instalments and payable within thirty days of the invoice date."),
-            ("para", "4. Breach. Either party may terminate this Agreement if the other party materially breaches it "
-                     "and fails to cure the breach within thirty (30) days of written notice."),
-            ("para", "5. Liability. Each party's aggregate liability is limited to the fees paid in the twelve "
-                     "months preceding the claim."),
+                     "party gives at least ninety (90) days' written notice before the end of the current term.")),
+            ("para", ("3. Fees. The Customer shall pay a total fee of EUR 96,000.00, invoiced in eight equal "
+                     "quarterly instalments and payable within thirty days of the invoice date.")),
+            ("para", ("4. Breach. Either party may terminate this Agreement if the other party materially breaches it "
+                     "and fails to cure the breach within thirty (30) days of written notice.")),
+            ("para", ("5. Liability. Each party's aggregate liability is limited to the fees paid in the twelve "
+                     "months preceding the claim.")),
             ("para", "6. Governing Law. This Agreement is governed by the laws of Ireland."),
             ("gap", 40),
             ("cols", ["For Helix Data Systems Ltd", "Name: Ciara Byrne", "Title: Managing Director"],

eval/download_real_samples.py

@@ -228,7 +228,7 @@ def download_rvl_cdip(n: int) -> None:
         (OUT_DIR / f"{stem}.expected.json").write_text(json.dumps({"doc_type": doc_type}, indent=2))
 
 
-_BILL_TITLE_RE = re.compile(r"^(invoice|bill|billing|billed)\b", re.I)
+_BILL_TITLE_RE = re.compile(r"^(invoice|bill|billing|billed)\b", re.IGNORECASE)
 
 
 def _titled_as_invoice(image) -> bool:

eval/metrics.py

@@ -281,7 +281,7 @@ def citation_coverage(result, expected: dict) -> dict:
         from docket.catalog import get_schema
         spec = get_schema(result.schema_id) if result.schema_id else None
         spec_items = spec.line_items if spec is not None else None
-    except Exception:
+    except Exception:  # noqa: BLE001 — an unknown schema just has no rows to grade
         spec_items = None
     if spec_items is not None:
         item_fields = [

eval/run_eval.py

@@ -15,16 +15,16 @@
 
 sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "src"))
 
-from metrics import (  # noqa: E402
+from metrics import (
+    error_breakdown,
     field_accuracy,
     field_precision_recall_f1,
-    error_breakdown,
     review_safety_metrics,
     validation_detection_metrics,
 )
 
-from docket import config  # noqa: E402
-from docket.pipeline import process_document  # noqa: E402
+from docket import config
+from docket.pipeline import process_document
 
 DEFAULT_DIR = Path(__file__).parent / "golden_dataset"
 RESULTS_DIR = Path(__file__).parent / "results"

examples/export_einvoice.py

@@ -10,7 +10,14 @@
 """
 import sys
 
-from docket import ExportError, ExportOptions, ProcessOptions, ReviewOptions, export_document, process_document
+from docket import (
+    ExportError,
+    ExportOptions,
+    ProcessOptions,
+    ReviewOptions,
+    export_document,
+    process_document,
+)
 
 path, fmt = sys.argv[1], sys.argv[2] if len(sys.argv) > 2 else "xrechnung-ubl"
 result = process_document(path, ProcessOptions(document_type="invoice", review=ReviewOptions(enqueue=False)))

examples/streamlit_demo.py

@@ -15,8 +15,7 @@
 
 import streamlit as st
 
-from docket import config, llm_client, pdf
-from docket import DocumentResult, process_document
+from docket import DocumentResult, config, llm_client, pdf, process_document
 
 # An application, so it reads .env and ./docket.toml like the CLI does
 # (importing docket reads only the environment). Once per session: the
@@ -71,9 +70,8 @@ def _pdf_preview(path: str, mtime: float) -> tuple[list, int]:
         uploaded = st.file_uploader("Document", type=["pdf", "png", "jpg", "jpeg", "txt"])
         if uploaded is not None:
             suffix = Path(uploaded.name).suffix
-            tmp = tempfile.NamedTemporaryFile(delete=False, suffix=suffix)
-            tmp.write(uploaded.read())
-            tmp.close()
+            with tempfile.NamedTemporaryFile(delete=False, suffix=suffix) as tmp:
+                tmp.write(uploaded.read())
             uploaded_path = Path(tmp.name)
 
     st.divider()

pyproject.toml

@@ -98,6 +98,9 @@ dev = [
     "pymupdf>=1.24",
     "build>=1.2",
     "twine>=5.0",
+    # Pinned: new releases add default rules; bump deliberately, with the fixes.
+    "ruff==0.16.8",
+    "mypy==1.18.2",
 ]
 
 [project.scripts]
@@ -130,6 +133,21 @@ docket = [
     "einvoice/resources/cii-d16b/**/*",
 ]
 
+[tool.ruff.lint]
+ignore = [
+    "C408",  # dict(key=...) reads better than a literal in kwargs-style test factories
+    "DTZ",   # document dates are calendar dates, not instants; there is no zone to attach
+]
+
+[tool.ruff.lint.flake8-bugbear]
+# FastAPI declares request parameters as call defaults by design.
+extend-immutable-calls = ["fastapi.File", "fastapi.Form", "fastapi.Depends", "fastapi.Header"]
+
+[tool.mypy]
+files = ["src/docket"]
+# Optional extras (paddleocr, docling, saxonche, langfuse, ...) ship no stubs.
+ignore_missing_imports = true
+
 [tool.pytest.ini_options]
 testpaths = ["tests"]
 markers = [

scripts/update_einvoice_resources.py

@@ -31,7 +31,13 @@
 FIXTURES = ROOT / "tests" / "fixtures" / "einvoice" / "official"
 
 sys.path.insert(0, str(ROOT / "src"))
-from docket.einvoice.sources import SCHXSLT, SOURCES, SourceError, install, sha256  # noqa: E402
+from docket.einvoice.sources import (
+    SCHXSLT,
+    SOURCES,
+    SourceError,
+    install,
+    sha256,
+)
 
 
 def build(cache: Path) -> None: