From 71d2889292a4c1dd3ce65d6e44d6ae50aaa85713 Mon Sep 17 00:00:00 2001 From: KazKozDev Date: Thu, 24 Sep 2026 14:26:51 +0200 Subject: [PATCH] chore: real token usage, Retry-After, mypy in CI, ruff 0.16 - llm_client records provider-reported input/output tokens and models per document; ProcessingMetrics and BatchMetrics expose them. - Retries honour Retry-After (capped at 60 s) or back off exponentially with jitter. - mypy over src/docket runs in the CI lint job; 115 errors fixed. - ruff 0.16.8 (pinned in the dev extra); 201 findings fixed, C408 and DTZ ignored and FastAPI's File/Form defaults allowed, with reasons in config. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/ci.yml | 5 +- CHANGELOG.md | 12 ++ docs/openapi.json | 27 ++++ eval/benchmark_competitors.py | 8 +- eval/benchmark_methods.py | 10 +- eval/benchmark_ocr.py | 16 +- eval/benchmark_variance.py | 2 +- eval/build_golden.py | 24 +-- eval/download_real_samples.py | 2 +- eval/metrics.py | 2 +- eval/run_eval.py | 8 +- examples/export_einvoice.py | 9 +- examples/streamlit_demo.py | 8 +- pyproject.toml | 18 +++ scripts/update_einvoice_resources.py | 8 +- src/docket/__init__.py | 182 +++++++++++------------ src/docket/api.py | 27 ++-- src/docket/batch.py | 23 ++- src/docket/catalog/__init__.py | 18 +-- src/docket/catalog/builtin.py | 14 +- src/docket/catalog/corpus.py | 2 +- src/docket/catalog/registry.py | 28 ++-- src/docket/catalog/rules.py | 1 - src/docket/checksums.py | 4 +- src/docket/classify.py | 3 +- src/docket/cli.py | 18 +-- src/docket/config.py | 14 +- src/docket/corroboration.py | 3 +- src/docket/einvoice/__init__.py | 32 ++-- src/docket/einvoice/facturx_pdf.py | 12 +- src/docket/einvoice/validator.py | 9 +- src/docket/export/__init__.py | 16 +- src/docket/export/en16931.py | 16 +- src/docket/export/facturae.py | 1 + src/docket/export/tabular.py | 16 +- src/docket/extract.py | 4 +- src/docket/job_store.py | 9 +- src/docket/layout/__init__.py | 9 +- src/docket/layout/analysis.py | 17 ++- src/docket/layout/models.py | 11 +- src/docket/limits.py | 2 +- src/docket/llm_client.py | 134 +++++++++++++---- src/docket/matching.py | 71 +++++---- src/docket/ocr/__init__.py | 4 +- src/docket/ocr/acquire.py | 17 ++- src/docket/ocr/base.py | 4 +- src/docket/ocr/deskew.py | 5 +- src/docket/ocr/docling.py | 13 +- src/docket/ocr/languages.py | 2 +- src/docket/ocr/paddle.py | 8 +- src/docket/ocr/pdftext.py | 8 +- src/docket/ocr/registry.py | 14 +- src/docket/ocr/source.py | 13 +- src/docket/ocr_quality.py | 2 +- src/docket/options.py | 14 +- src/docket/pii.py | 5 +- src/docket/pipeline.py | 17 ++- src/docket/result.py | 10 +- src/docket/review_queue.py | 33 +++- src/docket/review_reasons.py | 8 +- src/docket/templates.py | 22 +-- src/docket/validate.py | 32 ++-- tests/factories.py | 8 +- tests/security/test_hostile_documents.py | 4 +- tests/snapshots/public_api.json | 10 +- tests/test_batch.py | 15 +- tests/test_catalog_rules.py | 1 - tests/test_classify.py | 1 - tests/test_classify_tfidf.py | 1 - tests/test_config.py | 5 +- tests/test_custom_schemas.py | 14 +- tests/test_document_result.py | 5 +- tests/test_einvoice.py | 6 +- tests/test_escalation.py | 8 +- tests/test_eval_scripts.py | 15 +- tests/test_export.py | 2 +- tests/test_http_api.py | 3 +- tests/test_integration_api.py | 10 +- tests/test_layout.py | 4 +- tests/test_line_item_citations.py | 5 +- tests/test_llm_retries.py | 38 ++++- tests/test_long_documents.py | 1 - tests/test_matching.py | 2 +- tests/test_metrics.py | 4 +- tests/test_model_selection.py | 6 +- tests/test_ocr_deskew.py | 2 +- tests/test_ocr_paddle.py | 10 +- tests/test_page_validation.py | 3 +- tests/test_pipeline_options.py | 7 +- tests/test_public_api.py | 4 +- tests/test_review_queue.py | 2 +- tests/test_schemas.py | 11 +- tests/test_templates.py | 14 +- tests/test_tracing.py | 13 +- tests/test_validate.py | 12 +- tests/test_witness.py | 6 +- 96 files changed, 864 insertions(+), 474 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 940f69c..54d2cca 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,8 +15,11 @@ jobs: - uses: actions/setup-python@v5 with: python-version: "3.12" - - run: pip install ruff==0.12.2 # bump deliberately: new releases add default rules + cache: "pip" + # Versions pinned in the dev extra; mypy needs the runtime deps' types. + - run: pip install -e ".[dev,review]" - run: ruff check src tests eval scripts examples + - run: mypy test: runs-on: ubuntu-latest diff --git a/CHANGELOG.md b/CHANGELOG.md index 4c9e8ff..5614035 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,6 +31,13 @@ exported from `docket`, the `docket` / `docket-api` commands, the HTTP API in ### Added +- Real LLM usage per document: `ProcessingMetrics.llm_input_tokens` / + `llm_output_tokens` from what the provider reports (Ollama + `prompt_eval_count`/`eval_count`, OpenAI `usage`), `llm_unreported_calls` + for calls that reported nothing, and `llm_models` for the models used. + `BatchMetrics` sums them. The chars/4 `llm_estimated_tokens` stays. +- CI type-checks `src/docket` with mypy (pinned, like ruff, in the `dev` + extra); the codebase was brought to zero errors. - `corroborate(result, purchase_order=..., transactions=...)`: a purchase order total, or a bank payment of the total in the same currency, confirms that field; a payment whose text carries the invoice number confirms the @@ -49,6 +56,11 @@ exported from `docket`, the `docket` / `docket-api` commands, the HTTP API in ### Changed +- LLM retries honour `Retry-After` (seconds or an HTTP date, capped at + 60 s) and otherwise back off exponentially with jitter, instead of a fixed + linear delay. +- ruff moved to 0.16.8 and its wider default rule set; the codebase is clean + under it. - The low-OCR-confidence review rule looks only at the words holding the value's own digits, not the label beside it or another value on the same line; names are no longer gated; subtotal and total that reconcile across diff --git a/docs/openapi.json b/docs/openapi.json index debe4e8..c6198e0 100644 --- a/docs/openapi.json +++ b/docs/openapi.json @@ -4292,9 +4292,36 @@ "title": "Llm Calls", "default": 0 }, + "llm_models": { + "items": { + "type": "string" + }, + "type": "array", + "title": "Llm Models", + "description": "Models called, in first-use order." + }, + "llm_input_tokens": { + "type": "integer", + "title": "Llm Input Tokens", + "description": "Prompt tokens the provider reported.", + "default": 0 + }, + "llm_output_tokens": { + "type": "integer", + "title": "Llm Output Tokens", + "description": "Completion tokens the provider reported.", + "default": 0 + }, + "llm_unreported_calls": { + "type": "integer", + "title": "Llm Unreported Calls", + "description": "Calls whose provider reported no token counts; not in the two sums above.", + "default": 0 + }, "llm_estimated_tokens": { "type": "integer", "title": "Llm Estimated Tokens", + "description": "Characters / 4 over every call, reported or not.", "default": 0 }, "extract_attempts": { diff --git a/eval/benchmark_competitors.py b/eval/benchmark_competitors.py index 49b0e8b..3b00e3f 100644 --- a/eval/benchmark_competitors.py +++ b/eval/benchmark_competitors.py @@ -49,7 +49,7 @@ ROOT = EVAL.parent sys.path.insert(0, str(EVAL)) -from metrics import field_accuracy, source_of # noqa: E402 +from metrics import field_accuracy, source_of DATASETS = [ROOT / "eval" / "golden_dataset", ROOT / "eval" / "real_samples"] SCANS = {".jpg", ".jpeg", ".png", ".pdf"} @@ -149,7 +149,9 @@ def fail(path, expected, field_map, exc, started) -> dict: def run_invoice2data(docs, env): from invoice2data.api import extract_data - from invoice2data.input import paddleocr as paddle_reader # its tesseract reader needs imagemagick; paddle is in the venv anyway + from invoice2data.input import ( + paddleocr as paddle_reader, # its tesseract reader needs imagemagick; paddle is in the venv anyway + ) for path, expected in docs: if expected.get("doc_type") != "invoice" or path.suffix.lower() == ".pdf": @@ -182,7 +184,7 @@ def run_docpick(docs, env): pipeline = DocpickPipeline() for path, expected in docs: - doc_type, schema, field_map = { + _doc_type, schema, field_map = { "invoice": ("invoice", InvoiceSchema, DOCPICK_INVOICE), "receipt": ("receipt", ReceiptSchema, DOCPICK_RECEIPT), }.get(expected.get("doc_type"), (None, None, None)) diff --git a/eval/benchmark_methods.py b/eval/benchmark_methods.py index 3120786..7167da5 100644 --- a/eval/benchmark_methods.py +++ b/eval/benchmark_methods.py @@ -20,11 +20,11 @@ sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "src")) -from docket.classify import classify_llm, classify_rules # noqa: E402 -from docket.classify_tfidf import classify_tfidf # noqa: E402 -from docket.llm_client import LLMError, vision_transcribe # noqa: E402 -from docket.ocr import DocumentSource # noqa: E402 -from docket.ocr.tesseract import TesseractBackend # noqa: E402 +from docket.classify import classify_llm, classify_rules +from docket.classify_tfidf import classify_tfidf +from docket.llm_client import LLMError, vision_transcribe +from docket.ocr import DocumentSource +from docket.ocr.tesseract import TesseractBackend ROOT = Path(__file__).parent DIRS = [ROOT / "golden_dataset", ROOT / "real_samples"] diff --git a/eval/benchmark_ocr.py b/eval/benchmark_ocr.py index 4cb89df..d824ee3 100644 --- a/eval/benchmark_ocr.py +++ b/eval/benchmark_ocr.py @@ -42,7 +42,7 @@ sys.path.insert(0, str(ROOT / "src")) sys.path.insert(0, str(Path(__file__).resolve().parent)) -from metrics import ( # noqa: E402 +from metrics import ( citation_coverage, citation_coverage_summary, expected_cells, @@ -55,11 +55,11 @@ word_scores, ) -from docket import __version__, catalog, config # noqa: E402 -from docket.ocr import AcquisitionOptions, OcrSettings, acquire # noqa: E402 -from docket.options import OcrOptions, ProcessOptions, ReviewOptions # noqa: E402 -from docket.pipeline import process_document # noqa: E402 -from docket.review_reasons import key_field_confidence # noqa: E402 +from docket import __version__, catalog, config +from docket.ocr import AcquisitionOptions, OcrSettings, acquire +from docket.options import OcrOptions, ProcessOptions, ReviewOptions +from docket.pipeline import process_document +from docket.review_reasons import key_field_confidence DATASETS = [ROOT / "eval" / "golden_dataset", ROOT / "eval" / "real_samples"] IMAGE_SUFFIXES = {".png", ".jpg", ".jpeg", ".tif", ".tiff", ".bmp", ".webp"} @@ -332,11 +332,11 @@ def version(module: str) -> str | None: return None try: - tesseract = subprocess.run(["tesseract", "--version"], capture_output=True, text=True).stdout.split("\n")[0] + tesseract = subprocess.run(["tesseract", "--version"], capture_output=True, text=True, check=False).stdout.split("\n")[0] except OSError: tesseract = None try: - commit = subprocess.run(["git", "rev-parse", "--short", "HEAD"], capture_output=True, text=True, + commit = subprocess.run(["git", "rev-parse", "--short", "HEAD"], capture_output=True, text=True, check=False, cwd=ROOT).stdout.strip() except OSError: commit = None diff --git a/eval/benchmark_variance.py b/eval/benchmark_variance.py index 660c1d8..332f8ff 100644 --- a/eval/benchmark_variance.py +++ b/eval/benchmark_variance.py @@ -25,7 +25,7 @@ ROOT = Path(__file__).resolve().parent.parent sys.path.insert(0, str(ROOT / "src")) -from docket import __version__, process_document # noqa: E402 +from docket import __version__, process_document DEFAULT_DOCUMENT = ROOT / "eval" / "golden_dataset" / "receipt_taxed.txt" DEFAULT_OUT = ROOT / "eval" / "results" / "llm_variance.json" diff --git a/eval/build_golden.py b/eval/build_golden.py index de9f249..13f503b 100644 --- a/eval/build_golden.py +++ b/eval/build_golden.py @@ -405,22 +405,22 @@ class Doc: pages=[[ ("title", "SOFTWARE MAINTENANCE AGREEMENT"), ("gap", 30), - ("para", "This Software Maintenance Agreement (the \"Agreement\") is entered into as of " + ("para", ("This Software Maintenance Agreement (the \"Agreement\") is entered into as of " "January 1, 2026 (the \"Effective Date\") by and between Helix Data Systems Ltd, a company " "registered in Ireland (the \"Provider\"), and Greenfield Municipal Utilities, a public " - "utility (the \"Customer\")."), + "utility (the \"Customer\").")), ("gap", 20), - ("para", "1. Services. The Provider shall maintain and support the Customer's billing platform, " - "including corrective releases, security patches and a service desk available on business days."), - ("para", "2. Term. The Agreement runs for twenty-four months from the Effective Date and ends on " + ("para", ("1. Services. The Provider shall maintain and support the Customer's billing platform, " + "including corrective releases, security patches and a service desk available on business days.")), + ("para", ("2. Term. The Agreement runs for twenty-four months from the Effective Date and ends on " "December 31, 2027. It renews automatically for successive twelve-month periods unless either " - "party gives at least ninety (90) days' written notice before the end of the current term."), - ("para", "3. Fees. The Customer shall pay a total fee of EUR 96,000.00, invoiced in eight equal " - "quarterly instalments and payable within thirty days of the invoice date."), - ("para", "4. Breach. Either party may terminate this Agreement if the other party materially breaches it " - "and fails to cure the breach within thirty (30) days of written notice."), - ("para", "5. Liability. Each party's aggregate liability is limited to the fees paid in the twelve " - "months preceding the claim."), + "party gives at least ninety (90) days' written notice before the end of the current term.")), + ("para", ("3. Fees. The Customer shall pay a total fee of EUR 96,000.00, invoiced in eight equal " + "quarterly instalments and payable within thirty days of the invoice date.")), + ("para", ("4. Breach. Either party may terminate this Agreement if the other party materially breaches it " + "and fails to cure the breach within thirty (30) days of written notice.")), + ("para", ("5. Liability. Each party's aggregate liability is limited to the fees paid in the twelve " + "months preceding the claim.")), ("para", "6. Governing Law. This Agreement is governed by the laws of Ireland."), ("gap", 40), ("cols", ["For Helix Data Systems Ltd", "Name: Ciara Byrne", "Title: Managing Director"], diff --git a/eval/download_real_samples.py b/eval/download_real_samples.py index 177a154..4721e64 100644 --- a/eval/download_real_samples.py +++ b/eval/download_real_samples.py @@ -228,7 +228,7 @@ def download_rvl_cdip(n: int) -> None: (OUT_DIR / f"{stem}.expected.json").write_text(json.dumps({"doc_type": doc_type}, indent=2)) -_BILL_TITLE_RE = re.compile(r"^(invoice|bill|billing|billed)\b", re.I) +_BILL_TITLE_RE = re.compile(r"^(invoice|bill|billing|billed)\b", re.IGNORECASE) def _titled_as_invoice(image) -> bool: diff --git a/eval/metrics.py b/eval/metrics.py index f9f2c10..4fdffe9 100644 --- a/eval/metrics.py +++ b/eval/metrics.py @@ -281,7 +281,7 @@ def citation_coverage(result, expected: dict) -> dict: from docket.catalog import get_schema spec = get_schema(result.schema_id) if result.schema_id else None spec_items = spec.line_items if spec is not None else None - except Exception: + except Exception: # noqa: BLE001 — an unknown schema just has no rows to grade spec_items = None if spec_items is not None: item_fields = [ diff --git a/eval/run_eval.py b/eval/run_eval.py index 9c15b77..37d5bf6 100644 --- a/eval/run_eval.py +++ b/eval/run_eval.py @@ -15,16 +15,16 @@ sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "src")) -from metrics import ( # noqa: E402 +from metrics import ( + error_breakdown, field_accuracy, field_precision_recall_f1, - error_breakdown, review_safety_metrics, validation_detection_metrics, ) -from docket import config # noqa: E402 -from docket.pipeline import process_document # noqa: E402 +from docket import config +from docket.pipeline import process_document DEFAULT_DIR = Path(__file__).parent / "golden_dataset" RESULTS_DIR = Path(__file__).parent / "results" diff --git a/examples/export_einvoice.py b/examples/export_einvoice.py index a9ea561..cf5148b 100644 --- a/examples/export_einvoice.py +++ b/examples/export_einvoice.py @@ -10,7 +10,14 @@ """ import sys -from docket import ExportError, ExportOptions, ProcessOptions, ReviewOptions, export_document, process_document +from docket import ( + ExportError, + ExportOptions, + ProcessOptions, + ReviewOptions, + export_document, + process_document, +) path, fmt = sys.argv[1], sys.argv[2] if len(sys.argv) > 2 else "xrechnung-ubl" result = process_document(path, ProcessOptions(document_type="invoice", review=ReviewOptions(enqueue=False))) diff --git a/examples/streamlit_demo.py b/examples/streamlit_demo.py index de782c8..4dc1b30 100644 --- a/examples/streamlit_demo.py +++ b/examples/streamlit_demo.py @@ -15,8 +15,7 @@ import streamlit as st -from docket import config, llm_client, pdf -from docket import DocumentResult, process_document +from docket import DocumentResult, config, llm_client, pdf, process_document # An application, so it reads .env and ./docket.toml like the CLI does # (importing docket reads only the environment). Once per session: the @@ -71,9 +70,8 @@ def _pdf_preview(path: str, mtime: float) -> tuple[list, int]: uploaded = st.file_uploader("Document", type=["pdf", "png", "jpg", "jpeg", "txt"]) if uploaded is not None: suffix = Path(uploaded.name).suffix - tmp = tempfile.NamedTemporaryFile(delete=False, suffix=suffix) - tmp.write(uploaded.read()) - tmp.close() + with tempfile.NamedTemporaryFile(delete=False, suffix=suffix) as tmp: + tmp.write(uploaded.read()) uploaded_path = Path(tmp.name) st.divider() diff --git a/pyproject.toml b/pyproject.toml index 36fae85..fea8fef 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -98,6 +98,9 @@ dev = [ "pymupdf>=1.24", "build>=1.2", "twine>=5.0", + # Pinned: new releases add default rules; bump deliberately, with the fixes. + "ruff==0.16.8", + "mypy==1.18.2", ] [project.scripts] @@ -130,6 +133,21 @@ docket = [ "einvoice/resources/cii-d16b/**/*", ] +[tool.ruff.lint] +ignore = [ + "C408", # dict(key=...) reads better than a literal in kwargs-style test factories + "DTZ", # document dates are calendar dates, not instants; there is no zone to attach +] + +[tool.ruff.lint.flake8-bugbear] +# FastAPI declares request parameters as call defaults by design. +extend-immutable-calls = ["fastapi.File", "fastapi.Form", "fastapi.Depends", "fastapi.Header"] + +[tool.mypy] +files = ["src/docket"] +# Optional extras (paddleocr, docling, saxonche, langfuse, ...) ship no stubs. +ignore_missing_imports = true + [tool.pytest.ini_options] testpaths = ["tests"] markers = [ diff --git a/scripts/update_einvoice_resources.py b/scripts/update_einvoice_resources.py index 32640cc..e378fc8 100644 --- a/scripts/update_einvoice_resources.py +++ b/scripts/update_einvoice_resources.py @@ -31,7 +31,13 @@ FIXTURES = ROOT / "tests" / "fixtures" / "einvoice" / "official" sys.path.insert(0, str(ROOT / "src")) -from docket.einvoice.sources import SCHXSLT, SOURCES, SourceError, install, sha256 # noqa: E402 +from docket.einvoice.sources import ( + SCHXSLT, + SOURCES, + SourceError, + install, + sha256, +) def build(cache: Path) -> None: diff --git a/src/docket/__init__.py b/src/docket/__init__.py index ebb8dde..8d47c16 100644 --- a/src/docket/__init__.py +++ b/src/docket/__init__.py @@ -7,22 +7,7 @@ __version__ = "0.4.0" -from .export import ( - ExportError, - ExportOptions, - ExportResult, - export_document, - get_exporter, - list_exporters, - register_exporter, - export_to_facturae_xml, -) -from .matching import ( - match_invoice_to_po, - match_invoices_to_contract, - match_receipt_to_transactions, - match_three_way, -) +from .batch import BatchError, BatchMetrics, BatchOptions, BatchResult, process_batch from .catalog import ( Address, BankAccount, @@ -53,6 +38,35 @@ register_schema, unregister_schema, ) +from .corroboration import corroborate +from .einvoice import ( + EInvoiceResourcesMissing, + EInvoiceUnavailable, + EInvoiceValidationOptions, + EInvoiceValidationResult, + FacturXPdfUnavailable, + FacturXRoundTripResult, + PdfAValidationIssue, + PdfAValidationResult, + extract_facturx_xml, + fetch_einvoice_resources, + generate_facturx_pdf, + validate_einvoice, + validate_pdfa, + verify_facturx_round_trip, +) +from .einvoice import Profile as EInvoiceProfile +from .errors import ConfigurationError +from .export import ( + ExportError, + ExportOptions, + ExportResult, + export_document, + export_to_facturae_xml, + get_exporter, + list_exporters, + register_exporter, +) from .layout import ( BoundingBox, DocumentLayout, @@ -62,6 +76,12 @@ TextLine, WordToken, ) +from .matching import ( + match_invoice_to_po, + match_invoices_to_contract, + match_receipt_to_transactions, + match_three_way, +) from .ocr import ( BackendUnavailable, Capabilities, @@ -71,30 +91,9 @@ list_ocr_backends, register_ocr_backend, ) -from .batch import BatchError, BatchMetrics, BatchOptions, BatchResult, process_batch -from .einvoice import ( - EInvoiceResourcesMissing, - EInvoiceUnavailable, - EInvoiceValidationOptions, - EInvoiceValidationResult, - FacturXPdfUnavailable, - FacturXRoundTripResult, - PdfAValidationIssue, - PdfAValidationResult, - extract_facturx_xml, - fetch_einvoice_resources, - generate_facturx_pdf, - validate_pdfa, - validate_einvoice, - verify_facturx_round_trip, -) -from .einvoice import Profile as EInvoiceProfile -from .errors import ConfigurationError from .options import OcrOptions, ProcessOptions, ReviewOptions -from .pipeline import process_document -from .verification import verify -from .corroboration import corroborate from .pii import PiiCategory, PiiField, pii_fields +from .pipeline import process_document from .result import ( DocumentError, DocumentResult, @@ -120,35 +119,18 @@ register_vendor_template, unregister_vendor_template, ) +from .verification import verify __all__ = [ - "__version__", - "EInvoiceProfile", - "EInvoiceResourcesMissing", - "EInvoiceUnavailable", - "EInvoiceValidationOptions", - "EInvoiceValidationResult", - "FacturXPdfUnavailable", - "FacturXRoundTripResult", - "PdfAValidationIssue", - "PdfAValidationResult", - "extract_facturx_xml", - "fetch_einvoice_resources", - "generate_facturx_pdf", - "validate_pdfa", - "validate_einvoice", - "verify_facturx_round_trip", - "BatchError", - "BatchMetrics", - "BatchOptions", - "BatchResult", - "process_batch", - "add_validator", "Address", "BackendUnavailable", "BankAccount", "BankStatement", "BankTransaction", + "BatchError", + "BatchMetrics", + "BatchOptions", + "BatchResult", "BoundingBox", "Capabilities", "Citation", @@ -164,47 +146,36 @@ "DocumentReference", "DocumentResult", "DocumentStatus", - "export_document", - "export_to_facturae_xml", + "EInvoiceProfile", + "EInvoiceResourcesMissing", + "EInvoiceUnavailable", + "EInvoiceValidationOptions", + "EInvoiceValidationResult", "ExportError", "ExportOptions", "ExportResult", - "get_exporter", - "get_ocr_backend", - "get_schema", + "FacturXPdfUnavailable", + "FacturXRoundTripResult", + "FieldRule", "Invoice", - "keywords", + "ItemsRule", "LineItem", - "list_exporters", - "list_ocr_backends", - "list_schemas", - "load_schema", - "match_invoice_to_po", - "match_invoices_to_contract", - "match_receipt_to_transactions", - "match_three_way", - "MatchingStatus", "MatchResult", - "migrate", + "MatchingStatus", "Money", "OcrBackend", "OcrOptions", "OcrSettings", "PageLayout", "Party", - "process_document", - "corroborate", - "verify", - "pii_fields", + "PdfAValidationIssue", + "PdfAValidationResult", "PiiCategory", "PiiField", - "ProcessingMetrics", "ProcessOptions", + "ProcessingMetrics", "PurchaseOrder", "Receipt", - "register_exporter", - "register_ocr_backend", - "register_schema", "ReviewOptions", "SchemaError", "SchemaInfo", @@ -214,16 +185,45 @@ "TableCell", "TaxIdentifier", "TextLine", - "FieldRule", - "ItemsRule", + "ValidationContext", + "ValidationIssue", "VendorTemplate", + "Waybill", + "WordToken", + "__version__", + "add_validator", + "corroborate", + "export_document", + "export_to_facturae_xml", + "extract_facturx_xml", + "fetch_einvoice_resources", + "generate_facturx_pdf", + "get_exporter", + "get_ocr_backend", + "get_schema", "get_vendor_template", + "keywords", + "list_exporters", + "list_ocr_backends", + "list_schemas", "list_vendor_templates", + "load_schema", + "match_invoice_to_po", + "match_invoices_to_contract", + "match_receipt_to_transactions", + "match_three_way", + "migrate", + "pii_fields", + "process_batch", + "process_document", + "register_exporter", + "register_ocr_backend", + "register_schema", "register_vendor_template", - "unregister_vendor_template", "unregister_schema", - "ValidationContext", - "ValidationIssue", - "Waybill", - "WordToken", + "unregister_vendor_template", + "validate_einvoice", + "validate_pdfa", + "verify", + "verify_facturx_round_trip", ] diff --git a/src/docket/api.py b/src/docket/api.py index c08898f..1d21854 100644 --- a/src/docket/api.py +++ b/src/docket/api.py @@ -25,15 +25,16 @@ import asyncio import io +from collections.abc import Iterator from contextlib import asynccontextmanager from pathlib import Path -from typing import Iterator +from typing import Any from uuid import uuid4 from fastapi import Depends, FastAPI, File, Form, Header, Request, UploadFile from fastapi.exceptions import RequestValidationError -from fastapi.responses import FileResponse, JSONResponse, Response, StreamingResponse from fastapi.middleware.cors import CORSMiddleware +from fastapi.responses import FileResponse, JSONResponse, Response, StreamingResponse from pydantic import BaseModel from starlette.concurrency import run_in_threadpool from starlette.exceptions import HTTPException as StarletteHTTPException @@ -41,11 +42,15 @@ from . import __version__, catalog, config, job_store, review_queue from .batch import BatchOptions, process_batch from .catalog import SchemaError, SchemaInfo -from .einvoice import EInvoiceUnavailable, EInvoiceValidationOptions, EInvoiceValidationResult, Profile +from .einvoice import ( + EInvoiceUnavailable, + EInvoiceValidationOptions, + EInvoiceValidationResult, + Profile, +) from .einvoice import validate_einvoice as run_einvoice_validation from .errors import ConfigurationError -from .export import list_exporters -from .export import tabular +from .export import list_exporters, tabular from .job_store import Job, JobOptions from .logging_setup import configure, get_logger from .ocr import SUPPORTED_SUFFIXES, list_ocr_backends @@ -73,7 +78,7 @@ class ErrorResponse(BaseModel): error: ErrorBody -_ERRORS = { +_ERRORS: dict[int | str, dict[str, Any]] = { 400: {"model": ErrorResponse, "description": "Bad request"}, 401: {"model": ErrorResponse, "description": "Missing or invalid API key"}, 404: {"model": ErrorResponse, "description": "Not found"}, @@ -187,7 +192,7 @@ async def _stage(file: UploadFile, budget: list[int]) -> Path: if suffix == ".pdf": try: pages = page_count(path) - except Exception as exc: # noqa: BLE001 — pdfium raises its own error type + except Exception as exc: raise ApiError(400, "unreadable_pdf", f"{file.filename!r} is not a readable PDF") from exc if pages > config.MAX_PDF_PAGES: raise ApiError(413, "too_many_pages", f"{file.filename!r} has {pages} pages; limit is {config.MAX_PDF_PAGES}") @@ -235,7 +240,8 @@ def _job_options( def _process_options(options: JobOptions) -> ProcessOptions: return ProcessOptions( - ocr=OcrOptions(backend=options.ocr_backend, fallbacks=options.ocr_fallbacks, languages=options.ocr_languages), + ocr=OcrOptions(backend=options.ocr_backend, fallbacks=list(options.ocr_fallbacks) if options.ocr_fallbacks is not None else None, + languages=options.ocr_languages), document_type=options.document_type, schema_version=options.schema_version, include_layout=options.include_layout, @@ -290,7 +296,7 @@ async def _run_job(job_id: str) -> Job: job = job_store.update(job_id, status="running", error=None) try: await run_in_threadpool(_run_batch, job) - except Exception as exc: # noqa: BLE001 — a job must end in a state, never hang + except Exception as exc: log.exception("job failed", extra={"job_id": job_id}) job = job_store.update(job_id, status="failed", error=f"{type(exc).__name__}: {exc}") else: @@ -632,7 +638,8 @@ def update_review(document_id: str, update: ReviewUpdate) -> dict: except review_queue.ReviewConflict as exc: raise ApiError(409, "review_conflict", str(exc)) from exc except review_queue.ReviewValidationError as exc: - return JSONResponse(status_code=422, content={"error": {"code": "review_validation_failed", "message": str(exc)}, "issues": exc.issues}) + # FastAPI derives the response model from `-> dict`; this error body carries the issues too. + return JSONResponse(status_code=422, content={"error": {"code": "review_validation_failed", "message": str(exc)}, "issues": exc.issues}) # type: ignore[return-value] except ValueError as exc: raise ApiError(422, "invalid_review_update", str(exc)) from exc diff --git a/src/docket/batch.py b/src/docket/batch.py index 2c062b8..ce4a8d4 100644 --- a/src/docket/batch.py +++ b/src/docket/batch.py @@ -32,9 +32,9 @@ import threading import time from collections import deque +from collections.abc import Callable, Iterable, Iterator from concurrent.futures import Future, ThreadPoolExecutor from pathlib import Path -from typing import Callable, Iterable, Iterator, Union from pydantic import BaseModel, ConfigDict, Field @@ -47,7 +47,7 @@ log = get_logger() -SourceSpec = Union[str, Path, Iterable[Union[str, Path]]] +SourceSpec = str | Path | Iterable[str | Path] ResultCallback = Callable[[int, DocumentResult], None] @@ -77,6 +77,9 @@ class BatchMetrics(BaseModel): documents_resumed: int = 0 pages: int = 0 llm_calls: int = 0 + llm_input_tokens: int = 0 + llm_output_tokens: int = 0 + llm_unreported_calls: int = 0 llm_estimated_tokens: int = 0 escalated_to_vlm: int = 0 vlm_pages: int = 0 @@ -114,9 +117,12 @@ def _scan(directory: Path, pattern: str, recursive: bool) -> Iterator[Path]: for entry in entries: if entry.is_dir(follow_symlinks=False): subdirs.append(Path(entry.path)) - elif entry.is_file() and Path(entry.name).suffix.lower() in SUPPORTED_SUFFIXES: - if Path(entry.name).match(pattern): - yield Path(entry.path) + elif ( + entry.is_file() + and Path(entry.name).suffix.lower() in SUPPORTED_SUFFIXES + and Path(entry.name).match(pattern) + ): + yield Path(entry.path) if recursive: for sub in subdirs: yield from _scan(sub, pattern, recursive) @@ -182,7 +188,7 @@ def record(self, result: DocumentResult) -> None: def _run_one(path: Path, options: ResolvedOptions) -> DocumentResult: try: return process_document(path, options) - except Exception as exc: # noqa: BLE001 — one document must not take the batch down + except Exception as exc: log.exception("document crashed", extra={"document": path.name}) return DocumentResult( source=str(path), @@ -238,7 +244,7 @@ def fill() -> None: while window: i, path, pending = window.popleft() resumed = isinstance(pending, DocumentResult) - result = pending if resumed else pending.result() + result: DocumentResult = pending if isinstance(pending, DocumentResult) else pending.result() if not resumed and checkpoint is not None: checkpoint.record(result) if batch.fail_fast and result.status == DocumentStatus.FAILED: @@ -279,6 +285,9 @@ def process_batch( metrics.documents_processed += 1 seconds.append(result.metrics.elapsed_seconds) metrics.llm_calls += result.metrics.llm_calls + metrics.llm_input_tokens += result.metrics.llm_input_tokens + metrics.llm_output_tokens += result.metrics.llm_output_tokens + metrics.llm_unreported_calls += result.metrics.llm_unreported_calls metrics.llm_estimated_tokens += result.metrics.llm_estimated_tokens for stage, value in result.metrics.stage_seconds.items(): stages[stage] = stages.get(stage, 0.0) + value diff --git a/src/docket/catalog/__init__.py b/src/docket/catalog/__init__.py index a81a310..c59506c 100644 --- a/src/docket/catalog/__init__.py +++ b/src/docket/catalog/__init__.py @@ -19,16 +19,16 @@ TaxIdentifier, ) from .models import ( - BankStatementTransaction, - ReceiptItem, - WaybillItem, BankStatement, + BankStatementTransaction, Contract, CreditNote, Invoice, PurchaseOrder, Receipt, + ReceiptItem, Waybill, + WaybillItem, ) from .registry import ( ENTRY_POINT_GROUP, @@ -60,19 +60,18 @@ register_builtins() __all__ = [ - "BankStatementTransaction", - "ReceiptItem", - "WaybillItem", - "Address", "BUILTIN_SCHEMAS", + "ENTRY_POINT_GROUP", + "UNKNOWN", + "Address", "BankAccount", "BankStatement", + "BankStatementTransaction", "Citation", "CitedDocument", "Contract", "CreditNote", "DocumentReference", - "ENTRY_POINT_GROUP", "Invoice", "Keyword", "LineItem", @@ -81,14 +80,15 @@ "Party", "PurchaseOrder", "Receipt", + "ReceiptItem", "SchemaError", "SchemaInfo", "SchemaSpec", "TaxIdentifier", - "UNKNOWN", "ValidationContext", "Validator", "Waybill", + "WaybillItem", "add_validator", "adhoc", "custom_schemas", diff --git a/src/docket/catalog/builtin.py b/src/docket/catalog/builtin.py index 98d8fed..d547f92 100644 --- a/src/docket/catalog/builtin.py +++ b/src/docket/catalog/builtin.py @@ -13,6 +13,7 @@ """ from __future__ import annotations +from typing import cast from .corpus import CORPUS from .models import ( @@ -24,7 +25,15 @@ Receipt, Waybill, ) -from .registry import LineItems, Migration, SchemaSpec, _register, keywords, pattern +from .registry import ( + LineItems, + Migration, + SchemaSpec, + Validator, + _register, + keywords, + pattern, +) from .rules import ( validate_credit_note, ) @@ -333,7 +342,8 @@ def register_builtins() -> None: _register( replace( spec, - validators=rules[spec.schema_id], + # Each validator takes its own schema; the registry pairs them. + validators=cast("tuple[Validator, ...]", rules[spec.schema_id]), examples=tuple(CORPUS[spec.schema_id]), builtin=True, ) diff --git a/src/docket/catalog/corpus.py b/src/docket/catalog/corpus.py index c2752a6..94df4ed 100644 --- a/src/docket/catalog/corpus.py +++ b/src/docket/catalog/corpus.py @@ -7,7 +7,7 @@ Custom schemas contribute their own through `SchemaSpec.examples`. """ -CORPUS: dict[str, tuple[str, ...]] = { # schema_id -> paraphrased sample sentences +CORPUS: dict[str, list[str]] = { # schema_id -> paraphrased sample sentences "invoice": [ 'Please remit payment within 30 days of the statement date to the account below.', 'Balance owed for services rendered last month, itemized by hour and rate.', diff --git a/src/docket/catalog/registry.py b/src/docket/catalog/registry.py index a0021f6..9e4276d 100644 --- a/src/docket/catalog/registry.py +++ b/src/docket/catalog/registry.py @@ -22,9 +22,10 @@ import re import typing +from collections.abc import Callable, Iterable from dataclasses import dataclass, field, replace from importlib.metadata import entry_points -from typing import TYPE_CHECKING, Callable, Iterable, Literal +from typing import TYPE_CHECKING, Literal from pydantic import BaseModel @@ -74,13 +75,13 @@ def text(self) -> str: def keywords(*phrases: str, weight: float = DEFAULT_KEYWORD_WEIGHT) -> tuple[Keyword, ...]: """Literal phrases matched on word boundaries, one Keyword each.""" return tuple( - Keyword(re.compile(rf"\b{re.escape(p)}\b", re.I), weight) for p in phrases + Keyword(re.compile(rf"\b{re.escape(p)}\b", re.IGNORECASE), weight) for p in phrases ) def pattern(regex: str, weight: float = DEFAULT_KEYWORD_WEIGHT) -> Keyword: """One Keyword from a regular expression (case-insensitive).""" - return Keyword(re.compile(regex, re.I), weight) + return Keyword(re.compile(regex, re.IGNORECASE), weight) @dataclass(frozen=True) @@ -182,7 +183,7 @@ def exporters(self) -> list[str]: return [e.name for e in list_exporters() if issubclass(self.model, e.accepts)] - def info(self) -> "SchemaInfo": + def info(self) -> SchemaInfo: return SchemaInfo( schema_id=self.schema_id, version=self.version, @@ -281,7 +282,7 @@ def check_model(model: object, *, cited_fields: Iterable[str] | None = None) -> raise SchemaError(f"a schema must be a Pydantic BaseModel subclass, got {model!r}") try: model.model_json_schema() - except Exception as exc: # noqa: BLE001 — pydantic raises several types here + except Exception as exc: raise SchemaError( f"{model.__name__} cannot be described as JSON Schema, which extraction needs: {exc}" ) from exc @@ -330,12 +331,14 @@ def _register(spec: SchemaSpec, *, replace_existing: bool = False) -> SchemaSpec for other in list_schemas(all_versions=True): if other.model is spec.model and other.schema_id != spec.schema_id: raise SchemaError(f"{spec.model.__name__} is already registered as {other.schema_id!r}") + version = spec.version + assert version is not None # only ad-hoc specs are unversioned, and they are never registered versions = _REGISTRY.setdefault(spec.schema_id, {}) - existing = versions.get(spec.version) + existing = versions.get(version) if existing is not None and (existing.builtin or not replace_existing): - raise SchemaError(f"schema {spec.schema_id!r} version {spec.version} is already registered") + raise SchemaError(f"schema {spec.schema_id!r} version {version} is already registered") spec = replace(spec, registered=True, display_name=spec.title) - versions[spec.version] = spec + versions[version] = spec return spec @@ -379,7 +382,7 @@ def _load_plugins() -> None: for spec in produced or (): if not isinstance(spec, SchemaSpec): raise SchemaError(f"entry point {ep.name!r} produced {spec!r}, not a SchemaSpec") - existing = _REGISTRY.get(spec.schema_id, {}).get(spec.version) + existing = _REGISTRY.get(spec.schema_id, {}).get(spec.version or "") if existing is None or existing.model is not spec.model: _register(spec) @@ -484,6 +487,7 @@ def resolve( f"document type {schema_id!r} uses {spec.model.__name__}, not {model.__name__}" ) return spec + assert model is not None # both None returned above spec = for_model(model) or adhoc(model) if version is not None and spec.version != version: raise SchemaError(f"{model.__name__} is version {spec.version}, not {version}") @@ -516,16 +520,16 @@ def migrate(schema_id: str, data: dict, from_version: str, to_version: str | Non __all__ = [ "ENTRY_POINT_GROUP", - "Keyword", "LINE_ITEM_COLUMNS", + "SUMMARY_COLUMNS", + "UNKNOWN", + "Keyword", "LineItems", "Migration", "MigrationInfo", "SchemaError", "SchemaInfo", "SchemaSpec", - "SUMMARY_COLUMNS", - "UNKNOWN", "ValidationContext", "Validator", "add_validator", diff --git a/src/docket/catalog/rules.py b/src/docket/catalog/rules.py index ff1a383..34bb1b9 100644 --- a/src/docket/catalog/rules.py +++ b/src/docket/catalog/rules.py @@ -10,7 +10,6 @@ from .models import CreditNote from .registry import ValidationContext - # ---- credit note ------------------------------------------------------------------ diff --git a/src/docket/checksums.py b/src/docket/checksums.py index 85217b5..ad0d0d8 100644 --- a/src/docket/checksums.py +++ b/src/docket/checksums.py @@ -462,9 +462,9 @@ def validate_tax_id( hint == "CA" and ( re.match(r"^\d{9}$", cleaned) - or re.match(r"^\d{9}\s*[A-Z]{2}\s*\d{4}$", cleaned, re.I) + or re.match(r"^\d{9}\s*[A-Z]{2}\s*\d{4}$", cleaned, re.IGNORECASE) ) - ) or (bool(re.match(r"^\d{9}\s*[A-Z]{2}\s*\d{4}$", cleaned, re.I))): + ) or (bool(re.match(r"^\d{9}\s*[A-Z]{2}\s*\d{4}$", cleaned, re.IGNORECASE))): return validate_ca_bn(cleaned), "Canadian BN" # Brazilian CNPJ / CPF: diff --git a/src/docket/classify.py b/src/docket/classify.py index 3971261..56858a0 100644 --- a/src/docket/classify.py +++ b/src/docket/classify.py @@ -9,7 +9,6 @@ """ from __future__ import annotations - from . import catalog, config from .classify_tfidf import classify_tfidf from .llm_client import LLMError, chat_json @@ -86,7 +85,7 @@ def classify_llm(text: str) -> ClassificationResult: votes[candidate] = votes.get(candidate, 0.0) + float( result.get("confidence", 0.0) ) - doc_type = max(votes, key=votes.get) + doc_type = max(votes, key=votes.__getitem__) total = sum(votes.values()) confidence = votes[doc_type] / total if total else 0.0 return ClassificationResult( diff --git a/src/docket/cli.py b/src/docket/cli.py index 990d2cd..f45f48d 100644 --- a/src/docket/cli.py +++ b/src/docket/cli.py @@ -105,14 +105,15 @@ def __init__(self, args: argparse.Namespace): self.args = args self.tabular = tabular - self.stream = open(args.output, "w", encoding="utf-8", newline="") if args.output else sys.stdout + # Open for the writer's lifetime; close() below releases them. + self.stream = open(args.output, "w", encoding="utf-8", newline="") if args.output else sys.stdout # noqa: SIM115 self.items_stream = None self.count = 0 if args.format == "csv": self.summary = tabular.CsvWriter(self.stream, tabular.RESULT_COLUMNS) items_path = args.line_items or (f"{Path(args.output).with_suffix('')}.line_items.csv" if args.output else None) if items_path: - self.items_stream = open(items_path, "w", encoding="utf-8", newline="") + self.items_stream = open(items_path, "w", encoding="utf-8", newline="") # noqa: SIM115 self.items = tabular.CsvWriter(self.items_stream, tabular.ITEM_COLUMNS) elif args.format == "json": self.stream.write("[") @@ -175,12 +176,11 @@ def _cmd_process(args: argparse.Namespace) -> int: def _cmd_batch(args: argparse.Namespace) -> int: - from .batch import BatchOptions, process_batch + import glob as globlib + from .batch import BatchOptions, process_batch from .options import resolve - import glob as globlib - options = resolve(_options(args)) # configuration errors before any file is opened if not Path(args.input).exists() and not globlib.has_magic(args.input): raise ConfigurationError(f"input {args.input!r} does not exist") @@ -190,7 +190,7 @@ def _cmd_batch(args: argparse.Namespace) -> int: glob=args.glob, workers=args.workers, fail_fast=args.fail_fast, - checkpoint=checkpoint, + checkpoint=Path(checkpoint) if checkpoint else None, keep_results=False, ) sink = _Sink(args) @@ -256,10 +256,10 @@ def _cmd_templates(args: argparse.Namespace) -> int: return EXIT_OK if args.template_id is None: raise ConfigurationError("`docket templates show` needs a template id") - template = get_vendor_template(args.template_id) - if template is None: + found = get_vendor_template(args.template_id) + if found is None: raise ConfigurationError(f"unknown vendor template {args.template_id!r}") - _print_json(template.model_dump(mode="json")) + _print_json(found.model_dump(mode="json")) return EXIT_OK diff --git a/src/docket/config.py b/src/docket/config.py index 16b1ff8..3bb9ecd 100644 --- a/src/docket/config.py +++ b/src/docket/config.py @@ -34,9 +34,10 @@ import os import sys +from collections.abc import Mapping from dataclasses import dataclass from pathlib import Path -from typing import Any, Mapping +from typing import Any from dotenv import load_dotenv @@ -237,20 +238,21 @@ def _parse(setting: Setting, raw: Any, from_env: bool, base: Path | None) -> Any return raw.strip().lower() in _TRUE raise ValueError("expected true or false") if kind in ("int", "float"): + number: int | float if from_env: try: - value = int(raw.strip()) if kind == "int" else float(raw.strip()) + number = int(raw.strip()) if kind == "int" else float(raw.strip()) except ValueError: raise ValueError(f"expected {'an integer' if kind == 'int' else 'a number'}") from None elif isinstance(raw, bool) or not isinstance(raw, (int, float)) or (kind == "int" and isinstance(raw, float)): raise ValueError(f"expected {'an integer' if kind == 'int' else 'a number'}") else: - value = float(raw) if kind == "float" else raw - if setting.minimum is not None and value < setting.minimum: + number = float(raw) if kind == "float" else raw + if setting.minimum is not None and number < setting.minimum: raise ValueError(f"must be at least {setting.minimum:g}") - if setting.maximum is not None and value > setting.maximum: + if setting.maximum is not None and number > setting.maximum: raise ValueError(f"must be at most {setting.maximum:g}") - return value + return number if kind == "list": if isinstance(raw, str): items = raw.split(",") diff --git a/src/docket/corroboration.py b/src/docket/corroboration.py index 9757821..262a32a 100644 --- a/src/docket/corroboration.py +++ b/src/docket/corroboration.py @@ -61,8 +61,7 @@ def amount(name: str) -> float | None: for label, tx_currency, tx_amount, text in _payments(transactions): if total is None or tx_currency != currency or abs(abs(tx_amount) - abs(total)) > amount_tolerance: continue - names_number = number_field is not None and _alnum(getattr(document, number_field)) in _alnum(text) - if names_number: + if number_field is not None and _alnum(getattr(document, number_field)) in _alnum(text): confirmed[number_field] = label confirmed["total_amount"] = label break diff --git a/src/docket/einvoice/__init__.py b/src/docket/einvoice/__init__.py index 0a0a831..c1cde24 100644 --- a/src/docket/einvoice/__init__.py +++ b/src/docket/einvoice/__init__.py @@ -6,6 +6,16 @@ once (fetch.py). """ from .artifacts import ArtifactError +from .facturx_pdf import ( + FacturXPdfUnavailable, + FacturXRoundTripResult, + PdfAValidationIssue, + PdfAValidationResult, + extract_facturx_xml, + generate_facturx_pdf, + validate_pdfa, + verify_facturx_round_trip, +) from .fetch import fetch_einvoice_resources from .validator import ( PROFILE_IDS, @@ -19,36 +29,26 @@ available, validate_einvoice, ) -from .facturx_pdf import ( - FacturXPdfUnavailable, - FacturXRoundTripResult, - PdfAValidationIssue, - PdfAValidationResult, - extract_facturx_xml, - generate_facturx_pdf, - validate_pdfa, - verify_facturx_round_trip, -) __all__ = [ + "PROFILE_IDS", "ArtifactError", "EInvoiceIssue", "EInvoiceResourcesMissing", "EInvoiceUnavailable", "EInvoiceValidationOptions", "EInvoiceValidationResult", - "LayerReport", - "PROFILE_IDS", - "Profile", - "available", - "fetch_einvoice_resources", - "validate_einvoice", "FacturXPdfUnavailable", "FacturXRoundTripResult", + "LayerReport", "PdfAValidationIssue", "PdfAValidationResult", + "Profile", + "available", "extract_facturx_xml", + "fetch_einvoice_resources", "generate_facturx_pdf", + "validate_einvoice", "validate_pdfa", "verify_facturx_round_trip", ] diff --git a/src/docket/einvoice/facturx_pdf.py b/src/docket/einvoice/facturx_pdf.py index ec5f25b..4205aaa 100644 --- a/src/docket/einvoice/facturx_pdf.py +++ b/src/docket/einvoice/facturx_pdf.py @@ -12,7 +12,12 @@ from pydantic import BaseModel, Field from ..errors import ConfigurationError -from .validator import EInvoiceValidationOptions, EInvoiceValidationResult, Profile, validate_einvoice +from .validator import ( + EInvoiceValidationOptions, + EInvoiceValidationResult, + Profile, + validate_einvoice, +) Source = str | Path | bytes FacturXLevel = Literal["minimum", "basicwl", "basic", "en16931", "extended", "autodetect"] @@ -150,9 +155,8 @@ def validate_pdfa( path: Path | None = None try: if isinstance(source_pdf, bytes): - handle = tempfile.NamedTemporaryFile(suffix=".pdf", delete=False) - handle.write(source_pdf) - handle.close() + with tempfile.NamedTemporaryFile(suffix=".pdf", delete=False) as handle: + handle.write(source_pdf) path = Path(handle.name) else: path = Path(source_pdf) diff --git a/src/docket/einvoice/validator.py b/src/docket/einvoice/validator.py index 508ca45..aeffb7a 100644 --- a/src/docket/einvoice/validator.py +++ b/src/docket/einvoice/validator.py @@ -36,7 +36,7 @@ from dataclasses import dataclass from enum import Enum from pathlib import Path -from typing import Literal +from typing import Any, Literal from pydantic import BaseModel, Field @@ -238,7 +238,7 @@ def __init__(self) -> None: self._processor = PySaxonProcessor(license=False) self._xslt = self._processor.new_xslt30_processor() self._lock = threading.Lock() - self._stylesheets: dict[str, object] = {} + self._stylesheets: dict[str, Any] = {} self._schemas: dict[str, object] = {} @property @@ -316,7 +316,8 @@ def _parse_svrl(svrl: bytes, source: str) -> tuple[list[EInvoiceIssue], int]: for tag, default in (("failed-assert", "error"), ("successful-report", "warning")): for node in root.iter(f"{{{_NS['svrl']}}}{tag}"): flag = (node.get("flag") or node.get("role") or default).lower() - severity = {"fatal": "fatal", "error": "error", "warning": "warning", "information": "info", "info": "info"}.get( + severity: Literal["fatal", "error", "warning", "info"] = { # type: ignore[assignment] + "fatal": "fatal", "error": "error", "warning": "warning", "information": "info", "info": "info"}.get( flag, "error" ) text = " ".join("".join(node.itertext()).split()) @@ -476,13 +477,13 @@ def finish(**updates) -> EInvoiceValidationResult: __all__ = [ + "PROFILE_IDS", "EInvoiceIssue", "EInvoiceResourcesMissing", "EInvoiceUnavailable", "EInvoiceValidationOptions", "EInvoiceValidationResult", "LayerReport", - "PROFILE_IDS", "Profile", "available", "validate_einvoice", diff --git a/src/docket/export/__init__.py b/src/docket/export/__init__.py index e2e54dd..c87bfee 100644 --- a/src/docket/export/__init__.py +++ b/src/docket/export/__init__.py @@ -17,19 +17,19 @@ from __future__ import annotations import json +from collections.abc import Callable from dataclasses import dataclass from functools import partial from importlib.metadata import entry_points -from typing import TYPE_CHECKING, Any, Callable +from typing import TYPE_CHECKING, Any from pydantic import BaseModel, Field from ..catalog.models import CreditNote, Invoice +from ..einvoice.validator import EInvoiceValidationResult from . import en16931 from .facturae import export_to_facturae_xml -from ..einvoice.validator import EInvoiceValidationResult - if TYPE_CHECKING: from ..result import DocumentResult @@ -138,7 +138,7 @@ class ExportResult(BaseModel): def export_document( - source: "DocumentResult | BaseModel", format: str, options: ExportOptions | None = None + source: DocumentResult | BaseModel, format: str, options: ExportOptions | None = None ) -> ExportResult: """Render a processed document in the named format. @@ -180,10 +180,14 @@ def export_document( if options.validate_einvoice: if exporter.einvoice_profile is None: raise ExportError(f"{exporter.name} is not an e-invoice format; there are no official rules to validate it against") - from ..einvoice.validator import EInvoiceValidationOptions, validate_einvoice + from ..einvoice.validator import ( + EInvoiceValidationOptions, + Profile, + validate_einvoice, + ) validation = validate_einvoice( - content.encode("utf-8"), EInvoiceValidationOptions(profile=exporter.einvoice_profile) + content.encode("utf-8"), EInvoiceValidationOptions(profile=Profile(exporter.einvoice_profile)) ) return ExportResult( format=exporter.name, media_type=exporter.media_type, content=content, einvoice_validation=validation diff --git a/src/docket/export/en16931.py b/src/docket/export/en16931.py index 30f73d6..1f37494 100644 --- a/src/docket/export/en16931.py +++ b/src/docket/export/en16931.py @@ -119,8 +119,8 @@ def unit_code(unit: str | None) -> str: class VatGroup: category: str rate: Decimal - taxable: Decimal = Decimal("0") - tax: Decimal = Decimal("0") + taxable: Decimal = Decimal(0) + tax: Decimal = Decimal(0) @dataclass @@ -172,7 +172,7 @@ def _rate(doc) -> Decimal | None: return Decimal(str(doc.tax_rate_percent)) base = money(doc.subtotal) - money(doc.discount_amount) + money(doc.shipping_amount) if doc.tax_amount == 0: - return Decimal("0") + return Decimal(0) if base == 0: return None rate = (money(doc.tax_amount) / base * 100).quantize(_CENT, rounding=ROUND_HALF_UP) @@ -214,7 +214,7 @@ def semantic(doc: Invoice | CreditNote) -> Semantic: only.taxable += charge - allowance for group in groups.values(): group.tax = money(group.taxable * group.rate / 100) - tax_total = sum((g.tax for g in groups.values()), Decimal("0")) + tax_total = sum((g.tax for g in groups.values()), Decimal(0)) stated_tax = money(doc.tax_amount) if len(groups) == 1: # One rate: the stated tax is the breakdown's tax. EN 16931 allows @@ -232,7 +232,7 @@ def semantic(doc: Invoice | CreditNote) -> Semantic: raise EN16931Error( f"VAT by rate sums to {_fmt(tax_total)}, but the document states tax {_fmt(stated_tax)}" ) - line_total = sum((line.net for line in lines), Decimal("0")) + line_total = sum((line.net for line in lines), Decimal(0)) if abs(line_total - money(doc.subtotal)) > _CENT: raise EN16931Error(f"lines sum to {_fmt(line_total)}, subtotal is {_fmt(money(doc.subtotal))} (BR-CO-10)") tax_exclusive = line_total - allowance + charge @@ -240,7 +240,7 @@ def semantic(doc: Invoice | CreditNote) -> Semantic: account = doc.payment_account return Semantic( credit_note=credit_note, - number=doc.credit_note_number if credit_note else doc.invoice_number, + number=getattr(doc, "credit_note_number" if credit_note else "invoice_number"), issue_date=doc.issue_date.isoformat(), type_code="381" if credit_note else "380", currency=doc.currency.upper(), @@ -263,7 +263,7 @@ def semantic(doc: Invoice | CreditNote) -> Semantic: tax_total=tax_total, tax_inclusive=tax_inclusive, payable=tax_inclusive, - notes=[doc.reason] if credit_note and doc.reason else [], + notes=[doc.reason] if isinstance(doc, CreditNote) and doc.reason else [], ) @@ -588,4 +588,4 @@ def render(doc: Invoice | CreditNote, profile: str) -> str: return (_Ubl() if spec.syntax == "ubl" else _Cii()).render(sem, spec) -__all__ = ["EN16931Error", "PROFILES", "ProfileSpec", "Semantic", "render", "semantic", "unit_code"] +__all__ = ["PROFILES", "EN16931Error", "ProfileSpec", "Semantic", "render", "semantic", "unit_code"] diff --git a/src/docket/export/facturae.py b/src/docket/export/facturae.py index 710de8f..2f4db04 100644 --- a/src/docket/export/facturae.py +++ b/src/docket/export/facturae.py @@ -7,6 +7,7 @@ from __future__ import annotations import xml.etree.ElementTree as ET + from ..catalog.models import Invoice from ._parties import address_line, iban, tax_number, vat_number diff --git a/src/docket/export/tabular.py b/src/docket/export/tabular.py index 174c7ee..f2cd03f 100644 --- a/src/docket/export/tabular.py +++ b/src/docket/export/tabular.py @@ -25,9 +25,15 @@ import csv import json -from typing import IO, Iterable, Iterator - -from ..catalog.registry import LINE_ITEM_COLUMNS, SUMMARY_COLUMNS, SchemaSpec, get_schema +from collections.abc import Iterable, Iterator +from typing import IO, cast + +from ..catalog.registry import ( + LINE_ITEM_COLUMNS, + SUMMARY_COLUMNS, + SchemaSpec, + get_schema, +) from ..result import DocumentResult RESULT_COLUMNS: tuple[str, ...] = ( @@ -110,7 +116,7 @@ def line_item_rows(result: DocumentResult) -> Iterator[dict[str, object]]: spec = _spec(result) if spec is None or spec.line_items is None or not result.extracted: return - items = _at(result.extracted, spec.line_items.path) or [] + items = cast(list, _at(result.extracted, spec.line_items.path) or []) for n, item in enumerate(items, start=1): row: dict[str, object] = { "document_id": result.document_id, @@ -159,9 +165,9 @@ def write_jsonl(results: Iterable[DocumentResult], stream: IO[str], *, include_l __all__ = [ - "CsvWriter", "ITEM_COLUMNS", "RESULT_COLUMNS", + "CsvWriter", "jsonl_line", "line_item_rows", "result_row", diff --git a/src/docket/extract.py b/src/docket/extract.py index 38ddca6..b8bb775 100644 --- a/src/docket/extract.py +++ b/src/docket/extract.py @@ -88,7 +88,7 @@ def _validated_call( attempt += 1 try: raw = chat_json(prompt, schema=schema) - except Exception as exc: + except Exception as exc: # noqa: BLE001 — any model failure is one failed attempt, retried below log.warning( "extraction request failed", extra={"attempt": attempt, "error_type": type(exc).__name__}, @@ -192,7 +192,7 @@ def extract_pages( ), schema=schema, ) - except Exception: + except Exception: # noqa: BLE001 — one failed chunk must not lose the others candidate = {"_chunk_error": True} candidates.append({"candidate": candidate, "source": chunk}) diff --git a/src/docket/job_store.py b/src/docket/job_store.py index f693d6f..3e0beb1 100644 --- a/src/docket/job_store.py +++ b/src/docket/job_store.py @@ -14,9 +14,10 @@ import shutil import threading +from collections.abc import Iterator from datetime import datetime, timezone from pathlib import Path -from typing import Iterator, Literal +from typing import Literal from uuid import uuid4 from pydantic import BaseModel, Field @@ -201,9 +202,9 @@ def results(job: Job) -> Iterator[tuple[JobDocument, DocumentResult]]: continue by_stored[Path(result.source).name] = result for document in job.documents: - result = by_stored.get(document.stored_as) - if result is not None: - yield document, result.model_copy(update={"source": document.filename}) + stored = by_stored.get(document.stored_as) + if stored is not None: + yield document, stored.model_copy(update={"source": document.filename}) def remove_uploads(job_id: str) -> None: diff --git a/src/docket/layout/__init__.py b/src/docket/layout/__init__.py index 10afafd..728843c 100644 --- a/src/docket/layout/__init__.py +++ b/src/docket/layout/__init__.py @@ -1,7 +1,14 @@ """Layout-first document model: words, lines, blocks, columns and tables with normalized coordinates, plus the analysis that builds them and the serialization the LLM reads.""" -from .analysis import CellHint, RawWord, TableHint, build_page, cells_from_boxes, text_only_page +from .analysis import ( + CellHint, + RawWord, + TableHint, + build_page, + cells_from_boxes, + text_only_page, +) from .locate import locate_all, locate_quote from .models import ( BoundingBox, diff --git a/src/docket/layout/analysis.py b/src/docket/layout/analysis.py index eef0a5d..3a03324 100644 --- a/src/docket/layout/analysis.py +++ b/src/docket/layout/analysis.py @@ -27,6 +27,7 @@ import re from dataclasses import dataclass, field +from itertools import pairwise from statistics import median from .. import config @@ -291,7 +292,7 @@ def _find_aligned_tables( full_rows = sum(1 for r in run if len(rows[r].segments) >= len(bands or ())) regular = bands is not None and len(bands) >= MIN_TABLE_COLUMNS and full_rows >= MIN_TABLE_ROWS two_column = bands is not None and len(bands) == 2 and _is_two_column_table(run, rows, words) - if len(run) >= MIN_TABLE_ROWS and (regular or two_column): + if len(run) >= MIN_TABLE_ROWS and bands is not None and (regular or two_column): tables.append((run, bands)) k = j else: @@ -555,7 +556,7 @@ def build_page( start = min(g.start for g in group) end = max(g.end for g in group) edges = [(-float("inf"), group[0].lo)] - edges += [(a.hi, b.lo) for a, b in zip(group, group[1:])] + edges += [(a.hi, b.lo) for a, b in pairwise(group)] edges += [(group[-1].hi, float("inf"))] base = len(column_boxes) column_boxes.extend([] for _ in edges) @@ -667,17 +668,17 @@ def _table_lines(t: dict, rows: list[_Row], words: list[RawWord], t_index: int) ) out.append(_Line(words=members, text=text, table=t_index)) return out - by_row: dict[int, list[tuple]] = {} + grid_rows: dict[int, list[tuple]] = {} for cell, members, text in t["cells"]: - by_row.setdefault(cell.row, []).append((cell.column, members, text)) + grid_rows.setdefault(cell.row, []).append((cell.column, members, text)) out = [] - for row_index in sorted(by_row): - cells = sorted(by_row[row_index], key=lambda c: c[0]) - members = [i for _, m, _ in cells for i in m] + for row_index in sorted(grid_rows): + row_cells = sorted(grid_rows[row_index], key=lambda c: c[0]) + members = [i for _, m, _ in row_cells for i in m] if not members: continue out.append( - _Line(words=members, text=" | ".join(text for _, _, text in cells), table=t_index) + _Line(words=members, text=" | ".join(text for _, _, text in row_cells), table=t_index) ) return out diff --git a/src/docket/layout/models.py b/src/docket/layout/models.py index 4ab27c5..e3a03a0 100644 --- a/src/docket/layout/models.py +++ b/src/docket/layout/models.py @@ -10,7 +10,8 @@ """ from __future__ import annotations -from typing import Iterable, Literal +from collections.abc import Iterable +from typing import Literal from pydantic import BaseModel, ConfigDict, Field, model_validator @@ -28,7 +29,7 @@ class BoundingBox(BaseModel): y1: float = Field(ge=0.0, le=1.0) @model_validator(mode="after") - def _ordered(self) -> "BoundingBox": + def _ordered(self) -> BoundingBox: if self.x1 < self.x0 or self.y1 < self.y0: raise ValueError(f"inverted box: {self}") return self @@ -36,7 +37,7 @@ def _ordered(self) -> "BoundingBox": @classmethod def from_absolute( cls, x0: float, y0: float, x1: float, y1: float, width: float, height: float - ) -> "BoundingBox": + ) -> BoundingBox: """Normalize a box given in the page's own units, clamping to the page.""" if width <= 0 or height <= 0: raise ValueError("page width and height must be positive") @@ -49,7 +50,7 @@ def clamp(v: float) -> float: return cls(x0=nx0, y0=ny0, x1=nx1, y1=ny1) @classmethod - def union(cls, boxes: Iterable["BoundingBox"]) -> "BoundingBox": + def union(cls, boxes: Iterable[BoundingBox]) -> BoundingBox: boxes = list(boxes) if not boxes: raise ValueError("union of no boxes") @@ -76,7 +77,7 @@ def height(self) -> float: def center_y(self) -> float: return (self.y0 + self.y1) / 2 - def contains_center_of(self, other: "BoundingBox") -> bool: + def contains_center_of(self, other: BoundingBox) -> bool: cx = (other.x0 + other.x1) / 2 return self.x0 <= cx <= self.x1 and self.y0 <= other.center_y <= self.y1 diff --git a/src/docket/limits.py b/src/docket/limits.py index c64498f..d7b7c02 100644 --- a/src/docket/limits.py +++ b/src/docket/limits.py @@ -9,8 +9,8 @@ from __future__ import annotations import threading +from collections.abc import Iterator from contextlib import contextmanager -from typing import Iterator from . import config diff --git a/src/docket/llm_client.py b/src/docket/llm_client.py index f47e1c2..62a12b2 100644 --- a/src/docket/llm_client.py +++ b/src/docket/llm_client.py @@ -7,8 +7,9 @@ Two pieces of production plumbing live here rather than upstream, because every LLM call in the pipeline goes through this module: -- `usage`: a per-run counter of call count and estimated tokens, read by - the eval harness to report latency/cost per document (see eval/run_eval.py). +- `usage`: a per-document counter of calls, the models used, the tokens + the provider reported (Ollama's prompt_eval_count/eval_count, OpenAI's + usage block) and a chars/4 estimate for providers that report nothing. - Langfuse tracing: if LANGFUSE_PUBLIC_KEY/SECRET_KEY are set, every call is wrapped in a Langfuse generation span. If they aren't, or the langfuse package isn't installed, tracing is a no-op — this stays local-first by @@ -18,10 +19,12 @@ import base64 import json +import random import re import time from contextvars import ContextVar -from dataclasses import dataclass +from dataclasses import dataclass, field +from email.utils import parsedate_to_datetime import httpx @@ -32,23 +35,43 @@ class LLMError(RuntimeError): pass +@dataclass +class _Reply: + """A model's answer and the token counts its provider reported, if any.""" + + content: str + input_tokens: int | None = None + output_tokens: int | None = None + + @dataclass class _Usage: calls: int = 0 estimated_tokens: int = 0 + input_tokens: int = 0 + output_tokens: int = 0 + unreported_calls: int = 0 + models: list[str] = field(default_factory=list) - def record(self, *texts: str) -> None: + def record(self, *texts: str, model: str | None = None, reply: _Reply | None = None) -> None: self.calls += 1 - # Rough token estimate (chars/4) — good enough for a relative - # cost/latency comparison, not meant to match a real tokenizer. + # Rough token estimate (chars/4), kept for providers that report + # nothing; not meant to match a real tokenizer. self.estimated_tokens += sum(len(t) for t in texts) // 4 - - def reset(self) -> None: - self.calls = 0 - self.estimated_tokens = 0 + if reply is not None and reply.input_tokens is not None and reply.output_tokens is not None: + self.input_tokens += reply.input_tokens + self.output_tokens += reply.output_tokens + else: + self.unreported_calls += 1 + if model and model not in self.models: + self.models.append(model) def snapshot(self) -> dict: - return {"calls": self.calls, "estimated_tokens": self.estimated_tokens} + return { + "calls": self.calls, "estimated_tokens": self.estimated_tokens, + "input_tokens": self.input_tokens, "output_tokens": self.output_tokens, + "unreported_calls": self.unreported_calls, "models": list(self.models), + } _usage_var: ContextVar[_Usage | None] = ContextVar("docket_llm_usage", default=None) @@ -72,8 +95,24 @@ def calls(self) -> int: def estimated_tokens(self) -> int: return self._current().estimated_tokens - def record(self, *texts: str) -> None: - self._current().record(*texts) + @property + def input_tokens(self) -> int: + return self._current().input_tokens + + @property + def output_tokens(self) -> int: + return self._current().output_tokens + + @property + def unreported_calls(self) -> int: + return self._current().unreported_calls + + @property + def models(self) -> list[str]: + return list(self._current().models) + + def record(self, *texts: str, model: str | None = None, reply: _Reply | None = None) -> None: + self._current().record(*texts, model=model, reply=reply) def reset(self) -> None: _usage_var.set(_Usage()) @@ -100,7 +139,7 @@ def _get_langfuse(): secret_key=config.LANGFUSE_SECRET_KEY, host=config.LANGFUSE_HOST, ) - except Exception: + except Exception: # noqa: BLE001 — a broken tracing setup must not break extraction return None @@ -124,8 +163,8 @@ def _trace(name: str, model: str, input_: object, output: object, start: float) }, ).end() client.flush() - except Exception: - pass # tracing must never break the pipeline + except Exception: # noqa: BLE001, S110 — tracing must never break the pipeline + pass def list_models(*, vision_only: bool = False, timeout: float = 10.0) -> list[str]: @@ -177,7 +216,7 @@ def chat_json( model = model or config.TEXT_MODEL start = time.monotonic() if config.LLM_PROVIDER == "openai": - content = _openai_chat( + reply = _openai_chat( model, [{"role": "user", "content": prompt}], timeout=timeout, @@ -194,9 +233,10 @@ def chat_json( "think": config.ENABLE_THINKING, "options": {"temperature": 0}, } - content = _ollama_chat(payload, timeout=timeout) + reply = _ollama_chat(payload, timeout=timeout) - usage.record(prompt, content) + content = reply.content + usage.record(prompt, content, model=model, reply=reply) _trace("chat_json", model, prompt, content, start) try: # Some cloud responses wrap otherwise valid JSON in a Markdown fence. @@ -244,7 +284,7 @@ def vision_transcribe( {"type": "image_url", "image_url": {"url": f"data:{mime};base64,{b64}"}}, ], } - text = _openai_chat(model, [message], timeout=timeout).strip() + reply = _openai_chat(model, [message], timeout=timeout) else: payload = { "model": model, @@ -253,9 +293,10 @@ def vision_transcribe( "think": config.ENABLE_THINKING, "options": {"temperature": 0}, } - text = _ollama_chat(payload, timeout=timeout).strip() + reply = _ollama_chat(payload, timeout=timeout) - usage.record(prompt_text, text) + text = reply.content.strip() + usage.record(prompt_text, text, model=model, reply=reply) _trace("vision_transcribe", model, "", text, start) return text @@ -267,6 +308,30 @@ def vision_transcribe( # retried after one: a laptop can honestly need minutes for a dense page. _RETRY_STATUS = {429, 500, 502, 503, 504} _BACKOFF_S = 2.0 +# A Retry-After longer than this is not waited out: the document fails with +# the provider's error instead of holding a worker for minutes. +_MAX_RETRY_AFTER_S = 60.0 + + +def _retry_delay(attempt: int, resp: httpx.Response | None = None) -> float: + """How long to wait before retry `attempt` + 1. + + The provider's Retry-After (seconds or an HTTP date) wins when it sends + one, capped at _MAX_RETRY_AFTER_S. Otherwise exponential backoff with + jitter, so parallel workers throttled together don't retry together. + """ + header = resp.headers.get("retry-after") if resp is not None else None + if header: + try: + seconds = float(header) + except ValueError: + try: + seconds = parsedate_to_datetime(header).timestamp() - time.time() + except (TypeError, ValueError): + seconds = None + if seconds is not None: + return min(max(seconds, 0.0), _MAX_RETRY_AFTER_S) + return _BACKOFF_S * (2 ** attempt) * random.uniform(0.5, 1.5) def _hosted(model: str) -> bool: @@ -275,7 +340,7 @@ def _hosted(model: str) -> bool: if config.LLM_PROVIDER == "openai": host = httpx.URL(config.LLM_BASE_URL).host return host not in ("localhost", "127.0.0.1", "::1") - return model.endswith(":cloud") or model.endswith("-cloud") + return model.endswith((":cloud", "-cloud")) def _post(url: str, *, model: str, json: dict, timeout: float, headers: dict | None = None) -> httpx.Response: @@ -291,7 +356,7 @@ def _post(url: str, *, model: str, json: dict, timeout: float, headers: dict | N try: resp = httpx.post(url, json=json, headers=headers, timeout=timeout) if resp.status_code in _RETRY_STATUS and not last: - time.sleep(_BACKOFF_S * (attempt + 1)) + time.sleep(_retry_delay(attempt, resp)) continue resp.raise_for_status() return resp @@ -301,21 +366,22 @@ def _post(url: str, *, model: str, json: dict, timeout: float, headers: dict | N except httpx.TransportError: if last: raise - time.sleep(_BACKOFF_S * (attempt + 1)) + time.sleep(_retry_delay(attempt)) raise AssertionError("unreachable") -def _ollama_chat(payload: dict, *, timeout: float) -> str: +def _ollama_chat(payload: dict, *, timeout: float) -> _Reply: with limits.slot("llm"): return _ollama_request(payload, timeout=timeout) -def _ollama_request(payload: dict, *, timeout: float) -> str: +def _ollama_request(payload: dict, *, timeout: float) -> _Reply: try: resp = _post(f"{config.OLLAMA_HOST}/api/chat", model=payload["model"], json=payload, timeout=timeout) except httpx.HTTPError as exc: raise LLMError(f"Ollama request failed: {exc}") from exc - return resp.json()["message"]["content"] + body = resp.json() + return _Reply(body["message"]["content"], body.get("prompt_eval_count"), body.get("eval_count")) def _openai_headers() -> dict: @@ -324,14 +390,14 @@ def _openai_headers() -> dict: def _openai_chat( model: str, messages: list[dict], *, timeout: float, json_mode: bool = False -) -> str: +) -> _Reply: with limits.slot("llm"): return _openai_request(model, messages, timeout=timeout, json_mode=json_mode) def _openai_request( model: str, messages: list[dict], *, timeout: float, json_mode: bool = False -) -> str: +) -> _Reply: payload: dict = {"model": model, "messages": messages, "temperature": 0} if json_mode: # JSON mode rather than strict json_schema: Pydantic schemas rarely @@ -345,7 +411,13 @@ def _openai_request( headers=_openai_headers(), timeout=timeout, ) - return resp.json()["choices"][0]["message"]["content"] or "" + body = resp.json() + reported = body.get("usage") or {} + return _Reply( + body["choices"][0]["message"]["content"] or "", + reported.get("prompt_tokens"), + reported.get("completion_tokens"), + ) except httpx.HTTPError as exc: raise LLMError(f"LLM request failed: {exc}") from exc except (KeyError, IndexError, ValueError) as exc: diff --git a/src/docket/matching.py b/src/docket/matching.py index 61420cd..19b436b 100644 --- a/src/docket/matching.py +++ b/src/docket/matching.py @@ -7,11 +7,24 @@ """ from __future__ import annotations -from typing import Sequence +from collections.abc import Sequence from .catalog.common import LineItem -from .catalog.models import Contract, Invoice, PurchaseOrder, Receipt, Waybill, WaybillItem -from .schemas import BankTransaction, Discrepancy, DiscrepancyType, MatchingStatus, MatchResult +from .catalog.models import ( + Contract, + Invoice, + PurchaseOrder, + Receipt, + Waybill, + WaybillItem, +) +from .schemas import ( + BankTransaction, + Discrepancy, + DiscrepancyType, + MatchingStatus, + MatchResult, +) from .validate import _core_name, _isclose @@ -327,27 +340,28 @@ def match_invoices_to_contract( # 4. Cumulative Budget Ceiling Check variance = 0.0 matched_amount = total_billed - if contract.contract_value is not None: - if total_billed > contract.contract_value and not _isclose( - total_billed, contract.contract_value - ): - overage = round(total_billed - contract.contract_value, 2) - variance = overage - matched_amount = contract.contract_value - discrepancies.append( - Discrepancy( - type=DiscrepancyType.BUDGET_EXCEEDED, - field="contract_value", - expected=contract.contract_value, - actual=total_billed, - difference=overage, - severity="error", - message=( - f"Cumulative invoices total ({total_billed:.2f}) exceeds " - f"contract value limit ({contract.contract_value:.2f}) by {overage:.2f}" - ), - ) + if ( + contract.contract_value is not None + and total_billed > contract.contract_value + and not _isclose(total_billed, contract.contract_value) + ): + overage = round(total_billed - contract.contract_value, 2) + variance = overage + matched_amount = contract.contract_value + discrepancies.append( + Discrepancy( + type=DiscrepancyType.BUDGET_EXCEEDED, + field="contract_value", + expected=contract.contract_value, + actual=total_billed, + difference=overage, + severity="error", + message=( + f"Cumulative invoices total ({total_billed:.2f}) exceeds " + f"contract value limit ({contract.contract_value:.2f}) by {overage:.2f}" + ), ) + ) has_errors = any(d.severity == "error" for d in discrepancies) status = MatchingStatus.DISCREPANCY if has_errors else MatchingStatus.MATCHED @@ -398,9 +412,8 @@ def match_receipt_to_transactions( continue # Card check (if both have card_last_four) - if receipt.card_last_four and tx.card_last_four: - if receipt.card_last_four != tx.card_last_four: - continue + if receipt.card_last_four and tx.card_last_four and receipt.card_last_four != tx.card_last_four: + continue # Date tolerance check days_diff = abs((tx.transaction_date - receipt.transaction_date).days) @@ -559,9 +572,9 @@ def match_three_way( po_items[k] = item wb_items: dict[str, WaybillItem] = {} - for item in waybill.items: - k = item.sku.strip().lower() if item.sku else item.item_name.strip().lower() - wb_items[k] = item + for wb_item in waybill.items: + k = wb_item.sku.strip().lower() if wb_item.sku else wb_item.item_name.strip().lower() + wb_items[k] = wb_item # 3. Check Invoice items against Waybill (fulfillment) and PO (authorization & price) for idx, inv_item in enumerate(invoice.line_items): diff --git a/src/docket/ocr/__init__.py b/src/docket/ocr/__init__.py index 4b806ec..b09e6a4 100644 --- a/src/docket/ocr/__init__.py +++ b/src/docket/ocr/__init__.py @@ -33,6 +33,8 @@ __all__ = [ "AUTO_ORDER", + "ENTRY_POINT_GROUP", + "SUPPORTED_SUFFIXES", "Acquisition", "AcquisitionError", "AcquisitionOptions", @@ -43,14 +45,12 @@ "BackendUnavailable", "Capabilities", "DocumentSource", - "ENTRY_POINT_GROUP", "OcrBackend", "OcrBackendError", "OcrError", "OcrSettings", "PageAcquisition", "PageSource", - "SUPPORTED_SUFFIXES", "UnknownLanguage", "UnsupportedDocument", "acquire", diff --git a/src/docket/ocr/acquire.py b/src/docket/ocr/acquire.py index 977cda9..e2580cc 100644 --- a/src/docket/ocr/acquire.py +++ b/src/docket/ocr/acquire.py @@ -26,7 +26,7 @@ import time from dataclasses import dataclass from pathlib import Path -from typing import Literal, Union +from typing import Literal from pydantic import BaseModel, ConfigDict, Field @@ -38,7 +38,7 @@ from .source import DocumentSource, PageSource, UnsupportedDocument from .witness import confident_amounts -BackendSpec = Union[str, OcrBackend] +BackendSpec = str | OcrBackend # `auto` picks the first of these that can run here. AUTO_ORDER = ("tesseract", "paddle") @@ -55,7 +55,7 @@ class AcquisitionOptions(BaseModel): default="auto", description="Primary OCR backend: a name, 'auto', or an OcrBackend instance." ) fallbacks: list[BackendSpec] = Field( - default_factory=lambda: ["vlm"], description="Tried in order when the primary reading is rejected." + default_factory=lambda: list[BackendSpec](["vlm"]), description="Tried in order when the primary reading is rejected." ) use_pdf_text: bool = True min_confidence: float = Field(default=0.60, ge=0.0, le=1.0) @@ -191,15 +191,16 @@ def _read_page( if page.kind == "pdf" and options.use_pdf_text and not options.escalate: start = time.monotonic() + text_layer: PageLayout | None try: - layout = pdf_text.recognize_page(page) - problem = text_layer_problem(layout) + text_layer = pdf_text.recognize_page(page) + problem = text_layer_problem(text_layer) except OcrError as exc: - layout, problem = None, str(exc) + text_layer, problem = None, str(exc) seconds = round(time.monotonic() - start, 3) - if layout is not None and problem is None: + if text_layer is not None and problem is None: attempts.append(Attempt(backend=pdf_text.name, outcome="accepted", seconds=seconds)) - return layout, PageAcquisition(page=page.number, backend=pdf_text.name, attempts=attempts) + return text_layer, PageAcquisition(page=page.number, backend=pdf_text.name, attempts=attempts) attempts.append(Attempt(backend=pdf_text.name, outcome="rejected", reason=problem, seconds=seconds)) runnable = [b for b in chain if "image" in b.capabilities.inputs] diff --git a/src/docket/ocr/base.py b/src/docket/ocr/base.py index f82323e..72a0237 100644 --- a/src/docket/ocr/base.py +++ b/src/docket/ocr/base.py @@ -115,10 +115,10 @@ def availability(self) -> BackendStatus: """Whether this backend can run here — cheap, no page is read.""" @abstractmethod - def recognize_page(self, page: "PageSource") -> PageLayout: + def recognize_page(self, page: PageSource) -> PageLayout: """Read one page. Raise OcrError on failure, never return None.""" - def recognize_document(self, document: "DocumentSource") -> DocumentLayout: + def recognize_document(self, document: DocumentSource) -> DocumentLayout: """Read every page. Override when the engine is faster on a whole file.""" return DocumentLayout(pages=[self.recognize_page(p) for p in document.pages()]) diff --git a/src/docket/ocr/deskew.py b/src/docket/ocr/deskew.py index 00d4dcb..01a4b19 100644 --- a/src/docket/ocr/deskew.py +++ b/src/docket/ocr/deskew.py @@ -1,14 +1,17 @@ """Small, dependency-free fine-angle deskew for OCR page images.""" from __future__ import annotations +from typing import cast + from PIL import Image, ImageOps def _projection_score(image: Image.Image, angle: float) -> float: rotated = image.rotate(angle, resample=Image.Resampling.BILINEAR, expand=False, fillcolor=255) pixels = rotated.load() + assert pixels is not None # a loaded grayscale image width, height = rotated.size - rows = [sum(255 - pixels[x, y] for x in range(width)) for y in range(height)] + rows = [sum(255 - cast(int, pixels[x, y]) for x in range(width)) for y in range(height)] return sum((rows[i] - rows[i - 1]) ** 2 for i in range(1, height)) diff --git a/src/docket/ocr/docling.py b/src/docket/ocr/docling.py index 3f74b1b..a129be7 100644 --- a/src/docket/ocr/docling.py +++ b/src/docket/ocr/docling.py @@ -152,8 +152,15 @@ def availability(self) -> BackendStatus: def _converter(self): from docling.datamodel.base_models import InputFormat - from docling.datamodel.pipeline_options import PdfPipelineOptions, TableFormerMode - from docling.document_converter import DocumentConverter, ImageFormatOption, PdfFormatOption + from docling.datamodel.pipeline_options import ( + PdfPipelineOptions, + TableFormerMode, + ) + from docling.document_converter import ( + DocumentConverter, + ImageFormatOption, + PdfFormatOption, + ) options = PdfPipelineOptions( do_ocr=True, do_table_structure=True, generate_page_images=False, @@ -187,7 +194,7 @@ def recognize_document(self, document: DocumentSource) -> DocumentLayout: layout = document_from_docling(result.document, backend=self.name) self._document_cache = {cache_key: layout} return layout - except Exception as exc: # noqa: BLE001 + except Exception as exc: raise OcrError(f"Docling failed on {document.path.name}: {exc}") from exc def recognize_page(self, page: PageSource): diff --git a/src/docket/ocr/languages.py b/src/docket/ocr/languages.py index bc80cf4..944f989 100644 --- a/src/docket/ocr/languages.py +++ b/src/docket/ocr/languages.py @@ -71,7 +71,7 @@ def paddle_language(languages: list[str]) -> tuple[str, str]: codes = set(languages) if codes == {"en"}: return "en", "en_PP-OCRv5_mobile_rec" - for family, (members, model) in _PADDLE_FAMILIES.items(): + for members, model in _PADDLE_FAMILIES.values(): if codes <= members: primary = next((c for c in languages if c != "en"), languages[0]) return _PADDLE_LANG_CODE.get(primary, primary), model diff --git a/src/docket/ocr/paddle.py b/src/docket/ocr/paddle.py index e908e9d..833b1c5 100644 --- a/src/docket/ocr/paddle.py +++ b/src/docket/ocr/paddle.py @@ -54,7 +54,7 @@ def _engine(key: tuple, factory): def _words_of_line(tokens: list[str], boxes: list, line_key: int, score: float) -> list[RawWord]: """Join PaddleOCR's per-token boxes into whitespace-separated words.""" words: list[RawWord] = [] - text, parts = "", [] + text, parts = "", list[tuple]() def flush() -> None: nonlocal text, parts @@ -237,7 +237,7 @@ def recognize_page(self, page: PageSource) -> PageLayout: try: with lock: result = engine.predict(bgr)[0] - except Exception as exc: # noqa: BLE001 — engine errors surface as OcrError + except Exception as exc: raise OcrError(f"PaddleOCR failed on page {page.number}: {exc}") from exc preprocessed = result.get("doc_preprocessor_res") or {} @@ -271,12 +271,12 @@ def _table_hints(self, image, ocr_result, line_height: float) -> list[TableHint] try: with lock: table_result = pipeline.predict(image, use_ocr_model=False, overall_ocr_res=ocr_result)[0] - except Exception as exc: # noqa: BLE001 + except Exception as exc: raise OcrError(f"PaddleOCR table recognition failed: {exc}") from exc hints = [] tolerance = max(3.0, 0.4 * line_height) for table in table_result.get("table_res_list") or []: - boxes = [tuple(float(v) for v in b[:4]) for b in table.get("cell_box_list") or []] + boxes = [(float(b[0]), float(b[1]), float(b[2]), float(b[3])) for b in table.get("cell_box_list") or []] cells = cells_from_boxes(boxes, tolerance) if len({c.row for c in cells}) < 2 or len({c.column for c in cells}) < 2: continue diff --git a/src/docket/ocr/pdftext.py b/src/docket/ocr/pdftext.py index b22992d..f402c78 100644 --- a/src/docket/ocr/pdftext.py +++ b/src/docket/ocr/pdftext.py @@ -53,7 +53,7 @@ def _fix_rotation(chars: list[dict]) -> int: turn = {(1, 0): 0, (0, -1): 270, (-1, 0): 180, (0, 1): 90}.get((a, b)) if turn is not None: votes[turn] = votes.get(turn, 0) + 1 - return max(votes, key=votes.get) if votes else 0 + return max(votes, key=votes.__getitem__) if votes else 0 def _turn(x0: float, y0: float, x1: float, y1: float, turn: int, w: float, h: float): @@ -76,14 +76,14 @@ def _table_hints(plumber_page, turn: int, w: float, h: float) -> list[TableHint] for table in tables: # Index rows and columns on the upright boxes, so a turned page's # table reads in its upright order. - cells = [_turn(*c, turn, w, h) for c in table.cells if c] + cells = [_turn(c[0], c[1], c[2], c[3], turn, w, h) for c in table.cells if c] if not cells: continue hint_cells = cells_from_boxes(cells, tolerance=0.5) # A table with one row or one column is a box around text, not a table. if len({c.row for c in hint_cells}) < 2 or len({c.column for c in hint_cells}) < 2: continue - bx = _turn(*table.bbox, turn, w, h) + bx = _turn(table.bbox[0], table.bbox[1], table.bbox[2], table.bbox[3], turn, w, h) hints.append(TableHint(x0=bx[0], y0=bx[1], x1=bx[2], y1=bx[3], cells=hint_cells)) return hints @@ -122,7 +122,7 @@ def recognize_page(self, page: PageSource) -> PageLayout: raw = extract_words(upright, keep_blank_chars=False) else: raw = plumber_page.extract_words(keep_blank_chars=False) - except Exception as exc: # noqa: BLE001 — malformed content streams + except Exception as exc: raise OcrError(f"could not read the text layer of page {page.number}: {exc}") from exc words = [ RawWord(text=item["text"], x0=item["x0"], y0=item["top"], x1=item["x1"], y1=item["bottom"]) diff --git a/src/docket/ocr/registry.py b/src/docket/ocr/registry.py index 65a7f17..c04edc9 100644 --- a/src/docket/ocr/registry.py +++ b/src/docket/ocr/registry.py @@ -15,13 +15,19 @@ import importlib import re +from collections.abc import Callable from importlib.metadata import entry_points -from typing import Callable, Union from pydantic import BaseModel from ..errors import ConfigurationError -from .base import BackendStatus, BackendUnavailable, Capabilities, OcrBackend, OcrSettings +from .base import ( + BackendStatus, + BackendUnavailable, + Capabilities, + OcrBackend, + OcrSettings, +) ENTRY_POINT_GROUP = "docket.ocr_backends" @@ -35,7 +41,7 @@ "docling": "docket.ocr.docling:DoclingBackend", "vlm": "docket.ocr.vlm:VlmBackend", } -_REGISTRY: dict[str, Union[str, BackendFactory]] = dict(_BUILTIN) +_REGISTRY: dict[str, str | BackendFactory] = dict(_BUILTIN) _plugins_loaded = False @@ -142,9 +148,9 @@ def list_ocr_backends(settings: OcrSettings | None = None) -> list[BackendInfo]: __all__ = [ + "ENTRY_POINT_GROUP", "BackendFactory", "BackendInfo", - "ENTRY_POINT_GROUP", "OcrBackendError", "backend_names", "get_ocr_backend", diff --git a/src/docket/ocr/source.py b/src/docket/ocr/source.py index c24678e..2bc5ebb 100644 --- a/src/docket/ocr/source.py +++ b/src/docket/ocr/source.py @@ -6,8 +6,9 @@ from __future__ import annotations import io +from collections.abc import Iterator from pathlib import Path -from typing import Iterator, Literal +from typing import Literal import pdfplumber from PIL import Image @@ -93,12 +94,12 @@ def text_pages(self) -> list[str]: self._text_pages = self.path.read_text(encoding="utf-8").split("\f") return self._text_pages - def page(self, number: int) -> "PageSource": + def page(self, number: int) -> PageSource: if not 1 <= number <= self.page_count: raise IndexError(f"page {number} out of range 1..{self.page_count}") return PageSource(self, number) - def pages(self) -> Iterator["PageSource"]: + def pages(self) -> Iterator[PageSource]: for number in range(1, self.page_count + 1): yield self.page(number) @@ -107,7 +108,7 @@ def close(self) -> None: self._plumber.close() self._plumber = None - def __enter__(self) -> "DocumentSource": + def __enter__(self) -> DocumentSource: # noqa: PYI034 — typing.Self needs Python 3.11 return self def __exit__(self, *exc) -> None: @@ -156,9 +157,9 @@ def text(self) -> str: __all__ = [ - "DocumentSource", "IMAGE_SUFFIXES", - "PageSource", "SUPPORTED_SUFFIXES", + "DocumentSource", + "PageSource", "UnsupportedDocument", ] diff --git a/src/docket/ocr_quality.py b/src/docket/ocr_quality.py index 43305b6..67dd692 100644 --- a/src/docket/ocr_quality.py +++ b/src/docket/ocr_quality.py @@ -44,7 +44,7 @@ from .llm_client import ( LLMError, chat_json, -) # noqa: F401 (LLMError re-exported for tests) +) from .logging_setup import get_logger log = get_logger() diff --git a/src/docket/options.py b/src/docket/options.py index f217448..e93f767 100644 --- a/src/docket/options.py +++ b/src/docket/options.py @@ -19,15 +19,21 @@ import importlib.util from dataclasses import dataclass from pathlib import Path -from typing import Literal, Union +from typing import Literal from pydantic import BaseModel, ConfigDict, Field from . import config from .errors import ConfigurationError -from .ocr import AcquisitionOptions, OcrBackend, OcrSettings, parse_languages, resolve_chain - -BackendSpec = Union[str, OcrBackend] +from .ocr import ( + AcquisitionOptions, + OcrBackend, + OcrSettings, + parse_languages, + resolve_chain, +) + +BackendSpec = str | OcrBackend class OcrOptions(BaseModel): diff --git a/src/docket/pii.py b/src/docket/pii.py index dc5bed2..fe455ed 100644 --- a/src/docket/pii.py +++ b/src/docket/pii.py @@ -2,7 +2,7 @@ from __future__ import annotations -from typing import Literal +from typing import Literal, cast from pydantic import BaseModel @@ -30,7 +30,8 @@ def _walk(value: object, path: str, found: list[PiiField]) -> None: for name, field in value.__class__.model_fields.items(): child = getattr(value, name) child_path = f"{path}.{name}" if path else name - category = (field.json_schema_extra or {}).get("pii") + extra = field.json_schema_extra if isinstance(field.json_schema_extra, dict) else {} + category = cast("PiiCategory | None", extra.get("pii")) if category and child not in (None, "", [], {}): if isinstance(child, list): found.extend(PiiField(path=f"{child_path}[{i}]", category=category) for i in range(len(child))) diff --git a/src/docket/pipeline.py b/src/docket/pipeline.py index 73f55e0..86a4e36 100644 --- a/src/docket/pipeline.py +++ b/src/docket/pipeline.py @@ -19,8 +19,9 @@ import hashlib import time +from collections.abc import Callable from pathlib import Path -from typing import Any, Callable +from typing import Any, Literal, cast from pydantic import BaseModel @@ -41,7 +42,6 @@ ) from .ocr_quality import looks_garbled from .options import ProcessOptions, ResolvedOptions, resolve -from .review_reasons import reasons_for from .result import ( DocumentError, DocumentResult, @@ -49,6 +49,7 @@ ProcessingMetrics, SourceLocation, ) +from .review_reasons import reasons_for from .schemas import ClassificationResult, ValidationIssue from .validate import validate @@ -66,7 +67,7 @@ class _Stages: def __init__(self) -> None: self.seconds: dict[str, float] = {} - def timed(self, stage: str) -> "_Timer": + def timed(self, stage: str) -> _Timer: return _Timer(self.seconds, stage) @@ -146,7 +147,7 @@ def _resolve_sources(extracted: dict, acquisition: Acquisition) -> dict[str, Sou if not located: sources[field] = SourceLocation(page=page_number, quote=quote) continue - status = "conflicting" if len(located) > 1 and located[0].match_score >= 1.0 else ( + status: Literal["verified", "fuzzy", "conflicting"] = "conflicting" if len(located) > 1 and located[0].match_score >= 1.0 else ( "verified" if located[0].match_score >= 1.0 else "fuzzy" ) regions = [SourceRegion(page=page_number, bbox=match.bbox, word_ids=match.word_ids) for match in located] @@ -229,11 +230,11 @@ def _read( """Schema selection, extraction and validation of one acquired reading.""" doc = {"document": path.name} with stages.timed("classify"), log_stage(log, "select_schema", **doc): - doc_type, classification = select_schema(acquisition, options.document_type) + doc_type, classification = select_schema(acquisition, cast("SchemaSpec | None", options.document_type)) _notify(on_stage, "classify", classification) language, _ = detect_language(acquisition.text) - common = { + common: dict[str, Any] = { "source": str(path), "document_id": document_id, "status": DocumentStatus.SUCCEEDED, @@ -395,6 +396,10 @@ def process_document( "stage_seconds": {k: round(v, 3) for k, v in stages.seconds.items()}, "pages": len(result.layout.pages) if result.layout else 0, "llm_calls": llm_usage.calls, + "llm_models": llm_usage.models, + "llm_input_tokens": llm_usage.input_tokens, + "llm_output_tokens": llm_usage.output_tokens, + "llm_unreported_calls": llm_usage.unreported_calls, "llm_estimated_tokens": llm_usage.estimated_tokens, "escalated_to_vlm": escalated, } diff --git a/src/docket/result.py b/src/docket/result.py index 413f75c..fd38f86 100644 --- a/src/docket/result.py +++ b/src/docket/result.py @@ -99,7 +99,13 @@ class ProcessingMetrics(BaseModel): stage_seconds: dict[str, float] = Field(default_factory=dict) pages: int = 0 llm_calls: int = 0 - llm_estimated_tokens: int = 0 + llm_models: list[str] = Field(default_factory=list, description="Models called, in first-use order.") + llm_input_tokens: int = Field(default=0, description="Prompt tokens the provider reported.") + llm_output_tokens: int = Field(default=0, description="Completion tokens the provider reported.") + llm_unreported_calls: int = Field( + default=0, description="Calls whose provider reported no token counts; not in the two sums above." + ) + llm_estimated_tokens: int = Field(default=0, description="Characters / 4 over every call, reported or not.") extract_attempts: int = 0 escalated_to_vlm: bool = Field( default=False, @@ -150,7 +156,7 @@ def is_valid(self) -> bool: i.severity == "error" for i in self.validation_issues ) - def highlights(self, page: int | None = None) -> list[tuple[str, "SourceLocation", "SourceRegion"]]: + def highlights(self, page: int | None = None) -> list[tuple[str, SourceLocation, SourceRegion]]: """(field, source, region) triples for drawing provenance boxes over the original pages — everything a viewer needs, straight from the result, optionally filtered to one page. Fields without geometry diff --git a/src/docket/review_queue.py b/src/docket/review_queue.py index c936044..72f0e74 100644 --- a/src/docket/review_queue.py +++ b/src/docket/review_queue.py @@ -10,7 +10,21 @@ from typing import Any from uuid import uuid4 -from sqlalchemy import JSON, Column, Float, Integer, MetaData, String, Table, Text, create_engine, delete, insert, select, update as sql_update +from sqlalchemy import ( + JSON, + Column, + Float, + Integer, + MetaData, + String, + Table, + Text, + create_engine, + delete, + insert, + select, +) +from sqlalchemy import update as sql_update from sqlalchemy.engine import Engine from . import config @@ -75,6 +89,14 @@ def _history(conn, document_id: str) -> list[dict]: return [dict(row) for row in rows] +def _existing(conn, document_id: str) -> dict: + """The record of a task this transaction has just written.""" + record = _record(conn, document_id) + if record is None: + raise KeyError(document_id) + return record + + def _record(conn, document_id: str) -> dict | None: row = conn.execute(select(_TASKS).where(_TASKS.c.document_id == document_id)).mappings().first() if row is None: @@ -125,7 +147,7 @@ def list_pending(*, database_url: str | None = None) -> list[dict]: with _engine(database_url).connect() as conn: ids = list(conn.execute(select(_TASKS.c.document_id).where( _TASKS.c.status.in_(("pending", "in_review", "corrected"))).order_by(_TASKS.c.queued_at)).scalars()) - return [_record(conn, document_id) for document_id in ids] + return [record for document_id in ids if (record := _record(conn, document_id)) is not None] def get(document_id: str, *, database_url: str | None = None) -> dict | None: @@ -155,7 +177,7 @@ def claim(document_id: str, *, actor: str, lease_seconds: int | None = None, if result.rowcount != 1: raise ReviewConflict("task was claimed concurrently") _revision(conn, document_id, "claimed", version, actor=actor) - return _record(conn, document_id) + return _existing(conn, document_id) def _require_lock(row: Any, token: str | None) -> None: @@ -173,7 +195,7 @@ def release(document_id: str, *, lock_token: str, actor: str, database_url: str conn.execute(sql_update(_TASKS).where(_TASKS.c.document_id == document_id).values( status=status, lock_owner=None, lock_token=None, lock_expires_at=None, updated_at=_now(), version=version)) _revision(conn, document_id, "released", version, actor=actor) - return _record(conn, document_id) + return _existing(conn, document_id) def _merge(base: dict, changes: dict) -> dict: @@ -185,6 +207,7 @@ def _merge(base: dict, changes: dict) -> dict: def _revalidate(row: Any, corrections: dict | None) -> list[dict]: from pydantic import ValidationError + from . import catalog from .validate import validate saved = DocumentResult.model_validate(row["result"]) @@ -223,7 +246,7 @@ def update(document_id: str, *, status: str, corrections: dict | None = None, ac lock_expires_at=None if terminal else row["lock_expires_at"])) _revision(conn, document_id, status, version, actor=actor, note=note, corrections=corrections, validation_issues=issues) - return _record(conn, document_id) + return _existing(conn, document_id) def revalidate(document_id: str, *, lock_token: str, actor: str = "reviewer", diff --git a/src/docket/review_reasons.py b/src/docket/review_reasons.py index c3c107d..895a79d 100644 --- a/src/docket/review_reasons.py +++ b/src/docket/review_reasons.py @@ -106,7 +106,8 @@ def key_field_confidence(result: DocumentResult) -> dict[str, float | None]: # A line can hold two values ("Invoice INV-7 date 2026-03-01"); # each field answers for the words whose digits are its own. own = [w for w in digit_words if any(_digits(w.text) in form for form in forms)] or digit_words - readings.append(min(w.confidence for w in own) if own else source.confidence) + scores = [w.confidence for w in own if w.confidence is not None] + readings.append(min(scores) if scores else source.confidence) confidences[field] = min((c for c in readings if c is not None), default=None) return confidences @@ -163,9 +164,10 @@ def quote(name: str) -> str | None: rows = value_at(document, spec.line_items.path) or [] column = spec.line_items.columns["total"] row_totals = [getattr(row, column, None) for row in rows] - if len(row_totals) >= 2 and all(isinstance(t, (int, float)) for t in row_totals): + numeric = [float(t) for t in row_totals if isinstance(t, (int, float))] + if len(numeric) >= 2 and len(numeric) == len(row_totals): for name in ("subtotal", "total_amount"): - if amount(name) and abs(sum(row_totals) - amount(name)) <= 0.01: + if amount(name) and abs(sum(numeric) - amount(name)) <= 0.01: reconciled.add(name) return reconciled diff --git a/src/docket/templates.py b/src/docket/templates.py index b54fe84..4fac8dd 100644 --- a/src/docket/templates.py +++ b/src/docket/templates.py @@ -39,7 +39,7 @@ import re from datetime import date -from typing import get_args, get_origin, Union +from typing import Any, Union, get_args, get_origin from pydantic import BaseModel, Field @@ -47,7 +47,6 @@ from .catalog import SchemaSpec from .layout.models import DocumentLayout - # ---- the template model ---------------------------------------------------------------------- @@ -196,6 +195,7 @@ def _find_rule(layout: DocumentLayout, rule: FieldRule) -> tuple[int, str, str] def _read_items(layout: DocumentLayout, rule: ItemsRule, spec: SchemaSpec): + assert spec.line_items is not None # an items rule is only accepted for schemas with rows page = layout.page(rule.page) if page is None or rule.table >= len(page.tables): return [], {} @@ -230,6 +230,7 @@ def _read_items(layout: DocumentLayout, rule: ItemsRule, spec: SchemaSpec): def _coerce_item_attr(spec: SchemaSpec, attr: str, raw: str): + assert spec.line_items is not None item_model = _annotation_of(spec.model, spec.line_items.path) if isinstance(raw, str) and _kind_of(item_model, attr) == "number": return parse_amount(raw) @@ -244,10 +245,10 @@ def _convention(text: str) -> str | None: def _annotation_of(model: type[BaseModel], path: str): """The model class or field annotation a dotted path points into.""" - current: object = model + current: Any = model for part in path.split("."): name, _, index = part.partition("[") - annotation = current.model_fields[name].annotation # type: ignore[union-attr] + annotation = current.model_fields[name].annotation if get_origin(annotation) is list: annotation = get_args(annotation)[0] if index: @@ -262,6 +263,7 @@ def _annotation_of(model: type[BaseModel], path: str): def _kind_of(model_or_annotation: object, name: str) -> str: """'date', 'number' or 'text' — how to coerce a captured string.""" + annotation: Any if isinstance(model_or_annotation, type) and issubclass(model_or_annotation, BaseModel): annotation = model_or_annotation.model_fields[name].annotation else: @@ -310,21 +312,21 @@ def _set_path(data: dict, path: str, value) -> None: for part in parts[:-1]: name, _, index = part.partition("[") if index: - index = int(index.rstrip("]")) + position = int(index.rstrip("]")) bucket = current.setdefault(name, []) - while len(bucket) <= index: + while len(bucket) <= position: bucket.append({}) - current = bucket[index] + current = bucket[position] else: current = current.setdefault(name, {}) leaf = parts[-1] name, _, index = leaf.partition("[") if index: - index = int(index.rstrip("]")) + position = int(index.rstrip("]")) bucket = current.setdefault(name, []) - while len(bucket) <= index: + while len(bucket) <= position: bucket.append({}) - bucket[index] = value + bucket[position] = value else: current[name] = value diff --git a/src/docket/validate.py b/src/docket/validate.py index 9439955..eae7537 100644 --- a/src/docket/validate.py +++ b/src/docket/validate.py @@ -11,7 +11,6 @@ import re from datetime import date, datetime - from . import amounts, checksums from .catalog.common import Party from .catalog.models import ( @@ -25,7 +24,7 @@ from .catalog.registry import SchemaSpec, ValidationContext from .schemas import ValidationIssue -_TAX_ID_RE = re.compile(r"^[A-Z0-9][A-Z0-9\-\.]{4,20}$", re.I) +_TAX_ID_RE = re.compile(r"^[A-Z0-9][A-Z0-9\-\.]{4,20}$", re.IGNORECASE) _BIC_RE = re.compile(r"^[A-Z]{4}[A-Z]{2}[A-Z0-9]{2}([A-Z0-9]{3})?$") _AMOUNT_TOLERANCE = 0.01 @@ -1230,17 +1229,17 @@ def _check_contract_against_raw_text( ) ) - for field, value in ( + for field, stated in ( ("contract_title", c.contract_title), ("governing_law", c.governing_law), ("payment_terms", c.payment_terms), ("liability_cap", c.liability_cap), ): - if value and value.strip() and not _appears_in(value, normalized_text): + if stated and stated.strip() and not _appears_in(stated, normalized_text): issues.append( ValidationIssue( field=field, - message=f"{value!r} does not appear in the document text", + message=f"{stated!r} does not appear in the document text", severity="warning", ) ) @@ -1255,19 +1254,19 @@ def _check_contract_against_raw_text( ) ) - for field, value in ( + for field, when in ( ("effective_date", c.effective_date), ("expiration_date", c.expiration_date), ): - if value is None: + if when is None: continue - if any(_appears_in(r, normalized_text) for r in _date_renderings(value)): + if any(_appears_in(r, normalized_text) for r in _date_renderings(when)): continue issues.append( ValidationIssue( field=field, message=( - f"{value.isoformat()} could not be found in the document text in any common " + f"{when.isoformat()} could not be found in the document text in any common " f"date format — contracts often spell dates out, so this is worth a look " f"rather than proof of an error" ), @@ -1539,15 +1538,14 @@ def validate_bank_statement(stmt: BankStatement, ctx: ValidationContext) -> list ) # IBAN validation - if stmt.account_iban: - if not checksums.validate_iban(stmt.account_iban): - issues.append( - ValidationIssue( - field="account_iban", - message=f"{stmt.account_iban!r} fails IBAN checksum or format", - severity="error", - ) + if stmt.account_iban and not checksums.validate_iban(stmt.account_iban): + issues.append( + ValidationIssue( + field="account_iban", + message=f"{stmt.account_iban!r} fails IBAN checksum or format", + severity="error", ) + ) # Balance reconciliation formula: # opening_balance + total_deposits - total_withdrawals == closing_balance diff --git a/tests/factories.py b/tests/factories.py index 5d5238d..e3374f2 100644 --- a/tests/factories.py +++ b/tests/factories.py @@ -3,7 +3,13 @@ from pathlib import Path -from docket.layout import DocumentLayout, PageLayout, RawWord, build_page, text_only_page +from docket.layout import ( + DocumentLayout, + PageLayout, + RawWord, + build_page, + text_only_page, +) from docket.ocr import ( Acquisition, AcquisitionReport, diff --git a/tests/security/test_hostile_documents.py b/tests/security/test_hostile_documents.py index c1bb9e1..f753d36 100644 --- a/tests/security/test_hostile_documents.py +++ b/tests/security/test_hostile_documents.py @@ -7,8 +7,8 @@ from pypdf import PdfWriter -from docket import OcrOptions, ProcessOptions, ReviewOptions, process_document -from docket import config, extract as extract_module +from docket import OcrOptions, ProcessOptions, ReviewOptions, config, process_document +from docket import extract as extract_module def _options() -> ProcessOptions: diff --git a/tests/snapshots/public_api.json b/tests/snapshots/public_api.json index d7a1352..8a2e5e3 100644 --- a/tests/snapshots/public_api.json +++ b/tests/snapshots/public_api.json @@ -1,6 +1,6 @@ { "functions": { - "export_document": "(source: \"'DocumentResult | BaseModel'\", format: 'str', options: 'ExportOptions | None' = None) -> 'ExportResult'", + "export_document": "(source: 'DocumentResult | BaseModel', format: 'str', options: 'ExportOptions | None' = None) -> 'ExportResult'", "get_ocr_backend": "(name: 'str', settings: 'OcrSettings | None' = None) -> 'OcrBackend'", "get_schema": "(schema_id: 'str', version: 'str | None' = None) -> 'SchemaSpec | None'", "list_exporters": "() -> 'list[Exporter]'", @@ -40,14 +40,14 @@ "validation_issues": {"annotation": "list[docket.schemas.ValidationIssue]", "required": false} }, "OcrOptions": { - "backend": {"annotation": "typing.Union[str, docket.ocr.base.OcrBackend, NoneType]", "required": false}, + "backend": {"annotation": "str | docket.ocr.base.OcrBackend | None", "required": false}, "deskew": {"annotation": "bool | None", "required": false}, "detect_rotation": {"annotation": "bool | None", "required": false}, "device": {"annotation": "str | None", "required": false}, "docling_cell_matching": {"annotation": "bool | None", "required": false}, "docling_table_mode": {"annotation": "typing.Optional[typing.Literal['fast', 'accurate']]", "required": false}, "dpi": {"annotation": "int | None", "required": false}, - "fallbacks": {"annotation": "list[typing.Union[str, docket.ocr.base.OcrBackend]] | None", "required": false}, + "fallbacks": {"annotation": "list[str | docket.ocr.base.OcrBackend] | None", "required": false}, "languages": {"annotation": "list[str] | str | None", "required": false}, "min_confidence": {"annotation": "float | None", "required": false}, "paddle_model": {"annotation": "typing.Optional[typing.Literal['mobile', 'medium']]", "required": false}, @@ -70,6 +70,10 @@ "extract_attempts": {"annotation": "int", "required": false}, "llm_calls": {"annotation": "int", "required": false}, "llm_estimated_tokens": {"annotation": "int", "required": false}, + "llm_input_tokens": {"annotation": "int", "required": false}, + "llm_models": {"annotation": "list[str]", "required": false}, + "llm_output_tokens": {"annotation": "int", "required": false}, + "llm_unreported_calls": {"annotation": "int", "required": false}, "pages": {"annotation": "int", "required": false}, "stage_seconds": {"annotation": "dict[str, float]", "required": false}, "template_id": {"annotation": "str | None", "required": false} diff --git a/tests/test_batch.py b/tests/test_batch.py index 78299f6..4642dc8 100644 --- a/tests/test_batch.py +++ b/tests/test_batch.py @@ -12,8 +12,15 @@ import pytest -from docket import BatchOptions, ProcessOptions, ReviewOptions, config, limits, process_batch -from docket import cli +from docket import ( + BatchOptions, + ProcessOptions, + ReviewOptions, + cli, + config, + limits, + process_batch, +) from docket import extract as extract_module from docket.batch import iter_batch, iter_sources from docket.export import tabular @@ -207,7 +214,7 @@ def slow_request(payload, *, timeout): time.sleep(0.03) prompt = payload["messages"][0]["content"] number = prompt.split("Invoice no: ")[1].split("\n")[0] - return json.dumps(_payload(number)) + return llm_client._Reply(json.dumps(_payload(number)), input_tokens=100, output_tokens=20) monkeypatch.setattr(extract_module, "chat_json", llm_client.chat_json) monkeypatch.setattr(config, "LLM_PROVIDER", "ollama") @@ -219,6 +226,8 @@ def slow_request(payload, *, timeout): # Each result counts only its own call: usage is per document, not shared. assert {r.metrics.llm_calls for r in batch.results} == {1} assert batch.metrics.llm_calls == 10 + assert {(r.metrics.llm_input_tokens, r.metrics.llm_output_tokens) for r in batch.results} == {(100, 20)} + assert batch.metrics.llm_input_tokens == 1000 and batch.metrics.llm_unreported_calls == 0 def test_no_temporary_files_are_left(options, invoices, tmp_path): diff --git a/tests/test_catalog_rules.py b/tests/test_catalog_rules.py index ee5e371..184818e 100644 --- a/tests/test_catalog_rules.py +++ b/tests/test_catalog_rules.py @@ -5,7 +5,6 @@ import json from pathlib import Path - from docket.catalog import get_schema from docket.validate import validate diff --git a/tests/test_classify.py b/tests/test_classify.py index 0b642c6..dba1f0f 100644 --- a/tests/test_classify.py +++ b/tests/test_classify.py @@ -3,7 +3,6 @@ from docket import classify as classify_module from docket.classify import classify, classify_rules - INVOICE_TEXT = """ INVOICE Invoice #: INV-2026-0042 diff --git a/tests/test_classify_tfidf.py b/tests/test_classify_tfidf.py index a428f8a..bd0145d 100644 --- a/tests/test_classify_tfidf.py +++ b/tests/test_classify_tfidf.py @@ -1,7 +1,6 @@ from docket.classify_tfidf import classify_tfidf - def test_returns_a_classification_result(): result = classify_tfidf( "Please remit payment for services rendered within 30 days." diff --git a/tests/test_config.py b/tests/test_config.py index 3dd513d..7c42f5d 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -225,9 +225,8 @@ def test_http_server_refuses_to_start(monkeypatch): from docket import api monkeypatch.setattr(config, "ERRORS", ["DOCKET_MAX_CONCURRENT_JOBS='0': must be at least 1"]) - with pytest.raises(ConfigurationError, match="MAX_CONCURRENT_JOBS"): - with TestClient(api.app): - pass + with pytest.raises(ConfigurationError, match="MAX_CONCURRENT_JOBS"), TestClient(api.app): + pass def test_docket_api_command_exits_with_code_3(tmp_path, monkeypatch, capsys): diff --git a/tests/test_custom_schemas.py b/tests/test_custom_schemas.py index ef4707e..9b76bbc 100644 --- a/tests/test_custom_schemas.py +++ b/tests/test_custom_schemas.py @@ -7,7 +7,6 @@ from fastapi.testclient import TestClient from pydantic import BaseModel -import docket.catalog.registry as registry import docket.export as export_module from docket import ( CitedDocument, @@ -18,14 +17,23 @@ SchemaSpec, ValidationIssue, add_validator, + api, export_document, keywords, process_document, register_exporter, register_schema, + validate, +) +from docket import classify as classify_module +from docket import extract as extract_module +from docket.catalog import ( + get_schema, + list_schemas, + migrate, + registry, + unregister_schema, ) -from docket import api, classify as classify_module, extract as extract_module, validate -from docket.catalog import get_schema, list_schemas, migrate, unregister_schema from docket.catalog.registry import Migration from docket.result import DocumentResult, SourceLocation from docket.schemas import ClassificationResult diff --git a/tests/test_document_result.py b/tests/test_document_result.py index 31fab52..fee8653 100644 --- a/tests/test_document_result.py +++ b/tests/test_document_result.py @@ -4,7 +4,8 @@ import pymupdf as fitz import pytest -from docket import extract as extract_module, pipeline +from docket import extract as extract_module +from docket import pipeline from docket.ocr import BackendUnavailable, OcrBackendError, UnknownLanguage from docket.options import OcrOptions, ProcessOptions, ReviewOptions from docket.result import DocumentResult, DocumentStatus @@ -85,7 +86,7 @@ def test_every_cited_field_resolves_to_a_region(invoice_result): assert " ".join(page.word(i).text for i in total.word_ids) == "Total due: EUR 121.00" # Lines were laid out top to bottom, so the boxes are too. assert sources["invoice_number"].bbox.y0 < sources["total_amount"].bbox.y0 - x0, y0, x1, y1 = total.bbox.to_absolute(page.width, page.height) + x0, _y0, _x1, y1 = total.bbox.to_absolute(page.width, page.height) assert 70 <= x0 <= 74 and y1 <= page.height diff --git a/tests/test_einvoice.py b/tests/test_einvoice.py index 6447496..d78eb0f 100644 --- a/tests/test_einvoice.py +++ b/tests/test_einvoice.py @@ -27,10 +27,10 @@ extract_facturx_xml, fetch, generate_facturx_pdf, - validate_pdfa, validate_einvoice, - verify_facturx_round_trip, + validate_pdfa, validator, + verify_facturx_round_trip, ) from docket.export import ExportError, ExportOptions, export_document @@ -306,6 +306,7 @@ def _facturx_pdf(tmp_path: Path, xml: bytes, name: str = "factur-x.xml") -> Path def _blank_pdf() -> bytes: from io import BytesIO + from pypdf import PdfWriter output = BytesIO() @@ -317,6 +318,7 @@ def _blank_pdf() -> bytes: def test_factur_x_pdf_generation_embeds_xml_and_xmp(): from io import BytesIO + from pypdf import PdfReader xml = export_document(complete_invoice(), "factur-x-en16931").content.encode() diff --git a/tests/test_escalation.py b/tests/test_escalation.py index a26ddaa..8b98d50 100644 --- a/tests/test_escalation.py +++ b/tests/test_escalation.py @@ -15,7 +15,13 @@ from docket.ocr import AcquisitionError from docket.result import DocumentResult from docket.schemas import ValidationIssue -from tests.factories import acquisition, flat_invoice, make_result, text_acquisition, words_page +from tests.factories import ( + acquisition, + flat_invoice, + make_result, + text_acquisition, + words_page, +) _ERROR = [ValidationIssue(field="total_amount", message="does not add up")] diff --git a/tests/test_eval_scripts.py b/tests/test_eval_scripts.py index 5903efb..bcb486b 100644 --- a/tests/test_eval_scripts.py +++ b/tests/test_eval_scripts.py @@ -33,14 +33,17 @@ def test_eval_script_imports_cleanly(script): [ sys.executable, "-c", - f"import importlib.util,sys; sys.path.insert(0, {str(ROOT / 'src')!r}); " - f"sys.path.insert(0, {str(ROOT / 'eval')!r}); " - f"spec=importlib.util.spec_from_file_location('m', {str(script)!r}); " - f"m=importlib.util.module_from_spec(spec); spec.loader.exec_module(m)", + ( + f"import importlib.util,sys; sys.path.insert(0, {str(ROOT / 'src')!r}); " + f"sys.path.insert(0, {str(ROOT / 'eval')!r}); " + f"spec=importlib.util.spec_from_file_location('m', {str(script)!r}); " + f"m=importlib.util.module_from_spec(spec); spec.loader.exec_module(m)" + ), ], capture_output=True, text=True, timeout=120, + check=False, ) assert result.returncode == 0, result.stderr[-1500:] @@ -200,6 +203,7 @@ def test_pipeline_benchmark_resumes_from_its_checkpoint(tmp_path, monkeypatch): """A long run killed half-way must continue where it stopped, not restart.""" sys.path.insert(0, str(ROOT / "eval")) import benchmark_ocr + from tests.factories import make_result calls = [] @@ -225,9 +229,10 @@ def fake_process(path, options): def test_pipeline_benchmark_records_source_and_key_field_confidence(tmp_path, monkeypatch): """One run must hold what tuning DOCKET_MIN_SOURCE_CONFIDENCE needs.""" sys.path.insert(0, str(ROOT / "eval")) - import benchmark_ocr from datetime import date + import benchmark_ocr + from docket.result import SourceLocation from tests.factories import flat_invoice, make_result diff --git a/tests/test_export.py b/tests/test_export.py index aa93925..5156196 100644 --- a/tests/test_export.py +++ b/tests/test_export.py @@ -3,8 +3,8 @@ import xml.etree.ElementTree as ET from datetime import date -from docket.export import export_to_facturae_xml from docket.catalog import BankStatement, BankStatementTransaction, Invoice, LineItem +from docket.export import export_to_facturae_xml from tests.factories import flat_invoice diff --git a/tests/test_http_api.py b/tests/test_http_api.py index 60526e5..a3efeb2 100644 --- a/tests/test_http_api.py +++ b/tests/test_http_api.py @@ -11,7 +11,8 @@ import pytest from fastapi.testclient import TestClient -from docket import api, config, extract as extract_module, job_store, review_queue +from docket import api, config, job_store, review_queue +from docket import extract as extract_module INVOICE = ( "INVOICE\nInvoice no: INV-7\nDate: 2026-03-02\nFrom: Acme GmbH\nTo: Beta SA\n" diff --git a/tests/test_integration_api.py b/tests/test_integration_api.py index 667e24f..6d76073 100644 --- a/tests/test_integration_api.py +++ b/tests/test_integration_api.py @@ -5,8 +5,8 @@ import pypdfium2 as pdfium import pytest -from docket import config, llm_client, pdf -from docket import cli +from docket import cli, config, llm_client, pdf +from docket.catalog import Invoice from docket.export import ( ExportError, export_document, @@ -14,8 +14,6 @@ list_exporters, register_exporter, ) -from docket.catalog import Invoice - from tests.test_export import sample_bank_statement, sample_invoice @@ -93,7 +91,7 @@ def json(self) -> dict: def test_openai_compatible_backend(monkeypatch): captured: dict = {} - def fake_post(url, json=None, headers=None, timeout=None): # noqa: ANN001 + def fake_post(url, json=None, headers=None, timeout=None): captured.update(url=url, payload=json, headers=headers) return _Response({"choices": [{"message": {"content": '{"ok": true}'}}]}) @@ -114,7 +112,7 @@ def test_openai_vision_sends_data_url(monkeypatch, tmp_path): image.write_bytes(b"img") captured: dict = {} - def fake_post(url, json=None, headers=None, timeout=None): # noqa: ANN001 + def fake_post(url, json=None, headers=None, timeout=None): captured.update(json) return _Response({"choices": [{"message": {"content": " text "}}]}) diff --git a/tests/test_layout.py b/tests/test_layout.py index 2150532..f59099c 100644 --- a/tests/test_layout.py +++ b/tests/test_layout.py @@ -91,9 +91,9 @@ def test_engine_line_grouping_keeps_skewed_lines_apart(): """Two skewed lines whose word boxes brush against each other must not interleave: the engine's own line identity wins over a flat band.""" words = [] - for i, token in enumerate("the quick brown fox jumps".split()): + for i, token in enumerate(["the", "quick", "brown", "fox", "jumps"]): words.append(RawWord(token, 20 + 60 * i, 20 + 3 * i, 70 + 60 * i, 32 + 3 * i, 0.9, line_key="a")) - for i, token in enumerate("over the lazy dog again".split()): + for i, token in enumerate(["over", "the", "lazy", "dog", "again"]): words.append(RawWord(token, 20 + 60 * i, 30 + 3 * i, 70 + 60 * i, 42 + 3 * i, 0.9, line_key="b")) page = _page(words) assert [line.text for line in page.lines] == [ diff --git a/tests/test_line_item_citations.py b/tests/test_line_item_citations.py index 399a2b9..c27d636 100644 --- a/tests/test_line_item_citations.py +++ b/tests/test_line_item_citations.py @@ -8,7 +8,8 @@ import pymupdf as fitz import pytest -from docket import extract as extract_module, pipeline +from docket import extract as extract_module +from docket import pipeline from docket.options import OcrOptions, ProcessOptions, ReviewOptions from docket.result import DocumentStatus @@ -215,7 +216,7 @@ def test_highlights_draw_everything_from_the_result(result): # Drawable in page coordinates: the union box of the first item's row. page = result.layout.pages[0] _, _, region = next(h for h in highlights if h[0] == "line_items[0].quantity") - x0, y0, x1, y1 = region.bbox.to_absolute(page.width, page.height) + x0, _y0, _x1, y1 = region.bbox.to_absolute(page.width, page.height) assert 70 <= x0 <= 78 and y1 <= page.height diff --git a/tests/test_llm_retries.py b/tests/test_llm_retries.py index 9a2afd4..aed9340 100644 --- a/tests/test_llm_retries.py +++ b/tests/test_llm_retries.py @@ -21,8 +21,10 @@ def __call__(self, url, *, json, headers=None, timeout): outcome = self.outcomes.pop(0) if isinstance(outcome, Exception): raise outcome + status, headers_out = outcome if isinstance(outcome, tuple) else (outcome, {}) request = httpx.Request("POST", url) - return httpx.Response(outcome, json={"message": {"content": '{"ok": true}'}}, request=request) + body = {"message": {"content": '{"ok": true}'}, "prompt_eval_count": 50, "eval_count": 5} + return httpx.Response(status, json=body, headers=headers_out, request=request) @pytest.fixture @@ -82,3 +84,37 @@ def test_openai_compatible_endpoints_are_hosted_unless_local(monkeypatch, base_u monkeypatch.setattr(config, "LLM_PROVIDER", "openai") monkeypatch.setattr(config, "LLM_BASE_URL", base_url) assert llm_client._hosted("any-model") is hosted + + +def test_retry_after_is_honoured_and_capped(monkeypatch, server): + slept = [] + monkeypatch.setattr(llm_client.time, "sleep", slept.append) + server((429, {"Retry-After": "7"}), (503, {"Retry-After": "3600"}), 200) + assert llm_client.chat_json("x", model="qwen3:8b") == {"ok": True} + assert slept == [7.0, llm_client._MAX_RETRY_AFTER_S] + + +def test_retry_after_as_an_http_date(): + from datetime import datetime, timedelta, timezone + from email.utils import format_datetime + + when = format_datetime(datetime.now(timezone.utc) + timedelta(seconds=30), usegmt=True) + resp = httpx.Response(429, headers={"Retry-After": when}) + assert 25 <= llm_client._retry_delay(0, resp) <= 30 + + +def test_backoff_without_retry_after_grows_with_jitter(monkeypatch): + monkeypatch.setattr(llm_client, "_BACKOFF_S", 2.0) + first = [llm_client._retry_delay(0) for _ in range(50)] + third = [llm_client._retry_delay(2) for _ in range(50)] + assert all(1.0 <= d <= 3.0 for d in first) and len(set(first)) > 1 + assert all(4.0 <= d <= 12.0 for d in third) + + +def test_provider_token_counts_are_recorded(server): + server(200) + llm_client.usage.reset() + llm_client.chat_json("x", model="qwen3:8b") + assert (llm_client.usage.input_tokens, llm_client.usage.output_tokens) == (50, 5) + assert llm_client.usage.models == ["qwen3:8b"] + diff --git a/tests/test_long_documents.py b/tests/test_long_documents.py index 6d83d36..6daee0b 100644 --- a/tests/test_long_documents.py +++ b/tests/test_long_documents.py @@ -1,6 +1,5 @@ from datetime import date - from docket import extract as extract_module from docket.catalog import Invoice diff --git a/tests/test_matching.py b/tests/test_matching.py index 4813881..35d1b47 100644 --- a/tests/test_matching.py +++ b/tests/test_matching.py @@ -1,12 +1,12 @@ from datetime import date +from docket.catalog import Contract, LineItem, Receipt, Waybill, WaybillItem from docket.matching import ( match_invoice_to_po, match_invoices_to_contract, match_receipt_to_transactions, match_three_way, ) -from docket.catalog import Contract, LineItem, Receipt, Waybill, WaybillItem from docket.schemas import BankTransaction, DiscrepancyType, MatchingStatus from tests.factories import flat_invoice, flat_po diff --git a/tests/test_metrics.py b/tests/test_metrics.py index 3541483..989f822 100644 --- a/tests/test_metrics.py +++ b/tests/test_metrics.py @@ -3,10 +3,10 @@ sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "eval")) -from metrics import ( # noqa: E402 +from metrics import ( + error_breakdown, field_accuracy, field_precision_recall_f1, - error_breakdown, review_safety_metrics, validation_detection_metrics, ) diff --git a/tests/test_model_selection.py b/tests/test_model_selection.py index b4b7519..22751b8 100644 --- a/tests/test_model_selection.py +++ b/tests/test_model_selection.py @@ -19,7 +19,7 @@ def json(self) -> dict: def _capture_payload(monkeypatch) -> dict: captured: dict = {} - def fake_post(url, json=None, headers=None, timeout=None): # noqa: ANN001 + def fake_post(url, json=None, headers=None, timeout=None): captured.update(json) return _FakeResponse() @@ -84,7 +84,7 @@ def raise_for_status(self) -> None: def json(self) -> dict: return {"models": models} - return lambda url, timeout=None: Resp() # noqa: ANN001 + return lambda url, timeout=None: Resp() MODELS = [ @@ -106,7 +106,7 @@ def test_vision_list_requires_both_capabilities(monkeypatch): def test_unreachable_ollama_returns_no_choices_rather_than_raising(monkeypatch): - def boom(url, timeout=None): # noqa: ANN001 + def boom(url, timeout=None): raise llm_client.httpx.ConnectError("connection refused") monkeypatch.setattr(llm_client.httpx, "get", boom) diff --git a/tests/test_ocr_deskew.py b/tests/test_ocr_deskew.py index 18845fe..e0a85f8 100644 --- a/tests/test_ocr_deskew.py +++ b/tests/test_ocr_deskew.py @@ -1,5 +1,5 @@ -from PIL import Image, ImageDraw import pytest +from PIL import Image, ImageDraw from docket.ocr.deskew import deskew_image diff --git a/tests/test_ocr_paddle.py b/tests/test_ocr_paddle.py index 734fffe..fb0f710 100644 --- a/tests/test_ocr_paddle.py +++ b/tests/test_ocr_paddle.py @@ -12,7 +12,13 @@ import pytest from PIL import Image -from docket.ocr import BackendUnavailable, DocumentSource, OcrSettings, UnknownLanguage, get_ocr_backend +from docket.ocr import ( + BackendUnavailable, + DocumentSource, + OcrSettings, + UnknownLanguage, + get_ocr_backend, +) from docket.ocr import paddle as paddle_module from docket.ocr.languages import paddle_language from docket.ocr.paddle import PaddleOCRBackend, page_confidence, words_from_result @@ -247,7 +253,7 @@ def find_spec(self, name, path, target=None): assert not any(m.split(".")[0] in {"paddle", "paddleocr", "paddlex"} for m in sys.modules) """ result = subprocess.run( - [sys.executable, "-c", code], capture_output=True, text=True, timeout=120, cwd=ROOT + [sys.executable, "-c", code], capture_output=True, text=True, timeout=120, cwd=ROOT, check=False ) assert result.returncode == 0, result.stderr[-2000:] assert result.stdout.strip().endswith("OK") diff --git a/tests/test_page_validation.py b/tests/test_page_validation.py index 509e513..3895d41 100644 --- a/tests/test_page_validation.py +++ b/tests/test_page_validation.py @@ -2,11 +2,10 @@ from docket import pipeline from docket.catalog import Receipt +from docket.layout import text_only_page from docket.schemas import ClassificationResult from docket.validate import validate from tests.factories import acquisition, words_page -from docket.layout import text_only_page - TEXT = "Example Shop\n2026-03-18\nSUBTOTAL 167.93\nTAX 12.59\nTOTAL 180.52" diff --git a/tests/test_pipeline_options.py b/tests/test_pipeline_options.py index 2ea049e..adebefe 100644 --- a/tests/test_pipeline_options.py +++ b/tests/test_pipeline_options.py @@ -20,11 +20,12 @@ export_document, pipeline, ) -from docket import classify as classify_module, extract as extract_module -from docket.catalog import SchemaError as DocumentTypeError, load_schema +from docket import classify as classify_module +from docket import extract as extract_module +from docket.catalog import Invoice, load_schema +from docket.catalog import SchemaError as DocumentTypeError from docket.options import resolve from docket.result import DocumentStatus -from docket.catalog import Invoice INVOICE_TEXT = ( "INVOICE\nInvoice no: INV-7\nDate: 2026-03-02\nFrom: Acme GmbH\nTo: Beta SA\n" diff --git a/tests/test_public_api.py b/tests/test_public_api.py index c94e299..0bfa287 100644 --- a/tests/test_public_api.py +++ b/tests/test_public_api.py @@ -4,8 +4,9 @@ import json from pathlib import Path -import docket import pytest + +import docket from docket import ( DocumentError, DocumentResult, @@ -16,7 +17,6 @@ ) from docket._deprecation import warn_deprecated - SNAPSHOT = Path(__file__).parent / "snapshots" / "public_api.json" KEY_FUNCTIONS = ( "export_document", diff --git a/tests/test_review_queue.py b/tests/test_review_queue.py index 8578840..fc2dacf 100644 --- a/tests/test_review_queue.py +++ b/tests/test_review_queue.py @@ -4,8 +4,8 @@ import pytest from docket import review_queue, review_reasons -from docket.schemas import ClassificationResult, ValidationIssue from docket.result import DocumentResult +from docket.schemas import ClassificationResult, ValidationIssue from tests.factories import acquisition, flat_invoice, make_result, words_page diff --git a/tests/test_schemas.py b/tests/test_schemas.py index e36a60f..7b4b725 100644 --- a/tests/test_schemas.py +++ b/tests/test_schemas.py @@ -3,7 +3,16 @@ import pytest from pydantic import ValidationError -from docket.catalog import BankStatement, BankStatementTransaction, Contract, LineItem, Receipt, ReceiptItem, Waybill, WaybillItem +from docket.catalog import ( + BankStatement, + BankStatementTransaction, + Contract, + LineItem, + Receipt, + ReceiptItem, + Waybill, + WaybillItem, +) from tests.factories import flat_invoice, flat_po diff --git a/tests/test_templates.py b/tests/test_templates.py index 23a2dd0..e880c95 100644 --- a/tests/test_templates.py +++ b/tests/test_templates.py @@ -8,8 +8,16 @@ import pytest import docket.templates as templates_module -from docket import catalog, extract as extract_module, pipeline -from docket.layout.models import BoundingBox, DocumentLayout, PageLayout, Table, TableCell, TextLine +from docket import catalog, pipeline +from docket import extract as extract_module +from docket.layout.models import ( + BoundingBox, + DocumentLayout, + PageLayout, + Table, + TableCell, + TextLine, +) from docket.options import OcrOptions, ProcessOptions, ReviewOptions from docket.result import DocumentStatus from docket.templates import ( @@ -93,7 +101,7 @@ def _pdf(path, lines=LINES): def test_lines_after_reads_the_line_below_the_label(): layout = _layout("Bill to:\nIberia Mantenimiento SA") rule = FieldRule(field="buyer.name", label="Bill to", value=r"^(.+?)\s*$", lines_after=1) - page, quote, captured = templates_module._find_rule(layout, rule) + _page, quote, captured = templates_module._find_rule(layout, rule) assert captured == "Iberia Mantenimiento SA" assert quote.startswith("Bill to:") # the citation names the label line diff --git a/tests/test_tracing.py b/tests/test_tracing.py index 5110e63..a649fa1 100644 --- a/tests/test_tracing.py +++ b/tests/test_tracing.py @@ -59,4 +59,15 @@ def test_usage_counter_accumulates_and_resets(): assert llm_client.usage.estimated_tokens == 20 llm_client.usage.reset() - assert llm_client.usage.snapshot() == {"calls": 0, "estimated_tokens": 0} + assert llm_client.usage.snapshot() == {"calls": 0, "estimated_tokens": 0, "input_tokens": 0, + "output_tokens": 0, "unreported_calls": 0, "models": []} + + +def test_usage_sums_provider_reported_tokens_and_counts_the_rest(): + llm_client.usage.reset() + llm_client.usage.record("p", "c", model="m1", reply=llm_client._Reply("c", 120, 30)) + llm_client.usage.record("p", "c", model="m2", reply=llm_client._Reply("c")) + llm_client.usage.record("p", "c", model="m1", reply=llm_client._Reply("c", 80, 10)) + snap = llm_client.usage.snapshot() + assert (snap["input_tokens"], snap["output_tokens"], snap["unreported_calls"]) == (200, 40, 1) + assert snap["models"] == ["m1", "m2"] diff --git a/tests/test_validate.py b/tests/test_validate.py index 25b1f58..79c6e7d 100644 --- a/tests/test_validate.py +++ b/tests/test_validate.py @@ -1,6 +1,16 @@ from datetime import date -from docket.catalog import BankStatement, BankStatementTransaction, Contract, Invoice, LineItem, Receipt, ReceiptItem, Waybill, WaybillItem +from docket.catalog import ( + BankStatement, + BankStatementTransaction, + Contract, + Invoice, + LineItem, + Receipt, + ReceiptItem, + Waybill, + WaybillItem, +) from docket.validate import assess_contract_risks, validate from tests.factories import flat_invoice, flat_po diff --git a/tests/test_witness.py b/tests/test_witness.py index 29bd499..337c47a 100644 --- a/tests/test_witness.py +++ b/tests/test_witness.py @@ -15,11 +15,11 @@ """ from datetime import date +from docket.catalog import LineItem from docket.layout import build_page, text_only_page from docket.ocr import AcquisitionOptions, acquire from docket.ocr.tesseract import page_confidence, words_from_data from docket.ocr.witness import confident_amounts -from docket.catalog import LineItem from docket.validate import validate from tests.factories import ScriptedBackend, flat_invoice, words_page, write_png @@ -61,10 +61,10 @@ def _invoice(**overrides): ) # What Tesseract independently read of the same page. -WITNESS_AGREES = [ +WITNESS_AGREES = [( "Prototype | 2 | 4000.00 | 8000.00\n" "Subtotal: USD 8000.00\nSales Tax: USD 450.00\nTotal: USD 8,450.00" -] +)] def _data(texts, confs, lefts, widths, lines=None):