From 15000a75a9d4260717d177e19674e11727cbf701 Mon Sep 17 00:00:00 2001 From: KazKozDev Date: Thu, 24 Sep 2026 16:51:28 +0200 Subject: [PATCH] feat: crop phone photos, re-read small text enlarged, preprocess hook - [photo] extra (OpenCV): find and flatten the document in a photo; refuse crops that would cut text; record crop_quad on the page layout. - Tesseract re-reads pages with small text enlarged, keeping the enlarged reading only when it is at least as confident. - OcrOptions.preprocess: a caller's (image, page) -> image step before OCR. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 13 ++ README.md | 5 +- docket.example.toml | 7 ++ docs/openapi.json | 25 ++++ pyproject.toml | 8 +- src/docket/config.py | 7 ++ src/docket/layout/models.py | 7 ++ src/docket/ocr/acquire.py | 4 + src/docket/ocr/base.py | 18 ++- src/docket/ocr/preprocess.py | 123 ++++++++++++++++++ src/docket/ocr/source.py | 25 +++- src/docket/ocr/tesseract.py | 52 ++++++-- src/docket/options.py | 14 +++ tests/snapshots/public_api.json | 3 + tests/test_preprocess.py | 215 ++++++++++++++++++++++++++++++++ 15 files changed, 506 insertions(+), 20 deletions(-) create mode 100644 src/docket/ocr/preprocess.py create mode 100644 tests/test_preprocess.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 5614035..68efdf3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,6 +31,19 @@ exported from `docket`, the `docket` / `docket-api` commands, the HTTP API in ### Added +- Phone photos: with the new `[photo]` extra (OpenCV), a photographed + receipt or invoice is cropped to the paper and flattened before OCR + (`DOCKET_OCR_CROP_PHOTOS`, on by default). Crops that would cut text — a + table frame on a scan, a page that already fills the frame — are refused. + `PageLayout.crop_quad` records where the page was in the original photo. +- Small text: when Tesseract's typical word is shorter than + `DOCKET_OCR_MIN_TEXT_HEIGHT` (20 px), the page is re-read enlarged (up to + 3×) and the enlarged reading is kept only if Tesseract is at least as sure + of it. On an 11 px synthetic invoice this turns 16/03/2028 and 316.00 back + into the printed 15/03/2026 and 315.00. +- `OcrOptions(preprocess=fn)`: your own `(image, page_number) -> image` step + on every page image, after cropping and before OCR (Docling reads the + file itself and skips it). - Real LLM usage per document: `ProcessingMetrics.llm_input_tokens` / `llm_output_tokens` from what the provider reports (Ollama `prompt_eval_count`/`eval_count`, OpenAI `usage`), `llm_unreported_calls` diff --git a/README.md b/README.md index 7146b42..314c4ab 100644 --- a/README.md +++ b/README.md @@ -112,7 +112,7 @@ Method, per-source results and how to rerun: [docs/BENCHMARKS.md](https://github ## How it works -Text comes from the cheapest source that works, page by page: PDF text layer, then OCR (Tesseract, PaddleOCR, Docling or a plugin), then a vision model only when OCR is unusable. Classification tries keyword rules, TF-IDF, then an LLM. Extraction fills a Pydantic schema and cites the verbatim line for every value; schema errors go back to the model. Validation never calls a model: arithmetic to the cent, dates, IBAN, VAT and tax-ID check digits, and that every cited line contains the value. Anything uncertain goes to review instead of being silently fixed. +Text comes from the cheapest source that works, page by page: PDF text layer, then OCR (Tesseract, PaddleOCR, Docling or a plugin), then a vision model only when OCR is unusable. Before OCR, a phone photo of a receipt or invoice is cropped to the paper and flattened (the `[photo]` extra), and Tesseract re-reads small text enlarged; `OcrOptions(preprocess=fn)` adds your own step on every page image. Classification tries keyword rules, TF-IDF, then an LLM. Extraction fills a Pydantic schema and cites the verbatim line for every value; schema errors go back to the model. Validation never calls a model: arithmetic to the cent, dates, IBAN, VAT and tax-ID check digits, and that every cited line contains the value. Anything uncertain goes to review instead of being silently fixed. ``` document → text layer / OCR / VLM → classify → extract + cite → validate → JSON or review @@ -131,6 +131,8 @@ Priority, lowest to highest: defaults, a TOML file (`--config` or `DOCKET_CONFIG | `DOCKET_OCR_BACKEND` | `auto` | `tesseract`, `paddle`, `docling`, `auto` or a plugin name | | `DOCKET_OCR_FALLBACKS` | `vlm` | Backends tried when a page's reading is rejected | | `DOCKET_OCR_LANGUAGES` | `en` | ISO 639-1 codes, e.g. `en,de,fr` | +| `DOCKET_OCR_CROP_PHOTOS` | `true` | Crop and flatten the document in a photo (needs `[photo]`) | +| `DOCKET_OCR_MIN_TEXT_HEIGHT` | `20` | Tesseract re-reads a page enlarged when words are shorter than this (px); `0` off | | `DOCKET_MIN_CONFIDENCE` | `0.55` | Classification confidence below which a document goes to review | | `DOCKET_MIN_SOURCE_CONFIDENCE` | `0.8` | OCR confidence a key field's cited words need, or the document goes to review | | `DOCKET_REVIEW_QUEUE_ENABLED` | `false` | Persist flagged documents in the review queue (`[review]` extra) | @@ -163,6 +165,7 @@ pip install "docket-idp[einvoice]" # official e-invoice validation pip install "docket-idp[review]" # persistent review queue (SQLAlchemy; [postgres] for PostgreSQL) pip install "docket-idp[paddle]" # PaddleOCR backend pip install "docket-idp[docling]" # Docling/TableFormer backend +pip install "docket-idp[photo]" # crop and flatten documents in phone photos (OpenCV) pip install "docket-idp[all]" # + Langfuse tracing and e-invoice validation ``` diff --git a/docket.example.toml b/docket.example.toml index 8969edf..175fd35 100644 --- a/docket.example.toml +++ b/docket.example.toml @@ -55,6 +55,13 @@ # detect_rotation = true # Correct fine scan skew before raster OCR. Env: DOCKET_OCR_DESKEW. # deskew = true +# Find the document in a photo (receipt or invoice on a table) and flatten it +# before OCR; needs the [photo] extra, otherwise photos are read as they are. +# Env: DOCKET_OCR_CROP_PHOTOS. +# crop_photos = true +# Tesseract re-reads a page enlarged when its typical word is shorter than +# this many pixels (low-resolution scans); 0 disables. Env: DOCKET_OCR_MIN_TEXT_HEIGHT. +# min_text_height = 20.0 # Resolution PDF pages are rendered at for OCR. Env: DOCKET_OCR_DPI. # dpi = 200 # Tesseract page-segmentation mode (3 automatic; 6 can suit dense tables). Env: diff --git a/docs/openapi.json b/docs/openapi.json index c6198e0..b5eec47 100644 --- a/docs/openapi.json +++ b/docs/openapi.json @@ -4197,6 +4197,31 @@ "description": "Fine clockwise correction in degrees, applied after orthogonal rotation.", "default": 0.0 }, + "crop_quad": { + "anyOf": [ + { + "items": { + "prefixItems": [ + { + "type": "number" + }, + { + "type": "number" + } + ], + "type": "array", + "maxItems": 2, + "minItems": 2 + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Crop Quad", + "description": "When the page was cropped out of a photo: its corners (top-left, top-right, bottom-right, bottom-left) in the original image, normalized 0..1. Word boxes are on the flattened page." + }, "backend": { "type": "string", "title": "Backend", diff --git a/pyproject.toml b/pyproject.toml index fea8fef..788c0aa 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -50,6 +50,10 @@ api = [ "uvicorn>=0.29", "python-multipart>=0.0.9", ] +# Crop and flatten documents in phone photos before OCR. +photo = [ + "opencv-python-headless>=4.8", +] # Persistent review queue (ReviewOptions(enqueue=True)); SQLite by default. review = [ "SQLAlchemy>=2.0,<3", @@ -83,13 +87,13 @@ einvoice = [ "saxonche>=12.5", ] all = [ - "docket-idp[api,tracing,einvoice,postgres]", + "docket-idp[api,tracing,einvoice,postgres,photo]", ] eval = [ "datasets>=2.19", ] dev = [ - "docket-idp[api,eval,einvoice]", + "docket-idp[api,eval,einvoice,photo]", # The Streamlit demo (examples/streamlit_demo.py) is a repo-only dev tool. "streamlit>=1.49", "pytest>=8.0", diff --git a/src/docket/config.py b/src/docket/config.py index 3bb9ecd..f443835 100644 --- a/src/docket/config.py +++ b/src/docket/config.py @@ -119,6 +119,11 @@ def _cpu_half() -> int: "A page reading is accepted at this confidence (0..1).", minimum=0, maximum=1), Setting("OCR_DETECT_ROTATION", "ocr.detect_rotation", "DOCKET_OCR_DETECT_ROTATION", "bool", True, "Detect and undo page rotation on scans (Tesseract OSD)."), + Setting("OCR_CROP_PHOTOS", "ocr.crop_photos", "DOCKET_OCR_CROP_PHOTOS", "bool", True, + "Find the document in a photo and flatten it before OCR (needs the [photo] extra)."), + Setting("OCR_MIN_TEXT_HEIGHT", "ocr.min_text_height", "DOCKET_OCR_MIN_TEXT_HEIGHT", "float", 20.0, + "Tesseract re-reads a page enlarged when its typical word is shorter than this (px); 0 disables.", + minimum=0, maximum=200), Setting("OCR_DESKEW", "ocr.deskew", "DOCKET_OCR_DESKEW", "bool", True, "Correct fine scan skew before raster OCR."), Setting("OCR_DPI", "ocr.dpi", "DOCKET_OCR_DPI", "int", 200, @@ -419,6 +424,8 @@ def describe() -> list[dict[str, Any]]: OCR_MIN_CONFIDENCE: float OCR_DETECT_ROTATION: bool OCR_DESKEW: bool +OCR_CROP_PHOTOS: bool +OCR_MIN_TEXT_HEIGHT: float OCR_DPI: int TESSERACT_PSM: int OCR_CONCURRENCY: int diff --git a/src/docket/layout/models.py b/src/docket/layout/models.py index e3a03a0..829c211 100644 --- a/src/docket/layout/models.py +++ b/src/docket/layout/models.py @@ -176,6 +176,13 @@ class PageLayout(BaseModel): le=15.0, description="Fine clockwise correction in degrees, applied after orthogonal rotation.", ) + crop_quad: list[tuple[float, float]] | None = Field( + default=None, + description=( + "When the page was cropped out of a photo: its corners (top-left, top-right, bottom-right, " + "bottom-left) in the original image, normalized 0..1. Word boxes are on the flattened page." + ), + ) backend: str = Field(description="OCR backend that produced this page.") confidence: float | None = Field( default=None, diff --git a/src/docket/ocr/acquire.py b/src/docket/ocr/acquire.py index e2580cc..b86e27a 100644 --- a/src/docket/ocr/acquire.py +++ b/src/docket/ocr/acquire.py @@ -288,9 +288,13 @@ def acquire(path: str | Path, options: AcquisitionOptions | None = None) -> Acqu dpi=options.settings.dpi, max_pages=options.max_pages, max_pixels=options.max_pixels, + crop_photos=options.settings.crop_photos, + preprocess=options.settings.preprocess, ) as source: for page in source.pages(): layout, record = _read_page(page, chain, options, pdf_text) + if page.crop_quad is not None: + layout = layout.model_copy(update={"crop_quad": page.crop_quad}) pages.append(layout) records.append(record) except UnsupportedDocument: diff --git a/src/docket/ocr/base.py b/src/docket/ocr/base.py index 72a0237..b6daff3 100644 --- a/src/docket/ocr/base.py +++ b/src/docket/ocr/base.py @@ -19,9 +19,11 @@ def recognize_page(self, page: PageSource) -> PageLayout: ... from __future__ import annotations from abc import ABC, abstractmethod +from collections.abc import Callable from typing import TYPE_CHECKING, ClassVar, Literal -from pydantic import BaseModel, Field +from PIL import Image +from pydantic import BaseModel, ConfigDict, Field from ..errors import ConfigurationError from ..layout import DocumentLayout, PageLayout @@ -76,6 +78,8 @@ class BackendStatus(BaseModel): class OcrSettings(BaseModel): """Engine-independent settings each backend maps to its own options.""" + model_config = ConfigDict(arbitrary_types_allowed=True) + languages: list[str] = Field( default_factory=lambda: ["en"], description="ISO 639-1 codes, e.g. ['en', 'de']." ) @@ -86,6 +90,18 @@ class OcrSettings(BaseModel): default=True, description="Correct small scan angles before OCR; orthogonal rotation is controlled separately.", ) + crop_photos: bool = Field( + default=True, + description="Find the document in a photo and flatten it before OCR (needs the [photo] extra).", + ) + min_text_height: float = Field( + default=20.0, ge=0.0, + description="Tesseract re-reads a page enlarged when its typical word is shorter than this, in px; 0 disables.", + ) + preprocess: Callable[[Image.Image, int], Image.Image] | None = Field( + default=None, + description="Your own step on every page image (image, page number) -> image, after cropping, before OCR.", + ) word_confidence_floor: float = Field( default=0.60, ge=0.0, le=1.0, description="Words below this confidence neither count toward page confidence nor act as witnesses.", diff --git a/src/docket/ocr/preprocess.py b/src/docket/ocr/preprocess.py new file mode 100644 index 0000000..03b2119 --- /dev/null +++ b/src/docket/ocr/preprocess.py @@ -0,0 +1,123 @@ +"""Page image preparation before OCR: photo cropping and small-text upscaling. + +A phone photo of a receipt or invoice holds the document somewhere in a +frame of table, hand and shadow, at an angle. OCR reads the table grain as +characters and the tilted lines as a slope. `crop_document` finds the +paper's four corners and warps it flat. It needs OpenCV (the `[photo]` +extra); without it photos are read as they are. + +Tesseract reads text best when capital letters are 20–30 pixels tall; a +low-resolution scan gives it 10. `upscale_factor` says how far to enlarge a +page whose words came out that small (see TesseractBackend). + +Every step here is conservative: when in doubt it leaves the image alone, +because a wrong crop loses text and nothing downstream can bring it back. +""" +from __future__ import annotations + +from collections.abc import Callable +from statistics import median + +from PIL import Image + +# Page image, page number -> page image. Runs on every rendered PDF page and +# image page before any OCR engine sees it (Docling reads the file itself). +Preprocessor = Callable[[Image.Image, int], Image.Image] + +# Four corners (top-left, top-right, bottom-right, bottom-left) of the +# document in the original image, normalized to 0..1. +Quad = list[tuple[float, float]] + +# A crop must keep at least this share of the frame (smaller is more likely +# a label, a table or a card than the page), and leave out at least this +# much (otherwise the page already fills the frame: a scan). +_MIN_AREA, _MAX_AREA = 0.20, 0.92 +# The frame outside the page must be background, not more paper: the page +# must be clearly brighter than what surrounds it (median grey levels), and +# the surroundings must not be busy with edges. A table frame on a scanned +# page fails the first test — white paper on both sides of the line. +_MIN_PAPER_CONTRAST = 25 +_MAX_OUTSIDE_EDGES = 0.02 +# Work on a copy this wide; corner finding needs shape, not detail. +_DETECT_WIDTH = 800 + + +def crop_document(image: Image.Image) -> tuple[Image.Image, Quad | None]: + """The document flattened out of a photo, and where it was; or the image + unchanged and None when no page outline is clear enough to trust.""" + try: + import cv2 + import numpy as np + except ImportError: + return image, None + + rgb = np.asarray(image.convert("RGB")) + height, width = rgb.shape[:2] + scale = min(1.0, _DETECT_WIDTH / width) + small = cv2.resize(rgb, (round(width * scale), round(height * scale))) if scale < 1 else rgb + gray = cv2.GaussianBlur(cv2.cvtColor(small, cv2.COLOR_RGB2GRAY), (5, 5), 0) + edges = cv2.Canny(gray, 50, 150) + closed = cv2.dilate(edges, np.ones((3, 3), np.uint8), iterations=2) + contours, _ = cv2.findContours(closed, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE) + frame = float(small.shape[0] * small.shape[1]) + + quad = None + for contour in sorted(contours, key=cv2.contourArea, reverse=True)[:5]: + approx = cv2.approxPolyDP(contour, 0.02 * cv2.arcLength(contour, True), True) + if len(approx) == 4 and cv2.isContourConvex(approx): + share = cv2.contourArea(approx) / frame + if _MIN_AREA <= share <= _MAX_AREA: + quad = approx.reshape(4, 2).astype("float32") + break # only the largest outline can be the page + if quad is None: + return image, None + + mask = np.zeros(gray.shape, np.uint8) + cv2.fillConvexPoly(mask, quad.astype(np.int32), (255,)) + # Leave the outline's own edge band out of the "outside". + inside = cv2.dilate(mask, np.ones((9, 9), np.uint8)) + outside = inside == 0 + if not outside.any(): + return image, None + contrast = float(np.median(gray[mask > 0])) - float(np.median(gray[outside])) + if contrast < _MIN_PAPER_CONTRAST or (edges[outside] > 0).mean() > _MAX_OUTSIDE_EDGES: + return image, None + + corners = _ordered(quad / scale) + top = np.linalg.norm(corners[1] - corners[0]) + bottom = np.linalg.norm(corners[2] - corners[3]) + left = np.linalg.norm(corners[3] - corners[0]) + right = np.linalg.norm(corners[2] - corners[1]) + out_w, out_h = round(max(top, bottom)), round(max(left, right)) + target = np.array([[0, 0], [out_w - 1, 0], [out_w - 1, out_h - 1], [0, out_h - 1]], dtype="float32") + warp = cv2.getPerspectiveTransform(corners.astype("float32"), target) + flat = cv2.warpPerspective(rgb, warp, (out_w, out_h), flags=cv2.INTER_CUBIC, borderValue=(255, 255, 255)) + normalized = [(float(x) / width, float(y) / height) for x, y in corners] + return Image.fromarray(flat), normalized + + +def _ordered(points): + """Corners as top-left, top-right, bottom-right, bottom-left.""" + import numpy as np + + by_sum = points.sum(axis=1) + by_diff = np.diff(points, axis=1).ravel() + return np.array([ + points[by_sum.argmin()], points[by_diff.argmin()], points[by_sum.argmax()], points[by_diff.argmax()], + ]) + + +def upscale_factor(word_heights: list[float], *, min_height: float, target: float = 30.0, + max_factor: float = 3.0) -> float: + """How much to enlarge a page whose typical word is `word_heights` tall: + 1.0 when it is at least `min_height` px (or nothing was read), else + enough to reach `target`, at most `max_factor`.""" + if min_height <= 0 or not word_heights: + return 1.0 + typical = median(word_heights) + if typical <= 0 or typical >= min_height: + return 1.0 + return min(max_factor, target / typical) + + +__all__ = ["Preprocessor", "Quad", "crop_document", "upscale_factor"] diff --git a/src/docket/ocr/source.py b/src/docket/ocr/source.py index 2bc5ebb..4fc9e98 100644 --- a/src/docket/ocr/source.py +++ b/src/docket/ocr/source.py @@ -14,6 +14,7 @@ from PIL import Image from .. import pdf as pdf_render +from .preprocess import Preprocessor, Quad, crop_document PDF_SUFFIXES = {".pdf"} IMAGE_SUFFIXES = {".png", ".jpg", ".jpeg", ".tif", ".tiff", ".bmp", ".webp"} @@ -35,8 +36,12 @@ def __init__( dpi: int = 200, max_pages: int | None = None, max_pixels: int | None = None, + crop_photos: bool = False, + preprocess: Preprocessor | None = None, ): self.path = Path(path) + self.crop_photos = crop_photos + self.preprocess = preprocess suffix = self.path.suffix.lower() if suffix in PDF_SUFFIXES: self.kind: SourceKind = "pdf" @@ -122,6 +127,7 @@ def __init__(self, document: DocumentSource, number: int): self.document = document self.number = number self._image: Image.Image | None = None + self.crop_quad: Quad | None = None @property def kind(self) -> SourceKind: @@ -132,17 +138,24 @@ def plumber_page(self): return self.document.plumber.pages[self.number - 1] def image(self) -> Image.Image: + """The page as OCR sees it: rendered or decoded, cropped out of a + photo when that is on, then through the caller's preprocessor.""" if self._image is None: if self.kind == "pdf": - self._image = pdf_render.render_page( - self.document.path, self.number - 1, self.document.dpi - ) + image = pdf_render.render_page(self.document.path, self.number - 1, self.document.dpi) elif self.kind == "image": - with Image.open(self.document.path) as image: - image.seek(self.number - 1) - self._image = image.convert("RGB") + with Image.open(self.document.path) as source: + source.seek(self.number - 1) + image = source.convert("RGB") + # A rendered PDF page is already the page; only a picture + # can hold the document inside a larger frame. + if self.document.crop_photos: + image, self.crop_quad = crop_document(image) else: raise UnsupportedDocument("a plain-text source has no page image") + if self.document.preprocess is not None: + image = self.document.preprocess(image, self.number) + self._image = image return self._image def image_png(self) -> bytes: diff --git a/src/docket/ocr/tesseract.py b/src/docket/ocr/tesseract.py index 1a2da83..db0f9e0 100644 --- a/src/docket/ocr/tesseract.py +++ b/src/docket/ocr/tesseract.py @@ -21,9 +21,25 @@ from .base import BackendStatus, Capabilities, OcrBackend, OcrError from .deskew import deskew_image from .languages import TESSERACT, tesseract_codes +from .preprocess import upscale_factor from .source import PageSource _ROTATE = re.compile(r"Rotate:\s*(\d+)") +# An enlarged page is never allowed past this many pixels. +_MAX_UPSCALED_PIXELS = 40_000_000 + + +def word_heights(data: dict) -> list[float]: + """Box heights of the words Tesseract actually recognized.""" + heights = [] + for i, raw in enumerate(data.get("text", [])): + try: + conf = float(data["conf"][i]) + except (TypeError, ValueError): + continue + if (raw or "").strip() and conf >= 0: + heights.append(float(data["height"][i])) + return heights def page_confidence(data: dict, floor: float) -> float: @@ -135,15 +151,20 @@ def recognize_page(self, page: PageSource) -> PageLayout: # PIL rotates counter-clockwise; OSD reports a clockwise fix. image = image.rotate(-rotation, expand=True) image, deskew_angle = deskew_image(image) if self.settings.deskew else (image, 0.0) - try: - data = pytesseract.image_to_data( - image, - lang="+".join(tesseract_codes(self.settings.languages)), - config=f"--psm {self.settings.tesseract_psm}", - output_type=pytesseract.Output.DICT, - ) - except pytesseract.TesseractError as exc: - raise OcrError(f"tesseract failed on page {page.number}: {exc}") from exc + floor = self.settings.word_confidence_floor + data = self._read(image, page.number) + confidence = page_confidence(data, floor) + # Small text (a low-resolution scan, a receipt shot from afar) is + # read again enlarged; the enlarged reading is kept only if Tesseract + # is at least as sure of it, so this can't make a page worse. + factor = upscale_factor(word_heights(data), min_height=self.settings.min_text_height) + factor = min(factor, (_MAX_UPSCALED_PIXELS / (image.width * image.height)) ** 0.5) + if factor > 1.05: + enlarged = image.resize((round(image.width * factor), round(image.height * factor)), Image.Resampling.LANCZOS) + enlarged_data = self._read(enlarged, page.number) + enlarged_confidence = page_confidence(enlarged_data, floor) + if enlarged_confidence >= confidence: + image, data, confidence = enlarged, enlarged_data, enlarged_confidence return build_page( page_number=page.number, width=float(image.width), @@ -152,9 +173,20 @@ def recognize_page(self, page: PageSource) -> PageLayout: rotation=rotation, deskew_angle=deskew_angle, backend=self.name, - confidence=page_confidence(data, self.settings.word_confidence_floor), + confidence=confidence, words=words_from_data(data), ) + def _read(self, image: Image.Image, page_number: int) -> dict: + try: + return pytesseract.image_to_data( + image, + lang="+".join(tesseract_codes(self.settings.languages)), + config=f"--psm {self.settings.tesseract_psm}", + output_type=pytesseract.Output.DICT, + ) + except pytesseract.TesseractError as exc: + raise OcrError(f"tesseract failed on page {page_number}: {exc}") from exc + __all__ = ["TesseractBackend", "page_confidence", "words_from_data"] diff --git a/src/docket/options.py b/src/docket/options.py index e93f767..7b7a73d 100644 --- a/src/docket/options.py +++ b/src/docket/options.py @@ -32,6 +32,7 @@ parse_languages, resolve_chain, ) +from .ocr.preprocess import Preprocessor BackendSpec = str | OcrBackend @@ -51,6 +52,16 @@ class OcrOptions(BaseModel): min_confidence: float | None = Field(default=None, ge=0.0, le=1.0) detect_rotation: bool | None = None deskew: bool | None = None + crop_photos: bool | None = Field( + default=None, description="Crop and flatten the document in a photo. Env: DOCKET_OCR_CROP_PHOTOS." + ) + min_text_height: float | None = Field( + default=None, ge=0, description="Re-read small text enlarged below this word height (px). Env: DOCKET_OCR_MIN_TEXT_HEIGHT." + ) + preprocess: Preprocessor | None = Field( + default=None, + description="Your own step on every page image, (image, page number) -> image, after cropping and before OCR.", + ) use_pdf_text: bool = Field(default=True, description="Take a usable PDF text layer without OCR.") dpi: int | None = Field(default=None, ge=50, le=600) device: str | None = Field(default=None, description="PaddleOCR device. Env: DOCKET_PADDLE_DEVICE.") @@ -149,6 +160,9 @@ def resolve(options: ProcessOptions | None = None) -> ResolvedOptions: dpi=_pick(ocr.dpi, config.OCR_DPI), detect_rotation=_pick(ocr.detect_rotation, config.OCR_DETECT_ROTATION), deskew=_pick(ocr.deskew, config.OCR_DESKEW), + crop_photos=_pick(ocr.crop_photos, config.OCR_CROP_PHOTOS), + min_text_height=_pick(ocr.min_text_height, config.OCR_MIN_TEXT_HEIGHT), + preprocess=ocr.preprocess, tesseract_psm=config.TESSERACT_PSM, paddle_model=_pick(ocr.paddle_model, config.PADDLE_MODEL), paddle_tables=_pick(ocr.paddle_tables, config.PADDLE_TABLES), diff --git a/tests/snapshots/public_api.json b/tests/snapshots/public_api.json index 8a2e5e3..be8694d 100644 --- a/tests/snapshots/public_api.json +++ b/tests/snapshots/public_api.json @@ -41,6 +41,9 @@ }, "OcrOptions": { "backend": {"annotation": "str | docket.ocr.base.OcrBackend | None", "required": false}, + "crop_photos": {"annotation": "bool | None", "required": false}, + "min_text_height": {"annotation": "float | None", "required": false}, + "preprocess": {"annotation": "collections.abc.Callable[[PIL.Image.Image, int], PIL.Image.Image] | None", "required": false}, "deskew": {"annotation": "bool | None", "required": false}, "detect_rotation": {"annotation": "bool | None", "required": false}, "device": {"annotation": "str | None", "required": false}, diff --git a/tests/test_preprocess.py b/tests/test_preprocess.py new file mode 100644 index 0000000..eb8d0c4 --- /dev/null +++ b/tests/test_preprocess.py @@ -0,0 +1,215 @@ +"""Page image preparation: photo cropping, small-text re-reading, and the +caller's own preprocessing hook — on synthetic images, no LLM.""" +from __future__ import annotations + +import shutil +from typing import ClassVar + +import pytest +from PIL import Image, ImageDraw, ImageFont + +from docket.layout import PageLayout, build_page +from docket.ocr import AcquisitionOptions, OcrSettings, acquire +from docket.ocr.base import BackendStatus, Capabilities, OcrBackend +from docket.ocr.preprocess import crop_document, upscale_factor +from docket.ocr.source import DocumentSource + +needs_tesseract = pytest.mark.skipif(shutil.which("tesseract") is None, reason="tesseract not installed") + + +def _text_page(size, lines, font_size, origin=(40, 60), step=60) -> Image.Image: + page = Image.new("RGB", size, "white") + draw = ImageDraw.Draw(page) + font = ImageFont.load_default(size=font_size) + for i, line in enumerate(lines): + draw.text((origin[0], origin[1] + i * step), line, fill="black", font=font) + return page + + +def _photo() -> tuple[Image.Image, list[tuple[float, float]]]: + """An invoice page photographed at an angle on a dark, grainy table.""" + cv2 = pytest.importorskip("cv2") + import numpy as np + + page = np.asarray(_text_page((600, 800), ["INVOICE 2026-17", "Acme GmbH", "Total 1815.00 EUR"], 28)) + corners = np.float32([[180, 140], [760, 200], [700, 1050], [120, 980]]) + warp = cv2.getPerspectiveTransform(np.float32([[0, 0], [600, 0], [600, 800], [0, 800]]), corners) + table = np.random.default_rng(0).integers(40, 90, (1200, 900, 3)).astype("uint8") + placed = cv2.warpPerspective(page, warp, (900, 1200)) + footprint = cv2.warpPerspective(np.full((800, 600), 255, np.uint8), warp, (900, 1200)) + table[footprint > 0] = placed[footprint > 0] + return Image.fromarray(table), [(x / 900, y / 1200) for x, y in corners] + + +# ---- cropping --------------------------------------------------------------------------------- + + +def test_a_photographed_page_is_found_and_flattened(): + photo, corners = _photo() + + flat, quad = crop_document(photo) + + assert quad is not None + assert all(abs(qx - cx) < 0.02 and abs(qy - cy) < 0.02 for (qx, qy), (cx, cy) in zip(quad, corners)) + # Flattened back to (roughly) the page's own 3:4 proportions. + assert abs(flat.width / flat.height - 600 / 800) < 0.08 + + +def test_a_scan_is_left_alone(): + pytest.importorskip("cv2") + scan = _text_page((850, 1100), ["Line of invoice text 12.00"] * 20, 22, origin=(60, 60), step=45) + assert crop_document(scan) == (scan, None) + + +def test_a_table_frame_on_a_scan_is_not_mistaken_for_the_page(): + """Cropping to the frame would cut away the header and footer text.""" + pytest.importorskip("cv2") + scan = _text_page((850, 1100), ["Line of invoice text 12.00"] * 20, 22, origin=(60, 60), step=45) + ImageDraw.Draw(scan).rectangle((100, 300, 750, 800), outline="black", width=3) + assert crop_document(scan)[1] is None + + +class _ImageReader(OcrBackend): + """Records the image OCR is handed; reads one fixed word.""" + + name = "image-reader" + capabilities = Capabilities(confidence=True, word_coordinates=True, lines=True, tables=False, rotation=False) + seen: ClassVar[list[Image.Image]] = [] + + def availability(self) -> BackendStatus: + return BackendStatus(name=self.name, available=True) + + def recognize_page(self, page) -> PageLayout: + image = page.image() + type(self).seen.append(image) + from docket.layout import RawWord + + word = RawWord(text="INVOICE", x0=10, y0=10, x1=90, y1=30, confidence=0.95) + return build_page(page_number=page.number, width=image.width, height=image.height, unit="px", + backend=self.name, confidence=0.95, words=[word]) + + +def test_acquisition_records_where_the_page_was_in_the_photo(tmp_path): + photo, _ = _photo() + path = tmp_path / "receipt.jpg" + photo.save(path) + _ImageReader.seen = [] + + reading = acquire(path, AcquisitionOptions(backend=_ImageReader(), fallbacks=[], + settings=OcrSettings(crop_photos=True))) + + page = reading.layout.pages[0] + assert page.crop_quad is not None and len(page.crop_quad) == 4 + assert _ImageReader.seen[0].size != photo.size # OCR saw the flattened page + + +def test_cropping_can_be_turned_off(tmp_path): + photo, _ = _photo() + path = tmp_path / "receipt.jpg" + photo.save(path) + _ImageReader.seen = [] + + reading = acquire(path, AcquisitionOptions(backend=_ImageReader(), fallbacks=[], + settings=OcrSettings(crop_photos=False))) + + assert reading.layout.pages[0].crop_quad is None + assert _ImageReader.seen[0].size == photo.size + + +# ---- the caller's hook ------------------------------------------------------------------------ + + +def test_the_preprocess_hook_sees_every_page_and_its_result_is_what_ocr_reads(tmp_path): + path = tmp_path / "page.png" + _text_page((300, 200), ["INVOICE"], 20).save(path) + calls = [] + + def grayscale(image: Image.Image, page_number: int) -> Image.Image: + calls.append(page_number) + return image.convert("L").convert("RGB").resize((150, 100)) + + _ImageReader.seen = [] + acquire(path, AcquisitionOptions(backend=_ImageReader(), fallbacks=[], + settings=OcrSettings(preprocess=grayscale))) + + assert calls == [1] + assert _ImageReader.seen[0].size == (150, 100) + + +def test_the_hook_reaches_acquisition_from_process_options(): + from docket import OcrOptions, ProcessOptions + from docket.options import resolve + + def hook(image, page_number): + return image + + settings = resolve(ProcessOptions(ocr=OcrOptions(preprocess=hook, crop_photos=False, min_text_height=0))).acquisition.settings + assert (settings.preprocess, settings.crop_photos, settings.min_text_height) == (hook, False, 0) + + +def test_the_hook_runs_once_per_page_however_often_it_is_read(tmp_path): + path = tmp_path / "page.png" + _text_page((300, 200), ["INVOICE"], 20).save(path) + calls = [] + with DocumentSource(path, preprocess=lambda image, n: calls.append(n) or image) as source: + page = source.page(1) + page.image() + page.image() + assert calls == [1] + + +# ---- small text ------------------------------------------------------------------------------- + + +@pytest.mark.parametrize("heights, min_height, expected", [ + ([10, 11, 9], 20, 3.0), # capped + ([15, 15, 16], 20, 2.0), # to the 30 px target + ([24, 26], 20, 1.0), # already legible + ([], 20, 1.0), # nothing read: nothing to measure + ([10, 10], 0, 1.0), # disabled +]) +def test_upscale_factor(heights, min_height, expected): + assert upscale_factor(heights, min_height=min_height) == pytest.approx(expected) + + +SMALL_INVOICE = ["INVOICE INV-2026-0417", "Seller Acme Handels GmbH", "Date 15/03/2026", + "Subtotal 1500.00", "VAT 315.00", "Total 1815.00 EUR"] + + +@needs_tesseract +def test_small_text_is_reread_enlarged_and_reads_right(tmp_path): + """At 11 px Tesseract misreads 15/03/2026 as 16/03/2028 and 315.00 as + 316.00 — digits changed, nothing flagged. Enlarged, it reads them right.""" + from docket.ocr.tesseract import TesseractBackend + + path = tmp_path / "small.png" + _text_page((420, 260), SMALL_INVOICE, 11, origin=(12, 12), step=18).save(path) + + def read(min_height): + settings = OcrSettings(min_text_height=min_height, deskew=False, detect_rotation=False) + with DocumentSource(path) as source: + return TesseractBackend(settings).recognize_page(source.page(1)) + + as_is, enlarged = read(0), read(20) + + assert as_is.width == 420 and enlarged.width > 420 + assert enlarged.confidence >= as_is.confidence + for value in ("15/03/2026", "315.00", "1815.00"): + assert value in enlarged.text + + +@needs_tesseract +def test_legible_text_is_read_once(tmp_path, monkeypatch): + from docket.ocr import tesseract as tesseract_module + + path = tmp_path / "big.png" + _text_page((900, 500), SMALL_INVOICE[:3], 32).save(path) + reads = [] + original = tesseract_module.TesseractBackend._read + monkeypatch.setattr(tesseract_module.TesseractBackend, "_read", + lambda self, image, n: reads.append(image.size) or original(self, image, n)) + + with DocumentSource(path) as source: + tesseract_module.TesseractBackend(OcrSettings(deskew=False, detect_rotation=False)).recognize_page(source.page(1)) + + assert reads == [(900, 500)]