Adopt canonical output contract; drop frontmatter + fix keying + dead --task

.github/workflows/ci.yml

@@ -0,0 +1,47 @@
+name: CI
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["3.11"]
+    steps:
+      - uses: actions/checkout@v4
+      - uses: astral-sh/setup-uv@v3
+        with:
+          enable-cache: true
+          cache-dependency-glob: "**/pyproject.toml"
+      - name: Set up Python ${{ matrix.python-version }}
+        run: uv python install ${{ matrix.python-version }}
+      - name: Install dependencies
+        run: uv sync --extra dev
+      - name: Lint (advisory)
+        run: uv run ruff check .
+        continue-on-error: true
+      - name: Format check
+        run: uv run ruff format --check .
+      - name: Run tests
+        run: uv run pytest -m "not integration" -q
+
+  typecheck:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: astral-sh/setup-uv@v3
+        with:
+          enable-cache: true
+          cache-dependency-glob: "**/pyproject.toml"
+      - name: Set up Python
+        run: uv python install 3.11
+      - name: Install dependencies
+        run: uv sync --extra dev
+      - name: Type check (advisory)
+        run: uv run mypy deepseek_ocr/ --ignore-missing-imports
+        continue-on-error: true

README.md

@@ -10,12 +10,12 @@ Command-line tool for OCR using DeepSeek vision models. Supports Ollama (local)
 
 - **Multi-backend**: Ollama (local, free) and vLLM (OpenAI-compatible API)
 - Supports PDFs and images (JPG, PNG, WEBP, GIF, BMP, TIFF)
-- Per-document output folders with figures
-- Batch processing with incremental resume (skips already-processed files)
+- Canonical output via the shared [`ocr-output-contract`](https://github.com/r-uben/ocr-output-contract): one `<root>/<rel/dir>/<stem>/<stem>.md` per document under `## Page N` headers, dual `metadata.json` (per-doc sidecar + root index), input-relative keying (no basename collisions)
+- Batch processing of directory trees with incremental resume (skips already-completed documents; re-runs when the input, model, backend, task, or prompt changes)
+- Truncation detection: a length-truncated page is recorded `status=partial/failed`, never a silent `completed`
 - Retry with exponential backoff for transient failures
-- Parallel page processing for faster PDF OCR
-- `--dry-run` to preview files before processing
-- Clean markdown output with HTML tables converted to markdown
+- `--dry-run` to preview the exact documents that will be processed
+- Clean markdown output with HTML tables converted to markdown (`--raw` keeps the model's verbatim text)
 
 ## Choosing an OCR tool
 
@@ -67,10 +67,10 @@ deepseek-ocr document.jpg
 # Process a PDF
 deepseek-ocr paper.pdf
 
-# Process all files in a directory
-deepseek-ocr ./documents/ --recursive
+# Process a directory tree (always walked recursively)
+deepseek-ocr ./documents/
 
-# Preview files without processing
+# Preview the documents that would be processed
 deepseek-ocr ./documents/ --dry-run
 
 # Custom output directory
@@ -79,8 +79,11 @@ deepseek-ocr doc.pdf -o ./results/
 # Use vLLM backend
 deepseek-ocr paper.pdf --backend vllm --vllm-url http://gpu-server:8000/v1
 
-# Parallel processing for faster PDF OCR
-deepseek-ocr large-document.pdf -w 2
+# Raise the per-page token budget if dense pages truncate
+deepseek-ocr large-document.pdf --max-tokens 16384
+
+# Keep the model's verbatim output (skip the cleaner)
+deepseek-ocr paper.pdf --raw
 
 # Extract and analyze embedded figures
 deepseek-ocr paper.pdf --analyze-figures
@@ -95,23 +98,24 @@ deepseek-ocr paper.pdf -q
 deepseek-ocr [OPTIONS] INPUT_PATH
 
 Options:
-  -o, --output-dir PATH           Output directory for results
-  -r, --recursive                 Recursively process directories
+  -o, --output-dir PATH           Output root (default: <input-parent>/ocr/)
+  -r, --recursive                 Accepted for compatibility; batch trees are
+                                  ALWAYS walked recursively
   --model TEXT                    Model name (default: deepseek-ocr)
-  --prompt TEXT                   Custom prompt for OCR
+  --prompt TEXT                   Custom prompt for OCR (overrides --task)
   --task [convert|ocr|layout|extract|parse]
                                   OCR task type
-  --extract-images                Extract and save page images from PDFs
-  --no-metadata                   Exclude metadata from output
   --dpi INTEGER                   PDF rendering DPI (default: 200)
-  -w, --workers INTEGER           Parallel workers for PDF pages (default: 1)
   --analyze-figures               Extract and analyze embedded figures with AI
+  --raw                           Keep verbatim model output (skip the cleaner)
+  --max-tokens INTEGER            Max tokens per page (default: 8192). Raise if
+                                  dense pages truncate
   --max-dim INTEGER               Max image dimension (default: 1920, 0 to disable)
   --backend [ollama|vllm]         Backend to use (default: ollama)
   --vllm-url TEXT                 vLLM API URL (default: http://localhost:8000/v1)
-  --reprocess                     Force reprocessing of already-done files
-  --dry-run                       Preview files without processing
-  -q, --quiet                     Suppress output, print paths only
+  --reprocess                     Force reprocessing of already-done documents
+  --dry-run                       Preview documents without processing
+  -q, --quiet                     Suppress output, print one .md path per line
   --verbose                       Enable verbose output
   --help                          Show this message and exit.
 ```
@@ -138,36 +142,44 @@ deepseek-ocr info
 
 ## Output Format
 
-Each document gets its own folder:
+Output follows the shared `ocr-output-contract`. The default output root is
+`<input-parent>/ocr/` for a single file and `<input>/ocr/` for a directory
+(override with `-o`). Each document gets its own folder, mirroring the input
+subtree so same-named files in different directories never collide:
 
 ```
-output/
+ocr/
+├── metadata.json           # root index, keyed by input-relative path
 └── document/
-    ├── document.md          # OCR markdown
-    └── figures/             # Extracted figures (if --analyze-figures)
-        └── page1_fig1.png
+    ├── document.md         # OCR markdown
+    ├── metadata.json       # per-document sidecar (provenance)
+    └── figures/            # extracted figures (if --analyze-figures)
+        └── figure_1_page1.png
 ```
 
-The markdown includes metadata:
+The markdown body carries **no YAML frontmatter** — all provenance lives in the
+JSON sidecars. Pages are separated by `## Page N` headers:
 
 ```markdown
----
-source: /path/to/document.pdf
-processed: 2025-12-01T15:30:00
-pages: 3
-processing_time: 18.45s
-model: deepseek-ocr
-backend: ollama
----
-
 ## Page 1
 
+[Extracted content...]
+
+## Page 2
+
 [Extracted content...]
 ```
 
+The per-document `metadata.json` records the ratified schema (`status`,
+`checksum`, `model`, `backend`, `processing_time`, `timestamp` (UTC),
+`output_path`, `pages`, plus a run `fingerprint`).
+
 ### Batch Resume
 
-Batch processing saves `metadata.json` in the output directory. On re-run, already-processed files are skipped automatically. Use `--reprocess` to force reprocessing.
+The root `metadata.json` records every processed document. On re-run, a document
+is skipped only when the input is unchanged, its `.md` still exists on disk, and
+the run configuration (model, backend, task, prompt) is unchanged. Use
+`--reprocess` to force reprocessing.
 
 ## Configuration
 
@@ -176,33 +188,30 @@ Create a `.env` file or set environment variables with `DEEPSEEK_OCR_` prefix:
 ```bash
 DEEPSEEK_OCR_BACKEND=ollama
 DEEPSEEK_OCR_MODEL_NAME=deepseek-ocr
-DEEPSEEK_OCR_OUTPUT_DIR=output
 DEEPSEEK_OCR_OLLAMA_URL=http://localhost:11434
 DEEPSEEK_OCR_VLLM_BASE_URL=http://localhost:8000/v1
 DEEPSEEK_OCR_MAX_DIMENSION=1920
+DEEPSEEK_OCR_MAX_TOKENS=8192
 DEEPSEEK_OCR_MAX_RETRIES=3
 DEEPSEEK_OCR_RETRY_DELAY=1.0
+DEEPSEEK_OCR_LOG_LEVEL=INFO
 ```
 
 ## Programmatic Usage
 
 ```python
 from pathlib import Path
-from deepseek_ocr import create_backend, OCRProcessor
+from deepseek_ocr import create_backend, process
 
 backend = create_backend(backend_type="ollama", model_name="deepseek-ocr")
 backend.load_model()
 
-processor = OCRProcessor(
-    backend=backend,
-    output_dir=Path("./results"),
-    workers=2,
-)
-
-result = processor.process_file(Path("document.pdf"))
-print(result.output_text)
+# process() routes all output through the contract and returns a RunOutcome.
+outcome = process(Path("document.pdf"), backend, output_dir=Path("./results"))
+for md_path in outcome.outputs:
+    print("wrote", md_path)
+print("exit code:", outcome.exit_code)  # nonzero if any document/page failed
 
-processor.save_result(result)
 backend.unload_model()
 ```
 

deepseek_ocr/__init__.py

@@ -5,10 +5,11 @@
 __license__ = "MIT"
 
 from deepseek_ocr.backends import Backend, OllamaBackend, VLLMBackend, create_backend
-from deepseek_ocr.processor import OCRProcessor
+from deepseek_ocr.processor import discover_documents, process
 
 __all__ = [
-    "OCRProcessor",
+    "process",
+    "discover_documents",
     "Backend",
     "OllamaBackend",
     "VLLMBackend",

deepseek_ocr/__main__.py

@@ -2,7 +2,5 @@
 
 from .cli import main
 
-
 if __name__ == "__main__":
     main()
-

deepseek_ocr/backends/__init__.py

@@ -13,6 +13,7 @@ def create_backend(
     max_dimension: int | None = None,
     max_retries: int | None = None,
     retry_delay: float | None = None,
+    max_tokens: int | None = None,
     **kwargs,
 ) -> Backend:
     """Factory function to create the appropriate backend.
@@ -23,6 +24,7 @@ def create_backend(
         max_dimension: Maximum image dimension for resizing
         max_retries: Maximum number of retries for transient errors
         retry_delay: Base delay in seconds between retries
+        max_tokens: Maximum tokens per page response (truncation control)
         **kwargs: Additional backend-specific arguments
 
     Returns:
@@ -36,6 +38,7 @@ def create_backend(
             max_dimension=max_dimension,
             max_retries=max_retries,
             retry_delay=retry_delay,
+            max_tokens=max_tokens,
             ollama_url=kwargs.get("ollama_url"),
         )
     elif backend_type == "vllm":
@@ -44,6 +47,7 @@ def create_backend(
             max_dimension=max_dimension,
             max_retries=max_retries,
             retry_delay=retry_delay,
+            max_tokens=max_tokens,
             base_url=kwargs.get("vllm_base_url"),
         )
     else:

deepseek_ocr/backends/base.py

@@ -5,7 +5,6 @@
 import time
 from abc import ABC, abstractmethod
 from pathlib import Path
-from typing import Union
 
 from PIL import Image
 
@@ -68,7 +67,7 @@ def unload_model(self) -> None:
     @abstractmethod
     def process_image(
         self,
-        image: Union[Image.Image, Path, str],
+        image: Image.Image | Path | str,
         prompt: str | None = None,
         task: str = "convert",
         return_raw: bool = False,
@@ -86,6 +85,28 @@ def process_image(
         """
         ...
 
+    def process_image_with_meta(
+        self,
+        image: Image.Image | Path | str,
+        prompt: str | None = None,
+        task: str = "convert",
+        return_raw: bool = False,
+    ) -> tuple[str, object | None]:
+        """Process an image and return ``(text, finish_reason)``.
+
+        The ``finish_reason`` is the model's completion stop reason (e.g.
+        ``"length"`` / ``"stop"`` / ``None``). The processor feeds it to the
+        contract's :func:`ocr_output_contract.is_truncated` so a length-truncated
+        non-empty page is recorded as a per-page failure (status=partial/failed)
+        rather than a silent ``completed`` (content loss).
+
+        The default delegates to :meth:`process_image` and reports no finish
+        reason, so a backend that does not surface one (or a mock) keeps working
+        with truncation detection simply disabled. Backends that expose a stop
+        reason override this to return it.
+        """
+        return self.process_image(image, prompt=prompt, task=task, return_raw=return_raw), None
+
     def _retry(self, func, *args, **kwargs):
         """Execute func with exponential backoff retry on TransientError.
 
@@ -105,7 +126,7 @@ def _retry(self, func, *args, **kwargs):
             except TransientError as e:
                 last_error = e
                 if attempt < self.max_retries:
-                    delay = self.retry_delay * (2 ** attempt) + random.uniform(0, 0.5)
+                    delay = self.retry_delay * (2**attempt) + random.uniform(0, 0.5)
                     logger.warning(
                         f"[{self.backend_name}] Transient error (attempt {attempt + 1}/"
                         f"{self.max_retries + 1}): {e}. Retrying in {delay:.1f}s..."
@@ -115,9 +136,7 @@ def _retry(self, func, *args, **kwargs):
                     logger.error(
                         f"[{self.backend_name}] Max retries ({self.max_retries}) exhausted: {e}"
                     )
-        raise RuntimeError(
-            f"Max retries ({self.max_retries}) exhausted: {last_error}"
-        )
+        raise RuntimeError(f"Max retries ({self.max_retries}) exhausted: {last_error}")
 
     def process_images_batch(
         self,
@@ -135,7 +154,7 @@ def process_images_batch(
             results.append(result)
         return results
 
-    def describe_figure(self, image: Union[Image.Image, Path, str]) -> str:
+    def describe_figure(self, image: Image.Image | Path | str) -> str:
         """Generate a description of a figure/chart/diagram."""
         return self.process_image(
             image,