diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index f74e30f74..e1c5f5c50 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -16,6 +16,8 @@ jobs: python-version: ['3.10', '3.11', '3.12', '3.13', '3.14', '3.15'] steps: - uses: actions/checkout@v7 + - name: Check performance documentation budgets and links + run: python3 scripts/tools/update_performance_docs.py --check - name: Install system fonts # The renderer substitutes a SYSTEM face for every font a PDF names but # does not embed, and the ubuntu-24.04 image ships essentially none diff --git a/.gitignore b/.gitignore index 3374a6b7c..21c1e7c5a 100644 --- a/.gitignore +++ b/.gitignore @@ -29,8 +29,10 @@ scratch* docs/performance_benchmarks/*.csv # Published benchmark figures and compact provenance. -!docs/performance_benchmarks/2026-11-bo767-m3-max/*.png -!docs/performance_benchmarks/2026-11-bo767-m3-max/provenance.json +!docs/performance_benchmarks/*/*/assets/*.png +!docs/performance_benchmarks/*/*/provenance.json +!docs/performance_benchmarks/*/*/publication.json +!scripts/tools/update_performance_docs_artifacts/performance_docs_limits.json # Created by https://www.toptal.com/developers/gitignore/api/python,macos,emacs,cmake,virtualenv # Edit at https://www.toptal.com/developers/gitignore?templates=python,macos,emacs,cmake,virtualenv diff --git a/.plans/active/reduce-memory-footprint.md b/.plans/active/reduce-memory-footprint.md new file mode 100644 index 000000000..3b2a12381 --- /dev/null +++ b/.plans/active/reduce-memory-footprint.md @@ -0,0 +1,4778 @@ +# Reduce the native parser's memory footprint + +Date: 2026-11-11. +Status: COMPLETE. The user reports all final tests passing after comparison-code cleanup; the saved final smoke passed 80 measurements and all six intended PNG-equivalence checks. The controlled Type3 comparison, post-merge workload validation and comparison/dead-code retirement are complete. The optional benchmark memory-reporting feature is subsequent work. +Shared immutable clip history removes the measured quadratic retained geometry. The renderer cursor resolves the chain's render-only slowdown; at 1,000 clips, rendering is 33.85% faster than the shared-history implementation without the cursor. Post-merge workload scaling/replay memory validation and final rebuilt-source verification passed. +The legacy Type3 cleanup is now quantified with a current-library reference: 54 measurements per side, matching logical output and six identical PNG pairs. At 250,000 lines, Type3 decode/render peaks fall about 72%, and shapes-disabled peaks about 81%; every Type3 case is faster. +The earlier Type3 comparison rejected on changed dependencies is superseded by this controlled pair; it is retained below as investigation history. +MyPy/Ruff pass; step 4b appearance/widget streaming validated by native experiments and 171 focused tests. +Step 4a Form streaming is validated by native experiments and 153 focused tests. +Step 2 native recheck and carried-forward semantic coverage passed. +Step 3 native smoke/scaling, Form timing recheck and focused semantic coverage passed. +Step 1 core scaling is reviewed. The user completed final build/native/unit/corpus verification and the 80-case cleanup smoke. The three-repeat bo20 bundle validates the merged implementation before mechanical cleanup; earlier Nov-11 observations remain historical context. +Initial C++ reproduction and structural investigation are complete. + +Related issues: + +- [docling-parse#448](https://github.com/docling-project/docling-parse/issues/448): + quadratic dash-array growth, fixed by + [#452](https://github.com/docling-project/docling-parse/pull/452). +- [docling-parse#345](https://github.com/docling-project/docling-parse/issues/345): + high native memory consumption on vector-dense pages. +- [docling#4058](https://github.com/docling-project/docling/issues/4058): + original dense-grid reproduction and downstream operational impact. + +## Working agreement: Codex implements; the user compiles and experiments + +The user explicitly wants **Codex to write the code, the user to compile and run +the experiments, and both to strategize together from the intermediate results**. + +The user reaffirmed "Keep builds and experiments with me" during the six-item +completion pass. The earlier delegated 174-test run does not change that rule. + +This is an iterative collaboration, not authorization to implement all five +steps in one pass. + +For each implementation cycle: + +1. Codex reads the current code and the most recent results. +2. Codex states the hypothesis, proposes one bounded change, identifies affected + files, and shows the proposed diff whenever possible. +3. The user approves that change before Codex modifies the code. Approval already + given for a specific change does not need to be requested again. +4. Codex implements the approved change and reviews the diff and ownership, + state, error-handling, and timing implications. +5. Codex supplies exact build, test, and experiment commands, identifying which + commands already work and which require the new probe features. +6. **The user compiles, runs the correctness checks and experiments, and returns + the build output and measurement artifacts. Codex does not launch builds, + tests, or benchmark runs unless the user explicitly delegates them.** +7. Codex analyzes the returned data, explains what it establishes and what remains + uncertain, and records the result in this plan. +8. Together, choose to keep, revise, or revert that individual change and agree + on the next implementation cycle. + +A compilation failure, an output difference, or an unexpected memory curve is a +result to investigate together. Do not automatically advance to the next step +just because code has been written. Do not update regression ground truth to +make a behavior difference disappear. + +Writing this plan does not start step 1 or authorize production-code changes. + +## Objective and scope + +Remove the unnecessary retention and copying of complete content-stream token +lists in docling-parse while preserving extracted page content, graphics-state +semantics, damaged-document recovery, and rendered pixels. + +The first investigation and measurements remain in native C++ to isolate this +work from Python model construction and bindings. The existing Python-facing +tests still provide important correctness coverage once implementation begins. + +The primary scope is: + +- Audit reusable state for unintended accumulation and missing reset/replacement operations, separately from the streaming refactor. +- Distinguish QPDF's own parsing cost from our decision to collect every token. +- Interpret instructions as QPDF delivers them instead of waiting for a complete + vector of parsed objects. +- Remove unnecessary token-vector copies in nested content interpretation. +- Keep intentional caches useful without copying their contents on every use. +- Demonstrate improvements on synthetic fixtures and real documents. +- Verify that the gains survive the actual threaded parsing path. + +Separate follow-up decisions, informed by the measurements: + +- A per-page segment, decoded-byte, or memory budget with a degraded-page result. +- Scheduling heavy pages to limit concurrent working memory. +- Reducing the separate page-shape and render-instruction representations. +- QPDF changes, if QPDF-only measurements reveal a remaining problem at that layer. + +These follow-ups must not be folded into the initial streaming change. The +existing dash-array fix must remain intact. + +## Terminology + +| Term | Meaning in this plan | +| --- | --- | +| QPDF parsing with a discard callback | QPDF decodes the content bytes and delivers parsed objects; the callback counts and releases them without accumulating a token list. This has not yet been measured. | +| Collecting tokens | Our callback retains each QPDF object handle and its type/text representation in a vector of qpdf_stream_instruction records. This is what the initial probe called "tokens" or "tokenization only." | +| Interpretation | Execute PDF operators, maintaining graphics/text state and constructing page elements and render instructions. | +| Full native parsing / decode | Content decoding and object parsing, interpretation, and the usual native page processing; no Python model materialization. | +| Rendering | Execute render instructions to produce a raster canvas, at the specified scale. | +| Peak RSS | A process-lifetime resident-memory high-water mark, including temporary allocations. | +| Current RSS | Resident memory at a particular observation point; it may include allocations retained by the allocator after objects have been freed. | +| Retained payload | Bytes or objects we can count in live structures. This is not interchangeable with RSS. | + +QPDF already delivers parsed objects individually through ParserCallbacks. +In the version inspected, it first buffers the decompressed content-stream bytes. +Consuming callbacks immediately therefore removes our accumulated object list; +it does not make the whole parser constant-memory. + +## Starting evidence + +### Code and experiment identity + +The initial investigation used HEAD +511c57a87e4cc72a25d6ac613a095839288ba34e, macOS arm64, a Release build with +-O3, and QPDF v12.4.2. Future cycles must record their own identities; these +numbers are not universal memory guarantees. + +The native probe is +[tests/cpp/test_memory_on_lines.cpp](../../tests/cpp/test_memory_on_lines.cpp). +Its optional CMake target is test_memory_on_lines, enabled with +DOCLING_PARSE_BUILD_MEMORY_PROBE=ON. It belongs under tests/cpp, not app. + +Initial artifacts, local to the investigation machine: + +- [Detailed report](/private/tmp/docling-line-memory/report.txt). +- [Summary CSV](/private/tmp/docling-line-memory/summary.csv). +- [Raw measurements](/private/tmp/docling-line-memory/measurements.jsonl). +- [Build and dependency identity](/private/tmp/docling-line-memory/metadata.json). +- [Scaling plot](/private/tmp/docling-line-memory/line_memory.png). + +The /private/tmp paths are temporary artifacts, not durable repository fixtures. +The initial experiment generated 16 PDF variant/size combinations and ran 108 +measurements: three fresh processes for each of 36 configurations. Generated +PDFs were produced in separate processes, outside the measured parsing process. + +### Dash-array finding: already fixed + +"Old dash handler" means current code with only grph.h substituted from +immediately before #452. The relevant header differs only by the added +dash_array.clear(). It does not mean an entire historical release. + +"Current code" in the initial report means the checkout above, already including +#452. + +Median peak RSS through rendering, in MiB: + +| Dashed lines | Without the clear | With the clear | +| ---: | ---: | ---: | +| 1,000 | 82.06 | 24.19 | +| 2,000 | 263.75 | 29.72 | +| 4,000 | 884.09 | 33.94 | +| 8,000 | 3,245.97 | 49.09 | + +Both versions produce exactly N shape render instructions and N page-shape +records. Before the fix, shape i receives 4i dash entries. Two retained copies +of those double arrays occupy 32 × N × (N + 1) bytes, excluding overhead and +temporary allocations. With the fix, the same payload is 64 × N bytes. + +The quadratic behavior is in each instruction's growing payload, not in the +number of render instructions. This finding is established; it does not need +to be rediscovered during the new memory work. Use the fixed code as the normal +baseline and retain dash controls as correctness coverage. + +### Dense-grid finding: large linear cost + +The C++ port of the published path-storm generator rounds its grid dimensions, +so the actual line count is slightly smaller than the requested count. + +Median peak RSS, in MiB: + +| Actual lines | Collect tokens only | Parse with shapes disabled | Full parse | Full parse + render | +| ---: | ---: | ---: | ---: | ---: | +| 124,861 | 226.47 | 229.98 | 303.64 | 303.23 | +| 249,652 | 441.20 | 445.22 | 570.78 | 573.67 | +| 499,204 | 873.06 | 878.38 | 1,109.42 | 1,119.55 | +| 999,108 | 1,734.98 | 1,742.52 | 2,202.89 | 2,218.86 | + +These are separate process measurements, not additive memory buckets. Small +differences between full parse and full parse + render can reflect run-to-run +variation. RSS differences are not an exact breakdown of live allocations. + +At 999,108 lines: + +- 999,108 shape render instructions and page-shape records. +- 1,998,216 path points, with no stored dash entries. +- 6,993,758 top-level tokens: 7 × N + 2 for this generator. +- Token-vector capacity of 8,388,608. +- sizeof(qpdf_stream_instruction) of 64 bytes on this build, giving 512 MiB for + the vector's inline allocation alone; the referenced QPDF objects and other + allocations are additional. + +The measurements show approximately linear growth on these inputs. They +establish a substantial cost before page elements are created, but do not yet +separate QPDF's own working memory from our retained token objects. No QPDF leak +has been established. + +The first measurements use one page, one process, do_thread_safe=false, default +sanitization, and render scale 1.0. They do not reproduce the real large Form +XObject mentioned in #345 or the threaded backend's aggregate memory. + +## Source map and current structure + +Line numbers can move; use the named methods when implementing. + +| File | Relevant responsibility | +| --- | --- | +| [src/parse/qpdf/stream_decoder.h](../../src/parse/qpdf/stream_decoder.h) | QPDF ParserCallbacks, token normalization, per-token type/text construction, stream.push_back, damaged-content recovery. | +| [src/parse/qpdf/stream_instruction.h](../../src/parse/qpdf/stream_instruction.h) | QPDF handle plus key and val strings retained for each token. | +| [src/parse/pdf_decoders/stream.h](../../src/parse/pdf_decoders/stream.h) | Stored stream vector, interpretation loop, graphics state, inline images, nested content, and stream = stream_ copy in the vector-taking interpretation overload. | +| [src/parse/pdf_decoders/page.h](../../src/parse/pdf_decoders/page.h) | Page content decoding followed by interpretation; annotation appearance decoding; timing attribution. | +| [src/parse/pdf_resources/page_xobject_form.h](../../src/parse/pdf_resources/page_xobject_form.h) | parse_stream() materializes and returns a local Form token vector. | +| [src/parse/pdf_resources/page_pattern.h](../../src/parse/pdf_resources/page_pattern.h) | Cached parsed tiling-pattern content; requires a deliberate cache/replay policy. | +| [src/parse/pdf_states/shape.h](../../src/parse/pdf_states/shape.h) | Constructs page-shape records and render subpaths. | +| [src/parse/page_items/render_instructions.h](../../src/parse/page_items/render_instructions.h) | Retained render instructions and geometry. | +| [src/parse/utils/pdf_timings.h](../../src/parse/utils/pdf_timings.h) | Existing timing keys and attribution hierarchy. | + +Current sequence for page content: + +1. Read /Contents, including a possible array of streams. +2. QPDF buffers the decompressed bytes and parses objects. +3. Our callback appends every normalized object to a token vector. +4. The interpreter traverses that vector and creates page output. +5. The stream decoder is destroyed when content decoding finishes. +6. Page output remains available for extraction and rendering. + +On the Form path, parse_stream() creates a local vector; the receiving +interpreter then copies it with stream = stream_. This is an additional +linear allocation. It is not a copy from a Form token cache. Tiling-pattern +caching is a separate case. + +## Measurement protocol for every cycle + +### Keep comparisons controlled + +- Use the same generated PDF bytes for baseline and candidate. Record input + hashes and actual emitted line counts. +- Use matching compiler, optimization mode, architecture, dependency versions, + and parser/render settings. Avoid dependency upgrades within a comparison. +- Record source revision, relevant uncommitted diff, binary hash, build flags, + QPDF version, and the exact command. +- Run each measured case in a fresh process. Generate the PDFs in a separate + process and keep generation outside the measured run. +- Begin with a small diagnostic selection. Expand only after the user returns + a successful build and the first results. +- Use three repetitions for reported comparisons, preserving each observation + and reporting median plus range. Keep baseline and candidate run conditions + comparable; investigate noisy results instead of drawing a conclusion from + one convenient run. +- Keep single-process, single-page native measurements first. Add threaded and + Python integration measurements at the relevant later gates. +- Preserve raw logs, settings, failures, and output counts. Give each iteration + its own output directory so the previous result is still available. + +### Record the quantities that explain memory + +Collect scalar counters without accumulating an event log per token: + +- Decompressed content bytes, token and operator counts, and executed operators. +- Peak RSS and current RSS at defined stages; normalize units to bytes in data. +- Existing token-vector size/capacity and any nested-vector copies. +- Maximum pending operand count and, where practical, large operand payload size. +- Active nested content depth and the kind of stream being interpreted. +- Page-shape, render-instruction, subpath, segment, and point counts. +- Dash-array entry counts, decoded bitmap sizes, and retained output sizes where + practical. +- Total process wall time, full parse time, render time, and relevant stage times. +- Parse/load failures, skipped operators, degraded output, and unexpected warnings. + +The existing probe reports elapsed time at snapshots before process teardown; +do not silently treat this as the entire subprocess lifetime. If end-to-end time +is needed, record it separately around the child process. External peak-memory +measurement can corroborate the in-process high-water readings. + +A top-level token can be a large array, dictionary, string, or inline-image +payload. A small token count does not prove a small byte footprint. The constant +pending-operand expectation applies to the simple line generator, not every PDF. + +### Compare slopes and output, not just one large-page number + +For the grid, use the current requested sizes 125k, 250k, 500k, and 1M and compare +actual counts. Retain smaller cases for quick feedback and larger cases only as +the user finds practical. + +Compare memory versus actual lines, memory per line after accounting for fixed +overhead, and the number of retained token objects. Do not claim constant total +memory: QPDF's decompressed bytes and the page outputs still scale with input. + +For repeated and nested Forms, record both unique stream size and total +invocation/output count. Replaying a Form R times can legitimately create R times +the output; distinguish that from unnecessarily retaining R token-list copies. + +## Preliminary audit — Find unintended accumulation and missing resets + +The user requested an additional investigation for bugs similar to the missing +dash_array.clear() fixed by +[commit 9b09a05](https://github.com/docling-project/docling-parse/commit/9b09a057722edb56ad55bd9a6767a3ae4960eebd). + +Run this audit before the interpreter refactor and revisit relevant containers +as steps 2–4 change their lifetimes. Keep the five main implementation stages +in scope; this audit is an additional preliminary investigation, not a +replacement for isolating the token-list cost in step 1. + +The same working agreement applies: Codex investigates and proposes the code; +the user approves changes, compiles and runs reproductions; both review the +intermediate results before deciding what to fix or investigate next. + +### Hypothesis and distinction to preserve + +A setter or reset operation may accidentally append to a reusable container +instead of replacing its previous contents. If subsequent page elements copy +that state, a small mistake can produce: + +- Wrong PDF semantics, such as an empty dash array failing to restore solid lines. +- Retained state proportional to the number of prior setter calls. +- Total copied payload proportional to the number of paints multiplied by the + accumulated state size, potentially giving quadratic memory growth. + +There are also legitimate accumulations with expensive copying. Those are a +different category. Correct clipping history, page output, resource scopes and +intentional caches must not be cleared simply because their containers grow. + +A search result without a nearby clear is a lead, not a confirmed bug. +Assignment, construction of a fresh object, swap, replacement of a smart +pointer, or an owner with a shorter lifetime may already provide the reset. + +### Audit procedure + +1. Inventory member containers, strings and owned buffers in reusable parser + state, graphics/text state, path handling, resources and decode helpers. + Distinguish them from local scratch objects and page-result collections. +2. Locate their mutations: push_back, emplace_back, insert, append, operator+=, + resize and indexed assignment, as well as clear/assign/swap/replacement. +3. For each candidate, identify the semantic operation: replace a setting, + append a path segment, intersect a clip, emit a page element, merge a resource + scope, replay cached content, or finish a pending operation. +4. Trace actual callers and object lifetimes. Establish whether the same + instance can reach that operation repeatedly. Check constructor, reset, + copy/assignment, q/Q restoration, nested-stream entry/exit and page unloading. +5. Check empty values, explicit defaults and malformed operands. An empty value + that should reset a setting is especially useful for exposing stale state. + Also verify partial-update semantics: an ExtGState dictionary changes the + properties it supplies, so missing fields must not blindly reset other state. +6. Identify downstream snapshots/copies and their lifetimes. Record both current + state size and the total state payload copied into emitted elements. +7. Classify each finding as confirmed incorrect accumulation, suspected pending + reproduction, intentional accumulation with potential copy amplification, + safe because replacement/fresh construction was verified, or unresolved. +8. For a confirmed candidate, propose a minimal semantic fix and a focused + regression fixture. Fix and measure it separately from streaming changes so + its effect remains attributable. + +### Initial areas to inspect + +| Area | Reset/reuse questions | Evidence required | +| --- | --- | --- | +| Graphics-state settings and ExtGState application | Does each supplied setter replace its own value? Do empty patterns/default values work? Are unspecified properties preserved? | Repeated and alternating settings followed by visible paint, including explicit empty values and q/Q restoration. | +| Current paths and temporary subpaths | Are painted/discarded paths released at the correct operator? Are retained current-path portions required by PDF semantics? | Repeated m/l/S and m/l/n cases, multiple subpaths, closing operators, clip-plus-paint and nested state scopes. | +| Clipping state and snapshots | Which history must accumulate, and is the entire history copied into every render instruction? | Repeated clip-plus-paint with and without q/Q, clip counts/points per instruction, total copied clip bytes, unchanged pixels. | +| Text state and pending Type 3 procedures | Are pending procedures drained after each show operation, and are appropriate per-operation fields reset without destroying persistent text state? | Repeated text and Type 3 painting, nested content and recoverable failures, with emitted glyph/procedure counts. | +| Color-space, font and other resource parsing | Is the object reused, replaced, or freshly constructed? Can repeated initialization append to old arrays/maps or leave stale flags? | Actual call-path evidence plus a supported repeated-use reproduction; do not treat an unreachable reuse pattern as an established page-memory bug. | +| Inline images and decoding scratch buffers | Are headers/payloads replaced between images and after failure? Are reused buffers holding content or only capacity? | Multiple images with different settings, a damaged image followed by a valid image, and buffer/payload counters. | +| Marked content, operands and other stacks | Are scopes popped/drained at the proper logical boundary? Does recovery leave stale pending work? | Balanced and malformed streams, nested Forms, split /Contents, usable suffixes after a bad operator. | +| Page/document and cache lifetimes | Does repeated processing retain output or resources beyond their intended owner? | Repeated processing/unloading in one process, separated from fresh-process scaling. | + +### Initial source observations: not additional confirmed bugs + +The first targeted source inspection established the following leads and +non-findings. It was not a complete audit. + +- The direct d-operator bug is already fixed in pdf_states/grph.h. +- ExtGState /D parsing in pdf_resources/page_grph.h uses a fresh local + std::vector and assigns it to dash_array_. The local push_back loop + is not itself the same missing-clear bug. +- Resource collection setters examined in page_grphs.h and page_colorspaces.h + construct a fresh individual resource before assigning it into the map. Any + claim about a missing reset must account for that construction and the map's + intended scope. +- pdf_state::capture_pending_clip() intentionally appends clip paths: + successive clipping operations intersect the current clip. + pdf_state::get_clip_state() builds a snapshot from that history, which + painting can retain in a render instruction. This is a candidate for + legitimate-state copy amplification, not evidence that the history should + be cleared. Its scaling has not yet been measured. +- No second missing-clear defect has been confirmed by this preliminary pass. + +### Reproduction and measurement strategy + +For each selected candidate, Codex should supply a small deterministic PDF +generator/fixture and exact commands for the user. Prefer the native probe for +memory measurements and focused semantic tests for correctness. + +Compare: + +- Repeating the same setting before every paint. +- Setting it once, followed by the same number of paints. +- Alternating two distinct values, to expose stale contents rather than only + memory growth. +- Setting a nonempty value followed by its defined empty/reset value. +- A save/restore control using q/Q, where appropriate for that state. +- A fresh resource/object versus a supported repeated-use path. + +Use increasing operation counts. Keep the number of settings and emitted paint +operations explicit; they may vary independently. + +Record: + +- Final and maximum current-state entry/byte counts. +- Number of emitted page elements and render instructions. +- Per-element snapshot sizes and total retained/copied state payload. +- Peak RSS and runtime in fresh processes. +- Output values and pixels that prove the reset/replacement semantics. +- For cross-page/document retention, a separate repeated-use process experiment. + +If current state grows with repeated replacement operations, investigate a +missing reset. If current state must grow by definition but snapshots multiply +it across outputs, consider immutable sharing or another correctness-preserving +representation instead. Do not label allocator-retained capacity as retained +logical state: clear() normally preserves capacity, while shrink_to_fit() is +not a universal fix and should not be added to hot paths without evidence. + +### Acceptance and integration with the five stages + +The audit deliverable is a candidate table with semantic expectations, +owner/lifetime evidence, reproduction status, measured scaling and a proposed +decision. + +A missing-reset fix needs both a semantic regression test and a before/after +measurement on its reproduction. Passing a memory check alone is insufficient +if the proposed clear removes required PDF state. + +Keep each confirmed fix independently reviewable and obtain approval before +editing production code. The user compiles/runs each candidate, and we decide +together whether to retain it. Refresh the memory baseline after accepting such +a fix before attributing later gains to callback-driven interpretation. + +If the audit finds no additional missing resets, record the inspected scope and +move on with the five-step plan. Do not make speculative clears to produce a +smaller graph. + +## Step 1 — Complete the baseline and isolate QPDF + +**Purpose:** determine how much memory belongs to QPDF's own parsing path and +how much comes from retaining its delivered objects in docling-parse. + +### Proposed code changes + +Extend tests/cpp/test_memory_on_lines.cpp, retaining the existing optional CMake +target and current modes. + +Add a clearly named QPDF-only/discard mode: + +- Open the same file and select the content stream under test. +- Supply ParserCallbacks that count delivered objects/operators but retain no + object handles, per-token strings, or growing containers. +- Release each delivered object when its callback returns. +- Record decompressed byte count through the supported callback, and call counts. +- Use the same QPDF object-level parsing API as the collecting path. A lower-level + lexical tokenizer would answer a different question. +- Label this mode unambiguously. Keep the old tokens mode identifiable as the + collecting callback, not as bare QPDF. + +Add a Form-based generator: + +- Put the same line operators in a Form XObject invoked from the page. +- Match page size, geometry, paint settings, and compression to the direct case. +- Initially invoke the Form once; add repeated and nested variants separately. +- In a QPDF-only Form measurement, inspect the actual Form content, not just the + page's small /Fm0 Do wrapper. +- Preserve deterministic PDF generation and report actual output counts. + +Avoid changing production interpreter behavior in this step. If result +aggregation needs a helper, Codex should provide it with exact usage; any Python +helper is orchestration/reporting only, never the parser under measurement. + +### Experiments for the user + +1. Compile the probe using the existing target. +2. Run a small direct-page case with discard, collect, full-parse, shapes-disabled, + and render modes to verify all commands and counts. +3. Repeat at increasing line counts for the direct page. +4. Compare direct and single-Form full parsing at matched line counts. +5. Run repeated/nested Forms at modest sizes to expose copy count and lifetime. +6. Keep the fixed-dash controls; do not routinely rebuild the pre-#452 handler. + +### Expected evidence and decision gate + +- Discard and collect modes deliver matching token/operator counts for matching + logical content. +- Discard mode has no growing container of callback objects. +- Direct and equivalent Form cases produce matching geometry and rendered output. +- Explain the memory gap between discard and collect modes, rather than assigning + the entire initial 1.69 GiB to QPDF. +- Quantify the added Form cost before choosing the order of nested-path changes. + +If discard mode itself consumes unexpectedly large memory, investigate its +source and object lifetimes before assuming the rest of the plan will provide +the predicted gain. Decide together whether a QPDF-side investigation is needed. +Do not set an absolute memory target until this baseline is available. + +## Step 2 — Extract instruction execution from storage + +**Purpose:** make the interpreter consume one instruction at a time while +initially keeping the existing buffered delivery path. + +### Proposed code changes + +Refactor pdf_decoder so stream lifecycle and dispatch are explicit: + +- Begin a logical content stream: establish the initial state and capture its + base transformation before executing its first instruction. +- Consume one normalized instruction: preserve existing inline-image handling, + operand accumulation, null handling, operator execution, and operand cleanup. +- Finish a logical content stream: handle trailing operands, unresolved marked + content, and current end-of-stream diagnostics at the same semantic boundary. + +The existing vector loop should call these operations. This patch changes the +organization of execution, not whether QPDF tokens are retained. + +Preserve: + +- Graphics/text state and current transformation behavior. +- Resource resolution, including Form-local resources and inherited fallbacks. +- Operand order and current validation/normalization semantics. +- Recoverable per-operator errors and propagation of resource-exhaustion errors. +- Inline-image header/data state transitions. +- /ActualText, artifact and MCID linkage, and end-of-stream cleanup. +- Existing output order and unknown-operator reporting. + +Do not silently add eager object-to-string conversions or extra copies in the +new helper. Treat ownership and reference lifetimes explicitly. + +### Experiments and correctness checks for the user + +- Compile and run the small native comparison matrix from step 1. +- Compare parsed output and rendered pixels before/after the refactor. +- Run focused split-content-stream, damaged-content, inline-image, graphics-state, + text, clipping, and actual-text coverage. +- Review warning/error changes as well as successful outputs. + +### Expected evidence and decision gate + +Memory and runtime should remain within the measured baseline variation; this +is a behavior-preserving preparation step, not the main memory optimization. +Investigate any systematic difference before proceeding. Accept the refactor +only after output equivalence and error-handling behavior are established. + +## Step 3 — Interpret page instructions as QPDF delivers them + +**Purpose:** remove the complete collected token list from ordinary page +content interpretation. + +### Proposed code changes + +Allow the QPDF adapter to deliver normalized instructions to the execution +operations from step 2 instead of always appending them to a vector. + +For ordinary page content: + +1. Set up stream state before the first delivered instruction. +2. Consume operand objects as they arrive. +3. When an operator arrives, execute it and release its operands. +4. Release the delivered instruction when its callback returns unless execution + explicitly needs ownership of its payload. +5. Finalize once at the end of the logical content stream. + +Keep buffered delivery temporarily as an internal test/comparison path. Prefer +a narrow internal selector or test facility over a permanent public parser option. +The public Python API and normal decode configuration should not need to change +just to compare implementations. + +Retain normalization in one place so buffered and callback delivery cannot drift. +Do not replace the QPDF parser with a new PDF tokenizer as part of this work. + +### Error recovery and stream boundaries: required design work + +The existing recovery code uses stream.empty() to decide whether parsing +delivered any tokens before failing. That check becomes invalid if tokens are +consumed without being stored. + +Replace it with explicit delivery progress: + +- Track whether/how many objects have been delivered for the relevant attempt. +- Distinguish failure before any callback from failure after a prefix was consumed. +- Retry a damaged stream or its intact component streams only under conditions + that cannot execute the same prefix twice. +- Preserve the existing behavior of retaining usable content before corruption. +- Keep operator-execution errors separate from QPDF parser/decompression errors. +- Do not let a broad exception handler reinterpret allocation failures or + interpreter failures as a reason to retry the source stream. + +A /Contents array is one logical content stream. Continue to support operands +and compound tokens split across its physical streams. Do not reset graphics, +text, pending operands, or marked-content state at each physical boundary. + +Specify when finishing occurs after a recoverable error versus an unrecoverable +failure. A cleanup routine must not accidentally emit or replay content on a +failed attempt. + +### Timing and instrumentation + +Callback execution interleaves QPDF parsing and interpretation. An outer parsing +timer will now include callback work. + +Define the new accounting before comparing old stage keys: + +- Keep end-to-end full parse wall time as the primary runtime comparison. +- Separate QPDF-exclusive time from callback interpretation where feasible. +- Account for nested interpretation consistently. +- Avoid double counting and negative exclusive timings. +- Document any changed interpretation of existing timing fields. + +Do not add per-token logging to the measured fast path. Diagnostic counters +should be bounded and disabled or inexpensive in ordinary operation. + +### Experiments and correctness checks for the user + +Run buffered and callback delivery on identical input bytes: + +- Direct solid-line grids at increasing sizes. +- Fixed repeated dashes and q/Q controls. +- Shapes-disabled parsing to expose the remaining QPDF/input-buffer cost. +- Text, bitmaps, inline images, clipping and large operand-array cases. +- /Contents arrays with a token spanning physical streams. +- Damaged streams that fail before any object and after a usable prefix. +- A bad operator followed by valid content, ensuring recovery does not end the + page and does not duplicate earlier content. + +### Expected evidence and decision gate + +For the simple line fixtures: + +- Maximum pending numeric operands stays bounded as N increases; it should track + the operands needed by the operator, not the entire content-stream history. +- No complete vector of all page tokens remains live on the callback path. +- Geometry, page elements, and render instructions match the buffered path. +- Peak memory falls substantially relative to the collecting baseline. +- Shapes-disabled measurements move toward the QPDF-discard baseline plus the + remaining interpreter state; do not assume equality or promise a fixed MiB cap. +- Full-parse memory still includes legitimate page output. + +If memory does not fall as expected, inspect retained handles, large operands, +allocator effects, and remaining container lifetimes before moving on. + +## Step 4 — Remove unnecessary retention in nested content + +**Purpose:** carry the improvement into the Form-heavy inputs relevant to #345 +and the other nested content entry points. + +This step may require several separately reviewed patches. Agree on their order +from step 1 and step 3 results; do not combine all nested mechanisms at once. + +### Form XObjects first + +Replace the materialize-local-vector-then-copy sequence with callback-driven +interpretation where the Form is interpreted once. + +Preserve: + +- Caller graphics state and save/restore semantics. +- Form /Matrix, /BBox, clipping, and base transformation capture. +- Local fonts, graphics state, colorspaces, shadings, patterns, and XObjects, + including inherited resource fallback. +- Marked-content linkage and text replacement behavior. +- Transparency-group and soft-mask behavior. +- The lifetime of the caller's callback, operands, QPDF object and source bytes + while a nested parse is active. + +Review QPDF reentrancy and the lifetime of every borrowed object. Verify nested +parsing against the actual supported API rather than assuming that a callback +can safely retain references into a parent parser's temporary objects. + +### Other entry points + +Audit and convert in bounded groups: + +- Annotation/widget appearance streams in pdf_decoder. +- Type 3 character procedures. +- Soft-mask group streams. +- Tiling-pattern parsing and replay. + +The Type3 audit also found eager init_charprocs() description-vector retention. +Remove unnecessary storage/copies while preserving eager validation and damaged- +stream repair before glyph analysis; see the ordered continuation below. + +Ordinary Form parse_stream() currently returns a newly built vector. Tiling +patterns intentionally cache content and must be treated differently. + +For intentional replay/caching: + +- Identify whether the existing parsed-token cache or decoded render-instruction + cache is needed for repeated painting. +- Preserve useful reuse without repeatedly copying cached token vectors. +- If a token representation remains necessary, evaluate an immutable borrowed + view with a clearly proven owner/lifetime. +- Do not move from a cache that another invocation needs, or invalidate a view + during nested resource resolution. +- Measure both retained cache bytes and repeated execution time. +- Avoid reparsing a frequently used pattern on every paint just to lower one + snapshot's retained memory. + +Review uses of stream = stream_, copies of pattern.parse_stream() results, and +other vector-taking interpretation entry points. A byte-saving local fix must +not change state inheritance or introduce repeated decoding. + +### Experiments and correctness checks for the user + +- Direct page versus equivalent single Form at matching sizes and settings. +- One Form invoked repeatedly, and nested Forms at controlled depth. +- Different transforms and local resources on repeated invocations. +- Widget appearances and inherited AcroForm resources. +- Type 3 glyphs painted repeatedly and at different transforms. +- Soft-mask groups and transparency. +- Reused colored and uncolored tiling patterns, including nested content. +- The existing focused tests and selected real regression pages covering each + mechanism touched by the individual patch. + +### Expected evidence and decision gate + +- Equivalent page elements and pixels. +- Unnecessary local/interpreter token copies disappear on the converted paths. +- Memory differences between direct and Form input become explainable by actual + source buffers, state, nesting, resources, and retained output. +- Repeated use does not cause unintended cache growth or an unexplained runtime + regression. +- No claim of constant total memory with nesting: active QPDF content buffers + and genuine output can still accumulate. + +Record an entry-point audit showing which paths now use callbacks, which retain +an intentional immutable representation, and why. + +## Step 5 — Validate broadly and retire the comparison path + +**Purpose:** establish correctness and memory improvements beyond the diagnostic +fixtures, then remove the temporary old execution path. + +### Validation sequence for the user + +1. Complete the synthetic direct/nested matrix and focused correctness checks. +2. Run the existing self-contained parser and renderer unit tests. +3. Select representative real vector-heavy and Form-heavy pages from the + regression corpus. Record document names and page numbers. +4. Compare baseline/candidate extracted content, render instructions where + meaningful, and rendered output. +5. Run the broader relevant parser and renderer regression suites. +6. Exercise the actual threaded path with one, two and four parser threads, + using a multi-page fixture that contains several heavy pages. +7. Run a bounded set through the Python-facing API to confirm integration and + preserve public behavior, after the native comparison has passed. + +Use the repository's existing build/test workflow. Codex supplies commands for +the user's environment after the code for that cycle is ready; the commands +below are reference starting points, not a request for Codex to execute them. + +Real-document evidence is required for a contribution under CONTRIBUTING.md. +Use the existing dataset first. The real 5.6M-segment Form page mentioned in #345 +has not been supplied or reproduced, so do not label a synthetic Form fixture +as validation of that exact file. + +### Threaded measurements + +Use matching thread counts and settings for baseline/candidate: + +- Per-process peak memory across active workers and retained results. +- Throughput, wall time, failures, and output counts. +- Peak behavior with several heavy pages eligible to decode simultaneously. +- Results queue settings and consumer behavior, recorded explicitly. +- Repeated document/page processing to distinguish intended caches from + cumulative retention. + +A shared content stream in a PDF does not imply that concurrent page decoding +will share its working buffers. Measure the actual backend. Do not claim that +max_concurrent_results limits active page decoding. + +### Retirement and acceptance + +After the user returns successful validation and we agree to accept the result: + +- Remove the temporary buffered comparison selector and unused production paths. +- Keep the useful QPDF-discard and collecting diagnostics in the optional native + probe if they remain valuable for future investigations. +- Remove dead token-vector APIs and copies only after the entry-point audit. +- Keep targeted regression tests for streaming-specific failure modes. +- Recheck timing semantics, public configuration and API compatibility. +- Provide the user a final bounded verification command set for the cleaned-up + implementation. + +Acceptance requires: + +- Equivalent expected parsed content, graphics-state semantics, and output order. +- Matching rendered output for the diagnostic fixtures and compliance with the + existing regression tolerances for the real corpus. +- No silent increase in failed, skipped, or degraded pages. +- A documented, reproducible reduction in peak memory on the relevant inputs. +- The user's explicit priorities are lower memory and no significant slowdown. + The user now explicitly accepts up to +5% slower. Use a configurable +5% + review threshold with repetition spread; this supersedes the provisional +3%. +- No complete per-page token list on ordinary streamed interpretation paths. +- An explanation for intentional retained representations and remaining memory. +- No material unexplained runtime regression; interpret changes against the + observed repetition spread and agree on any practical tradeoff together. +- No recurrence of quadratic dash payload growth. +- A documented result for both the native and threaded/integration paths. + +## Correctness coverage to preserve + +Use existing tests where they already express the relevant behavior. Add focused +fixtures only for behavior that the streaming change newly risks breaking. + +| Behavior | Existing starting points / additional checks | +| --- | --- | +| Split /Contents and compound tokens | tests/test_unit_split_content_streams.py; arrays/operands spanning streams. | +| Corrupted streams and operator recovery | tests/test_unit_damaged_content_streams.py; before-first-token and after-prefix failures; no replayed output. | +| Malformed numbers and null handling | Existing normalization in qpdf/stream_decoder.h; preserve equivalent results through both delivery paths during development. | +| Inline images and regular image XObjects | tests/test_unit_image_decode.py and relevant codec/image tests; BI/ID/EI transitions and payload lifetime. | +| ActualText, MCIDs and artifacts | tests/test_unit_actual_text.py, tests/test_unit_structure_tree.py. | +| Graphics state, clipping and coordinate systems | tests/test_unit_clipping.py, page-geometry and shape-frame tests; q/Q and Form inheritance. | +| Dash semantics | Existing dash coverage in tests/test_unit_patterns.py and tests/test_unit_dash_sanitization.py; fixed repeated-pattern and empty-array controls in the native probe. | +| Form and tiling-pattern resources | tests/test_unit_patterns.py and representative Form-based regression pages. | +| Widgets and appearance streams | tests/test_unit_widget_appearances.py, tests/test_unit_acroform_widgets.py. | +| Type 3 fonts | tests/test_unit_type3_fonts.py. | +| Transparency and masks | tests/test_unit_transparency.py, tests/test_unit_soft_masks.py. | +| Threaded behavior and output equivalence | tests/test_regression_threaded_parse.py, tests/test_regression_threaded_render.py, tests/test_unit_threaded_page_range.py. | + +Do not use changed ground truth or weakened tolerances as a substitute for +understanding an output difference. + +## User-run command starting points + +These commands describe the probe that already exists. The new discard/Form/ +callback-comparison options will be specified with the step 1–3 patches; they +must not be presented as already implemented. + +Configure and compile: + +~~~sh +cmake -S . -B build -DDOCLING_PARSE_BUILD_MEMORY_PROBE=ON +cmake --build build --target test_memory_on_lines --parallel 2 +~~~ + +Create one deterministic grid, outside every measured process: + +~~~sh +mkdir -p /private/tmp/docling-memory-baseline +./build/test_memory_on_lines generate pathstorm 1000000 \ + /private/tmp/docling-memory-baseline/grid.pdf +~~~ + +Run each mode in a separate process, repeating each three times for reported +results and preserving stdout/stderr under distinct names: + +~~~sh +./build/test_memory_on_lines measure tokens \ + /private/tmp/docling-memory-baseline/grid.pdf +./build/test_memory_on_lines measure decode \ + /private/tmp/docling-memory-baseline/grid.pdf +./build/test_memory_on_lines measure skip-shapes \ + /private/tmp/docling-memory-baseline/grid.pdf +./build/test_memory_on_lines measure render \ + /private/tmp/docling-memory-baseline/grid.pdf +~~~ + +The current tokens mode uses our collecting callback. It is not the proposed +QPDF-only/discard mode. + +The optional pause_seconds argument is for external inspection only. Keep it +zero for timing comparisons. The optional PNG output is useful for correctness +comparison; PNG encoding happens after the probe's rendered snapshot. + +After the user rebuilds the Python extension using the project's established +workflow, focused tests can start with: + +~~~sh +uv run pytest tests/test_unit_split_content_streams.py \ + tests/test_unit_damaged_content_streams.py \ + tests/test_unit_actual_text.py \ + tests/test_unit_patterns.py \ + tests/test_unit_widget_appearances.py \ + tests/test_unit_type3_fonts.py \ + tests/test_unit_soft_masks.py -q +~~~ + +Broaden the selection according to the files changed and the stage's acceptance +gate. Running against an old extension is not validation of newly edited C++. + +## Results and joint decision record + +For every cycle, record: + +| Field | Required content | +| --- | --- | +| Cycle / step | Smallest implemented change and its hypothesis. | +| Approval | User instruction approving that bounded change. | +| Build identity | Source revision/diff, compiler/flags, dependency versions, binary hash. | +| Inputs | PDF hashes, document/page selection, actual counts and generator settings. | +| Configuration | Delivery mode, parser flags, render scale, thread count, consumer/queue settings. | +| User commands | Exact build, correctness and measurement commands. | +| Correctness | Output comparisons, tests, warnings/failures, and any differences needing explanation. | +| Memory | Individual observations, median/range, relevant structural counts, and scaling. | +| Runtime | Individual observations and median/range; distinguish stage and process timings. | +| Interpretation | What the evidence establishes, alternative explanations and remaining uncertainty. | +| Joint decision | Keep/revise/revert, and the agreed next experiment or patch. | + +Suggested comparison rows: + +| Cycle | Input / actual lines | Mode | Threads | Peak RSS baseline → candidate | Parse/runtime baseline → candidate | Max pending operands | Output comparison | Decision | +| --- | --- | --- | ---: | --- | --- | ---: | --- | --- | +| Step 3 | Direct, 999,108 lines | Decode | 1 | 2,200.23 → 1,148.80 MiB | 3.178 → 2.963 process s | 2 | Counts and PNGs matched | Retain page streaming; semantic suite pending | +| Step 3 | Direct, 999,108 lines | Shapes disabled | 1 | 1,742.58 → 108.47 MiB | 2.366 → 2.049 process s | 2 | Expected zero shapes | Native memory/runtime gain verified | +| Step 3 | Direct, 999,108 lines | Render | 1 | 2,219.50 → 1,169.59 MiB | 3.563 → 3.197 process s | 2 | Byte-identical PNG | Native memory/runtime gain verified | + +### Progress + +- [x] Reproduce the original dash-payload growth in native C++. +- [x] Measure current direct-page dense-grid memory and inspect structural origins. +- [x] Agree on the five-step strategy and document the user/Codex working loop. +- [x] Write the first measurement patch and user-run launcher; native smoke/scaling passed. +- [x] Preliminary audit: state/reset lifetimes reviewed; quadratic clip snapshot retention reproduced, fixed and validated. +- [x] Step 1 core checkpoint: QPDF-discard/collect scaling and single-Form overhead measured and reviewed; broader Form reuse sweeps remain optional follow-ups. +- [x] Step 2 checkpoint: buffered refactor, matching grid outputs, scaling and targeted timing recheck accepted; carried-forward semantic suite passed. +- [x] Write step 2's buffered refactor and baseline comparison. +- [x] Write step 3 page callback implementation, boundedness diagnostics and recovery regressions; native user runs passed. +- [x] Step 3 checkpoint: ordinary page memory gain verified; fresh Form callback timing within +5%; focused semantic suite passed. +- [x] Step 4a checkpoint: Form-only callback implementation, native smoke/scaling/reuse and 153 focused semantic tests passed. +- [x] Step 4b checkpoint: appearance/widget callback implementation, native smoke/scaling and 171 focused semantic tests passed. +- [x] Repair launcher typing; full MyPy (110 sources) and Ruff checks pass. +- [x] Step 4c procedural Type3 checkpoint: native smoke/scaling and 174 focused semantic tests passed. +- [x] Legacy Type3 font-token cleanup and eager-repair bounds regression: controlled measurements and semantic tests accepted. +- [x] Step 4: Nested content and intentional-cache audit; results verified. +- [x] Step 5: Real-document, corpus and threaded validation; comparison path retired and final checks passed. + +## Cycle 1 handoff — Step 1 measurement implementation + +Date: 2026-11-11. +Authorization: the user said "good, then let's start implementing." +Status: the user subsequently ran the small and core scaling matrices successfully; +see the result records below. Step 2 is proposed, not implemented. + +### Implemented scope + +- tests/cpp/test_memory_on_lines.cpp: + - qpdf-discard: parse objects with QPDF and count/release callback objects. + - collect: parse through the same QPDF entry point, forwarding delivered + objects to our existing collecting adapter. + - tokens remains an alias for collect. + - Both raw modes report delivered objects/operators, decompressed bytes, + content-size notifications, EOF, retained tokens and token-vector capacity. + - Record the runtime QPDF version in baseline snapshots. + - pathstorm-form adds a Form-based grid generator, with optional invocation + count and nesting depth. The grid leaf has the same content bytes and + compression as the direct pathstorm case. + - Generated Form chains are explicitly marked in the page dictionary. Raw + token modes follow that marked chain and parse the leaf once; they do not + pretend to execute the page's Do operators. + - Full parse and render modes still use the production interpreter, executing + all Form invocations and nested content. + - Report copied clip paths, points and payload bytes in shape instructions + as additional bounded audit counters. +- tests/cpp/run_memory_on_lines.py: + - Launch an already-built native probe in separate subprocesses. + - Generate PDFs outside measured processes. + - Cover direct, single-Form, repeated-Form and nested-Form layouts. + - Check raw leaf counts, generated shape/segment counts, absence of dash + payload on solid grids, and the direct versus single/nested Form PNG bytes. + - Record each subprocess lifetime separately from the probe's snapshot timer. + - Save input/binary/source/dependency hashes, source snapshots, available + CMake flags/link command, raw JSON/logs, PNGs, and summary.csv. + - Refuse to overwrite an existing output directory. + - Use the Python standard library for launching/reporting and tabulate for + the console summary; no PDF bindings, compilation, dependency installation + or plotting dependencies. + +Production parser code has not been modified. The existing optional CMake +target from the initial reproduction remains sufficient. + +For repeated-Form fixtures, unique_lines is the line count inside the leaf, +while lines is the expected total painted output after all invocations. +Raw parsing counts the leaf once even when the full page invokes it repeatedly. +Do not compare that raw token count with total emitted page shapes as if the +two measurements performed the same amount of interpretation. + +Clipping byte counters describe the retained instruction snapshots. They are +not an exhaustive heap profiler and do not demonstrate a missing clear. + +### Preliminary reset audit: current candidate record + +This is a targeted source audit, not a completed repository-wide review. + +| Container / operation | Observed lifetime or reset | Classification / next action | +| --- | --- | --- | +| Direct d-operator dash array | clear before replacement, with null/error branches assigning an empty array. | Existing #452 fix retained; no new defect here. | +| ExtGState /D local dash array | Fresh local vector, then assignment to the resource member. | Not the same missing-clear pattern. | +| Current shape paths | register_paths clears after painting; n/discard_paths clear and restore the appropriate scratch state. | Required reset is present in inspected paths; retain semantic coverage. | +| Pending Type 3 procedures | paint_type3_glyphs swaps pending work into a local vector before nested execution/state copies. | Queue drain is present; malformed/reentrant cases still belong to later correctness validation. | +| Inline-image state | begin_inline_image replaces the entry; successful and incomplete-image endings replace it again. | Reset is present in the inspected transitions. | +| Graphics-state and color-space resource setters | Resource collections construct a fresh individual resource before assigning it into their map. | Do not infer a missing member clear without proving a reused instance on a supported path. | +| Page-shape init_from optional graphics fields | Geometry vectors are assigned; optional graphics fields are changed only when their JSON keys are present. | Reuse candidate for stale optional state, not a confirmed memory-growth defect. Trace callers and reproduce before proposing a change. | +| Clip history and snapshots | History intentionally accumulates intersections; snapshots can copy it into paint instructions. | Potential copy amplification; no missing-clear fix proposed and no scaling measurement yet. | +| Tiling-pattern token cache | One parsed stream is intentionally cached and returned for reuse. | Intentional retention; inspect copying and lifetime in step 4. | + +No second missing-clear defect has been confirmed. Further resource/font, +malformed-input and cross-page lifetime inspection is pending. A confirmed +independent reset fix would be proposed and measured as a separate cycle. + +### First user action: compile and run the small matrix + +Use the repository root. The target was already configured in the initial +investigation; rebuild it after these source changes: + +~~~sh +cmake --build build --target test_memory_on_lines --parallel 2 +~~~ + +If the build directory has not been configured for this optional target: + +~~~sh +cmake -S . -B build -DDOCLING_PARSE_BUILD_MEMORY_PROBE=ON +cmake --build build --target test_memory_on_lines --parallel 2 +~~~ + +Then run the small matrix, using a fresh output directory: + +~~~sh +python3 tests/cpp/run_memory_on_lines.py \ + --output /private/tmp/docling-memory-step1-smoke \ + --sizes 1000 \ + --repeats 1 +~~~ + +This runs 20 measurements: five modes across four layouts. The default repeated +Form is invoked four times; the nested Form has depth three. Generated direct, +single-Form and nested-Form images should match. Repeated painting is not +required to match one painting at antialiased boundaries. + +The runner stops and records a failure if compilation was skipped and the +binary does not support the new modes, if a child fails/times out, if a count +check fails, or if the compared PNG bytes differ. PNG-byte differences require +inspection of pixels/encoding rather than automatic ground-truth updates. + +Return: + +- Build output, especially any compiler error. +- status.json. +- summary.csv if the matrix completed. +- measurements.jsonl and metadata.json. +- image-comparisons.json if present. +- For a failure, its named .log and any generated PNGs. + +Codex has reviewed source changes and whitespace only. No build, Python script, +correctness test, fixture generation or benchmark has been run by Codex for +this cycle, in accordance with the working agreement. + +### After the smoke results: decide the scaling experiment together + +The next suggested run, after we jointly review the smoke result, is three +repetitions for direct and single-Form cases at the previous grid sizes: + +~~~sh +python3 tests/cpp/run_memory_on_lines.py \ + --output /private/tmp/docling-memory-step1-scaling \ + --sizes 125000,250000,500000,1000000 \ + --layouts direct,form \ + --repeats 3 +~~~ + +Keep repeated/nested exploration at smaller sizes initially: + +~~~sh +python3 tests/cpp/run_memory_on_lines.py \ + --output /private/tmp/docling-memory-step1-form-reuse \ + --sizes 1000,8000 \ + --layouts direct,form,form-repeat,form-nested \ + --repeats 3 +~~~ + +The larger commands are proposed follow-ups, not completed experiments or a +reason to proceed automatically. We first review the user's build and smoke +output, then agree on the next run. Step 2 remains unstarted. + +### Pending joint decision + +| Item | Status | +| --- | --- | +| Source patch | Implemented for review. | +| Compilation | The user's updated native probe ran successfully; compiler log not supplied. | +| Small-matrix counts and images | Passed: 20 measurements and both direct/Form PNG comparisons. | +| QPDF-discard versus collecting memory | Scaling reviewed: 100.73 versus 1,734.95 MiB at the 1M target. | +| Direct versus Form copy overhead | Scaling reviewed: +427.13 MiB with shapes disabled; identified vector copy is 426.87 MiB. | +| Keep/revise this patch | Pending joint review. | +| Advance to the refactor | Subsequently approved by the user; cycle 2 implementation is awaiting validation. | + +### User smoke results and console-format follow-up + +The user ran the documented --sizes 1000 --repeats 1 matrix. Local artifacts in +/private/tmp/docling-memory-step1-smoke confirm status=passed, 20 measurements, +and identical PNG bytes for direct versus single-Form and nested-Form output. +The generator emitted 980 unique lines, 6,862 leaf tokens and 28,866 +decompressed leaf bytes. Repeated-Form full parsing emits 3,920 lines. + +Peak RSS from this single repetition, in MiB: + +| Layout | QPDF discard | Collect | Full parse | Shapes disabled | Parse + render | +| --- | ---: | ---: | ---: | ---: | ---: | +| Direct | 7.53 | 9.25 | 12.72 | 12.03 | 19.17 | +| Single Form | 7.55 | 9.30 | 13.28 | 12.22 | 19.58 | +| Repeated Form | 7.59 | 9.22 | 17.58 | 12.27 | 23.66 | +| Nested Form | 7.53 | 9.27 | 13.36 | 12.20 | 19.58 | + +These results establish that the first native fixtures, count checks and image +comparisons work on the user's build. They do not establish large-page memory +scaling or a useful runtime comparison: this is one small run, and startup/ +first-use costs are visible, especially in rendering. + +At the user's request, the console summary now uses tabulate with two decimal +places and grouped integer counts. CSV fields and measurement behavior are +unchanged. tabulate is already declared in the project's dev/perf dependency +groups. This formatting change needs no C++ rebuild; Codex has not rerun the +experiments or the changed launcher. + +### User scaling results — QPDF retention and Form-copy attribution + +Date: 2026-11-11. +Artifact directory: /private/tmp/docling-memory-step1-scaling. +User-run matrix: requested sizes 125k, 250k, 500k, 1M; layouts direct and form; +five modes; three fresh subprocesses per configuration. +Status: all 120 measurements passed. Direct and single-Form PNG bytes matched +at every size. Runtime QPDF version was 12.4.2. + +Build identity and individual observations are preserved in metadata.json, +measurements.jsonl, summary.csv and the per-run logs in that directory. +The native binary SHA-256 was +e5c75a7c7eba3434c419bf47b5a1b71acd77ccddbd5e8c7868bb0527c742b57b. +HEAD remained 511c57a87e4cc72a25d6ac613a095839288ba34e with the measurement +working-tree changes. This was not a production interpreter optimization. + +Median peak RSS for the direct-page raw parsing comparison: + +| Actual lines | QPDF discard MiB | Collect MiB | +| ---: | ---: | ---: | +| 124,861 | 19.31 | 226.50 | +| 249,652 | 31.03 | 441.22 | +| 499,204 | 54.36 | 873.14 | +| 999,108 | 100.73 | 1,734.95 | + +At the largest size, QPDF delivers the same 6,993,758 top-level objects from +29,473,719 decompressed leaf bytes in both modes. The discard callback retains +no token vector; the collector retains all 6,993,758 records. + +Collecting peaks 1,634.22 MiB higher, approximately 17.22 times the discard +peak. The discard run has approximately 94.2% lower peak RSS in this isolated +comparison. This is measured evidence about the two raw-parsing modes, not +a claim that full parsing will eventually use only 100.73 MiB or improve by +94.2%. Full parsing still needs page output and interpreter/resource state. +The QPDF-discard curve itself grows approximately linearly with input size. + +The Form results provide evidence for the independently identified token-vector +copy. It is most clearly visible with shapes disabled: + +| Actual lines | Direct shapes-disabled MiB | Form shapes-disabled MiB | Form minus direct MiB | Token count × 64 bytes, MiB | +| ---: | ---: | ---: | ---: | ---: | +| 124,861 | 229.95 | 283.41 | 53.45 | 53.35 | +| 249,652 | 446.67 | 552.08 | 105.41 | 106.66 | +| 499,204 | 878.50 | 1,091.89 | 213.39 | 213.28 | +| 999,108 | 1,742.45 | 2,169.58 | 427.13 | 426.87 | + +In pdf_decoders/stream.h, do_form materializes the local vector returned by +xobj.parse_stream(), and the vector-taking interprete overload copies it with +stream = stream_. At 999,108 lines, the extra inline record allocation is +6,993,758 × 64 = 447,600,512 bytes, or 426.865 MiB. Its size closely matches the +observed 427.125 MiB shapes-disabled Form overhead. This copy shares the QPDF +objects through their handles; it does not duplicate the complete QPDF object +graph. Differences of independent peak-RSS observations still are not an exact +heap accounting. + +With full shape output enabled at the largest size: + +- Direct full parse: 2,200.27 MiB; Form full parse: 2,399.69 MiB. +- Direct parse + render: 2,218.33 MiB; Form parse + render: 2,401.11 MiB. + +The full-parse Form/direct peak difference is smaller than the isolated +shapes-disabled copy difference. Different allocation lifetimes, vector growth, +allocator reuse and peak timing prevent treating these RSS buckets as additive. +Do not infer that a particular local allocation always adds its entire size to +the final process high-water mark. + +At the largest size, median full subprocess times were: + +| Layout | QPDF discard s | Collect s | Full parse s | Shapes disabled s | Parse + render s | +| --- | ---: | ---: | ---: | ---: | ---: | +| Direct | 0.642 | 1.045 | 3.136 | 2.279 | 3.560 | +| Form | 0.651 | 1.110 | 3.249 | 2.439 | 3.530 | + +These provide runtime baselines. The probe also records earlier in-process +snapshot times separately; do not substitute those for subprocess lifetime. + +#### Interpretation and next proposed checkpoint + +The evidence supports fixing docling-parse's retained token list and nested +vector copies first. It does not establish a QPDF memory bug. QPDF's remaining +buffer/object-working-memory behavior can be investigated separately if it +becomes the dominant cost after these allocations are removed. + +Step 1's core comparison is now measured and reviewed. Larger repeated/nested +Form sweeps remain unperformed; the small matrix establishes their functional +fixture checks, and their broader investigation can be scheduled jointly. + +The next recommended patch is step 2 only: + +- Extract explicit begin/consume-one-instruction/finish operations. +- Continue feeding them from the existing stored vector. +- Preserve operand validation, inline images, state capture, per-operator + recovery and logical end-of-stream cleanup. +- Hand off focused correctness and buffered-baseline comparison commands. +- Expect approximately unchanged memory; the substantial retention reduction + belongs to step 3. + +No step 2 production edits, builds or experiments were performed while +analyzing these results. The user's decision on starting that patch is pending. +The broader missing-reset audit remains in progress; this data does not confirm +another missing clear. + +## Cycle 2 handoff — Step 2 buffered execution refactor + +Date: 2026-11-11. +Authorization: the user explicitly requested implementing step 2 and clarified +the priorities: "(1) the memory is lower and (2) we do not get significantly +slower (up to a few % is ok)". +Status: native smoke/scaling comparisons passed; see the result record below. +Focused semantic-test results and a targeted timing review remain pending. + +### Acceptance criteria clarified by the user + +The eventual optimized parser must reduce peak memory and preserve throughput +within a few percent of the comparable baseline. A large memory reduction alone +is not sufficient if it introduces a substantial slowdown. + +For reporting, start with a configurable +3% wall-time review threshold. This +is a working interpretation of "a few percent", not an exact threshold the user +specified or a statistical significance test. Values above the threshold require +review/repetition; they do not automatically make the run's correctness fail. +Change the threshold explicitly with --slowdown-limit-pct if we jointly choose +a different working budget. + +For step 2 specifically, memory is expected to remain approximately unchanged: +the complete token vector and Form vector copy still exist. This checkpoint +establishes that the execution refactor preserves output and does not add an +unacceptable runtime cost. The substantial token-retention reduction belongs +to step 3 and must be measured after that change. + +### Production change + +Only src/parse/pdf_decoders/stream.h changes production behavior organization: + +- begin_interpretation() contains the existing base-CTM capture. +- consume_instruction() contains the existing dispatch for inline-image data, + operand accumulation, null filtering, operator execution, recoverable + per-operator errors and operand cleanup. +- finish_interpretation() contains the existing marked-content end cleanup. +- interprete_stream() remains a buffered index loop and calls those operations. + +The instruction consumer is inline and accepts the instruction and operands by +reference. No new per-token callback abstraction, std::function, counter, +container allocation or record copy is added. The compiler's inlining choice +and its actual runtime effect still need measurement; inline is not a guarantee +of zero call overhead. + +Caller-owned pending operands, the public ensure_stack() call, nested-stack +rebasing, caller-specific trailing-operand diagnostics, exception types and +timing boundaries are preserved. There is no new unconditional cleanup on an +exception and no altered damaged-stream retry policy in this patch. + +The only remaining source changes for this cycle are the benchmark launcher +and this plan. Step 3 streaming delivery has not been implemented. + +### Launcher comparison + +The existing launcher adds: + +- --baseline RESULT_DIRECTORY: read the user's existing observations without + executing or rebuilding the baseline. +- --slowdown-limit-pct 3: configurable timing review threshold. +- Preflight check that the baseline passed and contains the requested cases. +- Matching generated input hashes, probe configuration, dependency hashes and + runtime QPDF version. +- Matching semantic shape/segment/dash/clip counters. +- Baseline/candidate PNG-byte comparison, in addition to the direct/Form checks + already performed within a run. +- comparison.csv with before/after peak MiB, peak-memory percentage change, + median process wall seconds, timing percentage change, timing ranges and + review flags. +- baseline-comparison.json and a tabulate comparison table. +- An explicit identical-binary flag for repeat/control comparisons. +- The comparison report embedded in status.json, so status=passed cannot be + mistaken for evidence that no timing cases require review. + +Negative memory/time percentage changes are improvements. Overlapping observed +timing ranges are labeled when the median exceeds the review threshold. +Comparisons with fewer than three repetitions are labeled diagnostic. + +The comparison uses process_wall_s from the same launch protocol for both +versions, including process teardown. The individual in-process snapshot +elapsed times remain available separately. Raw QPDF-discard and collect modes +also serve as controls: their parsing path was not changed by this refactor. + +### User commands: preserve baseline, rebuild, run sanity comparison + +Before rebuilding the current native binary, optionally preserve it for a fresh +baseline timing run if the historical measurements later look noisy: + +~~~sh +cp -n build/test_memory_on_lines /private/tmp/docling-memory-before-step2 +~~~ + +The intended baseline binary is the one from the accepted step-1 scaling +experiment, hash +e5c75a7c7eba3434c419bf47b5a1b71acd77ccddbd5e8c7868bb0527c742b57b. +Copying an already-rebuilt step-2 binary would not preserve that baseline. +No binary copy or compilation has been performed by Codex for this cycle. + +Rebuild the native target: + +~~~sh +cmake --build build --target test_memory_on_lines --parallel 2 +~~~ + +Run the small existing fixture matrix and compare it with the saved smoke run: + +~~~sh +python3 tests/cpp/run_memory_on_lines.py \ + --output /private/tmp/docling-memory-step2-smoke \ + --baseline /private/tmp/docling-memory-step1-smoke \ + --sizes 1000 \ + --repeats 1 +~~~ + +This is a build/count/image check across all four layouts. Its timing flags +are diagnostic because there is only one repetition. Use a new output directory; +the launcher refuses to overwrite existing results. + +### User commands: compare memory and runtime at scale + +Once the native sanity comparison passes, run the same scaling matrix as step 1: + +~~~sh +python3 tests/cpp/run_memory_on_lines.py \ + --output /private/tmp/docling-memory-step2-scaling \ + --baseline /private/tmp/docling-memory-step1-scaling \ + --sizes 125000,250000,500000,1000000 \ + --layouts direct,form \ + --repeats 3 \ + --slowdown-limit-pct 3 +~~~ + +This performs 120 candidate measurements and compares them with the 120 saved +baseline measurements. It does not rerun the old binary. Review: + +- Peak-memory changes: approximately flat is expected at this preparation step. +- Full decode and render process wall-time changes, especially at larger sizes. +- Timing range and repetition spread. +- Raw QPDF-mode controls for general machine/load differences. +- Exact input, output-counter and PNG checks. + +A flag above +3% calls for investigation, not an immediate conclusion that the +refactor is slower. If necessary, agree on a fresh, interleaved baseline/candidate +comparison using the preserved baseline executable and extra repetitions. Do +not introduce compiler-specific forced-inlining or other optimizations before +the returned data shows a need. + +### User correctness checks beyond line fixtures + +The synthetic grids do not exercise text, damaged operands, split compound +tokens, inline images, nested text state or ActualText. Before accepting this +production refactor, rebuild the extension in the established project environment +and run focused existing tests. + +The existing documented build workflow is: + +~~~sh +uv run --no-sync python local_build.py +~~~ + +Then: + +~~~sh +uv run --no-sync pytest \ + tests/test_unit_split_content_streams.py \ + tests/test_unit_damaged_content_streams.py \ + tests/test_unit_actual_text.py \ + tests/test_unit_image_decode.py \ + tests/test_unit_clipping.py \ + tests/test_unit_patterns.py \ + tests/test_unit_widget_appearances.py \ + tests/test_unit_type3_fonts.py \ + tests/test_unit_soft_masks.py -q +~~~ + +Keep build mode and dependencies aligned with the baseline; no dependency upgrade +is part of this patch. A stale extension does not validate the edited C++. +The broader real-document/corpus gate in step 5 remains necessary; these checks +do not replace it. + +No additional implementation-mirroring unit tests were introduced for the +mechanical extraction: existing content-semantic tests and direct baseline output +comparisons cover this checkpoint. + +### Evidence to return and pending decision + +Return compilation errors if any; otherwise status.json, comparison.csv, +baseline-comparison.json and the focused test result. Preserve measurements.jsonl, +metadata.json and logs for further analysis. + +Codex performed read-only source review and whitespace checks. Codex did not +compile, generate PDFs, run the launcher, run correctness tests or collect +candidate measurements for this cycle. + +| Item | Status | +| --- | --- | +| User acceptance priorities | Lower memory and up to +5% slowdown; explicitly confirmed after the recheck. | +| Step 2 implementation | Written; buffered delivery retained. | +| Native/extension builds | Native candidate ran successfully; rebuilt-extension semantic results not yet reported. | +| Small-fixture baseline comparison | Passed 20 measurements and all four baseline image comparisons. | +| Scaling memory/runtime comparison | Passed 120 measurements; memory approximately flat; three timing flags reviewed below. | +| Focused semantic tests | Awaiting user. | +| Keep/revise refactor | User authorized proceeding after the seven-repetition timing recheck; semantic checks carry forward. | +| Step 3 callback delivery | Source written; see Cycle 3 for validation and pending measurements. | + +### User step-2 results — Buffered refactor comparison + +Date: 2026-11-11. +Source of evidence: the user's comparison table and saved artifacts under +/private/tmp/docling-memory-step2-smoke and +/private/tmp/docling-memory-step2-scaling. + +The small 20-measurement matrix passed its input/counter/image comparisons +against step 1. The scaling matrix passed all 120 measurements, comparing 40 +three-repetition configurations against the saved baseline. All eight +direct/Form render outputs at the four larger sizes matched the corresponding +baseline PNG bytes. Baseline and candidate binaries were different, as expected. + +Candidate native binary SHA-256: +1f037251bd0b8938449d3eb2e86d0b934afc241b442a0c1df579f5db26bbb551. + +#### Memory conclusion + +Peak-memory changes across the scaling comparison ranged from -0.785% to ++0.106%. Memory is effectively unchanged at this checkpoint, as expected for the +buffered refactor. The retained token list and Form copy have not been removed. +Do not count this as the project's required memory reduction. + +#### Main decode/render timing conclusion + +All 16 full-decode and render configurations are within the +3% review budget. +The largest observed slowdown among those modes is +1.342%. + +At the largest requested size: + +| Layout | Mode | Memory change % | Process-time change % | +| --- | --- | ---: | ---: | +| Direct | Decode | -0.001 | +1.342 | +| Direct | Render | +0.053 | +0.080 | +| Form | Decode | +0.021 | -0.530 | +| Form | Render | -0.034 | -0.819 | + +Many smaller cases measured faster, but unchanged QPDF control paths also +shifted. Do not attribute all apparent speedups to this mechanical refactor. + +#### Three timing review flags + +| Layout / mode at 1M target | Median change % | Baseline process range s | Candidate process range s | Interpretation | +| --- | ---: | --- | --- | --- | +| Direct collect | +4.266 | 1.0446–1.0776 | 1.0671–1.0914 | Ranges overlap; collecting does not execute the refactored interpreter. | +| Direct shapes disabled | +3.815 | 2.2650–2.2977 | 2.3167–2.4263 | Ranges do not overlap; this path executes the refactor and needs a targeted repeat. | +| Form QPDF discard | +4.375 | 0.6506–0.6590 | 0.6603–0.6832 | Does not execute the refactored interpreter; demonstrates that an unchanged control can also shift between these runs. | + +The control shifts make machine/load variation a plausible explanation, not +proof that the shapes-disabled flag is harmless. A small per-instruction cost +could be more visible with shape construction disabled. Resolve that specific +question with a fresh comparison rather than forcing inlining or rerunning +the entire scaling matrix immediately. + +#### Next proposed user experiment: targeted fresh baseline and candidate + +The preserved binary at /private/tmp/docling-memory-before-step2 exists. +Codex verified its file hash by read-only inspection: it matches the original +step-1 binary hash e5c75a7c7eba3434c419bf47b5a1b71acd77ccddbd5e8c7868bb0527c742b57b. + +Run the old binary on the direct 1M case, with QPDF-discard as a control and +shapes-disabled parsing as the case under review: + +~~~sh +python3 tests/cpp/run_memory_on_lines.py \ + --exe /private/tmp/docling-memory-before-step2 \ + --output /private/tmp/docling-memory-step2-recheck-baseline \ + --sizes 1000000 \ + --layouts direct \ + --modes qpdf-discard,skip-shapes \ + --repeats 7 +~~~ + +Immediately follow with the current binary and compare the fresh results: + +~~~sh +python3 tests/cpp/run_memory_on_lines.py \ + --output /private/tmp/docling-memory-step2-recheck \ + --baseline /private/tmp/docling-memory-step2-recheck-baseline \ + --sizes 1000000 \ + --layouts direct \ + --modes qpdf-discard,skip-shapes \ + --repeats 7 +~~~ + +These are two short back-to-back batches, not a fully interleaved statistical +experiment. They refresh the machine conditions and increase the repetition +count without changing code. If they still leave the question unresolved, +decide jointly whether alternating the executables or profiling their generated +code is warranted. + +The native source snapshot saved while running the old executable describes +the current working tree, not the historical source compiled into that old +binary. Its executable hash, the original step-1 metadata and known step-2 diff +establish the baseline identity. + +#### Correctness and next implementation gate + +Focused semantic-test results after rebuilding the extension have not yet been +reported in this conversation. Grid counters and matching pixels do not +substitute for the damaged-stream, split-content, text/ActualText, inline-image, +pattern and nested-content coverage already listed for step 2. + +Recommendation: provisionally retain the refactor, obtain the focused semantic +test result, and resolve the direct shapes-disabled timing flag before accepting +the step and starting callback delivery. This is a recommendation, not recorded +joint approval for step 3. + +No candidate builds, tests or benchmark executions were run by Codex while +analyzing these results. Only saved artifacts and hashes were inspected. +No production code changes were made in response to this result. + +## Cycle 3 handoff — Step 3 callback-driven page content + +Date: 2026-11-11. +Authorization: after the seven-repetition recheck, the user said: +"Up to 5% slower is totally acceptable. I would go to step 3 now." +Status: step 3 source code written; user compilation and all new validation pending. + +### Step 2 recheck and updated runtime agreement + +The user's fresh recheck compared the preserved original baseline with the +step-2 binary on the direct 1M case, using seven repetitions per mode: + +| Mode | Baseline peak MiB | Candidate peak MiB | Baseline process s | Candidate process s | Time change % | +| --- | ---: | ---: | ---: | ---: | ---: | +| QPDF discard | 100.77 | 100.72 | 0.643 | 0.639 | -0.68 | +| Shapes disabled | 1,742.58 | 1,742.58 | 2.230 | 2.251 | +0.96 | + +The earlier shapes-disabled timing flag did not reproduce at the same size +under this refreshed comparison. The native checkpoint supports proceeding +with the mechanical refactor. No forced-inlining change was made. + +The current runtime allowance is explicitly **up to +5% slower** for the +optimized implementation, together with lower peak memory and equivalent +output. This supersedes the earlier provisional +3% review threshold. +The launcher's default is now 5%; the threshold remains configurable. + +Focused semantic-test results for the rebuilt extension have not been reported. +The user explicitly authorized step 3; carry those checks forward into step-3 +validation rather than treating them as already passed. + +### Production changes + +1. qpdf/stream_decoder.h supports either its existing vector collector or a + borrowed function-pointer consumer and context. +2. Both delivery paths use the same normalization logic for QPDF objects, null + operands and malformed numeric tokens. +3. pdf_decoder::decode_and_interprete() establishes stream state, + consumes normalized objects immediately and finalizes the logical stream + once when decoding/recovery completes. +4. pdf_decoder::decode_contents() uses immediate consumption by default. +5. The temporary DOCLING_PARSE_BUFFERED_PAGE_CONTENTS compile definition keeps + the buffered page path only for a standalone comparison executable. No public + Python configuration switch or environment variable is introduced. + +QPDF parsing still buffers its decompressed input. The new path removes our +whole-page vector of rich token records; output geometry, text, bitmaps and +required current state still use memory. + +Nested Forms, widget appearances, Type 3 character procedures, soft masks and +cached tiling-pattern content retain their existing buffered entry points in +this step. Parent page callbacks now execute their nested operations before +the parent QPDF parse returns, so Form/pattern/image tests are important even +though conversion of their own source streams remains step 4. + +Source inspection of QPDF v12.4.2 shows that each parseContentStream invocation +has its own content buffer, OffsetBuffer and Tokenizer. This supports the +reentrant design; it is not a substitute for executing the nested fixtures and +semantic tests on the user's build. + +### Recovery and exception ownership + +- Delivered-object progress replaces stream.empty() / stream.size() as the + recovery decision signal. +- A source failure before any delivered objects can still recover component + streams or a usable decompressed prefix. +- A prefix that has already been delivered is not replayed. +- A consumer exception is retained as an exception_ptr and rethrown instead + of being mistaken for damaged PDF data. +- std::bad_alloc propagates through all three decoder recovery catch sites. + Buffered tokenization also receives this allocation-failure correction. +- Callback/context references are borrowed only during synchronous decoding. + Operands and output that must outlive a callback hold their own values or + QPDF handles; they do not retain a reference to its temporary token record. +- QPDF's component EOF callbacks do not perform interpreter cleanup. Logical + stream finishing happens once after the entire page /Contents operation. +- Fatal interpretation/allocation errors leave through the exception path; + finishing is not invoked through an unconditional destructor that could + emit/replay content during failure. + +### Timing semantics + +Callback delivery interleaves parsing and execution. The streaming path reports +content_parse_interprete_total: combined parsing/operator self-time after +subtracting already-attributed nested work. This key is registered in the native +timing tree and the performance-analysis breakdown. + +The buffered comparison keeps content_decode_total and interprete_ops_total. +Do not compare one of those old component keys directly with the new combined +key. Overall parse/process time is the acceptance metric. + +No clock calls were added to the per-object/per-operator callback to reconstruct +the old split. Pending-operand high-water counting is enabled only for the +optional native probe targets, using DOCLING_PARSE_CONTENT_DIAGNOSTICS. Production +extension builds do not pay for that high-water update. + +### New diagnostics and their scope + +The probes report: + +- page_content_delivery: callback or buffered. +- page_source_objects and page_source_bytes_observed. +- page_buffered_tokens and page_buffered_token_capacity. +- page_operand_diagnostics and page_max_pending_operands. + +These page_* values cover the page's /Contents only, not a total over nested +Form sources. + +For direct grid fixtures: + +- Both implementations consume 7 × unique_lines + 2 page-source objects. +- Buffered delivery retains that many source records. +- Callback delivery retains zero source records and zero token-vector capacity. +- The pending operand high-water mark is two, independent of line count. + +For Form fixtures: + +- The page /Contents wrapper consumes four objects per Form invocation and has + a pending operand high-water mark of one. +- Raw QPDF-discard/collect modes continue to inspect the Form's leaf once. +- Large Form-source vectors remain live in the existing nested path. A zero + page_buffered_tokens reading must not be described as zero nested retention. + +The launcher validates these structural claims on generated fixtures. Its CSV +now contains the page-source/delivery/operand fields. The CMake flags saved with +each run come from the actual executable target's directory, so buffered and +callback builds can be distinguished. + +### Native comparison and regression targets + +With DOCLING_PARSE_BUILD_MEMORY_PROBE=ON, build: + +- test_memory_on_lines: normal callback page delivery. +- test_memory_on_lines_buffered: the same source/dependencies with buffered + page delivery, useful for controlled comparison and diagnosing failures. +- stream_callback_regression: focused native consumer/recovery checks. + +The native regression covers: + +- Delivery order and normalization equivalence with buffered collection. +- Inline-image object delivery. +- Consumer failures and std::bad_alloc before/after a delivered prefix. +- Those failures on valid Flate, damaged stream arrays and recovered Flate + prefixes, covering different recovery contexts. +- Decoder reuse after a consumer failure. +- Preserving intact streams before and after a damaged array component exactly + once. + +tests/test_unit_split_content_streams.py additionally checks pending operands +crossing physical streams and a damaged middle stream between two text-bearing +streams without duplicate prefix output. + +All checks are written but have not been compiled or executed by Codex. + +### First user commands: configure, compile and run native recovery checks + +The additional targets require CMake regeneration: + +~~~sh +cmake -S . -B build -DDOCLING_PARSE_BUILD_MEMORY_PROBE=ON + +cmake --build build --parallel 2 --target \ + test_memory_on_lines \ + test_memory_on_lines_buffered \ + stream_callback_regression + +ctest --test-dir build --output-on-failure -R '^stream_callback_regression$' +~~~ + +Do not infer success from the existence of an older native binary. The new +targets, delivery labels and diagnostics identify the new build. + +### First user experiment: compare the two builds on small fixtures + +Create a fresh same-source buffered reference: + +~~~sh +python3 tests/cpp/run_memory_on_lines.py \ + --exe build/test_memory_on_lines_buffered \ + --output /private/tmp/docling-memory-step3-buffered-smoke \ + --sizes 1000 \ + --repeats 1 +~~~ + +Then run callback delivery and compare: + +~~~sh +python3 tests/cpp/run_memory_on_lines.py \ + --output /private/tmp/docling-memory-step3-smoke \ + --baseline /private/tmp/docling-memory-step3-buffered-smoke \ + --sizes 1000 \ + --repeats 1 +~~~ + +Each matrix contains 20 runs across direct, single-Form, repeated-Form and +nested-Form layouts. The expected first gate is matching inputs/counters/PNG +bytes, zero buffered page tokens on the callback path and the expected pending +operand bounds. One repetition at this size is not a runtime-budget test. + +Return any compilation/native regression failure first. If successful, return +status.json, comparison.csv, baseline-comparison.json, summary.csv and relevant +logs for the two small runs. + +### Scaling after the small gate + +The primary end-to-end comparison should remain against the user's step-2 +baseline, so shared changes in a newly built buffered reference do not hide an +overall regression relative to the prior implementation: + +~~~sh +python3 tests/cpp/run_memory_on_lines.py \ + --output /private/tmp/docling-memory-step3-scaling \ + --baseline /private/tmp/docling-memory-step2-scaling \ + --sizes 125000,250000,500000,1000000 \ + --layouts direct,form \ + --repeats 3 \ + --slowdown-limit-pct 5 +~~~ + +Expected hypothesis, not a measured result: + +- Direct-page peak memory falls substantially, especially with shapes disabled. +- Full output memory remains approximately linear in emitted elements. +- Form-heavy pages retain most of their nested-source cost until step 4. +- Full parse/render runtime remains within the user's +5% allowance, or improves. +- Counts/pixels remain equivalent. + +If the historical comparison is noisy or a change needs attribution, collect +the matching larger buffered matrix with --exe build/test_memory_on_lines_buffered +and compare callback delivery against that fresh reference as well. Both probe +builds have operand diagnostics enabled, whereas the historical step-2 build did +not; account for instrumentation and machine variation when interpreting a small +timing difference. Do not credit shared buffered-path changes as a callback-only +benefit. + +Agree on extra measurements from the returned results; do not launch them +automatically. + +### Semantic validation before accepting step 3 + +Rebuild the extension using the established workflow, then run the focused +existing tests plus the new split/recovery cases: + +~~~sh +uv run --no-sync python local_build.py + +uv run --no-sync pytest \ + tests/test_unit_split_content_streams.py \ + tests/test_unit_damaged_content_streams.py \ + tests/test_unit_actual_text.py \ + tests/test_unit_image_decode.py \ + tests/test_unit_clipping.py \ + tests/test_unit_patterns.py \ + tests/test_unit_widget_appearances.py \ + tests/test_unit_type3_fonts.py \ + tests/test_unit_soft_masks.py \ + tests/test_unit_transparency.py -q +~~~ + +These must exercise the new extension, not an older installed binary. In +particular, verify operator-error recovery, the usable prefix of a corrupted +Flate stream, physical stream splits, inline images, ActualText finalization and +nested content invoked while a parent QPDF callback is active. + +Do not update ground truth, weaken tolerances or accept a success count that +silently drops content. The larger corpus and real-document checks remain in +the final validation stage. + +### Pending decision + +| Item | Status | +| --- | --- | +| Step-2 targeted native recheck | Reviewed; +0.96% on shapes-disabled parsing, unchanged peak memory. | +| User runtime allowance | Explicit maximum +5% slowdown; default review threshold updated. | +| Step-3 source implementation | Written for review. | +| New native builds/regressions | Awaiting user. | +| Small buffered/callback comparison | Awaiting user. | +| Scaling memory/runtime comparison | Awaiting user after the small gate. | +| Rebuilt-extension semantic tests | Awaiting user; earlier results were not reported. | +| Accept/revise callback delivery | Pending joint review. | +| Step 4 nested-content conversion | Not started. | + +No build, test, fixture generation or benchmark was run by Codex for this cycle. +Only source inspection and whitespace checks were performed. No memory-reduction +or speedup measurement is claimed for the written implementation. + +## Cycle 4 handoff — Step 4a: stream ordinary Form XObjects + +Date: 2026-11-11. +Authorization: after reviewing the step-3 Form recheck, the user said: +"Yes, please go ahead with step 4." +The immediately proposed next patch was Form XObjects only. +Status: Form source conversion written; user compilation and experiments pending. +Step 4 remains open for its other entry points and intentional-cache audit. + +### Step-3 evidence reviewed before this patch + +The user ran the two 20-measurement smoke matrices, the 120-measurement scaling +matrix, and two 56-measurement Form rechecks. Saved artifacts were inspected: + +- /private/tmp/docling-memory-step3-buffered-smoke +- /private/tmp/docling-memory-step3-smoke +- /private/tmp/docling-memory-step3-scaling +- /private/tmp/docling-memory-step3-form-recheck-buffered +- /private/tmp/docling-memory-step3-form-recheck + +Both smoke matrices passed; their four baseline/candidate PNG pairs were +byte-identical. The 120 scaling measurements passed their fixture assertions +and all eight scaling PNG comparisons matched byte-for-byte. All 72 decoded +scaling measurements reported callback page delivery, zero buffered page token +size/capacity, and pending operand maxima of two (direct) or one (Form wrapper). +The saved CTest log also reports stream_callback_regression passed. + +At the 1M direct-page target: + +| Mode | Step-2 peak MiB | Step-3 peak MiB | Memory change % | Process-time change % | +| --- | ---: | ---: | ---: | ---: | +| Decode | 2,200.23 | 1,148.80 | -47.79 | -6.79 | +| Shapes disabled | 1,742.58 | 108.47 | -93.78 | -13.38 | +| Render | 2,219.50 | 1,169.59 | -47.30 | -10.28 | + +Shapes-disabled memory is close to the QPDF-discard result of 100.73 MiB. +Direct decode/render/skip-shapes process times remain within +5% at all four +sizes. Form source memory remains approximately unchanged, as expected because +step 3 converts only the page /Contents. + +The historical comparison flagged smaller Form cases. A fresh comparison of +step-3 buffered-page and callback-page executables, seven repetitions per case, +gave: + +| Form target | Decode time change % | Shapes-disabled time change % | +| --- | ---: | ---: | +| 125,000 | +0.83 | +0.49 | +| 250,000 | +1.41 | +2.70 | + +Both fresh runs passed all 56 measurements. All eight comparisons are within ++5%, with overlapping repetition ranges; no PNG comparison was requested in +that recheck. Raw controls differed by +2.87% to +3.70%. This supports a small +callback-specific cost on these Form inputs. It does not attribute the earlier +historical gap to either machine variation or shared code overhead. + +The user authorized moving on. Focused rebuilt-extension semantic test results +have still not been reported; carry those tests into this cycle. Do not mark +the complete semantic validation or step 5 as passed from grid evidence. + +### Hypothesis and production change + +For an ordinary Form Do, the old path does both: + +1. xobj.parse_stream() builds a local vector of normalized source records. +2. new_stream.interprete(insts, parameters) copies that vector into the child + interpreter and then executes it. + +The new path copies a QPDF stream handle from the resource, sets up the child +interpreter exactly as before, and calls decode_and_interprete(content, parameters). +It reuses the step-3 normalizer, callback ownership, recovery and logical-stream +lifecycle rather than duplicating the operator dispatcher. + +This should remove both Form token vectors. It is a hypothesis about peak +memory and performance until the user runs the new binaries. + +### State and ownership review + +The Form still creates parent-linked resource sets for fonts, ExtGState, +colorspaces, shadings, patterns and XObjects. Local resources are populated in +the same order before source interpretation. + +The existing parent q(), transparency-group entry, /Matrix application, +update_stack() resource rebase, inherited MCID/artifact linkage, shared soft-mask +cache/depth, child Q() and parent Q() remain in place. The begin hook captures +the Form base CTM after its matrix and inherited stack have been established. + +The returned QPDFObjectHandle is a value held until synchronous decoding returns. +The owning QPDF document must remain alive; that owner is unchanged. No token +reference or consumer/context is cached after decode returns. Nested Form +parsing leaves each parent callback and its operand vector alive on its own +stack. The nested tests must verify that behavior in the supported build. + +Existing clipping and /BBox behavior is retained. This patch does not implement +the preexisting /BBox/page-boundary TODO in do_form(). +The ordinary Form path has no new token cache. Repeated Do invocations parse +the source again just as before; measure that cost rather than assuming reuse +or removing useful caches elsewhere. + +Consumer failures and allocation failures use the step-3 propagation rules. +The legacy buffered resource helper now also rethrows std::bad_alloc explicitly, +rather than wrapping it as a logic_error that operator recovery could swallow. +Recoverable operator errors, damaged-source prefix repair and trailing operand +reporting continue through their existing mechanisms. + +### Timing semantics + +The streaming Form path records form_parse_interprete_total, a combined Form +parsing/operator self-time after subtracting work already attributed during +nested execution. It notes that self-time as attributed so enclosing Forms and +the page do not count it again. + +Form machinery remains total time minus resource setup and complete child +source/execution wall time. The buffered comparison retains parse_stream_total +and its old interpretation path. + +The new key is registered in pdf_timings and the performance-analysis breakdown. +No per-token timer is introduced. Use full process/parse runtime, not an old +tokenization-only bucket, for the +5% acceptance decision. + +### Comparison targets + +With DOCLING_PARSE_BUILD_MEMORY_PROBE=ON: + +| Executable | Page source | Ordinary Form source | +| --- | --- | --- | +| test_memory_on_lines | Callback | Callback | +| test_memory_on_lines_buffered_forms | Callback | Buffered, including the old vector copy | +| test_memory_on_lines_buffered | Buffered | Buffered, including the old vector copy | + +The Form-only reference isolates this patch while keeping page streaming. +Both comparison definitions are restricted to standalone native targets. +Normal production builds use streamed pages and streamed ordinary Forms. + +Codex preserved the original step-3 executable at: + +- /private/tmp/docling-memory-before-step4 +- SHA-256: 4222c234f884d09389da15e523f39911a193bd5700ae658144029580f2a3711b + +Its hash matches both the step-3 scaling and final Form-recheck metadata. +Copying/checking this binary did not execute it. Use it for fresh historical +rechecks if needed; the primary scaling baseline can remain the saved step-3 +results. A source snapshot collected while executing this historical binary +would describe today's working tree, so retain the original metadata/hash too. + +### Form diagnostics + +Probe targets enable scalar aggregation over Do calls reached from the page's +ordinary Form tree: + +- form_content_delivery: callback or buffered. +- form_diagnostics: whether probe aggregation is enabled. +- form_executions: total Form invocations, including nested/repeated calls. +- form_source_objects: normalized source objects consumed over those calls. +- form_materialized_tokens: cumulative records materialized across the legacy + local and interpreter vectors. This is not peak live tokens or allocated bytes. +- form_max_pending_operands and form_max_depth. + +Existing page_* fields still describe only the page /Contents. +These Form counters do not aggregate across separate buffered appearance, +Type 3, soft-mask or pattern interpreter entry points. Generated fixtures use +the ordinary Form tree and therefore have exact expectations. + +Let L be unique grid lines, D the Form depth and R page-level repetitions: + +- Direct pages: zero Form executions, objects, materializations and depth. +- Form fixtures: D * R executions and maximum depth D. +- Form source objects: R * (7 * L + 2 + 4 * (D - 1)). +- Maximum pending operands inside Forms: two. +- Callback Form materializations: zero. +- Buffered Form materializations: twice the consumed Form source-object count. + +The launcher validates these fields on every decoded measurement and writes +them to summary.csv. Existing tabulate memory/runtime summaries remain. +It also snapshots the relevant production headers alongside probe sources, +flags and hashes. Older saved baselines without these diagnostic fields remain +readable, so the step-3 data can be compared without regeneration. + +Aggregation and operand high-water updates are enabled only for native probes; +production does not perform the aggregation after each Do. + +### New semantic regression coverage + +tests/test_unit_form_streaming.py adds four cases: + +1. Repeated, nested Forms with distinct matrices and shadowed ExtGState names: + verify local opacity, restored outer opacity, continued outer painting and + restored page color/transform. +2. A damaged Flate Form with a recoverable prefix, invoked twice, followed by + parent text: verify both executions and the suffix appear exactly once. +3. A Form's closed ActualText scope applies its replacement before parent text. +4. An unclosed Form ActualText scope preserves literal text and cannot capture + the parent suffix. + +These are meaningful behavior checks for nested callback execution; they have +not been compiled/run by Codex. Existing transparency, soft-mask, pattern, +widget and Type 3 tests remain important because they may invoke ordinary Forms +even while their own source entry points stay buffered. + +### First user commands: configure and build + +~~~sh +cmake -S . -B build -DDOCLING_PARSE_BUILD_MEMORY_PROBE=ON + +cmake --build build --parallel 2 --target \ + test_memory_on_lines \ + test_memory_on_lines_buffered_forms \ + stream_callback_regression + +ctest --test-dir build --output-on-failure -R '^stream_callback_regression$' +~~~ + +This CTest target covers generic callback normalization/recovery. It does not +replace the generated Form comparison or extension semantic tests. + +### First user experiment: small Form matrix + +~~~sh +uv run python3 tests/cpp/run_memory_on_lines.py \ + --exe build/test_memory_on_lines_buffered_forms \ + --output /private/tmp/docling-memory-step4-buffered-forms-smoke \ + --sizes 1000 --repeats 1 + +uv run python3 tests/cpp/run_memory_on_lines.py \ + --output /private/tmp/docling-memory-step4-smoke \ + --baseline /private/tmp/docling-memory-step4-buffered-forms-smoke \ + --sizes 1000 --repeats 1 +~~~ + +Each run has 20 measurements across direct, single-Form, repeated-Form and +nested-Form fixtures. Require matching counters and all four baseline PNGs, +expected execution/depth counts, zero callback Form materialization, and +two pending Form operands independent of line count. + +One repetition at 1,000 lines is a correctness/structural smoke check, not a +runtime-budget measurement. Return compilation or assertion failures first. + +### Scaling after the small gate + +Keep the historical step-3 comparison to assess the complete change: + +~~~sh +uv run python3 tests/cpp/run_memory_on_lines.py \ + --output /private/tmp/docling-memory-step4-scaling \ + --baseline /private/tmp/docling-memory-step3-scaling \ + --sizes 125000,250000,500000,1000000 \ + --layouts direct,form --repeats 3 --slowdown-limit-pct 5 +~~~ + +Expected hypothesis: + +- Direct results remain close to step 3. +- Single-Form peak memory falls substantially, approaching direct-page memory + plus explainable Form state/resource/source-buffer costs. +- Shapes-disabled Forms lose both large token vectors. +- Runtime stays within +5% or improves. +- Counts, order and rendered pixels remain equivalent. + +If needed, compare against freshly run test_memory_on_lines_buffered_forms or +the preserved original step-3 binary. The former isolates callback Form delivery; +the latter detects shared overhead relative to the historical implementation. + +After reviewing that scaling result, a bounded repeated/nested sweep is available: + +~~~sh +uv run python3 tests/cpp/run_memory_on_lines.py \ + --exe build/test_memory_on_lines_buffered_forms \ + --output /private/tmp/docling-memory-step4-buffered-forms-reuse \ + --sizes 125000,250000 --layouts form-repeat,form-nested \ + --modes decode,skip-shapes,render --repeats 3 + +uv run python3 tests/cpp/run_memory_on_lines.py \ + --output /private/tmp/docling-memory-step4-reuse \ + --baseline /private/tmp/docling-memory-step4-buffered-forms-reuse \ + --sizes 125000,250000 --layouts form-repeat,form-nested \ + --modes decode,skip-shapes,render --repeats 3 +~~~ + +The default repeated fixture paints four times as many lines as the target; +the nested fixture has depth three. Each reuse matrix has 36 measurements. +Account for emitted-output growth and active QPDF buffers when interpreting +depth/repetition results. Agree on these additional runs from the first results; +do not execute them automatically. + +### User semantic validation + +Rebuild the extension, then run the new Form cases together with the pending +step-3 suite: + +~~~sh +uv run --no-sync python local_build.py + +uv run --no-sync pytest \ + tests/test_unit_form_streaming.py \ + tests/test_unit_split_content_streams.py \ + tests/test_unit_damaged_content_streams.py \ + tests/test_unit_actual_text.py \ + tests/test_unit_image_decode.py \ + tests/test_unit_clipping.py \ + tests/test_unit_patterns.py \ + tests/test_unit_widget_appearances.py \ + tests/test_unit_type3_fonts.py \ + tests/test_unit_soft_masks.py \ + tests/test_unit_transparency.py -q +~~~ + +The tests must use the rebuilt extension. Report failures without updating +ground truth or weakening assertions. The native grid alone is not semantic +acceptance for resource shadowing, ActualText or transparency. + +### Entry-point audit and next decision + +| Entry point | Current implementation | This cycle | +| --- | --- | --- | +| Ordinary page /Contents | Callback | Retained from step 3 | +| Ordinary Form Do | Callback | Native smoke/scaling/reuse and focused semantic coverage passed | +| Appearance/widget source | Buffered | Later bounded step-4 patch | +| Type 3 CharProc source | Buffered | Later bounded step-4 patch | +| Soft-mask group source | Buffered | Later bounded step-4 patch | +| Tiling-pattern source/replay | Intentional cached tokens and copies | Audit/optimize separately; preserve repeated-use benefit | + +A Form Do encountered inside one of those other interpreters uses the converted +do_form() path, but that does not convert the caller's own source storage. + +Return status.json, comparison.csv, baseline-comparison.json, summary.csv and +the focused test result. Keep metadata, measurements.jsonl, source snapshots and +logs for analysis. Decide jointly whether to retain/revise this Form patch and +which remaining step-4 entry point to tackle next. + +Codex performed source review, preserved/hashed the old executable, and checked +whitespace. Codex did not compile, execute the native regression, generate PDFs, +run the launcher or run pytest. No step-4 memory or runtime result is claimed. + +### User results — Step 4a native and focused semantic checkpoint + +Date: 2026-11-11. +The user returned the smoke, single-Form scaling and repeated/nested scaling +results, then reported 153 focused tests passing in 6.91 seconds. These results +complete the current page/Form implementation checkpoint. They do not complete +all step-4 conversions or the broad step-5 validation. + +Evidence directories inspected: + +- /private/tmp/docling-memory-step4-buffered-forms-smoke: 20 measurements passed. +- /private/tmp/docling-memory-step4-smoke: 20 measurements passed; all four + baseline/candidate PNG comparisons are byte-identical. +- /private/tmp/docling-memory-step4-scaling: 120 measurements passed; all eight + baseline/candidate PNG comparisons are byte-identical. Direct/single-Form + PNGs also match at all four sizes. +- /private/tmp/docling-memory-step4-buffered-forms-reuse: 36 measurements passed. +- /private/tmp/docling-memory-step4-reuse: 36 measurements passed; all four + baseline/candidate PNG comparisons are byte-identical; no timing flags. +- The saved CTest log reports stream_callback_regression passed. + +#### Single-Form scaling + +At the 1M target (999,108 painted lines): + +| Mode | Step-3 peak MiB | Step-4a peak MiB | Memory change % | Process-time change % | +| --- | ---: | ---: | ---: | ---: | +| Decode | 2,402.66 | 1,148.91 | -52.18 | -10.23 | +| Shapes disabled | 2,169.59 | 108.73 | -94.99 | -10.28 | +| Render | 2,402.62 | 1,168.91 | -51.35 | -6.89 | + +Direct-page memory at the same target is 1,148.73 MiB for decode, 108.52 MiB +with shapes disabled and 1,168.83 MiB for render. The extra memory of this +single-Form fixture is consequently about 0.08–0.21 MiB rather than the large +token-vector gap present before conversion. + +All direct and Form decode/render/skip-shapes comparisons satisfy the +5% +allowance. The sole flag is raw Form-leaf collect at 1M, +6.10%; that measurement +bypasses do_form() and therefore does not identify a callback Form regression. +Do not attribute all observed speed differences to the patch: raw controls also +vary between batches. + +All 72 decoded scaling measurements report zero Form materializations. +The 1M Form consumes 6,993,758 normalized source objects and emits 999,108 shape +instructions. Pending operands inside Forms remain bounded at two. + +#### Repeated and nested Forms + +At the 250k leaf target: + +| Layout | Mode | Buffered peak MiB | Callback peak MiB | Memory change % | Process-time change % | +| --- | --- | ---: | ---: | ---: | ---: | +| Four Form invocations | Decode | 1,785.33 | 1,136.33 | -36.35 | -7.73 | +| Four Form invocations | Shapes disabled | 552.66 | 38.97 | -92.95 | -8.50 | +| Four Form invocations | Render | 1,788.84 | 1,142.70 | -36.12 | -4.29 | +| Three Form levels | Decode | 673.73 | 300.08 | -55.46 | -6.88 | +| Three Form levels | Shapes disabled | 552.34 | 35.50 | -93.57 | -17.09 | +| Three Form levels | Render | 677.70 | 308.45 | -54.49 | -10.27 | + +All 12 reuse comparisons are faster, by 4.29–17.09%, with matching output +counters and PNGs. All 36 decoded callback measurements have zero materialized +Form tokens and a maximum of two pending Form operands. + +The repeated layout has four Form executions at depth one and paints 998,608 +lines at the 250k target. It consumes 6,990,264 source objects. The nested +layout has three Form executions at depth three, consumes 1,747,574 source +objects and paints 249,652 lines. Their output sizes differ by a factor of four; +do not compare their total RSS as though their emitted output were identical. + +Doubling the repeated fixture's leaf target from 125k to 250k approximately +doubles retained-output memory (574.97 to 1,136.33 MiB for decode). This is +consistent with linear scaling for the measured cases. It does not establish a +universal bound for arbitrary PDFs, deeper nesting or other entry points. + +#### Focused semantic validation + +The user reported all 153 tests passing in 6.91 seconds: + +| Test module | Passed | +| --- | ---: | +| test_unit_form_streaming.py | 4 | +| test_unit_split_content_streams.py | 4 | +| test_unit_damaged_content_streams.py | 6 | +| test_unit_actual_text.py | 8 | +| test_unit_image_decode.py | 42 | +| test_unit_clipping.py | 12 | +| test_unit_patterns.py | 35 | +| test_unit_widget_appearances.py | 3 | +| test_unit_type3_fonts.py | 10 | +| test_unit_soft_masks.py | 22 | +| test_unit_transparency.py | 7 | + +This covers the four new Form cases and the focused coverage carried forward +from steps 2–3. Read-only inspection of the local compiled Python extension +found the new form_parse_interprete_total key, consistent with a step-4 build. +Codex did not rerun pytest or import/execute the extension during this review. + +#### Joint checkpoint and remaining work + +Retain page and ordinary Form callback delivery. The current patch meets its +native memory/runtime goals, its fixture counts/images match, and the focused +semantic gate passes. No further synthetic performance sweeps are required for +this Form checkpoint unless a new change or failure justifies them. + +Step 4 remains open for separately reviewed appearance/widget, Type 3 and +soft-mask source conversions, plus the intentional tiling-pattern cache/copy +audit. A bounded next patch can address appearance/widget source delivery first. +Decide on each patch's scope before editing; retain useful pattern replay +caching rather than reparsing it on every paint. + +Step 5 remains open for real-document, corpus and threaded/integration coverage +and eventual removal of the temporary buffered comparison paths. The missing- +reset audit also remains open for unexamined lifetimes. Do not infer a second +missing-clear defect from legitimate clipping history or remaining output cost. + +## Cycle 5 handoff — Step 4b: stream annotation appearances + +Date: 2026-11-11. +Authorization: after the Form checkpoint passed, Codex proposed appearance/widget +streams as the next bounded patch and the user said "continue". +Status: appearance-source conversion and validation harness written; compilation, +semantic tests and measurements await the user. + +### Production change and ordering + +decode_ap_stream() formerly decoded its entire source into the STREAM decoder's +token vector, installed the appearance matrix, and interpreted the vector. +It now installs that same matrix first and calls decode_and_interprete(). + +The matrix must be installed before decoding because a QPDF callback can execute +the first drawing/text operator immediately. set_base_matrix() also establishes +the existing pattern-space anchor; the callback lifecycle preserves that state. + +The existing shared decode_ap_stream() path covers widget and rendered markup +annotation appearances. The patch preserves: + +- /AP/N stream selection and /AS lookup for appearance-state dictionaries. +- AP-local font lookup followed by AcroForm /DR and page-font fallback. +- Existing graphics-state, colorspace, shading, pattern and XObject resolution. +- The composed /Matrix and transformed /BBox-to-/Rect fit. +- Temporary appearance cells/shapes/images/render instructions and existing + merge order, widget cell tagging and widget-frame/background filtering. +- Step-3 logical-stream finalization, damaged-source recovery and consumer/ + allocation exception ownership. +- The ordinary Form callback path when an appearance invokes a Form. + +No user-facing configuration switch is added. The buffered appearance path is +retained only behind DOCLING_PARSE_BUFFERED_AP_CONTENTS in native comparison +targets. This is one buffered vector in the old appearance decoder; unlike the +old ordinary Form path, it does not add a second interpreter vector copy. + +The existing appearance resource/merge behavior is kept; this conversion does +not expand supported annotation rendering or fix unrelated resource/merge gaps. +Type 3 and soft-mask source entry points and cached pattern replay remain later +step-4 work. + +### Ownership, output memory and timing + +decode_ap_stream() already receives its QPDF handle by value. The owning document, +handle, local resources, interpreter and operand vector remain alive throughout +synchronous callback decoding. The child output objects retain their values as +before; no temporary source-token reference is cached. + +Appearance execution still builds temporary output containers and copies selected +results into the page. Their lifetimes can overlap, and render instructions and +page elements still consume memory. Removing the token list does not imply that +appearance memory will equal direct-page memory or become independent of output. + +The existing decode_annots timing includes source decoding, execution and merging. +No additional per-token clocks or artificial split between parsing and execution +are introduced. Native process wall time and peak RSS remain the acceptance +measurements, with a +5% slowdown allowance. + +### Native comparison targets + +| Executable | Page source | Ordinary Form source | Appearance source | +| --- | --- | --- | --- | +| test_memory_on_lines | Callback | Callback | Callback | +| test_memory_on_lines_buffered_appearances | Callback | Callback | Buffered | +| test_memory_on_lines_buffered_forms | Callback | Buffered | Callback | +| test_memory_on_lines_buffered | Buffered | Buffered | Buffered | + +All probe targets enable the same diagnostic definition. Normal production +builds use callback appearances without probe aggregation. + +The original step-4a executable is preserved at: + +- /private/tmp/docling-memory-before-step4b +- SHA-256: 20193d8cace138734c51c32efe8488104f97131e8bf955717d845e17f76f6d72 + +The hash matches the saved step-4 scaling metadata. It is useful for old-layout +regression rechecks. That old executable cannot generate/select the new +appearance fixtures, so the new same-source buffered-appearance target is the +primary reference for this cycle. A snapshot of today's source while running +an old executable must not be mistaken for that executable's compiled source. + +### Probe diagnostics and scope + +PAGE exposes scalar appearance statistics through a native-only getter. +Aggregation is compiled only into diagnostic probes, after each appearance +source returns. Counters reset when annotation decoding begins. + +The probe reports: + +- appearance_content_delivery and appearance_diagnostics. +- appearance_executions and appearance_widget_executions. +- appearance_source_objects and appearance_source_bytes_observed. +- appearance_materialized_tokens: cumulative token records stored by all + top-level appearance decoders, not simultaneous live tokens or allocated bytes. +- appearance_max_buffered_token_capacity: maximum final source-vector capacity + across those decoders; this is not a peak measure of every parser allocation. +- appearance_max_pending_operands. + +These counters cover selected top-level annotation appearance sources. They do +not roll Forms invoked inside an AP into its own source-object count. The +existing page_* fields still cover page /Contents; form_* fields cover the +ordinary page/Form invocation tree, excluding separate AP interpreter trees. + +For valid generated AP grids with L unique lines and R selected appearances: + +- Page /Contents is an empty stream: zero page source objects/bytes/operands. +- AP executions = R; widget executions = R only for the widget layout. +- AP source objects = R * (7 * L + 2). +- AP observed source bytes = R * leaf content_bytes. +- Callback AP materialization and final buffered capacity = zero. +- Buffered AP materialization = the AP source-object count, with enough vector + capacity for one leaf's token list. +- Pending AP operands peak at two; page-tree Form execution counters are zero. +- keep_shapes=false yields no emitted shape output but still consumes the AP. + +The byte-count equality is for valid generated fixtures. Recovery can report +source bytes across more than one attempt; do not treat it as unique bytes for +arbitrary damaged PDFs. + +### New native fixtures and launcher behavior + +The C++ generator adds: + +- pathstorm-appearance N PDF [appearances=1]: a /Stamp AP with the grid. +- pathstorm-widget N PDF [appearances=1]: a /Btn widget AP with the grid. + +The AP /BBox and annotation /Rect match the direct grid page, and its matrix is +identity. The page source is empty. Repeated annotations share one AP stream +but each selected appearance executes; generation stays in a separate process. + +The launcher offers these opt-in layouts: + +| Layout | Selected APs | Role | +| --- | ---: | --- | +| appearance | 1 | Markup annotation | +| widget | 1 | Widget selection and frame-filter path | +| appearance-repeat | 4 by default | Shared AP, repeatedly interpreted | + +--appearance-repeats controls the last layout. Existing default layouts remain +direct/form/form-repeat/form-nested, so earlier commands retain their scope. + +Raw qpdf-discard/collect modes use the generated marker to select the first AP +source once. They do not execute annotations or multiply token counts by AP +invocations. All raw layouts should expose the same grid source signature. + +The launcher validates the new counters on every decoded run, stores them in +summary.csv and keeps the tabulate memory/runtime tables. Metadata records the +appearance repeat setting and snapshots the new regression source too. Older +saved direct/Form baselines remain readable. + +The image checker compares direct versus single appearance/widget output when +those layouts are in one matrix, as well as baseline/candidate images. Repeated +appearance pixels are compared to the corresponding repeated baseline. + +### New focused semantic tests + +tests/test_unit_appearance_streaming.py adds six cases: + +1. A /Stamp AP with a rotated, nonzero BBox and an inherited page Form: the + fitted matrix must be active from the first callback, and the nested Form + must see the AP's local half-opacity resource. +2. The corresponding /Widget case exercises the same transform and local-state + requirements through widget handling. +3–5. Two widgets select the same /Yes appearance at distinct rectangles; only + the selected content is emitted twice. The font is supplied respectively by + AP /Resources, AcroForm /DR, or page resources. +6. A recoverable Flate prefix with an unclosed ActualText scope keeps literal + text, then a following annotation still emits its own text exactly once. + +These supplement existing widget frame/background/mark tests and AcroForm +metadata tests. They are written and syntax-checked, not executed by Codex. + +### First user commands: configure and build + +~~~sh +cmake -S . -B build -DDOCLING_PARSE_BUILD_MEMORY_PROBE=ON + +cmake --build build --parallel 2 --target \ + test_memory_on_lines \ + test_memory_on_lines_buffered_appearances \ + stream_callback_regression + +ctest --test-dir build --output-on-failure -R '^stream_callback_regression$' +~~~ + +The generic callback regression is still useful but does not replace the new +AP fixtures and semantic tests. + +### First user experiment: small appearance matrix + +~~~sh +uv run python3 tests/cpp/run_memory_on_lines.py \ + --exe build/test_memory_on_lines_buffered_appearances \ + --output /private/tmp/docling-memory-step4b-buffered-appearances-smoke \ + --layouts direct,form,appearance,widget,appearance-repeat \ + --sizes 1000 --repeats 1 + +uv run python3 tests/cpp/run_memory_on_lines.py \ + --output /private/tmp/docling-memory-step4b-smoke \ + --baseline /private/tmp/docling-memory-step4b-buffered-appearances-smoke \ + --layouts direct,form,appearance,widget,appearance-repeat \ + --sizes 1000 --repeats 1 +~~~ + +Each run has 25 measurements. Require matched counters and all five baseline +PNGs, equivalent direct/single-Form/single-AP/widget images, correct AP selection +counts, zero callback AP vector materialization/capacity, and bounded operands. + +One repetition at 1,000 lines is a structural/correctness gate. Return compile, +counter or image failures first; do not interpret these short process times as +a runtime regression or established speedup. + +### User semantic validation + +Rebuild the extension, then run the new cases and the established focused suite, +adding AcroForm metadata coverage: + +~~~sh +uv run --no-sync python local_build.py + +uv run --no-sync pytest \ + tests/test_unit_appearance_streaming.py \ + tests/test_unit_widget_appearances.py \ + tests/test_unit_acroform_widgets.py \ + tests/test_unit_form_streaming.py \ + tests/test_unit_split_content_streams.py \ + tests/test_unit_damaged_content_streams.py \ + tests/test_unit_actual_text.py \ + tests/test_unit_image_decode.py \ + tests/test_unit_clipping.py \ + tests/test_unit_patterns.py \ + tests/test_unit_type3_fonts.py \ + tests/test_unit_soft_masks.py \ + tests/test_unit_transparency.py -q +~~~ + +Do not count the previous 153 passing tests as validation of this newly compiled +appearance path. Return new failures without changing ground truth or weakening +assertions. + +### Scaling after the small gate + +Use a fresh buffered-appearance reference with the same current source, since +the earlier result matrices do not contain the new AP layouts: + +~~~sh +uv run python3 tests/cpp/run_memory_on_lines.py \ + --exe build/test_memory_on_lines_buffered_appearances \ + --output /private/tmp/docling-memory-step4b-buffered-appearances-scaling \ + --sizes 125000,250000 --layouts direct,appearance,widget \ + --modes decode,skip-shapes,render --repeats 3 + +uv run python3 tests/cpp/run_memory_on_lines.py \ + --output /private/tmp/docling-memory-step4b-scaling \ + --baseline /private/tmp/docling-memory-step4b-buffered-appearances-scaling \ + --sizes 125000,250000 --layouts direct,appearance,widget \ + --modes decode,skip-shapes,render --repeats 3 --slowdown-limit-pct 5 +~~~ + +Each scaling matrix has 54 measurements. Start at these sizes because AP output +temporaries and page-output copies may make its memory larger than a direct or +ordinary Form case even after source token retention disappears. + +Hypotheses to measure, not claimed results: + +- Callback APs materialize no complete source-token vector. +- Shapes-disabled APs show a clear memory reduction. +- Full output peak memory falls, with remaining temporary/page copies explained. +- Direct controls remain similar and interpretation/render time stays within +5%. +- Counters, output order and PNGs remain equivalent. + +After reviewing those results, a bounded shared-AP reuse check can use +--sizes 125000 --layouts appearance-repeat --modes decode,skip-shapes,render +--repeats 3 in fresh baseline/candidate directories. Four APs then paint roughly +500k lines. Agree on larger/reuse runs from the intermediate evidence. + +### Pending decision and remaining entry points + +| Item | Status | +| --- | --- | +| Page/Form checkpoint from earlier cycles | Native measurements and 153 focused tests passed | +| Appearance/widget source implementation | Written for this cycle | +| New native/AP smoke and semantic checks | Passed; six new AP cases and 171 focused tests total | +| AP memory/runtime scaling | Passed two 54-measurement matrices; matching PNGs and no timing flags | +| Accept/revise AP patch | Retain; native and focused semantic gates passed | +| Type 3 and soft-mask source conversion | Not started | +| Tiling-pattern cache/copy audit | Not started | +| Broad corpus/threaded validation and fallback retirement | Step 5, not started | + +Codex performed source review, Python AST parsing and whitespace checks, and +preserved/hashed the old native executable. Codex did not compile, run CTest or +pytest, generate PDFs or collect new performance measurements in this cycle. + +### User results — Step 4b native and focused semantic checkpoint + +Date: 2026-11-11. +The user reported both AP smoke matrices, both scaling matrices and the rebuilt +extension's focused suite: 171 tests passed in 6.86 seconds. This completes the +appearance/widget implementation checkpoint; retain the callback path. + +#### Native evidence + +Saved artifacts were inspected under: + +- /private/tmp/docling-memory-step4b-buffered-appearances-smoke +- /private/tmp/docling-memory-step4b-smoke +- /private/tmp/docling-memory-step4b-buffered-appearances-scaling +- /private/tmp/docling-memory-step4b-scaling + +Both smoke matrices passed 25 measurements. All five baseline PNG pairs matched +byte-for-byte, including four selected appearances sharing one source. Direct, +single-Form, single-appearance and widget output PNGs also matched each other. +The saved native callback/recovery regression passed. + +Both scaling matrices passed 54 measurements. All six baseline PNG pairs matched +byte-for-byte, and direct/single-appearance/widget PNGs matched at both sizes. +All 18 scaling comparisons satisfied the +5% runtime allowance; no timing flags +were reported. + +At the 250k target (249,652 painted lines): + +| Layout | Mode | Buffered peak MiB | Callback peak MiB | Memory change % | Process-time change % | +| --- | --- | ---: | ---: | ---: | ---: | +| Appearance | Decode | 730.44 | 437.38 | -40.12 | -4.09 | +| Appearance | Shapes disabled | 445.38 | 35.36 | -92.06 | -5.40 | +| Appearance | Render | 730.52 | 438.12 | -40.03 | -1.83 | +| Widget | Decode | 730.61 | 437.62 | -40.10 | -4.68 | +| Widget | Shapes disabled | 445.48 | 35.44 | -92.05 | -7.70 | +| Widget | Render | 730.48 | 437.59 | -40.10 | -3.27 | + +Across both sizes, appearance/widget full-output peak memory fell approximately +38–40%, shapes-disabled memory fell approximately 90–92%, and those process +timings improved by 1.83–8.04%. The single-repetition smoke times remain +diagnostic rather than runtime evidence. + +All 36 decoded appearance/widget scaling measurements had zero materialized +appearance tokens, zero final source-vector capacity and a pending operand +maximum of two. At 250k, each source consumed 1,747,566 normalized objects and +emitted 249,652 shape instructions. + +Direct-page decode memory at 250k was 298.66 MiB, below the AP's 437.38 MiB. +With shapes disabled the corresponding results were 35.25 and 35.36 MiB. +This is consistent with remaining temporary appearance output containers and +page-output copies contributing to the full-output gap. No allocation profile +was collected to assign that gap exactly to individual representations. + +A large shared-AP reuse sweep was not run. The smoke fixture and shared selected +appearance semantic cases cover repeated execution for this checkpoint. Further +single-AP scaling is unnecessary unless a new change or failure justifies it. + +#### Focused semantic evidence + +The user reported 171 tests passing in 6.86 seconds: + +| Test module | Passed | +| --- | ---: | +| test_unit_appearance_streaming.py | 6 | +| test_unit_widget_appearances.py | 3 | +| test_unit_acroform_widgets.py | 12 | +| test_unit_form_streaming.py | 4 | +| test_unit_split_content_streams.py | 4 | +| test_unit_damaged_content_streams.py | 6 | +| test_unit_actual_text.py | 8 | +| test_unit_image_decode.py | 42 | +| test_unit_clipping.py | 12 | +| test_unit_patterns.py | 35 | +| test_unit_type3_fonts.py | 10 | +| test_unit_soft_masks.py | 22 | +| test_unit_transparency.py | 7 | + +This includes the six new AP cases and the previous 153-test coverage, plus +12 existing AcroForm tests. It covers transformed appearance execution, AP-local +state during nested Form execution, all three font fallback scopes, shared +selected appearances, damaged-prefix recovery and isolation from the following +annotation. Widget chrome filtering and AcroForm metadata coverage also pass. + +Codex reviewed the user-provided pytest report and saved native artifacts. +Codex did not compile, rerun pytest or execute new measurements in this review. + +#### Decision and next bounded patch + +Retain streamed page, ordinary Form and annotation appearance sources. +Their respective native checkpoints and the current 171-test focused suite +pass. Overall step 4 remains open. + +The next proposed bounded conversion is Type 3 CharProc source delivery, with +glyph transforms, font resources, procedure draining and recursion limits +preserved. Soft-mask group sources follow separately. Pattern cache/copy +handling requires a separate audit that preserves efficient replay. + +Real-document/corpus/threaded validation, the remaining state-lifetime audit and +retirement of the native buffered comparison paths remain outstanding. Do not +treat the focused suite as completion of step 5. + +## Cycle 6 handoff — MyPy repair and step 4c Type 3 CharProc streaming + +Date: 2026-11-11. +Authorization: the user asked to correct the reported MyPy failures and then +implement Type 3 CharProc streaming. +Status: MyPy/Ruff checks pass; procedural Type 3 source conversion, probe and +regressions are written. User compilation and runtime validation are pending. + +### MyPy repair and completed static checks + +The launcher had eight reported errors from three causes: + +1. Timeout stdout/stderr variables were inferred as bytes and then assigned + decoded strings, making string concatenation invalid to MyPy. +2. A raw-source string expectation and a later numeric shape-count expectation + reused the same variable name. +3. Empty fallback dictionaries and the raw-signature cache needed annotations. + +Timeout output now normalizes directly into string variables, handling bytes, +text and missing output. Raw source selection uses expected_stream. Summary +dictionaries and measurement/image/signature collections have explicit types. +The signature cache also distinguishes ordinary grids from the metric-prefixed +Type 3 source family. + +No type ignores were added and no MyPy settings were weakened. Existing +formatter changes from the user's pre-commit run were preserved. + +After adding the Type 3 code and new typed regression module, Codex ran the +project's installed tools directly from .venv/bin: + +| Check | Result | +| --- | --- | +| mypy docling_parse tests scripts | Success; 110 source files | +| ruff check docling_parse tests scripts | All checks passed | +| ruff format --check docling_parse tests scripts | 115 files already formatted | +| git diff --check | Passed | + +Existing annotation-unchecked notes about other untyped functions remain +informational. The complete pre-commit wrapper/uv-lock hook was not executed; +the Ruff/MyPy checks above use the installed project tools and normal config. +No build, CTest, pytest or benchmark was executed by Codex. + +### Production source conversion + +do_type3_glyph() previously decoded the CharProc into a local rich instruction +vector and copied that vector into a child interpreter. The normal path now +holds a QPDF handle and calls new_stream.decode_and_interprete(charproc, parameters) +after establishing the same glyph resources and graphics state. + +Preserved behavior: + +- paint_type3_glyphs() swaps the pending queue into a local vector before glyph + execution. Each queued procedure is consumed once; inherited states do not + carry pending work back into themselves. +- CharProc lookup uses the font's resources with existing parent fallback. +- Glyph placement combines FontMatrix, font size/rise, text placement and the + inherited page CTM in the existing order. +- The child stack is rebound to glyph resources before callbacks execute. +- Glyph cells, shapes and images stay in private temporary containers; only + render instructions reach the page, alongside the already-created text cell. +- Bitmap glyph classification, unknown-operator handling and the thread-local + recursion limit of eight remain in place. +- The recursion counter is decremented on normal and exceptional source exits. +- The parent q()/Q() boundary is now also restored if child setup or execution + throws, so operator-level recovery cannot continue with the glyph transform. +- The old buffered source reference explicitly propagates std::bad_alloc rather + than warning/returning as though a glyph were merely unreadable. + +Cached image-mask/vector glyph rendering and font extraction are unchanged. +Soft-mask source conversion and pattern cache/copy work remain separate patches. + +### Important independent Type 3 memory stage + +get_type3_glyph() first examines the decoded CharProc to decide whether it can +cache a rasterized glyph. Its lightweight vector analyzer currently builds a +temporary vector of strings and scans the source even when it eventually +chooses procedural fallback. + +That first-use tokenization, source-buffer copies and any useful cached masks +remain part of process-lifetime peak RSS. The new interpreter diagnostics do +not measure that analyzer's temporary tokens. + +Therefore a streamed CharProc can report zero rich instruction materialization +while peak RSS still scales with source size, particularly with shapes disabled. +Do not promise a QPDF-discard-level memory floor or describe this patch as +removing every Type 3 token list. Use returned measurements to decide whether +the independent analyzer deserves a later bounded optimization. + +### Comparison targets and old binary + +The new test_memory_on_lines_buffered_type3 target streams page, Form and +appearance sources but keeps the old two-vector procedural CharProc path. +test_memory_on_lines streams all four converted source paths. The fully +buffered native target now defines the Type 3 comparison macro as well. + +DOCLING_PARSE_BUFFERED_TYPE3_CONTENTS is restricted to standalone native +comparison targets. There is no new public runtime/Python switch. + +The exact earlier executable is preserved at: + +- /private/tmp/docling-memory-before-step4c +- SHA-256: 19bc36ff06cb57a931c8a5730bdc1ee8744fe5c05560b2d6efa4d5725ec84203 + +Its hash matches the step-4b scaling metadata. It can recheck historical +direct/Form/AP fixtures, but cannot generate or select the new Type 3 layouts. +Use the new buffered-Type3 reference for the new source family. + +### Native Type 3 fixtures + +The C++ generator adds pathstorm-type3 N PDF [glyphs=1]. It embeds a Type 3 +font with identity FontMatrix and displays it at font size one with a fresh +identity text matrix for each glyph invocation, mapping grid ink into the same +page coordinates as the direct fixture. + +The CharProc contains a 595 0 d0 metrics prefix followed by the normal stroked +grid. Strokes are unsupported by the lightweight cached-mask rasterizer, so the +probe requires actual procedural executions. This detects accidentally +benchmarking a cached glyph mask instead of the converted interpreter. + +New opt-in launcher layouts: + +- type3: one procedural glyph. +- type3-repeat: four invocations by default, controlled by --type3-repeats. + +Repeated invocations reset their text placement and paint the same grid again. +They produce repeated render geometry, while glyph outlines remain excluded +from the public page-shape collection. This is intentional Type 3 semantics, +not silently dropped output. + +Raw qpdf-discard/collect modes select the marked CharProc once and do not execute +Tj or multiply source counts by glyph invocations. Their source has three extra +objects and one extra operator for d0, so its raw signature is grouped separately +from ordinary grids. Default layouts remain the existing four page/Form layouts. + +### Diagnostics and exact expectations + +The probes report type3_content_delivery, type3_diagnostics, type3_executions, +type3_source_objects, type3_materialized_tokens, type3_max_pending_operands, +type3_max_depth and type3_recursion_skips, plus text_instructions. + +Counters aggregate procedural fallback execution through the page/Form/Type3 +interpreter tree, including CharProcs invoked through Forms. They do not include +cached mask draws or separate AP/soft-mask interpreter trees. Aggregation is +compiled only for diagnostic native probes. + +For L unique grid lines and R procedural glyph invocations: + +- One CharProc source contains 7 * L + 5 objects and 3 * L + 2 operators. +- Executed CharProc objects total R * (7 * L + 5). +- Type 3 executions = R, maximum procedural depth = 1, recursion skips = 0. +- CharProc pending operands peak at two. +- The page source has 5 + 9 * R objects, known generated page_content_bytes, + and an operand maximum of six due to Tm. +- Normal decode/render emits R * L shape instructions/subpaths/segments and + twice that many points, with R Type 3 text instructions. +- Public page_shapes remains zero: the path marks are glyph ink. +- Shapes-disabled runs emit zero shape instructions but still execute CharProcs. +- Streamed interpreter materialization = zero. +- Buffered interpreter materialization = twice the executed CharProc objects, + counting both legacy vectors cumulatively, not peak live bytes. + +These do not count the independent first-use glyph analyzer's token strings. +The launcher validates counters, preserves matching-input/output checks, writes +the fields to summary.csv and retains tabulate memory/runtime summaries. + +### New focused semantic coverage + +tests/test_unit_type3_streaming.py adds three cases: + +1. Two glyphs queued by one Tj share the font-local half-opacity graphics + resource. Pixel checks verify each stroke paints once, glyph transforms stay + local and the parent page retains its own blue/quarter-opacity stroke. +2. The same behavior with a recoverable damaged Flate CharProc checks prefix + recovery and the second queued execution of the shared, repaired source. +3. A self-recursive glyph paints eight quarter-opacity strokes, then a plain + glyph and parent suffix still paint. Pixel opacity tests the depth bound; + later marks check that the recursion counter and graphics state were restored. + +Strokes and recursive text force these cases through the procedural fallback. +Existing Type 3 tests still cover cached vector/image-mask paths, image-XObject +resources, orientation and artifact classification. + +These cases are formatted and type-checked but have not been run by Codex. + +### First user commands: configure/build and generic native recovery + +~~~sh +cmake -S . -B build -DDOCLING_PARSE_BUILD_MEMORY_PROBE=ON + +cmake --build build --parallel 2 --target \ + test_memory_on_lines \ + test_memory_on_lines_buffered_type3 \ + stream_callback_regression + +ctest --test-dir build --output-on-failure -R '^stream_callback_regression$' +~~~ + +The generic callback regression does not replace Type 3 fixture/semantic checks. + +### First user experiment: small Type 3 matrix + +~~~sh +uv run python3 tests/cpp/run_memory_on_lines.py \ + --exe build/test_memory_on_lines_buffered_type3 \ + --output /private/tmp/docling-memory-step4c-buffered-type3-smoke \ + --layouts direct,form,type3,type3-repeat --sizes 1000 --repeats 1 + +uv run python3 tests/cpp/run_memory_on_lines.py \ + --output /private/tmp/docling-memory-step4c-smoke \ + --baseline /private/tmp/docling-memory-step4c-buffered-type3-smoke \ + --layouts direct,form,type3,type3-repeat --sizes 1000 --repeats 1 +~~~ + +Each matrix has 20 measurements. Require baseline/candidate PNG equality for +all four layouts, equivalent direct/Form/single-Type3 pixels, correct glyph +execution and ink counts, zero streamed rich-token materialization and bounded +operands. A small single repetition is a structural gate, not runtime evidence. + +Return compilation, counter, image or runtime errors first. Do not relax the +assertions or change ground truth to make a mismatch pass. + +### User semantic validation + +Rebuild the extension, then run the three new cases plus the established focused +suite: + +~~~sh +uv run --no-sync python local_build.py + +uv run --no-sync pytest \ + tests/test_unit_type3_streaming.py \ + tests/test_unit_type3_fonts.py \ + tests/test_unit_appearance_streaming.py \ + tests/test_unit_widget_appearances.py \ + tests/test_unit_acroform_widgets.py \ + tests/test_unit_form_streaming.py \ + tests/test_unit_split_content_streams.py \ + tests/test_unit_damaged_content_streams.py \ + tests/test_unit_actual_text.py \ + tests/test_unit_image_decode.py \ + tests/test_unit_clipping.py \ + tests/test_unit_patterns.py \ + tests/test_unit_soft_masks.py \ + tests/test_unit_transparency.py -q +~~~ + +The 171 earlier passing tests describe the previous compiled extension; they do +not count as executing this Type 3 change. Report the new result explicitly. + +### Scaling after the small gate + +Start with a controlled same-source comparison at two sizes: + +~~~sh +uv run python3 tests/cpp/run_memory_on_lines.py \ + --exe build/test_memory_on_lines_buffered_type3 \ + --output /private/tmp/docling-memory-step4c-buffered-type3-scaling \ + --sizes 125000,250000 --layouts direct,type3 \ + --modes decode,skip-shapes,render --repeats 3 + +uv run python3 tests/cpp/run_memory_on_lines.py \ + --output /private/tmp/docling-memory-step4c-scaling \ + --baseline /private/tmp/docling-memory-step4c-buffered-type3-scaling \ + --sizes 125000,250000 --layouts direct,type3 \ + --modes decode,skip-shapes,render --repeats 3 --slowdown-limit-pct 5 +~~~ + +Each scaling matrix has 36 measurements. Measure the combined first-use glyph +analysis, procedural interpretation and retained output; do not infer the whole +process peak from the interpreter counters alone. + +Expected hypotheses are lower peak RSS, equivalent output and process time +within +5%. A shared analysis/output cost may limit observed gains. If that cost +dominates, decide jointly whether to profile/optimize it later. Repeated-glyph +scaling is available with --layouts type3-repeat at a bounded target after the +first scaling result; it is not automatically launched. + +### Pending decision + +| Item | Status | +| --- | --- | +| Reported MyPy errors | Fixed; full 110-file MyPy check passes | +| Ruff lint/format | Pass | +| Procedural Type 3 conversion and three regressions | Written | +| User native build/smoke and rebuilt-extension tests | Native smoke/scaling passed; 174 focused tests passed in 5.93s at the user's request | +| Type 3 memory/runtime scaling | Two 36-measurement matrices passed; lower peak memory and all times within +5% | +| Cached glyph analyzer/mask pipeline | Preserved; separate first-use memory cost documented | +| Accept/revise Type 3 patch | Joint review after user results | +| Soft-mask source conversion and pattern cache/copy audit | Not started | +| Broad corpus/threaded validation and comparison retirement | Step 5, not started | + +Codex ran only source/static checks and copied/hashed the historical executable. +No native build, PDF generation, benchmark, CTest or pytest was run by Codex. + +## Type 3 native results and remaining font-initialization retention + +Date: 2026-11-11. +The user ran both step-4c smoke matrices and both scaling matrices. + +- Both smoke runs passed 20 measurements, all four baseline PNG comparisons + matched byte-for-byte, and direct/Form/single-Type3 PNGs matched. +- Both scaling runs passed 36 measurements and all four baseline PNG comparisons + matched. Direct and Type3 PNGs matched at both sizes. +- All 12 scaling comparisons meet the +5% allowance. The six Type3 comparisons + improve process time by 0.58–3.50%. +- All 18 decoded Type3 scaling measurements have zero interpreter token + materialization, a maximum of two CharProc operands and no recursion skips. +- The native callback/recovery regression also passed. + +At the 250k leaf target: + +| Type3 mode | Buffered peak MiB | Callback peak MiB | Memory change % | Process-time change % | +| --- | ---: | ---: | ---: | ---: | +| Decode | 980.14 | 791.47 | -19.25 | -0.58 | +| Shapes disabled | 860.12 | 676.72 | -21.32 | -1.12 | +| Render | 978.14 | 790.47 | -19.19 | -2.84 | + +The procedural interpreter's native goal is met, but focused rebuilt-extension +test results for this patch have not yet been supplied. The bo20 benchmark +refresh discussed below runs damaged-page tests, not the new Type3 semantic +suite, so it does not close that gate. + +### Additional structural finding in font initialization + +The remaining shapes-disabled footprint prompted inspection beyond the +lightweight first-use glyph analyzer: + +1. PAGE_FONT::set() calls init_charprocs(). +2. init_charprocs() eagerly decodes every CharProc into a complete vector of + qpdf_stream_instruction records. +3. char_processor::parse(stream) copies that vector into char_description. + It is called twice: the first return value is discarded; the second is + assigned into name_to_descr. +4. The only reader of name_to_descr found under src/, app/ and tests/ logs its + size. The retained instructions have no functional reader in this repository. + +At 250k the CharProc has 1,747,569 objects. With 64-byte instruction records, +one size-only copied token array is about 106.66 MiB before referenced objects, +strings and excess capacity. Additional eager vector/copy/cache work is thus a +strong remaining memory candidate. Exact attribution of the 676.72 MiB peak +still needs phase measurements or a controlled follow-up comparison. + +These vectors are outside the procedural interpreter counters. Zero +type3_materialized_tokens does not mean the complete font pipeline has no +retained token list. + +The eager decoder also has observable side effects: damaged CharProcs may be +repaired in place before first-use glyph analysis, and invalid entries are +checked during font initialization. Do not simply delete the eager call based +on the unused map alone. + +The next bounded Type3 cleanup should preserve eager validation/recovery through +the existing normalized discard-consumer path while removing the description +vector copies/cache and its size-only debug dependency. Keep the independent +cached glyph-mask analyzer separate. Preserve a pre-cleanup executable and +compare the complete process footprint rather than only the callback counter. +This belongs to step 4's existing unnecessary-retention audit. + +## Ordered continuation — finish parser plan, then benchmark memory reporting + +Date: 2026-11-11. +The user requested this order: + +1. Finish the steps in this parser-memory plan. +2. Update scripts/benchmarking/refresh_performance_benchmarks.sh so the benchmark + report shows memory evolution, especially peak RSS, with optional time-based + memory profiles. + +The second item is an explicitly requested follow-up. It does not authorize +skipping the parser's remaining semantic/corpus gates. The user still compiles +and runs experiments; Codex implements and analyzes. + +### Remaining parser sequence + +1. Obtain the focused Type3 semantic result for the rebuilt extension. +2. Remove unnecessary eager Type3 description retention while preserving its + validation/recovery behavior; measure and validate that bounded patch. +3. Convert soft-mask group source delivery, preserving group state, mask + semantics, recursion/budgets and useful decoded-mask reuse. +4. Audit tiling-pattern token/cache copies and change only the representation + needed to remove redundant copies while preserving efficient replay. +5. Finish the missing-reset/lifetime audit and record confirmed/non-defect + findings; do not clear legitimate clipping state to reduce one snapshot. +6. Complete the existing step-5 self-contained tests, real-document output + comparisons, broader corpus and actual threaded-path validation. +7. Retire temporary production comparison selectors/dead source paths after + acceptance, keeping useful native diagnostics and preserved measurement + artifacts. Supply final user-run checks for the cleaned-up result. + +The existing step-5 threaded-memory evidence can use the Python replay peaks +already captured and bounded manual/native OS measurements. The permanent +launcher/reporting extension follows parser acceptance; do not create a circular +dependency that requires that extension before the parser plan can be finished. + +### New bo20 bundle: what it establishes now + +User command: + +~~~sh +REPEATS=1 THREADS=1,4,8,12 RUN_COMPARISON=0 DATASET=bo20 \ + ./scripts/benchmarking/refresh_performance_benchmarks.sh +~~~ + +Bundle: scratch-performance-refresh-20261111-161912-r7M86J. +Report: report/summary.md. +Dataset: performance-dataset-bo20 at +65b9971b3fa939fe4ef89be678237e6928049b67, 20 PDFs / 496 pages. +Report source revision: 04e31ded07866d6741efb99a059056f94ee8759b. +The bundle also records working-tree and binary provenance; a revision label +alone does not identify uncommitted source compiled into the binaries. + +This provides real-workload scaling observations for native render-only and +render-with-text profiles and the Python backend-operation replay. It is one +observation per case, not a variability estimate or a before/after comparison. +The refresh's pre-test is test_unit_damaged_pages.py. Runtime/count validation +does not substitute for pixel/content regression comparisons or the new Type3 +focused tests. + +### Peak RSS is already available for the Python replay + +run_performance_benchmarking_with_docling_settings.py already calls +resource.getrusage(RUSAGE_SELF).ru_maxrss and records peak_rss_mib plus a +process-lifetime scope in each runs[] entry. + +The launcher runs each backend/thread profile in a fresh process, making the +following existing values usable for those individual runs: + +| Backend replay | Threads | Existing peak RSS MiB | Raw file under the bundle | +| --- | ---: | ---: | --- | +| pypdfium2 | 1 | 623.08 | replay/r1/pdfium/report.json | +| docling-parse | 1 | 710.33 | replay/r1/t1/report.json | +| docling-parse | 4 | 861.80 | replay/r1/t4/report.json | +| docling-parse | 8 | 1,023.56 | replay/r1/t8/report.json | +| docling-parse | 12 | 1,176.50 | replay/r1/t12/report.json | + +These are whole-process peaks, including Python/native libraries, startup, +loading, processing and retained output. They are not parser-only heap sizes +or sums of per-page peaks. + +run_performance_report.py currently omits these fields from its summary/runs +view. app/run_scaling.cpp does not currently emit a corresponding peak RSS. +There are no saved sampled RSS timelines to recover retroactively. + +### Benchmark follow-up A: default per-run peak RSS + +Implement only after the parser plan's acceptance gate: + +- Carry existing replay peak_rss_mib through report ingestion, runs.csv, + summary tables and machine-readable output. +- Add native per-process OS peak capture, reusing the native probe's verified + platform/unit handling where appropriate. +- Capture every repetition/backend/profile/thread combination separately. + Do not wrap the entire launcher and report its peak: build, downloads and + reporting processes are outside the requested benchmark measurement. +- Track the actual benchmark process, including all its threads. Do not report + the RSS of uv, the logger or tee as the backend's memory. +- Normalize native OS values to canonical bytes at collection boundaries and + display MiB. Darwin ru_maxrss is bytes; the Linux value is KiB, matching the + existing replay normalization. +- Record method, units, process identity, scope, mode and configuration with + the existing source/binary/dataset provenance. +- Label whole-process lifetime peaks explicitly. Native wall time excludes + document preload, while a lifetime peak includes it; do not imply matching + timing/memory boundaries or subtract two high-water values as phase memory. +- Preserve current workload/profile separation and recorded result-queue, + output-consumption and native-memory-release policies. +- For multiple repetitions show the median and observed min/max of per-run + peaks, plus the largest observed run peak for capacity planning. +- Add peak-RSS-versus-thread-count figures alongside throughput/scaling figures. +- Keep old bundles readable: missing native peak data displays as unavailable, + never zero or an estimate. Existing Python replay peaks can be surfaced in + a newly generated report without rerunning those measurements. +- Avoid cumulative RUSAGE_CHILDREN or shared-process high-water reuse across + runs. Optional cross-package comparison currently runs multiple series in a + process; isolate them before claiming per-series peaks, or label memory + unavailable/shared-process there. The initial target is the native/replay + profiles used by RUN_COMPARISON=0. + +Peak capture should be enabled by default with low overhead; verify that +overhead instead of promising it is free. + +### Benchmark follow-up B: optional sampled RSS timeline + +Proposed interface, preserving environment-based invocation and adding explicit +CLI equivalents: + +~~~sh +# Peak RSS in every normal report. +MEMORY_MODE=peak REPEATS=3 THREADS=1,4,8,12 DATASET=bo20 \ + ./scripts/benchmarking/refresh_performance_benchmarks.sh + +# Optional timeline; also retains the OS peak. +REPEATS=3 THREADS=1,4,8,12 DATASET=bo20 \ + ./scripts/benchmarking/refresh_performance_benchmarks.sh \ + --memory-mode profile --memory-interval-ms 50 +~~~ + +MEMORY_MODE values: off, peak (default), profile. +MEMORY_INTERVAL_MS sets a positive sampling interval, proposed default 50 ms. +CLI values override environment values. These options are proposed and not yet +implemented; existing launcher commands still reject extra arguments. + +Requirements: + +- Sample current RSS against monotonic elapsed time for each actual benchmark + process. An external sampler should continue observing while native work or + the Python GIL is busy. +- Write raw timestamp/RSS/PID samples and a per-run summary beside its existing + measurement artifacts; include interval/method/identity metadata. +- Keep OS peak RSS and sampled maximum as distinct fields. Sampling may miss + brief spikes and cannot replace the OS high-water mark. +- Record lifecycle/measurement boundaries when available, so plots distinguish + initialization/preload, measured processing and teardown. +- Produce RSS-versus-elapsed-time plots by backend, profile, threads and + repetition without confusing independent clocks or averaging unrelated runs. +- Record sampling as instrumentation. Check its effect on runtime and avoid + mixing sampled and unsampled runs in a performance claim. +- Bound the monitor's own memory with streamed sample output. Ensure cleanup + and useful artifacts on child failure, timeout or interruption. +- Preserve completed-series checkpoints and recovery provenance. A resumed + series must not silently claim a missing timeline or inherit another + process's high-water value. + +### Benchmark follow-up implementation and validation order + +1. Add a small collection/schema layer and meaningful unit coverage for unit + conversion, process isolation, missing/legacy data and invalid options. +2. Surface existing replay peaks and add native per-run peaks; integrate the + normal launcher, report tables/CSV and memory-scaling plots. +3. Add optional sampling and timeline figures after default peaks are correct. +4. Validate recovery/backward compatibility and scoped failure artifacts. +5. User runs a bounded bo20 smoke at one/four threads, then paired off/peak/ + profile runs to assess capture overhead and peak agreement. +6. User runs the requested one/four/eight/twelve-thread sweep with repeated + observations, preserving dataset/source/settings for comparison. + +Codex has not modified the benchmark launcher/report implementation or rerun the +benchmark in this review. The immediate existing peak-RSS table above was read +from saved JSON reports. + +## Completion pass — item 1: discard eager Type 3 validation tokens + +Date: 2026-11-11. +Authorization: the user said "ok, finish step 1 to 6", referring to the six +remaining parser items, then explicitly confirmed: +"Keep builds and experiments with me." + +All six items are authorized for continuation. Do not ask for source-change +approval again for those listed items. Keep the established measurement and +acceptance gates: Codex writes/reviews code and runs static checks; the user +compiles and runs experiments. Retire comparison paths after broad validation, +not merely because their replacement code has been written. + +### Prior Type 3 semantic gate is complete + +At the user's explicit request, Codex ran the full focused suite against the +then-current extension: 174 tests passed in 5.93 seconds. That includes the +three procedural Type 3 cases and closes the semantic gate for step 4c. +This one delegated test run does not authorize subsequent native builds or +experiments; the user has reaffirmed the original division of work. + +### Item 1 source change + +PAGE_FONT now calls validate_charprocs() at the same point in font setup. +It still visits every CharProc, checks that it is a stream, and uses the same +qpdf_stream_decoder normalization and damaged-stream recovery, but delivers +objects to a no-op synchronous consumer. + +The unused name_to_descr map, discarded char_processor::parse() call, second +description copy and logging of the map size are removed. Validation retains +no complete rich-token vector. + +The independent lazy glyph-mask/vector analyzer remains unchanged. It still +has its own first-use temporary storage, so total RSS is not expected to become +constant or equal to QPDF-discard memory. + +The old char_description/char_processor helper definitions remain temporarily +included but have no functional callers. Retire their dead includes/files with +the final source-path cleanup after validation; they do not retain objects now. + +### Recovery regression added + +test_eager_repair_preserves_damaged_cached_glyph_bounds() supplies a damaged +Flate CharProc with a usable filled-rectangle prefix and a deliberately loose +FontBBox. It expects actual ink dimensions (60 by 100 points). + +This checks an observable side effect of the old eager decoder: repair must +happen before lazy glyph analysis computes the text-cell bounds. Simply +removing eager decoding could preserve rendered ink while changing extraction +geometry, so the regression checks the parsed cell rather than only pixels. + +The previously passing focused suite gains one case. The new case and updated +source have not been built or executed by Codex. + +### Exact before-change reference + +Preserved executable: + +- /private/tmp/docling-memory-before-type3-font-cleanup +- SHA-256: 24459b9fb6b9450e7d60ae774412d36cea15e03ebf49510b151672aa8b3aa2aa + +Its hash matches the user's step-4c scaling metadata. Both the preserved binary +and existing step-4c data have callback CharProc interpretation plus the old +eager description retention. This isolates the new cleanup from the earlier +streaming patch. + +The launcher now also snapshots page_font.h, so subsequent result bundles +capture the changed source explicitly. + +### Static checks + +- MyPy: no issues in 110 source files. +- Ruff lint: passed. +- Ruff format check: 115 files already formatted. +- git diff --check: passed. + +These are source/static checks, not C++ compilation or runtime validation. + +### User native build and recovery check + +The optional targets already exist; an incremental build suffices: + +~~~sh +cmake --build build --parallel 2 --target \ + test_memory_on_lines stream_callback_regression + +ctest --test-dir build --output-on-failure -R '^stream_callback_regression$' +~~~ + +If using a new build directory, configure first with +`cmake -S . -B build -DDOCLING_PARSE_BUILD_MEMORY_PROBE=ON`. + +### User small comparison + +Compare to the saved callback-Type3 smoke, not the buffered-Type3 reference: + +~~~sh +uv run python3 tests/cpp/run_memory_on_lines.py \ + --output /private/tmp/docling-memory-type3-font-cleanup-smoke \ + --baseline /private/tmp/docling-memory-step4c-smoke \ + --layouts direct,form,type3,type3-repeat \ + --sizes 1000 --repeats 1 +~~~ + +Require matching source hashes, geometry counters and all four PNG pairs. +The existing Type3 interpreter materialization counter is already zero on both +sides; it does not count eager font initialization. Source inspection plus +process peak measurements establish this cleanup's effect. + +### User scaling comparison after the small gate + +~~~sh +uv run python3 tests/cpp/run_memory_on_lines.py \ + --output /private/tmp/docling-memory-type3-font-cleanup-scaling \ + --baseline /private/tmp/docling-memory-step4c-scaling \ + --sizes 125000,250000 --layouts direct,type3 \ + --modes decode,skip-shapes,render --repeats 3 \ + --slowdown-limit-pct 5 +~~~ + +This has 36 measurements. Assess memory and runtime against the old eager +retention, with direct pages as controls. The earlier 250k Type3 callback +baseline was 791.47 MiB decode and 676.72 MiB with shapes disabled. + +If dependency hashes/configuration differ, the historical binary cannot provide +a controlled reference against newly rebuilt dependencies. Rebuild the +pre-cleanup source against the same current dependencies and compiler settings +as the candidate, in an isolated reference build, before collecting a new pair. +Do not bypass comparison safeguards or relabel an old executable with current +archive hashes. Preserve its known hash and original metadata. + +### User rebuilt-extension semantic checks + +~~~sh +uv run --no-sync python local_build.py + +uv run --no-sync pytest \ + tests/test_unit_type3_streaming.py \ + tests/test_unit_type3_fonts.py \ + tests/test_unit_appearance_streaming.py \ + tests/test_unit_widget_appearances.py \ + tests/test_unit_acroform_widgets.py \ + tests/test_unit_form_streaming.py \ + tests/test_unit_split_content_streams.py \ + tests/test_unit_damaged_content_streams.py \ + tests/test_unit_actual_text.py \ + tests/test_unit_image_decode.py \ + tests/test_unit_clipping.py \ + tests/test_unit_patterns.py \ + tests/test_unit_soft_masks.py \ + tests/test_unit_transparency.py -q +~~~ + +The added bounds regression must run against the rebuilt extension. Keep +existing assertions and ground truth unchanged when reporting failures. + +### Remaining six-item completion status + +| Item | Status / next gate | +| --- | --- | +| 1. Legacy Type3 token cleanup | Complete: semantic regression, controlled smoke/scaling, six identical scaling PNG pairs and timing budget passed; current-library reference isolates eager font storage | +| 2. Soft-mask group source streaming | Complete: native smoke/scaling, cache reuse, PNG equivalence and 86 focused semantic tests passed | +| 3. Pattern cache/replay copies | Focused validation complete: native smoke/scaling, PNG/cache checks, control recheck and the user's 131-test semantic suite passed | +| 4. State-lifetime/reset audit | Focused validation complete: native exception/clip-history checks, shared-clip memory/pixel scaling, renderer cursor comparison, ordinary-page controls and 131 semantic tests passed; small timing variations remain recorded for broad validation | +| 5. Broader corpus and threaded validation | Accepted before cleanup: post-merge font/corpus failures resolved; three-repeat bo20 at 1/4/8/12 threads completed all pages without errors; native/replay scaling and replay process peaks reviewed | +| 6. Comparison-path/dead-API retirement | Complete: temporary targets/selectors, unused stream storage/APIs, Form token API and CharProc helpers removed; user reports all final tests passing and 80-case smoke passed | + +The subsequent permanent benchmark peak-RSS/timeline work remains after these +parser items, as requested. This checkpoint does not mark all six complete. + +## Completion pass — item 2: stream soft-mask group sources + +Date: 2026-11-11. +The user explicitly requested starting item 2 after the Type3-cleanup smoke. +The unresolved item-1 comparison does not block this authorized implementation; +it remains a separate acceptance gate. + +### Item-1 intermediate result and comparison limitation + +The bundle `/private/tmp/docling-memory-type3-font-cleanup-smoke` contains all +20 requested measurements, but status.json reports `failed`: the baseline and +candidate dependency_sha256 maps differ. The comparison stops before accepting +cross-run output or memory equivalence. Single-repeat timing remains diagnostic. + +Changed archives relative to the step-4c smoke are: + +- libblend2d.a, libloguru.a and libpdfium_jbig2.a; +- libfreetype.a, libjpeg.a, liblcms2.a and libopenjp2.a; +- libqpdf.a and libturbojpeg.a. + +The numbers are useful as a smoke observation but do not isolate the Type3 +cleanup's memory or runtime effect. Do not mark item 1 accepted, bypass hashes, +or use this failed comparison bundle as a baseline. Its native executable has +been rebuilt, but the new Type3 bounds regression still needs user execution. +The reference-build instruction above supersedes the earlier suggestion that +simply rerunning the preserved historical executable would suffice. + +### Source change and retained ownership + +decode_soft_mask() establishes local resources, its own output containers, the +shared page mask cache, recursion depth and the composed mask-space/gs matrix +before invoking decode_and_interprete(). It no longer accumulates a complete +group-source token vector in the normal path. + +The useful decoded soft_mask_instruction remains cached by spec and CTM. Its +private render group is retained for reuse by page paint instructions. This +change removes source-token retention, not the geometry needed to render a +mask. Alpha/luminosity selection, backdrop/transfer handling, intersections, +the nested-mask guard and the existing 512-entry limit stay in force. + +std::bad_alloc propagates rather than being reported as an unreadable mask and +silently continuing unmasked. Existing malformed-source recovery still uses +the same qpdf_stream_decoder normalization and prefix-repair path. + +### Native reference and diagnostics + +New temporary target: `test_memory_on_lines_buffered_soft_masks`. +It enables only DOCLING_PARSE_BUFFERED_SOFT_MASK_CONTENTS; pages, Forms, +appearances, Type3 interpretation and font validation match the candidate. +Build this reference and the candidate together against the current archives. +The existing fully buffered target also enables the new mask selector. + +New opt-in layouts: + +- `soft-mask`: one N-line stroked grid used as an alpha-mask group; the page + paints one filled rectangle under it. +- `soft-mask-repeat`: four installations/paints of the same entry at the same + CTM by default (`--soft-mask-repeats` changes the count). The mask source must + decode once, with three cache hits. Page output has four rectangles, while + the private decoded mask contains N strokes, not 4N. + +The native generator marks the selected group, so QPDF-discard/collect visit +that source once. It has exactly 7N+2 objects and 3N+1 operators, matching the +direct/Form grid controls. `unique_lines` is N; the generic `lines` field is +N times the number of mask applications and does not describe stored geometry. + +Probe-only counters follow the shared /Contents/ordinary-Form mask cache: +executions, cache hits, source objects/bytes, materialized tokens, maximum +buffered capacity, pending operands and private mask shape instructions. +They do not claim to cover separate appearance-decoder caches or replace a +process RSS measurement. Source counts describe each decoded group's own +source, not token counts in nested Form sources. + +For these fixtures, normal delivery must report zero mask materialized tokens +and capacity, with at most two pending operands. The reference reports 7N+2 +materialized tokens for its one decode. Shapes-disabled still executes gs and +decodes the source, but emits no mask shapes. Both paths must preserve source +counts, geometry, cache reuse and PNG bytes. + +### Semantic regression + +`tests/test_unit_soft_mask_streaming.py` adds four cases: alpha/luminosity, +each with intact/damaged Flate group content. A group matrix composes with the +CTM at gs; a nested Form uses group-local resources; two gs applications share +coverage; parent paint resumes with its original color and transform. + +Half-opacity mask content makes duplicate delivery during damaged-prefix +recovery observable in the rendered pixels. Points outside the group box and +inside its unpainted portion must retain the red background. The existing +soft-mask suite supplies transfer, clipping, cache-limit and nested-mask cases. + +### User build and native smoke + +Static validation of this implementation passed: MyPy (111 source files), +Ruff lint, Ruff formatting (116 files) and git diff --check. These do not +establish C++ compilation, rendered equivalence or a memory/runtime improvement. + +Codex has not compiled or executed these paths. Keep builds and experiments +with the user. Regenerate the existing build directory to register the new +reference target, retaining its other cached compiler/dependency options: + +~~~sh +cmake -S . -B build -DDOCLING_PARSE_BUILD_MEMORY_PROBE=ON + +cmake --build build --parallel 2 --target \ + test_memory_on_lines test_memory_on_lines_buffered_soft_masks \ + stream_callback_regression + +ctest --test-dir build --output-on-failure -R '^stream_callback_regression$' + +uv run python3 tests/cpp/run_memory_on_lines.py \ + --exe build/test_memory_on_lines_buffered_soft_masks \ + --output /private/tmp/docling-memory-soft-mask-buffered-smoke \ + --layouts direct,form,soft-mask,soft-mask-repeat \ + --sizes 1000 --repeats 1 + +uv run python3 tests/cpp/run_memory_on_lines.py \ + --output /private/tmp/docling-memory-soft-mask-smoke \ + --baseline /private/tmp/docling-memory-soft-mask-buffered-smoke \ + --layouts direct,form,soft-mask,soft-mask-repeat \ + --sizes 1000 --repeats 1 +~~~ + +Each smoke run has 20 measurements. Use fresh directories, and require both +status.json files to pass. Preserve the compiler flags, exact binary hashes, +archives, inputs and source snapshots in these bundles. Do not reuse the old +Type3 reference for this experiment. + +### User scaling after the small gate + +~~~sh +uv run python3 tests/cpp/run_memory_on_lines.py \ + --exe build/test_memory_on_lines_buffered_soft_masks \ + --output /private/tmp/docling-memory-soft-mask-buffered-scaling \ + --sizes 125000,250000 --layouts direct,soft-mask,soft-mask-repeat \ + --modes decode,skip-shapes,render --repeats 3 + +uv run python3 tests/cpp/run_memory_on_lines.py \ + --output /private/tmp/docling-memory-soft-mask-scaling \ + --baseline /private/tmp/docling-memory-soft-mask-buffered-scaling \ + --sizes 125000,250000 --layouts direct,soft-mask,soft-mask-repeat \ + --modes decode,skip-shapes,render --repeats 3 \ + --slowdown-limit-pct 5 +~~~ + +Each side has 54 measurements. Seek reduced mask memory and no meaningful +slowdown beyond the accepted 5%. Direct pages are controls. Repeated mask +applications must preserve cache hits and avoid multiplying private geometry. +Do not predict the RSS reduction from token-record sizes alone: temporary +page-shape containers, output geometry, allocator behavior and rasterization +also contribute to the peak. + +### User rebuilt-extension semantic gate + +~~~sh +uv run --no-sync python local_build.py + +uv run --no-sync pytest \ + tests/test_unit_soft_mask_streaming.py \ + tests/test_unit_soft_masks.py \ + tests/test_unit_transparency.py \ + tests/test_unit_form_streaming.py \ + tests/test_unit_patterns.py \ + tests/test_unit_type3_streaming.py \ + tests/test_unit_type3_fonts.py -q +~~~ + +This also exercises the pending Type3 bounds regression against the rebuilt +extension. Broad corpus/threaded validation and comparison-path retirement +remain later gates. Await these results before accepting item 2 or changing +the next pattern-cache ownership path. + +## Completion pass — item 2 accepted results + +The user completed the paired soft-mask smoke (20 measurements per executable) +and scaling (54 measurements per executable). Both candidate bundles passed +the dependency/input checks, geometry assertions, cache diagnostics and PNG +comparisons. All six scaling PNG pairs were byte-identical; no timing case +exceeded the accepted 5% review threshold. + +At a 250,000-line target: + +| Layout / mode | Before MiB | After MiB | Memory change | Process-time change | +| --- | ---: | ---: | ---: | ---: | +| soft-mask / decode | 566.91 | 245.84 | -56.63% | -4.65% | +| soft-mask / skip-shapes | 445.55 | 35.36 | -92.06% | -7.43% | +| soft-mask / render | 566.91 | 245.75 | -56.65% | -5.36% | +| soft-mask-repeat / decode | 566.86 | 245.88 | -56.63% | -4.98% | +| soft-mask-repeat / skip-shapes | 445.47 | 35.38 | -92.06% | -11.33% | +| soft-mask-repeat / render | 566.86 | 245.89 | -56.62% | -6.10% | + +At 125,000, decode/render memory fell about 46–49%, and shapes-disabled memory +about 90%. The direct-page controls stayed within the runtime budget. + +The repeated layouts decoded once with three cache hits, retained zero source +tokens/capacity and reached only two pending operands. Private mask geometry +was 124,861 and 249,652 strokes at the two targets, shared by four page paints. +The corresponding source counts were 874,029 and 1,747,566 objects. + +The user then rebuilt the extension and ran the seven-file focused semantic +suite: **86 passed in 4.55 seconds**, including all four soft-mask callback +cases and the Type3 eager-repair bounds regression. This closes item 2 and the +item-1 semantic gate; item-1 performance attribution remains unresolved. + +## Completion pass — item 3: borrow cached pattern source during replay + +The user approved this bounded change with "do it" after reviewing the +two-copy finding. Builds and experiments remain with the user. + +### Source and ownership change + +Both the direct lattice path and cached tile creation previously copied the +entire pattern source into a local `insts` vector, then copied it again into +the child decoder's `stream`. The normal path now calls interprete_cached(), +which borrows the resource-owned const vector for the synchronous call. + +The helper runs the same begin/consume/finish lifecycle. It copies one current +instruction because QPDF accessors and the consumer take mutable objects; +cached instruction records remain untouched. Pending operands still use the +existing bounded-per-operator vector. It does not retain a view after return. + +The resource's parsed-token cache remains owned by PAGE_PATTERN, including +its referenced QPDF objects. Its decoded render-cell cache and uncoloured +paint-colour keys also remain. No token cache is cleared or reparsed on each +tile, and no decoded cell is intentionally duplicated by this change. + +The pattern resource remains alive through the parent-linked resource maps +of these synchronous child decoders. Nested replay receives an already parsed +cache; normal interpretation does not replace or resize its source vector. +Normal replay also drops any previous child-owned source buffer before use. + +### Native reference and instrumentation + +New temporary target: `test_memory_on_lines_copied_patterns`, enabling only +DOCLING_PARSE_COPY_PATTERN_CONTENTS. Its two original vector copies and their +call-site lifetimes are preserved. The candidate and reference otherwise use +the same current code and dependencies, including accepted soft-mask streaming +and the Type3 font cleanup. The fully buffered reference also enables this +selector until final comparison-path retirement. + +Probe-only shared counters cover the /Contents decoder tree, including its +ordinary Forms, procedural CharProcs, mask groups and pattern child decoders. +They do not aggregate separate appearance decoder roots. The shared statistics +allocation and updates are absent in production builds. + +Counters distinguish: + +- Source parses/cache hits and retained cached token count/capacity. +- Decoded-cell construction/cache hits and its private shape count. +- Direct lattice executions and cumulative source objects interpreted. +- Cumulative tokens materialized in complete replay vectors, maximum capacity + of a child replay vector, and maximum pending operands. + +The vector materialization count excludes the useful resource cache, the +single working instruction and pending operands. It measures copying work, +not simultaneous peak live memory. The intentional cache remains linear in +source size in both executables. Source/copy counters cannot substitute for +measured process RSS or runtime. + +### Native fixtures and invariants + +All fixtures reuse the same generated grid source, with N actual strokes, +7N+2 QPDF objects and 3N+1 operators. Raw modes select the marked pattern stream +once. New opt-in layouts are: + +- `pattern`: one fill using a page-sized tile. One source parse, one decoded + cell containing N strokes, and one page rectangle. +- `pattern-repeat`: four fills by default (`--pattern-repeats` changes this). + One source parse and one decoded cell, three source-cache hits and three + decoded-cell-cache hits. The private geometry stays N; page output is four + rectangles. +- `pattern-lattice`: both cell steps are twice the page extent, selecting + direct replay at [0,1] by [0,1]. Four executions share one parsed source. + Render output contains 4N stroke instructions, although only the first + placement contributes on-page ink. The outline is retained as an additional + page shape and does not become an unresolved solid-fill render instruction. + +For E actual cell interpretations and T=7N+2 cached objects, the reference +reports 2ET materialized replay tokens; the candidate reports zero. Cached +tokens/capacity, source/replay counts, page/private geometry and PNG bytes +must match. These grid cells reach at most two pending operands. + +With shapes disabled, all three layouts must skip pattern parsing and replay: +zero caches, executions, replay copies and pattern geometry. This preserves +the existing CAD/text-extraction performance behavior. + +### Added semantic coverage + +`tests/test_unit_pattern_replay.py` has four cases: cached tile/direct lattice, +each using intact/damaged Flate source. An uncoloured stencil paints successive +regions red, blue and red, exercising colour-specific decoded-cell ownership +and reuse. Local half-opacity resources shadow an opaque parent resource. + +The source leaves a translation active; each subsequent replay and the parent +suffix must retain their own placement/state. Half-opacity pixels expose +duplicate recovered-prefix execution. The existing pattern suite continues +to cover pattern/base matrices, clipping, nested resources, shape skipping, +overlapping cells and image reporting. + +### Static review and checks + +- MyPy: no issues in 112 source files. +- Ruff lint: passed. +- Ruff format check: 117 files already formatted. +- git diff --check: passed. + +Codex has not compiled these C++ paths or run the new semantic cases/native +experiments. Four source cases are added to the previously passing 86-test +suite; no 90-test result is claimed. + +The lifetime audit still needs to address exception restoration in existing +pattern code: lattice depth/q-Q scopes and cached-cell decoding flags are +restored manually, and an allocation or nonstandard exception can bypass +restoration. This source finding is carried to item 4; it is not evidence of +a new measured quadratic-memory defect. This patch isolates token borrowing +from that separate state-restoration change. + +### User build and smoke gate + +Regenerate the existing build directory to register the reference target while +retaining its other cached compiler/dependency settings: + +~~~sh +cmake -S . -B build -DDOCLING_PARSE_BUILD_MEMORY_PROBE=ON + +cmake --build build --parallel 2 --target \ + test_memory_on_lines test_memory_on_lines_copied_patterns + +uv run python3 tests/cpp/run_memory_on_lines.py \ + --exe build/test_memory_on_lines_copied_patterns \ + --output /private/tmp/docling-memory-pattern-copied-smoke \ + --layouts direct,pattern,pattern-repeat,pattern-lattice \ + --sizes 1000 --repeats 1 + +uv run python3 tests/cpp/run_memory_on_lines.py \ + --output /private/tmp/docling-memory-pattern-smoke \ + --baseline /private/tmp/docling-memory-pattern-copied-smoke \ + --layouts direct,pattern,pattern-repeat,pattern-lattice \ + --sizes 1000 --repeats 1 +~~~ + +Each run has 20 measurements. Require passing status.json, unchanged cached +source counts, expected reuse, zero candidate replay vectors and four identical +PNG pairs. Single-repeat timing remains diagnostic. + +### User cached-tile scaling after the small gate + +~~~sh +uv run python3 tests/cpp/run_memory_on_lines.py \ + --exe build/test_memory_on_lines_copied_patterns \ + --output /private/tmp/docling-memory-pattern-copied-scaling \ + --sizes 125000,250000 --layouts direct,pattern,pattern-repeat \ + --modes decode,skip-shapes,render --repeats 3 + +uv run python3 tests/cpp/run_memory_on_lines.py \ + --output /private/tmp/docling-memory-pattern-scaling \ + --baseline /private/tmp/docling-memory-pattern-copied-scaling \ + --sizes 125000,250000 --layouts direct,pattern,pattern-repeat \ + --modes decode,skip-shapes,render --repeats 3 \ + --slowdown-limit-pct 5 +~~~ + +Each executable contributes 54 measurements. Seek lower RSS for cell decoding +and no meaningful slowdown beyond 5%; decoded-cell reuse must remain intact. +The source cache remains, so do not expect page/soft-mask streaming percentages. + +### User bounded direct-lattice scaling + +This path emits four placements of every grid stroke. Use smaller source +sizes to keep output geometry and runtime bounded: + +~~~sh +uv run python3 tests/cpp/run_memory_on_lines.py \ + --exe build/test_memory_on_lines_copied_patterns \ + --output /private/tmp/docling-memory-pattern-lattice-copied-scaling \ + --sizes 32000,64000 --layouts direct,pattern-lattice \ + --modes decode,skip-shapes,render --repeats 3 + +uv run python3 tests/cpp/run_memory_on_lines.py \ + --output /private/tmp/docling-memory-pattern-lattice-scaling \ + --baseline /private/tmp/docling-memory-pattern-lattice-copied-scaling \ + --sizes 32000,64000 --layouts direct,pattern-lattice \ + --modes decode,skip-shapes,render --repeats 3 \ + --slowdown-limit-pct 5 +~~~ + +Each side has 36 measurements. Compare cumulative copying, RSS and process wall +time together; the four executions are sequential and their allocation totals +must not be reported as simultaneous live memory. + +### User rebuilt-extension semantic gate + +~~~sh +uv run --no-sync python local_build.py + +uv run --no-sync pytest \ + tests/test_unit_pattern_replay.py \ + tests/test_unit_patterns.py \ + tests/test_unit_soft_mask_streaming.py \ + tests/test_unit_soft_masks.py \ + tests/test_unit_transparency.py \ + tests/test_unit_form_streaming.py \ + tests/test_unit_type3_streaming.py \ + tests/test_unit_type3_fonts.py +~~~ + +Review the native and semantic results together before accepting item 3. +Item 4 then closes the state/reset audit and any separately reproduced fixes; +items 5–6 cover broader corpus/threaded validation and source-path retirement. +The permanent benchmark peak-RSS/timeline work still follows parser acceptance. + +## Completion pass — item 3 native scaling accepted + +The user completed cached-tile scaling (54 measurements per executable) and +direct-lattice scaling (36 per executable). All dependency/input and structural +checks passed. All ten scaling PNG pairs were byte-identical. + +Cached-tile decode/render memory fell 26–28%, with 3–7% faster process times. +At 250k, pattern decode fell 780.30 to 566.94 MiB, and repeated-pattern decode +782.27 to 566.97 MiB. The source contains 1,747,566 64-byte token records: +two size-only vector copies predict 213.33 MiB, closely matching the observed +213.36 MiB reduction for one fill. This does not account for every allocator +effect or generalize that exact percentage to other content. + +Lattice decode/render memory fell 11–12%, with 3–6% faster process times. +At 64k, decode fell 467.44 to 413.06 MiB and rendering 476.48 to 420.36 MiB. +Both executables retained the same source cache. The candidate materialized +zero replay vectors; repeated cached tiles decoded once with three cache hits, +and the lattice still executed four placements. + +The cached-tile matrix flagged direct/125k/skip-shapes at +6.26% and the roughly +20 ms pattern-repeat/250k/skip-shapes case at +7.82% with overlapping ranges. +Both bypass pattern replay. The user rechecked the direct control seven times +per executable: 0.274 to 0.276 seconds (+0.77%), with 23.02 to 23.08 MiB +(+0.27%). The earlier direct slowdown did not reproduce. The very short +skipped-pattern timing remains inconclusive; no replay-path slowdown was found. + +The user then said "continue to item 4". This authorizes the audit without +asserting that the pending 90-case semantic suite has passed. + +## Completion pass — item 4: state/reset lifetime audit + +This is a targeted source audit of the parser state, nested content lifetimes, +resource reuse and output ownership relevant to this memory investigation. +It is not a claim that every C++ subsystem has been exhaustively verified. +Codex has not compiled or executed this iteration. + +Static checks passed: MyPy (112 source files), Ruff lint, Ruff formatting +(117 files) and git diff --check. These do not establish native compilation, +exception-test success, output equivalence or runtime overhead. + +### Audit findings and classification + +| Area | Verified reset/ownership or finding | Classification / action | +| --- | --- | --- | +| Direct dash replacement | d clears the array before replacement; null/error assigns an empty array. | Existing #452 fix is retained; no new normal-path append defect. | +| ExtGState dash parsing | A fresh local array is assigned to the supplied /D field. Missing ExtGState fields preserve existing properties. | Correct replacement/partial-update behavior in inspected paths. | +| Current paths | register_paths, n and discard_paths clear scratch paths after their prescribed painting/clipping work. | Existing resets retained. | +| Clip history | capture_pending_clip appends successive intersections; get_clip_state copies the current history into each painted instruction. | Intentional state with a structural quadratic snapshot case; bounded native fixtures added below. No history clear introduced. | +| Graphics-state copies | GLOBAL copy construction rebinds state references to the new frame; assignment preserves destination resources. SHAPE copies current paths/clip history. | Required scoped state copying; retain resource and q/Q semantics. | +| Internal graphics saves | Form and pattern/shading paths restored manually only on normal return. | Confirmed exception-restoration gaps; scope guards added. | +| Pattern recursion bookkeeping | Lattice depth had no exceptional cleanup. Cached-cell flags/depth could survive a nonstandard exception or a failure while logging/allocating fallback output. | Confirmed exceptional state lifetime defects; scope guards added. | +| Pattern source-cache completion | stream_parsed_ was set before parsing. Allocation failure was caught as unreadable input and could leave a failed attempt treated as complete. | Commit the flag after completion; release partial tokens and propagate allocation/nonstandard failures. | +| Type3 recursion and placement | Existing catch-all cleanup already restored the glyph depth and parent frame. | Moved to the same scope guards; no newly discovered normal-path defect claimed. Native regression preserves that guarantee. | +| Pending Type3 procedures | paint_type3_glyphs swaps work into a local vector before nested state copies. TEXT assignment omits the transient queue; actual child rebase starts with a fresh state. | No independent successful-show accumulation found. Partial text-generation failures are not claimed exhaustively covered. | +| Inline-image scratch | BI replaces the entry; completed/incomplete EI replaces it again. A throwing bitmap decode can retain one entry until the next BI/decoder destruction. | Bounded scratch retention, without a confirmed per-image accumulation or quadratic copy defect. | +| Marked content and operands | EMC pops scopes; logical-stream finish clears unbalanced marked content. Handled operators clear pending operands. | Existing logical stream boundaries retained. Large explicit operands still require their own storage. | +| Resource setters | Inspected collection setters construct fresh resources before map replacement; parent-linked maps preserve inherited ownership. | A missing reset in a hypothetical reused individual resource is not established on these production callers. | +| Shape JSON initialization | PAGE_SHAPES clears/resizes fresh destination entries before optional fields are read. | Optional member reuse was not reproduced on this inspected path. | +| stack_count | Historical cumulative q counter is also emitted as text-cell metadata; Q does not decrement it. | Not treated as live stack depth; unchanged to preserve extracted metadata. Scope guards restore actual frames. | +| Page/document ownership | Page decoders are intentionally cached; unload_page/unload_pages erase those owners. Threaded document unload removes document/schedule owners; held results can retain instructions. | No missing-clear defect established by these ownership paths. Multi-page/threaded RSS remains item 5 validation. | +| Font, mask and pattern caches | Glyph data, decoded masks, parsed pattern tokens and decoded tiles have explicit reuse owners. Unused eager Type3 descriptions were removed in item 1. | Intentional caches retained; token-copy costs are measured separately from cache residency. | + +No second normal-path setter bug equivalent to dash_array's old missing clear +was confirmed. The exception defects above and the clip snapshot structure are +separate findings with separate evidence/validation requirements. + +### Implemented exception-restoration patch + +An internal graphics_state_scope records the current frame count, calls q, +and removes the frames it owns on normal or exceptional exit. Its destructor +performs no allocation, logging, state copying or Q call. Explicit restore() +retains the old normal-return lifetime at each site. The historical stack_count +behavior and actual PDF q/Q operator implementation remain unchanged. + +The guard now covers ordinary Forms, lattice parent/placement scopes, shading +and Coons pattern scopes, and procedural Type3 parent placement. Child decoder +stacks remain independently owned and die with their decoder. + +A nesting_scope saves/restores the prior thread-local depth and, for cached +tiles, the prior resource decoding flag. It protects lattice, cached-tile and +Type3 interpretation even if a later catch block itself throws. + +Cached tile interpretation now propagates std::bad_alloc. Pattern source +parsing commits stream_parsed_ after completion or handled unreadable input; +allocation/nonstandard failures discard partial cache records/capacity and +leave parsing retryable. Ordinary unreadable-input behavior is retained. + +These fixes protect recovery/state ownership. They do not promise lower peak +RSS for normal documents or an additional normal-path quadratic-memory fix. + +### Native failure-injection regression + +New optional target/CTest: `stream_state_lifetime_regression`, source +`tests/cpp/test_stream_state_lifetimes.cpp`. + +It has 14 scenario groups, covering recoverable runtime errors, allocation +failure and a nonstandard exception during cached tiles, direct lattice, +ordinary Forms and Type3 procedures, plus failed source-cache parsing/retry. +Glyph scenarios repeat failures beyond the recursion cap before trying again. + +Assertions check parent frame count, matrix, clip isolation, bitmap glyph mode, +rendered suffix coordinates, pattern depth/flags and preserved exception types. +Source parse failures seed a partial record, then require an empty/uncommitted +cache and a successful retry with the original source. + +Private fault checkpoints are compiled only with +DOCLING_PARSE_STATE_LIFETIME_TESTING in this dedicated target. They are absent +from production and every memory-probe executable. Failures are deterministic; +the tests do not exhaust actual memory. Assertions remain active in Release. +No runtime pass is claimed until the user compiles and executes it. + +### Clip snapshot experiment + +New opt-in layouts `clip-chain` and `clip-scoped` use N independent strokes and +the same full-page rectangular clip before every stroke: + +- clip-chain leaves all intersections active. The instruction count is N, + but retained clip-path occurrences total N(N+1)/2, with 5N(N+1)/2 points. +- clip-scoped saves/restores each clip-and-paint with q/Q. Each instruction + retains one clip, so totals are N paths and 5N points. + +Both layouts should have identical pixels. The harness checks those PNG pairs, +the exact geometry/source counters and the clip totals. Source object counts +are 14N versus 16N (the scoped version adds q/Q); these are linear. The fixture +target is capped at 4000 lines to keep a diagnostic run bounded. + +This demonstrates the distinction between a linear instruction count and a +quadratic copied payload. Generic clipping intersections cannot simply be +cleared. A future shared/persistent clip representation or semantic clip +canonicalization would be a separate optimization, informed by the measured +importance of this case in real documents. No clip representation change is +included here. Actual RSS/timing curves remain to be measured by the user. + +### Preserved reference and normal-path comparison + +Before modifying the executable, Codex preserved the existing probe: + +- `/private/tmp/docling-memory-before-state-lifetimes` +- SHA-256: 959dbf38567d8d6873f50e27111ec2e6f088d4ee54db7876d35bc21fbe0c1994 + +It exactly matches the executable in the successful pattern control-after +bundle. Current dependency archives were also checked against that bundle: +none had changed. The original bundle retains its compiled-source provenance; +the current source snapshots recorded by a run of the preserved executable +are not a claim about that old binary's source. + +Before rebuilding, collect the normal-path reference while those dependencies +still match: + +~~~sh +uv run python3 tests/cpp/run_memory_on_lines.py \ + --exe /private/tmp/docling-memory-before-state-lifetimes \ + --output /private/tmp/docling-memory-state-lifetimes-before \ + --sizes 125000 --layouts direct,form,type3,pattern \ + --modes decode,skip-shapes,render --repeats 3 +~~~ + +This has 36 measurements. The existing dependency checks remain mandatory. +If rebuilding changes archives, do not relabel the preserved binary or bypass +the hashes: rebuild an isolated pre-audit reference against the candidate's +dependencies before comparing again. + +### User build and focused native gate + +~~~sh +cmake -S . -B build -DDOCLING_PARSE_BUILD_MEMORY_PROBE=ON + +cmake --build build --parallel 2 --target \ + test_memory_on_lines stream_callback_regression stream_state_lifetime_regression + +ctest --test-dir build --output-on-failure \ + -R '^stream_(callback|state_lifetime)_regression$' +~~~ + +After the tests pass, compare normal-path memory/runtime and images: + +~~~sh +uv run python3 tests/cpp/run_memory_on_lines.py \ + --output /private/tmp/docling-memory-state-lifetimes-after \ + --baseline /private/tmp/docling-memory-state-lifetimes-before \ + --sizes 125000 --layouts direct,form,type3,pattern \ + --modes decode,skip-shapes,render --repeats 3 \ + --slowdown-limit-pct 5 +~~~ + +Require unchanged geometry/PNGs, ordinary memory behavior and runtime within +the accepted budget. This comparison isolates item 4, not the older Type3 +eager-cache cleanup whose historical dependency mismatch remains unresolved. + +### User clipping smoke and bounded scaling + +~~~sh +uv run python3 tests/cpp/run_memory_on_lines.py \ + --output /private/tmp/docling-memory-clip-history-smoke \ + --layouts clip-chain,clip-scoped --sizes 100 --repeats 1 + +uv run python3 tests/cpp/run_memory_on_lines.py \ + --output /private/tmp/docling-memory-clip-history-scaling \ + --layouts clip-chain,clip-scoped --sizes 250,500,1000 \ + --modes decode,skip-shapes,render --repeats 3 +~~~ + +The smoke has ten measurements and one equivalent-layout PNG pair. Scaling +has 54 measurements and three equivalent-layout PNG pairs. Inspect retained +clip paths/points/bytes alongside process peak RSS, not instruction count alone. +The shapes-disabled cases must retain no render-instruction clip snapshots. + +### User rebuilt-extension semantic gate + +The item-3 semantic result was not supplied before item 4 was authorized. +Run its existing eight-file suite after rebuilding this version, and include +clipping, damaged-stream and pattern-related coverage: + +~~~sh +uv run --no-sync python local_build.py + +uv run --no-sync pytest \ + tests/test_unit_pattern_replay.py tests/test_unit_patterns.py \ + tests/test_unit_soft_mask_streaming.py tests/test_unit_soft_masks.py \ + tests/test_unit_transparency.py tests/test_unit_form_streaming.py \ + tests/test_unit_type3_streaming.py tests/test_unit_type3_fonts.py \ + tests/test_unit_clipping.py tests/test_unit_damaged_content_streams.py \ + tests/test_unit_split_content_streams.py tests/test_unit_actual_text.py \ + tests/test_unit_appearance_streaming.py tests/test_unit_widget_appearances.py +~~~ + +This closes the pending item-3 semantics and checks the revised state lifetimes +against existing outputs. Native compilation/tests, clipping measurements and +these semantic results are user gates before accepting item 4. Broader corpus +and actual threaded ownership/memory validation remain item 5. + +## Item-4 native results and clipping growth confirmation + +The user supplied passing CTest results for stream_callback_regression and +stream_state_lifetime_regression (0.81 seconds total). The paired normal-path +comparison completed 36 measurements per executable and passed all structural, +dependency and PNG checks. All four PNG pairs were identical, with no timing +review flags. The largest reported positive runtime change was +3.94%; positive +peak changes were below 0.8 MiB. This accepts native overhead for the scope +guards, without claiming a rebuilt-extension semantic result. + +The clip-history scaling bundle completed 54 measurements and passed all three +equivalent-layout PNG comparisons. Its results confirm the structural case: + +| N | Instructions | Chain clip occurrences | Scoped occurrences | Chain minus scoped decode MiB | +| ---: | ---: | ---: | ---: | ---: | +| 250 | 250 | 31,375 | 250 | 5.89 | +| 500 | 500 | 125,250 | 500 | 23.15 | +| 1,000 | 1,000 | 500,500 | 1,000 | 91.56 | + +Decoded chain peaks were 16.81, 34.42 and 103.64 MiB; scoped peaks 10.92, 11.27 +and 12.08 MiB. Render peaks followed the same growth (26.97, 44.42, 113.73 MiB +versus 21.17, 21.56, 22.25 MiB). Shapes-disabled peaks stayed about 10–11 MiB. +The chain's accounted clip payload was 4,518,000, 18,036,000 and 72,072,000 +bytes, with 156,875, 626,250 and 2,502,500 point occurrences. + +Source objects remain linear (14N for chain, 16N for scoped). Render instruction +counts are N; full copied histories account for N(N+1)/2 path occurrences. +Subtracting the scoped control gives approximately fourfold extra decode RSS +on each doubling, consistent with that exact structural count. This is a +specific native snapshot-retention finding, not a general attribution of all +high-memory PDFs to clipping. + +## Clipping follow-up — share immutable history + +The user approved proceeding after reviewing the clipping results. Codex +implements; builds, experiments and runtime correctness checks remain with +the user. This follow-up is separate from the already measured scope guards. + +### Representation and ownership + +Each nonempty clip capture becomes one immutable node containing its subpaths +and a shared predecessor. A paint snapshot and a q/Q state save retain the +relevant history head. Appending creates a new node; earlier snapshots and +restored branches keep their existing heads. Capture geometry is owned once +per node, independent of the number of later paints. + +SHAPE now owns this shared history instead of deep-copying a page-shape clip +list and a parallel group-id list. The clip bbox/intersection calculations, +capture order, per-subpath group ids and existing snapshot rule value remain. +get_clip_state() returns the shared prefix with the current rule. No clipping +intersection is cleared or deduplicated by this change. + +get_paths() exposes a read-only forward range rather than a vector. Existing +repository consumers were adapted: iteration, size and empty queries do not +retain flattened geometry. Explicit to_vector() is available for intentional +materialization and the temporary native reference. Code requiring a concrete +vector must request that copy explicitly; Python-facing APIs are unchanged. + +Traversal preserves oldest-capture and within-capture subpath order. Its +iterator uses inline storage for up to 16 capture pointers and transient +pointer storage for longer histories. No flattened cache lives in a snapshot. +Long uniquely owned suffixes are destroyed iteratively to avoid recursive +shared_ptr teardown exhausting the native stack. + +Shifted instruction replay shares a translation cache for that replay. A +prefix is translated once for a given offset; the cache retains source owners +as well as translated nodes so source addresses cannot be reused as keys. +Changing the offset clears that cache. Independently retained translated +snapshots remain valid. Translation does not mutate original coordinates. + +Renderer readers retain their existing clipping/rasterization algorithms. +Clip equality first checks shared-history identity, then compares ordered paths +when needed. The existing group-size map replaces a redundant nested scan +for same-group subpaths. No new rectangle-intersection simplification or +clip-mask representation is introduced. + +Storage for a simple N-capture chain is now N nodes/N unique paths, while all +N paints still logically see N(N+1)/2 path occurrences. Rasterization may +still traverse growing histories; this patch does not claim linear render +runtime. Memory and the accepted runtime budget require measurement. + +### Controlled reference and diagnostics + +New optional executable: `test_memory_on_lines_copied_clips`, with only +DOCLING_PARSE_COPY_CLIP_SNAPSHOTS enabled. It materializes a complete history +for each paint and uses per-instruction translated snapshots. All source-token +delivery, scope guards and renderer readers otherwise match the candidate. +The fully buffered reference also enables this selector until final retirement. + +Use a fresh paired reference/candidate run against matching current libraries. +This target isolates snapshot copying; it does not reproduce every historical +internal allocation of the old mutable clip-state representation. + +The probe now distinguishes: + +- instruction_clip_paths / instruction_clip_points: logical occurrences seen + by all instructions, preserving the earlier semantic counters. +- instruction_clip_logical_bytes: cumulative path-record/coordinate payload + corresponding to those logical occurrences. +- instruction_clip_unique_paths / instruction_clip_unique_points: geometry + retained in distinct history nodes, counted once across snapshots. +- instruction_clip_history_nodes: distinct retained history nodes. +- instruction_clip_bytes: accounted unique node/vector/coordinate storage. + This excludes shared_ptr control blocks and allocator overhead and is not + process peak RSS. Earlier bundles used this field for copied vector payload; + use clip_history_storage and the new fields to interpret the schema. + +Accounting visits each distinct node once, stopping at already counted prefixes. +Per-node cumulative totals avoid walking every logical occurrence during +diagnostics. Thus the probe itself does not flatten or recreate a quadratic +history while measuring the shared representation. + +decode_work_s and render_work_s isolate native page decoding and rendering +(including canvas flush) from the later diagnostics/JSON/image save work. +Their medians are added to summary.csv. The existing process-wall comparison +continues unchanged; inspect both focused work and process timing before +attributing a change to production execution. + +### Added regression coverage and static checks + +New CTest target `clip_history_regression`, source tests/cpp/test_clip_history.cpp: + +1. Earlier snapshots and independent branches retain their own geometry/rule. +2. Forward traversal preserves capture and same-group subpath order, including + a copied iterator on a history longer than its inline pointer buffer. +3. One thousand prefix snapshots expose 500,500 logical paths but retain only + 1,000 nodes and 1,000 unique paths. +4. Translation preserves originals, shares translated prefixes, and handles + offset changes without reusing stale coordinates. Shifted instruction replay + exercises the shared cache through the real dispatch path. +5. A 50,000-capture unique suffix can be released without recursive teardown. + +tests/test_unit_clipping.py adds W/W* pixel cases: an early paint sees a broad +clip; a q/Q child branches left; the restored parent branches right. These +check snapshot immutability and restored-branch isolation. Existing curved, +multi-subpath, image, shading, mask and appearance suites remain necessary. + +MyPy passes for 112 source files; Ruff lint and formatting pass for 117 files. +Codex has not built or run these native/semantic tests or the new experiments. + +### User build and native regression gate + +~~~sh +cmake -S . -B build -DDOCLING_PARSE_BUILD_MEMORY_PROBE=ON + +cmake --build build --parallel 2 --target \ + test_memory_on_lines test_memory_on_lines_copied_clips \ + clip_history_regression stream_callback_regression stream_state_lifetime_regression + +ctest --test-dir build --output-on-failure \ + -R '^(clip_history|stream_callback|stream_state_lifetime)_regression$' +~~~ + +### User paired smoke + +~~~sh +uv run python3 tests/cpp/run_memory_on_lines.py \ + --exe build/test_memory_on_lines_copied_clips \ + --output /private/tmp/docling-memory-shared-clips-copied-smoke \ + --layouts direct,clip-chain,clip-scoped --sizes 100 --repeats 1 + +uv run python3 tests/cpp/run_memory_on_lines.py \ + --output /private/tmp/docling-memory-shared-clips-smoke \ + --baseline /private/tmp/docling-memory-shared-clips-copied-smoke \ + --layouts direct,clip-chain,clip-scoped --sizes 100 --repeats 1 +~~~ + +Each side has 15 measurements. Require matching PDFs/PNGs/logical counts. +For N=100, the chain must retain 5,050 unique paths in the copied reference +and 100 in the shared candidate. Both expose 5,050 logical occurrences and +100 render instructions. The scoped layout has 100 unique paths on both sides. + +### User paired clipping scaling after smoke + +~~~sh +uv run python3 tests/cpp/run_memory_on_lines.py \ + --exe build/test_memory_on_lines_copied_clips \ + --output /private/tmp/docling-memory-shared-clips-copied-scaling \ + --layouts clip-chain,clip-scoped --sizes 250,500,1000 \ + --modes decode,skip-shapes,render --repeats 3 + +uv run python3 tests/cpp/run_memory_on_lines.py \ + --output /private/tmp/docling-memory-shared-clips-scaling \ + --baseline /private/tmp/docling-memory-shared-clips-copied-scaling \ + --layouts clip-chain,clip-scoped --sizes 250,500,1000 \ + --modes decode,skip-shapes,render --repeats 3 \ + --slowdown-limit-pct 5 +~~~ + +Each side has 54 measurements. At N=1000, logical clip paths remain 500,500, +but unique retained paths should drop to 1,000. Seek linear unique storage and +lower measured RSS, with unchanged pixels and runtime within the accepted 5%. +Inspect decode/render work medians as well as process-wall results. Do not +raise the existing fixture cap while the copied reference remains quadratic. + +### Ordinary-page overhead reference + +Codex preserved the executable used for the successful state-lifetime comparison: + +- `/private/tmp/docling-memory-before-shared-clips` +- SHA-256: ca7fb602142ff954be45493930a3b09b854262d3f1728060fe3f3389294eeae6 + +Its binary hash matches /private/tmp/docling-memory-state-lifetimes-after, and +the dependency archive hashes were unchanged at preservation. Original compiled +source provenance remains in that bundle; later current-source snapshots do +not relabel the old executable. Compare ordinary pages after rebuilding: + +~~~sh +uv run python3 tests/cpp/run_memory_on_lines.py \ + --output /private/tmp/docling-memory-shared-clips-controls \ + --baseline /private/tmp/docling-memory-state-lifetimes-after \ + --sizes 125000 --layouts direct,form,type3,pattern \ + --modes decode,skip-shapes,render --repeats 3 \ + --slowdown-limit-pct 5 +~~~ + +This supplies 36 measurements. Retain the normal dependency safeguards. If +libraries change, obtain a reference built against matching libraries; do not +bypass hashes or treat this as the older unresolved Type3 cleanup comparison. +The preserved executable is available for focused timing rechecks if needed. + +### User rebuilt-extension semantic gate + +~~~sh +uv run --no-sync python local_build.py + +uv run --no-sync pytest \ + tests/test_unit_clipping.py tests/test_unit_pattern_replay.py \ + tests/test_unit_patterns.py tests/test_unit_soft_mask_streaming.py \ + tests/test_unit_soft_masks.py tests/test_unit_transparency.py \ + tests/test_unit_form_streaming.py tests/test_unit_type3_streaming.py \ + tests/test_unit_type3_fonts.py tests/test_unit_damaged_content_streams.py \ + tests/test_unit_split_content_streams.py tests/test_unit_actual_text.py \ + tests/test_unit_appearance_streaming.py tests/test_unit_widget_appearances.py +~~~ + +The prior item-3/item-4 semantic result still has not been reported. This suite +must run against the rebuilt extension before accepting the shared-history +patch. Broader corpus/threaded validation, comparison-path retirement and the +permanent benchmark peak-RSS/timeline enhancement remain subsequent work. + +## Shared clip history — measured result and renderer traversal follow-up + +Date: 2026-11-12. Continuation of the authorized item-4 shared-history change. +Builds, native tests, Python semantic tests and experiments remain with the user. + +### Native memory and pixel evidence + +The paired smoke bundles each passed 15 measurements: + +- `/private/tmp/docling-memory-shared-clips-copied-smoke` +- `/private/tmp/docling-memory-shared-clips-smoke` + +All three baseline PNG pairs matched. At 100 clips, both representations expose +5,050 logical clip paths; unique retained paths drop from 5,050 to 100. + +The paired scaling bundles each passed 54 measurements: + +- `/private/tmp/docling-memory-shared-clips-copied-scaling` +- `/private/tmp/docling-memory-shared-clips-scaling` + +All six baseline PNG pairs matched. At 1,000 clips, chain decode peak RSS fell +from 93.92 to 12.19 MiB (-87.02%), and render peak from 104.20 to 22.88 MiB +(-78.05%). The scoped control stayed near 12 MiB decode and 22 MiB render. +The chain still exposes 500,500 logical paths, but retains 1,000 unique paths; +accounted unique node/vector/coordinate storage drops from 72,144,000 to +216,000 bytes. These accounted bytes exclude allocator/control-block overhead. +This confirms the intended linear retained geometry on this fixture. It does +not establish linear rasterization work. + +### Seven-repeat recheck: process gate passes, render-phase gate remains open + +The new bundles each passed 28 measurements, with all four baseline PNG pairs +and both chain/scoped PNG equivalence checks matching: + +- `/private/tmp/docling-memory-shared-clips-render-recheck-before` +- `/private/tmp/docling-memory-shared-clips-render-recheck-after` + +| Layout / clips | Peak MiB before → after | Process ms before → after | Render work ms before → after | Render work change | +| --- | ---: | ---: | ---: | ---: | +| Chain / 500 | 42.00 → 22.00 | 83.94 → 74.55 | 56.47 → 53.03 | -6.10% | +| Scoped / 500 | 21.75 → 21.78 | 68.55 → 66.23 | 46.52 → 46.06 | -1.00% | +| Chain / 1,000 | 104.06 → 22.95 | 137.44 → 110.59 | 80.56 → 86.10 | +6.88% | +| Scoped / 1,000 | 22.42 → 22.48 | 69.41 → 69.11 | 45.01 → 43.26 | -3.90% | + +At 1,000 chain clips, native decode work falls from 23.35 to 5.27 ms (-77.44%). +The render work ranges do not overlap: 77.72–82.28 ms before and 84.70–90.82 ms +after. The earlier three-repeat matrix showed the same direction (+7.06%). +Treat this as a remaining regression to address, despite the 19.54% process +improvement and 77.94% peak reduction. A passed process-wall comparison is not +a passed render-work gate; the runner currently flags process time only. + +### Hypothesis and bounded source change + +The immutable forward range reconstructs a temporary stack of predecessor +pointers for each traversal. Renderer helpers also rebuild clip-group counts +several times for a shape. This is a source-based performance hypothesis; +profiling has not established how much each operation contributes. + +`clip_path_cursor` keeps one active traversal per renderer: + +1. Ordered pointers refer to immutable path geometry. A shared head owner keeps + the referenced history alive; path coordinates are not copied. +2. The same head reuses the traversal. A direct successor appends just its new + capture. A skipped sequence appends its suffix in capture order. +3. A restored ancestor, sibling, independent history or translation rebuilds + the ordered references and group counts from the requested history. +4. Group counts include repeated group IDs across captures, as shading BBox + paths can reuse a group. Rule changes still use the instruction's rule. +5. A failed allocation clears partially updated state and rethrows. No partial + pointer list can be reused as a valid traversal. +6. `renderer::set_size()` releases the cursor before its invalid-size early + return. Space is proportional to one active history, not every snapshot. + +The rectangular clipping, non-rectangle detection, mask building, clip bounds +and shading helpers use this traversal. The geometry operations, capture order +and fill-rule selection remain the same. Helpers do not hold cursor iterators +across nested rendering calls; each helper prepares the requested history. +The independent comparison of two clip states keeps its existing iterators. + +The existing `clip_history_regression` adds checks for one/multiple-capture +extensions, address/order preservation, repeated group IDs, ancestor/sibling +restoration, translations, unrelated/empty histories, reset/reuse and source +owner release. These are written but have not been compiled or executed by Codex. + +### Preserved executable and next controlled user run + +Codex preserved the measured shared-history executable before rebuilding: + +- `/private/tmp/docling-memory-before-clip-cursor` +- SHA-256: `60e3984699ed1e23e72efc680412f6281f6a2d8015a72094d00cbf2f31fdf596` + +Its binary hash matches the recheck-after metadata. All nine dependency archive +hashes matched at preservation. The original compiled-source provenance stays +in the recheck-after bundle; a new runner snapshot does not relabel that binary. + +~~~sh +cmake --build build --parallel 2 --target \ + test_memory_on_lines clip_history_regression + +ctest --test-dir build --output-on-failure -R '^clip_history_regression$' + +uv run python3 tests/cpp/run_memory_on_lines.py \ + --exe /private/tmp/docling-memory-before-clip-cursor \ + --output /private/tmp/docling-memory-clip-cursor-before \ + --layouts clip-chain,clip-scoped --sizes 500,1000 \ + --modes render --repeats 7 + +uv run python3 tests/cpp/run_memory_on_lines.py \ + --output /private/tmp/docling-memory-clip-cursor-after \ + --baseline /private/tmp/docling-memory-clip-cursor-before \ + --layouts clip-chain,clip-scoped --sizes 500,1000 \ + --modes render --repeats 7 --slowdown-limit-pct 5 +~~~ + +Inspect both process-wall and `median_render_work_s` from summary.csv/raw JSON. +The fresh pair isolates the cursor, since both sides share clip geometry. +Also compare candidate render work against the original copied-history recheck; +keep the original +5% allowance. If dependencies change, rebuild a matching +reference instead of bypassing identity checks. Require matching PNGs/logical +counts and retained linear unique geometry, with no material new peak cost. + +At this handoff, the ordinary-page controls, semantic suite and new native +clip-history CTest were awaiting results. The checkpoint below records those +results. Broad corpus/threaded acceptance, comparison-path retirement and +permanent benchmark memory reporting remain subsequent work. + +## Focused acceptance checkpoint — cursor, ordinary-page controls and semantics + +Date: 2026-11-12. The user returned the cursor comparison, ordinary-page +controls and the rebuilt-extension semantic results, and explicitly accepted +the control table's overlapping timing recheck as a fluke. + +### Renderer cursor result + +Both bundles contain 28 successful measurements: + +- `/private/tmp/docling-memory-clip-cursor-before` +- `/private/tmp/docling-memory-clip-cursor-after` + +All four baseline PNG pairs and both chain/scoped equivalence checks match. +Dependencies and probe settings match the original copied-history recheck too. + +| Layout / clips | Peak MiB before → after | Process change | Render work ms before → after | Render work change | +| --- | ---: | ---: | ---: | ---: | +| Chain / 500 | 22.08 → 21.75 | -14.80% | 55.60 → 46.04 | -17.20% | +| Scoped / 500 | 21.72 → 21.89 | +1.66% | 45.44 → 45.55 | +0.24% | +| Chain / 1,000 | 22.86 → 22.52 | -25.98% | 86.04 → 56.92 | -33.85% | +| Scoped / 1,000 | 22.47 → 22.66 | +1.56% | 44.77 → 47.61 | +6.34% | + +For the 1,000-clip chain, the render-work ranges no longer overlap in the +direction of improvement: 83.96–88.95 ms before versus 55.66–63.93 ms after. +Compared with the original copied-history recheck, peak RSS is 78.36% lower, +process time is 40.78% lower and render work is 29.35% lower. + +The scoped 1,000-clip render-work change is 2.84 ms, with overlapping ranges +(40.39–47.79 ms versus 44.18–48.69 ms) and only +1.56% process time. Preserve +that observation rather than claiming every individual phase median is within +5%. No further focused timing recheck is scheduled at this checkpoint; broad +workload performance remains part of item 5. + +The saved `build/Testing/Temporary/LastTest.log` records +`clip_history_regression` passing in 0.50 seconds on Nov 12 at 07:06 CET. +Codex read that log; Codex did not launch the native regression. + +### Ordinary-page controls + +Bundle: `/private/tmp/docling-memory-shared-clips-controls`. +Reference: `/private/tmp/docling-memory-state-lifetimes-after`. +All 36 measurements completed, with matching dependencies/settings, unchanged +logical output counters, and four byte-identical PNG pairs. + +Direct, Form and pattern peaks stay close to their references. Type3 decode +peak falls from 175.22 to 134.81 MiB (-23.06%), and render peak from 180.34 +to 140.11 MiB (-22.31%). Both sides have zero instruction clip paths on that +Type3 control: do not attribute this observation to sharing nonempty retained +clip geometry or use it to claim an isolated eager-font-cleanup measurement. + +All process comparisons are within the configured +5% budget except pattern +with shapes disabled: approximately 19 to 20 ms, +5.86%, with overlapping +ranges. The user explicitly accepted that recheck as noise. No further +repetitions are requested for that case. + +### Focused semantic acceptance + +The user reported **131 passed in 5.83 seconds**, covering: + +- Clipping, including both new W/W* snapshot/branch pixel cases. +- Pattern replay and the existing pattern suite. +- Soft-mask streaming, masks and transparency. +- Form streaming and procedural Type3 streaming/fonts. +- Damaged/split content streams and ActualText. +- Appearance streaming and widget appearances. + +This closes the previously unreported semantic gates for completion items 3 +and 4. Keep the accepted code and its focused regression coverage. This does +not mark the full parser plan complete or authorize retiring references before +broader validation. + +### Item 5 handoff — existing broad validation first + +No new production change is needed to run the next checks. The user keeps +execution; use the existing extension built for the 131-test suite: + +~~~sh +uv run --no-sync pytest tests/test_unit_*.py -q + +uv run --no-sync pytest -n 2 --dist loadfile tests/test_regression_*.py +~~~ + +The second command is the repository's documented CI-style regression command. +It includes sequential corpus parsing, threaded parsing/rendering, real-page +image checks, embedded fonts and locale coverage. The tests use the pinned +regression snapshot and selected corpus pages; native parser thread counts +inside those tests are independent of pytest's two workers. Existing threading +tests cover one, two and four parser threads plus backpressure and unloading. +Keep the existing ground truth and tolerances when assessing failures. + +After correctness passes, collect current real-workload scaling with the +existing launcher, before changing its memory reporting: + +~~~sh +REPEATS=3 THREADS=1,2,4 RUN_COMPARISON=0 DATASET=bo20 \ + ./scripts/benchmarking/refresh_performance_benchmarks.sh +~~~ + +This supplies current native throughput/counts and fresh-process backend replay +peak RSS. The replay already records process peaks in each report.json. Native +peak RSS and sampled timelines are not yet implemented in that launcher. +The older Nov-11 bundle is historical context, not a matched before/after pair +for all later source/dependency changes. Assess any historical comparison only +after checking settings, libraries, inputs and compiled-source provenance. + +Item 1's isolated performance attribution remains unresolved because the +historical reference had changed dependencies. Item 5 still needs the broad +results above and any remaining controlled real-document evidence. Item 6 +retires comparison selectors/dead APIs after that acceptance. The requested +permanent benchmark peak-RSS/timeline implementation follows parser completion. + +## Finish parser acceptance before benchmark tooling changes + +The user reaffirmed this order explicitly: finish comparison-code cleanup, +post-merge workload scaling/memory validation and isolated Type3 cleanup +attribution first. Do not modify recover_performance_benchmarks.py or the +permanent benchmark memory-reporting implementation during this completion pass. +The original authorization to finish items 1–6 remains in effect; the user +continues to compile and execute all tests/experiments. + +### Post-merge correctness result + +The branch includes main through merge 852d6b6. The extension initially still +predated the merged font headers. After the user rebuilt it, the 38 tests in +test_unit_truetype_macroman.py and test_unit_gid_glyph_names.py passed in 0.89 s. +The following corpus tests all passed in 795.71 s: + +- test_regression_parse.py::test_reference_documents_from_filenames +- test_regression_threaded_parse.py::test_threaded_reference_documents_from_filenames +- test_regression_threaded_render.py::test_render_reference_documents_from_filenames + +This resolves the reported degree/upsilon and bullet/infinity failures on the +merged build. Codex made no font-decoding or regression-dataset changes in that +investigation. The pre-merge ground-truth hash observations do not describe the +newly downloaded post-merge snapshot. Do not use them to explain the new run. + +### Close item 1 with a matching current-library reference + +The old preserved executable still cannot isolate the cleanup against changed +libraries. A final temporary target, test_memory_on_lines_eager_type3_fonts, +now uses the current interpreter, font-decoding fixes, clip representation and +dependency archives. Its generated include overlay restores only the old eager +CharProc initializer and the retained name_to_descr member. Historical source +comes from pinned commit 56341b17c22fb6b579171bc0dfd443ea485944f6; it does not +replace the whole current font implementation. + +tests/cpp/prepare_type3_cleanup_reference.py creates the overlay during the +user's build. It records original/current/generated header hashes and historical +helper hashes in build/type3-font-reference/manifest.json. A compile-time marker +rejects a reference build that accidentally selects the normal font header. +The runner saves this manifest, generated header and historical helpers with +the result bundle and checks them before accepting measurements. Existing PDF, +pixel, geometry and dependency comparison guards remain enabled. + +The generated change was reviewed in memory. MyPy, Ruff lint and formatting +passed for the two changed Python files; git diff --check passed too. +Codex has not configured CMake, compiled or run the new reference. + +User build and small paired comparison (nine measurements on each side): + +~~~sh +cmake -S . -B build -DDOCLING_PARSE_BUILD_MEMORY_PROBE=ON +cmake --build build --parallel 2 --target \ + test_memory_on_lines test_memory_on_lines_eager_type3_fonts + +uv run python3 tests/cpp/run_memory_on_lines.py \ + --exe build/test_memory_on_lines_eager_type3_fonts \ + --output /private/tmp/docling-memory-type3-cleanup-controlled-before-smoke \ + --layouts direct,type3,type3-repeat --sizes 1000 \ + --modes decode,skip-shapes,render --repeats 1 + +uv run python3 tests/cpp/run_memory_on_lines.py \ + --output /private/tmp/docling-memory-type3-cleanup-controlled-after-smoke \ + --baseline /private/tmp/docling-memory-type3-cleanup-controlled-before-smoke \ + --layouts direct,type3,type3-repeat --sizes 1000 \ + --modes decode,skip-shapes,render --repeats 1 +~~~ + +After the smoke passes, collect 54 measurements on each side: + +~~~sh +uv run python3 tests/cpp/run_memory_on_lines.py \ + --exe build/test_memory_on_lines_eager_type3_fonts \ + --output /private/tmp/docling-memory-type3-cleanup-controlled-before \ + --layouts direct,type3,type3-repeat --sizes 125000,250000 \ + --modes decode,skip-shapes,render --repeats 3 + +uv run python3 tests/cpp/run_memory_on_lines.py \ + --output /private/tmp/docling-memory-type3-cleanup-controlled-after \ + --baseline /private/tmp/docling-memory-type3-cleanup-controlled-before \ + --layouts direct,type3,type3-repeat --sizes 125000,250000 \ + --modes decode,skip-shapes,render --repeats 3 --slowdown-limit-pct 5 +~~~ + +Require matching PNGs and logical output counters. Compare peak RSS and process +time, with direct pages as the control. Both sides already use callback Type3 +interpretation: this pair isolates eager font initialization/retention only. +The repeated layout paints the same glyph four times; retain that interpretation +when assessing memory attributable to one font versus painted output. + +### Item 1 accepted — controlled eager-font attribution + +The user compiled the reference and candidate and returned both smoke and +scaling results. The smoke passed nine measurements on each side, with three +identical PNG pairs. Scaling passed 54 measurements on each side, with matching +dependency/settings identities, matching logical output counters, six identical +PNG pairs and no timing-review cases. Both bundles record source HEAD 852d6b6 +and their working-tree/source/reference provenance: + +- `/private/tmp/docling-memory-type3-cleanup-controlled-before` +- `/private/tmp/docling-memory-type3-cleanup-controlled-after` + +Representative medians at target 250,000 lines: + +| Layout / mode | Before MiB | After MiB | Memory change | Process-time change | +| --- | ---: | ---: | ---: | ---: | +| Type3 / decode | 794.92 | 219.72 | -72.36% | -9.75% | +| Type3 / skip shapes | 672.39 | 126.41 | -81.20% | -10.88% | +| Type3 / render | 793.70 | 222.23 | -72.00% | -9.26% | +| Type3 repeated / decode | 849.48 | 553.80 | -34.81% | -4.51% | +| Type3 repeated / skip shapes | 672.41 | 126.48 | -81.19% | -4.35% | +| Type3 repeated / render | 853.06 | 523.42 | -38.64% | -2.44% | + +All twelve Type3 scaling cases are faster. Across the six direct controls, +process changes range from -1.31% to +2.39%, within the accepted 5% allowance; +peak differences are small. The shapes-disabled savings are approximately +546 MiB for both single and repeated painting. That supports the expected +font-level retention effect: the repeated layout executes the glyph four times +and retains more painted output, but eager font initialization occurs once. + +This closes completion item 1's isolated performance gap. The old mismatched- +dependency comparison is not used to claim these gains. Keep the current +discard-callback validation and its damaged-CharProc semantic regression. +Remaining gates are the current workload evidence below and reference/dead-code +retirement followed by final verification. Benchmark-reporting changes wait. + +### Close item 5 with current workload results + +Use the existing launcher unchanged to collect current native throughput/counts +and Python-backend process peak RSS at one, two and four threads: + +~~~sh +REPEATS=3 THREADS=1,2,4 RUN_COMPARISON=0 DATASET=bo20 \ + ./scripts/benchmarking/refresh_performance_benchmarks.sh +~~~ + +Return the new bundle path. Codex will inspect all repetitions, failure/page +counts, settings and source/binary/dependency provenance; assess scaling and +runtime spread; and extract existing peak_rss_mib from replay/r*/pdfium/report.json +and replay/r*/t*/report.json. These measure the actual backend processes, +including their native threads and retained outputs. They do not supply native +run_scaling.exe peaks or sampled timelines. This is sufficient to perform the +existing plan's threaded integration memory check without implementing the +later benchmark-tooling feature first. + +The old Nov-11 bundle is historical context, not a controlled attribution pair +for changes across the subsequent main merge and dependency changes. Do not +claim post-merge performance or memory gains before reviewing the new results. + +### Then close item 6 and verify the final source + +After the two measurements above are accepted, Codex will remove: + +- The temporary buffered/copy comparison targets and all corresponding source + selectors, including the final eager-font overlay target and generator. +- Dead vector-taking APIs whose last callers were comparison branches, after + checking all callers. Keep parsed pattern caching and useful native diagnostics. +- The unused char_description/char_processor headers and their parse.h includes. + +Keep the memory fixture runner, QPDF discard/collect measurements, callback and +state-lifetime regressions, clip-history tests and semantic coverage. Preserve +accepted experiment bundles and explicitly identify the final source revision. + +Codex will then supply the user a final build/native-regression/full-unit/corpus +verification set and a bounded memory comparison against the accepted candidate. +Only after those results pass can this parser plan be marked complete and the +requested permanent benchmark peak-RSS/timeline implementation begin. + +## Post-merge workload accepted; final comparison-code cleanup + +The user returned `scratch-performance-refresh-20261112-104813-yVM2JG`. +The report records the merged source 852d6b6 plus its working-tree/build +provenance, QPDF 12.4.2, docling-parse 7.25.0 and pypdfium2 5.14.0 on the +Apple M3 Max. Dataset: pinned bo20 65b9971b3fa939fe4ef89be678237e6928049b67, +20 PDFs / 496 pages. Actual thread counts are **1, 4, 8, 12**, with three +repetitions. This differs from the earlier proposed 1/2/4 sweep; do not report +a two-thread workload measurement that was not collected. Two-thread +correctness is exercised by the existing threading tests. + +All 39 native/replay runs completed all 496 pages with zero errors. Native +render-only medians are 6.641, 1.746, 0.904 and 0.642 seconds, giving 10.35x +speedup at twelve threads. Native render-with-text medians are 6.710, 1.779, +0.924 and 0.652 seconds (10.30x at twelve threads). Replay docling-parse +medians are 7.793, 3.012, 2.283 and 2.111 seconds, versus PDFium 11.822 +seconds. Workloads and timing boundaries remain distinct as documented. + +The existing replay report.json files provide per-process lifetime peak RSS: + +| Replay backend | Threads | Median MiB | Observed min–max MiB | +| --- | ---: | ---: | ---: | +| PDFium | 1 | 628.92 | 628.39–630.36 | +| docling-parse | 1 | 692.44 | 680.84–703.55 | +| docling-parse | 4 | 850.00 | 839.00–881.19 | +| docling-parse | 8 | 1,035.56 | 1,025.12–1,058.02 | +| docling-parse | 12 | 1,171.45 | 1,131.00–1,181.53 | + +These observations close the post-merge workload scaling/integration-memory +gate. They do not establish a controlled before/after delta against the old +Nov-11 bundle, native-only process peaks, or sampled memory timelines. + +### Retired comparison code + +The final source cleanup removes all seven buffered/copy comparison selectors, +their temporary executable targets and the eager-Type3 overlay target/generator. +Only test_memory_on_lines and the three native regression targets remain in +the optional memory-probe CMake block. + +Removed unused STREAM decode/interpret-vector/print entry points, its owned +token vector, the unused Form parse_stream() token API, and the unused +char_description/char_processor headers and includes. Kept callback execution, +borrowed pattern replay, useful parsed pattern caches, QPDF discard/collect +diagnostics, fixture counters and callback/state/clip regression coverage. +Production branches are the same branches selected by the previous normal +build; Form residual timing no longer subtracts an always-zero buffered-parse +variable. Legacy timing keys and old experiment bundles remain readable. + +The measured run_scaling executable and Python extension were preserved under +`/private/tmp/docling-before-comparison-cleanup`, with verified hashes matching +the new workload metadata. provenance.json includes the original metadata and +archive hashes observed after that run. Those archive observations are not +retroactive proof of an older binary's link inputs. + +The old Type3 probe executable still matches its accepted experiment hash, but +all nine archive hashes now differ from that experiment. Do not compare the +final rebuilt probe to that bundle as if dependencies matched, and do not +bypass the existing guard. Accepted Type3 before/after attribution remains +valid within its original controlled pair. + +### Final user verification before closing the plan + +Codex reviewed the cleanup diff and searched callers/selectors. MyPy, Ruff lint +and formatting pass for the remaining Python runner; git diff --check passes. +Codex has not configured, built or executed the cleaned implementation. + +~~~sh +cmake -S . -B build -DDOCLING_PARSE_BUILD_MEMORY_PROBE=ON +cmake --build build --parallel 2 --target \ + test_memory_on_lines stream_callback_regression \ + stream_state_lifetime_regression clip_history_regression + +ctest --test-dir build --output-on-failure \ + -R '^(stream_callback|stream_state_lifetime|clip_history)_regression$' + +uv run --no-sync python local_build.py +uv run --no-sync pytest tests/test_unit_*.py -q +uv run --no-sync pytest -n 2 --dist loadfile tests/test_regression_*.py + +uv run python3 tests/cpp/run_memory_on_lines.py \ + --output /private/tmp/docling-memory-final-cleanup-smoke \ + --layouts direct,form,form-repeat,form-nested,appearance,widget,appearance-repeat,type3,type3-repeat,soft-mask,soft-mask-repeat,pattern,pattern-repeat,pattern-lattice,clip-chain,clip-scoped \ + --sizes 1000 --repeats 1 +~~~ + +The final native smoke has 80 fresh-process measurements, preserving all source +and clip/pattern invariants checked by the runner. It is a correctness/retention +sanity check, not a one-repeat speed comparison. No mismatched-library baseline +is supplied. Existing pixel/content regression references remain unchanged. + +The user returned passing final checks. The saved final-cleanup smoke records +status passed, 80 measurements, and six byte-identical PNG equivalence checks +(direct versus Form/nested Form/appearance/widget/Type3, and chain versus scoped +clipping). This closes the final acceptance gate. The one-repeat smoke is not +a new speed claim, and no mismatched dependency comparison was used. + +All parser-plan work is complete. The requested optional benchmark peak/sampled +memory feature can now proceed separately; its validation remains with the user. + +## Questions to answer from the intermediate results + +1. How much of the initial token-stage memory remains with a callback that + releases every object? +2. How much extra memory is due to the Form's local vector and interpreter copy? +3. Does page callback delivery remove the predicted retention, and what remains? +4. Do damaged-document and split-stream outputs remain equivalent? +5. Which intentional caches improve repeated-use runtime enough to retain? +6. Is the remaining dominant cost the input bytes, page outputs, bitmap data, + clipping state, cache retention, or concurrent pages? +7. After the avoidable token cost is removed, is a budget, weighted scheduling, + output-representation optimization, or QPDF change the next justified step? +8. Are there additional missing-reset defects, or legitimate state histories whose repeated copying amplifies memory? Which can be reproduced and fixed independently? + +Answer these together as evidence arrives. Do not preselect later optimizations +or promise memory limits that the measurements have not established. diff --git a/CMakeLists.txt b/CMakeLists.txt index 6202adc9e..00a86d2ff 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -236,6 +236,35 @@ if(DOCLING_PARSE_BUILD_STRUCTURE_TESTS) target_link_libraries(structure_concurrency PRIVATE ${DEPENDENCIES} ${LIB_LINK}) endif() +option(DOCLING_PARSE_BUILD_MEMORY_PROBE "Build the native line-memory probe" OFF) +if(DOCLING_PARSE_BUILD_MEMORY_PROBE) + add_executable(test_memory_on_lines tests/cpp/test_memory_on_lines.cpp) + add_dependencies(test_memory_on_lines ${DEPENDENCIES}) + target_link_libraries(test_memory_on_lines PRIVATE ${DEPENDENCIES} ${LIB_LINK}) + target_compile_definitions(test_memory_on_lines PRIVATE DOCLING_PARSE_CONTENT_DIAGNOSTICS=1) + + add_executable(stream_callback_regression tests/cpp/test_stream_callbacks.cpp) + add_dependencies(stream_callback_regression ${DEPENDENCIES}) + target_link_libraries(stream_callback_regression PRIVATE ${DEPENDENCIES} ${LIB_LINK}) + enable_testing() + add_test(NAME stream_callback_regression COMMAND stream_callback_regression) + set_tests_properties(stream_callback_regression PROPERTIES TIMEOUT 30) + + add_executable(stream_state_lifetime_regression tests/cpp/test_stream_state_lifetimes.cpp) + add_dependencies(stream_state_lifetime_regression ${DEPENDENCIES}) + target_link_libraries(stream_state_lifetime_regression PRIVATE ${DEPENDENCIES} ${LIB_LINK}) + target_compile_definitions(stream_state_lifetime_regression PRIVATE + DOCLING_PARSE_CONTENT_DIAGNOSTICS=1 DOCLING_PARSE_STATE_LIFETIME_TESTING=1) + add_test(NAME stream_state_lifetime_regression COMMAND stream_state_lifetime_regression) + set_tests_properties(stream_state_lifetime_regression PROPERTIES TIMEOUT 30) + + add_executable(clip_history_regression tests/cpp/test_clip_history.cpp) + add_dependencies(clip_history_regression ${DEPENDENCIES}) + target_link_libraries(clip_history_regression PRIVATE ${DEPENDENCIES} ${LIB_LINK}) + add_test(NAME clip_history_regression COMMAND clip_history_regression) + set_tests_properties(clip_history_regression PROPERTIES TIMEOUT 30) +endif() + # Test the native library directly: the renderer's curve-flattening workaround # (#407) otherwise bypasses the stroker routines patched for #396/#400/#401. option(DOCLING_PARSE_BUILD_BLEND2D_TESTS "Build native Blend2D patch regression tests" OFF) diff --git a/README.md b/README.md index 40d70beba..c8e15165a 100644 --- a/README.md +++ b/README.md @@ -284,6 +284,10 @@ host. Run with `loglevel="info"` to see which file was chosen for each font. The tooling behind it lives under [`scripts/benchmarking/`](./scripts/benchmarking/README.md): +Publish an existing run with `uv run python scripts/tools/update_performance_docs.py --performance-run-dir RUN` +(preview), then add `--apply` after review. Results are organized by dataset and CPU/setup; +publication enforces documentation and image size budgets before writing files. + - [`scripts/benchmarking/run_performance_benchmarking.py`](./scripts/benchmarking/run_performance_benchmarking.py): measure — thread-scaling sweeps and the cross-package comparison - [`scripts/benchmarking/run_performance_eval.py`](./scripts/benchmarking/run_performance_eval.py): plot — per-page distributions, thread scaling, and per-package correlations - [`scripts/benchmarking/run_performance_analysis.py`](./scripts/benchmarking/run_performance_analysis.py): drill down — replay the slowest pages with C++ stage timings diff --git a/docs/performance_benchmarks.md b/docs/performance_benchmarks.md index d7926b2de..3d2c4720c 100644 --- a/docs/performance_benchmarks.md +++ b/docs/performance_benchmarks.md @@ -1,589 +1,126 @@ # Performance benchmarks -Measurements collected November 6–7, 2026, on an **Apple M3 Max with 64 GiB memory**, using **767 PDFs / 54,730 pages**. The Docling-settings workload creates **page-images and text**, including word/line models, bitmap bounding boxes and separator geometry. +Performance results are published separately for each dataset and machine setup. +Choose a result page below for measured tables, plots, package versions and provenance. +A publication updates only its dataset/setup; results from other setups remain available. -A **page-image** is a raster of the complete PDF page. It is distinct from an embedded bitmap extracted from inside the PDF. Embedded bitmaps are decoded as needed to render pages; the published profiles do not export their original image bytes as separate outputs. - -These results contain **one observation per configuration**. They do not establish run-to-run variability. Native rendering, Python with Docling settings and cross-package API workloads have separate timing boundaries and output contracts. - -| Workload | Threads | Wall time (s) | Pages/s | Parallel efficiency | -|---|---:|---:|---:|---:| -| Native C++, page-images | 12 | 104.020 | 526.1 | 86.4% | -| Native C++, page-images + word/line text geometry | 12 | 102.791 | 532.4 | 87.4% | -| Python, Docling settings: page-images + text | 12 | 248.806 | 220.0 | 36.9% | - -The following sections explain each workload before comparing its results. Speedup and efficiency always use a one-thread baseline from the same workload; subtracting these wall times does not isolate Python language overhead. + +| Dataset | Setup | Run | Workloads | +|---|---|---|---| +| bo20 | [Apple M3 Max / 64 GiB / macOS-26.6.2-arm64-arm-64bit](performance_benchmarks/bo20/apple-m3-max-64gib-macos-26-6-2-arm64-arm-64bit/index.md) | scratch-performance-refresh-20261112-123829-Q21HfP | native, replay | +| bo767 | [Apple M3 Max / 64 GiB / macOS-26.6.2-arm64-arm-64bit](performance_benchmarks/bo767/apple-m3-max-64gib-macos-26-6-2-arm64-arm-64bit/index.md) | scratch-performance-recovery-8vumw3sy | native, replay | + ## Workloads and timing boundaries -Three experiments answer different questions: - -| Experiment | Runner | What the reported wall time includes | -|---|---|---| -| Native C++ scaling | `build/run_scaling.exe` | Page decoder construction, decoding, word/line computation, rasterization, canvas extraction, result collection and timing CSV output. Document loading is outside the timer. | -| Python scaling with Docling settings | `run_performance_benchmarking_with_docling_settings.py` | Per-document parser construction/loading, processing, text-model and page-image consumption, separator queries, JSONL output and unloading. | -| Optional cross-package comparison | `run_performance_benchmarking.py` | Each selected package's complete corpus workload, including document loading. The Docling row uses the public threaded Python API. | - -Native C++ produces native results. The replay also materializes Python models and -RGB page-images, uses a different document lifecycle, and releases native memory every -128 pages. Their runtime difference is not an isolated measurement of Python -language overhead. - -The replay follows a pinned inspection of Docling revision -`07154cdc82500058ad9377bac92beed3de820458`. It excludes OCR, layout/table models, -inference and Docling framework construction. Exact parity with the actual -Docling backend has not been independently established. - -## Dataset and configuration - -Within each run, all experiments use the selected pinned dataset and file order. -The default is Bo767: - -| Parameter | Value | -|---|---| -| Dataset | `docling-project/performance-dataset-bo767` | -| Revision | `221c19e716d6ede5d18fd1560bc938ab19cc1a70` | -| Documents / pages | 767 / 54,730 | -| Render scale | 1.0: requested output at 72 dpi | -| Result queue capacity | 32 | -| Thread counts | 1, 4, 8, 12 by default | -| QPDF | Bundled version recorded after the build; current pin: 12.4.2 | -| Machine / operating system | Apple M3 Max, arm64, 64 GiB; macOS 26.6.2 | -| Python / Docling packages | Python 3.12.11; docling-parse 7.24.2; docling-core 2.103.1 | -| Native build | CMake Release, `-O3 -DNDEBUG`, C++20; compiler driver `/usr/bin/c++`; exact compiler version was not archived | -| Source and build identities | See the provenance section and compact JSON linked below | -| Repetitions | One per configuration; no empirical variability range | - -Set `DATASET=bo20` for 20 PDFs / 496 pages, or `DATASET=bo10k` for 10,000 PDFs / -213,538 pages. Each corpus has its own pinned revision, listed in -[the benchmarking tools README](../scripts/benchmarking/README.md#publication-bundle-and-plots). -`DATASET_REVISION` can override the selected corpus's pin; the requested revision -and resolved commit are saved in the run metadata. - -The earlier Bo767 snapshot `c684df54de2081a6de1192cee12713b54759c6bb` contained -753 PDFs / 54,584 pages. Restoring 14 PDFs added 146 pages. Existing reports from -that snapshot keep their original inputs and must not be mixed with measurements -of the complete corpus. The launcher records ordered filenames, page counts and -SHA-256 hashes so the exact inputs can be checked. - -Native rendering is measured twice: `render-only` disables word/line cell -construction, while `render-with-text` enables both. Both decode characters and -bitmap geometry needed for drawing and extract native RGBA canvases. Neither -materializes Python models or separate bitmap bounding-box outputs. -Python replay materializes word/line models and bitmap bounds, computes shapes -without materializing their models, and excludes bitmap image-byte export. -Both native profiles and replay use crop-box boundaries and enable same-font -enforcement. - -The replay requests scale-1 page-images from both backends. PDFium renders at 1.5 times -the requested scale and resizes; docling-parse renders at scale 1 and converts to -RGB. Equal output dimensions do not establish equal pixels or text segmentation. - -The corpus has 1–2,227 pages per document: median 24, mean 71.4. This uneven document size helps explain why document lifecycle and page scheduling matter when interpreting parallel efficiency. - -## Native C++ scaling - -Both profiles decode and render page-images into native RGBA canvases. `render-only` disables word/line cell construction; `render-with-text` enables both. Character and bitmap geometry required for drawing is computed in either profile. Python models and separate embedded-bitmap image outputs are not produced. - -The timer includes decoder construction, decoding, word/line computation when enabled, rasterization, canvas extraction, collection and timing CSV output. Document loading is outside the timer. Stage timing is enabled; glyph-cache diagnostics and saving rendered page-images as PNG files are disabled. - -| Profile | Threads | Wall time (s) | Pages/s | Speedup vs 1 thread | Efficiency | -|---|---:|---:|---:|---:|---:| -| Page-images | 1 | 1078.813 | 50.7 | 1.00x | 100.0% | -| Page-images | 4 | 281.504 | 194.4 | 3.83x | 95.8% | -| Page-images | 8 | 148.472 | 368.6 | 7.27x | 90.8% | -| Page-images | 12 | 104.020 | 526.1 | 10.37x | 86.4% | -| Page-images + word/line geometry | 1 | 1077.483 | 50.8 | 1.00x | 100.0% | -| Page-images + word/line geometry | 4 | 284.828 | 192.2 | 3.78x | 94.6% | -| Page-images + word/line geometry | 8 | 151.327 | 361.7 | 7.12x | 89.0% | -| Page-images + word/line geometry | 12 | 102.791 | 532.4 | 10.48x | 87.4% | - -At 12 threads, throughput reaches 526.1 pages/s for page-images and 532.4 pages/s with word/line construction. Relative to their own one-thread baselines, these are 10.37x and 10.48x speedups, with 86.4% and 87.4% efficiency. - -The two profiles have similar observed wall times. The slightly faster result with word/line construction does not establish that constructing text is free or beneficial: each configuration was run once. Stage totals in the one-thread text profile include 750.3 worker-seconds of decoding, 287.7 of rendering and 20.6 of word-cell construction; decoding is the largest measured stage. - -### Page-images - -#### Wall time and throughput - -Corpus wall time and pages per second. Document loading is excluded. The single observations do not provide a variability estimate. - -![Wall time and throughput](performance_benchmarks/2026-11-bo767-m3-max/native_render_only_runtime.png) - -#### Speedup and parallel efficiency - -Each profile uses its own one-thread result. Dashed lines show ideal speedup and 100% efficiency. - -![Speedup and parallel efficiency](performance_benchmarks/2026-11-bo767-m3-max/native_render_only_scaling.png) - -#### Where the native time goes - -Stacked bars sum page worker elapsed by stage. Workers overlap: the stacked total is neither corpus wall time nor CPU time. Internal waits are included in worker elapsed. - -![Where the native time goes](performance_benchmarks/2026-11-bo767-m3-max/native_render_only_phases.png) - -#### Page worker-duration distribution - -Page worker durations on logarithmic axes show the long tail and how it changes with thread count. These are not end-to-end page latencies or document wall times. - -![Page worker-duration distribution](performance_benchmarks/2026-11-bo767-m3-max/native_render_only_distribution.png) - -
-Reproduce the measurements for these figures - -Run from the repository root after building. Set `PDF_ROOT` to the pinned corpus directory. Each invocation below runs in a fresh process and writes to a new output directory. - -```bash -set -euo pipefail -PDF_ROOT=/path/to/performance-dataset-bo767/pdf -REPRO_OUT=$(mktemp -d "$PWD/scratch-performance-reproduce-XXXXXX") -for ((r=1; r<=1; r++)); do - for t in 1 4 8 12; do - ./build/run_scaling.exe "$PDF_ROOT" -v WARNING --recursive --mode render \ - --threads "$t" --max-concurrent-results 32 --scale 1.0 \ - --create-word-cells=false --create-line-cells=false \ - --enable-timing --timing-csv "$REPRO_OUT/r${r}-t${t}.csv" \ - 2>&1 | tee "$REPRO_OUT/r${r}-t${t}.log" - done -done -``` - -
- -### Page-images and word/line text geometry - -#### Wall time and throughput - -Corpus wall time and pages per second. Document loading is excluded. The single observations do not provide a variability estimate. - -![Wall time and throughput](performance_benchmarks/2026-11-bo767-m3-max/native_render_with_text_runtime.png) - -#### Speedup and parallel efficiency - -Each profile uses its own one-thread result. Dashed lines show ideal speedup and 100% efficiency. - -![Speedup and parallel efficiency](performance_benchmarks/2026-11-bo767-m3-max/native_render_with_text_scaling.png) - -#### Where the native time goes - -Stacked bars sum page worker elapsed by stage. Workers overlap: the stacked total is neither corpus wall time nor CPU time. Internal waits are included in worker elapsed. - -![Where the native time goes](performance_benchmarks/2026-11-bo767-m3-max/native_render_with_text_phases.png) - -#### Page worker-duration distribution - -Page worker durations on logarithmic axes show the long tail and how it changes with thread count. These are not end-to-end page latencies or document wall times. - -![Page worker-duration distribution](performance_benchmarks/2026-11-bo767-m3-max/native_render_with_text_distribution.png) - -
-Reproduce the measurements for these figures - -Run from the repository root after building. Set `PDF_ROOT` to the pinned corpus directory. Each invocation below runs in a fresh process and writes to a new output directory. - -```bash -set -euo pipefail -PDF_ROOT=/path/to/performance-dataset-bo767/pdf -REPRO_OUT=$(mktemp -d "$PWD/scratch-performance-reproduce-XXXXXX") -for ((r=1; r<=1; r++)); do - for t in 1 4 8 12; do - ./build/run_scaling.exe "$PDF_ROOT" -v WARNING --recursive --mode render \ - --threads "$t" --max-concurrent-results 32 --scale 1.0 \ - --create-word-cells=true --create-line-cells=true \ - --enable-timing --timing-csv "$REPRO_OUT/r${r}-t${t}.csv" \ - 2>&1 | tee "$REPRO_OUT/r${r}-t${t}.log" - done -done -``` - -
- -## Understanding the two Python experiments - -**These sections measure different workloads and Python output representations.** They share a corpus, but their pages/second values cannot be subtracted to measure isolated Python overhead or interpreted as a change in native renderer performance. Neither experiment runs OCR, model inference or complete document conversion. - -### What each experiment measures - -The Docling-settings replay uses `run_performance_benchmarking_with_docling_settings.py` to replay selected backend operations from a pinned Docling revision, including adapter processing. The cross-package experiment uses `run_performance_benchmarking.py` to exercise each package's supported extraction API, optionally followed by rendering. It does not construct a common Docling output model across packages. - -| Aspect | Python scaling with Docling settings | Python cross-package comparison | -|---|---|---| -| PDFium page-image generation | Render at 1.5 times the requested scale, copy the PIL image, then resize to the requested dimensions | Render directly at the requested scale and obtain a PIL image | -| PDFium text processing | Read character data, select text geometrically, normalize coordinates, merge fragmented cells and construct segmented-page models | Iterate PDFium text rectangles and call `get_text_bounded()`; no Docling page models or cell-merging adapter | -| Docling text output | Compute characters internally; materialize word and line models, without character models | Materialize character models; word/line construction is disabled | -| Docling page-image output | Obtain the rendered page-image and convert it to RGB | Obtain the rendered image without the replay's explicit RGB conversion | -| Additional geometry | Query separator lines, connected shapes and bitmap bounds | These adapter queries and separate bitmap-bound outputs are omitted | -| Docling document scheduling | Construct a parser, load, process and unload one document at a time; native page work within that document is threaded | Construct one parser and submit all documents before consuming results; native work can overlap submission | -| Native-memory release policy | Every 128 pages | No periodic release (`0`) | -| Reported wall time | Sum of document timers, including parser construction, loading, processing and unloading | Corpus timer including document loading and result consumption; parser construction occurs before this timer | -| Timed bookkeeping | Per-page metadata generation, JSONL writes and progress updates | Per-page timing collection and progress updates; checkpoint and final CSV writes occur after measurement | - -Repeated image and segmented-page requests in the replay check caching; they do not repeat the underlying work. Both render profiles decode the image content needed to draw a page, even when exporting bitmap bytes or bitmap bounding-box models is disabled. - -Text geometry is also different **within** the cross-package experiment: docling-parse materializes character cells; PDFium returns text rectangles; PyMuPDF returns `rawdict` character data; pdfplumber exposes `page.chars`; pdfminer.six walks `LTChar` objects; and pypdf's visitor exposes text-chunk positions. Its speedups compare these selected API workloads, not identical output granularity or extraction quality. - -### Reading this run's throughput - -Compare the replay with the cross-package **render** rows. Cross-package parse-only rows do not generate page-images. The table below pairs successful render measurements for the same package version and thread count; it does not assert equivalent workloads. - -| Backend | Version | Threads | Docling-settings replay, pages/s | Cross-package text + render, pages/s | -|---|---|---:|---:|---:| -| pypdfium2 | 5.14.0 | 1 | 36.9 | 95.8 | -| docling-parse | 7.24.2 | 1 | 49.7 | 42.5 | -| docling-parse | 7.24.2 | 4 | 151.3 | 63.4 | -| docling-parse | 7.24.2 | 8 | 201.4 | 65.3 | -| docling-parse | 7.24.2 | 12 | 220.0 | 64.9 | - -For PDFium, the replay performs additional text adaptation, geometry queries, model construction and image processing. Its intermediate image has approximately **2.25 times as many pixels** before resizing. These are verified workload differences; the measurements do not isolate their individual contributions to the timing gap. - -For Docling, the replay requests additional adapter operations but avoids constructing a Python model for every character. It instead materializes word and line models. The cross-package character-model construction runs through the Python result consumer; adding native workers does not parallelize that Python construction loop. Character-level output therefore introduces a possible consumer bottleneck, even though this profile omits several replay operations. - -In this run, cross-package Docling parse-only throughput is 69.5 pages/s at one thread and 69.1 pages/s at 12 threads. These observations help assess scaling for the character-output workload. - -**Bottleneck attribution remains an inference.** These runs do not separately time character materialization or queue stalls, so they cannot quantify how much of the gap comes from either. Document scheduling, memory policy and process/cache state also differ: the launcher uses a fresh process per replay profile/thread count, while each cross-package mode runs its references and Docling thread sweep in one process. Use each experiment's own one-thread baseline when interpreting speedup and parallel efficiency. - -## Python scaling with Docling settings - -This workload creates **RGB page-images and text**, consumes segmented text models, obtains bitmap bounding boxes and queries separator geometry. Docling computes characters internally but materializes word/line models; PDFium constructs merged textline cells using its adapter. The text segmentation and page-image pixels are not identical across the engines. - -The standalone replay follows Docling revision `07154cdc82500058ad9377bac92beed3de820458`. It excludes OCR, layout/table inference and Docling framework construction; exact parity with the full Docling backend has not been independently established. - -The reported wall time is the sum of document `total_s`, including per-document parser construction/loading, page processing, model and page-image consumption, JSONL metadata writes, progress updates and unloading. Imports, corpus verification and final report serialization are outside this boundary. - -| Backend | Version | Threads | Wall time (s) | Pages/s | Speedup vs PDFium (1t) | Speedup vs Docling (1t) | Efficiency | -|---|---|---:|---:|---:|---:|---:|---:| -| pypdfium2 | 5.14.0 | 1 | 1485.068 | 36.9 | 1.00x | 0.74x | — | -| docling-parse | 7.24.2 | 1 | 1100.957 | 49.7 | 1.35x | 1.00x | 100.0% | -| docling-parse | 7.24.2 | 4 | 361.817 | 151.3 | 4.10x | 3.04x | 76.1% | -| docling-parse | 7.24.2 | 8 | 271.772 | 201.4 | 5.46x | 4.05x | 50.6% | -| docling-parse | 7.24.2 | 12 | 248.806 | 220.0 | 5.97x | 4.42x | 36.9% | - -Docling reaches 220.0 pages/s at 12 threads: 4.42x its one-thread result and 5.97x the PDFium replay result. These speedups refer to this page-image-and-text workload. They should not be transferred to the cross-package API comparison below. - -The replay constructs and uses a parser for one document at a time. Native page work within that document is threaded, but documents are processed serially. Across the full corpus, Docling construction/load totals stay near 32.5–32.9 seconds while page-processing time decreases. At 12 threads, construction/load accounts for about 13.1% of the measured total. This is a serial contribution in this harness, not a general statement about every Docling deployment. - -| Docling threads | Construction/load (s) | Page processing (s) | Unload (s) | Total (s) | -|---:|---:|---:|---:|---:| -| 1 | 32.888 | 1067.594 | 0.474 | 1100.957 | -| 4 | 32.642 | 328.571 | 0.604 | 361.817 | -| 8 | 32.635 | 238.493 | 0.645 | 271.772 | -| 12 | 32.537 | 215.620 | 0.649 | 248.806 | - -On this corpus, the Docling replay materializes 12,207,024 word cells and 2,304,136 line cells, without character models. The PDFium replay returns 3,126,124 merged textline cells. These counts describe different segmentation contracts and do not establish extraction accuracy. - -### Page-images and text: figures - -#### Page-image-and-text wall time and throughput - -Backend totals include parser construction/load, page processing and unloading. PDFium is driven with one worker; Docling uses the thread sweep. - -![Page-image-and-text wall time and throughput](performance_benchmarks/2026-11-bo767-m3-max/replay_runtime.png) - -#### Python replay speedup and parallel efficiency - -Docling speedup is relative to its own one-thread replay. The work includes Python output construction and a document-by-document lifecycle. - -![Python replay speedup and parallel efficiency](performance_benchmarks/2026-11-bo767-m3-max/replay_scaling.png) - -#### Document construction, processing and unloading - -These are summed document phases, unlike the native worker-stage bars. Construction/load remains nearly constant while threaded page processing decreases. - -![Document construction, processing and unloading](performance_benchmarks/2026-11-bo767-m3-max/replay_phases.png) - -#### Whole-document duration distribution - -Histograms pool document total times on logarithmic axes. Replay JSONL contains page output metadata, not individual page timings; it cannot supply per-page latency histograms. - -![Whole-document duration distribution](performance_benchmarks/2026-11-bo767-m3-max/replay_distribution.png) - -
-Reproduce the measurements for these figures - -Run from the repository root after building. Set `PDF_ROOT` to the pinned corpus directory. Each invocation below runs in a fresh process and writes to a new output directory. - -```bash -set -euo pipefail -PDF_ROOT=/path/to/performance-dataset-bo767/pdf -REPRO_OUT=$(mktemp -d "$PWD/scratch-performance-reproduce-XXXXXX") -for ((r=1; r<=1; r++)); do - uv run --no-sync python scripts/benchmarking/run_performance_benchmarking_with_docling_settings.py "$PDF_ROOT" \ - --backend pdfium --dataset-revision 221c19e716d6ede5d18fd1560bc938ab19cc1a70 \ - --output-dir "$REPRO_OUT/r${r}-pdfium" 2>&1 | tee "$REPRO_OUT/r${r}-pdfium.log" - for t in 1 4 8 12; do - uv run --no-sync python scripts/benchmarking/run_performance_benchmarking_with_docling_settings.py "$PDF_ROOT" \ - --backend threaded --threads "$t" --dataset-revision 221c19e716d6ede5d18fd1560bc938ab19cc1a70 \ - --output-dir "$REPRO_OUT/r${r}-t${t}" 2>&1 | tee "$REPRO_OUT/r${r}-t${t}.log" - done -done -``` - -
- -## Python cross-package comparison - -This experiment measures the selected public text/geometry APIs, optionally followed by rendering a page-image. It creates no shared Docling output representation for the reference packages. Docling materializes character models, with word/line construction and separate bitmap-bound models disabled. Reference packages are driven with one worker in this harness; Docling uses the requested native page-thread sweep. - -The table and figures above explain why this workload differs from the Docling-settings replay. Its rates measure the chosen APIs and output granularity; they do not establish equal output fidelity or general limits on other packages' concurrency. - -| Package | Text/geometry API | Page-image rendering API | -|---|---|---| -| docling-parse | Threaded parser with materialized character cells | Native renderer and `get_image()` | -| PyMuPDF | `get_text("rawdict")` character data | `get_pixmap()` and `pil_image()` | -| pypdfium2 | Text-page rectangles and bounded text | `page.render()` and `to_pil()` | -| pdfplumber | `page.chars` | `page.to_image()`, using PDFium | -| pdfminer.six | `extract_pages()` and an `LTChar` walk | Unsupported | -| pypdf | `extract_text(visitor_text=...)` text-chunk positions | Unsupported | - -The corpus timer includes document loading and result consumption. Docling parser construction precedes the timer. Exact wall times come from completed-series checkpoints, whose provenance and CSV hashes are validated. Console rounding is checked for consistency; per-page worker timings and arrival gaps are not substituted for corpus wall time. - -### Text and geometry, without page-images - -| Backend | Version | Threads | Wall time (s) | Pages/s | Speedup vs PDFium (1t) | Speedup vs Docling (1t) | Efficiency | -|---|---|---:|---:|---:|---:|---:|---:| -| docling-parse | 7.24.2 | 1 | 787.445 | 69.5 | 0.25x | 1.00x | 100.0% | -| docling-parse | 7.24.2 | 4 | 776.090 | 70.5 | 0.25x | 1.01x | 25.4% | -| docling-parse | 7.24.2 | 8 | 784.243 | 69.8 | 0.25x | 1.00x | 12.6% | -| docling-parse | 7.24.2 | 12 | 792.253 | 69.1 | 0.24x | 0.99x | 8.3% | -| pdfminer.six | 20260107 | 1 | 1591.772 | 34.4 | 0.12x | 0.49x | — | -| pdfplumber | 0.11.10 | 1 | 1688.906 | 32.4 | 0.11x | 0.47x | — | -| pymupdf | 1.27.2.3 | 1 | 1422.534 | 38.5 | 0.14x | 0.55x | — | -| pypdfium2 | 5.14.0 | 1 | 193.079 | 283.5 | 1.00x | 4.08x | — | - -#### Text/geometry API wall time and throughput - -No page-images are created. Geometry varies from individual characters to rectangles or text chunks, depending on the package. - -![Text/geometry API wall time and throughput](performance_benchmarks/2026-11-bo767-m3-max/comparison_parse_runtime.png) - -#### Character-model workload speedup and efficiency - -The scalable series is Docling, relative to its one-thread API workload. The nearly flat throughput is consistent with a Python consumer bottleneck, but character materialization and queue stalls were not separately timed. - -![Character-model workload speedup and efficiency](performance_benchmarks/2026-11-bo767-m3-max/comparison_parse_scaling.png) - -
-Reproduce the measurements for these figures - -Run from the repository root after building. Set `PDF_ROOT` to the pinned corpus directory. Each invocation below runs in a fresh process and writes to a new output directory. - -```bash -set -euo pipefail -PDF_ROOT=/path/to/performance-dataset-bo767/pdf -REPRO_OUT=$(mktemp -d "$PWD/scratch-performance-reproduce-XXXXXX") -for ((r=1; r<=1; r++)); do - uv run --no-sync python scripts/benchmarking/run_performance_benchmarking.py "$PDF_ROOT" \ - --recursive --mode parse-without-bitmap --threads 1,4,8,12 --scale 1.0 --max-concurrent-results 32 \ - --3rd-party-backends all --only-threaded \ - --keep-char-cells=true --create-word-cells=false --create-line-cells=false \ - --keep-shapes=false --keep-bitmaps=false \ - --materialize-char-cells=true --materialize-word-cells=false \ - --materialize-line-cells=false --materialize-shapes=false --materialize-bitmaps=false \ - --output-dir "$REPRO_OUT/r${r}" 2>&1 | tee "$REPRO_OUT/r${r}.log" -done -``` - -
- -### Page-images plus text and geometry - -| Backend | Version | Threads | Wall time (s) | Pages/s | Speedup vs PDFium (1t) | Speedup vs Docling (1t) | Efficiency | -|---|---|---:|---:|---:|---:|---:|---:| -| docling-parse | 7.24.2 | 1 | 1287.637 | 42.5 | 0.44x | 1.00x | 100.0% | -| docling-parse | 7.24.2 | 4 | 863.604 | 63.4 | 0.66x | 1.49x | 37.3% | -| docling-parse | 7.24.2 | 8 | 838.376 | 65.3 | 0.68x | 1.54x | 19.2% | -| docling-parse | 7.24.2 | 12 | 842.685 | 64.9 | 0.68x | 1.53x | 12.7% | -| pdfplumber | 0.11.10 | 1 | 2309.746 | 23.7 | 0.25x | 0.56x | — | -| pymupdf | 1.27.2.3 | 1 | 1822.560 | 30.0 | 0.31x | 0.71x | — | -| pypdfium2 | 5.14.0 | 1 | 571.583 | 95.8 | 1.00x | 2.25x | — | - -All seven successful rendering series cover 54,730 pages. Against PDFium, none differ in canvas size by more than two pixels; the largest observed delta is one pixel. Matching output dimensions do not establish pixel equivalence or equal rendering quality. - -#### Page-image and text/geometry API wall time and throughput - -This includes rendering at the requested scale plus each package's extraction API. PDFium does not perform the replay's 1.5-times-scale render, copy/resize or Docling model construction. - -![Page-image and text/geometry API wall time and throughput](performance_benchmarks/2026-11-bo767-m3-max/comparison_parse_render_runtime.png) - -#### Page-image and character-model workload scaling - -Docling scales from 42.5 to 64.9 pages/s at 12 threads, with 1.53x speedup. This is a different consumer/output workload from the 220.0 pages/s Docling-settings replay. - -![Page-image and character-model workload scaling](performance_benchmarks/2026-11-bo767-m3-max/comparison_parse_render_scaling.png) - -
-Reproduce the measurements for these figures - -Run from the repository root after building. Set `PDF_ROOT` to the pinned corpus directory. Each invocation below runs in a fresh process and writes to a new output directory. - -```bash -set -euo pipefail -PDF_ROOT=/path/to/performance-dataset-bo767/pdf -REPRO_OUT=$(mktemp -d "$PWD/scratch-performance-reproduce-XXXXXX") -for ((r=1; r<=1; r++)); do - uv run --no-sync python scripts/benchmarking/run_performance_benchmarking.py "$PDF_ROOT" \ - --recursive --mode render --threads 1,4,8,12 --scale 1.0 --max-concurrent-results 32 \ - --3rd-party-backends all --only-threaded \ - --keep-char-cells=true --create-word-cells=false --create-line-cells=false \ - --keep-shapes=true --keep-bitmaps=true \ - --materialize-char-cells=true --materialize-word-cells=false \ - --materialize-line-cells=false --materialize-shapes=false --materialize-bitmaps=false \ - --output-dir "$REPRO_OUT/r${r}" 2>&1 | tee "$REPRO_OUT/r${r}.log" -done -``` - -
- -### Reference failures and recovery - -`pypdf 6.21.1` attempted all 54,730 pages but failed on 16, leaving 54,714 successful pages. Its series is excluded from performance tables, speedups and figures. The recorded failure cases are: - -| Document | Pages (1-based) | Error | -|---|---|---| -| `1067588.pdf` | 191 | Maximum tokens per stream reached | -| `1106182.pdf` | 21 | Maximum tokens per stream reached | -| `1161467.pdf` | 20–22, 24–30 | Multiple `/FontFile` entries | -| `1245654.pdf` | 20 | Maximum tokens per stream reached | -| `2216123.pdf` | 55, 153, 162 | Maximum tokens per stream reached | - -The original cross-package render run aborted in PyMuPDF 1.28.2 on Bo767 `1293644.pdf`, page 3. The reference was pinned to PyMuPDF 1.27.2.3, which rendered all 453 pages of that document. Recovery validated and reused the completed native/Python-replay measurements, then recollected both cross-package modes under separately recorded provenance. The failed original PyMuPDF run supplies no published performance result. - -All 28 included corpus measurements completed without page failures. Full error details and raw checkpoints remain in the retained local bundles. A reference failure is reported separately only after every expected page was attempted; missing coverage, changed provenance/checksums and native failures still fail report validation. - -## Provenance and scope - -The compact [publication provenance](performance_benchmarks/2026-11-bo767-m3-max/provenance.json) records the dataset identity, build/extension hashes, source revisions and diff hashes, package versions, artifact hashes and original bundle identifiers. The local raw measurements remain in: - -- Core measurements: `scratch-performance-refresh-20261106-132832-Ued2aI`. -- Completed comparison and report: `scratch-performance-recovery-c2rz8gq0`. - -The core build recorded source revision `929a2b6ed7cc9648d9b9c3153362d6ee9542402a` plus its working-tree diff. Replay recorded `6dade98230ee60aaaa01a5db7111bda45382695d` after those changes were committed during the run. The commit patch exactly matches the recorded build diff, and the replay script hash matches that commit. Comparison provenance records `6dade98230ee60aaaa01a5db7111bda45382695d` plus its later diff. Recovery verified that the native binary, extension, core packages, Python and machine matched the original core run. - -| Package | Version used | +| Experiment | What the reported wall time includes | |---|---| -| docling-parse | 7.24.2 | -| docling-core | 2.103.1 | -| pypdfium2 | 5.14.0 | -| Pillow | 12.3.0 | -| rtree | 1.4.1 | -| PyMuPDF, completed comparison | 1.27.2.3 | -| pdfplumber | 0.11.10 | -| pdfminer.six | 20260107 | -| pypdf, excluded reference | 6.21.1 | -| Matplotlib | 3.11.2 | - -Build settings were CMake Release, `-O3 -DNDEBUG` and C++20, using `/usr/bin/c++` and the macOS SDK selected by `xcrun`. The native compiler version was not archived; Python reports Clang 20.1.4 as its own build compiler, which must not be substituted for the native compiler version. - -The source bundles and full page-level CSV/JSONL data remain local. This documentation publishes the figures, tables and compact provenance rather than copying those data files into the repository. The figures total about 1.4 MB. - -Throughput is pages divided by corpus wall time. Amortized milliseconds/page are `1000 * wall / pages`, not individual-page latency. Speedup is `wall(1) / wall(N)` within the same workload/backend/task; parallel efficiency is `100 * speedup / N`. With one repetition, a median equals the single observation and a min–max range provides no empirical variability information. - -## Run the refresh - -From the repository root, launch the native and Python page-image-and-text experiments: - -```sh -bash scripts/benchmarking/refresh_performance_benchmarks.sh -``` - -To refresh every section of this page, including the slower cross-package suite, -and collect three repetitions: - -```sh -REPEATS=3 RUN_COMPARISON=1 \ - bash scripts/benchmarking/refresh_performance_benchmarks.sh -``` - -These are full-corpus measurements and can take hours. The script installs the -build/dev/perf dependency groups, checks/builds the Python extension and standalone -executable incrementally, runs the damaged-page unit tests, resolves -the pinned snapshot, and records provenance before timing. It does not edit this -page or publish results. - -`BUILD_MODE=incremental` is the default. Existing `build/` and `externals/` -directories are reused; CMake checks dependencies and recompiles affected -targets. Unchanged targets are reused. Cached repository, Python, SDK and -compiler paths are checked before and after dependency synchronization. -An incompatible cache stops the run and requests `BUILD_MODE=clean`. - -Use `BUILD_MODE=clean` after SDK/toolchain changes or bundled dependency pin/patch -upgrades. Existing directories are then moved to `previous-build/` and -`previous-externals/` under the new output directory before rebuilding from -scratch. Build work remains outside all benchmark timers; each bundle records -the selected mode and native binary/extension hashes. -Setup stages and experiments are labeled by repetition, backend and thread -count. Runner output, including page progress bars, is streamed to the terminal -and saved in the run log. `BENCHMARK_OUTPUT=compact` is the launcher default: -repeated config tables and native timing tables are hidden, while progress and -result tables remain visible. Native timing summaries are saved beside each CSV -as `.summary.log` and used by the report. Set -`BENCHMARK_OUTPUT=full` to print all benchmark tables. Standalone runners keep -their full output unless `--suppress-config` and, for C++, `--suppress-timings` -are requested. Completion messages report command elapsed time; -the report still uses each benchmark's own documented wall-time boundary. -On macOS, the launcher selects the current SDK with `xcrun`. A failed phase -prints its exit status, log path and final 40 log lines, then stops. - -Each native thread profile and each replay backend/thread profile runs in its -own process. Runs execute sequentially. Each cross-package mode runs in one -process per repetition and measures its selected packages sequentially. - -Run on an otherwise idle machine. Dataset verification reads the files before -measurement, so these are not cold-cache benchmarks. Repeat ranges expose -observed variation; they are not confidence intervals. Thread counts above the -machine's available physical/performance cores need separate interpretation. - -Optional launcher settings: +| Native C++ scaling | Decoder construction, decoding, optional word/line computation, rasterization, canvas extraction, result collection and timing CSV output. Document loading is outside the timer. | +| Python scaling with Docling settings | Per-document parser construction/loading, processing, text-model and page-image consumption, separator queries, JSONL output and unloading. | +| Cross-package comparison, when collected | Each selected package's corpus workload, including document loading. The Docling row uses the public threaded Python API. | + +Native profiles produce page-images in native RGBA canvases. `render-only` disables +word/line cell construction; `render-with-text` enables both. Both decode character +and bitmap geometry needed for drawing. Neither materializes Python models. + +The Python replay creates RGB page-images, materializes word/line models and bitmap +bounds, and queries separator geometry. It follows selected backend operations +from pinned Docling revision `07154cdc82500058ad9377bac92beed3de820458`. +It excludes OCR, layout/table inference and Docling framework construction. +Exact parity with the full Docling backend has not been independently established. +PDFium renders at 1.5 times the requested scale and resizes; docling-parse renders +at the requested scale and converts to RGB. Equal dimensions do not establish +equal pixels or text segmentation. + +Cross-package comparisons exercise package-specific text/geometry APIs, optionally +with rendering. Docling materializes character models; other packages expose +different geometry granularity. These experiments do not produce identical output +models. Subtracting wall times across workloads does not isolate Python overhead. + +A page-image is a raster of the entire PDF page, distinct from an embedded bitmap. +The published profiles do not export original embedded bitmap bytes separately. +Performance and extraction/rendering quality are separate: see +[quality benchmarks](quality_benchmarks.md). + +## Datasets and setups + +The launcher supports Bo20, Bo767 and Bo10K. Each publication records the exact +dataset revision, ordered input identity, document/page counts, CPU, architecture, +RAM, operating system, source/build identities, package versions, thread sweep, +render scale and queue capacity. Custom revisions retain their recorded identity; +a dataset name alone does not establish equivalent inputs. + +Bo767's earlier snapshot contained 753 PDFs; the complete snapshot contains 767. +Those measurements must not be mixed. Likewise, Bo20 results are not substitutes +for Bo767 or Bo10K measurements. + +Results from different CPUs appear on separate pages. Treat OS, RAM, dataset +revision, workload settings, package/build versions and memory instrumentation as +part of the experimental setup when comparing numbers. A setup page presents one +published run; it never combines missing sections with measurements from an older run. + +## Reading the results + +Throughput is pages divided by corpus wall time. Amortized milliseconds/page are +`1000 * wall / pages`, not individual-page latency. Speedup uses the one-thread +baseline from the same workload/backend/task; efficiency is `100 * speedup / threads`. +Missing baselines produce unavailable ratios. Repeated runs report median wall +time and observed min–max ranges; these ranges are not confidence intervals. +A single observation provides no empirical variability estimate. + +Worker-stage totals include overlapping work and waits; they are neither corpus +wall time nor CPU time. Native page-worker durations and replay whole-document +durations describe different units. Thread counts above available physical or +performance cores need separate interpretation. + +Memory summaries use OS process high-water marks including startup, loading and +teardown, excluding the monitor/logger. Missing observations are unavailable, +never zero. Compare timings under matching memory instrumentation and process +isolation. Detailed sampled RSS timelines remain in local reports. + +## Collect and publish results + +From the repository root: ```sh -# Small corpus for quick benchmark iterations; incremental build by default. -DATASET=bo20 THREADS=1,4 \ +DATASET=bo20 REPEATS=3 THREADS=1,4,8,12 \ bash scripts/benchmarking/refresh_performance_benchmarks.sh -# Shorter thread sweep; output directory must not already exist. -THREADS=1,4,8 REPEATS=1 OUT=scratch-performance-my-run \ - bash scripts/benchmarking/refresh_performance_benchmarks.sh - -# Force a fresh build, for example after a bundled dependency upgrade. -BUILD_MODE=clean DATASET=bo20 \ - bash scripts/benchmarking/refresh_performance_benchmarks.sh - -bash scripts/benchmarking/refresh_performance_benchmarks.sh --help -``` - -A successful launch prints the path to `report/summary.md`. Raw logs and page data -stay in that unique output directory. Inspect an active phase by tailing its log; -a nonzero exit stops the launcher and preserves completed measurements. - -## Regenerate a report or recover a failed run - -Regenerate the completed report without rebuilding or rerunning benchmarks: - -```sh -uv run --no-sync python scripts/benchmarking/run_performance_report.py \ - scratch-performance-recovery-c2rz8gq0 -``` - -A report-only run validates the saved measurements and regenerates tables and figures. Reference page failures are documented in its local `excluded_comparisons.csv` and `comparison_errors.csv`; the affected series is omitted from performance metrics across all repetitions. The published tables and figures are a reviewed snapshot, not automatically overwritten by the launcher. - -For an interrupted benchmark, recover into a new bundle: - -```sh -RECOVER_FROM=/absolute/path/to/scratch-performance-refresh-RUN \ +# Include the slower optional cross-package suite: +DATASET=bo767 REPEATS=3 RUN_COMPARISON=1 \ bash scripts/benchmarking/refresh_performance_benchmarks.sh -``` - -Recovery requires matching native binary/extension hashes, core packages, Python and machine, plus the original dataset hashes. It links the original native/replay artifacts, so keep the original bundle. It records comparison provenance separately and can reuse compatible completed checkpoints from an interrupted recovery. `BUILD_MODE=clean` is incompatible with recovery. -Reference-enabled refreshes preflight the known PyMuPDF page in a subprocess, when that PDF is present. Completed comparison series are checkpointed after their timers stop. If measurements finished and only reporting failed, use the report generator directly instead of rerunning recovery. +# Preview publication from an existing run; no benchmarks are rerun: +uv run python scripts/tools/update_performance_docs.py --performance-run-dir scratch-performance-refresh-RUN -To inspect complete native/replay results while excluding comparisons: +# Apply after reviewing the preview: +uv run python scripts/tools/update_performance_docs.py --performance-run-dir scratch-performance-refresh-RUN --apply -```sh -uv run --no-sync python scripts/benchmarking/run_performance_report.py \ - /absolute/path/to/scratch-performance-refresh-RUN --core-only +# Check published file sizes, manifests and links: +python3 scripts/tools/update_performance_docs.py --check ``` -This writes `report-core/`, explicitly labeling comparisons as excluded. See [the benchmarking tools README](../scripts/benchmarking/README.md) for detailed runner, output and recovery options. The older `run_performance_eval.py` uses arrival-gap-based scaling plots; use the report's corpus-wall-time figures for the comparisons on this page. - -Performance measurements are separate from rendering or extraction quality: see [quality benchmarks](quality_benchmarks.md). Earlier measurements are retained in [legacy performance benchmarks](legacy/legacy_performance_benchmarks.md). +Run on an otherwise idle machine. Dataset verification reads files before timing, +so these are not cold-cache benchmarks. Native/replay profiles run in fresh +processes. Memory-off cross-package suites retain their shared-process execution; +peak/profile comparison series use fresh processes. + +The publisher validates raw measurements using the report generator. Omitted +optional suites are labeled “not collected”; missing expected measurements fail +validation. An absent documentation placeholder is skipped and reported. +Custom `--setup-id` values support older bundles with incomplete machine metadata. +An older run requires `--allow-older` before replacing a newer publication. + +Only referenced summary PNGs, Markdown, compact provenance and a publication +manifest enter the repository. Raw logs, CSV/JSONL and detailed RSS timelines stay +local. Publication is preview-only by default and includes byte sizes and net size +change. Size limits are enforced before docs are written and checked in CI: +100 KiB per Markdown/JSON file, 500 KiB per image, 5 MiB per dataset/setup publication, +and 25 MiB for the performance documentation tree, including historical assets. +Limits live in [the publication configuration](../scripts/tools/update_performance_docs_artifacts/performance_docs_limits.json). +Replacing a publication removes obsolete files owned by its manifest; Git retains +history. Publish selected refreshes because replacing binary assets still grows Git history. + +See [the benchmarking tools README](../scripts/benchmarking/README.md) for build, +pinned datasets, standalone report generation and failed-run recovery. Earlier +V1/V2 measurements remain in [legacy performance benchmarks](legacy/legacy_performance_benchmarks.md). diff --git a/docs/performance_benchmarks/2026-11-bo767-m3-max/comparison_parse_render_runtime.png b/docs/performance_benchmarks/2026-11-bo767-m3-max/comparison_parse_render_runtime.png deleted file mode 100644 index 7b35ab2e2..000000000 Binary files a/docs/performance_benchmarks/2026-11-bo767-m3-max/comparison_parse_render_runtime.png and /dev/null differ diff --git a/docs/performance_benchmarks/2026-11-bo767-m3-max/comparison_parse_render_scaling.png b/docs/performance_benchmarks/2026-11-bo767-m3-max/comparison_parse_render_scaling.png deleted file mode 100644 index b481e6367..000000000 Binary files a/docs/performance_benchmarks/2026-11-bo767-m3-max/comparison_parse_render_scaling.png and /dev/null differ diff --git a/docs/performance_benchmarks/2026-11-bo767-m3-max/comparison_parse_runtime.png b/docs/performance_benchmarks/2026-11-bo767-m3-max/comparison_parse_runtime.png deleted file mode 100644 index 795a06965..000000000 Binary files a/docs/performance_benchmarks/2026-11-bo767-m3-max/comparison_parse_runtime.png and /dev/null differ diff --git a/docs/performance_benchmarks/2026-11-bo767-m3-max/comparison_parse_scaling.png b/docs/performance_benchmarks/2026-11-bo767-m3-max/comparison_parse_scaling.png deleted file mode 100644 index eb34df1d8..000000000 Binary files a/docs/performance_benchmarks/2026-11-bo767-m3-max/comparison_parse_scaling.png and /dev/null differ diff --git a/docs/performance_benchmarks/2026-11-bo767-m3-max/native_render_only_distribution.png b/docs/performance_benchmarks/2026-11-bo767-m3-max/native_render_only_distribution.png deleted file mode 100644 index 3be4be266..000000000 Binary files a/docs/performance_benchmarks/2026-11-bo767-m3-max/native_render_only_distribution.png and /dev/null differ diff --git a/docs/performance_benchmarks/2026-11-bo767-m3-max/native_render_only_phases.png b/docs/performance_benchmarks/2026-11-bo767-m3-max/native_render_only_phases.png deleted file mode 100644 index a04ca3ba9..000000000 Binary files a/docs/performance_benchmarks/2026-11-bo767-m3-max/native_render_only_phases.png and /dev/null differ diff --git a/docs/performance_benchmarks/2026-11-bo767-m3-max/native_render_only_runtime.png b/docs/performance_benchmarks/2026-11-bo767-m3-max/native_render_only_runtime.png deleted file mode 100644 index 5015c9549..000000000 Binary files a/docs/performance_benchmarks/2026-11-bo767-m3-max/native_render_only_runtime.png and /dev/null differ diff --git a/docs/performance_benchmarks/2026-11-bo767-m3-max/native_render_only_scaling.png b/docs/performance_benchmarks/2026-11-bo767-m3-max/native_render_only_scaling.png deleted file mode 100644 index 0e5e5d602..000000000 Binary files a/docs/performance_benchmarks/2026-11-bo767-m3-max/native_render_only_scaling.png and /dev/null differ diff --git a/docs/performance_benchmarks/2026-11-bo767-m3-max/native_render_with_text_distribution.png b/docs/performance_benchmarks/2026-11-bo767-m3-max/native_render_with_text_distribution.png deleted file mode 100644 index 6076c3c13..000000000 Binary files a/docs/performance_benchmarks/2026-11-bo767-m3-max/native_render_with_text_distribution.png and /dev/null differ diff --git a/docs/performance_benchmarks/2026-11-bo767-m3-max/native_render_with_text_phases.png b/docs/performance_benchmarks/2026-11-bo767-m3-max/native_render_with_text_phases.png deleted file mode 100644 index 418ff015f..000000000 Binary files a/docs/performance_benchmarks/2026-11-bo767-m3-max/native_render_with_text_phases.png and /dev/null differ diff --git a/docs/performance_benchmarks/2026-11-bo767-m3-max/native_render_with_text_runtime.png b/docs/performance_benchmarks/2026-11-bo767-m3-max/native_render_with_text_runtime.png deleted file mode 100644 index e7c3dc04a..000000000 Binary files a/docs/performance_benchmarks/2026-11-bo767-m3-max/native_render_with_text_runtime.png and /dev/null differ diff --git a/docs/performance_benchmarks/2026-11-bo767-m3-max/native_render_with_text_scaling.png b/docs/performance_benchmarks/2026-11-bo767-m3-max/native_render_with_text_scaling.png deleted file mode 100644 index 499fd797a..000000000 Binary files a/docs/performance_benchmarks/2026-11-bo767-m3-max/native_render_with_text_scaling.png and /dev/null differ diff --git a/docs/performance_benchmarks/2026-11-bo767-m3-max/provenance.json b/docs/performance_benchmarks/2026-11-bo767-m3-max/provenance.json deleted file mode 100644 index 25b63ed82..000000000 --- a/docs/performance_benchmarks/2026-11-bo767-m3-max/provenance.json +++ /dev/null @@ -1,208 +0,0 @@ -{ - "schema_version": 1, - "publication_date": "2026-11-07", - "measurement_dates": [ - "2026-11-06", - "2026-11-07" - ], - "source_bundles": { - "core": "scratch-performance-refresh-20261106-132832-Ued2aI", - "comparison": "scratch-performance-recovery-c2rz8gq0" - }, - "dataset": { - "repository": "docling-project/performance-dataset-bo767", - "revision": "221c19e716d6ede5d18fd1560bc938ab19cc1a70", - "documents": 767, - "pages": 54730, - "ordered_file_identity_sha256": "2599f474f8475ac9e51c14c22e2b6d946e99fc57d84fa3ce690e52942f0d601a", - "identity_serialization": "UTF-8 JSON list in dataset file order; each item has name/pages/sha256; sort_keys=True and separators=(comma,colon)", - "page_count_min": 1, - "page_count_median": 24, - "page_count_mean": 71.35593220338983, - "page_count_max": 2227 - }, - "machine": { - "cpu": "Apple M3 Max", - "architecture": "arm64", - "memory_bytes": 68719476736, - "platform": "macOS-26.6.2-arm64-arm-64bit", - "python": "3.12.11 (main, Jul 8 2025, 20:41:49) [Clang 20.1.4 ]" - }, - "configuration": { - "repeats": 1, - "threads": [ - 1, - 4, - 8, - 12 - ], - "scale": 1.0, - "max_concurrent_results": 32, - "native_profiles": { - "render-only": { - "create_word_cells": false, - "create_line_cells": false - }, - "render-with-text": { - "create_word_cells": true, - "create_line_cells": true - } - }, - "build_mode": "incremental", - "qpdf": "12.4.2" - }, - "native_build": { - "configuration": "Release", - "flags": [ - "-O3", - "-DNDEBUG", - "-std=gnu++20" - ], - "compiler_driver": "/usr/bin/c++", - "compiler_version": "not archived", - "native_sha256": "1cd27611eaf5004b84f7c7c87d5c102d0ebef9f5f7542109ee76744220f983ab", - "extension_sha256": "bfd4ef10f7508daafd106f3ed54abda1eb70ef4f5fe16312702d3d88f28b1a2c" - }, - "core_provenance": { - "source_revision": "929a2b6ed7cc9648d9b9c3153362d6ee9542402a", - "source_diff_sha256": "31c16c5902a48af34853811c38cf454a2d7567652244585edbcd7e0feec71ff1", - "packages": { - "docling-parse": "7.24.2", - "docling-core": "2.103.1", - "pypdfium2": "5.14.0", - "pillow": "12.3.0", - "rtree": "1.4.1", - "pymupdf": "1.28.2", - "pdfplumber": "0.11.10", - "pdfminer.six": "20260107", - "pypdf": "6.21.1", - "matplotlib": "3.11.2" - }, - "script_sha256": { - "scripts/benchmarking/run_performance_eval.py": "314e2d1bdc58b031be89380baa345fe821eb32a488d42270e66ffebcb7d7bca9", - "scripts/benchmarking/run_quality_benchmarking.py": "20fdedee0694cc5a5af1d1cbe7087465a9004534e68b98dccaa89e7bdf35c1f2", - "scripts/benchmarking/_common.py": "56c2fcf9e53c4d0ae7def5253030e63242dbbc9169f1c0385c9c0812302ab1eb", - "scripts/benchmarking/run_performance_benchmarking_with_docling_settings.py": "71c743fab5c6737d034853f35c493fd6a1f831352c9eb3b3a5e81d5dffde361a", - "scripts/benchmarking/run_performance_analysis.py": "b74046443a7a08b62947e17508ba34d23855ff4abd7ffa86b75fefef55a49963", - "scripts/benchmarking/run_performance_report.py": "b9da47ab455d7399dd4eaf90abf1afc85222b2e035043704be13c83f68a8d814", - "scripts/benchmarking/run_performance_benchmarking.py": "fd9853bf53ae61019df010f01d4e662cc9914046ceb10fd84f93cc54933dff9e" - }, - "launcher_sha256": "557e3afa2caef1041bc681f76d6c53f5f86eefa521708a64227167a5fbe95bcb" - }, - "replay_source_equivalence": { - "committed_revision": "6dade98230ee60aaaa01a5db7111bda45382695d", - "commit_patch_matches_core_diff": true, - "replay_script_matches_commit": true - }, - "comparison_provenance": { - "packages": { - "docling-parse": "7.24.2", - "docling-core": "2.103.1", - "pypdfium2": "5.14.0", - "pillow": "12.3.0", - "rtree": "1.4.1", - "pymupdf": "1.27.2.3", - "pdfplumber": "0.11.10", - "pdfminer.six": "20260107", - "pypdf": "6.21.1" - }, - "python": "3.12.11 (main, Jul 8 2025, 20:41:49) [Clang 20.1.4 ]", - "platform": "macOS-26.6.2-arm64-arm-64bit", - "machine": "arm64", - "cpu": "Apple M3 Max", - "source_revision": "6dade98230ee60aaaa01a5db7111bda45382695d", - "source_diff_sha256": "0c99c8cc6f0abd842aefe140ffb971d4df75b420daaa9516fab6e4569275cd9e", - "extension_sha256": "bfd4ef10f7508daafd106f3ed54abda1eb70ef4f5fe16312702d3d88f28b1a2c", - "script_sha256": { - "_common.py": "56c2fcf9e53c4d0ae7def5253030e63242dbbc9169f1c0385c9c0812302ab1eb", - "_performance_runtime.py": "78edd356b67beafa8eda3922f2e0a0160db40f8498e2682414a014b601776310", - "recover_performance_benchmarks.py": "08c77360552fc54b08351f122ada02937d035aa787cc1a14754e94e0d675ac74", - "run_performance_analysis.py": "b74046443a7a08b62947e17508ba34d23855ff4abd7ffa86b75fefef55a49963", - "run_performance_benchmarking.py": "ba6ccce6f133fa9d68e8db5d391227f5e6e459f3c27057083f4fcfe3ead59a4c", - "run_performance_benchmarking_with_docling_settings.py": "71c743fab5c6737d034853f35c493fd6a1f831352c9eb3b3a5e81d5dffde361a", - "run_performance_eval.py": "314e2d1bdc58b031be89380baa345fe821eb32a488d42270e66ffebcb7d7bca9", - "run_performance_report.py": "138801e87848eea2734c71c0e187bc0c639a691a778ce45a0164e637ae5a4147", - "run_quality_benchmarking.py": "20fdedee0694cc5a5af1d1cbe7087465a9004534e68b98dccaa89e7bdf35c1f2" - } - }, - "report_generator_sha256": "419c1645a89829c6ec7d77d8c8a847b31749256504a34c9fd57af8507b157395", - "validated_included_measurements": 28, - "excluded_reference": { - "backend": "pypdf", - "version": "6.21.1", - "task": "parse", - "threads": 1, - "attempted_pages": 54730, - "successful_pages": 54714, - "failed_pages": 16, - "excluded_from_performance_metrics": true - }, - "numeric_summary_sha256": "edba7dcff6f3460d0e75d685b8d33aa0c39552f34d7acdb15d036511fd95f7ef", - "published_figures": { - "comparison_parse_render_runtime.png": { - "sha256": "38d5eec0faaeb8aa4ece0348ee5d0c32facc0995e8e348a608737a371de39e97", - "bytes": 102091 - }, - "comparison_parse_render_scaling.png": { - "sha256": "3d12f86179830ab492bb15845bb2fd0e8353f007838182423bf81d7977eff152", - "bytes": 97206 - }, - "comparison_parse_runtime.png": { - "sha256": "8e144fa8613aa333e636cc48c41861ee5f488012cb81e4fae5ff69dce62e9d9b", - "bytes": 94177 - }, - "comparison_parse_scaling.png": { - "sha256": "ce4409648f5cd8752aa780ffe4f334989729954807e675514e22ea4b812dd659", - "bytes": 94945 - }, - "native_render_only_distribution.png": { - "sha256": "cb3b2f58236bbf6d875ac1a9725a3b145d77b18af95ca7121c271740a7bd334d", - "bytes": 80861 - }, - "native_render_only_phases.png": { - "sha256": "087ca9ac9d84ec1c35a1a1fc4d7365a5577ba8c3f8c5a90c6b866c84b293d895", - "bytes": 74329 - }, - "native_render_only_runtime.png": { - "sha256": "2c258acca140cad311a086e562186fa2c58de8e1dd108aea9744b62981326bff", - "bytes": 83869 - }, - "native_render_only_scaling.png": { - "sha256": "3e6820762fa677fc88f85907e9e78b64eff84dd16199508182e415218178bda1", - "bytes": 105284 - }, - "native_render_with_text_distribution.png": { - "sha256": "63955e7c75985ad28ac31b17ebe3327473cdd7516b0243b4292968c69222144b", - "bytes": 85977 - }, - "native_render_with_text_phases.png": { - "sha256": "430fe1f9ab5937f4ac2adfb472d7b9b4c295fdf078db2c379e8dd1213445023d", - "bytes": 79788 - }, - "native_render_with_text_runtime.png": { - "sha256": "10000de287d1fa50c6a132c8dee587b34ec0fcced41a97f1eb40afbb6da0caf9", - "bytes": 89469 - }, - "native_render_with_text_scaling.png": { - "sha256": "e5705ebab0c41cbb57febc5173b15d83f49e918c20e609b7348d232afaa7dd5d", - "bytes": 109016 - }, - "replay_distribution.png": { - "sha256": "c7274eb7518ae9e16925da80d91b8a6d7b2e4ec550a12b0127c36a9572e74790", - "bytes": 82051 - }, - "replay_phases.png": { - "sha256": "8fd89e7c47b0ab069bf941af1ed3769236604e8a196cb998eda1ceecd4f01f7e", - "bytes": 70392 - }, - "replay_runtime.png": { - "sha256": "a97686fb5324f9df181d297aa8b89b52103d9480060535b1e0e692751eb75039", - "bytes": 94548 - }, - "replay_scaling.png": { - "sha256": "a0b154ea542e8deb2d05c945251c7dc2dc7f81640902b65e04bccd56cb3f2f6d", - "bytes": 106266 - } - }, - "raw_artifacts": "Retained in original local bundles; CSV/JSONL data are not copied into documentation." -} diff --git a/docs/performance_benchmarks/2026-11-bo767-m3-max/replay_distribution.png b/docs/performance_benchmarks/2026-11-bo767-m3-max/replay_distribution.png deleted file mode 100644 index 0c62a88d1..000000000 Binary files a/docs/performance_benchmarks/2026-11-bo767-m3-max/replay_distribution.png and /dev/null differ diff --git a/docs/performance_benchmarks/2026-11-bo767-m3-max/replay_phases.png b/docs/performance_benchmarks/2026-11-bo767-m3-max/replay_phases.png deleted file mode 100644 index 0b03c3bf3..000000000 Binary files a/docs/performance_benchmarks/2026-11-bo767-m3-max/replay_phases.png and /dev/null differ diff --git a/docs/performance_benchmarks/2026-11-bo767-m3-max/replay_runtime.png b/docs/performance_benchmarks/2026-11-bo767-m3-max/replay_runtime.png deleted file mode 100644 index 68e911b1a..000000000 Binary files a/docs/performance_benchmarks/2026-11-bo767-m3-max/replay_runtime.png and /dev/null differ diff --git a/docs/performance_benchmarks/2026-11-bo767-m3-max/replay_scaling.png b/docs/performance_benchmarks/2026-11-bo767-m3-max/replay_scaling.png deleted file mode 100644 index 71050b335..000000000 Binary files a/docs/performance_benchmarks/2026-11-bo767-m3-max/replay_scaling.png and /dev/null differ diff --git a/docs/performance_benchmarks/bo20/apple-m3-max-64gib-macos-26-6-2-arm64-arm-64bit/assets/memory_native_render-only.png b/docs/performance_benchmarks/bo20/apple-m3-max-64gib-macos-26-6-2-arm64-arm-64bit/assets/memory_native_render-only.png new file mode 100644 index 000000000..9e28bc6d3 Binary files /dev/null and b/docs/performance_benchmarks/bo20/apple-m3-max-64gib-macos-26-6-2-arm64-arm-64bit/assets/memory_native_render-only.png differ diff --git a/docs/performance_benchmarks/bo20/apple-m3-max-64gib-macos-26-6-2-arm64-arm-64bit/assets/memory_native_render-with-text.png b/docs/performance_benchmarks/bo20/apple-m3-max-64gib-macos-26-6-2-arm64-arm-64bit/assets/memory_native_render-with-text.png new file mode 100644 index 000000000..302b0c911 Binary files /dev/null and b/docs/performance_benchmarks/bo20/apple-m3-max-64gib-macos-26-6-2-arm64-arm-64bit/assets/memory_native_render-with-text.png differ diff --git a/docs/performance_benchmarks/bo20/apple-m3-max-64gib-macos-26-6-2-arm64-arm-64bit/assets/memory_replay_render.png b/docs/performance_benchmarks/bo20/apple-m3-max-64gib-macos-26-6-2-arm64-arm-64bit/assets/memory_replay_render.png new file mode 100644 index 000000000..fae3bb9ab Binary files /dev/null and b/docs/performance_benchmarks/bo20/apple-m3-max-64gib-macos-26-6-2-arm64-arm-64bit/assets/memory_replay_render.png differ diff --git a/docs/performance_benchmarks/bo20/apple-m3-max-64gib-macos-26-6-2-arm64-arm-64bit/assets/native_render_only_distribution.png b/docs/performance_benchmarks/bo20/apple-m3-max-64gib-macos-26-6-2-arm64-arm-64bit/assets/native_render_only_distribution.png new file mode 100644 index 000000000..224d282d4 Binary files /dev/null and b/docs/performance_benchmarks/bo20/apple-m3-max-64gib-macos-26-6-2-arm64-arm-64bit/assets/native_render_only_distribution.png differ diff --git a/docs/performance_benchmarks/bo20/apple-m3-max-64gib-macos-26-6-2-arm64-arm-64bit/assets/native_render_only_phases.png b/docs/performance_benchmarks/bo20/apple-m3-max-64gib-macos-26-6-2-arm64-arm-64bit/assets/native_render_only_phases.png new file mode 100644 index 000000000..17d41d9ce Binary files /dev/null and b/docs/performance_benchmarks/bo20/apple-m3-max-64gib-macos-26-6-2-arm64-arm-64bit/assets/native_render_only_phases.png differ diff --git a/docs/performance_benchmarks/bo20/apple-m3-max-64gib-macos-26-6-2-arm64-arm-64bit/assets/native_render_only_runtime.png b/docs/performance_benchmarks/bo20/apple-m3-max-64gib-macos-26-6-2-arm64-arm-64bit/assets/native_render_only_runtime.png new file mode 100644 index 000000000..b072a300d Binary files /dev/null and b/docs/performance_benchmarks/bo20/apple-m3-max-64gib-macos-26-6-2-arm64-arm-64bit/assets/native_render_only_runtime.png differ diff --git a/docs/performance_benchmarks/bo20/apple-m3-max-64gib-macos-26-6-2-arm64-arm-64bit/assets/native_render_only_scaling.png b/docs/performance_benchmarks/bo20/apple-m3-max-64gib-macos-26-6-2-arm64-arm-64bit/assets/native_render_only_scaling.png new file mode 100644 index 000000000..dfd0ebd8f Binary files /dev/null and b/docs/performance_benchmarks/bo20/apple-m3-max-64gib-macos-26-6-2-arm64-arm-64bit/assets/native_render_only_scaling.png differ diff --git a/docs/performance_benchmarks/bo20/apple-m3-max-64gib-macos-26-6-2-arm64-arm-64bit/assets/native_render_with_text_distribution.png b/docs/performance_benchmarks/bo20/apple-m3-max-64gib-macos-26-6-2-arm64-arm-64bit/assets/native_render_with_text_distribution.png new file mode 100644 index 000000000..99eb8c588 Binary files /dev/null and b/docs/performance_benchmarks/bo20/apple-m3-max-64gib-macos-26-6-2-arm64-arm-64bit/assets/native_render_with_text_distribution.png differ diff --git a/docs/performance_benchmarks/bo20/apple-m3-max-64gib-macos-26-6-2-arm64-arm-64bit/assets/native_render_with_text_phases.png b/docs/performance_benchmarks/bo20/apple-m3-max-64gib-macos-26-6-2-arm64-arm-64bit/assets/native_render_with_text_phases.png new file mode 100644 index 000000000..1489bdd7f Binary files /dev/null and b/docs/performance_benchmarks/bo20/apple-m3-max-64gib-macos-26-6-2-arm64-arm-64bit/assets/native_render_with_text_phases.png differ diff --git a/docs/performance_benchmarks/bo20/apple-m3-max-64gib-macos-26-6-2-arm64-arm-64bit/assets/native_render_with_text_runtime.png b/docs/performance_benchmarks/bo20/apple-m3-max-64gib-macos-26-6-2-arm64-arm-64bit/assets/native_render_with_text_runtime.png new file mode 100644 index 000000000..2deece19c Binary files /dev/null and b/docs/performance_benchmarks/bo20/apple-m3-max-64gib-macos-26-6-2-arm64-arm-64bit/assets/native_render_with_text_runtime.png differ diff --git a/docs/performance_benchmarks/bo20/apple-m3-max-64gib-macos-26-6-2-arm64-arm-64bit/assets/native_render_with_text_scaling.png b/docs/performance_benchmarks/bo20/apple-m3-max-64gib-macos-26-6-2-arm64-arm-64bit/assets/native_render_with_text_scaling.png new file mode 100644 index 000000000..2ca909586 Binary files /dev/null and b/docs/performance_benchmarks/bo20/apple-m3-max-64gib-macos-26-6-2-arm64-arm-64bit/assets/native_render_with_text_scaling.png differ diff --git a/docs/performance_benchmarks/bo20/apple-m3-max-64gib-macos-26-6-2-arm64-arm-64bit/assets/replay_distribution.png b/docs/performance_benchmarks/bo20/apple-m3-max-64gib-macos-26-6-2-arm64-arm-64bit/assets/replay_distribution.png new file mode 100644 index 000000000..105a18c5b Binary files /dev/null and b/docs/performance_benchmarks/bo20/apple-m3-max-64gib-macos-26-6-2-arm64-arm-64bit/assets/replay_distribution.png differ diff --git a/docs/performance_benchmarks/bo20/apple-m3-max-64gib-macos-26-6-2-arm64-arm-64bit/assets/replay_phases.png b/docs/performance_benchmarks/bo20/apple-m3-max-64gib-macos-26-6-2-arm64-arm-64bit/assets/replay_phases.png new file mode 100644 index 000000000..3319195f5 Binary files /dev/null and b/docs/performance_benchmarks/bo20/apple-m3-max-64gib-macos-26-6-2-arm64-arm-64bit/assets/replay_phases.png differ diff --git a/docs/performance_benchmarks/bo20/apple-m3-max-64gib-macos-26-6-2-arm64-arm-64bit/assets/replay_runtime.png b/docs/performance_benchmarks/bo20/apple-m3-max-64gib-macos-26-6-2-arm64-arm-64bit/assets/replay_runtime.png new file mode 100644 index 000000000..1a0b855f0 Binary files /dev/null and b/docs/performance_benchmarks/bo20/apple-m3-max-64gib-macos-26-6-2-arm64-arm-64bit/assets/replay_runtime.png differ diff --git a/docs/performance_benchmarks/bo20/apple-m3-max-64gib-macos-26-6-2-arm64-arm-64bit/assets/replay_scaling.png b/docs/performance_benchmarks/bo20/apple-m3-max-64gib-macos-26-6-2-arm64-arm-64bit/assets/replay_scaling.png new file mode 100644 index 000000000..173239d12 Binary files /dev/null and b/docs/performance_benchmarks/bo20/apple-m3-max-64gib-macos-26-6-2-arm64-arm-64bit/assets/replay_scaling.png differ diff --git a/docs/performance_benchmarks/bo20/apple-m3-max-64gib-macos-26-6-2-arm64-arm-64bit/index.md b/docs/performance_benchmarks/bo20/apple-m3-max-64gib-macos-26-6-2-arm64-arm-64bit/index.md new file mode 100644 index 000000000..4a88b92e7 --- /dev/null +++ b/docs/performance_benchmarks/bo20/apple-m3-max-64gib-macos-26-6-2-arm64-arm-64bit/index.md @@ -0,0 +1,211 @@ + +# Performance results: Bo20 / Apple M3 Max / 64 GiB / macOS + + +[Methodology and all setups](../../../performance_benchmarks.md). + + +## Setup + +Run: `scratch-performance-refresh-20261112-123829-Q21HfP`; collected 2026-11-12T12:38:29. + +CPU: **Apple M3 Max**; architecture: arm64; RAM: 64 GiB; OS: macOS-26.6.2-arm64-arm-64bit. + +Dataset: `docling-project/performance-dataset-bo20` at `65b9971b3fa939fe4ef89be678237e6928049b67`; 20 PDFs / 496 pages. + +Repetitions: 3; threads: [1, 4, 8, 12]; render scale: 1.0; queue capacity: 32. + +Values are medians with observed min-max ranges. One repetition provides no variability estimate. Speedup and efficiency use the same workload/task one-thread baseline. + + + + + + + +## Native C++ scaling + +| Backend | Version | Task | Threads | Runs | Wall median (s) | Min-max (s) | Pages/s | Speedup vs Docling 1t | Efficiency | +|---------------|-----------|------------------|-----------|--------|-------------------|---------------|-----------|-------------------------|--------------| +| docling-parse | 7.25.0 | render-only | 1 | 3 | 6.587 | 6.572-6.622 | 75.3 | 1.00x | 100.0% | +| docling-parse | 7.25.0 | render-only | 4 | 3 | 1.755 | 1.752-1.755 | 282.7 | 3.75x | 93.9% | +| docling-parse | 7.25.0 | render-only | 8 | 3 | 0.910 | 0.909-0.914 | 544.9 | 7.24x | 90.5% | +| docling-parse | 7.25.0 | render-only | 12 | 3 | 0.645 | 0.645-0.646 | 768.4 | 10.21x | 85.0% | +| docling-parse | 7.25.0 | render-with-text | 1 | 3 | 6.719 | 6.715-6.789 | 73.8 | 1.00x | 100.0% | +| docling-parse | 7.25.0 | render-with-text | 4 | 3 | 1.789 | 1.784-1.799 | 277.2 | 3.76x | 93.9% | +| docling-parse | 7.25.0 | render-with-text | 8 | 3 | 0.930 | 0.926-0.931 | 533.6 | 7.23x | 90.4% | +| docling-parse | 7.25.0 | render-with-text | 12 | 3 | 0.660 | 0.653-0.663 | 751.0 | 10.17x | 84.8% | + +### Render page-images + +Timing: native wall_s; includes collection/CSV; excludes document preload. + +![native render only runtime](assets/native_render_only_runtime.png) + +![native render only scaling](assets/native_render_only_scaling.png) + +![native render only phases](assets/native_render_only_phases.png) + +![native render only distribution](assets/native_render_only_distribution.png) + +Stage totals sum overlapping worker elapsed time, not corpus wall or CPU time. Distributions describe native page-worker or replay document durations, not end-to-end page latency. + +
+Reproduce these measurements + +```bash +set -euo pipefail +PDF_ROOT=/Users/taa/.cache/huggingface/hub/datasets--docling-project--performance-dataset-bo20/snapshots/65b9971b3fa939fe4ef89be678237e6928049b67/pdf +REPRO_OUT=$(mktemp -d "$PWD/scratch-performance-reproduce-XXXXXX") +observe() { + local memory_output=$1; shift + .venv/bin/python scripts/benchmarking/_performance_memory.py --memory-mode profile --memory-interval-ms 50 \ + --output "${memory_output}.memory.json" -- "$@" +} +for ((r=1; r<=3; r++)); do + for t in 1 4 8 12; do + observe "$REPRO_OUT/r${r}-t${t}" ./build/run_scaling.exe "$PDF_ROOT" -v WARNING --recursive --mode render \ + --threads "$t" --max-concurrent-results 32 --scale 1.0 \ + --create-word-cells=false --create-line-cells=false \ + --enable-timing --timing-csv "$REPRO_OUT/r${r}-t${t}.csv" \ + 2>&1 | tee "$REPRO_OUT/r${r}-t${t}.log" + done +done +``` + +
+ +### Render page-images and construct word/line text geometry + +Timing: native wall_s; includes collection/CSV; excludes document preload. + +![native render with text runtime](assets/native_render_with_text_runtime.png) + +![native render with text scaling](assets/native_render_with_text_scaling.png) + +![native render with text phases](assets/native_render_with_text_phases.png) + +![native render with text distribution](assets/native_render_with_text_distribution.png) + +Stage totals sum overlapping worker elapsed time, not corpus wall or CPU time. Distributions describe native page-worker or replay document durations, not end-to-end page latency. + +
+Reproduce these measurements + +```bash +set -euo pipefail +PDF_ROOT=/Users/taa/.cache/huggingface/hub/datasets--docling-project--performance-dataset-bo20/snapshots/65b9971b3fa939fe4ef89be678237e6928049b67/pdf +REPRO_OUT=$(mktemp -d "$PWD/scratch-performance-reproduce-XXXXXX") +observe() { + local memory_output=$1; shift + .venv/bin/python scripts/benchmarking/_performance_memory.py --memory-mode profile --memory-interval-ms 50 \ + --output "${memory_output}.memory.json" -- "$@" +} +for ((r=1; r<=3; r++)); do + for t in 1 4 8 12; do + observe "$REPRO_OUT/r${r}-t${t}" ./build/run_scaling.exe "$PDF_ROOT" -v WARNING --recursive --mode render \ + --threads "$t" --max-concurrent-results 32 --scale 1.0 \ + --create-word-cells=true --create-line-cells=true \ + --enable-timing --timing-csv "$REPRO_OUT/r${r}-t${t}.csv" \ + 2>&1 | tee "$REPRO_OUT/r${r}-t${t}.log" + done +done +``` + +
+ + + +## Python scaling with Docling settings + +| Backend | Version | Task | Threads | Runs | Wall median (s) | Min-max (s) | Pages/s | Speedup vs Docling 1t | Efficiency | +|---------------|-----------|--------|-----------|--------|-------------------|---------------|-----------|-------------------------|--------------| +| pypdfium2 | 5.14.0 | render | 1 | 3 | 11.815 | 11.755-11.916 | 42.0 | 0.66x | — | +| docling-parse | 7.25.0 | render | 1 | 3 | 7.806 | 7.758-7.825 | 63.5 | 1.00x | 100.0% | +| docling-parse | 7.25.0 | render | 4 | 3 | 3.058 | 3.011-3.067 | 162.2 | 2.55x | 63.8% | +| docling-parse | 7.25.0 | render | 8 | 3 | 2.299 | 2.281-2.314 | 215.7 | 3.40x | 42.4% | +| docling-parse | 7.25.0 | render | 12 | 3 | 2.173 | 2.159-2.221 | 228.3 | 3.59x | 29.9% | + +### Page-images, word/line text geometry and bitmap bounding boxes + +Timing: sum of document total_s; includes construction/load, processing and unload. + +![replay runtime](assets/replay_runtime.png) + +![replay scaling](assets/replay_scaling.png) + +![replay phases](assets/replay_phases.png) + +![replay distribution](assets/replay_distribution.png) + +Stage totals sum overlapping worker elapsed time, not corpus wall or CPU time. Distributions describe native page-worker or replay document durations, not end-to-end page latency. + +
+Reproduce these measurements + +```bash +set -euo pipefail +PDF_ROOT=/Users/taa/.cache/huggingface/hub/datasets--docling-project--performance-dataset-bo20/snapshots/65b9971b3fa939fe4ef89be678237e6928049b67/pdf +REPRO_OUT=$(mktemp -d "$PWD/scratch-performance-reproduce-XXXXXX") +observe() { + local memory_output=$1; shift + .venv/bin/python scripts/benchmarking/_performance_memory.py --memory-mode profile --memory-interval-ms 50 \ + --output "${memory_output}.memory.json" -- "$@" +} +for ((r=1; r<=3; r++)); do + observe "$REPRO_OUT/r${r}-pdfium" .venv/bin/python scripts/benchmarking/run_performance_benchmarking_with_docling_settings.py "$PDF_ROOT" \ + --backend pdfium --dataset-revision 65b9971b3fa939fe4ef89be678237e6928049b67 \ + --output-dir "$REPRO_OUT/r${r}-pdfium" 2>&1 | tee "$REPRO_OUT/r${r}-pdfium.log" + for t in 1 4 8 12; do + observe "$REPRO_OUT/r${r}-t${t}" .venv/bin/python scripts/benchmarking/run_performance_benchmarking_with_docling_settings.py "$PDF_ROOT" \ + --backend threaded --threads "$t" --dataset-revision 65b9971b3fa939fe4ef89be678237e6928049b67 \ + --output-dir "$REPRO_OUT/r${r}-t${t}" 2>&1 | tee "$REPRO_OUT/r${r}-t${t}.log" + done +done +``` + +
+ + + +## Python cross-package comparison + +Not collected in this run. + + + +## Process Memory + +OS high-water marks cover one backend process and all its threads, including startup, loading and teardown. They exclude the monitor/logger and are not parser-only heap sizes. Timing retains each workload's existing boundaries. Missing observations are unavailable, never zero. + +| Workload | Backend | Task | Threads | Peaks/runs | Median MiB | Min MiB | Largest peak MiB | Capture | +|------------|---------------|------------------|-----------|--------------|--------------|-----------|--------------------|-----------| +| native | docling-parse | render-only | 1 | 3/3 | 543.78 | 541.97 | 545.81 | profile | +| native | docling-parse | render-only | 4 | 3/3 | 734.17 | 727.98 | 756.83 | profile | +| native | docling-parse | render-only | 8 | 3/3 | 913.77 | 912.28 | 918.95 | profile | +| native | docling-parse | render-only | 12 | 3/3 | 1034.33 | 1008.66 | 1044.36 | profile | +| native | docling-parse | render-with-text | 1 | 3/3 | 546.31 | 545.61 | 548.48 | profile | +| native | docling-parse | render-with-text | 4 | 3/3 | 744.83 | 729.03 | 751.19 | profile | +| native | docling-parse | render-with-text | 8 | 3/3 | 909.89 | 888.56 | 929.27 | profile | +| native | docling-parse | render-with-text | 12 | 3/3 | 1037.03 | 1034.78 | 1057.19 | profile | +| replay | pdfium | render | 1 | 3/3 | 626.72 | 623.80 | 631.19 | profile | +| replay | threaded | render | 1 | 3/3 | 700.47 | 680.89 | 704.22 | profile | +| replay | threaded | render | 4 | 3/3 | 844.12 | 834.98 | 845.12 | profile | +| replay | threaded | render | 8 | 3/3 | 1008.50 | 996.47 | 1040.77 | profile | +| replay | threaded | render | 12 | 3/3 | 1169.38 | 1152.45 | 1171.09 | profile | + +`existing` denotes a previously recorded replay RUSAGE_SELF peak. Reused runs have no new timeline. Peak/profile comparison series run in fresh processes; memory-off comparisons retain their historical shared-process execution. Compare timing only under matching instrumentation and isolation. Sampled maxima are separate from OS peaks and can miss short spikes. Methods and scopes are recorded in publication provenance. + + +![Process peak RSS; median and observed range](assets/memory_native_render-only.png) + +![Process peak RSS; median and observed range](assets/memory_native_render-with-text.png) + +![Process peak RSS; median and observed range](assets/memory_replay_render.png) + + + +## Provenance + +[Dataset, source, build, packages and memory methods](provenance.json). Raw measurements and detailed RSS timelines remain in the local run bundle. + diff --git a/docs/performance_benchmarks/bo20/apple-m3-max-64gib-macos-26-6-2-arm64-arm-64bit/provenance.json b/docs/performance_benchmarks/bo20/apple-m3-max-64gib-macos-26-6-2-arm64-arm-64bit/provenance.json new file mode 100644 index 000000000..605df9ce2 --- /dev/null +++ b/docs/performance_benchmarks/bo20/apple-m3-max-64gib-macos-26-6-2-arm64-arm-64bit/provenance.json @@ -0,0 +1,90 @@ +{ + "build_mode": "incremental", + "canvas_checks": [], + "comparison_memory": { + "interval_ms": 50, + "mode": "profile" + }, + "dataset": { + "repository": "docling-project/performance-dataset-bo20", + "requested_revision": "65b9971b3fa939fe4ef89be678237e6928049b67", + "revision": "65b9971b3fa939fe4ef89be678237e6928049b67" + }, + "documents": 20, + "excluded_comparisons": [], + "extension_sha256": "017556b0bc007d831a0d40ccbda9b09285d4680378102de7b10f5b44143ef9cf", + "identity_serialization": "UTF-8 JSON list in dataset order; name/pages/sha256; sort_keys=True; separators=(comma,colon)", + "launcher_sha256": "259ffce1494f2e82c72381df5cf71f26ae6d9820cd228fb3283c9575c916e99a", + "machine": { + "cpu": "Apple M3 Max", + "machine": "arm64", + "memory_bytes": "68719476736", + "platform": "macOS-26.6.2-arm64-arm-64bit" + }, + "max_concurrent_results": 32, + "measured_at": "2026-11-12T12:38:29", + "memory_collection": { + "interval_ms": 50, + "mode": "profile" + }, + "memory_methods": [ + [ + "profile", + "wait4.ru_maxrss", + "process lifetime including startup and teardown" + ] + ], + "native_profiles": { + "render-only": { + "create_line_cells": false, + "create_word_cells": false + }, + "render-with-text": { + "create_line_cells": true, + "create_word_cells": true + } + }, + "native_sha256": "ce85c518f6a9c11c83ce2dcf4ccf602ffa2d5bf37d357c53234a3a8c31ec2850", + "ordered_file_identity_sha256": "45d2a30858ab243be75e30ab9aebe5ae73296808cdcbea1d32ccac3f349c9f60", + "packages": { + "docling-core": "2.103.1", + "docling-parse": "7.25.0", + "matplotlib": "3.11.2", + "pdfminer.six": "20260107", + "pdfplumber": "0.11.10", + "pillow": "12.3.0", + "pymupdf": "1.27.2.3", + "pypdf": "6.21.1", + "pypdfium2": "5.14.0", + "rtree": "1.4.1" + }, + "pages": 496, + "python": "3.12.11 (main, Jul 8 2025, 20:41:49) [Clang 20.1.4 ]", + "qpdf": "12.4.2", + "repeats": 3, + "run": "scratch-performance-refresh-20261112-123829-Q21HfP", + "scale": 1.0, + "schema_version": 1, + "script_sha256": { + "scripts/benchmarking/_common.py": "56c2fcf9e53c4d0ae7def5253030e63242dbbc9169f1c0385c9c0812302ab1eb", + "scripts/benchmarking/_performance_memory.py": "1b9b2cfe76d44872916470c40999940782e7ea35ce7f4e165c6365130ce33baa", + "scripts/benchmarking/_performance_memory_report.py": "a4e057c511d1f2bb4e526fe51c345bffd3c88cbb8bd66ddbb5bdfe7b5f45faca", + "scripts/benchmarking/_performance_runtime.py": "50abe947f1cc1223da7dffc9da6f2bd03a4fe9d54db739a3caf613e167a28fc1", + "scripts/benchmarking/_performance_worker.py": "a3e272eaca78b35b68839a32d3b049cb63d0c3afdd7484a9f228aff87f0d2453", + "scripts/benchmarking/recover_performance_benchmarks.py": "f563f6d6b568a5f430f5ffd3267a1eb90943e651ee736f5f8deda64ae1307192", + "scripts/benchmarking/run_performance_analysis.py": "96453dc08db69ec4a90e5da506c31fb0c08194470d8fea3a15043fcce99da5a2", + "scripts/benchmarking/run_performance_benchmarking.py": "7d7fbf6542a07faa70493b244293336f91af3abb493bbb04a7115fa1cb67ddf0", + "scripts/benchmarking/run_performance_benchmarking_with_docling_settings.py": "71c743fab5c6737d034853f35c493fd6a1f831352c9eb3b3a5e81d5dffde361a", + "scripts/benchmarking/run_performance_eval.py": "314e2d1bdc58b031be89380baa345fe821eb32a488d42270e66ffebcb7d7bca9", + "scripts/benchmarking/run_performance_report.py": "6e28e951d290f604d3630c1653e098e4470896bd804392ce3b02a5b993c8a38c", + "scripts/benchmarking/run_quality_benchmarking.py": "20fdedee0694cc5a5af1d1cbe7087465a9004534e68b98dccaa89e7bdf35c1f2" + }, + "source_diff_sha256": "0a9d3f27c9ef2fc3f53c9d900d9737ffd7a1f91dd99b8f106b126be2cda286ce", + "source_revision": "852d6b66c327c01d41e1ce7b1edea31d538d0f01", + "threads": [ + 1, + 4, + 8, + 12 + ] +} diff --git a/docs/performance_benchmarks/bo20/apple-m3-max-64gib-macos-26-6-2-arm64-arm-64bit/publication.json b/docs/performance_benchmarks/bo20/apple-m3-max-64gib-macos-26-6-2-arm64-arm-64bit/publication.json new file mode 100644 index 000000000..27cee538d --- /dev/null +++ b/docs/performance_benchmarks/bo20/apple-m3-max-64gib-macos-26-6-2-arm64-arm-64bit/publication.json @@ -0,0 +1,37 @@ +{ + "dataset": "bo20", + "files": { + "assets/memory_native_render-only.png": "3de5f3eff25237ba598e31dd0db6a4fdf298918e0983fb6c56c2f0e625cfba82", + "assets/memory_native_render-with-text.png": "4c2968fba87ae178a1966c2d54014a2be6d2a41974a335e74597957eae5161b7", + "assets/memory_replay_render.png": "56118663fdbff3237c2f26869cb64b9ad2e12d495e16869574f322b197f15b57", + "assets/native_render_only_distribution.png": "03d3adfde2ab38143bbd0aa4939fbae7273fdaecdb719597f125c35d741e8195", + "assets/native_render_only_phases.png": "54b85f0d6b22df04bd5d7213bd930c9fbd58e27fbdaefb31974a11af6fd045b1", + "assets/native_render_only_runtime.png": "a3844604165f021eb4dfa3a61d28d6ca10cdde13b49f14e0c6cfeeaf66282acc", + "assets/native_render_only_scaling.png": "836f24e1e42c9e14daec8e9544798d2ad8e7b9efacb7465e063aa7001b3cdaac", + "assets/native_render_with_text_distribution.png": "39bde0b56b8015ca86fdfc0f927be48635abb0afb1a2f8cffc76578222456f74", + "assets/native_render_with_text_phases.png": "18d4e2791f2f322f22ff45be804af2ec95cec3f236faa9ae1d72d09f3070c0c5", + "assets/native_render_with_text_runtime.png": "85f67e66adc86499cb2a6dc1980f3cc53f721b0cc94348eb25af25c4f3badfa8", + "assets/native_render_with_text_scaling.png": "3aa5f2f160dfa95a72ceccc3ed941efd71ef828dc8ec80d38c23bea3d3ef4ef0", + "assets/replay_distribution.png": "3dbb7e443e450af014d55187b2664fe789a46fdf0fe917934c96a4737f1972fb", + "assets/replay_phases.png": "b36f7a68a8078b87d64bc98eb16c7ae137883d9e0519e7a0d5e39ccbdf89fa4c", + "assets/replay_runtime.png": "431a14d90c1aff906b536a5491dc407b023dfec2c21a598bd62a14594a7f1798", + "assets/replay_scaling.png": "94a679fcdb44b45026e48421921c659890fbd66b1c043a4e9ddbccc2b1f79fee", + "index.md": "dd1650bcc83cce0a47480eac7ebccc1885a3ab258a0ae6444ac3bef538196185", + "provenance.json": "5d3d280de7b014a752acc4bb23d2655cf239fa86cda352b4d3f0a4daf0e36374" + }, + "label": "Apple M3 Max / 64 GiB / macOS-26.6.2-arm64-arm-64bit", + "machine": { + "cpu": "Apple M3 Max", + "machine": "arm64", + "memory_bytes": 68719476736, + "platform": "macOS-26.6.2-arm64-arm-64bit" + }, + "measured_at": "2026-11-12T12:38:29", + "run": "scratch-performance-refresh-20261112-123829-Q21HfP", + "schema_version": 1, + "setup": "apple-m3-max-64gib-macos-26-6-2-arm64-arm-64bit", + "workloads": [ + "native", + "replay" + ] +} diff --git a/docs/performance_benchmarks/bo767/apple-m3-max-64gib-macos-26-6-2-arm64-arm-64bit/assets/memory_native_render-only.png b/docs/performance_benchmarks/bo767/apple-m3-max-64gib-macos-26-6-2-arm64-arm-64bit/assets/memory_native_render-only.png new file mode 100644 index 000000000..66d252d18 Binary files /dev/null and b/docs/performance_benchmarks/bo767/apple-m3-max-64gib-macos-26-6-2-arm64-arm-64bit/assets/memory_native_render-only.png differ diff --git a/docs/performance_benchmarks/bo767/apple-m3-max-64gib-macos-26-6-2-arm64-arm-64bit/assets/memory_native_render-with-text.png b/docs/performance_benchmarks/bo767/apple-m3-max-64gib-macos-26-6-2-arm64-arm-64bit/assets/memory_native_render-with-text.png new file mode 100644 index 000000000..69f6faae6 Binary files /dev/null and b/docs/performance_benchmarks/bo767/apple-m3-max-64gib-macos-26-6-2-arm64-arm-64bit/assets/memory_native_render-with-text.png differ diff --git a/docs/performance_benchmarks/bo767/apple-m3-max-64gib-macos-26-6-2-arm64-arm-64bit/assets/memory_replay_render.png b/docs/performance_benchmarks/bo767/apple-m3-max-64gib-macos-26-6-2-arm64-arm-64bit/assets/memory_replay_render.png new file mode 100644 index 000000000..6e396181c Binary files /dev/null and b/docs/performance_benchmarks/bo767/apple-m3-max-64gib-macos-26-6-2-arm64-arm-64bit/assets/memory_replay_render.png differ diff --git a/docs/performance_benchmarks/bo767/apple-m3-max-64gib-macos-26-6-2-arm64-arm-64bit/assets/native_render_only_distribution.png b/docs/performance_benchmarks/bo767/apple-m3-max-64gib-macos-26-6-2-arm64-arm-64bit/assets/native_render_only_distribution.png new file mode 100644 index 000000000..627494aec Binary files /dev/null and b/docs/performance_benchmarks/bo767/apple-m3-max-64gib-macos-26-6-2-arm64-arm-64bit/assets/native_render_only_distribution.png differ diff --git a/docs/performance_benchmarks/bo767/apple-m3-max-64gib-macos-26-6-2-arm64-arm-64bit/assets/native_render_only_phases.png b/docs/performance_benchmarks/bo767/apple-m3-max-64gib-macos-26-6-2-arm64-arm-64bit/assets/native_render_only_phases.png new file mode 100644 index 000000000..6f746d3ad Binary files /dev/null and b/docs/performance_benchmarks/bo767/apple-m3-max-64gib-macos-26-6-2-arm64-arm-64bit/assets/native_render_only_phases.png differ diff --git a/docs/performance_benchmarks/bo767/apple-m3-max-64gib-macos-26-6-2-arm64-arm-64bit/assets/native_render_only_runtime.png b/docs/performance_benchmarks/bo767/apple-m3-max-64gib-macos-26-6-2-arm64-arm-64bit/assets/native_render_only_runtime.png new file mode 100644 index 000000000..778c5b5e6 Binary files /dev/null and b/docs/performance_benchmarks/bo767/apple-m3-max-64gib-macos-26-6-2-arm64-arm-64bit/assets/native_render_only_runtime.png differ diff --git a/docs/performance_benchmarks/bo767/apple-m3-max-64gib-macos-26-6-2-arm64-arm-64bit/assets/native_render_only_scaling.png b/docs/performance_benchmarks/bo767/apple-m3-max-64gib-macos-26-6-2-arm64-arm-64bit/assets/native_render_only_scaling.png new file mode 100644 index 000000000..745ed23ad Binary files /dev/null and b/docs/performance_benchmarks/bo767/apple-m3-max-64gib-macos-26-6-2-arm64-arm-64bit/assets/native_render_only_scaling.png differ diff --git a/docs/performance_benchmarks/bo767/apple-m3-max-64gib-macos-26-6-2-arm64-arm-64bit/assets/native_render_with_text_distribution.png b/docs/performance_benchmarks/bo767/apple-m3-max-64gib-macos-26-6-2-arm64-arm-64bit/assets/native_render_with_text_distribution.png new file mode 100644 index 000000000..14aad6398 Binary files /dev/null and b/docs/performance_benchmarks/bo767/apple-m3-max-64gib-macos-26-6-2-arm64-arm-64bit/assets/native_render_with_text_distribution.png differ diff --git a/docs/performance_benchmarks/bo767/apple-m3-max-64gib-macos-26-6-2-arm64-arm-64bit/assets/native_render_with_text_phases.png b/docs/performance_benchmarks/bo767/apple-m3-max-64gib-macos-26-6-2-arm64-arm-64bit/assets/native_render_with_text_phases.png new file mode 100644 index 000000000..900d05897 Binary files /dev/null and b/docs/performance_benchmarks/bo767/apple-m3-max-64gib-macos-26-6-2-arm64-arm-64bit/assets/native_render_with_text_phases.png differ diff --git a/docs/performance_benchmarks/bo767/apple-m3-max-64gib-macos-26-6-2-arm64-arm-64bit/assets/native_render_with_text_runtime.png b/docs/performance_benchmarks/bo767/apple-m3-max-64gib-macos-26-6-2-arm64-arm-64bit/assets/native_render_with_text_runtime.png new file mode 100644 index 000000000..0e61a49e0 Binary files /dev/null and b/docs/performance_benchmarks/bo767/apple-m3-max-64gib-macos-26-6-2-arm64-arm-64bit/assets/native_render_with_text_runtime.png differ diff --git a/docs/performance_benchmarks/bo767/apple-m3-max-64gib-macos-26-6-2-arm64-arm-64bit/assets/native_render_with_text_scaling.png b/docs/performance_benchmarks/bo767/apple-m3-max-64gib-macos-26-6-2-arm64-arm-64bit/assets/native_render_with_text_scaling.png new file mode 100644 index 000000000..26ce43a8e Binary files /dev/null and b/docs/performance_benchmarks/bo767/apple-m3-max-64gib-macos-26-6-2-arm64-arm-64bit/assets/native_render_with_text_scaling.png differ diff --git a/docs/performance_benchmarks/bo767/apple-m3-max-64gib-macos-26-6-2-arm64-arm-64bit/assets/replay_distribution.png b/docs/performance_benchmarks/bo767/apple-m3-max-64gib-macos-26-6-2-arm64-arm-64bit/assets/replay_distribution.png new file mode 100644 index 000000000..8bb6eeaa4 Binary files /dev/null and b/docs/performance_benchmarks/bo767/apple-m3-max-64gib-macos-26-6-2-arm64-arm-64bit/assets/replay_distribution.png differ diff --git a/docs/performance_benchmarks/bo767/apple-m3-max-64gib-macos-26-6-2-arm64-arm-64bit/assets/replay_phases.png b/docs/performance_benchmarks/bo767/apple-m3-max-64gib-macos-26-6-2-arm64-arm-64bit/assets/replay_phases.png new file mode 100644 index 000000000..d2ad52eed Binary files /dev/null and b/docs/performance_benchmarks/bo767/apple-m3-max-64gib-macos-26-6-2-arm64-arm-64bit/assets/replay_phases.png differ diff --git a/docs/performance_benchmarks/bo767/apple-m3-max-64gib-macos-26-6-2-arm64-arm-64bit/assets/replay_runtime.png b/docs/performance_benchmarks/bo767/apple-m3-max-64gib-macos-26-6-2-arm64-arm-64bit/assets/replay_runtime.png new file mode 100644 index 000000000..cbec4599a Binary files /dev/null and b/docs/performance_benchmarks/bo767/apple-m3-max-64gib-macos-26-6-2-arm64-arm-64bit/assets/replay_runtime.png differ diff --git a/docs/performance_benchmarks/bo767/apple-m3-max-64gib-macos-26-6-2-arm64-arm-64bit/assets/replay_scaling.png b/docs/performance_benchmarks/bo767/apple-m3-max-64gib-macos-26-6-2-arm64-arm-64bit/assets/replay_scaling.png new file mode 100644 index 000000000..e3be645cd Binary files /dev/null and b/docs/performance_benchmarks/bo767/apple-m3-max-64gib-macos-26-6-2-arm64-arm-64bit/assets/replay_scaling.png differ diff --git a/docs/performance_benchmarks/bo767/apple-m3-max-64gib-macos-26-6-2-arm64-arm-64bit/index.md b/docs/performance_benchmarks/bo767/apple-m3-max-64gib-macos-26-6-2-arm64-arm-64bit/index.md new file mode 100644 index 000000000..091cd9664 --- /dev/null +++ b/docs/performance_benchmarks/bo767/apple-m3-max-64gib-macos-26-6-2-arm64-arm-64bit/index.md @@ -0,0 +1,211 @@ + +# Performance results: Bo767 / Apple M3 Max / 64 GiB / macOS + + +[Methodology and all setups](../../../performance_benchmarks.md). + + +## Setup + +Run: `scratch-performance-recovery-8vumw3sy`; collected 2026-11-12T17:27:10.253402. + +CPU: **Apple M3 Max**; architecture: arm64; RAM: 64 GiB; OS: macOS-26.6.2-arm64-arm-64bit. + +Dataset: `docling-project/performance-dataset-bo767` at `221c19e716d6ede5d18fd1560bc938ab19cc1a70`; 767 PDFs / 54,730 pages. + +Repetitions: 3; threads: [1, 4, 8, 12]; render scale: 1.0; queue capacity: 32. + +Values are medians with observed min-max ranges. One repetition provides no variability estimate. Speedup and efficiency use the same workload/task one-thread baseline. + + + + + + + +## Native C++ scaling + +| Backend | Version | Task | Threads | Runs | Wall median (s) | Min-max (s) | Pages/s | Speedup vs Docling 1t | Efficiency | +|---------------|-----------|------------------|-----------|--------|-------------------|-------------------|-----------|-------------------------|--------------| +| docling-parse | 7.25.0 | render-only | 1 | 3 | 1071.194 | 1069.906-1101.857 | 51.1 | 1.00x | 100.0% | +| docling-parse | 7.25.0 | render-only | 4 | 3 | 283.046 | 283.023-287.826 | 193.4 | 3.78x | 94.6% | +| docling-parse | 7.25.0 | render-only | 8 | 3 | 149.237 | 148.032-149.848 | 366.7 | 7.18x | 89.7% | +| docling-parse | 7.25.0 | render-only | 12 | 3 | 102.384 | 101.552-104.521 | 534.6 | 10.46x | 87.2% | +| docling-parse | 7.25.0 | render-with-text | 1 | 3 | 1101.388 | 1100.648-1115.716 | 49.7 | 1.00x | 100.0% | +| docling-parse | 7.25.0 | render-with-text | 4 | 3 | 290.714 | 290.281-293.247 | 188.3 | 3.79x | 94.7% | +| docling-parse | 7.25.0 | render-with-text | 8 | 3 | 153.550 | 151.612-154.036 | 356.4 | 7.17x | 89.7% | +| docling-parse | 7.25.0 | render-with-text | 12 | 3 | 106.039 | 104.173-106.791 | 516.1 | 10.39x | 86.6% | + +### Render page-images + +Timing: native wall_s; includes collection/CSV; excludes document preload. + +![native render only runtime](assets/native_render_only_runtime.png) + +![native render only scaling](assets/native_render_only_scaling.png) + +![native render only phases](assets/native_render_only_phases.png) + +![native render only distribution](assets/native_render_only_distribution.png) + +Stage totals sum overlapping worker elapsed time, not corpus wall or CPU time. Distributions describe native page-worker or replay document durations, not end-to-end page latency. + +
+Reproduce these measurements + +```bash +set -euo pipefail +PDF_ROOT=/Users/taa/.cache/huggingface/hub/datasets--docling-project--performance-dataset-bo767/snapshots/221c19e716d6ede5d18fd1560bc938ab19cc1a70/pdf +REPRO_OUT=$(mktemp -d "$PWD/scratch-performance-reproduce-XXXXXX") +observe() { + local memory_output=$1; shift + .venv/bin/python scripts/benchmarking/_performance_memory.py --memory-mode profile --memory-interval-ms 50 \ + --output "${memory_output}.memory.json" -- "$@" +} +for ((r=1; r<=3; r++)); do + for t in 1 4 8 12; do + observe "$REPRO_OUT/r${r}-t${t}" ./build/run_scaling.exe "$PDF_ROOT" -v WARNING --recursive --mode render \ + --threads "$t" --max-concurrent-results 32 --scale 1.0 \ + --create-word-cells=false --create-line-cells=false \ + --enable-timing --timing-csv "$REPRO_OUT/r${r}-t${t}.csv" \ + 2>&1 | tee "$REPRO_OUT/r${r}-t${t}.log" + done +done +``` + +
+ +### Render page-images and construct word/line text geometry + +Timing: native wall_s; includes collection/CSV; excludes document preload. + +![native render with text runtime](assets/native_render_with_text_runtime.png) + +![native render with text scaling](assets/native_render_with_text_scaling.png) + +![native render with text phases](assets/native_render_with_text_phases.png) + +![native render with text distribution](assets/native_render_with_text_distribution.png) + +Stage totals sum overlapping worker elapsed time, not corpus wall or CPU time. Distributions describe native page-worker or replay document durations, not end-to-end page latency. + +
+Reproduce these measurements + +```bash +set -euo pipefail +PDF_ROOT=/Users/taa/.cache/huggingface/hub/datasets--docling-project--performance-dataset-bo767/snapshots/221c19e716d6ede5d18fd1560bc938ab19cc1a70/pdf +REPRO_OUT=$(mktemp -d "$PWD/scratch-performance-reproduce-XXXXXX") +observe() { + local memory_output=$1; shift + .venv/bin/python scripts/benchmarking/_performance_memory.py --memory-mode profile --memory-interval-ms 50 \ + --output "${memory_output}.memory.json" -- "$@" +} +for ((r=1; r<=3; r++)); do + for t in 1 4 8 12; do + observe "$REPRO_OUT/r${r}-t${t}" ./build/run_scaling.exe "$PDF_ROOT" -v WARNING --recursive --mode render \ + --threads "$t" --max-concurrent-results 32 --scale 1.0 \ + --create-word-cells=true --create-line-cells=true \ + --enable-timing --timing-csv "$REPRO_OUT/r${r}-t${t}.csv" \ + 2>&1 | tee "$REPRO_OUT/r${r}-t${t}.log" + done +done +``` + +
+ + + +## Python scaling with Docling settings + +| Backend | Version | Task | Threads | Runs | Wall median (s) | Min-max (s) | Pages/s | Speedup vs Docling 1t | Efficiency | +|---------------|-----------|--------|-----------|--------|-------------------|-------------------|-----------|-------------------------|--------------| +| pypdfium2 | 5.14.0 | render | 1 | 3 | 1457.800 | 1433.924-1491.756 | 37.5 | 0.79x | — | +| docling-parse | 7.25.0 | render | 1 | 3 | 1154.735 | 1115.175-1160.067 | 47.4 | 1.00x | 100.0% | +| docling-parse | 7.25.0 | render | 4 | 3 | 367.296 | 363.292-372.417 | 149.0 | 3.14x | 78.6% | +| docling-parse | 7.25.0 | render | 8 | 3 | 279.370 | 274.860-285.899 | 195.9 | 4.13x | 51.7% | +| docling-parse | 7.25.0 | render | 12 | 3 | 254.840 | 250.698-262.578 | 214.8 | 4.53x | 37.8% | + +### Page-images, word/line text geometry and bitmap bounding boxes + +Timing: sum of document total_s; includes construction/load, processing and unload. + +![replay runtime](assets/replay_runtime.png) + +![replay scaling](assets/replay_scaling.png) + +![replay phases](assets/replay_phases.png) + +![replay distribution](assets/replay_distribution.png) + +Stage totals sum overlapping worker elapsed time, not corpus wall or CPU time. Distributions describe native page-worker or replay document durations, not end-to-end page latency. + +
+Reproduce these measurements + +```bash +set -euo pipefail +PDF_ROOT=/Users/taa/.cache/huggingface/hub/datasets--docling-project--performance-dataset-bo767/snapshots/221c19e716d6ede5d18fd1560bc938ab19cc1a70/pdf +REPRO_OUT=$(mktemp -d "$PWD/scratch-performance-reproduce-XXXXXX") +observe() { + local memory_output=$1; shift + .venv/bin/python scripts/benchmarking/_performance_memory.py --memory-mode profile --memory-interval-ms 50 \ + --output "${memory_output}.memory.json" -- "$@" +} +for ((r=1; r<=3; r++)); do + observe "$REPRO_OUT/r${r}-pdfium" .venv/bin/python scripts/benchmarking/run_performance_benchmarking_with_docling_settings.py "$PDF_ROOT" \ + --backend pdfium --dataset-revision 221c19e716d6ede5d18fd1560bc938ab19cc1a70 \ + --output-dir "$REPRO_OUT/r${r}-pdfium" 2>&1 | tee "$REPRO_OUT/r${r}-pdfium.log" + for t in 1 4 8 12; do + observe "$REPRO_OUT/r${r}-t${t}" .venv/bin/python scripts/benchmarking/run_performance_benchmarking_with_docling_settings.py "$PDF_ROOT" \ + --backend threaded --threads "$t" --dataset-revision 221c19e716d6ede5d18fd1560bc938ab19cc1a70 \ + --output-dir "$REPRO_OUT/r${r}-t${t}" 2>&1 | tee "$REPRO_OUT/r${r}-t${t}.log" + done +done +``` + +
+ + + +## Python cross-package comparison + +Not collected in this run. + + + +## Process Memory + +OS high-water marks cover one backend process and all its threads, including startup, loading and teardown. They exclude the monitor/logger and are not parser-only heap sizes. Timing retains each workload's existing boundaries. Missing observations are unavailable, never zero. + +| Workload | Backend | Task | Threads | Peaks/runs | Median MiB | Min MiB | Largest peak MiB | Capture | +|------------|---------------|------------------|-----------|--------------|--------------|-----------|--------------------|-----------| +| native | docling-parse | render-only | 1 | 3/3 | 11047.67 | 10996.17 | 11048.22 | profile | +| native | docling-parse | render-only | 4 | 3/3 | 12448.70 | 12433.69 | 12538.94 | profile | +| native | docling-parse | render-only | 8 | 3/3 | 13137.61 | 11913.42 | 13173.20 | profile | +| native | docling-parse | render-only | 12 | 3/3 | 15219.81 | 14862.02 | 15619.06 | profile | +| native | docling-parse | render-with-text | 1 | 3/3 | 11102.88 | 11097.03 | 11168.17 | profile | +| native | docling-parse | render-with-text | 4 | 3/3 | 12396.61 | 11282.56 | 12540.33 | profile | +| native | docling-parse | render-with-text | 8 | 3/3 | 13168.28 | 13057.22 | 13186.28 | profile | +| native | docling-parse | render-with-text | 12 | 3/3 | 15343.41 | 14762.14 | 16298.81 | profile | +| replay | pdfium | render | 1 | 3/3 | 2520.64 | 2493.22 | 2540.17 | profile | +| replay | threaded | render | 1 | 3/3 | 4847.92 | 4842.22 | 4909.92 | profile | +| replay | threaded | render | 4 | 3/3 | 6641.08 | 6470.36 | 6647.22 | profile | +| replay | threaded | render | 8 | 3/3 | 6108.83 | 5931.45 | 6190.11 | profile | +| replay | threaded | render | 12 | 3/3 | 7120.94 | 7118.30 | 7212.56 | profile | + +`existing` denotes a previously recorded replay RUSAGE_SELF peak. Reused runs have no new timeline. Peak/profile comparison series run in fresh processes; memory-off comparisons retain their historical shared-process execution. Compare timing only under matching instrumentation and isolation. Sampled maxima are separate from OS peaks and can miss short spikes. Methods and scopes are recorded in publication provenance. + + +![Process peak RSS; median and observed range](assets/memory_native_render-only.png) + +![Process peak RSS; median and observed range](assets/memory_native_render-with-text.png) + +![Process peak RSS; median and observed range](assets/memory_replay_render.png) + + + +## Provenance + +[Dataset, source, build, packages and memory methods](provenance.json). Raw measurements and detailed RSS timelines remain in the local run bundle. + diff --git a/docs/performance_benchmarks/bo767/apple-m3-max-64gib-macos-26-6-2-arm64-arm-64bit/provenance.json b/docs/performance_benchmarks/bo767/apple-m3-max-64gib-macos-26-6-2-arm64-arm-64bit/provenance.json new file mode 100644 index 000000000..eac3caa2f --- /dev/null +++ b/docs/performance_benchmarks/bo767/apple-m3-max-64gib-macos-26-6-2-arm64-arm-64bit/provenance.json @@ -0,0 +1,388 @@ +{ + "build_mode": "incremental", + "canvas_checks": [], + "comparison_memory": { + "interval_ms": 50, + "mode": "profile" + }, + "dataset": { + "repository": "docling-project/performance-dataset-bo767", + "requested_revision": "221c19e716d6ede5d18fd1560bc938ab19cc1a70", + "revision": "221c19e716d6ede5d18fd1560bc938ab19cc1a70" + }, + "documents": 767, + "excluded_comparisons": [], + "extension_sha256": "017556b0bc007d831a0d40ccbda9b09285d4680378102de7b10f5b44143ef9cf", + "identity_serialization": "UTF-8 JSON list in dataset order; name/pages/sha256; sort_keys=True; separators=(comma,colon)", + "launcher_sha256": "259ffce1494f2e82c72381df5cf71f26ae6d9820cd228fb3283c9575c916e99a", + "machine": { + "cpu": "Apple M3 Max", + "machine": "arm64", + "memory_bytes": "68719476736", + "platform": "macOS-26.6.2-arm64-arm-64bit" + }, + "max_concurrent_results": 32, + "measured_at": "2026-11-12T17:27:10.253402", + "memory_collection": { + "interval_ms": 50, + "mode": "profile" + }, + "memory_methods": [ + [ + "profile", + "wait4.ru_maxrss", + "process lifetime including startup and teardown" + ] + ], + "native_profiles": { + "render-only": { + "create_line_cells": false, + "create_word_cells": false + }, + "render-with-text": { + "create_line_cells": true, + "create_word_cells": true + } + }, + "native_sha256": "ce85c518f6a9c11c83ce2dcf4ccf602ffa2d5bf37d357c53234a3a8c31ec2850", + "ordered_file_identity_sha256": "2599f474f8475ac9e51c14c22e2b6d946e99fc57d84fa3ce690e52942f0d601a", + "packages": { + "docling-core": "2.103.1", + "docling-parse": "7.25.0", + "matplotlib": "3.11.2", + "pdfminer.six": "20260107", + "pdfplumber": "0.11.10", + "pillow": "12.3.0", + "pymupdf": "1.27.2.3", + "pypdf": "6.21.1", + "pypdfium2": "5.14.0", + "rtree": "1.4.1" + }, + "pages": 54730, + "python": "3.12.11 (main, Jul 8 2025, 20:41:49) [Clang 20.1.4 ]", + "qpdf": "12.4.2", + "recovered_replay_provenance": { + "replay/r2/t12/report.json": { + "cpu": "Apple M3 Max", + "extension_sha256": "017556b0bc007d831a0d40ccbda9b09285d4680378102de7b10f5b44143ef9cf", + "machine": "arm64", + "packages": { + "docling-core": "2.103.1", + "docling-parse": "7.25.0", + "pdfminer.six": "20260107", + "pdfplumber": "0.11.10", + "pillow": "12.3.0", + "pymupdf": "1.27.2.3", + "pypdf": "6.21.1", + "pypdfium2": "5.14.0", + "rtree": "1.4.1" + }, + "platform": "macOS-26.6.2-arm64-arm-64bit", + "python": "3.12.11 (main, Jul 8 2025, 20:41:49) [Clang 20.1.4 ]", + "script_sha256": { + "_common.py": "56c2fcf9e53c4d0ae7def5253030e63242dbbc9169f1c0385c9c0812302ab1eb", + "_performance_memory.py": "1b9b2cfe76d44872916470c40999940782e7ea35ce7f4e165c6365130ce33baa", + "_performance_memory_report.py": "c426c404042658333201900081cd5525ebe1d45e2e41ae6ab3fe5510c005a65f", + "_performance_runtime.py": "50abe947f1cc1223da7dffc9da6f2bd03a4fe9d54db739a3caf613e167a28fc1", + "_performance_worker.py": "a3e272eaca78b35b68839a32d3b049cb63d0c3afdd7484a9f228aff87f0d2453", + "recover_performance_benchmarks.py": "514f35937d765eea1a1e29322a3e526c862bf9317952df9690f479992ba87a22", + "run_performance_analysis.py": "96453dc08db69ec4a90e5da506c31fb0c08194470d8fea3a15043fcce99da5a2", + "run_performance_benchmarking.py": "7d7fbf6542a07faa70493b244293336f91af3abb493bbb04a7115fa1cb67ddf0", + "run_performance_benchmarking_with_docling_settings.py": "71c743fab5c6737d034853f35c493fd6a1f831352c9eb3b3a5e81d5dffde361a", + "run_performance_eval.py": "314e2d1bdc58b031be89380baa345fe821eb32a488d42270e66ffebcb7d7bca9", + "run_performance_report.py": "d863d051095879fcf7bf909d24cef3a60181541d9bc7e0e26b77bceae80cb6cb", + "run_quality_benchmarking.py": "20fdedee0694cc5a5af1d1cbe7087465a9004534e68b98dccaa89e7bdf35c1f2" + }, + "source_diff_sha256": "051fcccbd45d061a3f27cdf8ee6b43132f46cb9d5701e866db50f57148d639fc", + "source_revision": "d4e47594bdcbf06a26fe2616ef42c6acaf592994" + }, + "replay/r2/t4/report.json": { + "cpu": "Apple M3 Max", + "extension_sha256": "017556b0bc007d831a0d40ccbda9b09285d4680378102de7b10f5b44143ef9cf", + "machine": "arm64", + "packages": { + "docling-core": "2.103.1", + "docling-parse": "7.25.0", + "pdfminer.six": "20260107", + "pdfplumber": "0.11.10", + "pillow": "12.3.0", + "pymupdf": "1.27.2.3", + "pypdf": "6.21.1", + "pypdfium2": "5.14.0", + "rtree": "1.4.1" + }, + "platform": "macOS-26.6.2-arm64-arm-64bit", + "python": "3.12.11 (main, Jul 8 2025, 20:41:49) [Clang 20.1.4 ]", + "script_sha256": { + "_common.py": "56c2fcf9e53c4d0ae7def5253030e63242dbbc9169f1c0385c9c0812302ab1eb", + "_performance_memory.py": "1b9b2cfe76d44872916470c40999940782e7ea35ce7f4e165c6365130ce33baa", + "_performance_memory_report.py": "c426c404042658333201900081cd5525ebe1d45e2e41ae6ab3fe5510c005a65f", + "_performance_runtime.py": "50abe947f1cc1223da7dffc9da6f2bd03a4fe9d54db739a3caf613e167a28fc1", + "_performance_worker.py": "a3e272eaca78b35b68839a32d3b049cb63d0c3afdd7484a9f228aff87f0d2453", + "recover_performance_benchmarks.py": "514f35937d765eea1a1e29322a3e526c862bf9317952df9690f479992ba87a22", + "run_performance_analysis.py": "96453dc08db69ec4a90e5da506c31fb0c08194470d8fea3a15043fcce99da5a2", + "run_performance_benchmarking.py": "7d7fbf6542a07faa70493b244293336f91af3abb493bbb04a7115fa1cb67ddf0", + "run_performance_benchmarking_with_docling_settings.py": "71c743fab5c6737d034853f35c493fd6a1f831352c9eb3b3a5e81d5dffde361a", + "run_performance_eval.py": "314e2d1bdc58b031be89380baa345fe821eb32a488d42270e66ffebcb7d7bca9", + "run_performance_report.py": "d863d051095879fcf7bf909d24cef3a60181541d9bc7e0e26b77bceae80cb6cb", + "run_quality_benchmarking.py": "20fdedee0694cc5a5af1d1cbe7087465a9004534e68b98dccaa89e7bdf35c1f2" + }, + "source_diff_sha256": "051fcccbd45d061a3f27cdf8ee6b43132f46cb9d5701e866db50f57148d639fc", + "source_revision": "d4e47594bdcbf06a26fe2616ef42c6acaf592994" + }, + "replay/r2/t8/report.json": { + "cpu": "Apple M3 Max", + "extension_sha256": "017556b0bc007d831a0d40ccbda9b09285d4680378102de7b10f5b44143ef9cf", + "machine": "arm64", + "packages": { + "docling-core": "2.103.1", + "docling-parse": "7.25.0", + "pdfminer.six": "20260107", + "pdfplumber": "0.11.10", + "pillow": "12.3.0", + "pymupdf": "1.27.2.3", + "pypdf": "6.21.1", + "pypdfium2": "5.14.0", + "rtree": "1.4.1" + }, + "platform": "macOS-26.6.2-arm64-arm-64bit", + "python": "3.12.11 (main, Jul 8 2025, 20:41:49) [Clang 20.1.4 ]", + "script_sha256": { + "_common.py": "56c2fcf9e53c4d0ae7def5253030e63242dbbc9169f1c0385c9c0812302ab1eb", + "_performance_memory.py": "1b9b2cfe76d44872916470c40999940782e7ea35ce7f4e165c6365130ce33baa", + "_performance_memory_report.py": "c426c404042658333201900081cd5525ebe1d45e2e41ae6ab3fe5510c005a65f", + "_performance_runtime.py": "50abe947f1cc1223da7dffc9da6f2bd03a4fe9d54db739a3caf613e167a28fc1", + "_performance_worker.py": "a3e272eaca78b35b68839a32d3b049cb63d0c3afdd7484a9f228aff87f0d2453", + "recover_performance_benchmarks.py": "514f35937d765eea1a1e29322a3e526c862bf9317952df9690f479992ba87a22", + "run_performance_analysis.py": "96453dc08db69ec4a90e5da506c31fb0c08194470d8fea3a15043fcce99da5a2", + "run_performance_benchmarking.py": "7d7fbf6542a07faa70493b244293336f91af3abb493bbb04a7115fa1cb67ddf0", + "run_performance_benchmarking_with_docling_settings.py": "71c743fab5c6737d034853f35c493fd6a1f831352c9eb3b3a5e81d5dffde361a", + "run_performance_eval.py": "314e2d1bdc58b031be89380baa345fe821eb32a488d42270e66ffebcb7d7bca9", + "run_performance_report.py": "d863d051095879fcf7bf909d24cef3a60181541d9bc7e0e26b77bceae80cb6cb", + "run_quality_benchmarking.py": "20fdedee0694cc5a5af1d1cbe7087465a9004534e68b98dccaa89e7bdf35c1f2" + }, + "source_diff_sha256": "051fcccbd45d061a3f27cdf8ee6b43132f46cb9d5701e866db50f57148d639fc", + "source_revision": "d4e47594bdcbf06a26fe2616ef42c6acaf592994" + }, + "replay/r3/pdfium/report.json": { + "cpu": "Apple M3 Max", + "extension_sha256": "017556b0bc007d831a0d40ccbda9b09285d4680378102de7b10f5b44143ef9cf", + "machine": "arm64", + "packages": { + "docling-core": "2.103.1", + "docling-parse": "7.25.0", + "pdfminer.six": "20260107", + "pdfplumber": "0.11.10", + "pillow": "12.3.0", + "pymupdf": "1.27.2.3", + "pypdf": "6.21.1", + "pypdfium2": "5.14.0", + "rtree": "1.4.1" + }, + "platform": "macOS-26.6.2-arm64-arm-64bit", + "python": "3.12.11 (main, Jul 8 2025, 20:41:49) [Clang 20.1.4 ]", + "script_sha256": { + "_common.py": "56c2fcf9e53c4d0ae7def5253030e63242dbbc9169f1c0385c9c0812302ab1eb", + "_performance_memory.py": "1b9b2cfe76d44872916470c40999940782e7ea35ce7f4e165c6365130ce33baa", + "_performance_memory_report.py": "c426c404042658333201900081cd5525ebe1d45e2e41ae6ab3fe5510c005a65f", + "_performance_runtime.py": "50abe947f1cc1223da7dffc9da6f2bd03a4fe9d54db739a3caf613e167a28fc1", + "_performance_worker.py": "a3e272eaca78b35b68839a32d3b049cb63d0c3afdd7484a9f228aff87f0d2453", + "recover_performance_benchmarks.py": "514f35937d765eea1a1e29322a3e526c862bf9317952df9690f479992ba87a22", + "run_performance_analysis.py": "96453dc08db69ec4a90e5da506c31fb0c08194470d8fea3a15043fcce99da5a2", + "run_performance_benchmarking.py": "7d7fbf6542a07faa70493b244293336f91af3abb493bbb04a7115fa1cb67ddf0", + "run_performance_benchmarking_with_docling_settings.py": "71c743fab5c6737d034853f35c493fd6a1f831352c9eb3b3a5e81d5dffde361a", + "run_performance_eval.py": "314e2d1bdc58b031be89380baa345fe821eb32a488d42270e66ffebcb7d7bca9", + "run_performance_report.py": "d863d051095879fcf7bf909d24cef3a60181541d9bc7e0e26b77bceae80cb6cb", + "run_quality_benchmarking.py": "20fdedee0694cc5a5af1d1cbe7087465a9004534e68b98dccaa89e7bdf35c1f2" + }, + "source_diff_sha256": "051fcccbd45d061a3f27cdf8ee6b43132f46cb9d5701e866db50f57148d639fc", + "source_revision": "d4e47594bdcbf06a26fe2616ef42c6acaf592994" + }, + "replay/r3/t1/report.json": { + "cpu": "Apple M3 Max", + "extension_sha256": "017556b0bc007d831a0d40ccbda9b09285d4680378102de7b10f5b44143ef9cf", + "machine": "arm64", + "packages": { + "docling-core": "2.103.1", + "docling-parse": "7.25.0", + "pdfminer.six": "20260107", + "pdfplumber": "0.11.10", + "pillow": "12.3.0", + "pymupdf": "1.27.2.3", + "pypdf": "6.21.1", + "pypdfium2": "5.14.0", + "rtree": "1.4.1" + }, + "platform": "macOS-26.6.2-arm64-arm-64bit", + "python": "3.12.11 (main, Jul 8 2025, 20:41:49) [Clang 20.1.4 ]", + "script_sha256": { + "_common.py": "56c2fcf9e53c4d0ae7def5253030e63242dbbc9169f1c0385c9c0812302ab1eb", + "_performance_memory.py": "1b9b2cfe76d44872916470c40999940782e7ea35ce7f4e165c6365130ce33baa", + "_performance_memory_report.py": "c426c404042658333201900081cd5525ebe1d45e2e41ae6ab3fe5510c005a65f", + "_performance_runtime.py": "50abe947f1cc1223da7dffc9da6f2bd03a4fe9d54db739a3caf613e167a28fc1", + "_performance_worker.py": "a3e272eaca78b35b68839a32d3b049cb63d0c3afdd7484a9f228aff87f0d2453", + "recover_performance_benchmarks.py": "514f35937d765eea1a1e29322a3e526c862bf9317952df9690f479992ba87a22", + "run_performance_analysis.py": "96453dc08db69ec4a90e5da506c31fb0c08194470d8fea3a15043fcce99da5a2", + "run_performance_benchmarking.py": "7d7fbf6542a07faa70493b244293336f91af3abb493bbb04a7115fa1cb67ddf0", + "run_performance_benchmarking_with_docling_settings.py": "71c743fab5c6737d034853f35c493fd6a1f831352c9eb3b3a5e81d5dffde361a", + "run_performance_eval.py": "314e2d1bdc58b031be89380baa345fe821eb32a488d42270e66ffebcb7d7bca9", + "run_performance_report.py": "d863d051095879fcf7bf909d24cef3a60181541d9bc7e0e26b77bceae80cb6cb", + "run_quality_benchmarking.py": "20fdedee0694cc5a5af1d1cbe7087465a9004534e68b98dccaa89e7bdf35c1f2" + }, + "source_diff_sha256": "051fcccbd45d061a3f27cdf8ee6b43132f46cb9d5701e866db50f57148d639fc", + "source_revision": "d4e47594bdcbf06a26fe2616ef42c6acaf592994" + }, + "replay/r3/t12/report.json": { + "cpu": "Apple M3 Max", + "extension_sha256": "017556b0bc007d831a0d40ccbda9b09285d4680378102de7b10f5b44143ef9cf", + "machine": "arm64", + "packages": { + "docling-core": "2.103.1", + "docling-parse": "7.25.0", + "pdfminer.six": "20260107", + "pdfplumber": "0.11.10", + "pillow": "12.3.0", + "pymupdf": "1.27.2.3", + "pypdf": "6.21.1", + "pypdfium2": "5.14.0", + "rtree": "1.4.1" + }, + "platform": "macOS-26.6.2-arm64-arm-64bit", + "python": "3.12.11 (main, Jul 8 2025, 20:41:49) [Clang 20.1.4 ]", + "script_sha256": { + "_common.py": "56c2fcf9e53c4d0ae7def5253030e63242dbbc9169f1c0385c9c0812302ab1eb", + "_performance_memory.py": "1b9b2cfe76d44872916470c40999940782e7ea35ce7f4e165c6365130ce33baa", + "_performance_memory_report.py": "c426c404042658333201900081cd5525ebe1d45e2e41ae6ab3fe5510c005a65f", + "_performance_runtime.py": "50abe947f1cc1223da7dffc9da6f2bd03a4fe9d54db739a3caf613e167a28fc1", + "_performance_worker.py": "a3e272eaca78b35b68839a32d3b049cb63d0c3afdd7484a9f228aff87f0d2453", + "recover_performance_benchmarks.py": "514f35937d765eea1a1e29322a3e526c862bf9317952df9690f479992ba87a22", + "run_performance_analysis.py": "96453dc08db69ec4a90e5da506c31fb0c08194470d8fea3a15043fcce99da5a2", + "run_performance_benchmarking.py": "7d7fbf6542a07faa70493b244293336f91af3abb493bbb04a7115fa1cb67ddf0", + "run_performance_benchmarking_with_docling_settings.py": "71c743fab5c6737d034853f35c493fd6a1f831352c9eb3b3a5e81d5dffde361a", + "run_performance_eval.py": "314e2d1bdc58b031be89380baa345fe821eb32a488d42270e66ffebcb7d7bca9", + "run_performance_report.py": "d863d051095879fcf7bf909d24cef3a60181541d9bc7e0e26b77bceae80cb6cb", + "run_quality_benchmarking.py": "20fdedee0694cc5a5af1d1cbe7087465a9004534e68b98dccaa89e7bdf35c1f2" + }, + "source_diff_sha256": "051fcccbd45d061a3f27cdf8ee6b43132f46cb9d5701e866db50f57148d639fc", + "source_revision": "d4e47594bdcbf06a26fe2616ef42c6acaf592994" + }, + "replay/r3/t4/report.json": { + "cpu": "Apple M3 Max", + "extension_sha256": "017556b0bc007d831a0d40ccbda9b09285d4680378102de7b10f5b44143ef9cf", + "machine": "arm64", + "packages": { + "docling-core": "2.103.1", + "docling-parse": "7.25.0", + "pdfminer.six": "20260107", + "pdfplumber": "0.11.10", + "pillow": "12.3.0", + "pymupdf": "1.27.2.3", + "pypdf": "6.21.1", + "pypdfium2": "5.14.0", + "rtree": "1.4.1" + }, + "platform": "macOS-26.6.2-arm64-arm-64bit", + "python": "3.12.11 (main, Jul 8 2025, 20:41:49) [Clang 20.1.4 ]", + "script_sha256": { + "_common.py": "56c2fcf9e53c4d0ae7def5253030e63242dbbc9169f1c0385c9c0812302ab1eb", + "_performance_memory.py": "1b9b2cfe76d44872916470c40999940782e7ea35ce7f4e165c6365130ce33baa", + "_performance_memory_report.py": "c426c404042658333201900081cd5525ebe1d45e2e41ae6ab3fe5510c005a65f", + "_performance_runtime.py": "50abe947f1cc1223da7dffc9da6f2bd03a4fe9d54db739a3caf613e167a28fc1", + "_performance_worker.py": "a3e272eaca78b35b68839a32d3b049cb63d0c3afdd7484a9f228aff87f0d2453", + "recover_performance_benchmarks.py": "514f35937d765eea1a1e29322a3e526c862bf9317952df9690f479992ba87a22", + "run_performance_analysis.py": "96453dc08db69ec4a90e5da506c31fb0c08194470d8fea3a15043fcce99da5a2", + "run_performance_benchmarking.py": "7d7fbf6542a07faa70493b244293336f91af3abb493bbb04a7115fa1cb67ddf0", + "run_performance_benchmarking_with_docling_settings.py": "71c743fab5c6737d034853f35c493fd6a1f831352c9eb3b3a5e81d5dffde361a", + "run_performance_eval.py": "314e2d1bdc58b031be89380baa345fe821eb32a488d42270e66ffebcb7d7bca9", + "run_performance_report.py": "d863d051095879fcf7bf909d24cef3a60181541d9bc7e0e26b77bceae80cb6cb", + "run_quality_benchmarking.py": "20fdedee0694cc5a5af1d1cbe7087465a9004534e68b98dccaa89e7bdf35c1f2" + }, + "source_diff_sha256": "051fcccbd45d061a3f27cdf8ee6b43132f46cb9d5701e866db50f57148d639fc", + "source_revision": "d4e47594bdcbf06a26fe2616ef42c6acaf592994" + }, + "replay/r3/t8/report.json": { + "cpu": "Apple M3 Max", + "extension_sha256": "017556b0bc007d831a0d40ccbda9b09285d4680378102de7b10f5b44143ef9cf", + "machine": "arm64", + "packages": { + "docling-core": "2.103.1", + "docling-parse": "7.25.0", + "pdfminer.six": "20260107", + "pdfplumber": "0.11.10", + "pillow": "12.3.0", + "pymupdf": "1.27.2.3", + "pypdf": "6.21.1", + "pypdfium2": "5.14.0", + "rtree": "1.4.1" + }, + "platform": "macOS-26.6.2-arm64-arm-64bit", + "python": "3.12.11 (main, Jul 8 2025, 20:41:49) [Clang 20.1.4 ]", + "script_sha256": { + "_common.py": "56c2fcf9e53c4d0ae7def5253030e63242dbbc9169f1c0385c9c0812302ab1eb", + "_performance_memory.py": "1b9b2cfe76d44872916470c40999940782e7ea35ce7f4e165c6365130ce33baa", + "_performance_memory_report.py": "c426c404042658333201900081cd5525ebe1d45e2e41ae6ab3fe5510c005a65f", + "_performance_runtime.py": "50abe947f1cc1223da7dffc9da6f2bd03a4fe9d54db739a3caf613e167a28fc1", + "_performance_worker.py": "a3e272eaca78b35b68839a32d3b049cb63d0c3afdd7484a9f228aff87f0d2453", + "recover_performance_benchmarks.py": "514f35937d765eea1a1e29322a3e526c862bf9317952df9690f479992ba87a22", + "run_performance_analysis.py": "96453dc08db69ec4a90e5da506c31fb0c08194470d8fea3a15043fcce99da5a2", + "run_performance_benchmarking.py": "7d7fbf6542a07faa70493b244293336f91af3abb493bbb04a7115fa1cb67ddf0", + "run_performance_benchmarking_with_docling_settings.py": "71c743fab5c6737d034853f35c493fd6a1f831352c9eb3b3a5e81d5dffde361a", + "run_performance_eval.py": "314e2d1bdc58b031be89380baa345fe821eb32a488d42270e66ffebcb7d7bca9", + "run_performance_report.py": "d863d051095879fcf7bf909d24cef3a60181541d9bc7e0e26b77bceae80cb6cb", + "run_quality_benchmarking.py": "20fdedee0694cc5a5af1d1cbe7087465a9004534e68b98dccaa89e7bdf35c1f2" + }, + "source_diff_sha256": "051fcccbd45d061a3f27cdf8ee6b43132f46cb9d5701e866db50f57148d639fc", + "source_revision": "d4e47594bdcbf06a26fe2616ef42c6acaf592994" + } + }, + "recovery": { + "comparisons": "not requested", + "core_results": "completed configurations reused; missing/incomplete configurations resumed", + "created_at": "2026-11-12T17:27:10.250902+00:00", + "pending_core": [ + "replay/r2/t4/report.json", + "replay/r2/t8/report.json", + "replay/r2/t12/report.json", + "native/render-only/r3/t1.csv", + "native/render-only/r3/t4.csv", + "native/render-only/r3/t8.csv", + "native/render-only/r3/t12.csv", + "native/render-with-text/r3/t1.csv", + "native/render-with-text/r3/t4.csv", + "native/render-with-text/r3/t8.csv", + "native/render-with-text/r3/t12.csv", + "replay/r3/pdfium/report.json", + "replay/r3/t1/report.json", + "replay/r3/t4/report.json", + "replay/r3/t8/report.json", + "replay/r3/t12/report.json" + ], + "source_bundle": "/Users/taa/Documents/projects/docling-org/docling-parse/scratch-performance-refresh-20261112-135818-khBWhl" + }, + "repeats": 3, + "run": "scratch-performance-recovery-8vumw3sy", + "scale": 1.0, + "schema_version": 1, + "script_sha256": { + "scripts/benchmarking/_common.py": "56c2fcf9e53c4d0ae7def5253030e63242dbbc9169f1c0385c9c0812302ab1eb", + "scripts/benchmarking/_performance_memory.py": "1b9b2cfe76d44872916470c40999940782e7ea35ce7f4e165c6365130ce33baa", + "scripts/benchmarking/_performance_memory_report.py": "c426c404042658333201900081cd5525ebe1d45e2e41ae6ab3fe5510c005a65f", + "scripts/benchmarking/_performance_runtime.py": "50abe947f1cc1223da7dffc9da6f2bd03a4fe9d54db739a3caf613e167a28fc1", + "scripts/benchmarking/_performance_worker.py": "a3e272eaca78b35b68839a32d3b049cb63d0c3afdd7484a9f228aff87f0d2453", + "scripts/benchmarking/recover_performance_benchmarks.py": "f563f6d6b568a5f430f5ffd3267a1eb90943e651ee736f5f8deda64ae1307192", + "scripts/benchmarking/run_performance_analysis.py": "96453dc08db69ec4a90e5da506c31fb0c08194470d8fea3a15043fcce99da5a2", + "scripts/benchmarking/run_performance_benchmarking.py": "7d7fbf6542a07faa70493b244293336f91af3abb493bbb04a7115fa1cb67ddf0", + "scripts/benchmarking/run_performance_benchmarking_with_docling_settings.py": "71c743fab5c6737d034853f35c493fd6a1f831352c9eb3b3a5e81d5dffde361a", + "scripts/benchmarking/run_performance_eval.py": "314e2d1bdc58b031be89380baa345fe821eb32a488d42270e66ffebcb7d7bca9", + "scripts/benchmarking/run_performance_report.py": "39be67cdecf97051e41ebd43aeee7e20421261aeb65dcabf077f86766d0a4252", + "scripts/benchmarking/run_quality_benchmarking.py": "20fdedee0694cc5a5af1d1cbe7087465a9004534e68b98dccaa89e7bdf35c1f2" + }, + "source_diff_sha256": "6ce0da20ad146ba1c1407ebc511a6e35baf926a80ecfd8257feec1b11156802a", + "source_revision": "7cbe87d8b7b87fb47ffc90a374c912144857ff05", + "threads": [ + 1, + 4, + 8, + 12 + ] +} diff --git a/docs/performance_benchmarks/bo767/apple-m3-max-64gib-macos-26-6-2-arm64-arm-64bit/publication.json b/docs/performance_benchmarks/bo767/apple-m3-max-64gib-macos-26-6-2-arm64-arm-64bit/publication.json new file mode 100644 index 000000000..41f59deb1 --- /dev/null +++ b/docs/performance_benchmarks/bo767/apple-m3-max-64gib-macos-26-6-2-arm64-arm-64bit/publication.json @@ -0,0 +1,37 @@ +{ + "dataset": "bo767", + "files": { + "assets/memory_native_render-only.png": "727cd0e5c255088d4f7788bbba996ea4b7dc13b7abcbe4e151e3ae284511834f", + "assets/memory_native_render-with-text.png": "3788553be67df5aeb06f8b3b62e0d78afca2933703b816bf24d56cc1283a855c", + "assets/memory_replay_render.png": "b0a42d975fa22ecfb01687fe7f12c356e3b10515f6ef718320030143a9e17c76", + "assets/native_render_only_distribution.png": "2143aa241f0a8e20a154dbe5e59d54d8a1fcace79a2c871d4db902051d01afca", + "assets/native_render_only_phases.png": "591429a3415ccf6eb10e1f72cbc9ccef7ca6c0065851da3e36f06f4b2a8ee93a", + "assets/native_render_only_runtime.png": "a8a8eff3a505d8e92e016c2ebd4c208c1790a0b862112e1e7c3e47e354719bb6", + "assets/native_render_only_scaling.png": "c19c5b1d47377942e8f44e36d4ef4e5e395bc260fad9ad4b9f6b45334402d04d", + "assets/native_render_with_text_distribution.png": "ca34e463667a025c2a2f0c2866846ce4519470c5dcd08919335e4d7bb7e6e8f6", + "assets/native_render_with_text_phases.png": "2e31ab3c7be361aa93ec7f95cf408886535494ad7428fd3ad1a75cb3b39c50a0", + "assets/native_render_with_text_runtime.png": "2a580698eb56b8ce002ee3c359472bd4810f284334a02cf6eb261bfb88aa2a91", + "assets/native_render_with_text_scaling.png": "cb08e34a9607dfff09acab41dd0abf6e0f0d801d56d00d6878f46ffefa68b30f", + "assets/replay_distribution.png": "eb40864259df2846f5a810f0a2fdf1a560712b1be08f4099138390de90b35c5f", + "assets/replay_phases.png": "ba4adb80210fb9d2ec038ad3bdfa41d84a8604324b18d79c38e83c1878552744", + "assets/replay_runtime.png": "8dd25e7fee121dea880de7c88e7e0de4b77bf2a768f03907c9a95c9ab38265ce", + "assets/replay_scaling.png": "bbf5dd7f680bc0d5bae4907513d93504c08d9f38b2899f78110b098ded2ba7e7", + "index.md": "04597489181ca49724fd27d2391bc49f0b519e38632a67024698348ebaca3f31", + "provenance.json": "0c988a60b222e0ea6291c78617f079851549e20fb45023ac1965b26be861b6c8" + }, + "label": "Apple M3 Max / 64 GiB / macOS-26.6.2-arm64-arm-64bit", + "machine": { + "cpu": "Apple M3 Max", + "machine": "arm64", + "memory_bytes": 68719476736, + "platform": "macOS-26.6.2-arm64-arm-64bit" + }, + "measured_at": "2026-11-12T17:27:10.253402", + "run": "scratch-performance-recovery-8vumw3sy", + "schema_version": 1, + "setup": "apple-m3-max-64gib-macos-26-6-2-arm64-arm-64bit", + "workloads": [ + "native", + "replay" + ] +} diff --git a/scripts/benchmarking/README.md b/scripts/benchmarking/README.md index ae9e61859..ca0bf3bb1 100644 --- a/scripts/benchmarking/README.md +++ b/scripts/benchmarking/README.md @@ -18,7 +18,7 @@ It does not update the documentation. Full runs can take hours. |---|---|---|---| | `refresh_performance_benchmarks.sh` | Build and run the publication workflow | Pinned HF dataset; environment settings shown by `--help` | Raw measurements, provenance, generated report and PNGs | | `run_performance_report.py` | Validate and summarize a launcher bundle | Bundle containing `metadata.json` | `report/summary.md`, CSV tables, PNGs | -| `recover_performance_benchmarks.py` | Reuse completed core results and recollect comparisons | Existing bundle; normally invoked through `RECOVER_FROM` | New bundle with original core artifacts and separate comparison provenance | +| `recover_performance_benchmarks.py` | Resume missing configurations and reuse validated results | Existing bundle; normally invoked through `RECOVER_FROM` | New bundle with linked completed artifacts and resumed results | | `run_performance_benchmarking_with_docling_settings.py` | Standalone backend replay | PDFs or pinned HF revision | `report.json` and page metadata JSONL | | `run_performance_benchmarking.py` | Python API scaling and cross-package measurement | PDFs or HF dataset | Console tables and per-page CSV | | `run_performance_eval.py` | Explore Python comparison page timings | Comparison per-page CSV | Histograms, scaling/correlation PNGs, document statistics | @@ -114,6 +114,96 @@ The historical Bo767 snapshot `c684df54de2081a6de1192cee12713b54759c6bb` contained 753 PDFs / 54,584 pages; its existing reports retain those inputs. Compare runs with matching corpus revisions and file identities. +## Optional process memory measurements + +Memory collection is opt-in on macOS and Linux. `off` retains the existing +execution path; `peak` adds each backend process's OS RSS high-water mark; +`profile` also samples current RSS from a separate monitor. Both modes work for +native scaling, PDFium/Docling settings replay, and cross-package comparisons. + +```sh +# OS peaks, without polling for an RSS timeline. +REPEATS=3 THREADS=1,4,8,12 RUN_COMPARISON=0 DATASET=bo20 \ + bash scripts/benchmarking/refresh_performance_benchmarks.sh --memory-mode peak + +# OS peaks plus current RSS samples, nominally every 50 ms. +REPEATS=3 THREADS=1,4,8,12 RUN_COMPARISON=0 DATASET=bo20 \ + bash scripts/benchmarking/refresh_performance_benchmarks.sh \ + --memory-mode profile --memory-interval-ms 50 +``` + +`MEMORY_MODE=peak|profile|off` and `MEMORY_INTERVAL_MS=50` are equivalent +environment settings; explicit CLI values take precedence. The interval must +be a positive integer. Shorter intervals add monitoring work. Collection is +optional because its overhead should be measured on the actual workload. +Use matching settings and several repetitions for timing comparisons. + +Peaks cover one process and all its threads over its entire lifetime, including +imports, PDF loading, result collection and teardown. They exclude the monitor, +`uv`, and log piping. They are RSS, not allocator/heap usage or the sum of a +process tree. macOS uses `wait4.ru_maxrss` in bytes; Linux uses the same field +converted from KiB. Existing backend timing boundaries are preserved, so the +peak's lifetime is broader than the reported native processing timer. The +monitor's elapsed time is recorded separately and includes polling delay. + +Current RSS comes from macOS `proc_pidinfo` or Linux `/proc/PID/statm` in the +external monitor, so native code holding Python's GIL does not stop sampling. +Samples use a monotonic process clock; they include startup and are not aligned +to page events. The OS peak and sampled maximum are separate values: polling +can miss short spikes. A sufficiently short run may produce no samples while +still producing an OS peak. Repeated traces are plotted independently. + +With memory enabled, comparison backend/thread series each run in a fresh +Python process. With memory off, comparisons retain their historical shared +process. This changes cache isolation as well as enabling meaningful individual +peaks; do not attribute timing differences between these modes solely to the +sampler. The standalone comparison runner accepts the same two flags. Its +optional sequential parse baseline is timing-only; the measured series are +the registered reference backends and threaded Docling configurations. + +Outputs include: + +- Native/replay `*.memory.json` sidecars with PID, executed command, OS peak, + measurement scope, completion status, method and sample metadata. +- Profile `*.memory.rss.csv` files with elapsed seconds, RSS bytes and PID. +- Comparison `memory/` artifacts alongside series checkpoints, including + worker requests/results and the corresponding memory sidecars. +- `report/memory_runs.csv`, `memory_summary.csv`, peak-versus-threads figures, + and per-series RSS timeline figures. In `report/summary.md`, the `Parallel + Scaling` section contains native and Python scaling tables, followed by + `Process Memory` with peak-memory tables and figures. Elapsed-time RSS plots + are in `report/memory_traces.md`, linked from the summary with a return link. + +Timing tables use compact task labels, with full descriptions listed once below +each table. Peak-RSS and throughput axes start at `(0, 0)`. Throughput plots +retain measured points and add dashed ideal-scaling guides: measured one-thread +throughput multiplied by thread count. Guides run from the origin to the largest +measured thread count for backends measured at one thread and additional counts. + +Missing observations are reported as unavailable. Older replay results already +contain a process peak; reports expose it as `existing`, even with collection +off. This does not manufacture peaks for native/other historical runs or create +a historical timeline. OS peaks remain available in failed-run artifacts for +diagnosis, but failed measurements cannot become successful report rows. +Checkpoint reuse checks memory settings and hashes of sidecars and timelines. + +Recovery accepts the same flags: + +```sh +uv run --no-sync python scripts/benchmarking/recover_performance_benchmarks.py \ + /absolute/path/to/completed-or-interrupted-bundle \ + --memory-mode profile --memory-interval-ms 50 +``` + +Recovery retains the original native/replay results and their original memory +settings; it measures only new comparison series with the requested settings. +It cannot add memory observations to earlier core runs. Compatible checkpoints +and their memory artifacts are copied together. Changed settings or missing +artifacts cause affected comparisons to be recollected. Existing strict +core-binary/package provenance checks still apply; use a fresh refresh to +measure a newly rebuilt extension. Report regeneration reads saved artifacts +and does not rerun benchmarks. + ## Reference preflight, checkpoints and recovery The perf dependency group pins **PyMuPDF 1.27.2.3**. Version 1.28.2 aborts in @@ -139,27 +229,43 @@ RECOVER_FROM=/absolute/path/to/scratch-performance-refresh-RUN \ bash scripts/benchmarking/refresh_performance_benchmarks.sh ``` -Recovery creates a **new** bundle using the dataset, repetitions and thread -counts from the source bundle. It synchronizes the pinned dependencies while -retaining the installed local project, checks the recorded machine, Python, core -packages, native binary/extension hashes and QPDF version, then validates the -complete core results and dataset hashes. It links the original native/replay -artifacts, so **keep the original bundle**. It does not rebuild native targets. -If scripts were committed during collection, replay provenance is accepted only -when the commit patch exactly matches the recorded build diff and the replay -script hash matches that commit. The report explains this equivalence. - -Both comparison modes are recollected under the current reference versions. -Compatible checkpoints from a previous recovery are copied and reused; a -reference-version or source change prevents checkpoint reuse. The original -metadata is retained as `original-metadata.json`. `comparison_provenance` records -the new environment separately, and report tables show package versions. The -completed original parse comparison remains in its original bundle. - -If recovery is interrupted, use the newly printed recovery directory as -`RECOVER_FROM` to retain compatible completed comparison checkpoints. -`OUT` may select a new output directory; `BUILD_MODE=clean` is incompatible with -recovery because the recorded core binaries must remain unchanged. +Recovery creates a **new** bundle using the source dataset/revision, repetitions, +thread counts, workload settings and comparison enablement. `REPEATS`, `THREADS`, +`DATASET` and `RUN_COMPARISON` environment settings do not override the source. +Memory mode and sampling interval are inherited too; you do not need to repeat +`--memory-mode profile --memory-interval-ms 50` for an originally profiled run. + +Each native profile/thread/repetition and each replay backend/thread/repetition +is validated independently. Completed configurations are linked unchanged; +missing or incomplete configurations restart from their beginning. Partial +page-level timings are never combined with a fresh measurement. The original +bundle is untouched: **keep it**, including any earlier bundles it links to. +Recovery prints which configurations it will reuse and which it must run. + +Recovery synchronizes dependencies while retaining the installed local project; +it does not rebuild native targets. Machine, Python, core packages, native +binary/extension hashes, QPDF and dataset hashes must match. Resuming core work +also requires unchanged measurement/monitor scripts and recorded runtime +settings (including fallback fonts). Explicit memory flags must match original +core instrumentation while core work remains incomplete. Once core work is +complete, explicit memory flags may change comparison instrumentation. + +The original source/build identity remains intact. Resumed replay measurements +have artifact-specific source provenance in `recovered_replay_provenance`, so +unrelated documentation/tooling edits do not invalidate resumed measurements. +Changed measurement scripts still require a new full run. Mid-run commits of +original changes retain the report's exact-patch equivalence validation. + +Cross-package comparisons run only if the source requested them. When enabled, +compatible completed checkpoints are reused; changed source/reference versions +or memory settings cause affected comparisons to be recollected. Original +metadata is retained as `original-metadata.json`, and `comparison_provenance` +records the environment for recovered comparisons separately. + +If recovery is interrupted, use its newly printed recovery directory as +`RECOVER_FROM` so completed resumed work is retained. `OUT` may select a new output +directory; `BUILD_MODE=clean` is incompatible with recovery because recorded core +binaries must remain unchanged. You can also inspect the completed core results independently: @@ -339,3 +445,51 @@ Outputs include `render_quality.csv`, `text_quality.csv`, render comparison panels and quality-distribution plots. Quality evidence is separate from the performance refresh; the launcher's damaged-page preflight is not a complete render-quality regression run. + +## Publish performance documentation + +Publish a completed launcher/recovery bundle without rerunning measurements: + +```sh +uv run python scripts/tools/update_performance_docs.py --performance-run-dir scratch-performance-refresh-RUN +uv run python scripts/tools/update_performance_docs.py --performance-run-dir scratch-performance-refresh-RUN --apply +python3 scripts/tools/update_performance_docs.py --check +``` + +Preview is the default: it prints Markdown diffs, file byte sizes and total/net size. +The publisher validates raw results through `ReportInputs`, then stages summary +plots and tables through the report generator's structured publication interface. +It does not copy or parse an existing `report/summary.md`. A bundle's original +linked recovery artifacts and dataset files must remain available as required by +report validation. No builds, benchmark reruns or downloads occur. + +Results live at `docs/performance_benchmarks///index.md`. The setup +is inferred from CPU, architecture, RAM and OS. Use `--setup-id lowercase-slug` for +older bundles with missing machine metadata. An explicit ID cannot overwrite a +publication recording a different machine. `--allow-older` explicitly permits an +older run to replace a newer one; collection order uses the launcher timestamp in +the directory name, falling back to metadata file modification time for custom names. +Thread sweep, dataset revision, workload settings and package versions are recorded +in each publication rather than silently merged with prior measurements. + +New pages use `scripts/tools/update_performance_docs_artifacts/performance_page_template.md`. Sections between +`` and `` are generated; +text outside markers is retained. Supported slots are `title`, `setup`, `interpretation`, +`native`, `replay`, `comparison`, `memory` and `provenance`; the entry page uses +`availability`. Absent slots are skipped with a message; malformed/duplicate markers +fail. Omitted comparisons/memory are labeled not collected; an incomplete expected +suite fails validation. Updating a migrated historical snapshot replaces it with +the current template because it predates placeholders. + +Only images referenced by a result page enter the publication. Peak-memory summaries +are included; diagnostic RSS timelines, raw CSV/JSONL and logs remain local. +`publication.json` records owned files and hashes; obsolete owned assets are removed +on replacement, and unowned files are preserved. Compact provenance records input +identity, source/build hashes, packages, workload settings and memory methods. + +Hard limits live in `scripts/tools/update_performance_docs_artifacts/performance_docs_limits.json`: 100 KiB per +Markdown/JSON, 500 KiB per image, 5 MiB per publication and 25 MiB across the entire +performance tree plus entry page. All limits and links are checked against staged +output before writing. `--check` requires only the Python standard library and +runs in CI; it also detects modified or missing generated files. Limits have no +CLI bypass: change the reviewed configuration when policy needs to change. diff --git a/scripts/benchmarking/_performance_memory.py b/scripts/benchmarking/_performance_memory.py new file mode 100644 index 000000000..bf1c597b8 --- /dev/null +++ b/scripts/benchmarking/_performance_memory.py @@ -0,0 +1,328 @@ +"""Independent process RSS collection for manual benchmarks (macOS/Linux). + +The monitor owns wait4 for exactly one child. Its own memory and the logger's +memory are excluded. Sampling is external to Python/native worker execution. +""" + +from __future__ import annotations + +import argparse +import csv +import ctypes +import errno +import hashlib +import json +import os +import subprocess +import sys +import time +from pathlib import Path +from typing import Any, Callable + + +def positive_interval(value: str) -> int: + number = int(value) + if number <= 0: + raise argparse.ArgumentTypeError("Memory interval must be a positive integer") + return number + + +def add_memory_arguments(parser: argparse.ArgumentParser) -> None: + def memory_mode(value: str) -> str: + if value not in ("off", "peak", "profile"): + raise argparse.ArgumentTypeError("Memory mode must be off, peak or profile") + return value + + parser.add_argument( + "--memory-mode", + choices=("off", "peak", "profile"), + type=memory_mode, + default=os.environ.get("MEMORY_MODE", "off"), + help="off (default), per-process OS peak, or peak plus sampled RSS", + ) + parser.add_argument( + "--memory-interval-ms", + type=positive_interval, + default=os.environ.get("MEMORY_INTERVAL_MS", "50"), + help="Current-RSS sampling interval in milliseconds (profile mode)", + ) + + +def peak_bytes(value: int, platform: str) -> int: + if value < 0: + raise ValueError("Negative OS peak RSS") + if platform == "darwin": + return value + if platform.startswith("linux"): + return value * 1024 + raise ValueError("Memory collection supports macOS and Linux") + + +def rss_reader() -> Callable[[int], int | None]: + if sys.platform.startswith("linux"): + page_size = os.sysconf("SC_PAGE_SIZE") + + def linux(pid: int) -> int | None: + try: + fields = Path(f"/proc/{pid}/statm").read_text().split() + except (FileNotFoundError, ProcessLookupError): + return None + return int(fields[1]) * page_size + + return linux + if sys.platform == "darwin": + # Layout/constant from the SDK's sys/proc_info.h, proc_taskinfo and + # PROC_PIDTASKINFO. Only resident_size is read; all fields preserve ABI. + class TaskInfo(ctypes.Structure): + _fields_ = [ + (name, ctypes.c_uint64) + for name in ( + "virtual_size", + "resident_size", + "total_user", + "total_system", + "threads_user", + "threads_system", + ) + ] + [ + (name, ctypes.c_int32) + for name in ( + "policy", + "faults", + "pageins", + "cow_faults", + "messages_sent", + "messages_received", + "syscalls_mach", + "syscalls_unix", + "csw", + "threadnum", + "numrunning", + "priority", + ) + ] + + library = ctypes.CDLL("/usr/lib/libproc.dylib", use_errno=True) + query = library.proc_pidinfo + query.argtypes = [ + ctypes.c_int, + ctypes.c_int, + ctypes.c_uint64, + ctypes.c_void_p, + ctypes.c_int, + ] + query.restype = ctypes.c_int + + def darwin(pid: int) -> int | None: + info = TaskInfo() + ctypes.set_errno(0) + size = query(pid, 4, 0, ctypes.byref(info), ctypes.sizeof(info)) + if size == ctypes.sizeof(info): + return int(info.resident_size) + error = ctypes.get_errno() + if size == 0 and error in (0, errno.ESRCH): + return None # Child exited between wait4 and the RSS query. + raise OSError(error, "proc_pidinfo could not sample child RSS") + + return darwin + raise ValueError("Memory collection supports macOS and Linux") + + +def file_digest(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def run_measured( + command: list[str], + output: Path, + *, + mode: str, + interval_ms: int = 50, + cwd: Path | None = None, +) -> dict[str, Any]: + if mode not in ("peak", "profile") or interval_ms <= 0 or not command: + raise ValueError( + "Expected peak/profile mode, a positive interval and a command" + ) + peak_bytes(0, sys.platform) # Fail before launching on unsupported platforms. + read_rss = rss_reader() if mode == "profile" else None + output.parent.mkdir(parents=True, exist_ok=True) + timeline = output.with_suffix(".rss.csv") + if output.exists() or (mode == "profile" and timeline.exists()): + raise ValueError(f"Memory artifact exists; choose a fresh run path: {output}") + result: dict[str, Any] = { + "schema_version": 1, + "mode": mode, + "units": "bytes", + "method": "wait4.ru_maxrss", + "scope": "process lifetime including startup and teardown", + "command": command, + "cwd": str(cwd or Path.cwd()), + "platform": sys.platform, + "pid": None, + "complete": False, + "returncode": None, + "peak_rss_bytes": None, + "sampled_max_rss_bytes": None, + "sample_count": 0, + "interval_ms": interval_ms if read_rss else None, + "timeline": timeline.name if read_rss else None, + "sampling_method": ( + "proc_pidinfo.PROC_PIDTASKINFO" + if sys.platform == "darwin" + else "/proc/PID/statm" + ) + if read_rss + else None, + } + process: subprocess.Popen | None = None + stream = None + started = time.monotonic() + + def reap(block: bool) -> bool: + assert process is not None + while True: + try: + pid, status, usage = os.wait4(process.pid, 0 if block else os.WNOHANG) + break + except InterruptedError: + continue + if not pid: + return False + process.returncode = os.waitstatus_to_exitcode(status) + result["returncode"] = process.returncode + result["peak_rss_bytes"] = peak_bytes(usage.ru_maxrss, sys.platform) + result["complete"] = process.returncode == 0 + return True + + try: + writer = None + if read_rss: + stream = timeline.open("x", newline="") + writer = csv.writer(stream) + writer.writerow(["elapsed_s", "rss_bytes", "pid"]) + process = subprocess.Popen(command, cwd=cwd) + result["pid"] = process.pid + if read_rss is None: + reap(True) + else: + deadline = started + while not reap(False): + rss = read_rss(process.pid) + if rss is not None: + assert writer is not None and stream is not None + writer.writerow([time.monotonic() - started, rss, process.pid]) + stream.flush() + result["sample_count"] += 1 + result["sampled_max_rss_bytes"] = max( + result["sampled_max_rss_bytes"] or 0, rss + ) + deadline = max(deadline + interval_ms / 1000, time.monotonic()) + time.sleep(max(0.0, deadline - time.monotonic())) + except BaseException as exc: + result["error"] = f"{type(exc).__name__}: {exc}" + if process is not None and process.returncode is None: + try: + process.kill() + except ProcessLookupError: + pass + reap(True) + result["complete"] = False + raise + finally: + if stream is not None: + stream.close() + result["timeline_sha256"] = file_digest(timeline) + result["process_elapsed_s"] = time.monotonic() - started + temporary = output.with_suffix(output.suffix + ".tmp") + temporary.write_text(json.dumps(result, indent=2) + "\n") + temporary.replace(output) + return result + + +def read_memory( + path: Path, expected: dict[str, Any] | None = None +) -> dict[str, Any] | None: + if not path.is_file(): + return None + value = json.loads(path.read_text()) + if not isinstance(value, dict): + raise ValueError(f"Invalid memory measurement: {path}") + peak = value.get("peak_rss_bytes") + if ( + value.get("schema_version") != 1 + or value.get("units") != "bytes" + or value.get("mode") not in ("peak", "profile") + or not value.get("complete") + or value.get("returncode") != 0 + or type(peak) is not int + or peak < 0 + or not isinstance(value.get("scope"), str) + or value.get("method") != "wait4.ru_maxrss" + or type(value.get("pid")) is not int + or value["pid"] <= 0 + or type(value.get("sample_count")) is not int + or value["sample_count"] < 0 + ): + raise ValueError(f"Invalid/incomplete memory measurement: {path}") + if expected and expected.get("mode", "off") != "off": + if value["mode"] != expected["mode"] or ( + value["mode"] == "profile" + and value.get("interval_ms") != expected["interval_ms"] + ): + raise ValueError(f"Memory collection settings differ: {path}") + if value["mode"] == "profile": + maximum = value.get("sampled_max_rss_bytes") + if ( + type(value.get("interval_ms")) is not int + or value["interval_ms"] <= 0 + or (value["sample_count"] == 0 and maximum is not None) + or (value["sample_count"] > 0 and (type(maximum) is not int or maximum < 0)) + ): + raise ValueError(f"Invalid memory sampling metadata: {path}") + name = value.get("timeline") + if not isinstance(name, str) or Path(name).name != name: + raise ValueError(f"Invalid memory timeline path: {path}") + timeline = path.parent / name + if file_digest(timeline) != value.get("timeline_sha256"): + raise ValueError(f"Memory timeline changed: {timeline}") + elif ( + value["sample_count"] != 0 + or value.get("interval_ms") is not None + or value.get("sampled_max_rss_bytes") is not None + or value.get("timeline") is not None + ): + raise ValueError(f"Unexpected RSS samples in peak-only measurement: {path}") + return value + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + add_memory_arguments(parser) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("command", nargs=argparse.REMAINDER) + args = parser.parse_args() + command = args.command[1:] if args.command[:1] == ["--"] else args.command + if not command: + parser.error("A measured command is required after --") + if args.memory_mode == "off": + raise SystemExit(subprocess.call(command)) + try: + result = run_measured( + command, + args.output, + mode=args.memory_mode, + interval_ms=args.memory_interval_ms, + ) + except (OSError, ValueError) as exc: + parser.exit(1, f"Memory collection failed: {exc}\n") + code = result["returncode"] + raise SystemExit(code if code >= 0 else 128 - code) + + +if __name__ == "__main__": + main() diff --git a/scripts/benchmarking/_performance_memory_report.py b/scripts/benchmarking/_performance_memory_report.py new file mode 100644 index 000000000..216a34b4a --- /dev/null +++ b/scripts/benchmarking/_performance_memory_report.py @@ -0,0 +1,263 @@ +"""Render observed process peaks and independent RSS traces; never estimate gaps.""" + +from __future__ import annotations + +import csv +import math +import statistics +from collections import defaultdict +from pathlib import Path +from typing import Any + +from tabulate import tabulate + + +def timeline_rows(path: Path, memory: dict[str, Any]) -> list[tuple[float, int]]: + result = [] + previous = -1.0 + with path.open(newline="") as stream: + for row in csv.DictReader(stream): + elapsed, rss, pid = ( + float(row["elapsed_s"]), + int(row["rss_bytes"]), + int(row["pid"]), + ) + if ( + not math.isfinite(elapsed) + or elapsed < previous + or elapsed < 0 + or rss < 0 + or pid != memory["pid"] + ): + raise ValueError(f"Invalid RSS sample: {path}") + result.append((elapsed, rss)) + previous = elapsed + maximum = max((rss for _, rss in result), default=None) + if ( + len(result) != memory["sample_count"] + or maximum != memory["sampled_max_rss_bytes"] + ): + raise ValueError(f"RSS timeline summary mismatch: {path}") + return result + + +def write_memory_report( + measurements: list[Any], output: Path, *, include_traces: bool = True +) -> list[str]: + groups: dict[tuple, list[Any]] = defaultdict(list) + runs = [] + for m in measurements: + groups[m.group].append(m) + memory = m.memory or {} + runs.append( + { + "workload": m.workload, + "backend": m.backend, + "task": m.task, + "threads": m.threads, + "repetition": m.repetition, + "peak_rss_bytes": memory.get("peak_rss_bytes"), + "sampled_max_rss_bytes": memory.get("sampled_max_rss_bytes"), + "sample_count": memory.get("sample_count"), + "mode": memory.get("mode", "unavailable"), + "method": memory.get("method", ""), + "scope": memory.get("scope", ""), + "pid": memory.get("pid"), + "interval_ms": memory.get("interval_ms"), + "source": memory.get("_source", ""), + } + ) + summaries = [] + for (workload, backend, task, threads), values in sorted(groups.items()): + observed = [m.memory for m in values if m.memory is not None] + signatures = { + (m["mode"], m.get("interval_ms"), m["scope"], m["method"]) for m in observed + } + if len(signatures) > 1: + raise ValueError( + f"Mixed memory instrumentation in series: {(workload, backend, task, threads)}" + ) + peaks = [m["peak_rss_bytes"] / 2**20 for m in observed] + complete = len(peaks) == len(values) + summaries.append( + { + "workload": workload, + "backend": backend, + "task": task, + "threads": threads, + "runs": len(values), + "observed_peaks": len(peaks), + "median_peak_mib": statistics.median(peaks) if complete else None, + "min_peak_mib": min(peaks) if complete else None, + "max_peak_mib": max(peaks) if complete else None, + "memory_mode": observed[0]["mode"] if observed else "unavailable", + } + ) + for name, rows in (("memory_runs.csv", runs), ("memory_summary.csv", summaries)): + if rows: + with (output / name).open("w", newline="") as stream: + writer = csv.DictWriter(stream, fieldnames=list(rows[0])) + writer.writeheader() + writer.writerows(rows) + + def display(value: float | None) -> str: + return "unavailable" if value is None else f"{value:.2f}" + + lines = [ + "## Process Memory", + "", + "OS high-water marks cover one backend process and all its threads, including startup, " + "loading and teardown. They exclude the monitor/logger and are not parser-only heap sizes. " + "Timing retains each workload's existing boundaries. Missing observations are unavailable, never zero.", + "", + tabulate( + [ + [ + r["workload"], + r["backend"], + r["task"], + r["threads"], + f"{r['observed_peaks']}/{r['runs']}", + display(r["median_peak_mib"]), + display(r["min_peak_mib"]), + display(r["max_peak_mib"]), + r["memory_mode"], + ] + for r in summaries + ], + headers=[ + "Workload", + "Backend", + "Task", + "Threads", + "Peaks/runs", + "Median MiB", + "Min MiB", + "Largest peak MiB", + "Capture", + ], + tablefmt="github", + disable_numparse=True, + ), + "", + "`existing` denotes a previously recorded replay RUSAGE_SELF peak. Reused runs have no new timeline. " + "Peak/profile comparison series run in fresh processes; memory-off comparisons retain their historical " + "shared-process execution. Compare timing only under matching instrumentation and isolation. " + "Sampled maxima are separate from OS peaks and can miss short spikes. See memory_runs.csv for methods and scopes.", + "", + "[Elapsed-time memory traces](memory_traces.md).", + "", + ] + trace_sections: list[str] = [] + + def write_traces() -> None: + trace_lines = [ + "# Memory traces", + "", + "[Back to Process Memory](summary.md#process-memory).", + "", + "Current RSS is sampled over each process's own elapsed-time clock. " + "Repetitions are plotted independently, without averaging. Samples can miss " + "short spikes; OS peak measurements are in the summary.", + "", + *( + trace_sections + or ["No sampled RSS traces are available for this report.", ""] + ), + ] + (output / "memory_traces.md").write_text("\n".join(trace_lines) + "\n") + + if not any(m.memory is not None for m in measurements): + write_traces() + return lines + import matplotlib + + matplotlib.use("Agg") + import matplotlib.pyplot as plt + + def save(fig: Any, filename: str, caption: str) -> list[str]: + fig.tight_layout() + fig.savefig(output / filename, dpi=180) + plt.close(fig) + return [f"![{caption}]({filename})", ""] + + by_task: dict[tuple[str, str], list[dict[str, Any]]] = defaultdict(list) + for row in summaries: + if row["median_peak_mib"] is not None: + by_task[row["workload"], row["task"]].append(row) + for (workload, task), rows in sorted(by_task.items()): + fig, ax = plt.subplots(figsize=(8, 4)) + for backend in sorted({r["backend"] for r in rows}): + selected = sorted( + (r for r in rows if r["backend"] == backend), key=lambda r: r["threads"] + ) + ax.errorbar( + [r["threads"] for r in selected], + [r["median_peak_mib"] for r in selected], + yerr=[ + [r["median_peak_mib"] - r["min_peak_mib"] for r in selected], + [r["max_peak_mib"] - r["median_peak_mib"] for r in selected], + ], + marker="o", + capsize=3, + label=backend, + ) + ax.set( + xlabel="Threads", + ylabel="OS peak RSS (MiB)", + title=f"{workload}: {task}", + xticks=sorted({0, *(r["threads"] for r in rows)}), + ) + ax.set_xlim(left=0) + ax.set_ylim(bottom=0) + ax.legend() + lines += save( + fig, + f"memory_{workload}_{task.replace('+', '_')}.png", + "Process peak RSS; median and observed range", + ) + if not include_traces: + return [ + line.replace( + "See memory_runs.csv for methods and scopes.", + "Methods and scopes are recorded in publication provenance.", + ) + for line in lines + if "memory_traces.md" not in line + ] + for (workload, backend, task, threads), values in sorted(groups.items()): + profiled = [m for m in values if m.memory and m.memory.get("_timeline_path")] + if not profiled: + continue + fig, ax = plt.subplots(figsize=(8, 4)) + found = False + for m in profiled: + memory = m.memory + trace = timeline_rows(Path(memory["_timeline_path"]), memory) + if trace: + ax.plot( + [t for t, _ in trace], + [rss / 2**20 for _, rss in trace], + label=f"r{m.repetition}", + ) + found = True + if not found: + plt.close(fig) + continue + ax.set( + xlabel="Elapsed process time (s)", + ylabel="Sampled current RSS (MiB)", + title=f"{workload}: {backend}, {task}, {threads} thread(s)", + ) + ax.legend() + trace_sections += [ + f"## {workload}: {backend}, {task}, {threads} thread(s)", + "", + ] + trace_sections += save( + fig, + f"rss_{workload}_{backend}_{task.replace('+', '_')}_t{threads}.png", + "Current RSS over independent process clocks; repetitions are not averaged", + ) + write_traces() + return lines diff --git a/scripts/benchmarking/_performance_runtime.py b/scripts/benchmarking/_performance_runtime.py index 6244079ec..ddbe283b7 100644 --- a/scripts/benchmarking/_performance_runtime.py +++ b/scripts/benchmarking/_performance_runtime.py @@ -146,6 +146,7 @@ def __init__( total_pages: int, ): self.directory = directory + self.identity = identity directory.mkdir(parents=True, exist_ok=True) self.provenance = provenance self.total_pages = total_pages @@ -176,15 +177,22 @@ def save(self, run: Any) -> None: and len(run.samples) == self.total_pages and all(row.success for row in run.samples), "run": { - key: getattr(run, key) - for key in ( - "backend", - "task", - "threads", - "wall_s", - "per_page_source", - "doc_errors", - ) + **{ + key: getattr(run, key) + for key in ( + "backend", + "task", + "threads", + "wall_s", + "per_page_source", + "doc_errors", + ) + }, + **( + {"memory": run.memory} + if getattr(run, "memory", None) is not None + else {} + ), }, }, ) @@ -204,6 +212,24 @@ def load(self, backend: str, task: str, threads: int) -> dict[str, Any] | None: f"Checkpoint inputs, settings, source or packages changed: {json_path}; use a fresh output directory" ) run = state["run"] + memory = run.get("memory") + requested = self.identity.get("memory", {"mode": "off"}) + if requested["mode"] != "off" and memory is None: + return None + if memory is not None: + from _performance_memory import read_memory + + relative = Path(memory["artifact"]) + if relative.is_absolute() or ".." in relative.parts: + raise ValueError(f"Invalid checkpoint memory path: {json_path}") + artifact = self.directory.parent / relative + if not artifact.is_file() or digest(artifact) != memory["artifact_sha256"]: + return None + try: + if read_memory(artifact, requested) is None: + return None + except (OSError, ValueError): + return None rows = read_page_rows(csv_path) if (run["backend"], run["task"], run["threads"]) != (backend, task, threads): raise ValueError(f"Checkpoint series identity mismatch: {json_path}") diff --git a/scripts/benchmarking/_performance_worker.py b/scripts/benchmarking/_performance_worker.py new file mode 100644 index 000000000..8f29ee8ad --- /dev/null +++ b/scripts/benchmarking/_performance_worker.py @@ -0,0 +1,51 @@ +"""Run exactly one comparison series in a fresh process; internal entry point.""" + +from __future__ import annotations + +import json +import sys +from dataclasses import asdict +from pathlib import Path +from typing import Any, Callable + + +def main() -> None: + from run_performance_benchmarking import COMPARISON_BACKENDS + + request = json.loads(Path(sys.argv[1]).read_text()) + backend = request.pop("backend") + schedule = [(Path(p), pages) for p, pages in request.pop("pdf_schedule")] + total_pages = request.pop("total_pages") + runner: Callable[..., Any] = COMPARISON_BACKENDS[backend][0] # type: ignore[assignment] + try: + run = runner(schedule, total_pages, **request) + except ImportError as exc: + print(str(exc), file=sys.stderr) + raise SystemExit(5) + output = Path(sys.argv[2]) + temporary = output.with_suffix(".tmp") + # Stream the result: a second corpus-sized list/string would inflate the + # worker's lifetime peak solely to transport observations to its parent. + with temporary.open("w") as stream: + stream.write('{"samples":[') + for index, row in enumerate(run.samples): + if index: + stream.write(",") + json.dump(asdict(row), stream) + stream.write("]") + for key in ( + "backend", + "task", + "threads", + "wall_s", + "per_page_source", + "doc_errors", + ): + stream.write("," + json.dumps(key) + ":") + json.dump(getattr(run, key), stream) + stream.write("}\n") + temporary.replace(output) + + +if __name__ == "__main__": + main() diff --git a/scripts/benchmarking/recover_performance_benchmarks.py b/scripts/benchmarking/recover_performance_benchmarks.py index 31314daa5..dc77e3e01 100644 --- a/scripts/benchmarking/recover_performance_benchmarks.py +++ b/scripts/benchmarking/recover_performance_benchmarks.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""Reuse validated core results and recollect comparisons with explicit provenance.""" +"""Resume missing configurations, reusing validated results in a new bundle.""" from __future__ import annotations @@ -13,10 +13,12 @@ import subprocess import sys import tempfile +from dataclasses import dataclass from datetime import datetime, timezone from pathlib import Path from typing import Any +from _performance_memory import add_memory_arguments from _performance_runtime import ( ROOT, atomic_json, @@ -73,56 +75,277 @@ def validate_environment(metadata: dict[str, Any], provenance: dict[str, Any]) - ) +@dataclass(frozen=True) +class CoreJob: + workload: str + profile: str + threads: int + repetition: int + artifact: Path + + @property + def key(self) -> tuple: + backend = ( + "docling-parse" + if self.workload == "native" + else ("pdfium" if self.profile == "pdfium" else "threaded") + ) + task = self.profile if self.workload == "native" else "render" + return ((self.workload, backend, task, self.threads), self.repetition) + + +def core_jobs(metadata: dict[str, Any]) -> list[CoreJob]: + jobs = [] + for repetition in range(1, metadata["repeats"] + 1): + for profile in metadata.get("native_profiles", {"render": {}}): + for threads in metadata["threads"]: + parent = ( + Path("native") / profile if profile != "render" else Path("native") + ) + jobs.append( + CoreJob( + "native", + profile, + threads, + repetition, + parent / f"r{repetition}/t{threads}.csv", + ) + ) + for backend, threads in [ + ("pdfium", 1), + *(("threaded", t) for t in metadata["threads"]), + ]: + name = "pdfium" if backend == "pdfium" else f"t{threads}" + jobs.append( + CoreJob( + "replay", + backend, + threads, + repetition, + Path(f"replay/r{repetition}/{name}/report.json"), + ) + ) + return jobs + + +def inspect_job(inputs: ReportInputs, job: CoreJob) -> bool: + path = inputs.root / job.artifact + if not path.is_file(): + print(f"Pending: {job.artifact} (no completed artifact)", flush=True) + return False + try: + values = ( + inputs.native(path) if job.workload == "native" else inputs.replay(path) + ) + if [(m.group, m.repetition) for m in values] != [job.key] or any( + m.wall_s <= 0 for m in values + ): + raise ValueError("Unexpected configuration or nonpositive wall time") + except (ValueError, KeyError, OSError, TypeError) as exc: + # Source changes are a provenance violation, not an interrupted measurement. + if any( + word in str(exc) + for word in ("source revision", "source diff", "script hash") + ): + raise + print(f"Pending: {job.artifact} (incomplete/invalid: {exc})", flush=True) + return False + print(f"Reuse: {job.artifact}", flush=True) + return True + + +def link_job(source: Path, output: Path, job: CoreJob) -> None: + artifact = source / job.artifact + if job.workload == "replay": + candidates = [ + artifact.parent, + *artifact.parent.parent.glob(artifact.parent.name + ".memory.*"), + ] + else: + candidates = list(artifact.parent.glob(artifact.stem + ".*")) + for path in candidates: + dest = output / path.relative_to(source) + dest.parent.mkdir(parents=True, exist_ok=True) + dest.symlink_to(path.resolve(), target_is_directory=path.is_dir()) + + +def validate_resume_scripts(metadata: dict[str, Any], jobs: list[CoreJob]) -> None: + names = set() + if any(j.workload == "replay" for j in jobs): + names.add("run_performance_benchmarking_with_docling_settings.py") + if jobs and metadata.get("memory_collection", {}).get("mode", "off") != "off": + names.add("_performance_memory.py") + for name in names: + recorded = metadata.get("script_sha256", {}).get("scripts/benchmarking/" + name) + if recorded is None or digest(ROOT / "scripts/benchmarking" / name) != recorded: + raise ValueError( + f"Cannot resume core measurements: runner {name} changed or its original hash is unavailable" + ) + for key, value in metadata.get("environment", {}).items(): + if key != "SDKROOT" and os.environ.get(key) != value: + raise ValueError( + f"Cannot resume core measurements: environment {key} changed" + ) + + def prepare_recovery( - source: Path, output: Path, provenance: dict[str, Any] + source: Path, + output: Path, + provenance: dict[str, Any], + memory: dict[str, Any] | None = None, ) -> dict[str, Any]: inputs = ReportInputs(source) metadata = copy.deepcopy(inputs.metadata) validate_environment(metadata, provenance) - print("Validating completed native and Python replay results ...", flush=True) - measurements = inputs.load(include_comparison=False) - print(f"Validated {len(measurements)} reusable core measurements.", flush=True) + jobs = core_jobs(metadata) + print("Inspecting native and Python replay configurations ...", flush=True) + complete = [job for job in jobs if inspect_job(inputs, job)] + pending = [job for job in jobs if job not in complete] + if pending: + validate_resume_scripts(metadata, pending) + if memory is not None and memory != metadata.get( + "memory_collection", {"mode": "off", "interval_ms": None} + ): + raise ValueError( + "Cannot change memory settings while resuming core measurements; omit flags to inherit the original settings" + ) + print( + f"Recovery plan: {len(complete)} core configurations reused, {len(pending)} to run.", + flush=True, + ) print("Verifying original dataset hashes ...", flush=True) for item in metadata["dataset"]["files"]: if digest(Path(item["path"])) != item["sha256"]: raise ValueError( f"Recovery dataset differs from original manifest: {item['name']}" ) - preflight = reference_preflight(metadata) - for name in ("native", "replay"): - (output / name).symlink_to((source / name).resolve(), target_is_directory=True) + if metadata["comparison"]: + metadata["reference_preflight"] = reference_preflight(metadata) shutil.copy2(source / "metadata.json", output / "original-metadata.json") for name in ("CMakeCache.txt", "native-flags.make"): if (source / name).is_file(): shutil.copy2(source / name, output / name) - old_provenance = metadata.get("comparison_provenance") - if old_provenance == provenance: + for job in complete: + link_job(source, output, job) + # Artifact-specific provenance keeps old replay source identities intact. + recovered = metadata.setdefault("recovered_replay_provenance", {}) + for job in pending: + if job.workload == "replay": + recovered[str(job.artifact)] = provenance + compatible_memory = memory is None or metadata.get("comparison_memory") == memory + if ( + metadata["comparison"] + and metadata.get("comparison_provenance") == provenance + and compatible_memory + ): for directory in sorted((source / "comparison").glob("r*/*/checkpoints")): target = output / directory.relative_to(source) target.parent.mkdir(parents=True, exist_ok=True) shutil.copytree(directory, target) - print( - "Compatible comparison checkpoints copied; failed/missing series will be measured.", - flush=True, - ) - else: - print( - "Comparisons will be recollected with the current source/package versions.", - flush=True, - ) - metadata["comparison"] = True - metadata["comparison_provenance"] = provenance - metadata["reference_preflight"] = preflight + memory_source = directory.parent / "memory" + if memory_source.is_dir(): + shutil.copytree(memory_source, target.parent / "memory") + if metadata["comparison"]: + if memory is not None: + metadata["comparison_memory"] = memory + metadata["comparison_provenance"] = provenance metadata["recovery"] = { "source_bundle": str(source), "created_at": datetime.now(timezone.utc).isoformat(), - "core_results": "reused unchanged", - "comparisons": "current provenance; compatible checkpoints only", + "core_results": "completed configurations reused; missing/incomplete configurations resumed", + "pending_core": [str(j.artifact) for j in pending], + "comparisons": "current provenance; compatible checkpoints only" + if metadata["comparison"] + else "not requested", } atomic_json(output / "metadata.json", metadata) return metadata +def core_command( + metadata: dict[str, Any], job: CoreJob, output: Path, compact: bool +) -> tuple[list[str], Path]: + artifact = output / job.artifact + if job.workload == "native": + log = artifact.with_suffix(".log") + command = [ + str(ROOT / "build/run_scaling.exe"), + metadata["dataset"]["root"], + "--recursive", + "--mode", + "render", + "--threads", + str(job.threads), + "--max-concurrent-results", + str(metadata["max_concurrent_results"]), + "--scale", + str(metadata["scale"]), + "--enable-timing", + "--timing-csv", + str(artifact), + ] + settings = metadata.get("native_profiles", {}).get( + job.profile, {"create_word_cells": True, "create_line_cells": True} + ) + command += [ + f"--{name.replace('_', '-')}={str(value).lower()}" + for name, value in settings.items() + ] + if compact: + command += ["-v", "WARNING", "--suppress-config", "--suppress-timings"] + else: + log = artifact.parent.with_suffix(".log") + command = [ + sys.executable, + str( + ROOT + / "scripts/benchmarking/run_performance_benchmarking_with_docling_settings.py" + ), + metadata["dataset"]["root"], + "--backend", + job.profile, + "--dataset-revision", + metadata["dataset"]["revision"], + "--output-dir", + str(artifact.parent), + ] + if job.profile == "threaded": + command += ["--threads", str(job.threads)] + if compact: + command.append("--suppress-config") + memory = metadata.get("memory_collection", {"mode": "off"}) + if memory["mode"] != "off": + command = [ + sys.executable, + str(ROOT / "scripts/benchmarking/_performance_memory.py"), + "--memory-mode", + memory["mode"], + "--memory-interval-ms", + str(memory.get("interval_ms") or 50), + "--output", + str(log.with_suffix(".memory.json")), + "--", + *command, + ] + return command, log + + +def resume_core(metadata: dict[str, Any], output: Path, compact: bool) -> None: + pending = set(metadata["recovery"]["pending_core"]) + for job in core_jobs(metadata): + if str(job.artifact) not in pending: + continue + command, log = core_command(metadata, job, output, compact) + log.parent.mkdir(parents=True, exist_ok=True) + print( + f"Resume: repetition {job.repetition}/{metadata['repeats']}, {job.workload}, {job.profile}, {job.threads} thread(s)", + flush=True, + ) + run_logged(command, log, output / "commands.log") + if not inspect_job(ReportInputs(output), job): + raise ValueError(f"Resumed configuration failed validation: {job.artifact}") + + def comparison_command( metadata: dict[str, Any], directory: Path, mode: str, compact: bool ) -> list[str]: @@ -161,6 +384,10 @@ def comparison_command( ] if compact: command.append("--suppress-config") + if memory := metadata.get("comparison_memory"): + command += ["--memory-mode", memory["mode"]] + if memory["mode"] == "profile": + command += ["--memory-interval-ms", str(memory["interval_ms"])] return command @@ -168,6 +395,12 @@ def main() -> None: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("source", type=Path) parser.add_argument("--output-dir", type=Path) + add_memory_arguments(parser) + # None means inherit the source bundle, rather than silently switching to off. + parser.set_defaults( + memory_mode=os.environ.get("MEMORY_MODE"), + memory_interval_ms=os.environ.get("MEMORY_INTERVAL_MS"), + ) args = parser.parse_args() source = args.source.resolve() output_mode = os.environ.get("BENCHMARK_OUTPUT", "compact") @@ -217,18 +450,35 @@ def main() -> None: output / "dependencies.log", ) provenance = runtime_provenance() - metadata = prepare_recovery(source, output, provenance) - for repetition in range(1, metadata["repeats"] + 1): - for mode in ("parse-without-bitmap", "render"): - directory = output / "comparison" / f"r{repetition}" / mode - directory.mkdir(parents=True, exist_ok=True) - run_logged( - comparison_command( - metadata, directory, mode, output_mode == "compact" - ), - directory / "run.log", - output / "commands.log", + original = json.loads((source / "metadata.json").read_text()) + memory = None + if args.memory_mode is not None or args.memory_interval_ms is not None: + saved = original.get( + "memory_collection", {"mode": "off", "interval_ms": None} + ) + mode = args.memory_mode or saved["mode"] + memory = { + "mode": mode, + "interval_ms": ( + args.memory_interval_ms or saved.get("interval_ms") or 50 ) + if mode == "profile" + else None, + } + metadata = prepare_recovery(source, output, provenance, memory) + resume_core(metadata, output, output_mode == "compact") + if metadata["comparison"]: + for repetition in range(1, metadata["repeats"] + 1): + for mode in ("parse-without-bitmap", "render"): + directory = output / "comparison" / f"r{repetition}" / mode + directory.mkdir(parents=True, exist_ok=True) + run_logged( + comparison_command( + metadata, directory, mode, output_mode == "compact" + ), + directory / "run.log", + output / "commands.log", + ) inputs = ReportInputs(output) write_report(inputs, inputs.load(), output / "report") except ( diff --git a/scripts/benchmarking/refresh_performance_benchmarks.sh b/scripts/benchmarking/refresh_performance_benchmarks.sh index 6ccc03ef0..1422d71a0 100755 --- a/scripts/benchmarking/refresh_performance_benchmarks.sh +++ b/scripts/benchmarking/refresh_performance_benchmarks.sh @@ -4,7 +4,7 @@ set -Eeuo pipefail if [[ ${1:-} == --help ]]; then cat <<'HELP' -Usage: bash scripts/benchmarking/refresh_performance_benchmarks.sh +Usage: bash scripts/benchmarking/refresh_performance_benchmarks.sh [--memory-mode off|peak|profile] [--memory-interval-ms 50] Environment: REPEATS=1 Fresh-process repetitions (use 3 for publication) @@ -16,7 +16,9 @@ Environment: DATASET=bo767 Select bo20, bo767 or bo10k DATASET_REVISION= Override the selected dataset's pinned revision OUT= Default: a unique scratch-performance-refresh-* path - RECOVER_FROM= Reuse validated native/replay results; recollect comparisons in a new bundle + RECOVER_FROM= Resume missing configurations in a new bundle; inherit original settings + MEMORY_MODE=off off, independent OS peak, or peak plus sampled RSS (profile) + MEMORY_INTERVAL_MS=50 Positive sampling interval; CLI overrides environment Default dataset pins (PDFs / pages): bo20 65b9971b3fa939fe4ef89be678237e6928049b67 20 / 496 @@ -38,9 +40,26 @@ Full corpus sweeps can take hours, especially with RUN_COMPARISON=1 or REPEATS=3 HELP exit 0 fi -if [[ $# != 0 ]]; then - echo "Unexpected argument; use --help." >&2 - exit 2 +memory_mode_explicit=${MEMORY_MODE+x} +memory_interval_explicit=${MEMORY_INTERVAL_MS+x} +MEMORY_MODE=${MEMORY_MODE:-off} +MEMORY_INTERVAL_MS=${MEMORY_INTERVAL_MS:-50} +while [[ $# != 0 ]]; do + case "$1" in + --memory-mode|--memory-interval-ms) + if [[ $# -lt 2 ]]; then echo "Missing value for $1" >&2; exit 2; fi + if [[ $1 == --memory-mode ]]; then + MEMORY_MODE=$2; memory_mode_explicit=x + else + MEMORY_INTERVAL_MS=$2; memory_interval_explicit=x + fi + shift 2 ;; + *) echo "Unexpected argument: $1; use --help." >&2; exit 2 ;; + esac +done +case "$MEMORY_MODE" in off|peak|profile) ;; *) echo "Invalid memory mode." >&2; exit 2 ;; esac +if [[ ! $MEMORY_INTERVAL_MS =~ ^[1-9][0-9]*$ ]]; then + echo "MEMORY_INTERVAL_MS must be a positive integer." >&2; exit 2 fi REPO_ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd) @@ -52,8 +71,11 @@ if [[ -n ${RECOVER_FROM:-} ]]; then fi recovery=(uv run --no-sync python scripts/benchmarking/recover_performance_benchmarks.py "$RECOVER_FROM") if [[ -n ${OUT:-} ]]; then recovery+=(--output-dir "$OUT"); fi + if [[ -n $memory_mode_explicit ]]; then recovery+=(--memory-mode "$MEMORY_MODE"); fi + if [[ -n $memory_interval_explicit ]]; then recovery+=(--memory-interval-ms "$MEMORY_INTERVAL_MS"); fi exec "${recovery[@]}" fi +export MEMORY_MODE MEMORY_INTERVAL_MS REPEATS=${REPEATS:-1} THREADS=${THREADS:-1,4,8,12} RUN_COMPARISON=${RUN_COMPARISON:-0} @@ -128,6 +150,7 @@ echo "Repetitions: $REPEATS; threads: $THREADS; cross-package suite: $RUN_COMPAR echo "Dataset: $DATASET_REPOSITORY at $DATASET_REVISION" echo "Benchmark output: $BENCHMARK_OUTPUT" echo "Build mode: $BUILD_MODE" +echo "Memory collection: $MEMORY_MODE; sampling interval: $MEMORY_INTERVAL_MS ms" echo "Full corpus measurements can take hours. Progress and commands are saved below this directory." trap 'echo "Stopped. Inspect the latest log under $OUT; completed measurements are preserved." >&2' ERR @@ -340,7 +363,10 @@ metadata = { "pymupdf", "pdfplumber", "pdfminer.six", "pypdf", "matplotlib"]}, "script_sha256": {str(p): digest(p) for p in Path("scripts/benchmarking").glob("*.py")}, "launcher_sha256": digest("scripts/benchmarking/refresh_performance_benchmarks.sh"), + "memory_collection": {"mode": os.environ.get("MEMORY_MODE", "off"), + "interval_ms": int(os.environ.get("MEMORY_INTERVAL_MS", "50")) if os.environ.get("MEMORY_MODE") == "profile" else None}, } +metadata["comparison_memory"] = dict(metadata["memory_collection"]) for command, key in [(["sysctl", "-n", "machdep.cpu.brand_string"], "cpu"), (["sysctl", "-n", "hw.memsize"], "memory_bytes")]: if sys.platform == "darwin": @@ -381,7 +407,22 @@ run_experiment() { local description=$2 shift 2 experiment=$((experiment + 1)) - run_logged "$log" "Experiment $experiment/$total_experiments | $description" "$@" + local command=("$@") + if [[ $MEMORY_MODE != off ]]; then + if [[ $log == */comparison/*/run.log ]]; then + # The comparison runner isolates its backend/thread series internally. + command+=(--memory-mode "$MEMORY_MODE" --memory-interval-ms "$MEMORY_INTERVAL_MS") + else + # Observe the actual Python process, not uv and its child together. + if [[ ${command[0]} == uv ]]; then + command=("$REPO_ROOT/.venv/bin/python" "${command[@]:4}") + fi + command=("$REPO_ROOT/.venv/bin/python" scripts/benchmarking/_performance_memory.py + --memory-mode "$MEMORY_MODE" --memory-interval-ms "$MEMORY_INTERVAL_MS" + --output "${log%.log}.memory.json" -- "${command[@]}") + fi + fi + run_logged "$log" "Experiment $experiment/$total_experiments | $description" "${command[@]}" } for (( repetition=1; repetition<=REPEATS; repetition++ )); do diff --git a/scripts/benchmarking/run_performance_analysis.py b/scripts/benchmarking/run_performance_analysis.py index c865fb30c..cbcecffe2 100644 --- a/scripts/benchmarking/run_performance_analysis.py +++ b/scripts/benchmarking/run_performance_analysis.py @@ -111,11 +111,13 @@ class PageTimings: ], "decode_contents": [ "content_decode_total", + "content_parse_interprete_total", "interprete_ops_total", "decode_xobjects_total", "decode_grphs_total", "decode_fonts_total", "parse_stream_total", + "form_parse_interprete_total", "do_form_machinery_total", "do_image_total", ], diff --git a/scripts/benchmarking/run_performance_benchmarking.py b/scripts/benchmarking/run_performance_benchmarking.py index 2c583b775..1e7002811 100644 --- a/scripts/benchmarking/run_performance_benchmarking.py +++ b/scripts/benchmarking/run_performance_benchmarking.py @@ -46,6 +46,7 @@ import argparse import contextlib +import json import os import platform import re @@ -68,6 +69,7 @@ series_label, write_page_rows, ) +from _performance_memory import add_memory_arguments, run_measured from _performance_runtime import BenchmarkCheckpoints, digest, runtime_provenance from tabulate import tabulate @@ -920,6 +922,7 @@ class BackendRun: per_page_source: str samples: List[PageRow] = field(default_factory=list) doc_errors: int = 0 + memory: dict | None = None @property def durations(self) -> List[float]: @@ -2140,6 +2143,9 @@ def _run_one_mode( collected_runs: List[BackendRun], checkpoints: BenchmarkCheckpoints | None = None, resume: bool = False, + memory_mode: str = "off", + memory_interval_ms: int = 50, + memory_directory: Path | None = None, ) -> Tuple[List[Tuple[str, float]], List[Tuple[int, float]]]: baselines: List[Tuple[str, float]] = [] @@ -2168,7 +2174,70 @@ def obtain( f" Reusing completed checkpoint: {backend}, {task}, {threads} thread(s)" ) return BackendRun(**cached) - run = measure() + if memory_mode == "off": + run = measure() + else: + assert memory_directory is not None + memory_directory.mkdir(parents=True, exist_ok=True) + prefix = memory_directory / f"{backend}-{task}-{threads}" + request = Path(str(prefix) + ".request.json") + response = Path(str(prefix) + ".result.json") + artifact = Path(str(prefix) + ".memory.json") + # Missing/corrupt checkpoints may be recollected. Keep their old + # artifacts, and never associate them with this fresh process. + if artifact.exists() or request.exists() or response.exists(): + import tempfile + + prefix = ( + Path( + tempfile.mkdtemp(prefix=prefix.name + "-", dir=memory_directory) + ) + / "run" + ) + request = prefix.with_suffix(".request.json") + response = prefix.with_suffix(".result.json") + artifact = prefix.with_suffix(".memory.json") + request.write_text( + json.dumps( + { + "backend": backend, + "pdf_schedule": [(str(p), pages) for p, pages in pdf_schedule], + "total_pages": total_pages, + "render": render, + "scale": scale, + "threads": threads, + "max_concurrent_results": max_concurrent_results, + "decode_options": decode_options, + "materialization_options": materialization_options, + "bytesio": bytesio, + } + ) + + "\n" + ) + memory = run_measured( + [ + sys.executable, + str(Path(__file__).with_name("_performance_worker.py")), + str(request.resolve()), + str(response.resolve()), + ], + artifact, + mode=memory_mode, + interval_ms=memory_interval_ms, + ) + if memory["returncode"] == 5: + raise ImportError(f"Backend {backend} could not import in its worker") + if memory["returncode"]: + raise RuntimeError(f"Backend worker failed; inspect {artifact}") + data = json.loads(response.read_text()) + data["samples"] = [PageRow(**row) for row in data["samples"]] + run = BackendRun(**data) + base = ( + checkpoints.directory.parent if checkpoints else memory_directory.parent + ) + memory["artifact"] = str(artifact.relative_to(base)) + memory["artifact_sha256"] = digest(artifact) + run.memory = memory if checkpoints is not None: checkpoints.save(run) return run @@ -2417,6 +2486,7 @@ def main(argv: List[str]) -> int: ), ) + add_memory_arguments(ap) args = ap.parse_args(argv) if args.resume and args.checkpoint_dir is None: ap.error("--resume requires --checkpoint-dir") @@ -2538,6 +2608,12 @@ def main(argv: List[str]) -> int: "bytesio": args.bytesio, "decode": decode_options, "materialization": mode_materialization_options, + "memory": { + "mode": args.memory_mode, + "interval_ms": args.memory_interval_ms + if args.memory_mode == "profile" + else None, + }, }, checkpoint_provenance, total_pages, @@ -2557,6 +2633,13 @@ def main(argv: List[str]) -> int: collected_runs=collected_runs, checkpoints=checkpoints, resume=args.resume, + memory_mode=args.memory_mode, + memory_interval_ms=args.memory_interval_ms, + memory_directory=( + args.checkpoint_dir.parent / "memory" + if args.checkpoint_dir + else args.output_dir / "memory" / m + ), ) _print_table(title, baselines, threaded_results, total_pages) diff --git a/scripts/benchmarking/run_performance_report.py b/scripts/benchmarking/run_performance_report.py index e278d7a59..ad5f1a296 100644 --- a/scripts/benchmarking/run_performance_report.py +++ b/scripts/benchmarking/run_performance_report.py @@ -23,6 +23,9 @@ from pathlib import Path from typing import Any +from _performance_memory import read_memory +from _performance_memory_report import write_memory_report + WORKLOAD_LABELS = { "native": "Native C++ scaling", "replay": "Python scaling with Docling settings", @@ -86,6 +89,14 @@ def task_title(workload: str, task: str) -> str: ) +def task_label(workload: str, task: str) -> str: + if workload == "native": + return "Render" if task == "render-only" else "Render + text" + if workload == "replay": + return "Backend replay" + return "Extract" if task == "parse" else "Render + extract" + + def plot_title(workload: str, task: str) -> str: return f"{WORKLOAD_LABELS[workload]}\n{textwrap.fill(task_title(workload, task), width=70)}" @@ -131,6 +142,7 @@ class Measurement: phases: dict[str, float] = field(default_factory=dict) durations: list[float] = field(default_factory=list) package_version: str = "" + memory: dict[str, Any] | None = None @property def group(self) -> tuple[str, str, str, int]: @@ -166,6 +178,39 @@ def __init__(self, root: Path): self.excluded_comparisons: list[dict[str, Any]] = [] self.comparison_errors: list[dict[str, Any]] = [] + def memory( + self, + path: Path, + legacy: dict[str, Any] | None = None, + *, + comparison: bool = False, + ) -> dict[str, Any] | None: + collection = self.metadata.get("memory_collection", {"mode": "off"}) + if comparison: + collection = self.metadata.get("comparison_memory", collection) + value = read_memory(path, collection) + if value is None: + if collection["mode"] != "off": + raise ValueError(f"Requested memory measurement is missing: {path}") + if legacy is None or legacy.get("peak_rss_mib") is None: + return None + peak = float(legacy["peak_rss_mib"]) + require( + math.isfinite(peak) and peak >= 0, f"Invalid legacy replay RSS: {path}" + ) + return { + "mode": "existing", + "peak_rss_bytes": round(peak * 2**20), + "scope": legacy.get("rss_scope", "process lifetime"), + "method": "resource.getrusage(RUSAGE_SELF)", + "sample_count": 0, + "_source": "existing replay report", + } + value["_source"] = str(path.relative_to(self.root)) + if value.get("timeline"): + value["_timeline_path"] = str(path.parent / value["timeline"]) + return value + def package_version(self, backend: str, workload: str) -> str: package = {"threaded": "docling-parse", "pdfium": "pypdfium2"}.get( backend, backend @@ -300,6 +345,7 @@ def native(self, path: Path) -> list[Measurement]: stages, durations[t], package_version=self.package_version("docling-parse", "native"), + memory=self.memory(path.with_suffix(".memory.json")), ) ) return result @@ -307,15 +353,26 @@ def native(self, path: Path) -> list[Measurement]: def replay(self, path: Path) -> list[Measurement]: report = json.loads(path.read_text()) replay_git = report["provenance"]["parse_git"] - if replay_git["revision"] == self.metadata["source_revision"]: + source_metadata = self.metadata.get("recovered_replay_provenance", {}).get( + str(path.relative_to(self.root)), self.metadata + ) + if source_metadata is not self.metadata: require( - replay_git["diff_sha256"] == self.metadata["source_diff_sha256"], + report["provenance"]["script_sha256"] + == source_metadata["script_sha256"][ + "run_performance_benchmarking_with_docling_settings.py" + ], + f"Recovered replay script hash changed: {path}", + ) + if replay_git["revision"] == source_metadata["source_revision"]: + require( + replay_git["diff_sha256"] == source_metadata["source_diff_sha256"], f"Replay tracked source diff changed during the run: {path}", ) else: require( committed_source_matches( - self.metadata, + source_metadata, report["provenance"], Path(__file__).resolve().parents[2], ), @@ -403,6 +460,7 @@ def replay(self, path: Path) -> list[Measurement]: dict(phases), durations, package_version=self.package_version(settings["backend"], "replay"), + memory=self.memory(path.parent.with_suffix(".memory.json"), run), ) ) return result @@ -436,6 +494,29 @@ def comparison(self, path: Path) -> list[Measurement]: f"Comparison checkpoint CSV changed: {csv_path}", ) group = state["run"]["backend"], int(state["run"]["threads"]) + memory = state["run"].get("memory") + if ( + self.metadata.get( + "comparison_memory", self.metadata.get("memory_collection", {}) + ).get("mode", "off") + != "off" + ): + require( + memory is not None, f"Missing comparison memory: {checkpoint}" + ) + if memory is not None: + relative = Path(memory["artifact"]) + require( + not relative.is_absolute() and ".." not in relative.parts, + f"Invalid comparison memory path: {checkpoint}", + ) + artifact = path.parent / relative + require( + artifact.is_file() + and hashlib.sha256(artifact.read_bytes()).hexdigest() + == memory["artifact_sha256"], + f"Comparison memory changed: {artifact}", + ) require( group not in checkpoints, f"Duplicate comparison checkpoint: {checkpoint}", @@ -568,6 +649,16 @@ def comparison(self, path: Path) -> list[Measurement]: ), durations=durations[backend, t], package_version=self.package_version(backend, "comparison"), + memory=( + self.memory( + path.parent + / checkpoints[backend, t]["run"]["memory"]["artifact"], + comparison=True, + ) + if provenance is not None + and checkpoints[backend, t]["run"].get("memory") + else None + ), ) ) if sizes and (backend, t) in sizes and not errors["pypdfium2", 1]: @@ -599,6 +690,8 @@ def load(self, *, include_comparison: bool = True) -> list[Measurement]: self.comparison_errors.clear() measurements = [] for path in sorted((self.root / "native").rglob("*.csv")): + if path.name.endswith(".memory.rss.csv"): + continue measurements.extend(self.native(path)) for path in sorted((self.root / "replay").glob("r*/*/report.json")): measurements.extend(self.replay(path)) @@ -909,6 +1002,7 @@ def write_report( write_csv(output / "excluded_comparisons.csv", inputs.excluded_comparisons) write_csv(output / "comparison_errors.csv", inputs.comparison_errors) meta = inputs.metadata + memory_lines = write_memory_report(measurements, output) lines = [ "# Performance measurement report", "", @@ -952,7 +1046,8 @@ def write_report( if meta.get("recovery"): lines += [ f"Core measurements reused from `{meta['recovery']['source_bundle']}`. " - "Comparisons were recollected with their own recorded source/package provenance.", + "Missing core configurations were resumed where needed; recovered replay source identities " + "are recorded per artifact in metadata. Comparisons, when requested, retain separate provenance.", "", ] if meta.get("comparison_provenance") and not core_only: @@ -971,18 +1066,21 @@ def write_report( f"| {package} | {meta.get('packages', {}).get(package, '—')} | {version} |" ) lines += [""] + lines += python_experiment_explanation(summary) + sections: dict[str, list[str]] = {} for workload in ("native", "replay", "comparison"): rows = [row for row in summary if row["workload"] == workload] if not rows: continue - if workload == "replay": - lines += python_experiment_explanation(summary) - lines += [ - f"## {WORKLOAD_LABELS[workload]}", + section: list[str] = [] + sections[workload] = section + level = "##" if workload == "comparison" else "###" + section += [ + f"{level} {WORKLOAD_LABELS[workload]}", "", ] if workload == "replay": - lines += [ + section += [ "This standalone backend-operation replay creates page-images and text, and requests bitmap " "bounding boxes and separator geometry, including coordinate normalization and text " "cleanup. Docling materializes word/line models; PDFium builds merged textline models " @@ -991,7 +1089,7 @@ def write_report( "", ] elif workload == "comparison": - lines += [ + section += [ "This experiment measures package-specific text and geometry APIs, optionally with page " "page-images. Docling materializes character models, PDFium returns rectangle text, and other " "packages expose their own geometry granularity. Word/line construction and separate " @@ -999,7 +1097,7 @@ def write_report( "comparisons, with different outputs from the Docling-settings replay.", "", ] - lines += [ + section += [ rows[0]["timing_basis"] + ".", "", "| Backend | Version | Task | Threads | Runs | Wall median (s) | Min-max (s) | Pages/s | " @@ -1025,16 +1123,29 @@ def write_report( if row["efficiency_pct"] is not None else "—" ) - lines.append( + section.append( f"| {BACKEND_LABELS.get(row['backend'], row['backend'])} " f"| {row['package_version'] or '—'} " - f"| {task_title(workload, row['task'])} | {row['threads']} | {row['repetitions']} " + f"| {task_label(workload, row['task'])} | {row['threads']} | {row['repetitions']} " f"| {row['median_wall_s']:.3f} | {row['min_wall_s']:.3f}-{row['max_wall_s']:.3f} " f"| {row['pages_per_s']:.1f} | " + (f"{pdfium_speedup} | " if workload != "native" else "") + f"{speedup} | {efficiency} |" ) - lines += [""] + section += [""] + for task in sorted({row["task"] for row in rows}): + section.append( + f"- **{task_label(workload, task)}**: {task_title(workload, task)}." + ) + section += [""] + lines += [ + "## Parallel Scaling", + "", + *sections.get("native", []), + *sections.get("replay", []), + *memory_lines, + *sections.get("comparison", []), + ] if inputs.excluded_comparisons and not core_only: lines += [ "## Excluded reference comparisons", @@ -1101,19 +1212,37 @@ def reproduction_commands(meta: dict[str, Any], workload: str, task: str) -> lis revision = shlex.quote(meta["dataset"]["revision"]) scale = str(float(meta["scale"])) queue = str(int(meta["max_concurrent_results"])) + memory = meta.get("memory_collection", {"mode": "off"}) + if workload == "comparison": + memory = meta.get("comparison_memory", memory) + memory_flags = ( + f"--memory-mode {shlex.quote(memory['mode'])} " + f"--memory-interval-ms {int(memory.get('interval_ms') or 50)}" + if memory["mode"] != "off" + else "" + ) lines = [ "set -euo pipefail", f"PDF_ROOT={shlex.quote(root)}", 'REPRO_OUT=$(mktemp -d "$PWD/scratch-performance-reproduce-XXXXXX")', - f"for ((r=1; r<={int(meta['repeats'])}; r++)); do", ] + if memory_flags and workload != "comparison": + lines += [ + "observe() {", + " local memory_output=$1; shift", + f" .venv/bin/python scripts/benchmarking/_performance_memory.py {memory_flags} \\", + ' --output "${memory_output}.memory.json" -- "$@"', + "}", + ] + lines.append(f"for ((r=1; r<={int(meta['repeats'])}; r++)); do") if workload == "native": flags = NATIVE_PROFILES.get(task, NATIVE_PROFILES["render-with-text"]) words = str(flags["create_word_cells"]).lower() line_cells = str(flags["create_line_cells"]).lower() + observe = 'observe "$REPRO_OUT/r${r}-t${t}" ' if memory_flags else "" lines += [ f" for t in {threads}; do", - ' ./build/run_scaling.exe "$PDF_ROOT" -v WARNING --recursive --mode render \\', + f' {observe}./build/run_scaling.exe "$PDF_ROOT" -v WARNING --recursive --mode render \\', f' --threads "$t" --max-concurrent-results {queue} --scale {scale} \\', f" --create-word-cells={words} --create-line-cells={line_cells} \\", ' --enable-timing --timing-csv "$REPRO_OUT/r${r}-t${t}.csv" \\', @@ -1122,12 +1251,17 @@ def reproduction_commands(meta: dict[str, Any], workload: str, task: str) -> lis ] elif workload == "replay": command = "uv run --no-sync python scripts/benchmarking/run_performance_benchmarking_with_docling_settings.py" + pdfium_command = threaded_command = command + if memory_flags: + command = ".venv/bin/python scripts/benchmarking/run_performance_benchmarking_with_docling_settings.py" + pdfium_command = f'observe "$REPRO_OUT/r${{r}}-pdfium" {command}' + threaded_command = f'observe "$REPRO_OUT/r${{r}}-t${{t}}" {command}' lines += [ - f' {command} "$PDF_ROOT" \\', + f' {pdfium_command} "$PDF_ROOT" \\', f" --backend pdfium --dataset-revision {revision} \\", ' --output-dir "$REPRO_OUT/r${r}-pdfium" 2>&1 | tee "$REPRO_OUT/r${r}-pdfium.log"', f" for t in {threads}; do", - f' {command} "$PDF_ROOT" \\', + f' {threaded_command} "$PDF_ROOT" \\', f' --backend threaded --threads "$t" --dataset-revision {revision} \\', ' --output-dir "$REPRO_OUT/r${r}-t${t}" 2>&1 | tee "$REPRO_OUT/r${r}-t${t}.log"', " done", @@ -1140,7 +1274,7 @@ def reproduction_commands(meta: dict[str, Any], workload: str, task: str) -> lis lines += [ ' uv run --no-sync python scripts/benchmarking/run_performance_benchmarking.py "$PDF_ROOT" \\', f" --recursive --mode {mode} --threads {thread_csv} --scale {scale} --max-concurrent-results {queue} \\", - " --3rd-party-backends all --only-threaded \\", + f" --3rd-party-backends all --only-threaded {memory_flags} \\", " --keep-char-cells=true --create-word-cells=false --create-line-cells=false \\", f" --keep-shapes={scene} --keep-bitmaps={scene} \\", " --materialize-char-cells=true --materialize-word-cells=false \\", @@ -1160,7 +1294,7 @@ def figure_sections( "Each profile has its own one-thread baseline; speedup and efficiency never mix profiles.", "", "Run the commands from the repository root with the source revision, native build and " - "Python packages recorded above. Each reproduction block below covers all plots in its " + "Python packages recorded above. Each reproduction block below collects timing for its " "profile, uses the recorded thread counts and repetitions, and creates fresh output paths. " "The native and replay sweeps use a fresh process per thread count, as in the launcher.", "", @@ -1171,10 +1305,20 @@ def figure_sections( f"DATASET_REVISION={shlex.quote(meta['dataset']['revision'])} \\", f" REPEATS={int(meta['repeats'])} THREADS={','.join(str(int(t)) for t in meta['threads'])} " f"RUN_COMPARISON={int(meta['comparison'])} \\", + f" MEMORY_MODE={meta.get('memory_collection', {}).get('mode', 'off')} " + f"MEMORY_INTERVAL_MS={meta.get('memory_collection', {}).get('interval_ms') or 50} \\", " bash scripts/benchmarking/refresh_performance_benchmarks.sh", "```", "", ] + if meta.get("recovery"): + lines += [ + "Recovery may combine original core memory settings with different comparison settings. " + "Use `recover_performance_benchmarks.py --memory-mode ... --memory-interval-ms ...` " + "to reproduce that combination against the retained source bundle; the fresh launcher " + "command above applies its memory settings to every workload.", + "", + ] names = {figure.name for figure in figures} for workload in ("native", "replay", "comparison"): tasks = sorted({m.task for m in measurements if m.workload == workload}) @@ -1235,7 +1379,10 @@ def figure_sections( "runtime": ( "Wall time and throughput", "The left panel shows corpus wall time; the right shows pages per second. " - "Points are medians, and wall-time bars show the observed minimum and maximum across repetitions.", + "Points are medians, and wall-time bars show the observed minimum and maximum across repetitions. " + "Dashed throughput lines show ideal scaling: measured one-thread throughput " + "multiplied by the thread count. They are shown for backends measured at one " + "thread and additional thread counts.", ), "scaling": ( "Speedup and parallel efficiency", @@ -1322,20 +1469,36 @@ def save(fig: Any, name: str) -> None: capsize=3, label=BACKEND_LABELS.get(backend, backend), ) - axes[1].plot( + throughput = [r["pages_per_s"] for r in group] + (measured,) = axes[1].plot( x, - [r["pages_per_s"] for r in group], + throughput, "o-", label=BACKEND_LABELS.get(backend, backend), ) + one_thread = next( + (r["pages_per_s"] for r in group if r["threads"] == 1), None + ) + if one_thread is not None and len(set(x)) >= 2: + axes[1].plot( + [0, max(x)], + [0, one_thread * max(x)], + linestyle="--", + color=measured.get_color(), + label=f"{BACKEND_LABELS.get(backend, backend)} ideal", + ) + axes[1].set_xlim(left=0) + axes[1].set_ylim(bottom=0) + thread_ticks = sorted({r["threads"] for r in rows}) for ax, ylabel in zip(axes, ("Corpus wall time (s)", "Pages per second")): ax.set( xlabel="Threads", ylabel=ylabel, - xticks=sorted({r["threads"] for r in rows}), + xticks=thread_ticks, ) ax.grid(alpha=0.25) ax.legend() + axes[1].set_xticks([0, *thread_ticks]) fig.suptitle(plot_title(workload, task), fontsize=12) save(fig, suffix + "_runtime") scalable = [r for r in rows if r["efficiency_pct"] is not None] @@ -1440,6 +1603,125 @@ def save(fig: Any, name: str) -> None: return figures +def publication_sections( + inputs: ReportInputs, measurements: list[Measurement], output: Path +) -> dict[str, str]: + """Render validated measurements for marked documentation slots. + + Only summary figures are published. CSVs and detailed RSS traces stay local. + This interface deliberately does not parse the standalone Markdown report. + """ + from tabulate import tabulate + + output.mkdir(parents=True, exist_ok=True) + summary = summarize(measurements) + figures = plot(measurements, summary, output) + sections = {} + for workload, label in WORKLOAD_LABELS.items(): + rows = [r for r in summary if r["workload"] == workload] + lines = [f"## {label}", ""] + if not rows: + sections[workload] = "\n".join([*lines, "Not collected in this run.", ""]) + continue + lines += [ + tabulate( + [ + [ + BACKEND_LABELS.get(r["backend"], r["backend"]), + r["package_version"], + r["task"], + r["threads"], + r["repetitions"], + f"{r['median_wall_s']:.3f}", + f"{r['min_wall_s']:.3f}-{r['max_wall_s']:.3f}", + f"{r['pages_per_s']:.1f}", + "—" + if r["speedup_vs_docling_1t"] is None + else f"{r['speedup_vs_docling_1t']:.2f}x", + "—" + if r["efficiency_pct"] is None + else f"{r['efficiency_pct']:.1f}%", + ] + for r in rows + ], + headers=[ + "Backend", + "Version", + "Task", + "Threads", + "Runs", + "Wall median (s)", + "Min-max (s)", + "Pages/s", + "Speedup vs Docling 1t", + "Efficiency", + ], + tablefmt="github", + disable_numparse=True, + ), + "", + ] + for task in sorted({r["task"] for r in rows}): + prefix = figure_prefix(workload, task) + lines += [ + f"### {task_title(workload, task)}", + "", + "Timing: " + + next(r["timing_basis"] for r in rows if r["task"] == task) + + ".", + "", + ] + for figure in figures: + if figure.stem in { + prefix + "_" + kind + for kind in ("runtime", "scaling", "phases", "distribution") + }: + lines += [ + f"![{figure.stem.replace('_', ' ')}](assets/{figure.name})", + "", + ] + lines += [ + "Stage totals sum overlapping worker elapsed time, not corpus wall or CPU time. " + "Distributions describe native page-worker or replay document durations, not end-to-end page latency.", + "", + "
", + "Reproduce these measurements", + "", + "```bash", + *reproduction_commands(inputs.metadata, workload, task), + "```", + "", + "
", + "", + ] + if workload == "comparison" and inputs.excluded_comparisons: + lines += [ + "Affected reference series are excluded across all repetitions:", + "", + tabulate( + [ + [r["backend"], r["task"], r["threads"], r["repetition"]] + for r in inputs.excluded_comparisons + ], + headers=["Backend", "Task", "Threads", "Repetition"], + tablefmt="github", + ), + "", + ] + sections[workload] = "\n".join(lines) + if any(m.memory is not None for m in measurements): + memory = "\n".join( + write_memory_report(measurements, output, include_traces=False) + ) + sections["memory"] = re.sub( + r"\]\((memory_[^)]+\.png)\)", r"](assets/\1)", memory + ) + else: + sections["memory"] = "## Process memory\n\nNot collected in this run.\n" + sections["interpretation"] = "\n".join(python_experiment_explanation(summary)) + return sections + + def main() -> None: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument( diff --git a/scripts/tools/update_performance_docs.py b/scripts/tools/update_performance_docs.py new file mode 100644 index 000000000..b9ab22b3b --- /dev/null +++ b/scripts/tools/update_performance_docs.py @@ -0,0 +1,438 @@ +#!/usr/bin/env python3 +"""Publish a validated performance run. Preview by default; --apply writes docs. + +Use --check without a run to enforce publication size budgets and local links. +No benchmarks, builds, downloads, commits, or pushes are performed. +""" + +from __future__ import annotations + +import argparse +import difflib +import hashlib +import json +import os +import re +import sys +import tempfile +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +ROOT = Path(__file__).resolve().parents[2] +TOOLS = Path(__file__).resolve().parent +ARTIFACTS = TOOLS / "update_performance_docs_artifacts" +TREE = Path("docs/performance_benchmarks") +INDEX = Path("docs/performance_benchmarks.md") + + +def encoded(value: Any) -> bytes: + return (json.dumps(value, indent=2, sort_keys=True) + "\n").encode() + + +def digest(content: bytes) -> str: + return hashlib.sha256(content).hexdigest() + + +def slot(text: str, name: str, content: str) -> str: + start = f"" + end = f"" + if start not in text and end not in text: + print(f"Skip absent placeholder: {name}") + return text + if ( + text.count(start) != 1 + or text.count(end) != 1 + or text.index(start) > text.index(end) + ): + raise ValueError(f"Malformed/duplicate placeholder: {name}") + before, rest = text.split(start) + _, after = rest.split(end) + return before + start + "\n" + content.strip() + "\n" + end + after + + +def slug(value: str) -> str: + result = re.sub(r"[^a-z0-9]+", "-", value.lower()).strip("-") + if not result: + raise ValueError("Empty setup identifier") + return result + + +def memory_label(meta: dict[str, Any]) -> str: + value = meta.get("memory_bytes") + return f"{int(value) / 2**30:g} GiB" if value else "unknown RAM" + + +def identity( + meta: dict[str, Any], override: str | None +) -> tuple[str, str, dict[str, Any]]: + dataset = meta["dataset"]["repository"].rsplit("-", 1)[-1].lower() + if dataset not in {"bo20", "bo767", "bo10k"}: + raise ValueError(f"Unsupported dataset: {dataset}") + machine = {k: meta.get(k) for k in ("cpu", "machine", "platform", "memory_bytes")} + if not override and not all(machine.values()): + raise ValueError("Incomplete machine metadata; provide --setup-id explicitly") + ram = int(meta.get("memory_bytes") or 0) + machine["memory_bytes"] = ram or None + setup = override or f"{meta['cpu']}-{ram / 2**30:g}gib-{meta['platform']}" + if override and slug(override) != override: + raise ValueError( + "--setup-id must contain lowercase letters, digits and hyphens" + ) + return dataset, slug(setup), machine + + +def snapshot(root: Path) -> dict[Path, bytes]: + files = {} + for p in [root / INDEX, *(root / TREE).rglob("*")]: + if p.is_symlink(): + raise ValueError(f"Symlink in publication tree: {p}") + if p.is_file(): + files[p.relative_to(root)] = p.read_bytes() + return files + + +def checked_relative(value: str) -> Path: + path = Path(value) + if path.is_absolute() or ".." in path.parts or not path.parts: + raise ValueError(f"Unsafe publication path: {value}") + return path + + +def validate(files: dict[Path, bytes], limits: dict[str, int]) -> None: + total = 0 + for path, content in files.items(): + key = { + ".md": "markdown_bytes", + ".png": "image_bytes", + ".json": "json_bytes", + }.get(path.suffix) + if key is None: + # Historical documentation contains extensionless files. + key = "image_bytes" + if len(content) > limits[key]: + raise ValueError( + f"{path}: {len(content):,} bytes exceeds {key}={limits[key]:,}" + ) + total += len(content) + if total > limits["tree_bytes"]: + raise ValueError( + f"Performance tree: {total:,} bytes exceeds tree_bytes={limits['tree_bytes']:,}" + ) + for path, content in files.items(): + if path.name != "publication.json": + continue + manifest = json.loads(content) + base = path.parent + size = sum(len(data) for p, data in files.items() if p.is_relative_to(base)) + if size > limits["publication_bytes"]: + raise ValueError( + f"{base}: {size:,} bytes exceeds publication_bytes={limits['publication_bytes']:,}" + ) + for name, sha in manifest["files"].items(): + owned = base / checked_relative(name) + if owned not in files or digest(files[owned]) != sha: + raise ValueError(f"Missing or modified generated file: {owned}") + # Check links in managed pages and the entry page, including images. + pages = {INDEX} | { + p.parent / "index.md" for p in files if p.name == "publication.json" + } + for page in pages: + if page not in files: + continue + for link in re.findall(r"\]\(([^)]+)\)", files[page].decode()): + if ":" in link or link.startswith("#"): + continue + target = link.split("#", 1)[0] + resolved = Path(os.path.normpath(page.parent / target)) + if resolved not in files and not (ROOT / resolved).is_file(): + raise ValueError(f"Broken documentation link in {page}: {link}") + + +def availability(files: dict[Path, bytes]) -> str: + lines = ["| Dataset | Setup | Run | Workloads |", "|---|---|---|---|"] + for p, data in sorted(files.items()): + if p.name != "publication.json": + continue + m = json.loads(data) + link = (p.parent / "index.md").relative_to("docs") + lines.append( + f"| {m['dataset']} | [{m['label']}]({link}) | {m['run']} | {', '.join(m['workloads'])} |" + ) + return "\n".join(lines) + + +def measured_at(bundle: Path) -> str: + # Launcher names encode the collection start. Fallback supports custom OUT names. + match = re.search(r"(\d{8}-\d{6})", bundle.name) + if match: + return datetime.strptime(match[1], "%Y%m%d-%H%M%S").isoformat() + return ( + datetime.fromtimestamp((bundle / "metadata.json").stat().st_mtime, timezone.utc) + .replace(tzinfo=None) + .isoformat() + ) + + +def provenance( + inputs: Any, bundle: Path, memories: list[dict[str, Any]] +) -> dict[str, Any]: + meta = inputs.metadata + ordered = [{k: f[k] for k in ("name", "pages", "sha256")} for f in inputs.files] + keys = ( + "source_revision", + "source_diff_sha256", + "native_sha256", + "extension_sha256", + "packages", + "qpdf", + "python", + "script_sha256", + "launcher_sha256", + "threads", + "repeats", + "scale", + "max_concurrent_results", + "native_profiles", + "build_mode", + "memory_collection", + "comparison_memory", + "comparison_provenance", + "recovery", + "recovered_replay_provenance", + ) + result = {k: meta[k] for k in keys if k in meta} + result.update( + { + "schema_version": 1, + "run": bundle.name, + "measured_at": measured_at(bundle), + "dataset": { + k: v for k, v in meta["dataset"].items() if k not in ("files", "root") + }, + "documents": len(inputs.files), + "pages": len(inputs.expected), + "ordered_file_identity_sha256": digest( + json.dumps(ordered, sort_keys=True, separators=(",", ":")).encode() + ), + "identity_serialization": "UTF-8 JSON list in dataset order; name/pages/sha256; sort_keys=True; separators=(comma,colon)", + "machine": { + k: meta.get(k) for k in ("cpu", "machine", "platform", "memory_bytes") + }, + "excluded_comparisons": inputs.excluded_comparisons, + "canvas_checks": inputs.canvas_checks, + "memory_methods": sorted( + { + (m.get("mode", ""), m.get("method", ""), m.get("scope", "")) + for m in memories + } + ), + } + ) + return result + + +def plan( + bundle: Path, files: dict[Path, bytes], setup_id: str | None, allow_older: bool +) -> dict[Path, bytes]: + # Lazy import keeps --check independent of performance packages/native extension. + sys.path.insert(0, str(ROOT / "scripts/benchmarking")) + from run_performance_report import ReportInputs, publication_sections + + inputs = ReportInputs(bundle) + measurements = inputs.load() + memories = [m.memory for m in measurements if m.memory] + meta = inputs.metadata + dataset, setup, machine = identity(meta, setup_id) + base = TREE / dataset / setup + manifest_path = base / "publication.json" + old = json.loads(files[manifest_path]) if manifest_path in files else None + if old: + if old["machine"] != machine: + raise ValueError( + "Setup identity differs from existing publication; choose a different --setup-id" + ) + if measured_at(bundle) < old["measured_at"] and not allow_older: + raise ValueError( + "Incoming run is older; use --allow-older to replace explicitly" + ) + with tempfile.TemporaryDirectory(prefix="performance-publication-") as tmp: + output = Path(tmp) + sections = publication_sections(inputs, measurements, output) + dataset_label = {"bo20": "Bo20", "bo767": "Bo767", "bo10k": "Bo10K"}[dataset] + sections["title"] = ( + f"# Performance results: {dataset_label} / {meta.get('cpu', setup)} / " + f"{memory_label(meta)} / {meta['platform'].split('-')[0]}" + ) + sections["setup"] = "\n".join( + [ + "## Setup", + "", + f"Run: `{bundle.name}`; collected {measured_at(bundle)}.", + "", + f"CPU: **{meta.get('cpu', 'unknown')}**; architecture: {meta['machine']}; " + f"RAM: {memory_label(meta)}; OS: {meta['platform']}.", + "", + f"Dataset: `{meta['dataset']['repository']}` at `{meta['dataset']['revision']}`; " + f"{len(inputs.files):,} PDFs / {len(inputs.expected):,} pages.", + "", + f"Repetitions: {meta['repeats']}; threads: {meta['threads']}; render scale: {meta['scale']}; " + f"queue capacity: {meta['max_concurrent_results']}.", + "", + "Values are medians with observed min-max ranges. One repetition provides no variability estimate. " + "Speedup and efficiency use the same workload/task one-thread baseline.", + "", + ] + ) + sections["provenance"] = ( + "## Provenance\n\n[Dataset, source, build, packages and memory methods](provenance.json). " + "Raw measurements and detailed RSS timelines remain in the local run bundle.\n" + ) + template = (ARTIFACTS / "performance_page_template.md").read_bytes() + page = ( + template + if old and old.get("migrated_snapshot") + else files.get(base / "index.md", template) + ).decode() + # Upgrade pages created before the generated title placeholder existed. + if page.startswith("# Performance results\n"): + page = page.replace( + "# Performance results\n", + "\n# Performance results\n" + "\n", + 1, + ) + for name, content in sections.items(): + page = slot(page, name, content) + generated = { + "index.md": page.encode(), + "provenance.json": encoded(provenance(inputs, bundle, memories)), + } + for name in re.findall(r"\]\(assets/([^/)]+\.png)\)", page): + generated["assets/" + name] = (output / name).read_bytes() + if old: + for name, sha in old["files"].items(): + path = base / checked_relative(name) + if path in files and digest(files[path]) != sha and path.name != "index.md": + raise ValueError(f"Generated asset was edited: {path}") + files.pop(path, None) + for name, data in generated.items(): + dest = base / name + if dest in files and dest not in (base / "index.md",) and files[dest] != data: + raise ValueError(f"Refusing to overwrite unowned file: {dest}") + files[dest] = data + manifest = { + "schema_version": 1, + "dataset": dataset, + "setup": setup, + "machine": machine, + "label": f"{meta.get('cpu', setup)} / {memory_label(meta)} / {meta['platform']}", + "run": bundle.name, + "measured_at": measured_at(bundle), + "workloads": sorted({m.workload for m in measurements}), + "files": {name: digest(data) for name, data in generated.items()}, + } + files[manifest_path] = encoded(manifest) + files[INDEX] = slot( + files[INDEX].decode(), "availability", availability(files) + ).encode() + return files + + +def preview(before: dict[Path, bytes], after: dict[Path, bytes]) -> list[Path]: + changed = [] + for path in sorted(before.keys() | after.keys()): + old, new = before.get(path, b""), after.get(path, b"") + if old == new: + continue + changed.append(path) + print(f"{path}: {len(old):,} -> {len(new):,} bytes") + if path.suffix == ".md": + print( + "".join( + difflib.unified_diff( + old.decode().splitlines(True), + new.decode().splitlines(True), + fromfile=str(path), + tofile=str(path), + ) + ), + end="", + ) + print( + f"{len(changed)} changed files; total {sum(map(len, after.values())):,} bytes; " + f"net change {sum(map(len, after.values())) - sum(map(len, before.values())):+,} bytes" + ) + for path in sorted(after): + if path.name == "publication.json": + size = sum( + len(data) for p, data in after.items() if p.is_relative_to(path.parent) + ) + print(f"Publication {path.parent}: {size:,} bytes") + return changed + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + mode = parser.add_mutually_exclusive_group(required=True) + mode.add_argument( + "--performance-run-dir", + type=Path, + metavar="DIRECTORY", + help="Completed performance launcher/recovery output directory", + ) + parser.add_argument( + "--apply", action="store_true", help="Apply the validated publication plan" + ) + mode.add_argument( + "--check", + action="store_true", + help="Check existing sizes, manifests and links only", + ) + parser.add_argument( + "--setup-id", + help="Explicit stable identifier for incomplete/ambiguous machine metadata", + ) + parser.add_argument("--allow-older", action="store_true") + args = parser.parse_args() + if args.check and args.apply: + parser.error("--check cannot be combined with --apply") + try: + limits = json.loads((ARTIFACTS / "performance_docs_limits.json").read_text()) + before = snapshot(ROOT) + if args.check: + validate(before, limits) + print( + f"Performance docs OK: {len(before)} files, {sum(map(len, before.values())):,} bytes" + ) + return + after = plan( + args.performance_run_dir.resolve(), + dict(before), + args.setup_id, + args.allow_older, + ) + validate(after, limits) + changed = preview(before, after) + if args.apply: + # Catch edits made during validation/rendering, before any docs are written. + if snapshot(ROOT) != before: + raise ValueError( + "Documentation changed during publication; rerun the preview" + ) + for path in changed: + dest = ROOT / path + if path in after: + dest.parent.mkdir(parents=True, exist_ok=True) + dest.write_bytes(after[path]) + else: + dest.unlink() + print("Applied publication.") + else: + print("Preview only. Use --apply to write this publication.") + except (ValueError, KeyError, OSError) as error: + parser.exit(1, f"Cannot publish performance docs: {error}\n") + + +if __name__ == "__main__": + main() diff --git a/scripts/tools/update_performance_docs_artifacts/performance_docs_limits.json b/scripts/tools/update_performance_docs_artifacts/performance_docs_limits.json new file mode 100644 index 000000000..d2b8872a2 --- /dev/null +++ b/scripts/tools/update_performance_docs_artifacts/performance_docs_limits.json @@ -0,0 +1,7 @@ +{ + "markdown_bytes": 102400, + "image_bytes": 512000, + "json_bytes": 102400, + "publication_bytes": 5242880, + "tree_bytes": 26214400 +} diff --git a/scripts/tools/update_performance_docs_artifacts/performance_page_template.md b/scripts/tools/update_performance_docs_artifacts/performance_page_template.md new file mode 100644 index 000000000..4fbd62efa --- /dev/null +++ b/scripts/tools/update_performance_docs_artifacts/performance_page_template.md @@ -0,0 +1,26 @@ + +# Performance results + + +[Methodology and all setups](../../../performance_benchmarks.md). + + + + + + + + + + + + + + + + + + + + + diff --git a/src/parse.h b/src/parse.h index 0f115ade9..2d00f7635 100644 --- a/src/parse.h +++ b/src/parse.h @@ -94,8 +94,6 @@ #include #include #include -#include -#include #include #include diff --git a/src/parse/page_items/render_instructions.h b/src/parse/page_items/render_instructions.h index e5e8278c2..bb82af336 100644 --- a/src/parse/page_items/render_instructions.h +++ b/src/parse/page_items/render_instructions.h @@ -4,6 +4,7 @@ #define PAGE_ITEM_RENDER_INSTRUCTION_H #include +#include #include #include #include @@ -11,6 +12,8 @@ #include #include #include +#include +#include #include #include @@ -110,6 +113,255 @@ namespace pdflib clip_group(clip_group_) {} + namespace detail + { + // One immutable capture, shared by every later snapshot of its history. + // Only destruction can detach the predecessor of a uniquely owned node. + struct clip_history_node + { + std::shared_ptr previous; + const std::vector paths; + const std::size_t depth; + const std::size_t path_count; + std::size_t point_count; + std::size_t logical_bytes; + + clip_history_node(std::shared_ptr previous_, + std::vector paths_): + previous(std::move(previous_)), paths(std::move(paths_)), + depth(previous ? previous->depth + 1 : 1), + path_count(paths.size() + (previous ? previous->path_count : 0)), + point_count(previous ? previous->point_count : 0), + logical_bytes((previous ? previous->logical_bytes : 0) + + paths.size() * sizeof(clip_path_instruction)) + { + for(const auto& path : paths) + { + point_count += path.size(); + logical_bytes += (path.get_x().capacity() + path.get_y().capacity()) * sizeof(double); + } + } + + ~clip_history_node() noexcept + { + // A long unique suffix must not recursively destroy thousands of + // shared_ptr predecessors on the native stack. + auto tail = std::move(previous); + while(tail and tail.use_count() == 1) + { + auto next = std::move(tail->previous); + tail.reset(); + tail = std::move(next); + } + } + }; + } + + class clip_path_range + { + friend class clip_state_instruction; + friend class clip_path_cursor; + using node = detail::clip_history_node; + std::shared_ptr head; + + public: + class const_iterator + { + public: + using iterator_category = std::forward_iterator_tag; + using value_type = clip_path_instruction; + using difference_type = std::ptrdiff_t; + using pointer = const value_type*; + using reference = const value_type&; + + const_iterator() = default; + explicit const_iterator(const node* head): origin(head), remaining(head ? head->depth : 0) + { + if(remaining > inline_nodes.size()) + { + overflow.reserve(remaining); + for(auto p = head; p; p = p->previous.get()) { overflow.push_back(p); } + } + else + { + std::size_t i = 0; + for(auto p = head; p; p = p->previous.get()) { inline_nodes[i++] = p; } + } + } + reference operator*() const { return current()->paths[offset]; } + pointer operator->() const { return &**this; } + const_iterator& operator++() + { + if(++offset == current()->paths.size()) { offset = 0; --remaining; } + return *this; + } + const_iterator operator++(int) { auto copy = *this; ++*this; return copy; } + bool operator==(const const_iterator& other) const + { + if(remaining == 0 or other.remaining == 0) + { return remaining == other.remaining; } + return origin == other.origin and remaining == other.remaining and offset == other.offset; + } + bool operator!=(const const_iterator& other) const { return not (*this == other); } + + private: + const node* current() const + { return overflow.empty() ? inline_nodes[remaining - 1] : overflow[remaining - 1]; } + const node* origin = nullptr; + std::size_t remaining = 0, offset = 0; + std::array inline_nodes{}; + std::vector overflow; + }; + + std::size_t size() const { return head ? head->path_count : 0; } + bool empty() const { return not head; } + std::size_t point_count() const { return head ? head->point_count : 0; } + std::size_t logical_bytes() const { return head ? head->logical_bytes : 0; } + const_iterator begin() const { return const_iterator(head.get()); } + const_iterator end() const { return const_iterator(); } + + std::vector to_vector() const + { + std::vector result; + result.reserve(size()); + for(const auto& path : *this) { result.push_back(path); } + return result; + } + + // The visitor returns false at an already-accounted prefix. This lets a + // probe count unique retained allocations across all instruction snapshots + // without materializing or repeatedly walking every logical occurrence. + template + void visit_storage(Visitor&& visitor) const + { + for(auto p = head.get(); p; p = p->previous.get()) + { + const std::size_t prior_points = p->previous ? p->previous->point_count : 0; + const std::size_t prior_bytes = p->previous ? p->previous->logical_bytes : 0; + const std::size_t storage_bytes = sizeof(node) + p->logical_bytes - prior_bytes + + (p->paths.capacity() - p->paths.size()) * sizeof(clip_path_instruction); + if(not visitor(static_cast(p), p->paths.size(), + p->point_count - prior_points, storage_bytes)) { break; } + } + } + }; + + // One active renderer traversal, not a cache in every snapshot. Geometry + // stays in the immutable history; this keeps ordered references and group + // counts ready for repeated helper passes over the same clip. + class clip_path_cursor + { + using node = detail::clip_history_node; + using storage = std::vector; + std::shared_ptr head; + storage ordered; + std::unordered_map groups; + bool valid = false; + + void append_capture(const node& capture) + { + for(const auto& path : capture.paths) + { + ordered.push_back(&path); + ++groups[path.get_clip_group()]; + } + } + + public: + class const_iterator + { + storage::const_iterator position; + public: + using iterator_category = std::forward_iterator_tag; + using value_type = clip_path_instruction; + using difference_type = std::ptrdiff_t; + using pointer = const value_type*; + using reference = const value_type&; + const_iterator() = default; + explicit const_iterator(storage::const_iterator position): position(position) {} + reference operator*() const { return **position; } + pointer operator->() const { return *position; } + const_iterator& operator++() { ++position; return *this; } + const_iterator operator++(int) { auto copy = *this; ++*this; return copy; } + bool operator==(const const_iterator& other) const { return position == other.position; } + bool operator!=(const const_iterator& other) const { return not (*this == other); } + }; + + void prepare(const clip_path_range& paths) + { + if(valid and head == paths.head) { return; } + const bool can_extend = valid; + valid = false; + try + { + if(can_extend and paths.head and paths.head->previous == head) + { + // The common clip-chain case adds exactly one capture. + append_capture(*paths.head); + } + else + { + std::vector suffix; + auto next = paths.head.get(); + while(next and (not can_extend or next != head.get())) + { + suffix.push_back(next); + next = next->previous.get(); + } + if(not can_extend or next != head.get()) + { + // A restored prefix, sibling, translation or independent + // history must not inherit counts/references from a branch. + ordered.clear(); + groups.clear(); + } + for(auto it = suffix.rbegin(); it != suffix.rend(); ++it) { append_capture(**it); } + } + head = paths.head; + valid = true; + } + catch(...) + { + // A partially appended pointer list must never be reused after a + // failed allocation; reset does not dereference any stored pointer. + reset(); + throw; + } + } + + void reset() noexcept + { + valid = false; + storage().swap(ordered); + std::unordered_map().swap(groups); + head.reset(); + } + std::size_t size() const { return ordered.size(); } + bool empty() const { return ordered.empty(); } + std::size_t group_size(int group) const + { + const auto found = groups.find(group); + return found == groups.end() ? 0 : found->second; + } + const_iterator begin() const { return const_iterator(ordered.begin()); } + const_iterator end() const { return const_iterator(ordered.end()); } + }; + + class clip_translation_cache + { + friend class clip_state_instruction; + using node = detail::clip_history_node; + struct entry + { + // Retain the source too: its address cannot be reused as a cache key. + std::shared_ptr source; + std::shared_ptr translated; + }; + std::unordered_map entries; + bool initialized = false; + double dx = 0.0, dy = 0.0; + }; + class clip_state_instruction { public: @@ -118,27 +370,72 @@ namespace pdflib std::vector paths); clip_rule get_rule() const { return rule; } - const std::vector& get_paths() const { return paths; } + const clip_path_range& get_paths() const { return paths; } + bool shares_history(const clip_state_instruction& other) const + { return paths.head == other.paths.head; } + + clip_state_instruction with_rule(clip_rule rule_) const + { + auto result = *this; + result.rule = rule_; + return result; + } + + clip_state_instruction appended(clip_rule rule_, std::vector capture) const + { + auto result = with_rule(rule_); + if(not capture.empty()) + { + result.paths.head = std::make_shared(paths.head, std::move(capture)); + } + return result; + } bool has_clip() const { return rule != CLIP_RULE_NONE and not paths.empty(); } - clip_state_instruction translated(double dx, double dy) const + clip_state_instruction translated(double dx, double dy, + clip_translation_cache* shared_cache = nullptr) const { - std::vector paths_; - paths_.reserve(paths.size()); - for(const auto& path : paths) + if(paths.empty() or (dx == 0.0 and dy == 0.0)) { return *this; } + clip_translation_cache local_cache; + auto& cache = shared_cache ? *shared_cache : local_cache; + if(not cache.initialized or cache.dx != dx or cache.dy != dy) + { + cache.entries.clear(); + cache.initialized = true; + cache.dx = dx; + cache.dy = dy; + } + std::vector> pending; + auto source = paths.head; + std::shared_ptr translated; + while(source) { - paths_.push_back(path.translated(dx, dy)); + auto found = cache.entries.find(source.get()); + if(found != cache.entries.end()) { translated = found->second.translated; break; } + pending.push_back(source); + source = source->previous; } - return clip_state_instruction(rule, std::move(paths_)); + for(auto it = pending.rbegin(); it != pending.rend(); ++it) + { + std::vector capture; + capture.reserve((*it)->paths.size()); + for(const auto& path : (*it)->paths) { capture.push_back(path.translated(dx, dy)); } + translated = std::make_shared(translated, std::move(capture)); + cache.entries.emplace(it->get(), clip_translation_cache::entry{*it, translated}); + } + clip_state_instruction result; + result.rule = rule; + result.paths.head = std::move(translated); + return result; } private: clip_rule rule; - std::vector paths; + clip_path_range paths; }; inline clip_state_instruction::clip_state_instruction(): @@ -149,9 +446,11 @@ namespace pdflib inline clip_state_instruction::clip_state_instruction( clip_rule rule_, std::vector paths_): - rule(rule_), - paths(std::move(paths_)) - {} + rule(rule_) + { + if(not paths_.empty()) + { paths.head = std::make_shared(nullptr, std::move(paths_)); } + } enum RENDER_INSTRUCTION_NAME { SIZE_INSTRUCTION, // set the size of the canvas on which we render @@ -358,7 +657,7 @@ namespace pdflib // Copy shifted by (dx, dy); used to lift widget appearance-stream text // from AP-local into page coordinates. The glyph bbox (g_*) lives in // font units and is copied unshifted. - text_instruction translated(double dx, double dy) const + text_instruction translated(double dx, double dy, clip_translation_cache* cache = nullptr) const { text_instruction copy(text, font_enc, font_key, font_name, encoding_name, base_font, font_size, @@ -376,7 +675,7 @@ namespace pdflib copy.glyph_name_ = glyph_name_; copy.rgb_filling_ = rgb_filling_; copy.fill_alpha_ = fill_alpha_; - copy.clip_state_ = clip_state_.translated(dx, dy); + copy.clip_state_ = clip_state_.translated(dx, dy, cache); copy.blend_mode_ = blend_mode_; copy.rendering_mode_ = rendering_mode_; @@ -592,7 +891,7 @@ namespace pdflib const clip_state_instruction& get_clip_state() const { return clip_state; } bool has_clip_state() const { return clip_state.has_clip(); } - bitmap_instruction translated(double dx, double dy) const + bitmap_instruction translated(double dx, double dy, clip_translation_cache* cache = nullptr) const { bitmap_instruction copy(xobject_key, data, alpha_data, cmyk_conv, shape, fmt, image_mask, rgb_filling, fill_alpha, @@ -600,7 +899,7 @@ namespace pdflib r_x1 + dx, r_y1 + dy, r_x2 + dx, r_y2 + dy, r_x3 + dx, r_y3 + dy, - source, clip_state.translated(dx, dy)); + source, clip_state.translated(dx, dy, cache)); copy.blend_mode_ = blend_mode_; copy.filters_ = filters_; return copy; @@ -913,7 +1212,7 @@ namespace pdflib bool has_tiling_paint() const { return static_cast(tiling_paint_); } const tiling_paint& get_tiling_paint() const { return *tiling_paint_; } - shape_instruction translated(double dx, double dy) const + shape_instruction translated(double dx, double dy, clip_translation_cache* cache = nullptr) const { std::vector subpaths_; subpaths_.reserve(subpaths.size()); @@ -926,7 +1225,7 @@ namespace pdflib dash_array, dash_phase, rgb_stroking, rgb_filling, stroke_alpha, fill_alpha, - clip_state.translated(dx, dy)); + clip_state.translated(dx, dy, cache)); copy.blend_mode_ = blend_mode_; // the mask is anchored in page space, so a moved copy keeps it as is copy.soft_mask_ = soft_mask_; @@ -1033,7 +1332,7 @@ namespace pdflib const clip_state_instruction& get_clip_state() const { return clip_state; } bool has_clip_state() const { return clip_state.has_clip(); } - shading_instruction translated(double dx, double dy) const + shading_instruction translated(double dx, double dy, clip_translation_cache* cache = nullptr) const { // The shading coordinates live in shading space, so a page-space shift // belongs in the translation part of the matrix. @@ -1043,7 +1342,7 @@ namespace pdflib shading_instruction copy(shading_key, geometry, coords, matrix_, stops, extend_start, extend_end, fill_alpha, - clip_state.translated(dx, dy)); + clip_state.translated(dx, dy, cache)); copy.blend_mode_ = blend_mode_; // the mask is anchored in page space, so a moved copy keeps it as is copy.soft_mask_ = soft_mask_; @@ -1218,6 +1517,8 @@ namespace pdflib double dx, double dy) { const bool shifted = (dx != 0.0 or dy != 0.0); + clip_translation_cache clip_translations; + clip_translation_cache* translation_cache = &clip_translations; for(const auto& instr : instructions) { @@ -1227,28 +1528,28 @@ namespace pdflib { case TEXT_RENDER_INSTRUCTION: { - text_instruction_type copy = text_instructions.at(instr.index).translated(dx, dy); + text_instruction_type copy = text_instructions.at(instr.index).translated(dx, dy, translation_cache); renderer.render_text(copy); } break; case BITMAP_RENDER_INSTRUCTION: { - bitmap_instruction_type copy = bitmap_instructions.at(instr.index).translated(dx, dy); + bitmap_instruction_type copy = bitmap_instructions.at(instr.index).translated(dx, dy, translation_cache); renderer.render_bitmap(copy); } break; case SHAPE_RENDER_INSTRUCTION: { - shape_instruction_type copy = shape_instructions.at(instr.index).translated(dx, dy); + shape_instruction_type copy = shape_instructions.at(instr.index).translated(dx, dy, translation_cache); renderer.render_shape(copy); } break; case SHADING_RENDER_INSTRUCTION: { - shading_instruction_type copy = shading_instructions.at(instr.index).translated(dx, dy); + shading_instruction_type copy = shading_instructions.at(instr.index).translated(dx, dy, translation_cache); renderer.render_shading(copy); } break; diff --git a/src/parse/pdf_decoders/page.h b/src/parse/pdf_decoders/page.h index 4f8cce6a9..d06cc2242 100644 --- a/src/parse/pdf_decoders/page.h +++ b/src/parse/pdf_decoders/page.h @@ -18,6 +18,19 @@ namespace pdflib { + // Native probe diagnostics for top-level annotation appearance sources. + // Output containers and Forms invoked by an appearance have separate storage. + struct appearance_stream_statistics + { + std::size_t executions = 0; + std::size_t widget_executions = 0; + std::size_t source_objects = 0; + std::size_t source_bytes_observed = 0; + std::size_t materialized_tokens = 0; + std::size_t max_buffered_token_capacity = 0; + std::size_t max_pending_operands = 0; + }; + template<> class pdf_decoder { @@ -100,6 +113,11 @@ namespace pdflib // Get render instructions collected during decode pdf_render_instructions& get_instructions() { return instructions; } + const content_stream_statistics& get_content_statistics() const { return content_stats; } + const appearance_stream_statistics& get_appearance_statistics() const { return appearance_stats; } + const soft_mask_stream_statistics& get_soft_mask_statistics() const { return soft_mask_stats; } + const pattern_stream_statistics& get_pattern_statistics() const { return pattern_stats; } + // Export this page as a standalone one-page PDF. void save_pdf_page(std::filesystem::path const& out_path) const; @@ -202,6 +220,10 @@ namespace pdflib nlohmann::json json_annots; page_item page_dimension; + content_stream_statistics content_stats; + appearance_stream_statistics appearance_stats; + soft_mask_stream_statistics soft_mask_stats; + pattern_stream_statistics pattern_stats; page_item page_cells; page_item char_cells; @@ -1210,36 +1232,28 @@ namespace pdflib instructions, timings); - // Split decode_contents into: page content-stream tokenization - // (content_decode_total) vs. operator-execution self-time - // (interprete_ops_total). The latter is the interpretation wall time minus - // the sub-work attributed to other buckets (resource set(), parse_stream, - // do_image, do_form machinery) while interpreting -- see note_attributed(). - double interprete_seconds = 0.0; - double attributed_before = timings.attributed_total(); - + const double attributed_before = timings.attributed_total(); std::vector parameters; - { - utils::timer content_decode_timer; - stream_decoder.decode(contents); - timings.add_timing(pdf_timings::KEY_CONTENT_DECODE_TOTAL, content_decode_timer.get_time()); - } - //stream_decoder.print(); + // QPDF parsing and interpretation are interleaved. Report their combined + // self-time, excluding already-attributed nested work. Timing every token + // to reconstruct the old split would add overhead to the production path. { - utils::timer interprete_timer; - stream_decoder.interprete(parameters); - interprete_seconds += interprete_timer.get_time(); + utils::timer content_timer; + stream_decoder.decode_and_interprete(contents, parameters); + const double elapsed = content_timer.get_time(); + const double attributed_during = timings.attributed_total() - attributed_before; + timings.add_timing(pdf_timings::KEY_CONTENT_PARSE_INTERPRETE_TOTAL, + std::max(0.0, elapsed - attributed_during)); } + content_stats = stream_decoder.get_content_statistics(); + soft_mask_stats = stream_decoder.get_soft_mask_statistics(); + pattern_stats = stream_decoder.get_pattern_statistics(); if(parameters.size()>0) { LOG_S(WARNING) << "page content is ending with non-zero number of parameters"; } - - double attributed_during = timings.attributed_total() - attributed_before; - timings.add_timing(pdf_timings::KEY_INTERPRETE_OPS_TOTAL, - interprete_seconds - attributed_during); } void pdf_decoder::load_acroform_dr_fonts() @@ -1290,6 +1304,8 @@ namespace pdflib { LOG_S(INFO) << __FUNCTION__; + appearance_stats = {}; + load_acroform_dr_fonts(); if(not qpdf_page.isDictionary()) @@ -1866,9 +1882,22 @@ namespace pdflib const std::array ap_matrix = appearance_matrix(ap_stream, bbox); std::vector parameters; - stream_decoder.decode(ap_stream); + // Interpretation starts in the QPDF callback, so seed the transform first. stream_decoder.set_base_matrix(ap_matrix); - stream_decoder.interprete(parameters); + stream_decoder.decode_and_interprete(ap_stream, parameters); + +#ifdef DOCLING_PARSE_CONTENT_DIAGNOSTICS + const auto& source_stats = stream_decoder.get_content_statistics(); + ++appearance_stats.executions; + appearance_stats.widget_executions += is_widget; + appearance_stats.source_objects += source_stats.source_objects; + appearance_stats.source_bytes_observed += source_stats.source_bytes_observed; + appearance_stats.materialized_tokens += source_stats.buffered_tokens; + appearance_stats.max_buffered_token_capacity = std::max( + appearance_stats.max_buffered_token_capacity, source_stats.buffered_token_capacity); + appearance_stats.max_pending_operands = std::max( + appearance_stats.max_pending_operands, source_stats.max_pending_operands); +#endif // Shapes drawn by the appearance stream (field borders, backgrounds, // check marks drawn as paths, ...): keep them in the parsed output and diff --git a/src/parse/pdf_decoders/stream.h b/src/parse/pdf_decoders/stream.h index 9ef2000f2..8946f1829 100644 --- a/src/parse/pdf_decoders/stream.h +++ b/src/parse/pdf_decoders/stream.h @@ -6,6 +6,68 @@ namespace pdflib { + // Diagnostics for this decoder's own source stream, not its nested Forms. + // Operand high-water tracking is enabled only in the optional native probes. + struct content_stream_statistics + { + bool callback_delivery = false; + std::size_t source_objects = 0; + std::size_t source_bytes_observed = 0; + std::size_t buffered_tokens = 0; + std::size_t buffered_token_capacity = 0; + std::size_t max_pending_operands = 0; + // Probe-only aggregation over Do calls in this stream and its Form children. + // Materialized tokens count both legacy vectors across all invocations; + // this is cumulative work, not a simultaneous-live-memory measurement. + std::size_t form_executions = 0; + std::size_t form_source_objects = 0; + std::size_t form_materialized_tokens = 0; + std::size_t form_max_pending_operands = 0; + std::size_t form_max_depth = 0; + // Procedural Type3 fallbacks only; cached glyph masks do not enter here. + std::size_t type3_executions = 0; + std::size_t type3_source_objects = 0; + std::size_t type3_materialized_tokens = 0; + std::size_t type3_max_pending_operands = 0; + std::size_t type3_max_depth = 0; + std::size_t type3_recursion_skips = 0; +#ifdef DOCLING_PARSE_CONTENT_DIAGNOSTICS + static constexpr bool operand_diagnostics_enabled = true; +#else + static constexpr bool operand_diagnostics_enabled = false; +#endif + }; + + // Probe-only counters follow the shared mask cache, so a reused group is + // counted once even when multiple ordinary Form decoders refer to it. + struct soft_mask_stream_statistics + { + std::size_t executions = 0; + std::size_t cache_hits = 0; + std::size_t source_objects = 0; + std::size_t source_bytes_observed = 0; + std::size_t materialized_tokens = 0; + std::size_t max_buffered_token_capacity = 0; + std::size_t max_pending_operands = 0; + std::size_t shape_instructions = 0; + }; + + struct pattern_stream_statistics + { + std::size_t source_parses = 0; + std::size_t source_cache_hits = 0; + std::size_t cached_tokens = 0; + std::size_t cached_token_capacity = 0; + std::size_t decoded_cells = 0; + std::size_t decoded_cache_hits = 0; + std::size_t decoded_shape_instructions = 0; + std::size_t lattice_executions = 0; + std::size_t replay_source_objects = 0; + std::size_t replay_materialized_tokens = 0; + std::size_t max_replay_token_capacity = 0; + std::size_t max_pending_operands = 0; + }; + template<> class pdf_decoder { @@ -32,32 +94,100 @@ namespace pdflib ~pdf_decoder(); - void print(); - std::unordered_set get_unknown_operators(); - // decode the qpdf-stream - void decode(QPDFObjectHandle& content); + // Interpret ordinary page content as normalized QPDF objects arrive. + void decode_and_interprete(QPDFObjectHandle& content, + std::vector& parameters); - // methods used to interprete the stream - void interprete(std::vector& parameters); + const content_stream_statistics& get_content_statistics() const { return content_stats; } + const soft_mask_stream_statistics& get_soft_mask_statistics() const { return soft_mask_cache_->statistics; } + const pattern_stream_statistics& get_pattern_statistics() const + { +#ifdef DOCLING_PARSE_CONTENT_DIAGNOSTICS + return *pattern_stats; +#else + static const pattern_stream_statistics empty; + return empty; +#endif + } // Establishes the matrix this stream is interpreted under, so that a // stream living in its own space -- an annotation appearance -- emits // instructions already in page coordinates. Must be called before - // interprete(). + // content delivery. void set_base_matrix(const std::array& matrix); private: + // Internal save/restore scopes cannot depend on normal return: a nested + // interpreter may fail and operator-level recovery then resumes the page. + class graphics_state_scope + { + public: + explicit graphics_state_scope(pdf_decoder& owner): + owner(owner), original_size(owner.stack.size()) { owner.q(); } + ~graphics_state_scope() noexcept { restore(); } + graphics_state_scope(const graphics_state_scope&) = delete; + graphics_state_scope& operator=(const graphics_state_scope&) = delete; + void restore() noexcept + { + if(not active) { return; } + while(owner.stack.size() > original_size) { owner.stack.pop_back(); } + active = false; + } + private: + pdf_decoder& owner; + std::size_t original_size; + bool active = true; + }; + + class nesting_scope + { + public: + explicit nesting_scope(int& depth, pdf_resource* pattern = nullptr): + depth(depth), original_depth(depth), pattern(pattern), + original_decoding(pattern ? pattern->is_decoding() : false) + { + ++depth; + if(pattern) { pattern->set_decoding(true); } + } + ~nesting_scope() noexcept + { + depth = original_depth; + if(pattern) { pattern->set_decoding(original_decoding); } + } + nesting_scope(const nesting_scope&) = delete; + nesting_scope& operator=(const nesting_scope&) = delete; + private: + int& depth; + int original_depth; + pdf_resource* pattern; + bool original_decoding; + }; + +#ifdef DOCLING_PARSE_STATE_LIFETIME_TESTING + friend struct stream_state_lifetime_test_access; + // Only the dedicated native regression target has this fault checkpoint. + inline static thread_local void (*interpretation_test_hook)(pdf_decoder&) = nullptr; +#endif + bool update_stack(std::vector >& stack_, int stack_count_); - void interprete(std::vector& stream_, - std::vector& parameters_); - + // The resource owns the immutable cache for this synchronous replay. + void interprete_cached(const std::vector& cached, + std::vector& parameters); +#ifdef DOCLING_PARSE_CONTENT_DIAGNOSTICS + void note_pattern_replay(const pdf_decoder& child, bool lattice); +#endif - void interprete_stream(std::vector& parameters); + // Stream lifecycle is separate from delivery. Callers establish/rebase the + // graphics stack before beginning; pending operands stay with the caller. + void begin_interpretation(); + inline void consume_instruction(qpdf_stream_instruction& inst, + std::vector& parameters); + void finish_interpretation(); pdf_state& current_global_state(); // get current global state pdf_state& current_text_state(); // get current text state @@ -228,7 +358,12 @@ namespace pdflib std::unordered_set unknown_operators; - std::vector stream; + content_stream_statistics content_stats; +#ifdef DOCLING_PARSE_CONTENT_DIAGNOSTICS + // Shared only by native probes; no allocation/counter work in production. + std::shared_ptr pattern_stats = + std::make_shared(); +#endif std::vector > stack; int stack_count; @@ -256,6 +391,7 @@ namespace pdflib // by the two masks, so every paint under the same pair shares one. struct soft_mask_page_cache { + soft_mask_stream_statistics statistics; std::unordered_map decoded; std::map, std::shared_ptr> intersected; @@ -314,7 +450,6 @@ namespace pdflib timings(timings), unknown_operators({}), - stream({}), stack({}), stack_count(0) @@ -341,21 +476,37 @@ namespace pdflib return unknown_operators; } - void pdf_decoder::print() + void pdf_decoder::decode_and_interprete( + QPDFObjectHandle& content, + std::vector& parameters) { - LOG_S(INFO) << __FUNCTION__; - for(auto row:stream) + content_stats = {}; + content_stats.callback_delivery = true; + ensure_stack(); + begin_interpretation(); + + struct consumer_context + { + pdf_decoder* decoder; + std::vector* parameters; + }; + consumer_context context{this, ¶meters}; + const auto consumer = [](void* opaque, qpdf_stream_instruction& inst) { - LOG_S(INFO) << std::setw(12) << row.key << " | " << row.val; - } - } + auto& state = *static_cast(opaque); + state.decoder->consume_instruction(inst, *state.parameters); + }; - void pdf_decoder::decode(QPDFObjectHandle& qpdf_content) - { - LOG_S(INFO) << __FUNCTION__; + // No std::function allocation and no accumulated vector of source tokens. + // A consumer failure propagates through decode(), not damaged-PDF recovery. + qpdf_stream_decoder decoder(consumer, &context); + decoder.decode(content); + content_stats.source_objects = decoder.get_objects_delivered(); + content_stats.source_bytes_observed = decoder.get_source_bytes_observed(); - qpdf_stream_decoder decoder(stream); - decoder.decode(qpdf_content); + // QPDF may report EOF for each fallback component. Finalize exactly once + // here, after the entire logical page/Form stream has been consumed. + finish_interpretation(); } void pdf_decoder::ensure_stack() @@ -386,15 +537,6 @@ namespace pdflib base_ctm_valid_ = true; } - void pdf_decoder::interprete(std::vector& parameters) - { - LOG_S(INFO) << __FUNCTION__; - - ensure_stack(); - - interprete_stream(parameters); - } - // A form XObject establishes its own resource scope (PDF 32000-1, 8.10.2): // `Tf`, `gs`, `cs`/`CS` and `Do` inside it resolve against the form's // /Resources first and only then against the inherited ones. The child @@ -437,88 +579,120 @@ namespace pdflib return true; } - void pdf_decoder::interprete(std::vector& stream_, - std::vector& parameters_) + void pdf_decoder::interprete_cached( + const std::vector& cached, + std::vector& parameters) { - LOG_S(INFO) << __FUNCTION__; - - stream = stream_; - - interprete_stream(parameters_); - - if(parameters_.size()!=0) + content_stats = {}; + content_stats.source_objects = cached.size(); + ensure_stack(); + begin_interpretation(); + for(const auto& cached_instruction : cached) + { + // QPDF accessors and the instruction consumer take mutable objects. + // Keep those changes local; never cast away the cache's constness. + qpdf_stream_instruction instruction = cached_instruction; + consume_instruction(instruction, parameters); + } + finish_interpretation(); + if(not parameters.empty()) { - LOG_S(ERROR) << "Finishing a `Do` with nonzero number of parameters!"; + LOG_S(ERROR) << "Finishing a pattern replay with nonzero number of parameters!"; } } - void pdf_decoder::interprete_stream(std::vector& parameters) +#ifdef DOCLING_PARSE_CONTENT_DIAGNOSTICS + void pdf_decoder::note_pattern_replay(const pdf_decoder& child, + bool lattice) { - LOG_S(INFO) << __FUNCTION__; + const auto& child_stats = child.get_content_statistics(); + if(lattice) { ++pattern_stats->lattice_executions; } + else { ++pattern_stats->decoded_cells; } + pattern_stats->replay_source_objects += child_stats.source_objects; + // Count complete replay vectors, not the intentional resource cache or + // the single working instruction/pending operands of borrowed delivery. + pattern_stats->replay_materialized_tokens += 2 * child_stats.buffered_tokens; + pattern_stats->max_replay_token_capacity = std::max( + pattern_stats->max_replay_token_capacity, child_stats.buffered_token_capacity); + pattern_stats->max_pending_operands = std::max( + pattern_stats->max_pending_operands, child_stats.max_pending_operands); + } +#endif + void pdf_decoder::begin_interpretation() + { +#ifdef DOCLING_PARSE_STATE_LIFETIME_TESTING + if(interpretation_test_hook) { interpretation_test_hook(*this); } +#endif // Pattern space is anchored to the space in effect at the START of this // content stream, not to whatever the CTM happens to be when a pattern is // painted (ISO 32000-1, 8.7.3.1): for a page that is the default user // space, for a form XObject the space the form was invoked in. Every entry - // point lands here, so this is where it gets recorded -- capturing it in - // one of the interprete() overloads only left the other one, the one pages - // go through, with nothing to anchor to. + // point lands here, so callback interpretation and cached pattern replay + // capture the same initial space. capture_base_ctm(); + } - for(int l=0; l::consume_instruction( + qpdf_stream_instruction& inst, + std::vector& parameters) + { + if(inst.obj.isInlineImage()) { - qpdf_stream_instruction& inst = stream[l]; - - if(inst.obj.isInlineImage()) - { - read_inline_image_data(inst); - } - else if(inst.key=="operator") + read_inline_image_data(inst); + } + else if(inst.key=="operator") + { + for(auto itr=parameters.begin(); itr!=parameters.end(); ) { - for(auto itr=parameters.begin(); itr!=parameters.end(); ) + if(itr->key=="null" and itr->val=="null") // this can happen if you have an empty array/dict { - if(itr->key=="null" and itr->val=="null") // this can happen if you have an empty array/dict - { - LOG_S(ERROR) << "\t" << std::setw(12) << itr->key << " | " << itr->val << " => erasing ..."; - itr = parameters.erase(itr); - } - else - { - LOG_S(INFO) << "\t" << std::setw(12) << itr->key << " | " << itr->val; - itr++; - } - } - LOG_S(INFO) << " --> " << std::setw(12) << inst.key << " | " << inst.val; - - // An operand of the wrong type, a resource that fails to load, a - // malformed inline image: in a damaged stream any of these can - // surface as an exception from one operator. That operator is - // lost either way; the ones after it need not be. Log it and go - // on, as mupdf does ("ignoring error during interpretation"). - // Resource exhaustion (std::bad_alloc) is not caught here. - try - { - execute_operator(inst, parameters); - } - catch(const std::logic_error& e) - { - LOG_S(ERROR) << "operator " << inst.val << " failed (" << e.what() - << ") => skipping the operator"; + LOG_S(ERROR) << "\t" << std::setw(12) << itr->key << " | " << itr->val << " => erasing ..."; + itr = parameters.erase(itr); } - catch(const std::runtime_error& e) + else { - LOG_S(ERROR) << "operator " << inst.val << " failed (" << e.what() - << ") => skipping the operator"; + LOG_S(INFO) << "\t" << std::setw(12) << itr->key << " | " << itr->val; + itr++; } - - parameters.clear(); } - else + LOG_S(INFO) << " --> " << std::setw(12) << inst.key << " | " << inst.val; + + // An operand of the wrong type, a resource that fails to load, a + // malformed inline image: in a damaged stream any of these can + // surface as an exception from one operator. That operator is + // lost either way; the ones after it need not be. Log it and go + // on, as mupdf does ("ignoring error during interpretation"). + // Resource exhaustion (std::bad_alloc) is not caught here. + try + { + execute_operator(inst, parameters); + } + catch(const std::logic_error& e) + { + LOG_S(ERROR) << "operator " << inst.val << " failed (" << e.what() + << ") => skipping the operator"; + } + catch(const std::runtime_error& e) { - parameters.push_back(inst); + LOG_S(ERROR) << "operator " << inst.val << " failed (" << e.what() + << ") => skipping the operator"; } + + parameters.clear(); + } + else + { + parameters.push_back(inst); } +#ifdef DOCLING_PARSE_CONTENT_DIAGNOSTICS + content_stats.max_pending_operands = std::max(content_stats.max_pending_operands, + parameters.size()); +#endif + } + void pdf_decoder::finish_interpretation() + { if(marked_content_stack.size()>0) { LOG_S(WARNING) << "content stream ended with " << marked_content_stack.size() @@ -1301,7 +1475,7 @@ namespace pdflib return false; } - this->q(); + graphics_state_scope saved_state(*this); { std::vector no_parameters; if(even_odd) { current_shape_state().WStar(no_parameters); } @@ -1353,7 +1527,7 @@ namespace pdflib } } - this->Q(); + saved_state.restore(); LOG_S(INFO) << "pattern " << pattern_name << ": rendered Type 6 Coons mesh as " << triangle_count @@ -1400,7 +1574,7 @@ namespace pdflib return false; } - this->q(); + graphics_state_scope saved_state(*this); { std::vector no_parameters; if(even_odd) { current_shape_state().WStar(no_parameters); } @@ -1428,7 +1602,7 @@ namespace pdflib shinstr.set_soft_mask(current_graphic_state().get_soft_mask_instruction()); instructions.add_shading_instruction(std::move(shinstr)); - this->Q(); + saved_state.restore(); return true; } @@ -1447,7 +1621,19 @@ namespace pdflib return false; } +#ifdef DOCLING_PARSE_CONTENT_DIAGNOSTICS + const bool source_cached = pattern.has_parsed_stream(); +#endif const std::vector& cell = pattern.parse_stream(); +#ifdef DOCLING_PARSE_CONTENT_DIAGNOSTICS + if(source_cached) { ++pattern_stats->source_cache_hits; } + else + { + ++pattern_stats->source_parses; + pattern_stats->cached_tokens += cell.size(); + pattern_stats->cached_token_capacity += cell.capacity(); + } +#endif if(cell.empty()) { return false; @@ -1637,7 +1823,7 @@ namespace pdflib // clips are transformed by the matrix current when they are captured, and // after the `cm` below that is pattern space, not the space the path was // built in. - this->q(); + graphics_state_scope saved_state(*this); { std::vector no_parameters; if(even_odd) { current_shape_state().WStar(no_parameters); } @@ -1647,7 +1833,7 @@ namespace pdflib bool painted = false; - tiling_depth() += 1; + nesting_scope replay_depth(tiling_depth()); for(int iy = iy0; iy <= iy1; iy++) { for(int ix = ix0; ix <= ix1; ix++) @@ -1656,7 +1842,7 @@ namespace pdflib ix * xs, iy * ys, 1}; std::array place = mul(step, to_pattern); - this->q(); + graphics_state_scope cell_state(*this); const std::array place6 = {place[0], place[1], place[3], place[4], place[6], place[7]}; @@ -1686,23 +1872,26 @@ namespace pdflib cell_stream.soft_mask_cache_ = soft_mask_cache_; cell_stream.soft_mask_depth_ = soft_mask_depth_; +#ifdef DOCLING_PARSE_CONTENT_DIAGNOSTICS + cell_stream.pattern_stats = pattern_stats; +#endif bool updated_stack = cell_stream.update_stack(stack, stack_count); cell_stream.current_graphic_state().materialize_pattern_fill_color(); std::vector parameters; - std::vector insts = cell; - cell_stream.interprete(insts, parameters); + cell_stream.interprete_cached(cell, parameters); +#ifdef DOCLING_PARSE_CONTENT_DIAGNOSTICS + note_pattern_replay(cell_stream, true); +#endif if(updated_stack) { cell_stream.Q(); } - this->Q(); + cell_state.restore(); painted = true; } } - tiling_depth() -= 1; - - this->Q(); + saved_state.restore(); return painted; } @@ -1807,6 +1996,9 @@ namespace pdflib page_xobjects_, *decoded, timings); +#ifdef DOCLING_PARSE_CONTENT_DIAGNOSTICS + cell_stream.pattern_stats = pattern_stats; +#endif // A fresh state, not a copy of the fill's: the identity CTM keeps the // cell in pattern space, and no clip or path is inherited. @@ -1816,26 +2008,35 @@ namespace pdflib cell_stream.current_graphic_state().set_uncolored_pattern_rgb(rgb); } - pattern.set_decoding(true); - tiling_depth() += 1; - try - { - std::vector parameters; - std::vector insts = cell; - cell_stream.interprete(insts, parameters); - } - catch(const std::exception& e) - { - LOG_S(WARNING) << "pattern " << pattern_name << ": cell decode failed: " << e.what(); - decoded = std::make_shared(); - } - tiling_depth() -= 1; - pattern.set_decoding(false); + { + nesting_scope decoding(tiling_depth(), &pattern); + try + { + std::vector parameters; + cell_stream.interprete_cached(cell, parameters); +#ifdef DOCLING_PARSE_CONTENT_DIAGNOSTICS + note_pattern_replay(cell_stream, false); + pattern_stats->decoded_shape_instructions += decoded->get_shape_instructions().size(); +#endif + } + catch(const std::bad_alloc&) + { + throw; + } + catch(const std::exception& e) + { + LOG_S(WARNING) << "pattern " << pattern_name << ": cell decode failed: " << e.what(); + decoded = std::make_shared(); + } + } cell_stream.Q(); pattern.set_decoded_cell(color_key, decoded, std::make_shared >(cell_images)); } +#ifdef DOCLING_PARSE_CONTENT_DIAGNOSTICS + else { ++pattern_stats->decoded_cache_hits; } +#endif if(decoded->empty()) { @@ -1906,13 +2107,13 @@ namespace pdflib py.push_back(matrix[1]*xs[c] + matrix[3]*ys[c] + matrix[5]); } - std::vector paths = clip_state.get_paths(); - paths.emplace_back(std::move(px), std::move(py), CLOSED, RECTANGLE); + std::vector capture; + capture.emplace_back(std::move(px), std::move(py), CLOSED, RECTANGLE); // the /BBox always bounds, so an unclipped shading becomes clipped clip_rule rule = clip_state.has_clip() ? clip_state.get_rule() : CLIP_RULE_NONZERO; - clip_state = clip_state_instruction(rule, std::move(paths)); + clip_state = clip_state.appended(rule, std::move(capture)); } LOG_S(INFO) << "sh " << sh_name << ": " << to_string(shading->get_shading_type()) @@ -1948,12 +2149,10 @@ namespace pdflib // Time the whole call; the "machinery" cost (child-resource allocation, // graphics-state copies via q()/Q(), stack copy in update_stack(), ...) is - // derived as: total - set() - parse_stream() - nested interprete(). The - // subtracted parts are bucketed under their own keys, so this residual is - // disjoint from them. + // derived as total - resource setup - source parsing/interpretation. The + // callback path reports combined Form self-time without per-token clocks. utils::timer do_form_timer; double set_seconds = 0.0; - double parse_stream_seconds = 0.0; double interprete_seconds = 0.0; const pdf_resource& xobj = page_xobjects->get_form(xobj_name); @@ -2022,7 +2221,7 @@ namespace pdflib { // push-back the stack - this->q(); + graphics_state_scope saved_state(*this); // A transparency group is composited as a unit, so what is in force at // the `Do` applies to the group's result rather than to each operator @@ -2036,12 +2235,6 @@ namespace pdflib current_global_state().cm(xobj.get_matrix()); { - utils::timer parse_stream_timer; - std::vector insts = xobj.parse_stream(); - parse_stream_seconds = parse_stream_timer.get_time(); - timings.add_timing(pdf_timings::KEY_PARSE_STREAM_TOTAL, parse_stream_seconds); - timings.note_attributed(parse_stream_seconds); - pdf_decoder new_stream(config, page_dimension, @@ -2062,6 +2255,9 @@ namespace pdflib new_stream.soft_mask_cache_ = soft_mask_cache_; new_stream.soft_mask_depth_ = soft_mask_depth_; +#ifdef DOCLING_PARSE_CONTENT_DIAGNOSTICS + new_stream.pattern_stats = pattern_stats; +#endif bool updated_stack = new_stream.update_stack(stack, stack_count); // Form content inherits the caller's effective linkage. Local BDC @@ -2069,14 +2265,48 @@ namespace pdflib new_stream.inherited_marked_content.mcid = active_marked_content().mcid; new_stream.inherited_marked_content.artifact = active_marked_content().artifact; - // copy the stack std::vector parameters; { - utils::timer interprete_timer; - new_stream.interprete(insts, parameters); - interprete_seconds = interprete_timer.get_time(); + // Hold our own QPDF handle until the synchronous nested parse returns. + // update_stack() has already rebound state to the child resources; + // begin_interpretation() captures the transformed Form base CTM. + QPDFObjectHandle content = xobj.get_stream(); + const double attributed_before = timings.attributed_total(); + utils::timer content_timer; + new_stream.decode_and_interprete(content, parameters); + interprete_seconds = content_timer.get_time(); + const double attributed_during = timings.attributed_total() - attributed_before; + const double self_seconds = std::max(0.0, interprete_seconds - attributed_during); + timings.add_timing(pdf_timings::KEY_FORM_PARSE_INTERPRETE_TOTAL, self_seconds); + timings.note_attributed(self_seconds); + + if(not parameters.empty()) + { + LOG_S(ERROR) << "Finishing a `Do` with nonzero number of parameters!"; + } } +#ifdef DOCLING_PARSE_CONTENT_DIAGNOSTICS + const auto& child_stats = new_stream.get_content_statistics(); + content_stats.form_executions += 1 + child_stats.form_executions; + content_stats.form_source_objects += child_stats.source_objects + child_stats.form_source_objects; + content_stats.form_materialized_tokens += child_stats.buffered_tokens + child_stats.form_materialized_tokens; + content_stats.form_max_pending_operands = std::max( + content_stats.form_max_pending_operands, + std::max(child_stats.max_pending_operands, child_stats.form_max_pending_operands)); + content_stats.form_max_depth = std::max(content_stats.form_max_depth, + 1 + child_stats.form_max_depth); + // A CharProc may call a Form which shows another Type3 glyph. + content_stats.type3_executions += child_stats.type3_executions; + content_stats.type3_source_objects += child_stats.type3_source_objects; + content_stats.type3_materialized_tokens += child_stats.type3_materialized_tokens; + content_stats.type3_max_pending_operands = std::max( + content_stats.type3_max_pending_operands, child_stats.type3_max_pending_operands); + content_stats.type3_max_depth = std::max(content_stats.type3_max_depth, + child_stats.type3_max_depth); + content_stats.type3_recursion_skips += child_stats.type3_recursion_skips; +#endif + if(updated_stack) { new_stream.Q(); @@ -2090,12 +2320,12 @@ namespace pdflib } // pop-back the stack - this->Q(); + saved_state.restore(); } // residual = state copies, child-resource allocation, stack handling, ... double machinery_seconds = do_form_timer.get_time() - - set_seconds - parse_stream_seconds - interprete_seconds; + - set_seconds - interprete_seconds; timings.add_timing(pdf_timings::KEY_DO_FORM_MACHINERY, machinery_seconds); timings.note_attributed(machinery_seconds); @@ -2132,23 +2362,15 @@ namespace pdflib static thread_local int depth = 0; if(depth >= max_depth) { +#ifdef DOCLING_PARSE_CONTENT_DIAGNOSTICS + ++content_stats.type3_recursion_skips; +#endif LOG_S(WARNING) << "Type3 glyph procedures nest deeper than " << max_depth << " levels, the glyph is not painted"; return; } - std::vector insts; - try - { - qpdf_stream_decoder decoder(insts); - QPDFObjectHandle charproc = glyph.charproc; - decoder.decode(charproc); - } - catch(const std::exception& e) - { - LOG_S(WARNING) << "could not decode a Type3 glyph procedure: " << e.what(); - return; - } + QPDFObjectHandle charproc = glyph.charproc; // 9.6.5: names in the CharProc resolve against the font's /Resources, // and against the page's when the font has none. @@ -2201,55 +2423,66 @@ namespace pdflib page_item glyph_shapes; page_item glyph_images; - this->q(); - current_global_state().cm(glyph.matrix); - current_bitmap_state().paints_type3_glyph = true; - { - pdf_decoder new_stream(config, + graphics_state_scope saved_state(*this); + { + current_global_state().cm(glyph.matrix); + current_bitmap_state().paints_type3_glyph = true; + pdf_decoder new_stream(config, - page_dimension, - glyph_cells, - glyph_shapes, - glyph_images, + page_dimension, + glyph_cells, + glyph_shapes, + glyph_images, - page_fonts_, - page_grphs_, - page_colorspaces_, - page_shadings_, - page_patterns_, - page_xobjects_, + page_fonts_, + page_grphs_, + page_colorspaces_, + page_shadings_, + page_patterns_, + page_xobjects_, - instructions, + instructions, - timings); + timings); +#ifdef DOCLING_PARSE_CONTENT_DIAGNOSTICS + new_stream.pattern_stats = pattern_stats; +#endif - bool updated_stack = new_stream.update_stack(stack, stack_count); + bool updated_stack = new_stream.update_stack(stack, stack_count); - std::vector parameters; + std::vector parameters; - depth += 1; - try - { - new_stream.interprete(insts, parameters); - } - catch(...) { - depth -= 1; - throw; + nesting_scope glyph_depth(depth); + // Resources and glyph placement must be ready before the first + // QPDF callback executes. The handle/context live until decode ends. + new_stream.decode_and_interprete(charproc, parameters); } - depth -= 1; - if(updated_stack) - { - new_stream.Q(); - } +#ifdef DOCLING_PARSE_CONTENT_DIAGNOSTICS + const auto& child_stats = new_stream.get_content_statistics(); + content_stats.type3_executions += 1 + child_stats.type3_executions; + content_stats.type3_source_objects += child_stats.source_objects + child_stats.type3_source_objects; + content_stats.type3_materialized_tokens += child_stats.buffered_tokens + child_stats.type3_materialized_tokens; + content_stats.type3_max_pending_operands = std::max( + content_stats.type3_max_pending_operands, + std::max(child_stats.max_pending_operands, child_stats.type3_max_pending_operands)); + content_stats.type3_max_depth = std::max(content_stats.type3_max_depth, + 1 + child_stats.type3_max_depth); + content_stats.type3_recursion_skips += child_stats.type3_recursion_skips; +#endif - for(auto item:new_stream.get_unknown_operators()) - { - unknown_operators.insert(item); - } - } - this->Q(); + if(updated_stack) + { + new_stream.Q(); + } + + for(auto item:new_stream.get_unknown_operators()) + { + unknown_operators.insert(item); + } + } + saved_state.restore(); } void pdf_decoder::resolve_soft_mask() @@ -2282,6 +2515,9 @@ namespace pdflib auto found = decoded.find(cache_key); if(found != decoded.end()) { +#ifdef DOCLING_PARSE_CONTENT_DIAGNOSTICS + ++soft_mask_cache_->statistics.cache_hits; +#endif mask = found->second.mask; } else if(decoded.size() >= max_soft_mask_decodes) @@ -2399,13 +2635,30 @@ namespace pdflib timings); mask_decoder.soft_mask_cache_ = soft_mask_cache_; mask_decoder.soft_mask_depth_ = soft_mask_depth_ + 1; +#ifdef DOCLING_PARSE_CONTENT_DIAGNOSTICS + mask_decoder.pattern_stats = pattern_stats; +#endif QPDFObjectHandle group = spec.group; - mask_decoder.decode(group); + // The first callback must see the composed mask-space/gs matrix. mask_decoder.set_base_matrix(matrix); std::vector parameters; - mask_decoder.interprete(parameters); + mask_decoder.decode_and_interprete(group, parameters); + +#ifdef DOCLING_PARSE_CONTENT_DIAGNOSTICS + const auto& child_stats = mask_decoder.get_content_statistics(); + auto& stats = soft_mask_cache_->statistics; + ++stats.executions; + stats.source_objects += child_stats.source_objects; + stats.source_bytes_observed += child_stats.source_bytes_observed; + stats.materialized_tokens += child_stats.buffered_tokens; + stats.max_buffered_token_capacity = std::max(stats.max_buffered_token_capacity, + child_stats.buffered_token_capacity); + stats.max_pending_operands = std::max(stats.max_pending_operands, + child_stats.max_pending_operands); + stats.shape_instructions += mask->get_group().get_shape_instructions().size(); +#endif for(const auto& item : mask_decoder.get_unknown_operators()) { @@ -2414,6 +2667,11 @@ namespace pdflib return mask; } + catch(const std::bad_alloc&) + { + // Exhaustion is a consumer failure, not an unreadable mask source. + throw; + } catch(const std::exception& exc) { LOG_S(WARNING) << "ExtGState " << spec.key diff --git a/src/parse/pdf_resources/page_font.h b/src/parse/pdf_resources/page_font.h index 177644e64..346f7645f 100644 --- a/src/parse/pdf_resources/page_font.h +++ b/src/parse/pdf_resources/page_font.h @@ -216,7 +216,7 @@ namespace pdflib void init_differences(); - void init_charprocs(); + void validate_charprocs(); void init_space_index(); // The embedded program's builtin encoding as the source of reading text @@ -314,8 +314,6 @@ namespace pdflib int widths_all_zero_state = 0; std::unordered_map name_to_widths; - std::unordered_map name_to_descr; - bool cmap_initialized; bool diff_initialized; @@ -1226,7 +1224,7 @@ namespace pdflib { utils::timer font_timer; - init_charprocs(); + validate_charprocs(); init_differences(); @@ -3135,7 +3133,7 @@ namespace pdflib diff_numb_to_name[numb] = name.substr(1); } - LOG_S(INFO) << name << ", in cmap: " << cmap_numb_to_char.count(numb) << ", #-names: " << name_to_descr.size() << ", type: " << subtype; + LOG_S(INFO) << name << ", in cmap: " << cmap_numb_to_char.count(numb) << ", type: " << subtype; // Resolution order following PDF 32000-1 section 9.10.2: // the /ToUnicode cmap first, then glyph-name based methods @@ -3800,72 +3798,35 @@ namespace pdflib return glyph; } - void pdf_resource::init_charprocs() + void pdf_resource::validate_charprocs() { LOG_S(INFO) << __FUNCTION__; - QPDFObjectHandle qpdf_char_procs = qpdf_object::get_path(qpdf_font, {"/CharProcs"}); - - if(not qpdf_char_procs.isNull()) + QPDFObjectHandle procs = qpdf_object::get_path(qpdf_font, {"/CharProcs"}); + if(procs.isNull()) { return; } + if(not procs.isDictionary()) { - //assert(subtype==TYPE_3); - - LOG_S(WARNING) << "found CharProcs: " << qpdf_char_procs.getTypeName(); + LOG_S(WARNING) << "/CharProcs is not a dictionary"; + return; + } - if(not qpdf_char_procs.isDictionary()) + // Validation can repair a damaged stream before lazy glyph analysis reads + // it. Keep that ordering, but no unused description needs a token vector. + const auto discard_instruction = [](void*, qpdf_stream_instruction&) {}; + for(const auto& key : procs.getKeys()) + { + QPDFObjectHandle proc = procs.getKey(key); + if(not proc.isStream()) { - LOG_S(WARNING) << "/CharProcs is not a dictionary"; - return; + LOG_S(ERROR) << "not qpdf_obj.isStream()"; + throw std::logic_error("not qpdf_obj.isStream()"); } - for(auto& key : qpdf_char_procs.getKeys()) - { - if(qpdf_char_procs.hasKey(key)) - { - QPDFObjectHandle qpdf_char_proc = qpdf_char_procs.getKey(key); - //LOG_S(INFO) << "decoding: " << key << " -> " << qpdf_char_proc.getTypeName(); - - //assert(qpdf_char_proc.isStream()); - if(not qpdf_char_proc.isStream()) - { - std::string message = "not qpdf_obj.isStream()"; - LOG_S(ERROR) << message; - throw std::logic_error(message); - } - - std::vector stream={}; - - // decode the stream - { - qpdf_stream_decoder decoder(stream); - decoder.decode(qpdf_char_proc); - decoder.print(); - } - - LOG_S(INFO) << "key: " << key << " => #-streams: " << stream.size(); - - // interprete the stream - { - char_processor parser; - parser.parse(stream); - - name_to_descr[key] = parser.parse(stream); - //LOG_S(INFO) << key << ": " << name_to_descr.at(key); - - //parser.print(); - //cmap_numb_to_char = parser.get(); - - // FIXME: place-holder for now - //char_description desc; - //name_to_descr[key] = desc; - } - } - else - { - LOG_S(WARNING) << "could not find key: " << key; - } - } - } + qpdf_stream_decoder decoder(discard_instruction, nullptr); + decoder.decode(proc); + LOG_S(INFO) << "CharProc " << key << ": " + << decoder.get_objects_delivered() << " objects validated"; + } } void pdf_resource::init_space_index() diff --git a/src/parse/pdf_resources/page_font/char_description.h b/src/parse/pdf_resources/page_font/char_description.h deleted file mode 100644 index fa47a4c76..000000000 --- a/src/parse/pdf_resources/page_font/char_description.h +++ /dev/null @@ -1,30 +0,0 @@ -//-*-C++-*- - -#ifndef PDF_PAGE_FONT_CHAR_DESCRIPTION_H -#define PDF_PAGE_FONT_CHAR_DESCRIPTION_H - -namespace pdflib -{ - - class char_description - { - - public: - - char_description(); - ~char_description(); - - public: - - std::vector instructions; - }; - - char_description::char_description() - {} - - char_description::~char_description() - {} - -} - -#endif diff --git a/src/parse/pdf_resources/page_font/char_processor.h b/src/parse/pdf_resources/page_font/char_processor.h deleted file mode 100644 index 247c09c98..000000000 --- a/src/parse/pdf_resources/page_font/char_processor.h +++ /dev/null @@ -1,56 +0,0 @@ -//-*-C++-*- - -#ifndef PDF_PAGE_FONT_CHAR_PROCESSOR_H -#define PDF_PAGE_FONT_CHAR_PROCESSOR_H - -namespace pdflib -{ - - class char_processor - { - - public: - - char_processor(); - ~char_processor(); - - char_description parse(std::vector& instructions_); - - private: - - //std::vector instructions - }; - - char_processor::char_processor() - {} - - char_processor::~char_processor() - {} - - char_description char_processor::parse(std::vector& instructions_) - { - char_description desc; - desc.instructions = instructions_; - - return desc; - - /* - std::vector parameters; - - for(auto& item:instructions) - { - if(item.key!="operator") - { - parameters.push_back(item); - } - else - { - if(item.val=="Char_ProcessName") - } - } - */ - } - -} - -#endif diff --git a/src/parse/pdf_resources/page_pattern.h b/src/parse/pdf_resources/page_pattern.h index 99ad3b9e6..00f28cdce 100644 --- a/src/parse/pdf_resources/page_pattern.h +++ b/src/parse/pdf_resources/page_pattern.h @@ -85,12 +85,20 @@ namespace pdflib // Parsed once and cached: a hatch pattern is painted once per filled // region, and a CAD sheet has thousands of them. const std::vector& parse_stream() const; +#ifdef DOCLING_PARSE_CONTENT_DIAGNOSTICS + bool has_parsed_stream() const { return stream_parsed_; } +#endif // Shading pattern only. QPDFObjectHandle get_shading() const { return shading_; } private: +#ifdef DOCLING_PARSE_STATE_LIFETIME_TESTING + friend struct stream_state_lifetime_test_access; + inline static thread_local void (*parse_test_hook)(std::vector&) = nullptr; +#endif + std::string key_; int type_ = 0; int paint_type_ = 1; @@ -197,27 +205,40 @@ namespace pdflib { return stream_cache_; } - stream_parsed_ = true; - std::vector& insts = stream_cache_; QPDFObjectHandle stream = qpdf_pattern_; if(not stream.isStream()) { + stream_parsed_ = true; return insts; } try { +#ifdef DOCLING_PARSE_STATE_LIFETIME_TESTING + if(parse_test_hook) { parse_test_hook(insts); } +#endif qpdf_stream_decoder decoder(insts); decoder.decode(stream); } + catch(const std::bad_alloc&) + { + std::vector().swap(insts); + throw; + } catch(const std::exception& e) { + std::vector().swap(insts); LOG_S(WARNING) << "pattern " << key_ << ": content stream unreadable: " << e.what(); - insts.clear(); + } + catch(...) + { + std::vector().swap(insts); + throw; } + stream_parsed_ = true; return insts; } diff --git a/src/parse/pdf_resources/page_xobject_form.h b/src/parse/pdf_resources/page_xobject_form.h index 63746a583..25f901c62 100644 --- a/src/parse/pdf_resources/page_xobject_form.h +++ b/src/parse/pdf_resources/page_xobject_form.h @@ -52,7 +52,8 @@ namespace pdflib QPDFObjectHandle get_patterns() const; QPDFObjectHandle get_xobjects() const; - std::vector parse_stream() const; + // Copy the handle; the owning QPDF document must outlive parsing. + QPDFObjectHandle get_stream() const; private: @@ -258,29 +259,9 @@ namespace pdflib return QPDFObjectHandle::newNull(); } - std::vector pdf_resource::parse_stream() const + QPDFObjectHandle pdf_resource::get_stream() const { - std::vector stream; - - // decode the stream - try - { - qpdf_stream_decoder decoder(stream); - QPDFObjectHandle qpdf_xobject_ = qpdf_xobject; - decoder.decode(qpdf_xobject_); - - decoder.print(); - } - catch(const std::exception& exc) - { - std::stringstream ss; - ss << "encountered an error: " << exc.what(); - LOG_S(ERROR) << ss.str(); - - throw std::logic_error(ss.str()); - } - - return stream; + return qpdf_xobject; } void pdf_resource::parse_matrix() diff --git a/src/parse/pdf_states/shape.h b/src/parse/pdf_states/shape.h index e77ef7d9d..4a94c9b02 100644 --- a/src/parse/pdf_states/shape.h +++ b/src/parse/pdf_states/shape.h @@ -119,7 +119,7 @@ namespace pdflib void register_paths(shape_paint_mode paint_mode, shape_fill_rule fill_rule); // Consumes a pending W/W* clip by capturing the current path into - // `clippings`. Called from n() (shapes still in user space) and from + // shared history. Called from n() (shapes still in user space) and from // register_paths() (shapes already transformed to page space) — the // flag prevents double application of the CTM on the `W f`/`W S` path. void capture_pending_clip(bool already_transformed); @@ -167,9 +167,7 @@ namespace pdflib pdf_render_instructions& instructions; page_item curr_shapes; - page_item clippings; - // group id of each entry in `clippings`: one W/W* capture = one group - std::vector clipping_groups; + clip_state_instruction clipping_state; int next_clip_group = 0; // Box of the region the clip in force admits. Every W/W* capture (a clip @@ -202,7 +200,7 @@ namespace pdflib instructions(instructions_), curr_shapes(), - clippings(), + clipping_state(), clipping_path_mode(NO_CLIPPING_PATH_RULE), clipping_path_pending(false) @@ -235,8 +233,7 @@ namespace pdflib pdf_state& pdf_state::operator=(const pdf_state& other) { this->curr_shapes = other.curr_shapes; - this->clippings = other.clippings; - this->clipping_groups = other.clipping_groups; + this->clipping_state = other.clipping_state; this->next_clip_group = other.next_clip_group; this->has_clip_bbox = other.has_clip_bbox; this->clip_bbox = other.clip_bbox; @@ -480,7 +477,7 @@ namespace pdflib return clip_state_instruction(); } - if(clipping_path_mode == NO_CLIPPING_PATH_RULE or clippings.size() == 0) + if(clipping_path_mode == NO_CLIPPING_PATH_RULE or clipping_state.get_paths().empty()) { return clip_state_instruction(); } @@ -495,19 +492,7 @@ namespace pdflib rule = CLIP_RULE_EVEN_ODD; } - std::vector paths; - for(int l = 0; l < clippings.size(); l++) - { - const auto& shape = clippings[l]; - paths.emplace_back(shape.get_x(), - shape.get_y(), - shape.get_closing_type(), - shape.get_shape_type(), - l < static_cast(clipping_groups.size()) - ? clipping_groups[l] : 0); - } - - return clip_state_instruction(rule, std::move(paths)); + return clipping_state.with_rule(rule); } bool pdf_state::get_clip_bbox(std::array& bbox) const @@ -584,6 +569,7 @@ namespace pdflib // Every subpath of this capture belongs to one clip operation. const int group_id = next_clip_group++; + std::vector capture; // The capture admits the union of its subpaths: one box over all their // points. @@ -626,13 +612,20 @@ namespace pdflib group_box[3] = std::max(group_box[3], ys[i]); } - clippings.push_back(shape); - clipping_groups.push_back(group_id); + capture.emplace_back(shape.get_x(), shape.get_y(), + shape.get_closing_type(), shape.get_shape_type(), group_id); } // 0/1-point subpaths are the expected continuation left behind by // the `h` operator and are dropped silently } + if(not capture.empty()) + { + const clip_rule rule = clipping_path_mode == EVEN_ODD_RULE + ? CLIP_RULE_EVEN_ODD : CLIP_RULE_NONZERO; + clipping_state = clipping_state.appended(rule, std::move(capture)); + } + if(has_group_box) { intersect_clip_bbox(group_box); @@ -797,7 +790,7 @@ namespace pdflib void pdf_state::register_paths(shape_paint_mode paint_mode, shape_fill_rule fill_rule) { - // NOTE: the clip paths in `clippings` are already in page space (they + // NOTE: the captured clip paths are already in page space (they // are transformed once when captured); only the current path gets the // CTM applied here. diff --git a/src/parse/qpdf/stream_decoder.h b/src/parse/qpdf/stream_decoder.h index 0fa1d0790..9dbf99a5b 100644 --- a/src/parse/qpdf/stream_decoder.h +++ b/src/parse/qpdf/stream_decoder.h @@ -5,6 +5,8 @@ #include #include +#include +#include #include #include @@ -20,7 +22,13 @@ namespace pdflib { public: + using instruction_consumer = void (*)(void*, qpdf_stream_instruction&); + qpdf_stream_decoder(std::vector& stream_); + qpdf_stream_decoder(instruction_consumer consumer_, void* context_); + + std::size_t get_objects_delivered() const { return objects_delivered; } + std::size_t get_source_bytes_observed() const { return source_bytes_observed; } ~qpdf_stream_decoder(); void print(); @@ -38,11 +46,21 @@ namespace pdflib private: + qpdf_stream_decoder(); + void decode_each_stream(QPDFObjectHandle& contents); void decode_damaged_stream(QPDFObjectHandle& content); std::string inflate_prefix(QPDFObjectHandle& content); - std::vector& stream; + // Buffered delivery and immediate consumption share token normalization + // and damaged-stream recovery. The consumer/context are borrowed only for + // the synchronous decode call; no token reference escapes that call. + std::vector* stream = nullptr; + instruction_consumer consumer = nullptr; + void* context = nullptr; + std::exception_ptr consumer_failure; + std::size_t objects_delivered = 0; + std::size_t source_bytes_observed = 0; std::regex value_pattern_0; std::regex value_pattern_1; @@ -50,8 +68,21 @@ namespace pdflib }; qpdf_stream_decoder::qpdf_stream_decoder(std::vector& stream_): + qpdf_stream_decoder() + { + stream = &stream_; + } + + qpdf_stream_decoder::qpdf_stream_decoder(instruction_consumer consumer_, void* context_): + qpdf_stream_decoder() + { + if(not consumer_) { throw std::invalid_argument("missing content instruction consumer"); } + consumer = consumer_; + context = context_; + } + + qpdf_stream_decoder::qpdf_stream_decoder(): QPDFObjectHandle::ParserCallbacks(), - stream(stream_), value_pattern_0(R"(^(\d\.\d+)(\-\d+)$)"), value_pattern_1(R"(^((\-)?\d+\.\d*)(\-)(\d*)$)"), // need to deal with numbers that have random `-` in the middle @@ -63,7 +94,8 @@ namespace pdflib void qpdf_stream_decoder::print() { - for(auto row:stream) + if(not stream) { return; } + for(auto row:*stream) { LOG_S(INFO) << std::setw(12) << row.key << " | " << row.val; } @@ -73,14 +105,23 @@ namespace pdflib { LOG_S(INFO) << "start decoding content-stream: " << content.getTypeName() << " -> " << content.unparse(); - stream.clear(); + if(stream) { stream->clear(); } + objects_delivered = 0; + source_bytes_observed = 0; + consumer_failure = nullptr; try { QPDFObjectHandle::parseContentStream(content, this); } + catch(const std::bad_alloc&) + { + throw; + } catch(std::exception& e) { + // Interpreter failures are not source corruption and must never retry. + if(consumer_failure) { std::rethrow_exception(consumer_failure); } LOG_S(ERROR) << "QPDF encountered error (" << e.what() << ") during decoding"; // QPDF runs the whole stream through its filters before it hands @@ -88,11 +129,11 @@ namespace pdflib // through a damaged stream leaves nothing at all, although every // operator in front of the error inflated fine. A tokenizer error // instead arrives after the tokens before it, and those are kept. - if(stream.empty() and content.isArray()) + if(objects_delivered==0 and content.isArray()) { decode_each_stream(content); } - else if(stream.empty()) + else if(objects_delivered==0) { decode_damaged_stream(content); } @@ -114,17 +155,23 @@ namespace pdflib continue; } - std::size_t size_before = stream.size(); + std::size_t delivered_before = objects_delivered; try { QPDFObjectHandle::parseContentStream(item, this); } + catch(const std::bad_alloc&) + { + throw; + } catch(std::exception& e) { + // Interpreter failures are not source corruption and must never retry. + if(consumer_failure) { std::rethrow_exception(consumer_failure); } LOG_S(ERROR) << "QPDF encountered error (" << e.what() << ") during decoding of content stream " << i; - if(stream.size()==size_before) + if(objects_delivered==delivered_before) { decode_damaged_stream(item); } @@ -170,8 +217,14 @@ namespace pdflib QPDFObjectHandle::parseContentStream(content, this); } + catch(const std::bad_alloc&) + { + throw; + } catch(std::exception& e) { + // Interpreter failures are not source corruption and must never retry. + if(consumer_failure) { std::rethrow_exception(consumer_failure); } LOG_S(ERROR) << "QPDF encountered error (" << e.what() << ") while decoding the recovered content"; } @@ -356,12 +409,30 @@ namespace pdflib } } - stream.push_back(row); + if(stream) + { + stream->push_back(row); + } + else + { + try + { + consumer(context, row); + } + catch(...) + { + consumer_failure = std::current_exception(); + throw; + } + } + ++objects_delivered; } void qpdf_stream_decoder::contentSize(size_t len) { - //LOG_S(INFO) << __FUNCTION__ << ": " << len; + // QPDF can notify again during damaged-stream fallback; this is observed + // source bytes across attempts, not a claim of unique input bytes. + source_bytes_observed += len; } void qpdf_stream_decoder::handleEOF() diff --git a/src/parse/utils/pdf_timings.h b/src/parse/utils/pdf_timings.h index b588b63cc..cc99ff5a8 100644 --- a/src/parse/utils/pdf_timings.h +++ b/src/parse/utils/pdf_timings.h @@ -73,9 +73,12 @@ namespace pdflib // XObject timing keys static const std::string KEY_DECODE_XOBJECTS_TOTAL; - // Form xobject stream decompression + tokenization (do_form -> parse_stream) + // Buffered Form stream decompression + tokenization (native comparison). static const std::string KEY_PARSE_STREAM_TOTAL; + // Streamed Form parsing/operator self-time, excluding attributed nested work. + static const std::string KEY_FORM_PARSE_INTERPRETE_TOTAL; + // Form xobject machinery: child-resource allocation, graphics-state copies // (q()/Q()), stack copy (update_stack) -- i.e. do_form cost that is NOT // resource-parsing, stream-decoding or nested interpretation. @@ -98,6 +101,11 @@ namespace pdflib // Top-level page content-stream tokenization (decode_contents -> decode) static const std::string KEY_CONTENT_DECODE_TOTAL; + // Callback delivery interleaves page-source parsing and operator execution. + // Combined self-time excludes work already attributed to nested buckets; + // it replaces the separate two keys for ordinary streamed page content. + static const std::string KEY_CONTENT_PARSE_INTERPRETE_TOTAL; + // Operator-execution self-time: time spent interpreting operators across // all (recursion) levels, excluding the sub-work bucketed under the keys // above. Derived as a residual in decode_contents. @@ -484,6 +492,8 @@ namespace pdflib const std::string pdf_timings::KEY_DO_FORM_MACHINERY = "do_form_machinery_total"; const std::string pdf_timings::KEY_DO_IMAGE_TOTAL = "do_image_total"; const std::string pdf_timings::KEY_CONTENT_DECODE_TOTAL = "content_decode_total"; + const std::string pdf_timings::KEY_CONTENT_PARSE_INTERPRETE_TOTAL = "content_parse_interprete_total"; + const std::string pdf_timings::KEY_FORM_PARSE_INTERPRETE_TOTAL = "form_parse_interprete_total"; const std::string pdf_timings::KEY_INTERPRETE_OPS_TOTAL = "interprete_ops_total"; const std::string pdf_timings::KEY_DECODE_GRPHS_TOTAL = "decode_grphs_total"; @@ -603,9 +613,11 @@ namespace pdflib KEY_CMAP_PARSE_ENDCODESPACERANGE, KEY_DECODE_XOBJECTS_TOTAL, KEY_PARSE_STREAM_TOTAL, + KEY_FORM_PARSE_INTERPRETE_TOTAL, KEY_DO_FORM_MACHINERY, KEY_DO_IMAGE_TOTAL, KEY_CONTENT_DECODE_TOTAL, + KEY_CONTENT_PARSE_INTERPRETE_TOTAL, KEY_INTERPRETE_OPS_TOTAL, KEY_DECODE_GRPHS_TOTAL, KEY_TO_JSON_PAGE, @@ -712,11 +724,13 @@ namespace pdflib // --- decode_contents sub-timings --- {KEY_CONTENT_DECODE_TOTAL, KEY_DECODE_CONTENTS}, + {KEY_CONTENT_PARSE_INTERPRETE_TOTAL, KEY_DECODE_CONTENTS}, {KEY_INTERPRETE_OPS_TOTAL, KEY_DECODE_CONTENTS}, {KEY_DECODE_XOBJECTS_TOTAL, KEY_DECODE_CONTENTS}, {KEY_DECODE_GRPHS_TOTAL, KEY_DECODE_CONTENTS}, {KEY_DECODE_FONTS_TOTAL, KEY_DECODE_CONTENTS}, {KEY_PARSE_STREAM_TOTAL, KEY_DECODE_CONTENTS}, + {KEY_FORM_PARSE_INTERPRETE_TOTAL, KEY_DECODE_CONTENTS}, {KEY_DO_FORM_MACHINERY, KEY_DECODE_CONTENTS}, {KEY_DO_IMAGE_TOTAL, KEY_DECODE_CONTENTS}, diff --git a/src/render/blend2d_renderer.h b/src/render/blend2d_renderer.h index 3dd3dc375..5e45683ee 100644 --- a/src/render/blend2d_renderer.h +++ b/src/render/blend2d_renderer.h @@ -247,6 +247,14 @@ namespace pdflib bool shape_clip_empty_ = false; bool shape_clip_valid_ = false; + mutable clip_path_cursor clip_cursor_; + + const clip_path_cursor& prepared_clip_paths(const clip_state_instruction& state) const + { + clip_cursor_.prepare(state.get_paths()); + return clip_cursor_; + } + // Guards the one-shot canvas rotation, so that repeated canvas access // (get_canvas() followed by save(), for instance) rotates only once. mutable bool rotation_applied_ = false; @@ -1085,7 +1093,7 @@ namespace pdflib BLRect& rect) const { std::unordered_map group_bboxes; - for(const auto& clip_path : clip_state.get_paths()) + for(const auto& clip_path : prepared_clip_paths(clip_state)) { BLRect bbox; if(not clip_path_canvas_bbox(clip_path, bbox)) { continue; } @@ -1124,17 +1132,20 @@ namespace pdflib const clip_state_instruction& b) { if(a.get_rule() != b.get_rule()) { return false; } + if(a.shares_history(b)) { return true; } const auto& pa = a.get_paths(); const auto& pb = b.get_paths(); if(pa.size() != pb.size()) { return false; } - for(size_t i = 0; i < pa.size(); i++) + auto ia = pa.begin(); + for(const auto& path : pb) { - if(pa[i].get_clip_group() != pb[i].get_clip_group() or - pa[i].get_x() != pb[i].get_x() or - pa[i].get_y() != pb[i].get_y()) + if(ia->get_clip_group() != path.get_clip_group() or + ia->get_x() != path.get_x() or + ia->get_y() != path.get_y()) { return false; } + ++ia; } return true; } @@ -1146,18 +1157,13 @@ namespace pdflib { if(not clip_state.has_clip()) { return false; } - std::unordered_map group_sizes; - for(const auto& clip_path : clip_state.get_paths()) - { - group_sizes[clip_path.get_clip_group()]++; - } - - for(const auto& clip_path : clip_state.get_paths()) + const auto& paths = prepared_clip_paths(clip_state); + for(const auto& clip_path : paths) { // Subpaths of one W/W* operation only mean anything together (one // path under the clip's fill rule), which clip_to_rect cannot // express even when each subpath happens to be rectangular. - if(group_sizes[clip_path.get_clip_group()] > 1) { return true; } + if(paths.group_size(clip_path.get_clip_group()) > 1) { return true; } BLRect unused; if(not get_axis_aligned_clip_rect(clip_path, unused)) { return true; } @@ -1236,11 +1242,7 @@ namespace pdflib // art, a text-shaped clip). Only DIFFERENT captures intersect. // Intersecting the subpaths individually made any such stencil empty, // and everything filled through it vanished. - std::unordered_map group_sizes; - for(const auto& clip_path : clip_state.get_paths()) - { - group_sizes[clip_path.get_clip_group()]++; - } + const auto& paths = prepared_clip_paths(clip_state); // A group goes through the mask when it has a curved path or more than // one subpath (a union of rectangles is not a context clip either). @@ -1249,12 +1251,12 @@ namespace pdflib // caller painted the shape unclipped. std::unordered_map group_bboxes; std::unordered_map group_needs_mask; - for(const auto& clip_path : clip_state.get_paths()) + for(const auto& clip_path : paths) { BLRect rect; const bool is_rect = get_axis_aligned_clip_rect(clip_path, rect); const int group = clip_path.get_clip_group(); - if(not is_rect or group_sizes[group] > 1) { group_needs_mask[group] = true; } + if(not is_rect or paths.group_size(group) > 1) { group_needs_mask[group] = true; } BLRect bbox; if(clip_path_canvas_bbox(clip_path, bbox)) @@ -1289,7 +1291,7 @@ namespace pdflib std::vector > group_paths; [[maybe_unused]] size_t clip_index = 0; - for(const auto& clip_path : clip_state.get_paths()) + for(const auto& clip_path : paths) { BLRect unused; const bool is_rect = get_axis_aligned_clip_rect(clip_path, unused); @@ -1300,16 +1302,7 @@ namespace pdflib // rect paths that are alone in their group were already applied // through the context clip - bool grouped = false; - for(const auto& other : clip_state.get_paths()) - { - if(&other != &clip_path and - other.get_clip_group() == clip_path.get_clip_group()) - { - grouped = true; - break; - } - } + const bool grouped = paths.group_size(clip_path.get_clip_group()) > 1; if(is_rect and not grouped) { // LOG_S(INFO) << "build_clip_mask: path " << clip_index @@ -1463,18 +1456,14 @@ namespace pdflib return CLIP_NONE; } - std::unordered_map group_sizes; - for(const auto& clip_path : clip_state.get_paths()) - { - group_sizes[clip_path.get_clip_group()]++; - } + const auto& paths = prepared_clip_paths(clip_state); bool applied_clip = false; - for(const auto& clip_path : clip_state.get_paths()) + for(const auto& clip_path : paths) { // A subpath that shares its W/W* capture with others is not a clip // on its own; the whole group is handled through the coverage mask. - if(group_sizes[clip_path.get_clip_group()] > 1) { continue; } + if(paths.group_size(clip_path.get_clip_group()) > 1) { continue; } BLRect clip_rect; if(get_axis_aligned_clip_rect(clip_path, clip_rect)) @@ -1554,7 +1543,7 @@ namespace pdflib { shading_clip result; - for(const auto& clip_path : clip_state.get_paths()) + for(const auto& clip_path : prepared_clip_paths(clip_state)) { BLRect clip_rect; if(get_axis_aligned_clip_rect(clip_path, clip_rect)) @@ -2157,6 +2146,8 @@ namespace pdflib finish_page_context(); + clip_cursor_.reset(); + const auto& bbox = instr.crop_bbox; const double pdf_w = bbox[2] - bbox[0]; const double pdf_h = bbox[3] - bbox[1]; diff --git a/tests/cpp/run_memory_on_lines.py b/tests/cpp/run_memory_on_lines.py new file mode 100644 index 000000000..2c26152c3 --- /dev/null +++ b/tests/cpp/run_memory_on_lines.py @@ -0,0 +1,1228 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: MIT +"""Run the native memory probe; never compile it or import Python PDF bindings. + +Start small: + python3 tests/cpp/run_memory_on_lines.py --output /tmp/memory-smoke \ + --sizes 1000 --repeats 1 + +Use a NEW output directory for each iteration. Counts and PNG comparisons are +diagnostics for these generated grid fixtures, not general PDF equivalence tests. +Use --baseline RESULT_DIRECTORY to compare memory, process wall time and images. +The default 5% slowdown threshold reflects the user's accepted runtime budget. +Appearance fixtures are opt-in with --layouts appearance,widget,appearance-repeat. +Procedural Type3 fixtures are opt-in with --layouts type3,type3-repeat. +Soft-mask fixtures are opt-in with --layouts soft-mask,soft-mask-repeat. +Pattern fixtures are opt-in with --layouts pattern,pattern-repeat,pattern-lattice. +Clip-history fixtures are opt-in with --layouts clip-chain,clip-scoped (max 4000). +""" + +from __future__ import annotations + +import argparse +import csv +import hashlib +import json +import math +import platform +import statistics +import subprocess +import sys +import time +from collections import defaultdict +from pathlib import Path + +from tabulate import tabulate + +ROOT = Path(__file__).resolve().parents[2] +MODES = ("qpdf-discard", "collect", "decode", "skip-shapes", "render") +DEFAULT_LAYOUTS = ("direct", "form", "form-repeat", "form-nested") +APPEARANCE_LAYOUTS = ("appearance", "widget", "appearance-repeat") +TYPE3_LAYOUTS = ("type3", "type3-repeat") +SOFT_MASK_LAYOUTS = ("soft-mask", "soft-mask-repeat") +PATTERN_LAYOUTS = ("pattern", "pattern-repeat", "pattern-lattice") +CLIP_LAYOUTS = ("clip-chain", "clip-scoped") +LAYOUTS = ( + DEFAULT_LAYOUTS + + APPEARANCE_LAYOUTS + + TYPE3_LAYOUTS + + SOFT_MASK_LAYOUTS + + PATTERN_LAYOUTS + + CLIP_LAYOUTS +) + + +def require(condition: bool, message: str) -> None: + if not condition: + raise RuntimeError(message) + + +def sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + for block in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(block) + return digest.hexdigest() + + +def positive_int(value: str) -> int: + number = int(value) + if number <= 0: + raise argparse.ArgumentTypeError("must be a positive integer") + return number + + +def selection(value: str, choices: tuple[str, ...]) -> list[str]: + selected = value.split(",") + if len(set(selected)) != len(selected) or not all(x in choices for x in selected): + raise argparse.ArgumentTypeError( + "choose distinct values from " + ",".join(choices) + ) + return selected + + +def command(argv: list[str], log: Path, timeout: int) -> tuple[list[dict], float]: + started = time.monotonic() + try: + result = subprocess.run( + argv, capture_output=True, text=True, timeout=timeout, check=False + ) + except subprocess.TimeoutExpired as error: + stdout = ( + error.stdout.decode(errors="replace") + if isinstance(error.stdout, bytes) + else error.stdout or "" + ) + stderr = ( + error.stderr.decode(errors="replace") + if isinstance(error.stderr, bytes) + else error.stderr or "" + ) + log.write_text( + "COMMAND: " + + json.dumps(argv) + + "\nTIMEOUT\nSTDOUT:\n" + + stdout + + "\nSTDERR:\n" + + stderr + ) + raise RuntimeError(f"child timed out after {timeout}s; see {log}") from error + elapsed = time.monotonic() - started + log.write_text( + "COMMAND: " + + json.dumps(argv) + + f"\nEXIT: {result.returncode}\nSTDOUT:\n" + + result.stdout + + "\nSTDERR:\n" + + result.stderr + ) + require(result.returncode == 0, f"child exited {result.returncode}; see {log}") + try: + records = [ + json.loads(line) for line in result.stdout.splitlines() if line.strip() + ] + except json.JSONDecodeError as error: + raise RuntimeError(f"unexpected non-JSON probe output; see {log}") from error + require(bool(records), f"probe emitted no JSON; see {log}") + return records, elapsed + + +def validate_clip_snapshots(decoded: dict, info: dict, mode: str, layout: str) -> None: + if layout not in CLIP_LAYOUTS: + return + count = info["unique_lines"] if mode != "skip-shapes" else 0 + total = count * (count + 1) // 2 if layout == "clip-chain" else count + require(decoded["instruction_clip_paths"] == total, "wrong clip snapshot count") + require( + decoded["max_clip_paths"] + == (count if layout == "clip-chain" else min(count, 1)), + "wrong clip-history high-water mark", + ) + require(decoded["instruction_clip_points"] == 5 * total, "wrong clip point count") + if "clip_history_storage" in decoded: + storage = decoded["clip_history_storage"] + require(storage in ("shared", "copied"), "unknown clip history representation") + unique = count if storage == "shared" else total + require( + decoded["instruction_clip_unique_paths"] == unique, + "wrong unique clip path count", + ) + require( + decoded["instruction_clip_unique_points"] == 5 * unique, + "wrong unique clip points", + ) + require( + decoded["instruction_clip_history_nodes"] == count, + "wrong retained clip node count", + ) + + +def validate(stages: list[dict], info: dict, mode: str, layout: str) -> dict: + by_stage = {stage["stage"]: stage for stage in stages} + require("baseline" in by_stage and "loaded" in by_stage, "missing memory snapshots") + if mode in ("qpdf-discard", "collect"): + counts = by_stage["tokenized"] + unique = info["unique_lines"] + type3 = layout in TYPE3_LAYOUTS + clipping = layout in CLIP_LAYOUTS + expected_tokens = ( + (16 if layout == "clip-scoped" else 14) * unique + if clipping + else 7 * unique + (5 if type3 else 2) + ) + expected_operators = ( + (8 if layout == "clip-scoped" else 6) * unique + if clipping + else 3 * unique + (2 if type3 else 1) + ) + require( + counts["tokens"] == expected_tokens, + "unexpected source token count", + ) + require( + counts["operators"] == expected_operators, + "unexpected source operator count", + ) + require( + counts["decompressed_bytes"] == info["content_bytes"], "wrong decoded size" + ) + require(counts["eof_reached"], "QPDF did not report EOF") + require(counts["content_size_calls"] == 1, "expected one selected stream") + if layout in PATTERN_LAYOUTS: + expected_stream = "pattern_cell" + elif layout in SOFT_MASK_LAYOUTS: + expected_stream = "soft_mask_group" + elif layout in TYPE3_LAYOUTS: + expected_stream = "type3_charproc" + elif layout in APPEARANCE_LAYOUTS: + expected_stream = "appearance" + else: + expected_stream = "page" if layout == "direct" or clipping else "form_leaf" + require(counts["measured_stream"] == expected_stream, "wrong stream selected") + require( + counts["stream_executions"] == 1, "raw parsing must visit leaf only once" + ) + if mode == "qpdf-discard": + require(counts["retained_tokens"] == 0, "discard retained token objects") + require( + counts["token_inline_bytes"] == 0, "discard allocated a token vector" + ) + else: + require( + counts["retained_tokens"] == counts["tokens"], "collector lost tokens" + ) + return counts + + decoded = by_stage["decoded"] + soft_mask = layout in SOFT_MASK_LAYOUTS + pattern = layout in PATTERN_LAYOUTS + clipping = layout in CLIP_LAYOUTS + paints = info.get("soft_mask_paints", 0) + fills = info.get("pattern_fills", 0) + lattice_cells = info.get("pattern_lattice_cells_per_fill", 0) + # A mask's grid lives in its private instruction group. Page output is + # one filled rectangle per paint, with the decoded mask shared by all. + rectangles = soft_mask or (pattern and not lattice_cells) + expected = paints if soft_mask else fills if rectangles else info["lines"] + if lattice_cells: + expected *= lattice_cells + if mode == "skip-shapes": + expected = 0 + expected_page_shapes = 0 if layout in TYPE3_LAYOUTS else expected + if lattice_cells and mode != "skip-shapes": + # The unresolved outline is retained as a page shape but does not + # become a render instruction; the lattice already supplied the ink. + expected_page_shapes += fills + require( + decoded["page_shapes"] == expected_page_shapes, "unexpected page-shape count" + ) + for field in ("shape_instructions", "subpaths"): + require( + decoded[field] == expected, + f"{field}: expected {expected}, got {decoded[field]}", + ) + require( + decoded["segments"] == (3 if rectangles else 1) * expected, + "unexpected path segment count", + ) + require( + decoded["points"] == (4 if rectangles else 2) * expected, + "unexpected path point count", + ) + require(decoded["stored_dash_bytes"] == 0, "solid grid retained dash data") + validate_clip_snapshots(decoded, info, mode, layout) + if "page_content_delivery" in decoded: + if clipping: + expected_objects = (16 if layout == "clip-scoped" else 14) * info[ + "unique_lines" + ] + expected_bytes, expected_pending = info["page_content_bytes"], 4 + elif pattern: + expected_objects = 4 + 6 * fills + expected_bytes, expected_pending = info["page_content_bytes"], 4 + elif soft_mask: + expected_objects = 8 * paints + expected_bytes, expected_pending = info["page_content_bytes"], 4 + elif layout in TYPE3_LAYOUTS: + expected_objects = 5 + 9 * info["type3_repeats"] + expected_bytes, expected_pending = info["page_content_bytes"], 6 + elif layout in APPEARANCE_LAYOUTS: + expected_objects, expected_bytes, expected_pending = 0, 0, 0 + elif layout == "direct": + expected_objects = 7 * info["unique_lines"] + 2 + expected_bytes, expected_pending = info["content_bytes"], 2 + else: + expected_objects = 4 * info["form_repeats"] + expected_bytes, expected_pending = 12 * info["form_repeats"], 1 + require( + decoded["page_source_objects"] == expected_objects, + "wrong page /Contents object count", + ) + require( + decoded["page_source_bytes_observed"] == expected_bytes, + "wrong page /Contents byte count", + ) + if decoded["page_content_delivery"] == "callback": + require( + decoded["page_buffered_tokens"] == 0, + "callback page retained source tokens", + ) + require( + decoded["page_buffered_token_capacity"] == 0, + "callback page retained token capacity", + ) + else: + require( + decoded["page_buffered_tokens"] == expected_objects, + "buffered page lost source tokens", + ) + if decoded["page_operand_diagnostics"]: + require( + decoded["page_max_pending_operands"] == expected_pending, + "unexpected operand high-water mark", + ) + if decoded.get("form_diagnostics"): + depth, repeats = info["form_depth"], info["form_repeats"] + expected_form_objects = ( + repeats * (7 * info["unique_lines"] + 2 + 4 * (depth - 1)) if depth else 0 + ) + require( + decoded["form_executions"] == depth * repeats, "wrong Form execution count" + ) + require( + decoded["form_source_objects"] == expected_form_objects, + "wrong Form source-object count", + ) + require(decoded["form_max_depth"] == depth, "wrong Form nesting depth") + require( + decoded["form_max_pending_operands"] == (2 if depth else 0), + "unexpected Form operand high-water mark", + ) + require( + decoded["form_content_delivery"] in ("callback", "buffered"), + "unknown Form delivery mode", + ) + expected_materialized = ( + 2 * expected_form_objects + if decoded["form_content_delivery"] == "buffered" + else 0 + ) + require( + decoded["form_materialized_tokens"] == expected_materialized, + "unexpected Form token materialization", + ) + if decoded.get("appearance_diagnostics"): + count = info.get("appearance_count", 0) + expected_objects = count * (7 * info["unique_lines"] + 2) + require( + decoded["appearance_executions"] == count, + "wrong appearance execution count", + ) + require( + decoded["appearance_widget_executions"] == info.get("widget_count", 0), + "wrong widget appearance count", + ) + require( + decoded["appearance_source_objects"] == expected_objects, + "wrong appearance source-object count", + ) + require( + decoded["appearance_source_bytes_observed"] + == count * info["content_bytes"], + "wrong appearance source bytes", + ) + require( + decoded["appearance_max_pending_operands"] == (2 if count else 0), + "unexpected appearance operand high-water mark", + ) + delivery = decoded["appearance_content_delivery"] + require( + delivery in ("callback", "buffered"), "unknown appearance delivery mode" + ) + expected_materialized = expected_objects if delivery == "buffered" else 0 + require( + decoded["appearance_materialized_tokens"] == expected_materialized, + "unexpected appearance token materialization", + ) + capacity = decoded["appearance_max_buffered_token_capacity"] + if delivery == "callback" or count == 0: + require(capacity == 0, "unused appearance vector retained capacity") + else: + require( + capacity >= 7 * info["unique_lines"] + 2, + "buffered appearance capacity too small", + ) + if decoded.get("type3_diagnostics"): + repeats = info.get("type3_repeats", 0) + expected_type3_objects = repeats * (7 * info["unique_lines"] + 5) + require( + decoded["type3_executions"] == repeats, + "wrong procedural Type3 execution count", + ) + require( + decoded["type3_source_objects"] == expected_type3_objects, + "wrong Type3 source-object count", + ) + require( + decoded["type3_max_depth"] == (1 if repeats else 0), + "unexpected Type3 depth", + ) + require( + decoded["type3_recursion_skips"] == 0, "unexpected Type3 recursion skip" + ) + require( + decoded["type3_max_pending_operands"] == (2 if repeats else 0), + "unexpected Type3 operand high-water mark", + ) + delivery = decoded["type3_content_delivery"] + require(delivery in ("callback", "buffered"), "unknown Type3 delivery mode") + materialized = 2 * expected_type3_objects if delivery == "buffered" else 0 + require( + decoded["type3_materialized_tokens"] == materialized, + "unexpected Type3 token materialization", + ) + if repeats: + require( + decoded["text_instructions"] == repeats, + "unexpected Type3 text-instruction count", + ) + if decoded.get("soft_mask_diagnostics"): + require( + decoded["soft_mask_executions"] == (1 if paints else 0), + "soft-mask group must decode once per cache entry/CTM", + ) + require( + decoded["soft_mask_cache_hits"] == max(0, paints - 1), + "unexpected soft-mask cache reuse", + ) + objects = 7 * info["unique_lines"] + 2 if paints else 0 + require( + decoded["soft_mask_source_objects"] == objects, + "wrong soft-mask source-object count", + ) + require( + decoded["soft_mask_source_bytes_observed"] + == (info["content_bytes"] if paints else 0), + "wrong soft-mask source bytes", + ) + require( + decoded["soft_mask_shape_instructions"] + == (info["unique_lines"] if paints and mode != "skip-shapes" else 0), + "unexpected private mask geometry count", + ) + require( + decoded["soft_mask_max_pending_operands"] == (2 if paints else 0), + "unexpected soft-mask operand high-water mark", + ) + delivery = decoded["soft_mask_content_delivery"] + require(delivery in ("callback", "buffered"), "unknown soft-mask delivery mode") + require( + decoded["soft_mask_materialized_tokens"] + == (objects if delivery == "buffered" else 0), + "unexpected soft-mask token materialization", + ) + capacity = decoded["soft_mask_max_buffered_token_capacity"] + if delivery == "callback" or not paints: + require(capacity == 0, "unused soft-mask vector retained capacity") + else: + require(capacity >= objects, "buffered soft-mask capacity too small") + if decoded.get("pattern_diagnostics"): + active = pattern and mode != "skip-shapes" + source_objects = 7 * info["unique_lines"] + 2 if active else 0 + executions = (lattice_cells * fills if lattice_cells else 1) if active else 0 + checks = { + "source_parses": 1 if active else 0, + "source_cache_hits": fills - 1 if active else 0, + "cached_tokens": source_objects, + "decoded_cells": 1 if active and not lattice_cells else 0, + "decoded_cache_hits": fills - 1 if active and not lattice_cells else 0, + "decoded_shape_instructions": info["unique_lines"] + if active and not lattice_cells + else 0, + "lattice_executions": executions if lattice_cells else 0, + "replay_source_objects": executions * source_objects, + "max_pending_operands": 2 if active else 0, + } + for field, value in checks.items(): + require(decoded["pattern_" + field] == value, f"wrong pattern {field}") + delivery = decoded["pattern_content_delivery"] + require(delivery in ("borrowed", "copied"), "unknown pattern delivery mode") + require( + decoded["pattern_replay_materialized_tokens"] + == (2 * executions * source_objects if delivery == "copied" else 0), + "unexpected pattern replay vector materialization", + ) + cached_capacity = decoded["pattern_cached_token_capacity"] + require( + cached_capacity >= source_objects if active else cached_capacity == 0, + "unexpected pattern source-cache capacity", + ) + replay_capacity = decoded["pattern_max_replay_token_capacity"] + require( + replay_capacity >= source_objects + if active and delivery == "copied" + else replay_capacity == 0, + "unexpected pattern replay vector capacity", + ) + if mode == "render": + require("rendered" in by_stage, "missing render snapshot") + return decoded + + +def stage_median(group: list[dict], stage: str, field: str) -> float | None: + values = [ + s[field] + for r in group + for s in r["stages"] + if s["stage"] == stage and field in s + ] + return statistics.median(values) if values else None + + +def summarize(records: list[dict], output: Path) -> None: + grouped = defaultdict(list) + for record in records: + grouped[record["layout"], record["requested_lines"], record["mode"]].append( + record + ) + rows = [] + for (layout, requested, mode), group in grouped.items(): + last = [r["stages"][-1] for r in group] + decoded: dict = next( + (s for s in group[0]["stages"] if s["stage"] == "decoded"), {} + ) + tokenized: dict = next( + (s for s in group[0]["stages"] if s["stage"] == "tokenized"), {} + ) + rows.append( + { + "layout": layout, + "requested_lines": requested, + "unique_lines": group[0]["generation"]["unique_lines"], + "painted_lines": group[0]["generation"]["lines"], + "mode": mode, + "repeats": len(group), + "median_peak_mib": statistics.median(x["peak_rss_bytes"] for x in last) + / 2**20, + "min_peak_mib": min(x["peak_rss_bytes"] for x in last) / 2**20, + "max_peak_mib": max(x["peak_rss_bytes"] for x in last) / 2**20, + "median_snapshot_elapsed_s": statistics.median( + x["elapsed_s"] for x in last + ), + "median_process_wall_s": statistics.median( + r["process_wall_s"] for r in group + ), + "tokens": tokenized.get("tokens"), + "decompressed_bytes": tokenized.get("decompressed_bytes"), + "retained_tokens": tokenized.get("retained_tokens"), + "token_inline_bytes": tokenized.get("token_inline_bytes"), + "shape_instructions": decoded.get("shape_instructions"), + "instruction_clip_paths": decoded.get("instruction_clip_paths"), + "instruction_clip_bytes": decoded.get("instruction_clip_bytes"), + "instruction_clip_logical_bytes": decoded.get( + "instruction_clip_logical_bytes" + ), + "instruction_clip_unique_paths": decoded.get( + "instruction_clip_unique_paths" + ), + "instruction_clip_unique_points": decoded.get( + "instruction_clip_unique_points" + ), + "instruction_clip_history_nodes": decoded.get( + "instruction_clip_history_nodes" + ), + "clip_history_storage": decoded.get("clip_history_storage"), + "median_decode_work_s": stage_median(group, "decoded", "decode_work_s"), + "median_render_work_s": stage_median( + group, "rendered", "render_work_s" + ), + "page_content_delivery": decoded.get("page_content_delivery"), + "page_source_objects": decoded.get("page_source_objects"), + "page_buffered_tokens": decoded.get("page_buffered_tokens"), + "page_max_pending_operands": decoded.get("page_max_pending_operands"), + "form_content_delivery": decoded.get("form_content_delivery"), + "form_executions": decoded.get("form_executions"), + "form_source_objects": decoded.get("form_source_objects"), + "form_materialized_tokens": decoded.get("form_materialized_tokens"), + "form_max_pending_operands": decoded.get("form_max_pending_operands"), + "form_max_depth": decoded.get("form_max_depth"), + "type3_content_delivery": decoded.get("type3_content_delivery"), + "type3_executions": decoded.get("type3_executions"), + "type3_source_objects": decoded.get("type3_source_objects"), + "type3_materialized_tokens": decoded.get("type3_materialized_tokens"), + "type3_max_pending_operands": decoded.get("type3_max_pending_operands"), + "type3_max_depth": decoded.get("type3_max_depth"), + "type3_recursion_skips": decoded.get("type3_recursion_skips"), + "appearance_content_delivery": decoded.get( + "appearance_content_delivery" + ), + "appearance_executions": decoded.get("appearance_executions"), + "appearance_widget_executions": decoded.get( + "appearance_widget_executions" + ), + "appearance_source_objects": decoded.get("appearance_source_objects"), + "appearance_source_bytes_observed": decoded.get( + "appearance_source_bytes_observed" + ), + "appearance_materialized_tokens": decoded.get( + "appearance_materialized_tokens" + ), + "appearance_max_buffered_token_capacity": decoded.get( + "appearance_max_buffered_token_capacity" + ), + "appearance_max_pending_operands": decoded.get( + "appearance_max_pending_operands" + ), + **{ + "soft_mask_" + field: decoded.get("soft_mask_" + field) + for field in ( + "content_delivery", + "executions", + "cache_hits", + "source_objects", + "source_bytes_observed", + "materialized_tokens", + "max_buffered_token_capacity", + "max_pending_operands", + "shape_instructions", + ) + }, + **{ + "pattern_" + field: decoded.get("pattern_" + field) + for field in ( + "content_delivery", + "source_parses", + "source_cache_hits", + "cached_tokens", + "cached_token_capacity", + "decoded_cells", + "decoded_cache_hits", + "decoded_shape_instructions", + "lattice_executions", + "replay_source_objects", + "replay_materialized_tokens", + "max_replay_token_capacity", + "max_pending_operands", + ) + }, + } + ) + with (output / "summary.csv").open("w", newline="") as stream: + writer = csv.DictWriter(stream, fieldnames=list(rows[0])) + writer.writeheader() + writer.writerows(rows) + table = [ + [ + row["layout"], + row["requested_lines"], + row["mode"], + row["median_peak_mib"], + f"{row['min_peak_mib']:.2f}..{row['max_peak_mib']:.2f}", + ] + for row in rows + ] + print( + "\n" + + tabulate( + table, + headers=["Layout", "Target", "Mode", "Peak MiB", "Range MiB"], + tablefmt="simple", + floatfmt=".2f", + intfmt=",", + ), + flush=True, + ) + + +def baseline_data(path: Path, args: argparse.Namespace) -> dict: + """Read existing observations only; do not launch or rebuild the baseline.""" + require(path.resolve() != args.output.resolve(), "baseline and output must differ") + status = json.loads((path / "status.json").read_text()) + require(status["status"] == "passed", "baseline run did not pass its checks") + with (path / "summary.csv").open(newline="") as stream: + rows = { + (row["layout"], int(row["requested_lines"]), row["mode"]): row + for row in csv.DictReader(stream) + } + for size in args.sizes: + for layout in args.layouts: + for mode in args.modes: + require((layout, size, mode) in rows, "baseline lacks a requested case") + grouped = defaultdict(list) + for line in (path / "measurements.jsonl").read_text().splitlines(): + record = json.loads(line) + grouped[record["layout"], record["requested_lines"], record["mode"]].append( + record + ) + return { + "directory": path, + "rows": rows, + "records": grouped, + "metadata": json.loads((path / "metadata.json").read_text()), + } + + +def compare_baseline( + records: list[dict], output: Path, baseline: dict, slowdown_limit: float +) -> dict: + """Compare matched input/output and repeated process timings, not estimates.""" + current_metadata = json.loads((output / "metadata.json").read_text()) + previous_metadata = baseline["metadata"] + for setting in ("probe", "dependency_sha256"): + require( + previous_metadata[setting] == current_metadata[setting], + f"baseline/candidate {setting} differs; use a controlled comparison", + ) + grouped = defaultdict(list) + for record in records: + grouped[record["layout"], record["requested_lines"], record["mode"]].append( + record + ) + semantic_fields = ( + "page_shapes", + "shape_instructions", + "subpaths", + "segments", + "points", + "instruction_dash_entries", + "max_dash_entries", + "page_shape_dash_entries", + "instruction_clip_paths", + "max_clip_paths", + "instruction_clip_points", + ) + rows, image_checks = [], [] + for key, group in grouped.items(): + layout, requested, mode = key + before = baseline["records"][key] + first_before, first_after = before[0], group[0] + require( + first_before["generation"]["pdf_sha256"] + == first_after["generation"]["pdf_sha256"], + f"baseline/candidate PDF differs for {layout}-{requested}", + ) + require( + first_before["stages"][0]["qpdf_version"] + == first_after["stages"][0]["qpdf_version"], + "baseline/candidate QPDF version differs", + ) + if mode in ("decode", "skip-shapes", "render"): + old_output = next( + s for s in first_before["stages"] if s["stage"] == "decoded" + ) + new_output = next( + s for s in first_after["stages"] if s["stage"] == "decoded" + ) + require( + all( + old_output[field] == new_output[field] for field in semantic_fields + ), + f"baseline/candidate output counters differ for {layout}-{requested}-{mode}", + ) + if layout in SOFT_MASK_LAYOUTS: + require( + all( + old_output[field] == new_output[field] + for field in ( + "soft_mask_executions", + "soft_mask_cache_hits", + "soft_mask_source_objects", + "soft_mask_shape_instructions", + ) + ), + "baseline/candidate soft-mask source, geometry or reuse differs", + ) + if layout in PATTERN_LAYOUTS: + require( + all( + old_output["pattern_" + field] == new_output["pattern_" + field] + for field in ( + "source_parses", + "source_cache_hits", + "cached_tokens", + "cached_token_capacity", + "decoded_cells", + "decoded_cache_hits", + "decoded_shape_instructions", + "lattice_executions", + "replay_source_objects", + ) + ), + "baseline/candidate pattern source, geometry or reuse differs", + ) + if mode == "render": + filename = f"{layout}-{requested}.png" + same_image = sha256(baseline["directory"] / filename) == sha256( + output / filename + ) + image_checks.append( + { + "layout": layout, + "target": requested, + "identical_png_bytes": same_image, + } + ) + require(same_image, f"baseline/candidate PNG bytes differ: {filename}") + + old_peak = ( + statistics.median(r["stages"][-1]["peak_rss_bytes"] for r in before) / 2**20 + ) + new_peak = ( + statistics.median(r["stages"][-1]["peak_rss_bytes"] for r in group) / 2**20 + ) + old_times = [r["process_wall_s"] for r in before] + new_times = [r["process_wall_s"] for r in group] + old_wall, new_wall = statistics.median(old_times), statistics.median(new_times) + delta_time = 100 * (new_wall / old_wall - 1) + overlap = max(min(old_times), min(new_times)) <= min( + max(old_times), max(new_times) + ) + if min(len(before), len(group)) < 3: + timing = "diagnostic: <3 reps" + elif delta_time > slowdown_limit: + timing = "recheck: overlap" if overlap else "review slowdown" + else: + timing = "within budget" + rows.append( + { + "layout": layout, + "requested_lines": requested, + "mode": mode, + "baseline_peak_mib": old_peak, + "candidate_peak_mib": new_peak, + "memory_change_pct": 100 * (new_peak / old_peak - 1), + "baseline_process_wall_s": old_wall, + "candidate_process_wall_s": new_wall, + "time_change_pct": delta_time, + "baseline_min_wall_s": min(old_times), + "baseline_max_wall_s": max(old_times), + "candidate_min_wall_s": min(new_times), + "candidate_max_wall_s": max(new_times), + "timing_ranges_overlap": overlap, + "timing_review": timing, + } + ) + with (output / "comparison.csv").open("w", newline="") as stream: + writer = csv.DictWriter(stream, fieldnames=list(rows[0])) + writer.writeheader() + writer.writerows(rows) + report = { + "baseline": str(baseline["directory"]), + "same_binary": previous_metadata["binary_sha256"] + == current_metadata["binary_sha256"], + "slowdown_review_limit_pct": slowdown_limit, + "compared_cases": len(rows), + "image_checks": image_checks, + "timing_review_cases": sum(r["timing_review"] != "within budget" for r in rows), + "note": "Timing flags request review, not a significance test or automatic failure.", + } + (output / "baseline-comparison.json").write_text( + json.dumps(report, indent=2) + "\n" + ) + if report["same_binary"]: + print( + "\nIdentical binary hashes: this is a repeat/control comparison.", + flush=True, + ) + print( + "\nBaseline comparison (negative changes are improvements):\n" + + tabulate( + [ + [ + row["layout"], + row["requested_lines"], + row["mode"], + row["baseline_peak_mib"], + row["candidate_peak_mib"], + row["memory_change_pct"], + row["baseline_process_wall_s"], + row["candidate_process_wall_s"], + row["time_change_pct"], + row["timing_review"], + ] + for row in rows + ], + headers=[ + "Layout", + "Target", + "Mode", + "Before MiB", + "After MiB", + "Mem %", + "Before s", + "After s", + "Time %", + "Timing", + ], + tablefmt="simple", + floatfmt=("", "", "", ".2f", ".2f", ".2f", ".3f", ".3f", ".2f", ""), + intfmt=",", + ), + flush=True, + ) + return report + + +def arguments() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--exe", type=Path, default=ROOT / "build/test_memory_on_lines") + parser.add_argument("--output", type=Path, required=True) + parser.add_argument( + "--sizes", default="1000,8000", help="comma-separated line targets" + ) + parser.add_argument("--repeats", type=positive_int, default=3) + parser.add_argument( + "--layouts", type=lambda s: selection(s, LAYOUTS), default=list(DEFAULT_LAYOUTS) + ) + parser.add_argument( + "--modes", type=lambda s: selection(s, MODES), default=list(MODES) + ) + parser.add_argument("--form-repeats", type=positive_int, default=4) + parser.add_argument("--form-depth", type=positive_int, default=3) + parser.add_argument("--appearance-repeats", type=positive_int, default=4) + parser.add_argument("--type3-repeats", type=positive_int, default=4) + parser.add_argument("--soft-mask-repeats", type=positive_int, default=4) + parser.add_argument("--pattern-repeats", type=positive_int, default=4) + parser.add_argument("--timeout", type=positive_int, default=180) + parser.add_argument( + "--baseline", type=Path, help="compare with a previous result directory" + ) + parser.add_argument( + "--slowdown-limit-pct", + type=float, + default=5.0, + help="timing review threshold, not an automatic failure (default: 5)", + ) + args = parser.parse_args() + if not math.isfinite(args.slowdown_limit_pct) or args.slowdown_limit_pct < 0: + parser.error("--slowdown-limit-pct must be finite and nonnegative") + try: + sizes = [positive_int(value) for value in args.sizes.split(",")] + except (ValueError, argparse.ArgumentTypeError) as error: + parser.error(str(error)) + if not sizes or min(sizes) < 10 or max(sizes) > 2**31 - 1: + parser.error("line targets must be 10..INT_MAX") + if len(set(sizes)) != len(sizes): + parser.error("line targets must be distinct") + if args.form_depth > 32: + parser.error("--form-depth must be 1..32") + if "form-repeat" in args.layouts and args.form_repeats < 2: + parser.error("--form-repeats must be at least 2 for form-repeat") + if "form-nested" in args.layouts and args.form_depth < 2: + parser.error("--form-depth must be at least 2 for form-nested") + if "appearance-repeat" in args.layouts and args.appearance_repeats < 2: + parser.error("--appearance-repeats must be at least 2 for appearance-repeat") + if "type3-repeat" in args.layouts and args.type3_repeats < 2: + parser.error("--type3-repeats must be at least 2 for type3-repeat") + if "soft-mask-repeat" in args.layouts and args.soft_mask_repeats < 2: + parser.error("--soft-mask-repeats must be at least 2 for soft-mask-repeat") + if "pattern-repeat" in args.layouts and args.pattern_repeats < 2: + parser.error("--pattern-repeats must be at least 2 for pattern-repeat") + if any(layout in CLIP_LAYOUTS for layout in args.layouts) and max(sizes) > 4000: + parser.error("clip-history fixtures require line targets of at most 4000") + args.sizes = sizes + return args + + +def compare_layout_images( + image_hashes: dict[tuple[str, int], str], sizes: list[int] +) -> list[dict]: + pairs = [ + ("direct", name) + for name in ("form", "form-nested", "appearance", "widget", "type3") + ] + pairs.append(("clip-chain", "clip-scoped")) + comparisons = [] + for size in sizes: + for reference, layout in pairs: + first = image_hashes.get((reference, size)) + other = image_hashes.get((layout, size)) + if first and other: + comparisons.append( + { + "requested_lines": size, + "reference_layout": reference, + "layout": layout, + "identical_png_bytes": first == other, + } + ) + return comparisons + + +def main() -> None: + args = arguments() + sizes = args.sizes + baseline = baseline_data(args.baseline.resolve(), args) if args.baseline else None + exe = args.exe.resolve() + require(exe.is_file(), f"probe not found: {exe}; compile it first") + output = args.output.resolve() + output.mkdir(parents=True, exist_ok=False) + sources = output / "source" + sources.mkdir() + source_hashes = {} + for relative in ( + "tests/cpp/test_memory_on_lines.cpp", + "tests/cpp/run_memory_on_lines.py", + "CMakeLists.txt", + "src/parse/pdf_decoders/page.h", + "src/parse/pdf_decoders/stream.h", + "src/parse/pdf_resources/page_xobject_form.h", + "src/parse/pdf_resources/page_font.h", + "src/parse/pdf_resources/page_pattern.h", + "src/parse/pdf_states/shape.h", + "src/parse/page_items/render_instructions.h", + "src/render/blend2d_renderer.h", + "src/parse/qpdf/stream_decoder.h", + "src/parse/utils/pdf_timings.h", + "tests/test_unit_appearance_streaming.py", + "tests/test_unit_type3_streaming.py", + "tests/test_unit_soft_mask_streaming.py", + "tests/test_unit_pattern_replay.py", + "tests/cpp/test_stream_state_lifetimes.cpp", + "tests/cpp/test_clip_history.cpp", + "tests/test_unit_clipping.py", + ): + path = ROOT / relative + source_hashes[relative] = sha256(path) + (sources / path.name).write_bytes(path.read_bytes()) + head = subprocess.run( + ["git", "rev-parse", "HEAD"], + cwd=ROOT, + capture_output=True, + text=True, + check=False, + ) + diff = subprocess.run( + ["git", "--no-pager", "diff", "--no-ext-diff", "--no-textconv"], + cwd=ROOT, + capture_output=True, + text=True, + check=False, + ) + (output / "working-tree.diff").write_text(diff.stdout + diff.stderr) + for filename in ("flags.make", "link.txt"): + path = exe.parent / "CMakeFiles" / f"{exe.name}.dir" / filename + if path.is_file(): + (sources / filename).write_bytes(path.read_bytes()) + archives = list((ROOT / "externals/lib").glob("*.a")) + archives += [ + ROOT / "build/_deps/logurugitrepo-build/libloguru.a", + ROOT / "build/_deps/blend2d-build/libblend2d.a", + ROOT / "build/libpdfium_jbig2.a", + ] + metadata = { + "created": time.strftime("%Y-%m-%d %H:%M:%S %z"), + "head": head.stdout.strip(), + "platform": platform.platform(), + "executable": str(exe), + "binary_sha256": sha256(exe), + "source_sha256": source_hashes, + "dependency_sha256": { + str(path): sha256(path) for path in archives if path.is_file() + }, + "sizes": sizes, + "layouts": args.layouts, + "modes": args.modes, + "repeats": args.repeats, + "form_repeats": args.form_repeats, + "form_depth": args.form_depth, + "appearance_repeats": args.appearance_repeats, + "type3_repeats": args.type3_repeats, + "soft_mask_repeats": args.soft_mask_repeats, + "pattern_repeats": args.pattern_repeats, + "baseline": str(args.baseline.resolve()) if args.baseline else None, + "slowdown_review_limit_pct": args.slowdown_limit_pct, + "probe": {"do_thread_safe": False, "render_scale": 1.0, "pause_seconds": 0}, + "raw_token_scope": "one page, Form leaf, appearance, CharProc, mask or pattern source; not execution", + "note": "No compilation or PDF Python bindings; each child is a fresh process.", + } + (output / "metadata.json").write_text(json.dumps(metadata, indent=2) + "\n") + records: list[dict] = [] + image_hashes: dict[tuple[str, int], str] = {} + raw_signatures: dict[tuple[int, str], tuple[int, int, int]] = {} + comparisons = [] + try: + with (output / "measurements.jsonl").open("w") as stream: + for size in sizes: + for layout in args.layouts: + stem = f"{layout}-{size}" + pdf = output / f"{stem}.pdf" + if layout in CLIP_LAYOUTS: + variant = layout + elif layout in PATTERN_LAYOUTS: + variant = ( + "pathstorm-pattern-lattice" + if layout == "pattern-lattice" + else "pathstorm-pattern" + ) + elif layout in SOFT_MASK_LAYOUTS: + variant = "pathstorm-soft-mask" + elif layout in TYPE3_LAYOUTS: + variant = "pathstorm-type3" + elif layout in APPEARANCE_LAYOUTS: + variant = ( + "pathstorm-widget" + if layout == "widget" + else "pathstorm-appearance" + ) + else: + variant = ( + "pathstorm" if layout == "direct" else "pathstorm-form" + ) + argv = [str(exe), "generate", variant, str(size), str(pdf)] + if layout in PATTERN_LAYOUTS: + argv += [ + str( + args.pattern_repeats + if layout == "pattern-repeat" + else 1 + ) + ] + elif layout in SOFT_MASK_LAYOUTS: + repeat = ( + args.soft_mask_repeats + if layout == "soft-mask-repeat" + else 1 + ) + argv += [str(repeat)] + elif layout in TYPE3_LAYOUTS: + argv += [ + str(args.type3_repeats if layout == "type3-repeat" else 1) + ] + elif layout in APPEARANCE_LAYOUTS: + repeat = ( + args.appearance_repeats + if layout == "appearance-repeat" + else 1 + ) + argv += [str(repeat)] + elif layout != "direct" and layout not in CLIP_LAYOUTS: + repeat = args.form_repeats if layout == "form-repeat" else 1 + depth = args.form_depth if layout == "form-nested" else 1 + argv += [str(repeat), str(depth)] + generation, _ = command( + argv, output / f"{stem}-generate.log", args.timeout + ) + require(len(generation) == 1, "expected one generator record") + info = generation[0] + info["pdf_sha256"] = sha256(pdf) + (output / f"{stem}-generation.json").write_text( + json.dumps(info, indent=2) + "\n" + ) + for mode in args.modes: + for repetition in range(1, args.repeats + 1): + case = f"{stem}-{mode}-{repetition}" + argv = [str(exe), "measure", mode, str(pdf)] + png = output / f"{stem}.png" + if mode == "render" and repetition == 1: + argv += ["0", str(png)] + stages, wall = command( + argv, output / f"{case}.log", args.timeout + ) + counts = validate(stages, info, mode, layout) + if mode in ("qpdf-discard", "collect"): + signature = ( + counts["tokens"], + counts["operators"], + counts["decompressed_bytes"], + ) + signature_key = ( + size, + "type3" + if layout in TYPE3_LAYOUTS + else layout + if layout in CLIP_LAYOUTS + else "grid", + ) + require( + raw_signatures.setdefault(signature_key, signature) + == signature, + "discard/collect or selected-source counts differ", + ) + record = { + "layout": layout, + "requested_lines": size, + "mode": mode, + "repetition": repetition, + "generation": info, + "command": argv, + "process_wall_s": wall, + "stages": stages, + } + records.append(record) + stream.write(json.dumps(record) + "\n") + stream.flush() + if mode == "render" and repetition == 1: + image_hashes[layout, size] = sha256(png) + print( + f"{case}: {stages[-1]['peak_rss_bytes'] / 2**20:.2f} MiB; " + f"process {wall:.3f}s", + flush=True, + ) + summarize(records, output) + comparisons = compare_layout_images(image_hashes, sizes) + (output / "image-comparisons.json").write_text( + json.dumps(comparisons, indent=2) + "\n" + ) + require( + all(row["identical_png_bytes"] for row in comparisons), + "equivalent-layout PNG bytes differ; inspect pixels/encoding before accepting", + ) + comparison_report = None + if baseline: + comparison_report = compare_baseline( + records, output, baseline, args.slowdown_limit_pct + ) + (output / "status.json").write_text( + json.dumps( + { + "status": "passed", + "measurements": len(records), + "baseline_comparison": comparison_report, + }, + indent=2, + ) + + "\n" + ) + print(f"\nSaved {len(records)} measurements to {output}", flush=True) + except Exception as error: + (output / "status.json").write_text( + json.dumps( + { + "status": "failed", + "completed_measurements": len(records), + "error": str(error), + }, + indent=2, + ) + + "\n" + ) + raise + + +if __name__ == "__main__": + try: + main() + except (OSError, RuntimeError, ValueError) as error: + print(f"Memory experiment stopped: {error}", file=sys.stderr) + raise SystemExit(1) diff --git a/tests/cpp/test_clip_history.cpp b/tests/cpp/test_clip_history.cpp new file mode 100644 index 000000000..718836866 --- /dev/null +++ b/tests/cpp/test_clip_history.cpp @@ -0,0 +1,279 @@ +// SPDX-License-Identifier: MIT +// Persistent clip history: prefix/branch isolation, traversal, translation and +// nonrecursive destruction. Run through CTest; checks remain active in Release. +#include + +#include +#include + +namespace +{ +using namespace pdflib; + +void require(bool condition, const std::string& message) +{ + if(not condition) { throw std::runtime_error(message); } +} + +clip_path_instruction rectangle(int group, double x = 0.0) +{ + return clip_path_instruction({x,x+10,x+10,x,x}, {0,0,10,10,0}, CLOSED, RECTANGLE, group); +} + +struct storage_totals +{ + std::unordered_set nodes; + std::size_t paths = 0; + void add(const clip_state_instruction& state) + { + state.get_paths().visit_storage([&](const void* identity, std::size_t path_count, + std::size_t, std::size_t) + { + if(not nodes.insert(identity).second) { return false; } + paths += path_count; + return true; + }); + } +}; + +void snapshot_and_branch_isolation() +{ + clip_state_instruction empty; + const auto first = empty.appended(CLIP_RULE_NONZERO, {rectangle(0)}); + const auto left = first.appended(CLIP_RULE_NONZERO, {rectangle(1, 20)}); + const auto right = first.appended(CLIP_RULE_EVEN_ODD, {rectangle(1, 40), rectangle(1, 60)}); + require(empty.get_paths().empty() and first.get_paths().size()==1, + "appending changed an earlier snapshot"); + require(left.get_paths().size()==2 and right.get_paths().size()==3, + "branches did not retain separate histories"); + const auto values = right.get_paths().to_vector(); + require(values[0].get_x()[0]==0 and values[1].get_x()[0]==40 and values[2].get_x()[0]==60, + "forward traversal lost capture/subpath order"); + require(values[1].get_clip_group()==values[2].get_clip_group(), "capture grouping changed"); + require(first.get_rule()==CLIP_RULE_NONZERO and right.get_rule()==CLIP_RULE_EVEN_ODD, + "a later rule changed a previous snapshot"); + require(first.with_rule(CLIP_RULE_EVEN_ODD).shares_history(first), "rule copy cloned geometry"); + storage_totals totals; + for(const auto& state : {first, left, right}) { totals.add(state); } + require(totals.nodes.size()==3 and totals.paths==4, "branch prefixes were not shared"); +} + +std::vector snapshots(std::size_t count) +{ + std::vector result; + result.reserve(count); + clip_state_instruction current; + for(std::size_t i=0; i(i), double(i))}); + result.push_back(current); + } + return result; +} + +void linear_storage_and_forward_iterator() +{ + const auto states = snapshots(1000); + storage_totals totals; + std::size_t logical_paths = 0; + for(const auto& state : states) + { + logical_paths += state.get_paths().size(); + totals.add(state); + } + require(logical_paths==500500, "logical clip occurrences changed"); + require(totals.nodes.size()==1000 and totals.paths==1000, "unique storage is not linear"); + const auto& paths = states.back().get_paths(); + require(paths.point_count()==5000, "wrong cumulative point count"); + auto copy = paths.begin(); + auto advanced = copy; + ++advanced; + require(copy->get_clip_group()==0 and advanced->get_clip_group()==1, + "copying/advancing a forward iterator changed the original"); + int expected = 0; + for(const auto& path : paths) { require(path.get_clip_group()==expected++, "reversed history traversal"); } + require(expected==1000, "history traversal omitted a capture"); +} + +void require_cursor_matches(const clip_path_cursor& cursor, + const clip_state_instruction& state) +{ + const auto& paths = state.get_paths(); + require(cursor.size()==paths.size(), "cursor retained the wrong number of paths"); + auto expected = paths.begin(); + for(const auto& path : cursor) + { + require(expected!=paths.end() and &path==&*expected, + "cursor copied, reordered or retained stale geometry"); + ++expected; + } + require(expected==paths.end(), "cursor omitted a clip path"); +} + +void cursor_extends_history_and_reuses_geometry() +{ + clip_path_cursor cursor; + clip_state_instruction empty; + const auto first = empty.appended(CLIP_RULE_NONZERO, {rectangle(0)}); + const auto second = first.appended(CLIP_RULE_NONZERO, {rectangle(1,20), rectangle(1,30)}); + const auto third = second.appended(CLIP_RULE_NONZERO, {rectangle(2,40)}); + // A later capture can reuse an existing group (for example a shading BBox). + const auto fourth = third.appended(CLIP_RULE_NONZERO, {rectangle(0,50)}); + + cursor.prepare(empty.get_paths()); + require(cursor.empty() and cursor.group_size(0)==0, "empty cursor has geometry/counts"); + cursor.prepare(first.get_paths()); + require_cursor_matches(cursor, first); + cursor.prepare(second.get_paths()); + require_cursor_matches(cursor, second); + require(cursor.group_size(0)==1 and cursor.group_size(1)==2, + "single-capture extension lost subpath groups"); + // Skip the third snapshot: replay need not paint after every capture. + cursor.prepare(fourth.get_paths()); + require_cursor_matches(cursor, fourth); + require(cursor.group_size(0)==2 and cursor.group_size(1)==2 and cursor.group_size(2)==1, + "multi-capture extension did not update existing/new groups"); + + auto copy = cursor.begin(); + auto advanced = copy; + ++advanced; + require(copy->get_x()[0]==0 and advanced->get_x()[0]==20, + "cursor iterator copies are not independent"); + const auto rule_change = fourth.with_rule(CLIP_RULE_EVEN_ODD); + cursor.prepare(rule_change.get_paths()); + require_cursor_matches(cursor, fourth); + require(cursor.group_size(0)==2, "rule-only change invalidated geometry groups"); +} + +void cursor_restores_branches_and_handles_translation() +{ + clip_path_cursor cursor; + clip_state_instruction empty; + const auto root = empty.appended(CLIP_RULE_NONZERO, {rectangle(0)}); + const auto left = root.appended(CLIP_RULE_NONZERO, {rectangle(1,20)}); + const auto right = root.appended(CLIP_RULE_NONZERO, {rectangle(0,40), rectangle(0,60)}); + cursor.prepare(left.get_paths()); + require_cursor_matches(cursor, left); + cursor.prepare(root.get_paths()); + require_cursor_matches(cursor, root); + require(cursor.group_size(0)==1 and cursor.group_size(1)==0, + "restoring an ancestor retained child groups"); + cursor.prepare(left.get_paths()); + cursor.prepare(right.get_paths()); + require_cursor_matches(cursor, right); + require(cursor.group_size(0)==3 and cursor.group_size(1)==0, + "switching siblings retained the previous branch's groups"); + + const auto shifted = right.translated(10,20); + cursor.prepare(shifted.get_paths()); + require_cursor_matches(cursor, shifted); + require(cursor.begin()->get_x()[0]==10 and cursor.begin()->get_y()[0]==20, + "translated history reused unshifted coordinates"); + const clip_state_instruction independent(CLIP_RULE_NONZERO, {rectangle(9,100)}); + cursor.prepare(independent.get_paths()); + require_cursor_matches(cursor, independent); + require(cursor.group_size(9)==1 and cursor.group_size(0)==0, + "independent history retained old groups"); + cursor.prepare(empty.get_paths()); + require_cursor_matches(cursor, empty); + require(cursor.group_size(9)==0, "returning to an empty history retained group counts"); + cursor.prepare(right.get_paths()); + cursor.reset(); + require(cursor.empty() and cursor.group_size(0)==0, "page reset retained cursor state"); + cursor.prepare(left.get_paths()); + require_cursor_matches(cursor, left); +} + +void cursor_owns_referenced_history() +{ + clip_path_cursor cursor; + const clip_path_instruction* original = nullptr; + { + const clip_state_instruction temporary(CLIP_RULE_NONZERO, {rectangle(7,70)}); + original = &*temporary.get_paths().begin(); + cursor.prepare(temporary.get_paths()); + } + require(cursor.size()==1 and &*cursor.begin()==original and cursor.begin()->get_x()[0]==70, + "cursor references outlived their history owner"); + cursor.reset(); + require(cursor.empty() and cursor.begin()==cursor.end(), "reset left a traversable history"); +} + +void translated_prefixes_stay_shared() +{ + const auto source = snapshots(64); + clip_translation_cache cache; + std::vector translated; + storage_totals totals; + for(const auto& state : source) + { + translated.push_back(state.translated(10, 20, &cache)); + totals.add(translated.back()); + } + require(totals.nodes.size()==64 and totals.paths==64, "translation duplicated shared prefixes"); + require(source.front().get_paths().begin()->get_x()[0]==0, "translation changed source coordinates"); + require(translated.front().get_paths().begin()->get_x()[0]==10 and + translated.front().get_paths().begin()->get_y()[0]==20, "wrong translated coordinates"); + require(source.back().translated(10,20,&cache).shares_history(translated.back()), + "translation cache did not reuse the completed history"); + const auto shifted_again = source.front().translated(30,40,&cache); + require(shifted_again.get_paths().begin()->get_x()[0]==30 and + translated.front().get_paths().begin()->get_x()[0]==10, + "changing the translation reused stale coordinates"); +} + +struct recording_renderer +{ + std::vector clips; + void render_shape(const shape_instruction& shape) { clips.push_back(shape.get_clip_state()); } + void render_text(const text_instruction&) {} + void render_widget(const text_widget_instruction&) {} + void render_bitmap(const bitmap_instruction&) {} + void render_shading(const shading_instruction&) {} +}; + +void shifted_instruction_replay_uses_one_translation_cache() +{ + pdf_render_instructions instructions; + const auto states = snapshots(32); + for(const auto& state : states) + { + instructions.add_shape_instruction(shape_instruction({}, SHAPE_PAINT_FILL, + SHAPE_FILL_NONZERO, 1, 0, 0, 10, {}, 0, {0,0,0}, {0,0,0}, 1, 1, state)); + } + recording_renderer renderer; + instructions.iterate_over_content(renderer, 10, 20); + storage_totals totals; + for(const auto& state : renderer.clips) { totals.add(state); } + require(renderer.clips.size()==32 and totals.paths==32, + "shifted instruction replay rematerialized clip prefixes"); +} + +void deep_unique_suffix_destroys_iteratively() +{ + clip_state_instruction history; + for(int i=0; i<50000; ++i) { history = history.appended(CLIP_RULE_NONZERO, {rectangle(i)}); } + require(history.get_paths().size()==50000, "deep history was truncated"); + history = clip_state_instruction(); + require(not history.has_clip(), "deep history release failed"); +} +} + +int main() +{ + try + { + snapshot_and_branch_isolation(); + linear_storage_and_forward_iterator(); + cursor_extends_history_and_reuses_geometry(); + cursor_restores_branches_and_handles_translation(); + cursor_owns_referenced_history(); + translated_prefixes_stay_shared(); + shifted_instruction_replay_uses_one_translation_cache(); + deep_unique_suffix_destroys_iteratively(); + std::cout << "Shared clip history checks passed\n"; + } + catch(const std::exception& error) + { std::cerr << error.what() << "\n"; return 1; } + return 0; +} diff --git a/tests/cpp/test_memory_on_lines.cpp b/tests/cpp/test_memory_on_lines.cpp new file mode 100644 index 000000000..abf2b67a2 --- /dev/null +++ b/tests/cpp/test_memory_on_lines.cpp @@ -0,0 +1,693 @@ +// SPDX-License-Identifier: MIT +// Manual native memory experiment for issues #448 and #345 (macOS/Linux). +// Build: cmake -S . -B build -DDOCLING_PARSE_BUILD_MEMORY_PROBE=ON +// cmake --build build --target test_memory_on_lines -j2 +// Generate PDFs in a SEPARATE process so generation cannot affect peak RSS: +// test_memory_on_lines generate dash-each 8000 /tmp/lines.pdf +// test_memory_on_lines measure render /tmp/lines.pdf [pause_seconds] [out.png] +// Modes: qpdf-discard, collect (tokens is an alias), decode, render, skip-shapes. +// Raw QPDF modes select the generated Form's LEAF stream once; they do not run Do. +// Generate pathstorm-form N PDF [repeats=1] [depth=1] for Form fixtures. +// Each measurement needs a fresh process. RSS values are bytes; peak RSS is a +// process-lifetime high-water mark. +// Generation variants: solid, dash-each, dash-once, dash-q, empty-each, +// pathstorm (the grid generator from docling#4058, with its rounded line count). +// Generate pathstorm-appearance/pathstorm-widget N PDF [appearances=1]. +// Generate pathstorm-type3 N PDF [glyphs=1] for procedural CharProc delivery. +// Generate pathstorm-soft-mask N PDF [paints=1] for one cached mask group. +// Generate pathstorm-pattern[-lattice] N PDF [fills=1] for pattern replay. +// Generate clip-chain/clip-scoped N PDF (10..4000) for clip snapshot growth. +// page_* fields cover /Contents only. form_* diagnostics aggregate Do calls +// through that page's ordinary Form tree; appearances have separate counters. +#include +#include + +#include +#include +#include +#include + +#include + +#if defined(__APPLE__) +#include +#include +#elif defined(__linux__) +#include +#include +#else +#error "This manual memory probe currently supports macOS and Linux." +#endif + +namespace +{ +using json = nlohmann::json; +using clock_type = std::chrono::steady_clock; +const auto started = clock_type::now(); +constexpr const char* page_content_delivery = "callback"; +constexpr const char* form_content_delivery = "callback"; +constexpr const char* appearance_content_delivery = "callback"; +constexpr const char* type3_content_delivery = "callback"; +constexpr const char* soft_mask_content_delivery = "callback"; +constexpr const char* pattern_content_delivery = "borrowed"; +constexpr const char* clip_history_storage = "shared"; + +void snapshot(const std::string& stage, int pause_seconds, json details = json::object()) +{ + rusage usage{}; + if(getrusage(RUSAGE_SELF, &usage)!=0) + { throw std::runtime_error("getrusage failed"); } +#if defined(__APPLE__) + mach_task_basic_info_data_t info{}; + mach_msg_type_number_t count = MACH_TASK_BASIC_INFO_COUNT; + if(task_info(mach_task_self(), MACH_TASK_BASIC_INFO, + reinterpret_cast(&info), &count)!=KERN_SUCCESS) + { throw std::runtime_error("task_info failed"); } + details["rss_bytes"] = info.resident_size; + details["peak_rss_bytes"] = usage.ru_maxrss; +#else + std::ifstream statm("/proc/self/statm"); + std::size_t total_pages = 0, resident_pages = 0; + if(not (statm >> total_pages >> resident_pages)) + { throw std::runtime_error("cannot read /proc/self/statm"); } + details["rss_bytes"] = resident_pages * sysconf(_SC_PAGESIZE); + details["peak_rss_bytes"] = static_cast(usage.ru_maxrss) * 1024; +#endif + details["stage"] = stage; + details["elapsed_s"] = std::chrono::duration(clock_type::now()-started).count(); + std::cout << details.dump() << std::endl; + if(pause_seconds>0) { std::this_thread::sleep_for(std::chrono::seconds(pause_seconds)); } +} + +// Forward to the production collector only when requested. The discard case +// keeps scalar counters and no QPDF objects or per-object string representations. +class counting_callbacks : public QPDFObjectHandle::ParserCallbacks +{ +public: + explicit counting_callbacks(pdflib::qpdf_stream_decoder* collector_ = nullptr): + collector(collector_) {} + + using QPDFObjectHandle::ParserCallbacks::handleObject; + + void handleObject(QPDFObjectHandle obj, std::size_t offset, std::size_t length) override + { + ++tokens; + operators += obj.isOperator(); + if(collector) { collector->handleObject(obj, offset, length); } + } + + void contentSize(std::size_t size) override + { + decompressed_bytes += size; + ++content_size_calls; + } + + void handleEOF() override { eof_reached = true; } + + json statistics() const + { + return {{"tokens", tokens}, {"operators", operators}, + {"decompressed_bytes", decompressed_bytes}, + {"content_size_calls", content_size_calls}, {"eof_reached", eof_reached}}; + } + +private: + pdflib::qpdf_stream_decoder* collector; + std::size_t tokens = 0, operators = 0, decompressed_bytes = 0, content_size_calls = 0; + bool eof_reached = false; +}; + +// Select only our marked generated Form chain or first appearance source. +// Arbitrary PDFs keep first-page /Contents; this is not a general Do/AP executor. +QPDFObjectHandle selected_content(QPDFObjectHandle page, json& details) +{ + auto content = page.getKey("/Contents"); + details["measured_stream"] = "page"; + details["stream_executions"] = 1; + auto pattern_fills = page.getKey("/MemoryProbePatternFills"); + if(not pattern_fills.isNull()) + { + if(not pattern_fills.isInteger() or pattern_fills.getIntValue()<1) + { throw std::runtime_error("invalid generated pattern fill count"); } + auto cell = page.getKey("/Resources").getKey("/Pattern").getKey("/P0"); + if(not cell.isStream()) + { throw std::runtime_error("generated pattern cell is missing"); } + details["measured_stream"] = "pattern_cell"; + details["page_pattern_fills"] = pattern_fills.getIntValue(); + return cell; + } + auto mask_paints = page.getKey("/MemoryProbeSoftMaskPaints"); + if(not mask_paints.isNull()) + { + if(not mask_paints.isInteger() or mask_paints.getIntValue()<1) + { throw std::runtime_error("invalid generated soft-mask paint count"); } + auto group = page.getKey("/Resources").getKey("/ExtGState") + .getKey("/GS").getKey("/SMask").getKey("/G"); + if(not group.isStream()) + { throw std::runtime_error("generated soft-mask group is missing"); } + details["measured_stream"] = "soft_mask_group"; + details["page_soft_mask_paints"] = mask_paints.getIntValue(); + return group; + } + auto type3_repeats = page.getKey("/MemoryProbeType3Repeats"); + if(not type3_repeats.isNull()) + { + if(not type3_repeats.isInteger() or type3_repeats.getIntValue()<1) + { throw std::runtime_error("invalid generated Type3 repeat count"); } + auto font = page.getKey("/Resources").getKey("/Font").getKey("/T3"); + auto charproc = font.getKey("/CharProcs").getKey("/g"); + if(not charproc.isStream()) + { throw std::runtime_error("generated CharProc is missing"); } + details["measured_stream"] = "type3_charproc"; + details["page_type3_repeats"] = type3_repeats.getIntValue(); + return charproc; + } + auto appearance_count = page.getKey("/MemoryProbeAppearanceCount"); + if(not appearance_count.isNull()) + { + if(not appearance_count.isInteger() or appearance_count.getIntValue()<1) + { throw std::runtime_error("invalid generated appearance count"); } + auto annots = page.getKey("/Annots"); + if(not annots.isArray() or annots.getArrayNItems()!=appearance_count.getIntValue()) + { throw std::runtime_error("generated annotation count differs"); } + auto appearance = annots.getArrayItem(0).getKey("/AP").getKey("/N"); + if(not appearance.isStream()) + { throw std::runtime_error("generated appearance source is missing"); } + details["measured_stream"] = "appearance"; + details["page_appearance_count"] = appearance_count.getIntValue(); + return appearance; + } + auto depth_obj = page.getKey("/MemoryProbeFormDepth"); + if(depth_obj.isNull()) { return content; } + if(not depth_obj.isInteger()) + { throw std::runtime_error("invalid generated Form depth"); } + const auto depth = depth_obj.getIntValue(); + if(depth<1 or depth>32) + { throw std::runtime_error("generated Form depth must be 1..32"); } + details["measured_stream"] = "form_leaf"; + details["form_depth"] = depth; + auto repeats = page.getKey("/MemoryProbeFormRepeats"); + if(repeats.isInteger()) { details["page_form_repeats"] = repeats.getIntValue(); } + + auto owner = page; + for(long long level=0; levelstd::numeric_limits::max()) + { throw std::invalid_argument("count is out of range"); } + return static_cast(value); +} + +std::string stream_object(const std::string& content, bool compress, + const std::string& extra = "") +{ + std::string payload = content; + if(compress) + { + uLongf size = compressBound(content.size()); + payload.resize(size); + if(compress2(reinterpret_cast(payload.data()), &size, + reinterpret_cast(content.data()), + content.size(), Z_DEFAULT_COMPRESSION)!=Z_OK) + { throw std::runtime_error("PDF compression failed"); } + payload.resize(size); + } + return "<< /Length " + std::to_string(payload.size()) + extra + + (compress ? " /Filter /FlateDecode" : "") + " >>\nstream\n" + payload + "\nendstream"; +} + +void generate(const std::string& variant, std::size_t requested, + const std::string& filename, std::size_t repeats, std::size_t depth) +{ + if(requested<10 or requested>static_cast(std::numeric_limits::max())) + { throw std::invalid_argument("line count must be 10..INT_MAX"); } + const bool form = variant=="pathstorm-form"; + const bool widget = variant=="pathstorm-widget"; + const bool appearance = variant=="pathstorm-appearance" or widget; + const bool type3 = variant=="pathstorm-type3"; + const bool soft_mask = variant=="pathstorm-soft-mask"; + const bool lattice = variant=="pathstorm-pattern-lattice"; + const bool pattern = variant=="pathstorm-pattern" or lattice; + const bool clip_chain = variant=="clip-chain"; + const bool clip_scoped = variant=="clip-scoped"; + const bool clipping = clip_chain or clip_scoped; + if(clipping and requested>4000) + { throw std::invalid_argument("clip fixtures require at most 4000 lines"); } + if(depth<1 or depth>32) + { throw std::invalid_argument("Form depth must be 1..32"); } + if((not form and not appearance and not type3 and not soft_mask and not pattern and repeats!=1) or (not form and depth!=1)) + { throw std::invalid_argument("repeats require Forms/appearances/Type3/masks/patterns; depth requires Forms"); } + if(variant!="solid" and variant!="dash-each" and variant!="dash-once" and + variant!="dash-q" and variant!="empty-each" and variant!="pathstorm" and not form and not appearance and not type3 and not soft_mask and not pattern and not clipping) + { throw std::invalid_argument("unknown PDF variant"); } + + std::ostringstream ops; + std::size_t lines = 0; + const bool grid = variant=="pathstorm" or form or appearance or type3 or soft_mask or pattern; + if(grid) + { + // Match the published generator, including the rounding of each grid. + const int w = 595, h = 842; + const int rows = static_cast(std::sqrt(requested/2.0) * std::sqrt(double(h)/w)); + const int per_row = (requested/2)/rows; + const double dash = double(w)/per_row/1.6; + ops << "0.4 w\n" << std::fixed << std::setprecision(1); + for(int r=0; rstd::numeric_limits::max()/repeats) + { throw std::overflow_error("total painted line count is out of range"); } + const std::string bbox = grid ? "595 842" : "1190 842"; + std::string resources = "<< >>", marker; + std::string catalog_fields; + std::string page_content = content; + if(form) + { + resources = "<< /XObject << /Fm0 5 0 R >> >>"; + marker = " /MemoryProbeFormDepth " + std::to_string(depth) + + " /MemoryProbeFormRepeats " + std::to_string(repeats); + page_content.clear(); + for(std::size_t i=0; i>"; } + } + if(type3) + { + resources = "<< /Font << /T3 5 0 R >> >>"; + marker = " /MemoryProbeType3Repeats " + std::to_string(repeats); + page_content = "BT /T3 1 Tf\n"; + for(std::size_t i=0; i objects = { + "<< /Type /Catalog /Pages 2 0 R" + catalog_fields + " >>", + "<< /Type /Pages /Kids [3 0 R] /Count 1 >>", + "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 " + bbox + + "] /Resources " + resources + marker + " /Contents 4 0 R >>", + stream_object(page_content, grid) + }; + if(form) + { + for(std::size_t level=0; level>" : + "<< /XObject << /Fm0 " + std::to_string(6+level) + " 0 R >> >>"; + const std::string extra = " /Type /XObject /Subtype /Form /FormType 1" + " /BBox [0 0 " + bbox + "] /Matrix [1 0 0 1 0 0] /Resources " + local_resources; + objects.push_back(stream_object(leaf ? content : "q /Fm0 Do Q\n", true, extra)); + } + } + if(appearance) + { + objects.push_back(stream_object(content, true, + " /Type /XObject /Subtype /Form /BBox [0 0 " + bbox + + "] /Matrix [1 0 0 1 0 0] /Resources << >>")); + for(std::size_t i=0; i>" + + (widget ? " /FT /Btn /T (probe" + std::to_string(i) + ")" : "") + " >>"); + } + } + if(type3) + { + objects.push_back("<< /Type /Font /Subtype /Type3 /FontBBox [0 0 " + bbox + + "] /FontMatrix [1 0 0 1 0 0] /CharProcs 6 0 R" + " /Encoding << /Type /Encoding /Differences [97 /g] >>" + " /FirstChar 97 /LastChar 97 /Widths [595] /Resources << >> >>"); + objects.push_back("<< /g 7 0 R >>"); + objects.push_back(stream_object(content, true)); + } + if(soft_mask) + { + objects.push_back(stream_object(content, true, + " /Type /XObject /Subtype /Form /BBox [0 0 " + bbox + + "] /Resources << >> /Group << /S /Transparency /CS /DeviceGray >>")); + } + if(pattern) + { + // A page-sized step uses the decoded-tile cache. Doubling both steps + // selects lattice replay at indices [0,1] x [0,1]: four executions. + const std::string pattern_box = lattice ? "1190 1684" : "595 842"; + objects.push_back(stream_object(content, true, + " /Type /Pattern /PatternType 1 /PaintType 1 /TilingType 1" + " /BBox [0 0 " + pattern_box + "] /XStep " + (lattice ? "1190" : "595") + + " /YStep " + (lattice ? "1684" : "842") + " /Resources << >>")); + } + std::ofstream out(filename, std::ios::binary); + out.exceptions(std::ios::failbit | std::ios::badbit); + out << "%PDF-1.4\n"; + std::vector offsets; + for(std::size_t i=0; i>\nstartxref\n" << xref << "\n%%EOF\n"; + out.close(); + std::cout << json{{"variant", variant}, {"requested_lines", requested}, + {"unique_lines", lines}, {"lines", lines*repeats}, + {"form_repeats", form ? repeats : 1}, {"form_depth", form ? depth : 0}, + {"appearance_count", appearance ? repeats : 0}, + {"widget_count", widget ? repeats : 0}, + {"type3_repeats", type3 ? repeats : 0}, + {"soft_mask_paints", soft_mask ? repeats : 0}, + {"pattern_fills", pattern ? repeats : 0}, + {"pattern_lattice_cells_per_fill", lattice ? 4 : 0}, + {"clip_mode", clipping ? (clip_chain ? "chain" : "scoped") : "none"}, + {"page_content_bytes", page_content.size()}, + {"content_bytes", content.size()}, {"pdf_bytes", std::filesystem::file_size(filename)}}.dump() + << std::endl; +} + +void measure(std::string mode, std::string filename, int pause_seconds, const std::string& png) +{ + const std::string requested_mode = mode; + if(mode=="tokens") { mode = "collect"; } + if(mode!="qpdf-discard" and mode!="collect" and mode!="decode" and + mode!="render" and mode!="skip-shapes") + { throw std::invalid_argument("unknown measurement mode"); } + snapshot("baseline", pause_seconds, + {{"mode", mode}, {"requested_mode", requested_mode}, {"file", filename}, + {"qpdf_version", QPDF::QPDFVersion()}, + {"page_content_delivery", page_content_delivery}, + {"form_content_delivery", form_content_delivery}, + {"appearance_content_delivery", appearance_content_delivery}, + {"type3_content_delivery", type3_content_delivery}, + {"soft_mask_content_delivery", soft_mask_content_delivery}, + {"pattern_content_delivery", pattern_content_delivery}, + {"clip_history_storage", clip_history_storage}, + {"sizeof_token", sizeof(pdflib::qpdf_stream_instruction)}, + {"sizeof_page_shape", sizeof(pdflib::page_item)}, + {"sizeof_shape_instruction", sizeof(pdflib::shape_instruction)}, + {"sizeof_subpath", sizeof(pdflib::shape_subpath)}}); + if(mode=="qpdf-discard" or mode=="collect") + { + QPDF pdf; + pdf.processFile(filename.c_str()); + auto pages = pdf.getAllPages(); + json selected = json::object(); + auto content = selected_content(pages.at(0), selected); + snapshot("loaded", pause_seconds, selected); + if(mode=="qpdf-discard") + { + counting_callbacks callbacks; + QPDFObjectHandle::parseContentStream(content, &callbacks); + auto details = callbacks.statistics(); + details.update(selected); + details["collector"] = "count_and_discard"; + details["retained_tokens"] = 0; + details["token_capacity"] = 0; + details["token_inline_bytes"] = 0; + snapshot("tokenized", pause_seconds, details); + } + else + { + std::vector tokens; + pdflib::qpdf_stream_decoder collector(tokens); + counting_callbacks callbacks(&collector); + // Use the same QPDF entry point as discard. A raw-token comparison + // fails on parse errors; production damaged-stream recovery is unchanged. + QPDFObjectHandle::parseContentStream(content, &callbacks); + auto details = callbacks.statistics(); + details.update(selected); + details["collector"] = "docling_collect"; + details["retained_tokens"] = tokens.size(); + details["token_capacity"] = tokens.capacity(); + details["token_inline_bytes"] = tokens.capacity()*sizeof(tokens[0]); + snapshot("tokenized", pause_seconds, details); + } + return; + } + + json font_config = {{pdflib::pdf_resource::RESOURCE_DIR_KEY, + resource_utils::get_resources_dir(true).string()}}; + std::unordered_map font_timings; + pdflib::pdf_resource::initialise(font_config, font_timings); + pdflib::pdf_decoder document; + std::optional password; + if(not document.process_document_from_file(filename, password)) + { throw std::runtime_error(document.get_load_error_message()); } + snapshot("loaded", pause_seconds); + pdflib::decode_config config; + config.do_thread_safe = false; + config.keep_shapes = mode!="skip-shapes"; + // Preserve other defaults, including sanitization. There are no text/images. + const auto decode_started = clock_type::now(); + auto page = document.decode_page(0, config); + const double decode_work_s = std::chrono::duration(clock_type::now()-decode_started).count(); + if(not page) { throw std::runtime_error("page decode failed"); } + const auto& content_stats = page->get_content_statistics(); + const auto& appearance_stats = page->get_appearance_statistics(); + const auto& soft_mask_stats = page->get_soft_mask_statistics(); + const auto& pattern_stats = page->get_pattern_statistics(); + const auto& shapes = page->get_instructions().get_shape_instructions(); + std::size_t dash_entries = 0, max_dash = 0, points = 0, subpaths = 0, segments = 0; + std::size_t geometry_bytes = 0, page_dash_entries = 0; + std::size_t clip_paths = 0, max_clip_paths = 0, clip_points = 0, clip_bytes = 0; + // Physical accounting deduplicates history nodes and their geometry; it + // excludes allocator overhead and shared_ptr control blocks. Logical totals + // still count each path occurrence visible to each instruction. + std::size_t logical_clip_bytes = 0, unique_clip_paths = 0, unique_clip_points = 0; + std::unordered_set clip_nodes; + for(const auto& shape : shapes) + { + dash_entries += shape.get_dash_array().size(); + max_dash = std::max(max_dash, shape.get_dash_array().size()); + const auto& clips = shape.get_clip_state().get_paths(); + clip_paths += clips.size(); + max_clip_paths = std::max(max_clip_paths, clips.size()); + clip_points += clips.point_count(); + logical_clip_bytes += clips.logical_bytes(); + clips.visit_storage([&](const void* identity, std::size_t path_count, + std::size_t point_count, std::size_t storage_bytes) + { + if(not clip_nodes.insert(identity).second) { return false; } + clip_bytes += storage_bytes; + unique_clip_paths += path_count; + unique_clip_points += point_count; + return true; + }); + points += shape.size(); + subpaths += shape.get_subpaths().size(); + geometry_bytes += shape.get_subpaths().capacity()*sizeof(pdflib::shape_subpath); + for(const auto& path : shape.get_subpaths()) + { + segments += path.get_ops().size(); + geometry_bytes += path.get_ops().capacity()*sizeof(pdflib::shape_segment_op) + + (path.get_px().capacity()+path.get_py().capacity())*sizeof(double); + } + } + for(const auto& shape : page->get_page_shapes()) { page_dash_entries += shape.get_dash_array().size(); } + snapshot("decoded", pause_seconds, + {{"decode_work_s", decode_work_s}, + {"page_shapes", page->get_page_shapes().size()}, {"shape_instructions", shapes.size()}, + {"instruction_capacity", shapes.capacity()}, {"subpaths", subpaths}, + {"points", points}, {"segments", segments}, + {"instruction_inline_bytes", shapes.capacity()*sizeof(pdflib::shape_instruction)}, + {"instruction_geometry_bytes", geometry_bytes}, + {"instruction_dash_entries", dash_entries}, {"max_dash_entries", max_dash}, + {"page_shape_dash_entries", page_dash_entries}, + {"stored_dash_bytes", (dash_entries+page_dash_entries)*sizeof(double)}, + {"instruction_clip_paths", clip_paths}, {"max_clip_paths", max_clip_paths}, + {"instruction_clip_points", clip_points}, {"instruction_clip_bytes", clip_bytes}, + {"clip_history_storage", clip_history_storage}, + {"instruction_clip_logical_bytes", logical_clip_bytes}, + {"instruction_clip_history_nodes", clip_nodes.size()}, + {"instruction_clip_unique_paths", unique_clip_paths}, + {"instruction_clip_unique_points", unique_clip_points}, + {"page_content_delivery", content_stats.callback_delivery ? "callback" : "buffered"}, + {"page_source_objects", content_stats.source_objects}, + {"page_source_bytes_observed", content_stats.source_bytes_observed}, + {"page_buffered_tokens", content_stats.buffered_tokens}, + {"page_buffered_token_capacity", content_stats.buffered_token_capacity}, + {"page_operand_diagnostics", content_stats.operand_diagnostics_enabled}, + {"page_max_pending_operands", content_stats.max_pending_operands}, + {"form_content_delivery", form_content_delivery}, + {"form_diagnostics", content_stats.operand_diagnostics_enabled}, + {"form_executions", content_stats.form_executions}, + {"form_source_objects", content_stats.form_source_objects}, + {"form_materialized_tokens", content_stats.form_materialized_tokens}, + {"form_max_pending_operands", content_stats.form_max_pending_operands}, + {"form_max_depth", content_stats.form_max_depth}, + {"appearance_content_delivery", appearance_content_delivery}, + {"appearance_diagnostics", content_stats.operand_diagnostics_enabled}, + {"appearance_executions", appearance_stats.executions}, + {"appearance_widget_executions", appearance_stats.widget_executions}, + {"appearance_source_objects", appearance_stats.source_objects}, + {"appearance_source_bytes_observed", appearance_stats.source_bytes_observed}, + {"appearance_materialized_tokens", appearance_stats.materialized_tokens}, + {"appearance_max_buffered_token_capacity", appearance_stats.max_buffered_token_capacity}, + {"appearance_max_pending_operands", appearance_stats.max_pending_operands}, + {"text_instructions", page->get_instructions().get_text_instructions().size()}, + {"type3_content_delivery", type3_content_delivery}, + {"type3_diagnostics", content_stats.operand_diagnostics_enabled}, + {"type3_executions", content_stats.type3_executions}, + {"type3_source_objects", content_stats.type3_source_objects}, + {"type3_materialized_tokens", content_stats.type3_materialized_tokens}, + {"type3_max_pending_operands", content_stats.type3_max_pending_operands}, + {"type3_max_depth", content_stats.type3_max_depth}, + {"type3_recursion_skips", content_stats.type3_recursion_skips}, + {"soft_mask_content_delivery", soft_mask_content_delivery}, + {"soft_mask_diagnostics", content_stats.operand_diagnostics_enabled}, + {"soft_mask_executions", soft_mask_stats.executions}, + {"soft_mask_cache_hits", soft_mask_stats.cache_hits}, + {"soft_mask_source_objects", soft_mask_stats.source_objects}, + {"soft_mask_source_bytes_observed", soft_mask_stats.source_bytes_observed}, + {"soft_mask_materialized_tokens", soft_mask_stats.materialized_tokens}, + {"soft_mask_max_buffered_token_capacity", soft_mask_stats.max_buffered_token_capacity}, + {"soft_mask_max_pending_operands", soft_mask_stats.max_pending_operands}, + {"soft_mask_shape_instructions", soft_mask_stats.shape_instructions}, + {"pattern_content_delivery", pattern_content_delivery}, + {"pattern_diagnostics", content_stats.operand_diagnostics_enabled}, + {"pattern_source_parses", pattern_stats.source_parses}, + {"pattern_source_cache_hits", pattern_stats.source_cache_hits}, + {"pattern_cached_tokens", pattern_stats.cached_tokens}, + {"pattern_cached_token_capacity", pattern_stats.cached_token_capacity}, + {"pattern_decoded_cells", pattern_stats.decoded_cells}, + {"pattern_decoded_cache_hits", pattern_stats.decoded_cache_hits}, + {"pattern_decoded_shape_instructions", pattern_stats.decoded_shape_instructions}, + {"pattern_lattice_executions", pattern_stats.lattice_executions}, + {"pattern_replay_source_objects", pattern_stats.replay_source_objects}, + {"pattern_replay_materialized_tokens", pattern_stats.replay_materialized_tokens}, + {"pattern_max_replay_token_capacity", pattern_stats.max_replay_token_capacity}, + {"pattern_max_pending_operands", pattern_stats.max_pending_operands}}); + if(mode=="render") + { + const auto render_started = clock_type::now(); + pdflib::render_config render_config; + render_config.scale = 1.0; + pdflib::renderer renderer(render_config); + page->get_instructions().iterate_over_instructions(renderer); + // get_canvas flushes pending work and retrieves an independent pixel buffer. + const auto canvas = renderer.get_canvas(); + const double render_work_s = std::chrono::duration(clock_type::now()-render_started).count(); + snapshot("rendered", pause_seconds, {{"render_work_s", render_work_s}}); + if(not png.empty()) { renderer.save(png); } + } +} +} + +int main(int argc, char** argv) +{ + try + { + loguru::g_stderr_verbosity = loguru::Verbosity_ERROR; + if(argc>=5 and argc<=7 and std::string(argv[1])=="generate") + { + const auto count = positive_count(argv[3]); + const auto repeats = argc>=6 ? positive_count(argv[5]) : 1; + const auto depth = argc>=7 ? positive_count(argv[6]) : 1; + if(argc>6 and std::string(argv[2])!="pathstorm-form") + { throw std::invalid_argument("depth argument requires pathstorm-form"); } + generate(argv[2], count, argv[4], repeats, depth); + } + else if(argc>=4 and argc<=6 and std::string(argv[1])=="measure") + { + const std::string pause_text = argc>=5 ? argv[4] : "0"; + if(pause_text.empty() or pause_text.find_first_not_of("0123456789")!=std::string::npos) + { throw std::invalid_argument("pause must be 0..60 seconds"); } + const int pause = std::stoi(pause_text); + if(pause<0 or pause>60) { throw std::invalid_argument("pause must be 0..60 seconds"); } + measure(argv[2], argv[3], pause, argc==6 ? argv[5] : ""); + } + else + { + std::cerr << "Usage: " << argv[0] << " generate VARIANT N PDF [repeats] [form_depth]\n" + << " " << argv[0] << " measure {qpdf-discard|collect|tokens|decode|render|skip-shapes} PDF [pause_seconds] [out.png]\n"; + return 2; + } + } + catch(const std::exception& e) + { + std::cerr << e.what() << "\n"; + return 1; + } + return 0; +} diff --git a/tests/cpp/test_stream_callbacks.cpp b/tests/cpp/test_stream_callbacks.cpp new file mode 100644 index 000000000..f43df943b --- /dev/null +++ b/tests/cpp/test_stream_callbacks.cpp @@ -0,0 +1,203 @@ +// SPDX-License-Identifier: MIT +// Native callback/recovery regressions. Run through CTest after building the +// optional memory-probe targets. These checks remain active in Release builds. +#include + +#include +#include +#include +#include + +namespace +{ +void require(bool condition, const std::string& message) +{ + if(not condition) { throw std::runtime_error(message); } +} + +struct consumer_error : std::runtime_error +{ + consumer_error(): std::runtime_error("consumer failure marker") {} +}; + +struct recording_consumer +{ + std::vector values; + std::size_t calls = 0, inline_images = 0; + std::size_t throw_after = std::numeric_limits::max(); + bool exhaust_memory = false; + + static void consume(void* opaque, pdflib::qpdf_stream_instruction& row) + { + auto& self = *static_cast(opaque); + ++self.calls; + if(self.values.size()==self.throw_after) + { + if(self.exhaust_memory) { throw std::bad_alloc(); } + throw consumer_error(); + } + self.inline_images += row.obj.isInlineImage(); + self.values.push_back(row.key + ":" + row.val); + } +}; + +QPDFObjectHandle flate_stream(QPDF& pdf, const std::string& content) +{ + uLongf size = compressBound(content.size()); + std::string compressed(size, '\0'); + require(compress2(reinterpret_cast(compressed.data()), &size, + reinterpret_cast(content.data()), + content.size(), Z_DEFAULT_COMPRESSION)==Z_OK, "compression failed"); + compressed.resize(size); + auto stream = pdf.newStream(""); + stream.replaceStreamData(compressed, QPDFObjectHandle::newName("/FlateDecode"), + QPDFObjectHandle::newNull()); + return stream; +} + +QPDFObjectHandle broken_stream(QPDF& pdf) +{ + auto stream = pdf.newStream(""); + stream.replaceStreamData("this is not zlib data", + QPDFObjectHandle::newName("/FlateDecode"), + QPDFObjectHandle::newNull()); + return stream; +} + +QPDFObjectHandle broken_flate_with_prefix(QPDF& pdf, const std::string& content) +{ + // Sync-flush a valid non-final prefix, then append an invalid DEFLATE block + // type. QPDF's all-at-once decode fails; prefix recovery can still read it. + std::string compressed(compressBound(content.size())+64, '\0'); + z_stream encoder{}; + require(deflateInit(&encoder, Z_DEFAULT_COMPRESSION)==Z_OK, "deflateInit failed"); + encoder.next_in = reinterpret_cast(const_cast(content.data())); + encoder.avail_in = static_cast(content.size()); + encoder.next_out = reinterpret_cast(compressed.data()); + encoder.avail_out = static_cast(compressed.size()); + const int result = deflate(&encoder, Z_SYNC_FLUSH); + const auto size = encoder.total_out; + const bool complete = encoder.avail_in==0 and encoder.avail_out>0; + deflateEnd(&encoder); + require(result==Z_OK and complete, "prefix compression failed"); + compressed.resize(size); + compressed.push_back('\x07'); // BFINAL=1, BTYPE=3 (reserved/invalid). + auto stream = pdf.newStream(""); + stream.replaceStreamData(compressed, QPDFObjectHandle::newName("/FlateDecode"), + QPDFObjectHandle::newNull()); + return stream; +} + +void compare_deliveries(QPDF& pdf, const std::string& content, bool has_inline_image) +{ + auto input = pdf.newStream(content); + std::vector buffered; + pdflib::qpdf_stream_decoder collector(buffered); + collector.decode(input); + recording_consumer observed; + pdflib::qpdf_stream_decoder immediate(recording_consumer::consume, &observed); + immediate.decode(input); + + require(buffered.size()==observed.values.size(), "delivery counts differ"); + for(std::size_t i=0; i0)==has_inline_image, "inline image token was lost"); +} + +void consumer_failure_propagates(QPDF& pdf, std::size_t prefix, bool bad_alloc, + int source_case) +{ + auto input = flate_stream(pdf, "11 22 m 33 44 l S\n"); + if(source_case==1) + { + input = QPDFObjectHandle::newArray(std::vector{ + pdf.newStream("11 22 m\n"), broken_stream(pdf), pdf.newStream("33 44 l S\n") + }); + } + else if(source_case==2) + { + input = broken_flate_with_prefix(pdf, "11 22 m 33 44 l S\n"); + } + recording_consumer observed; + observed.throw_after = prefix; + observed.exhaust_memory = bad_alloc; + pdflib::qpdf_stream_decoder decoder(recording_consumer::consume, &observed); + bool propagated = false; + try + { + decoder.decode(input); + } + catch(const consumer_error&) + { + require(not bad_alloc, "allocation failure changed exception type"); + propagated = true; + } + catch(const std::bad_alloc&) + { + require(bad_alloc, "consumer failure changed exception type"); + propagated = true; + } + require(propagated, "consumer failure was swallowed"); + require(observed.calls==prefix+1, "consumer prefix was replayed"); + require(observed.values.size()==prefix, "unexpected output after consumer failure"); + require(decoder.get_objects_delivered()==prefix, "failed delivery counted as successful"); + + // Reusing a decoder must not retain the exception from an earlier consumer. + observed.throw_after = std::numeric_limits::max(); + observed.exhaust_memory = false; + auto broken = broken_stream(pdf); + decoder.decode(broken); + require(decoder.get_objects_delivered()==0, "broken stream delivered spurious objects"); +} + +void damaged_array_keeps_each_good_stream_once(QPDF& pdf) +{ + auto before = pdf.newStream("11 22 m\n"); + auto after = pdf.newStream("33 44 l S\n"); + auto broken = broken_stream(pdf); + auto contents = QPDFObjectHandle::newArray(std::vector{before, broken, after}); + recording_consumer observed; + pdflib::qpdf_stream_decoder decoder(recording_consumer::consume, &observed); + decoder.decode(contents); + const std::vector expected = { + "integer:11", "integer:22", "operator:m", "integer:33", + "integer:44", "operator:l", "operator:S" + }; + require(observed.values==expected, "damaged array dropped/replayed a good prefix or suffix"); + require(decoder.get_objects_delivered()==expected.size(), "wrong fallback progress count"); +} +} + +int main() +{ + try + { + loguru::g_stderr_verbosity = loguru::Verbosity_FATAL; + QPDF pdf; + pdf.emptyPDF(); + pdf.setSuppressWarnings(true); + compare_deliveries(pdf, "null [18 2 3 2] 0 d 11 22 m 33 44 l S\n", false); + compare_deliveries(pdf, "BI /W 1 /H 1 /CS /G /BPC 8 ID x EI\n", true); + for(std::size_t prefix : {std::size_t(0), std::size_t(3)}) + { + for(int source_case=0; source_case<3; ++source_case) + { + consumer_failure_propagates(pdf, prefix, false, source_case); + consumer_failure_propagates(pdf, prefix, true, source_case); + } + } + damaged_array_keeps_each_good_stream_once(pdf); + std::cout << "Callback delivery/recovery checks passed\n"; + } + catch(const std::exception& error) + { + std::cerr << error.what() << "\n"; + return 1; + } + return 0; +} diff --git a/tests/cpp/test_stream_state_lifetimes.cpp b/tests/cpp/test_stream_state_lifetimes.cpp new file mode 100644 index 000000000..bc897a072 --- /dev/null +++ b/tests/cpp/test_stream_state_lifetimes.cpp @@ -0,0 +1,280 @@ +// SPDX-License-Identifier: MIT +// Deterministic native exception/recovery checks; no actual memory exhaustion. +// Fault checkpoints exist only in this dedicated target, never in the extension +// or memory probe. Assertions remain active in Release builds. +#include + +#include +#include +#include + +namespace pdflib +{ +struct stream_state_lifetime_test_access +{ + using decoder = pdf_decoder; + enum class failure { recoverable, allocation, nonstandard }; + struct failure_marker {}; + inline static decoder* root = nullptr; + inline static failure kind = failure::recoverable; + inline static bool fired = false; + + static void throw_failure() + { + fired = true; + switch(kind) + { + case failure::recoverable: throw std::runtime_error("nested failure marker"); + case failure::allocation: throw std::bad_alloc(); + case failure::nonstandard: throw failure_marker{}; + } + } + + static void child_checkpoint(decoder& stream) + { + if(&stream==root) { return; } + decoder::interpretation_test_hook = nullptr; + throw_failure(); + } + + static void parse_checkpoint(std::vector& partial) + { + pdf_resource::parse_test_hook = nullptr; + auto token = QPDFObjectHandle::newInteger(777); + partial.emplace_back(token); + throw_failure(); + } + + static void inject(decoder& stream, failure selected) + { + root = &stream; + kind = selected; + fired = false; + decoder::interpretation_test_hook = child_checkpoint; + } + + static void inject_parse(failure selected) + { + kind = selected; + fired = false; + pdf_resource::parse_test_hook = parse_checkpoint; + } + + static void reset() + { + decoder::interpretation_test_hook = nullptr; + pdf_resource::parse_test_hook = nullptr; + root = nullptr; + } + + static std::size_t frames(const decoder& stream) { return stream.stack.size(); } + static int pattern_depth() { return decoder::tiling_depth(); } + static std::array matrix(decoder& stream) + { return stream.current_global_state().trafo_matrix; } + static bool paints_glyph(decoder& stream) + { return stream.current_bitmap_state().paints_type3_glyph; } + static bool cache_empty(const pdf_resource& pattern) + { return pattern.stream_cache_.empty() and pattern.stream_cache_.capacity()==0; } + static void glyph(decoder& stream, const pdf_state::type3_procedure& proc) + { stream.do_type3_glyph(proc); } +}; +} + +namespace +{ +using access = pdflib::stream_state_lifetime_test_access; +using failure = access::failure; + +void require(bool condition, const std::string& message) +{ + if(not condition) { throw std::runtime_error(message); } +} + +QPDFObjectHandle decorated_stream(QPDF& pdf, const std::string& content, + const std::string& dictionary) +{ + auto stream = pdf.newStream(content); + auto extra = QPDFObjectHandle::parse(dictionary); + auto dict = stream.getDict(); + for(const auto& key : extra.getKeys()) { dict.replaceKey(key, extra.getKey(key)); } + return stream; +} + +struct fixture +{ + QPDF pdf; + pdflib::decode_config config; + pdflib::page_item dimension; + pdflib::page_item cells; + pdflib::page_item shapes; + pdflib::page_item images; + std::shared_ptr> fonts = + std::make_shared>(); + std::shared_ptr> grphs = + std::make_shared>(); + std::shared_ptr> colorspaces = + std::make_shared>(); + std::shared_ptr> shadings = + std::make_shared>(); + std::shared_ptr> patterns = + std::make_shared>(); + std::shared_ptr> xobjects = + std::make_shared>(); + pdflib::pdf_render_instructions instructions; + pdflib::pdf_timings timings; + pdflib::pdf_decoder stream; + + fixture(): stream(config, dimension, cells, shapes, images, fonts, grphs, + colorspaces, shadings, patterns, xobjects, instructions, timings) + { + pdf.emptyPDF(); + pdf.setSuppressWarnings(true); + config.do_thread_safe = false; + dimension.execute(QPDFObjectHandle::parse("<< /MediaBox [0 0 200 200] >>")); + dimension.set_page_boundaries("crop_box"); + run("n\n"); + } + ~fixture() { access::reset(); } + + void run(const std::string& content) + { + auto source = pdf.newStream(content); + std::vector parameters; + stream.decode_and_interprete(source, parameters); + } + + void pattern(bool lattice) + { + const std::string step = lattice ? "300" : "20"; + auto cell = decorated_stream(pdf, "11 22 m 33 44 l S\n", + "<< /Type /Pattern /PatternType 1 /PaintType 1 /TilingType 1 /BBox [0 0 " + + step + " " + step + "] /XStep " + step + " /YStep " + step + " >>"); + auto resources = QPDFObjectHandle::newDictionary(); + resources.replaceKey("/P0", cell); + patterns->set(resources); + } + + void form() + { + auto form = decorated_stream(pdf, "11 22 m 33 44 l S\n", + "<< /Type /XObject /Subtype /Form /BBox [0 0 100 100] /Matrix [2 0 0 2 30 40] >>"); + auto resources = QPDFObjectHandle::newDictionary(); + resources.replaceKey("/Fm", form); + xobjects->set(resources, timings); + } + + void check_suffix() + { + require(access::frames(stream)==1, "nested failure leaked parent graphics frames"); + require(access::matrix(stream)==std::array{1,0,0,0,1,0,0,0,1}, + "nested failure leaked a parent transform"); + require(access::pattern_depth()==0, "pattern failure poisoned thread-local depth"); + require(not access::paints_glyph(stream), "glyph failure leaked bitmap paint mode"); + const auto& output = instructions.get_shape_instructions(); + require(not output.empty(), "page suffix did not paint"); + const auto& path = output.back().get_subpaths().at(0); + require(std::abs(path.get_x0()-70)<1e-9 and std::abs(path.get_y0()-80)<1e-9, + "page suffix was moved by the failed child"); + require(not output.back().get_clip_state().has_clip(), "child clip leaked into suffix"); + } +}; + +const std::string suffix = "n 0 0 1 RG 2 w 70 80 m 90 80 l S\n"; + +template +void expect_failure(Operation operation, failure kind) +{ + bool propagated = false; + try { operation(); } + catch(const std::bad_alloc&) + { require(kind==failure::allocation, "failure changed to bad_alloc"); propagated = true; } + catch(const access::failure_marker&) + { require(kind==failure::nonstandard, "failure changed to nonstandard"); propagated = true; } + catch(const std::runtime_error&) + { require(kind==failure::recoverable, "failure changed to runtime_error"); propagated = true; } + require(propagated, "resource/interpreter failure was swallowed"); + require(access::fired, "fault checkpoint did not execute"); +} + +void nested_recovery(bool pattern, bool lattice, failure kind) +{ + fixture f; + if(pattern) { f.pattern(lattice); } + else { f.form(); } + access::inject(f.stream, kind); + const std::string operation = pattern + ? "/Pattern cs /P0 scn 0 0 100 100 re f\n" : "/Fm Do\n"; + if(kind==failure::recoverable) + { + f.run(operation + suffix); + require(access::fired, "recoverable checkpoint did not execute"); + } + else + { + expect_failure([&] { f.run(operation); }, kind); + f.run(suffix); + } + f.check_suffix(); + if(pattern) + { + require(not (*f.patterns)["/P0"].is_decoding(), "failed cell left its recursion flag set"); + f.run(operation + suffix); + f.check_suffix(); + } +} + +void glyph_failure_restores_parent(failure kind) +{ + fixture f; + pdflib::pdf_state::type3_procedure glyph{ + f.pdf.newStream("11 22 m 33 44 l S\n"), QPDFObjectHandle::newDictionary(), + {2,0,0,2,30,40}}; + // More failures than the recursion cap reveal a depth leak on this thread. + for(int attempt=0; attempt<10; ++attempt) + { + access::inject(f.stream, kind); + expect_failure([&] { access::glyph(f.stream, glyph); }, kind); + } + access::glyph(f.stream, glyph); + f.run(suffix); + f.check_suffix(); +} + +void failed_pattern_parse_is_retryable(failure kind) +{ + fixture f; + f.pattern(false); + auto& pattern = (*f.patterns)["/P0"]; + access::inject_parse(kind); + expect_failure([&] { pattern.parse_stream(); }, kind); + require(not pattern.has_parsed_stream(), "failed source parse was committed as complete"); + require(access::cache_empty(pattern), "failed source retained partial tokens/capacity"); + const auto& source = pattern.parse_stream(); + require(pattern.has_parsed_stream() and source.size()==7, "source parse did not retry"); + auto first = source.front().obj; + require(first.getIntValue()==11, "retry reused the partial token prefix"); +} +} + +int main() +{ + try + { + loguru::g_stderr_verbosity = loguru::Verbosity_FATAL; + for(auto kind : {failure::recoverable, failure::allocation, failure::nonstandard}) + { + nested_recovery(true, false, kind); + nested_recovery(true, true, kind); + nested_recovery(false, false, kind); + glyph_failure_restores_parent(kind); + } + failed_pattern_parse_is_retryable(failure::allocation); + failed_pattern_parse_is_retryable(failure::nonstandard); + std::cout << "Stream state lifetime/recovery checks passed\n"; + } + catch(const std::exception& error) + { std::cerr << error.what() << "\n"; return 1; } + catch(...) + { std::cerr << "Unexpected nonstandard exception\n"; return 1; } + return 0; +} diff --git a/tests/test_unit_appearance_streaming.py b/tests/test_unit_appearance_streaming.py new file mode 100644 index 000000000..0a988ecc4 --- /dev/null +++ b/tests/test_unit_appearance_streaming.py @@ -0,0 +1,131 @@ +"""Appearance callback execution after its resource scope and matrix are set.""" + +from __future__ import annotations + +import zlib + +import pytest + +from tests.pdf_builder import ( + build_pdf, + content_stream, + parse_page, + render_page, + stream_object, +) + +FONT = ( + "<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica /Encoding /WinAnsiEncoding >>" +) + + +@pytest.mark.parametrize("widget", [False, True]) +def test_transformed_appearance_can_invoke_a_form_with_local_graphics_state( + widget: bool, +): + # The rotated BBox fits a 100x100 /Rect. Its nested Form must inherit the + # appearance's half opacity rather than the page's opaque /G1 resource. + catalog_fields = " /AcroForm << /Fields [5 0 R] >>" if widget else "" + subtype = "/Widget /FT /Btn /T (field)" if widget else "/Stamp" + pdf = build_pdf( + [ + f"<< /Type /Catalog /Pages 2 0 R{catalog_fields} >>", + "<< /Type /Pages /Kids [3 0 R] /Count 1 >>", + "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 200 200] " + "/Contents 4 0 R /Annots [5 0 R] /Resources << " + "/XObject << /Fm 7 0 R >> /ExtGState << /G1 << /ca 1 >> >> >> >>", + content_stream(""), + f"<< /Type /Annot /Subtype {subtype} /F 4 /Rect [20 40 120 140] " + "/AP << /N 6 0 R >> >>", + stream_object( + "/Type /XObject /Subtype /Form /BBox [10 20 110 70] " + "/Matrix [0 1 -1 0 70 -10] " + "/Resources << /ExtGState << /G1 << /ca 0.5 >> >> >>", + b"/G1 gs /Fm Do", + ), + stream_object( + "/Type /XObject /Subtype /Form /BBox [10 20 110 70]", + b"1 0 0 rg 20 30 40 20 re f", + ), + ] + ) + image = render_page(pdf, scale=1.0).get_image().convert("RGB") + + # The mark maps to page x=[60,100], y=[50,90]. Interior samples avoid edges. + assert image.getpixel((80, 129)) == pytest.approx((255, 128, 128), abs=2) + assert image.getpixel((30, 69)) == (255, 255, 255) + + +@pytest.mark.parametrize("font_scope", ["appearance", "acroform", "page"]) +def test_shared_selected_widget_appearance_uses_the_font_fallback_chain( + font_scope: str, +): + dr = " /DR << /Font << /F1 9 0 R >> >>" if font_scope == "acroform" else "" + page_resources = ( + " /Resources << /Font << /F1 9 0 R >> >>" if font_scope == "page" else "" + ) + ap_resources = ( + " /Resources << /Font << /F1 9 0 R >> >>" if font_scope == "appearance" else "" + ) + pdf = build_pdf( + [ + "<< /Type /Catalog /Pages 2 0 R /AcroForm " + f"<< /Fields [5 0 R 6 0 R]{dr} >> >>", + "<< /Type /Pages /Kids [3 0 R] /Count 1 >>", + "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 200 200] " + f"/Contents 4 0 R /Annots [5 0 R 6 0 R]{page_resources} >>", + content_stream(""), + "<< /Type /Annot /Subtype /Widget /FT /Btn /T (first) /F 4 " + "/Rect [20 120 120 160] /AS /Yes /AP << /N << /Off 7 0 R /Yes 8 0 R >> >> >>", + "<< /Type /Annot /Subtype /Widget /FT /Btn /T (second) /F 4 " + "/Rect [20 60 120 100] /AS /Yes /AP << /N << /Off 7 0 R /Yes 8 0 R >> >> >>", + stream_object( + "/Type /XObject /Subtype /Form /BBox [0 0 100 40] " + "/Resources << /Font << /F1 9 0 R >> >>", + b"BT /F1 12 Tf 10 15 Td (wrong) Tj ET", + ), + stream_object( + f"/Type /XObject /Subtype /Form /BBox [0 0 100 40]{ap_resources}", + b"BT /F1 12 Tf 10 15 Td (inside) Tj ET", + ), + FONT, + ] + ) + + assert [cell.text for cell in parse_page(pdf).textline_cells] == [ + "inside", + "inside", + ] + + +def test_damaged_appearance_prefix_does_not_capture_the_next_annotation(): + encoder = zlib.compressobj() + content = ( + b"/Span << /ActualText (wrong) >> BDC BT /F1 12 Tf 10 15 Td (before) Tj ET\n" + ) + damaged = encoder.compress(content) + encoder.flush(zlib.Z_SYNC_FLUSH) + b"\x07" + pdf = build_pdf( + [ + "<< /Type /Catalog /Pages 2 0 R >>", + "<< /Type /Pages /Kids [3 0 R] /Count 1 >>", + "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 200 200] " + "/Contents 4 0 R /Annots [5 0 R 6 0 R] " + "/Resources << /Font << /F1 9 0 R >> >> >>", + content_stream(""), + "<< /Type /Annot /Subtype /Stamp /F 4 /Rect [20 120 120 160] " + "/AP << /N 7 0 R >> >>", + "<< /Type /Annot /Subtype /Stamp /F 4 /Rect [20 60 120 100] " + "/AP << /N 8 0 R >> >>", + stream_object( + "/Type /XObject /Subtype /Form /BBox [0 0 100 40] /Filter /FlateDecode", + damaged, + ), + stream_object( + "/Type /XObject /Subtype /Form /BBox [0 0 100 40]", + b"BT /F1 12 Tf 10 15 Td (after) Tj ET", + ), + FONT, + ] + ) + + assert [cell.text for cell in parse_page(pdf).textline_cells] == ["before", "after"] diff --git a/tests/test_unit_clipping.py b/tests/test_unit_clipping.py index b2d1f375e..ec79615c4 100644 --- a/tests/test_unit_clipping.py +++ b/tests/test_unit_clipping.py @@ -56,6 +56,32 @@ def circle_path(cx: float, cy: float, r: float) -> str: ) +@pytest.mark.parametrize("operator", ["W", "W*"]) +def test_clip_snapshots_keep_earlier_paints_and_q_restore_branches(operator: str): + # The child branches left; after Q the parent branches right. The first + # paint retains the broad prefix, and each later paint keeps its own clip. + content = ( + f"20 20 160 160 re {operator} n " + "1 0 0 rg 0 140 200 20 re f " + f"q 20 20 60 160 re {operator} n " + "0 0 1 rg 0 80 200 20 re f Q " + f"110 20 70 160 re {operator} n " + "0 1 0 rg 0 40 200 20 re f" + ) + image = render_page(simple_page_pdf(content), scale=1.0).get_image().convert("RGB") + + def pixel(x: int, y: int): + return image.getpixel((x, image.height - 1 - y)) + + for x in (40, 150): + assert pixel(x, 150) == (255, 0, 0) + assert pixel(40, 90) == (0, 0, 255) + assert pixel(150, 90) == (255, 255, 255) + assert pixel(150, 50) == (0, 255, 0) + assert pixel(40, 50) == (255, 255, 255) + assert pixel(10, 150) == (255, 255, 255) + + def _red_image_object() -> bytes: return stream_object( "/Type /XObject /Subtype /Image /Width 4 /Height 4 " diff --git a/tests/test_unit_form_streaming.py b/tests/test_unit_form_streaming.py new file mode 100644 index 000000000..de838a8d9 --- /dev/null +++ b/tests/test_unit_form_streaming.py @@ -0,0 +1,94 @@ +"""Nested Form execution while the parent content parser is still active.""" + +from __future__ import annotations + +import zlib + +import pytest + +from tests.pdf_builder import parse_page, render_page, simple_page_pdf, stream_object + +FONT = ( + "<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica /Encoding /WinAnsiEncoding >>" +) + + +def test_repeated_nested_forms_keep_local_resources_and_restore_parent_state(): + # Both Forms shadow /G1. The outer one is translucent; the leaf is opaque. + # A transform left active in the leaf must not move its parent's next mark. + outer = stream_object( + "/Type /XObject /Subtype /Form /BBox [0 0 70 30] " + "/Matrix [1 0 0 1 20 30] " + "/Resources << /XObject << /Leaf 6 0 R >> " + "/ExtGState << /G1 << /ca 0.5 >> >> >>", + b"/G1 gs /Leaf Do 0 1 0 rg 30 0 10 10 re f", + ) + leaf = stream_object( + "/Type /XObject /Subtype /Form /BBox [0 0 10 10] " + "/Matrix [2 0 0 2 0 0] " + "/Resources << /ExtGState << /G1 << /ca 1 >> >> >>", + b"/G1 gs 1 0 0 rg 0 0 10 10 re f 1 0 0 1 50 0 cm", + ) + pdf = simple_page_pdf( + "/G1 gs 0 0 1 rg /Outer Do q 1 0 0 1 80 0 cm /Outer Do Q 10 100 10 10 re f", + resources="/XObject << /Outer 5 0 R >> /ExtGState << /G1 << /ca 1 >> >>", + extra_objects=[outer, leaf], + ) + image = render_page(pdf, scale=1.0).get_image().convert("RGB") + + def pixel(x: int, y: int): + return image.getpixel((x, image.height - 1 - y)) + + for x in (30, 110): + assert pixel(x, 40) == pytest.approx((255, 0, 0), abs=2) + for x in (55, 135): + assert pixel(x, 35) == pytest.approx((128, 255, 128), abs=2) + assert pixel(15, 105) == pytest.approx((0, 0, 255), abs=2) + assert pixel(75, 35) == (255, 255, 255) + + +def test_damaged_form_prefix_survives_repeated_invocation_and_parent_continues(): + # A valid non-final Flate prefix followed by a reserved block type. The + # first invocation repairs the source; the second must still execute it. + encoder = zlib.compressobj() + content = b"BT /F1 12 Tf 5 40 Td (inside) Tj ET\n" + damaged = encoder.compress(content) + encoder.flush(zlib.Z_SYNC_FLUSH) + b"\x07" + form = stream_object( + "/Type /XObject /Subtype /Form /BBox [0 0 100 60] /Filter /FlateDecode", + damaged, + ) + pdf = simple_page_pdf( + "q 1 0 0 1 20 120 cm /Fm Do Q " + "q 1 0 0 1 20 60 cm /Fm Do Q " + "BT /F1 12 Tf 20 20 Td (after) Tj ET", + resources="/XObject << /Fm 5 0 R >> /Font << /F1 6 0 R >>", + extra_objects=[form, FONT], + ) + + assert [cell.text for cell in parse_page(pdf).textline_cells] == [ + "inside", + "inside", + "after", + ] + + +@pytest.mark.parametrize("close_scope", [True, False]) +def test_form_actual_text_scope_finishes_before_parent_suffix(close_scope: bool): + content = ( + "/Span << /ActualText (replacement) >> BDC " + "BT /F1 12 Tf 20 120 Td (shown) Tj ET" + (" EMC" if close_scope else "") + ) + form = stream_object( + "/Type /XObject /Subtype /Form /BBox [0 0 200 200]", + content.encode("latin-1"), + ) + pdf = simple_page_pdf( + "/Fm Do BT /F1 12 Tf 20 20 Td (after) Tj ET", + resources="/XObject << /Fm 5 0 R >> /Font << /F1 6 0 R >>", + extra_objects=[form, FONT], + ) + + assert [cell.text for cell in parse_page(pdf).textline_cells] == [ + "replacement" if close_scope else "shown", + "after", + ] diff --git a/tests/test_unit_pattern_replay.py b/tests/test_unit_pattern_replay.py new file mode 100644 index 000000000..8ae6aab13 --- /dev/null +++ b/tests/test_unit_pattern_replay.py @@ -0,0 +1,50 @@ +"""Immutable pattern source replay and decoded-cell reuse by stencil colour.""" + +from __future__ import annotations + +import zlib + +import pytest + +from tests.pdf_builder import render_page, simple_page_pdf, stream_object + + +@pytest.mark.parametrize("lattice", [False, True]) +@pytest.mark.parametrize("damaged", [False, True]) +def test_repeated_stencil_keeps_source_colour_and_parent_state( + lattice: bool, damaged: bool +) -> None: + # Small cells use the decoded-cell cache; cells larger than the page + # replay directly. Red/blue/red exercises both colour keys and reuse. + step = 300 if lattice else 20 + source = f"/G1 gs 0 0 {step} {step} re f 1 0 0 1 100 0 cm\n".encode() + if damaged: + encoder = zlib.compressobj() + source = encoder.compress(source) + encoder.flush(zlib.Z_SYNC_FLUSH) + b"\x07" + pattern = stream_object( + "/Type /Pattern /PatternType 1 /PaintType 2 /TilingType 1 " + f"/BBox [0 0 {step} {step}] /XStep {step} /YStep {step} " + "/Resources << /ExtGState << /G1 << /ca 0.5 >> >> >>" + + (" /Filter /FlateDecode" if damaged else ""), + source, + ) + pdf = simple_page_pdf( + "/G1 gs /PCS cs " + "1 0 0 /P0 scn 20 20 30 30 re f " + "0 0 1 /P0 scn 80 20 30 30 re f " + "1 0 0 /P0 scn 140 20 30 30 re f " + "0 1 0 rg 20 100 30 30 re f", + resources="/Pattern << /P0 5 0 R >> /ColorSpace << /PCS [/Pattern /DeviceRGB] >> " + "/ExtGState << /G1 << /ca 1 >> >>", + extra_objects=[pattern], + ) + image = render_page(pdf, scale=1.0).get_image().convert("RGB") + + def pixel(x: int, y: int): + return image.getpixel((x, image.height - 1 - y)) + + for x in (35, 155): + assert pixel(x, 35) == pytest.approx((255, 128, 128), abs=3) + assert pixel(95, 35) == pytest.approx((128, 128, 255), abs=3) + assert pixel(35, 115) == (0, 255, 0) + assert pixel(65, 35) == (255, 255, 255) diff --git a/tests/test_unit_performance_docs.py b/tests/test_unit_performance_docs.py new file mode 100644 index 000000000..97fa42646 --- /dev/null +++ b/tests/test_unit_performance_docs.py @@ -0,0 +1,253 @@ +"""Publication must preserve setup isolation and reject oversized/stale output.""" + +import importlib.util +import json +import sys +from pathlib import Path +from types import SimpleNamespace + +import pytest + +_SPEC = importlib.util.spec_from_file_location( + "performance_docs", + Path(__file__).parents[1] / "scripts/tools/update_performance_docs.py", +) +assert _SPEC and _SPEC.loader +_DOCS = importlib.util.module_from_spec(_SPEC) +_SPEC.loader.exec_module(_DOCS) +LIMITS = json.loads((_DOCS.ARTIFACTS / "performance_docs_limits.json").read_text()) + + +def fixture_run(tmp_path, monkeypatch): + meta = { + "dataset": { + "repository": "docling-project/performance-dataset-bo20", + "revision": "pin", + }, + "cpu": "Test CPU", + "machine": "arm64", + "platform": "TestOS", + "memory_bytes": "68719476736", + "threads": [1, 4], + "repeats": 3, + "scale": 1.0, + "max_concurrent_results": 32, + } + bundle = tmp_path / "scratch-performance-refresh-20261112-123829-example" + bundle.mkdir() + (bundle / "metadata.json").write_text(json.dumps(meta)) + measurement = SimpleNamespace(workload="native", memory=None) + inputs = SimpleNamespace( + metadata=meta, + files=[{"name": "a.pdf", "pages": 2, "sha256": "hash"}], + expected={1, 2}, + excluded_comparisons=[], + canvas_checks=[], + load=lambda: [measurement], + ) + + def render(inputs, measurements, output): + (output / "plot.png").write_bytes(b"png") + (output / "rss_unused.png").write_bytes(b"diagnostic") + return { + "native": "## Native\n\n![Plot](assets/plot.png)", + "comparison": "## Comparison\n\nNot collected in this run.", + } + + monkeypatch.setitem( + sys.modules, + "run_performance_report", + SimpleNamespace(ReportInputs=lambda _: inputs, publication_sections=render), + ) + index = ( + "\n" + ) + return bundle, {_DOCS.INDEX: index.encode()}, meta + + +def test_slots_preserve_authored_text_and_skip_missing(capsys): + text = "Intro\nold\nTail" + assert _DOCS.slot(text, "native", "new") == text.replace(">old<", ">\nnew\n<") + assert _DOCS.slot(text, "memory", "ignored") == text + assert "Skip absent" in capsys.readouterr().out + with pytest.raises(ValueError, match="Malformed"): + _DOCS.slot(text + "", "native", "x") + + +def test_publication_is_repeatable_and_preserves_other_setups(tmp_path, monkeypatch): + bundle, before, meta = fixture_run(tmp_path, monkeypatch) + after = _DOCS.plan(bundle, dict(before), None, False) + _DOCS.validate(after, LIMITS) + assert before.keys() == {_DOCS.INDEX} + assert not any("rss_unused" in str(p) for p in after) + assert _DOCS.plan(bundle, dict(after), None, False) == after + meta["cpu"] = "Other CPU" + other = _DOCS.plan(bundle, dict(after), None, False) + assert len([p for p in other if p.name == "publication.json"]) == 2 + for p in after: + if p != _DOCS.INDEX: + assert other[p] == after[p] + + +def test_older_run_and_machine_collision_fail(tmp_path, monkeypatch): + bundle, before, meta = fixture_run(tmp_path, monkeypatch) + after = _DOCS.plan(bundle, before, "stable", False) + manifest = next(p for p in after if p.name == "publication.json") + value = json.loads(after[manifest]) + value["measured_at"] = "2099-01-01T00:00:00" + after[manifest] = _DOCS.encoded(value) + with pytest.raises(ValueError, match="older"): + _DOCS.plan(bundle, dict(after), "stable", False) + _DOCS.plan(bundle, dict(after), "stable", True) + meta["cpu"] = "Different" + with pytest.raises(ValueError, match="identity differs"): + _DOCS.plan(bundle, dict(after), "stable", True) + + +@pytest.mark.parametrize( + "key", + ["markdown_bytes", "image_bytes", "json_bytes", "publication_bytes", "tree_bytes"], +) +def test_each_size_budget_is_enforced(tmp_path, monkeypatch, key): + bundle, before, _ = fixture_run(tmp_path, monkeypatch) + after = _DOCS.plan(bundle, before, None, False) + with pytest.raises(ValueError, match="exceeds"): + _DOCS.validate(after, {**LIMITS, key: 1}) + + +def test_stale_owned_assets_removed_unowned_preserved(tmp_path, monkeypatch): + bundle, before, _ = fixture_run(tmp_path, monkeypatch) + after = _DOCS.plan(bundle, before, None, False) + manifest = next(p for p in after if p.name == "publication.json") + base = manifest.parent + value = json.loads(after[manifest]) + value["files"]["assets/stale.png"] = _DOCS.digest(b"stale") + after[base / "assets/stale.png"] = b"stale" + after[base / "notes.md"] = b"Author notes" + after[manifest] = _DOCS.encoded(value) + updated = _DOCS.plan(bundle, after, None, False) + assert base / "assets/stale.png" not in updated + assert updated[base / "notes.md"] == b"Author notes" + + +def test_modified_asset_and_broken_link_fail(tmp_path, monkeypatch): + bundle, before, _ = fixture_run(tmp_path, monkeypatch) + after = _DOCS.plan(bundle, before, None, False) + image = next(p for p in after if p.suffix == ".png") + bad = {**after, image: b"manual edit"} + with pytest.raises(ValueError, match="modified"): + _DOCS.validate(bad, LIMITS) + bad = {**after, _DOCS.INDEX: b"[Missing](does-not-exist.md)"} + with pytest.raises(ValueError, match="Broken"): + _DOCS.validate(bad, LIMITS) + + +def test_path_traversal_rejected(): + for value in ["../outside.png", "/absolute.png"]: + with pytest.raises(ValueError, match="Unsafe"): + _DOCS.checked_relative(value) + + +def test_check_cli_uses_no_performance_dependencies(): + import subprocess + + result = subprocess.run( + [ + sys.executable, + "-S", + str(_DOCS.TOOLS / "update_performance_docs.py"), + "--check", + ], + capture_output=True, + text=True, + ) + assert result.returncode == 0, result.stderr + + +def test_cli_preview_and_budget_failure_never_write_docs(tmp_path, monkeypatch): + bundle, before, _ = fixture_run(tmp_path, monkeypatch) + entry = tmp_path / _DOCS.INDEX + entry.parent.mkdir(parents=True) + entry.write_bytes(before[_DOCS.INDEX]) + monkeypatch.setattr(_DOCS, "ROOT", tmp_path) + monkeypatch.setattr( + sys, "argv", ["publisher", "--performance-run-dir", str(bundle)] + ) + _DOCS.main() + assert _DOCS.snapshot(tmp_path) == before + planned = _DOCS.plan(bundle, dict(before), None, False) + planned[_DOCS.INDEX] = b"x" * (LIMITS["markdown_bytes"] + 1) + monkeypatch.setattr(_DOCS, "plan", lambda *args: planned) + monkeypatch.setattr( + sys, "argv", ["publisher", "--performance-run-dir", str(bundle), "--apply"] + ) + with pytest.raises(SystemExit): + _DOCS.main() + assert _DOCS.snapshot(tmp_path) == before + + +def test_missing_machine_identity_needs_explicit_setup(): + meta = { + "dataset": {"repository": "docling-project/performance-dataset-bo20"}, + "machine": "arm64", + "platform": "macOS", + } + with pytest.raises(ValueError, match="Incomplete"): + _DOCS.identity(meta, None) + assert _DOCS.identity(meta, "historical-machine")[1] == "historical-machine" + + +def test_historical_snapshot_uses_current_template_on_refresh(tmp_path, monkeypatch): + bundle, before, _ = fixture_run(tmp_path, monkeypatch) + after = _DOCS.plan(bundle, before, None, False) + path = next(p for p in after if p.name == "publication.json") + value = json.loads(after[path]) + value["migrated_snapshot"] = True + after[path] = _DOCS.encoded(value) + after[path.parent / "index.md"] = b"# Historical snapshot without markers" + updated = _DOCS.plan(bundle, after, None, False) + assert b"" in updated[path.parent / "index.md"] + assert "migrated_snapshot" not in json.loads(updated[path]) + _DOCS.validate(updated, LIMITS) + + +def test_unknown_ram_is_unavailable_not_zero(): + assert _DOCS.memory_label({}) == "unknown RAM" + + +@pytest.mark.parametrize( + "arguments", + [ + [], + ["RUN"], + ["--apply"], + ["--performance-run-dir"], + ["--check", "--performance-run-dir", "RUN"], + ["--check", "--apply"], + ], +) +def test_cli_requires_named_run_directory_or_check(monkeypatch, arguments): + monkeypatch.setattr(sys, "argv", ["publisher", *arguments]) + with pytest.raises(SystemExit) as error: + _DOCS.main() + assert error.value.code == 2 + + +@pytest.mark.parametrize( + "dataset, label", [("bo20", "Bo20"), ("bo767", "Bo767"), ("bo10k", "Bo10K")] +) +def test_generated_title_identifies_dataset_and_setup( + tmp_path, monkeypatch, dataset, label +): + bundle, before, meta = fixture_run(tmp_path, monkeypatch) + meta["dataset"]["repository"] = f"docling-project/performance-dataset-{dataset}" + after = _DOCS.plan(bundle, before, None, False) + page = next(p for p in after if p.name == "index.md") + heading = f"# Performance results: {label} / Test CPU / 64 GiB / TestOS" + assert heading.encode() in after[page] + # A previously generated generic title upgrades without losing authored content. + after[page] = b"# Performance results\n\nAuthor notes.\n" + updated = _DOCS.plan(bundle, after, None, False) + assert heading.encode() in updated[page] + assert b"Author notes." in updated[page] + _DOCS.validate(updated, LIMITS) diff --git a/tests/test_unit_performance_memory.py b/tests/test_unit_performance_memory.py new file mode 100644 index 000000000..4745f72a9 --- /dev/null +++ b/tests/test_unit_performance_memory.py @@ -0,0 +1,380 @@ +"""Independent RSS observations, worker isolation, and recoverable memory artifacts.""" + +import argparse +import csv +import importlib +import json +import os +import subprocess +import sys +from pathlib import Path +from textwrap import dedent +from types import SimpleNamespace + +import pytest + +_SCRIPTS = Path(__file__).parents[1] / "scripts/benchmarking" +sys.path.insert(0, str(_SCRIPTS)) +try: + _MEMORY = importlib.import_module("_performance_memory") + _REPORT = importlib.import_module("_performance_memory_report") + _RUNTIME = importlib.import_module("_performance_runtime") + _WORKER = importlib.import_module("_performance_worker") + _COMMON = importlib.import_module("_common") +finally: + sys.path.pop(0) + +_SUPPORTED = sys.platform == "darwin" or sys.platform.startswith("linux") + + +@pytest.mark.parametrize("platform,factor", [("darwin", 1), ("linux", 1024)]) +def test_os_high_water_units(platform, factor): + assert _MEMORY.peak_bytes(123, platform) == 123 * factor + with pytest.raises(ValueError): + _MEMORY.peak_bytes(-1, platform) + + +@pytest.mark.parametrize( + "variable,value", [("MEMORY_MODE", "invalid"), ("MEMORY_INTERVAL_MS", "0")] +) +def test_invalid_environment_is_rejected(monkeypatch, variable, value): + monkeypatch.setenv(variable, value) + parser = argparse.ArgumentParser() + _MEMORY.add_memory_arguments(parser) + with pytest.raises(SystemExit) as error: + parser.parse_args([]) + assert error.value.code == 2 + + +@pytest.mark.skipif(not _SUPPORTED, reason="OS RSS collection supports macOS/Linux") +def test_peak_belongs_to_each_fresh_child(tmp_path: Path): + # Linux can retain pre-exec RSS inherited from the spawning process. Keep + # pytest's imports out of the measured children's startup high-water mark. + # Both children still share one monitor so a cumulative RUSAGE_CHILDREN + # measurement would incorrectly retain the large child's peak for the small. + monitor = dedent("""\ + import os + import sys + from pathlib import Path + + sys.path.insert(0, sys.argv[1]) + from _performance_memory import run_measured + + output = Path(sys.argv[2]) + command = [sys.executable, "-c", "import sys; b=bytearray(int(sys.argv[1]))"] + for name, size in (("large", 64 * 2**20), ("small", 1024)): + run_measured([*command, str(size)], output / (name + ".json"), mode="peak") + print(os.getpid()) + """) + monitor_pid = int( + subprocess.check_output( + [sys.executable, "-c", monitor, str(_SCRIPTS), str(tmp_path)], text=True + ) + ) + large = _MEMORY.read_memory(tmp_path / "large.json") + small = _MEMORY.read_memory(tmp_path / "small.json") + assert len({large["pid"], small["pid"], monitor_pid, os.getpid()}) == 4 + command = [sys.executable, "-c", "import sys; b=bytearray(int(sys.argv[1]))"] + assert large["peak_rss_bytes"] > small["peak_rss_bytes"] + 32 * 2**20 + assert _MEMORY.read_memory(tmp_path / "small.json") == small + with pytest.raises(ValueError, match="exists"): + _MEMORY.run_measured(command, tmp_path / "small.json", mode="peak") + + +@pytest.mark.skipif(not _SUPPORTED, reason="OS RSS collection supports macOS/Linux") +def test_profile_is_external_and_preserves_os_peak(tmp_path: Path): + path = tmp_path / "profile.json" + result = _MEMORY.run_measured( + [ + sys.executable, + "-c", + "import time; b=bytearray(16*1024*1024); time.sleep(.2)", + ], + path, + mode="profile", + interval_ms=10, + ) + assert result["pid"] != os.getpid() + assert result["sample_count"] >= 2 + assert result["peak_rss_bytes"] > 16 * 2**20 + validated = _MEMORY.read_memory(path, {"mode": "profile", "interval_ms": 10}) + timeline = path.parent / validated["timeline"] + rows = _REPORT.timeline_rows(timeline, validated) + assert len(rows) == result["sample_count"] + assert max(rss for _, rss in rows) == result["sampled_max_rss_bytes"] + with pytest.raises(ValueError, match="settings differ"): + _MEMORY.read_memory(path, {"mode": "profile", "interval_ms": 50}) + timeline.write_text(timeline.read_text() + "0,0,0\n") + with pytest.raises(ValueError, match="changed"): + _MEMORY.read_memory(path) + + +@pytest.mark.skipif(not _SUPPORTED, reason="OS RSS collection supports macOS/Linux") +def test_failed_child_keeps_diagnostic_peak_but_is_not_valid(tmp_path: Path): + path = tmp_path / "failed.json" + result = _MEMORY.run_measured( + [sys.executable, "-c", "raise SystemExit(7)"], path, mode="peak" + ) + assert result["returncode"] == 7 + assert result["complete"] is False + assert result["peak_rss_bytes"] > 0 + with pytest.raises(ValueError, match="incomplete"): + _MEMORY.read_memory(path) + + +@pytest.mark.skipif(not _SUPPORTED, reason="OS RSS collection supports macOS/Linux") +def test_sampler_failure_terminates_and_reaps_child(tmp_path: Path, monkeypatch): + def fail(_pid): + raise OSError("sampler unavailable") + + monkeypatch.setattr(_MEMORY, "rss_reader", lambda: fail) + path = tmp_path / "failed.json" + with pytest.raises(OSError, match="sampler unavailable"): + _MEMORY.run_measured( + [sys.executable, "-c", "import time; time.sleep(60)"], + path, + mode="profile", + ) + result = json.loads(path.read_text()) + assert result["complete"] is False + with pytest.raises(ChildProcessError): + os.waitpid(result["pid"], os.WNOHANG) + + +def test_worker_transports_options_and_rows_without_changing_timing( + tmp_path: Path, monkeypatch +): + calls = [] + + def backend(schedule, total_pages, **options): + calls.append((schedule, total_pages, options)) + return SimpleNamespace( + backend="fixture", + task="parse+render", + threads=4, + wall_s=1.234, + per_page_source="native-worker", + doc_errors=0, + samples=[ + _COMMON.PageRow("fixture", "parse+render", 4, "doc.pdf", 1, True, 0.2) + ], + ) + + monkeypatch.setitem( + sys.modules, + "run_performance_benchmarking", + SimpleNamespace(COMPARISON_BACKENDS={"fixture": (backend, ())}), + ) + request = tmp_path / "request.json" + response = tmp_path / "response.json" + options = { + "render": True, + "threads": 4, + "scale": 1.5, + "bytesio": True, + "decode_options": {"keep_shapes": True}, + "materialization_options": {"include_bitmap_bytes": False}, + "max_concurrent_results": 8, + } + request.write_text( + json.dumps( + { + "backend": "fixture", + "pdf_schedule": [["doc.pdf", [1]]], + "total_pages": 1, + **options, + } + ) + ) + monkeypatch.setattr(sys, "argv", ["worker", str(request), str(response)]) + _WORKER.main() + assert calls == [([(Path("doc.pdf"), [1])], 1, options)] + result = json.loads(response.read_text()) + assert result["wall_s"] == 1.234 + assert result["samples"][0]["elapsed_s"] == 0.2 + assert result["threads"] == 4 + + +def _artifact(path: Path, *, mode="profile"): + path.parent.mkdir(parents=True, exist_ok=True) + value = { + "schema_version": 1, + "mode": mode, + "units": "bytes", + "method": "wait4.ru_maxrss", + "scope": "process lifetime", + "pid": 123, + "complete": True, + "returncode": 0, + "peak_rss_bytes": 2**20, + "sample_count": 2 if mode == "profile" else 0, + "interval_ms": 50 if mode == "profile" else None, + "sampled_max_rss_bytes": 512 if mode == "profile" else None, + "timeline": path.with_suffix(".rss.csv").name if mode == "profile" else None, + } + if mode == "profile": + timeline = path.parent / value["timeline"] + timeline.write_text("elapsed_s,rss_bytes,pid\n0.01,256,123\n0.05,512,123\n") + value["timeline_sha256"] = _MEMORY.file_digest(timeline) + path.write_text(json.dumps(value)) + return value + + +def test_resume_recollects_when_profile_artifact_is_lost(tmp_path: Path): + artifact = tmp_path / "memory/run.json" + memory = _artifact(artifact) + memory.update(artifact="memory/run.json", artifact_sha256=_RUNTIME.digest(artifact)) + store = _RUNTIME.BenchmarkCheckpoints( + tmp_path / "checkpoints", + {"memory": {"mode": "profile", "interval_ms": 50}}, + {}, + 1, + ) + run = SimpleNamespace( + backend="fixture", + task="parse", + threads=1, + wall_s=1, + per_page_source="wall", + doc_errors=0, + memory=memory, + samples=[_COMMON.PageRow("fixture", "parse", 1, "doc.pdf", 1, True, 0.5)], + ) + store.save(run) + assert store.load("fixture", "parse", 1)["memory"]["pid"] == 123 + (artifact.parent / memory["timeline"]).unlink() + assert store.load("fixture", "parse", 1) is None + + +@pytest.mark.parametrize("damage", ["pid", "order", "count"]) +def test_timeline_rejects_inconsistent_samples(tmp_path: Path, damage): + path = tmp_path / "memory.json" + memory = _artifact(path) + timeline = path.parent / memory["timeline"] + if damage == "pid": + timeline.write_text(timeline.read_text().replace(",123", ",456")) + elif damage == "order": + timeline.write_text("elapsed_s,rss_bytes,pid\n1,256,123\n0,512,123\n") + else: + memory["sample_count"] += 1 + with pytest.raises(ValueError, match="RSS"): + _REPORT.timeline_rows(timeline, memory) + + +def test_missing_memory_is_unavailable_and_mixed_modes_are_rejected(tmp_path: Path): + def measurement(repetition, memory): + return SimpleNamespace( + group=("native", "docling-parse", "render", 1), + workload="native", + backend="docling-parse", + task="render", + threads=1, + repetition=repetition, + memory=memory, + ) + + lines = _REPORT.write_memory_report([measurement(1, None)], tmp_path) + assert "unavailable" in "\n".join(lines) + assert ( + "No sampled RSS traces are available" + in (tmp_path / "memory_traces.md").read_text() + ) + with (tmp_path / "memory_summary.csv").open() as stream: + row = next(csv.DictReader(stream)) + assert row["median_peak_mib"] == "" + profile = _artifact(tmp_path / "profile.json") + peak = _artifact(tmp_path / "peak.json", mode="peak") + with pytest.raises(ValueError, match="Mixed memory"): + _REPORT.write_memory_report( + [measurement(1, profile), measurement(2, peak)], tmp_path + ) + + +@pytest.mark.parametrize("mode", ["peak", "profile"]) +def test_profile_report_exports_observed_peaks_and_timelines( + tmp_path: Path, mode, monkeypatch +): + figure_class = pytest.importorskip("matplotlib.figure").Figure + savefig = figure_class.savefig + figures = {} + + def capture(figure, path, **kwargs): + figures[Path(path).name] = figure + savefig(figure, path, **kwargs) + + monkeypatch.setattr(figure_class, "savefig", capture) + path = tmp_path / "profile.json" + memory = _artifact(path, mode=mode) + if mode == "profile": + memory["_timeline_path"] = str(path.parent / memory["timeline"]) + memory["_source"] = path.name + measurement = SimpleNamespace( + group=("replay", "pdfium", "render", 1), + workload="replay", + backend="pdfium", + task="render", + threads=1, + repetition=1, + memory=memory, + ) + lines = "\n".join(_REPORT.write_memory_report([measurement], tmp_path)) + assert lines.startswith("## Process Memory\n") + assert "memory_replay_render.png" in lines + assert "[Elapsed-time memory traces](memory_traces.md)" in lines + assert "rss_replay_pdfium_render_t1.png" not in lines + assert (tmp_path / "memory_replay_render.png").is_file() + peak_axes = figures["memory_replay_render.png"].axes[0] + assert peak_axes.get_xlim()[0] == 0 + assert peak_axes.get_ylim()[0] == 0 + assert 0 in peak_axes.get_xticks() + traces = (tmp_path / "memory_traces.md").read_text() + assert "[Back to Process Memory](summary.md#process-memory)" in traces + if mode == "profile": + assert "rss_replay_pdfium_render_t1.png" in traces + assert (tmp_path / "rss_replay_pdfium_render_t1.png").is_file() + else: + assert "No sampled RSS traces are available" in traces + with (tmp_path / "memory_runs.csv").open() as stream: + row = next(csv.DictReader(stream)) + assert row["peak_rss_bytes"] == str(2**20) + assert row["sampled_max_rss_bytes"] == ("512" if mode == "profile" else "") + assert row["source"] == "profile.json" + + +@pytest.mark.parametrize("comparison", [False, True]) +def test_launcher_observes_actual_process_or_delegates_series( + tmp_path: Path, comparison +): + source = (_SCRIPTS / "refresh_performance_benchmarks.sh").read_text() + function = source.split("run_experiment() {", 1)[1].split("\n}\n", 1)[0] + log = tmp_path / ( + "comparison/r1/render/run.log" if comparison else "replay/r1/t1.log" + ) + command = "\n".join( + [ + "set -euo pipefail", + "REPO_ROOT=/fixture", + "MEMORY_MODE=profile", + "MEMORY_INTERVAL_MS=25", + "experiment=0", + "total_experiments=1", + "run_logged() { printf '%s\\n' \"$@\"; }", + "run_experiment() {" + function + "\n}", + 'run_experiment "$1" fixture uv run --no-sync python runner.py --threads 4', + ] + ) + result = subprocess.run( + ["bash", "-c", command, "fixture", str(log)], + capture_output=True, + text=True, + check=True, + ) + arguments = result.stdout.splitlines()[2:] + assert "--memory-mode" in arguments and "profile" in arguments + assert "--memory-interval-ms" in arguments and "25" in arguments + if comparison: + assert arguments[:5] == ["uv", "run", "--no-sync", "python", "runner.py"] + assert "_performance_memory.py" not in " ".join(arguments) + else: + observed = arguments[arguments.index("--") + 1 :] + assert observed == ["/fixture/.venv/bin/python", "runner.py", "--threads", "4"] diff --git a/tests/test_unit_performance_report.py b/tests/test_unit_performance_report.py index df0845f4c..3fae6d4e9 100644 --- a/tests/test_unit_performance_report.py +++ b/tests/test_unit_performance_report.py @@ -22,7 +22,11 @@ assert _SPEC is not None and _SPEC.loader is not None _REPORT = importlib.util.module_from_spec(_SPEC) sys.modules[_SPEC.name] = _REPORT -_SPEC.loader.exec_module(_REPORT) +sys.path.insert(0, str(Path(__file__).parents[1] / "scripts/benchmarking")) +try: + _SPEC.loader.exec_module(_REPORT) +finally: + sys.path.pop(0) _DATASET_PINS = { "bo20": "65b9971b3fa939fe4ef89be678237e6928049b67", @@ -65,6 +69,40 @@ def _inputs(tmp_path: Path): return _REPORT.ReportInputs(tmp_path) +def test_memory_does_not_invent_historical_observations(tmp_path: Path): + inputs = _inputs(tmp_path) + path = tmp_path / "missing.memory.json" + assert inputs.memory(path) is None + existing = inputs.memory(path, {"peak_rss_mib": 12.5}) + assert existing["mode"] == "existing" + assert existing["peak_rss_bytes"] == int(12.5 * 2**20) + inputs.metadata["memory_collection"] = {"mode": "profile", "interval_ms": 50} + with pytest.raises(ValueError, match="missing"): + inputs.memory(path, {"peak_rss_mib": 12.5}) + # Recovery can reuse uninstrumented core results while measuring new references. + inputs.metadata["memory_collection"] = {"mode": "off"} + inputs.metadata["comparison_memory"] = {"mode": "peak"} + assert inputs.memory(path) is None + with pytest.raises(ValueError, match="missing"): + inputs.memory(path, comparison=True) + + +@pytest.mark.parametrize( + "workload,task", + [("native", "render-only"), ("replay", "render"), ("comparison", "parse+render")], +) +def test_reproduction_preserves_memory_settings(tmp_path: Path, workload, task): + inputs = _profiled_bundle(tmp_path) + inputs.metadata["memory_collection"] = {"mode": "profile", "interval_ms": 25} + command = "\n".join(_REPORT.reproduction_commands(inputs.metadata, workload, task)) + assert "--memory-mode profile --memory-interval-ms 25" in command + if workload == "comparison": + assert "observe()" not in command + else: + assert "observe()" in command + assert "_performance_memory.py" in command + + def _native(tmp_path: Path, profile=None) -> Path: native_root = tmp_path / "native" if profile: @@ -356,9 +394,16 @@ def test_comparison_checkpoint_provenance_controls_version_labels( assert measurement.package_version == "1.27.2.3" -def test_profiled_report_figures_captions_and_phase_exports(tmp_path: Path): +@pytest.mark.parametrize("with_rss_timelines", [False, True]) +def test_profiled_report_figures_captions_and_phase_exports( + tmp_path: Path, with_rss_timelines +): pytest.importorskip("matplotlib") inputs = _profiled_bundle(tmp_path) + if with_rss_timelines: + for profile in _REPORT.NATIVE_PROFILES: + timeline = tmp_path / "native" / profile / "r1/t1.memory.rss.csv" + timeline.write_text("elapsed_s,rss_bytes,pid\n0.05,1048576,123\n") output = tmp_path / "report" _REPORT.write_report(inputs, inputs.load(), output) expected = { @@ -368,7 +413,38 @@ def test_profiled_report_figures_captions_and_phase_exports(tmp_path: Path): } assert {p.name for p in output.glob("*.png")} == expected assert all((output / name).stat().st_size > 0 for name in expected) - text = (output / "summary.md").read_text().split("## Figures\n", 1)[1] + tables, text = (output / "summary.md").read_text().split("## Figures\n", 1) + assert [line for line in tables.splitlines() if line.startswith("##")] == [ + "## Parallel Scaling", + "### Native C++ scaling", + "### Python scaling with Docling settings", + "## Process Memory", + ] + assert "[Elapsed-time memory traces](memory_traces.md)" in tables + timing_rows = [ + line + for line in tables.splitlines() + if line.startswith(("| docling-parse |", "| pypdfium2 |")) + ] + assert [line.split("|")[3].strip() for line in timing_rows] == [ + "Render", + "Render + text", + "Backend replay", + "Backend replay", + ] + assert "- **Render**: Render page-images." in tables + assert ( + "- **Render + text**: Render page-images and construct word/line text geometry." + in tables + ) + assert ( + "- **Backend replay**: Page-images, word/line text geometry and bitmap bounding boxes." + in tables + ) + assert ( + "No sampled RSS traces are available" + in (output / "memory_traces.md").read_text() + ) assert text.index("### Native C++ scaling") < text.index( "### Python scaling with Docling settings" ) @@ -391,6 +467,64 @@ def test_profiled_report_figures_captions_and_phase_exports(tmp_path: Path): assert float(rows["render-with-text"]["create_word_cells_s"]) == 0.2 +@pytest.mark.parametrize("has_one_thread", [True, False]) +def test_ideal_throughput_uses_one_thread_baseline_and_zero_origin( + tmp_path: Path, monkeypatch, has_one_thread +): + figure_class = pytest.importorskip("matplotlib.figure").Figure + figures = {} + + def capture(figure, path, **_kwargs): + figures[Path(path).name] = figure + + monkeypatch.setattr(figure_class, "savefig", capture) + summary = [] + for backend, threads, throughput in ( + ("threaded", 1, 10), + ("threaded", 4, 33), + ("threaded", 8, 50), + ("pdfium", 1, 8), + ): + if backend == "threaded" and threads == 1 and not has_one_thread: + continue + wall = 800 / throughput + summary.append( + { + "workload": "replay", + "backend": backend, + "task": "render", + "threads": threads, + "median_wall_s": wall, + "min_wall_s": wall, + "max_wall_s": wall, + "pages_per_s": throughput, + "speedup_vs_docling_1t": throughput / 10, + "efficiency_pct": 100 * throughput / (10 * threads) + if backend == "threaded" + else None, + } + ) + _REPORT.plot([], summary, tmp_path) + throughput_ax = figures["replay_runtime.png"].axes[1] + assert throughput_ax.get_xlim()[0] == 0 + assert throughput_ax.get_ylim()[0] == 0 + assert 0 in throughput_ax.get_xticks() + curves = {line.get_label(): line for line in throughput_ax.lines} + assert list(curves["docling-parse"].get_ydata()) == ( + [10, 33, 50] if has_one_thread else [33, 50] + ) + assert list(curves["pypdfium2"].get_ydata()) == [8] + assert "pypdfium2 ideal" not in curves + assert not any("linear fit" in label for label in curves) + if has_one_thread: + ideal = curves["docling-parse ideal"] + assert ideal.get_linestyle() == "--" + assert list(ideal.get_xdata()) == [0, 8] + assert list(ideal.get_ydata()) == [0, 80] + else: + assert "docling-parse ideal" not in curves + + def _checkpointed_comparison( inputs, backend="pymupdf", @@ -580,7 +714,7 @@ def test_report_preserves_failed_reference_audit_without_performance_metrics( assert "fixture page error" in text assert "## Understanding the two Python experiments" in text assert text.index("## Understanding the two Python experiments") < text.index( - "## Python scaling with Docling settings" + "### Python scaling with Docling settings" ) assert "Text and geometry via package APIs" in text assert "Images plus character text and bounding boxes" not in text @@ -1464,3 +1598,223 @@ def test_launcher_recovery_dispatch_keeps_existing_builds(tmp_path: Path, clean) assert str(source) in result.stdout.splitlines() assert str(output) in result.stdout.splitlines() assert "scripts/benchmarking/recover_performance_benchmarks.py" in result.stdout + + +def _recovery_module(): + spec = importlib.util.spec_from_file_location( + "performance_recovery_tests", + Path(__file__).parents[1] + / "scripts/benchmarking/recover_performance_benchmarks.py", + ) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + sys.path.insert(0, str(Path(__file__).parents[1] / "scripts/benchmarking")) + try: + spec.loader.exec_module(module) + finally: + sys.path.pop(0) + return module + + +def _recovery_fixture(tmp_path, monkeypatch): + recovery = _recovery_module() + source = tmp_path / "source" + source.mkdir() + inputs = _profiled_bundle(source) + meta = inputs.metadata + pdf = tmp_path / "doc.pdf" + pdf.write_bytes(b"fixture PDF identity") + meta["dataset"]["files"][0].update(path=str(pdf), sha256=recovery.digest(pdf)) + meta["script_sha256"] = { + "scripts/benchmarking/" + name: recovery.digest( + recovery.ROOT / "scripts/benchmarking" / name + ) + for name in ( + "run_performance_benchmarking_with_docling_settings.py", + "_performance_memory.py", + ) + } + meta["memory_collection"] = {"mode": "off", "interval_ms": None} + meta["comparison_memory"] = dict(meta["memory_collection"]) + (source / "metadata.json").write_text(json.dumps(meta)) + provenance = { + "source_revision": "recovery-revision", + "source_diff_sha256": "recovery-diff", + "script_sha256": {Path(k).name: v for k, v in meta["script_sha256"].items()}, + } + monkeypatch.setattr(recovery, "validate_environment", lambda *args: None) + monkeypatch.setattr( + recovery, + "reference_preflight", + lambda *args: pytest.fail("Comparisons were not requested"), + ) + return recovery, source, meta, provenance + + +@pytest.mark.parametrize("partial_row", [False, True]) +def test_recovery_resumes_only_partial_or_missing_core_results( + tmp_path, monkeypatch, partial_row +): + recovery, source, meta, provenance = _recovery_fixture(tmp_path, monkeypatch) + baseline = tmp_path / "baseline" + shutil.copytree(source, baseline) + missing = Path("replay/r1/pdfium/report.json") + (source / missing).unlink() + partial = Path("native/render-only/r1/t1.csv") + if partial_row: + lines = (source / partial).read_text().splitlines() + (source / partial).write_text( + lines[0] + "\n" + ",".join(lines[1].split(",")[:8]) + "\n" + ) + else: + (source / partial).write_text("schema_version,mode\n") + original = { + p.relative_to(source): p.read_bytes() for p in source.rglob("*") if p.is_file() + } + output = tmp_path / "recovered" + output.mkdir() + resumed = recovery.prepare_recovery(source, output, provenance) + assert set(resumed["recovery"]["pending_core"]) == {str(missing), str(partial)} + assert resumed["comparison"] is False + assert resumed["threads"] == meta["threads"] + assert resumed["repeats"] == meta["repeats"] + assert resumed["memory_collection"] == meta["memory_collection"] + assert (output / "replay/r1/t1").is_symlink() + assert not (output / "native").is_symlink() + executed = [] + + def execute(command, log, commands_path=None): + executed.append(command) + if "--timing-csv" in command: + artifact = Path(command[command.index("--timing-csv") + 1]) + shutil.copy2(baseline / partial, artifact) + shutil.copy2((baseline / partial).with_suffix(".log"), log) + else: + directory = Path(command[command.index("--output-dir") + 1]) + shutil.copytree(baseline / missing.parent, directory) + report = json.loads((directory / "report.json").read_text()) + report["provenance"] = { + "parse_git": { + "revision": provenance["source_revision"], + "diff_sha256": provenance["source_diff_sha256"], + }, + "script_sha256": provenance["script_sha256"][ + "run_performance_benchmarking_with_docling_settings.py" + ], + } + (directory / "report.json").write_text(json.dumps(report)) + + monkeypatch.setattr(recovery, "run_logged", execute) + recovery.resume_core(resumed, output, True) + assert len(executed) == 2 + assert len(recovery.ReportInputs(output).load()) == 4 + assert original == { + p.relative_to(source): p.read_bytes() for p in source.rglob("*") if p.is_file() + } + # A second interruption/recovery can reuse results collected in the first recovery. + next_output = tmp_path / "recovered-again" + next_output.mkdir() + again = recovery.prepare_recovery(output, next_output, provenance) + assert again["recovery"]["pending_core"] == [] + assert len(recovery.ReportInputs(next_output).load()) == 4 + + +def test_recovery_rejects_changed_replay_provenance(tmp_path, monkeypatch): + recovery, source, _, provenance = _recovery_fixture(tmp_path, monkeypatch) + report_path = source / "replay/r1/t1/report.json" + report = json.loads(report_path.read_text()) + report["provenance"]["parse_git"]["diff_sha256"] = "changed" + report_path.write_text(json.dumps(report)) + output = tmp_path / "out" + output.mkdir() + with pytest.raises(ValueError, match="source diff changed"): + recovery.prepare_recovery(source, output, provenance) + assert not (output / "metadata.json").exists() + + +def test_recovery_rejects_dataset_changes(tmp_path, monkeypatch): + recovery, source, meta, provenance = _recovery_fixture(tmp_path, monkeypatch) + Path(meta["dataset"]["files"][0]["path"]).write_bytes(b"changed PDF") + output = tmp_path / "out" + output.mkdir() + with pytest.raises(ValueError, match="dataset differs"): + recovery.prepare_recovery(source, output, provenance) + + +@pytest.mark.parametrize("change", ["script", "memory", "environment"]) +def test_incomplete_recovery_rejects_incompatible_resume_settings( + tmp_path, monkeypatch, change +): + recovery, source, meta, provenance = _recovery_fixture(tmp_path, monkeypatch) + (source / "replay/r1/pdfium/report.json").unlink() + override = None + if change == "script": + meta["script_sha256"][ + "scripts/benchmarking/run_performance_benchmarking_with_docling_settings.py" + ] = "changed" + elif change == "memory": + override = {"mode": "profile", "interval_ms": 50} + else: + meta["environment"] = {"DOCLING_PARSE_FALLBACK_FONT": "original-font.ttf"} + monkeypatch.delenv("DOCLING_PARSE_FALLBACK_FONT", raising=False) + (source / "metadata.json").write_text(json.dumps(meta)) + output = tmp_path / "out" + output.mkdir() + with pytest.raises(ValueError, match=r"changed|memory settings"): + recovery.prepare_recovery(source, output, provenance, override) + assert not (output / "metadata.json").exists() + + +def test_resume_command_preserves_original_profile_instrumentation( + tmp_path, monkeypatch +): + recovery, _, meta, _ = _recovery_fixture(tmp_path, monkeypatch) + meta["memory_collection"] = {"mode": "profile", "interval_ms": 75} + for job in recovery.core_jobs(meta): + command, log = recovery.core_command(meta, job, tmp_path, True) + assert command[command.index("--memory-mode") + 1] == "profile" + assert command[command.index("--memory-interval-ms") + 1] == "75" + assert command[command.index("--output") + 1] == str( + log.with_suffix(".memory.json") + ) + if job.workload == "native": + assert ( + f"--create-word-cells={str(job.profile != 'render-only').lower()}" + in command + ) + else: + assert command[command.index("--backend") + 1] == job.profile + + +@pytest.mark.skipif(os.name == "nt", reason="Bash launcher") +@pytest.mark.parametrize("explicit", [False, True]) +def test_recovery_launcher_preserves_inherited_memory_defaults(tmp_path, explicit): + _, launcher = _copy_launcher(tmp_path) + binaries = tmp_path / "bin" + binaries.mkdir() + uv = binaries / "uv" + uv.write_text( + '#!/bin/sh\nprintf "%s\\n" "$@"\nprintf "memory_env=%s\\n" "${MEMORY_MODE-unset}"\n' + ) + uv.chmod(0o755) + env = { + k: v + for k, v in os.environ.items() + if k not in ("MEMORY_MODE", "MEMORY_INTERVAL_MS", "BUILD_MODE", "OUT") + } + env.update( + PATH=str(binaries) + os.pathsep + os.environ["PATH"], RECOVER_FROM="source" + ) + arguments = ( + ["--memory-mode", "profile", "--memory-interval-ms", "75"] if explicit else [] + ) + result = subprocess.run( + ["bash", str(launcher), *arguments], env=env, text=True, capture_output=True + ) + assert result.returncode == 0, result.stderr + assert "memory_env=unset" in result.stdout + if explicit: + assert "--memory-mode\nprofile\n--memory-interval-ms\n75" in result.stdout + else: + assert "--memory-mode" not in result.stdout diff --git a/tests/test_unit_performance_runtime.py b/tests/test_unit_performance_runtime.py index 06d7e1ef5..13e5519f7 100644 --- a/tests/test_unit_performance_runtime.py +++ b/tests/test_unit_performance_runtime.py @@ -8,6 +8,7 @@ import sys from pathlib import Path from types import SimpleNamespace +from typing import Any import pytest @@ -141,7 +142,7 @@ def test_recovery_forks_core_without_relabeling_versions(tmp_path: Path, monkeyp packages = dict.fromkeys( ("docling-parse", "docling-core", "pypdfium2", "pillow", "rtree"), "fixture" ) - metadata = { + metadata: dict[str, Any] = { "extension_sha256": "extension", "native_sha256": "native", "python": "python", @@ -156,6 +157,11 @@ def test_recovery_forks_core_without_relabeling_versions(tmp_path: Path, monkeyp ] }, } + metadata.update(repeats=1, threads=[1], comparison=True) + for job in _RECOVERY.core_jobs(metadata): + artifact = source / job.artifact + artifact.parent.mkdir(parents=True, exist_ok=True) + artifact.write_text("unchanged measurements") (source / "metadata.json").write_text(json.dumps(metadata)) original = (source / "metadata.json").read_bytes() provenance = { @@ -175,6 +181,7 @@ def load(self, *, include_comparison): return ["validated core"] monkeypatch.setattr(_RECOVERY, "ReportInputs", Inputs) + monkeypatch.setattr(_RECOVERY, "inspect_job", lambda *_: True) monkeypatch.setattr(_RECOVERY, "validate_environment", lambda meta, prov: None) monkeypatch.setattr( _RECOVERY, "reference_preflight", lambda meta: {"status": "passed"} @@ -184,9 +191,65 @@ def load(self, *, include_comparison): assert result["comparison_provenance"]["packages"]["pymupdf"] == "1.27.2.3" assert (source / "metadata.json").read_bytes() == original assert (output / "original-metadata.json").read_bytes() == original + for job in _RECOVERY.core_jobs(metadata): + artifact = output / job.artifact + assert artifact.resolve() == (source / job.artifact).resolve() + assert artifact.read_text() == "unchanged measurements" + assert not (output / "native").is_symlink() + + +@pytest.mark.skipif(os.name == "nt", reason="Recovery uses Unix symlinks") +@pytest.mark.parametrize("same_settings", [True, False]) +def test_recovery_copies_memory_with_compatible_checkpoints( + tmp_path: Path, monkeypatch, same_settings +): + source, output = tmp_path / "source", tmp_path / "output" + source.mkdir() + output.mkdir() for name in ("native", "replay"): - assert (output / name).is_symlink() - assert (output / name / "raw").read_text() == "unchanged measurements" + (source / name).mkdir() + series = source / "comparison/r1/render" + (series / "checkpoints").mkdir(parents=True) + (series / "memory").mkdir() + (series / "checkpoints/run.json").write_text("original checkpoint") + (series / "memory/run.memory.rss.csv").write_text("original samples") + provenance = {"packages": {"pymupdf": "fixture"}} + metadata = { + "repeats": 1, + "comparison": True, + "dataset": {"files": [], "root": "/pdf"}, + "threads": [1, 4], + "scale": 1, + "max_concurrent_results": 32, + "memory_collection": {"mode": "peak", "interval_ms": None}, + "comparison_memory": {"mode": "profile", "interval_ms": 50}, + "comparison_provenance": provenance, + } + (source / "metadata.json").write_text(json.dumps(metadata)) + monkeypatch.setattr( + _RECOVERY, + "ReportInputs", + lambda _: SimpleNamespace( + metadata=metadata, load=lambda **_: ["existing core"] + ), + ) + monkeypatch.setattr(_RECOVERY, "validate_environment", lambda *_: None) + monkeypatch.setattr(_RECOVERY, "core_jobs", lambda _: []) + monkeypatch.setattr( + _RECOVERY, "reference_preflight", lambda _: {"status": "passed"} + ) + requested = {"mode": "profile", "interval_ms": 50 if same_settings else 25} + result = _RECOVERY.prepare_recovery(source, output, provenance, requested) + assert result["memory_collection"] == metadata["memory_collection"] + assert result["comparison_memory"] == requested + target = output / "comparison/r1/render" + assert (target / "checkpoints/run.json").exists() == same_settings + assert (target / "memory/run.memory.rss.csv").exists() == same_settings + command = _RECOVERY.comparison_command(result, target, "render", False) + assert command[command.index("--memory-mode") + 1] == "profile" + assert command[command.index("--memory-interval-ms") + 1] == str( + requested["interval_ms"] + ) @pytest.mark.parametrize( diff --git a/tests/test_unit_soft_mask_streaming.py b/tests/test_unit_soft_mask_streaming.py new file mode 100644 index 000000000..8715e67a8 --- /dev/null +++ b/tests/test_unit_soft_mask_streaming.py @@ -0,0 +1,53 @@ +"""Soft-mask callbacks while the page and nested Form parsers are active.""" + +from __future__ import annotations + +import zlib + +import pytest + +from tests.pdf_builder import render_page, simple_page_pdf, stream_object + + +@pytest.mark.parametrize("damaged", [False, True]) +@pytest.mark.parametrize("subtype", ["Alpha", "Luminosity"]) +def test_transformed_mask_keeps_local_resources_and_recovers_once( + damaged: bool, subtype: str +) -> None: + # /G1 means the mask at page level and half-opacity within its group. + # The group's matrix and the CTM at gs together place its white rectangle + # at [30,30,70,50]. Replaying a recovered prefix twice makes it too opaque. + source = b"/G1 gs /Leaf Do\n" + if damaged: + encoder = zlib.compressobj() + source = encoder.compress(source) + encoder.flush(zlib.Z_SYNC_FLUSH) + b"\x07" + group = stream_object( + "/Type /XObject /Subtype /Form /BBox [0 0 30 20] " + "/Matrix [2 0 0 1 10 20] /Group << /S /Transparency /CS /DeviceGray >> " + "/Resources << /ExtGState << /G1 << /ca 0.5 >> >> " + "/XObject << /Leaf 6 0 R >> >>" + (" /Filter /FlateDecode" if damaged else ""), + source, + ) + leaf = stream_object( + "/Type /XObject /Subtype /Form /BBox [0 0 20 20]", + b"1 g 0 0 20 20 re f 1 0 0 1 100 0 cm", + ) + pdf = simple_page_pdf( + "1 0 0 rg 0 0 200 200 re f 0 0 1 rg " + "q 1 0 0 1 20 10 cm /G1 gs 0 20 30 20 re f " + "/G1 gs 30 20 40 20 re f Q 150 20 10 10 re f", + resources=f"/ExtGState << /G1 << /SMask << /S /{subtype} /G 5 0 R >> >> >>", + extra_objects=[group, leaf], + ) + image = render_page(pdf, scale=1.0).get_image().convert("RGB") + + def pixel(x: int, y: int): + return image.getpixel((x, image.height - 1 - y)) + + for x in (40, 60): + assert pixel(x, 40) == pytest.approx((128, 0, 128), abs=3) + # One point outside the group's box and one inside its unpainted portion. + for x in (25, 80): + assert pixel(x, 40) == (255, 0, 0) + assert pixel(155, 25) == (0, 0, 255) + assert pixel(40, 70) == (255, 0, 0) diff --git a/tests/test_unit_split_content_streams.py b/tests/test_unit_split_content_streams.py index 84cecee3a..8f95ee881 100644 --- a/tests/test_unit_split_content_streams.py +++ b/tests/test_unit_split_content_streams.py @@ -53,3 +53,30 @@ def test_damaged_stream_does_not_lose_the_intact_one(): ) assert lines == ["intact"] + + +def test_operands_split_across_streams_reach_their_operator(): + lines = _lines( + content_stream("BT /F1 12 Tf 20 250 Td (before) Tj 0 -20"), + content_stream("Td (after) Tj ET"), + ) + + assert lines == ["before", "after"] + + +def test_damaged_middle_stream_keeps_prefix_and_suffix_exactly_once(): + pdf = build_pdf( + [ + "<< /Type /Catalog /Pages 2 0 R >>", + "<< /Type /Pages /Kids [3 0 R] /Count 1 >>", + "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 300]" + " /Resources << /Font << /F1 7 0 R >> >>" + " /Contents [4 0 R 5 0 R 6 0 R] >>", + content_stream("BT /F1 12 Tf 20 250 Td (before) Tj ET"), + stream_object("/Filter /FlateDecode", b"this is not zlib data"), + content_stream("BT /F1 12 Tf 20 200 Td (after) Tj ET"), + FONT, + ] + ) + + assert [cell.text for cell in parse_page(pdf).textline_cells] == ["before", "after"] diff --git a/tests/test_unit_type3_streaming.py b/tests/test_unit_type3_streaming.py new file mode 100644 index 000000000..f28b7f7a2 --- /dev/null +++ b/tests/test_unit_type3_streaming.py @@ -0,0 +1,117 @@ +"""Procedural Type 3 callbacks, including queue and recursion boundaries.""" + +from __future__ import annotations + +import zlib + +import pytest +from docling_core.types.doc.page import TextCellUnit + +from tests.pdf_builder import ( + build_pdf, + content_stream, + parse_page, + render_page, + stream_object, +) + + +@pytest.mark.parametrize("damaged", [False, True]) +def test_queued_charprocs_keep_font_resources_and_restore_parent_state( + damaged: bool, +) -> None: + # Strokes require the procedural fallback. Two glyphs share one Tj queue; + # half-opacity ink reveals an accidental replay as a darker stroke. + source = b"1000 0 d0 /G1 gs 1 0 0 RG 100 w 0 0 m 1000 1000 l S\n1 0 0 1 1000 0 cm\n" + if damaged: + encoder = zlib.compressobj() + source = encoder.compress(source) + encoder.flush(zlib.Z_SYNC_FLUSH) + b"\x07" + pdf = build_pdf( + [ + "<< /Type /Catalog /Pages 2 0 R >>", + "<< /Type /Pages /Kids [3 0 R] /Count 1 >>", + "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 200 200] " + "/Resources << /Font << /T3 5 0 R >> /ExtGState << /G1 << /CA 0.25 >> >> >> " + "/Contents 4 0 R >>", + content_stream( + "/G1 gs 0 0 1 RG BT /T3 40 Tf 20 30 Td (aa) Tj ET " + "4 w 140 20 m 180 60 l S" + ), + "<< /Type /Font /Subtype /Type3 /FontBBox [0 0 1000 1000] " + "/FontMatrix [0.001 0 0 0.001 0 0] /CharProcs 6 0 R " + "/Encoding << /Type /Encoding /Differences [97 /a] >> " + "/FirstChar 97 /LastChar 97 /Widths [1000] " + "/Resources << /ExtGState << /G1 << /CA 0.5 >> >> >> >>", + "<< /a 7 0 R >>", + stream_object("/Filter /FlateDecode" if damaged else "", source), + ] + ) + image = render_page(pdf, scale=1.0).get_image().convert("RGB") + + for x in (40, 80): + assert image.getpixel((x, 149)) == pytest.approx((255, 128, 128), abs=2) + assert image.getpixel((160, 159)) == pytest.approx((191, 191, 255), abs=2) + + +def test_eager_repair_preserves_damaged_cached_glyph_bounds() -> None: + # The font's declared box is deliberately loose. Eager validation must + # repair the usable prefix before lazy mask analysis computes ink bounds; + # otherwise analysis fails and the text cell falls back to the wrong box. + source = b"1 0 -10 -10 10 10 d1\n0 -0.2 0.6 1 re f\n" + encoder = zlib.compressobj() + damaged = encoder.compress(source) + encoder.flush(zlib.Z_SYNC_FLUSH) + b"\x07" + pdf = build_pdf( + [ + "<< /Type /Catalog /Pages 2 0 R >>", + "<< /Type /Pages /Kids [3 0 R] /Count 1 >>", + "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 200 200] " + "/Resources << /Font << /T3 5 0 R >> >> /Contents 4 0 R >>", + content_stream("BT /T3 100 Tf 50 50 Td (a) Tj ET"), + "<< /Type /Font /Subtype /Type3 /FontBBox [-10 -10 10 10] " + "/FontMatrix [1 0 0 1 0 0] /CharProcs 6 0 R " + "/Encoding << /Type /Encoding /Differences [97 /a] >> " + "/FirstChar 97 /LastChar 97 /Widths [1] /Resources << >> >>", + "<< /a 7 0 R >>", + stream_object("/Filter /FlateDecode", damaged), + ] + ) + cells = list(parse_page(pdf).iterate_cells(TextCellUnit.CHAR)) + assert len(cells) == 1 + box = cells[0].rect.to_bounding_box() + assert box.width == pytest.approx(60.0, abs=0.1) + assert box.height == pytest.approx(100.0, abs=0.1) + + +def test_recursive_charproc_stops_at_eight_and_the_next_glyph_still_paints() -> None: + recursive = ( + b"1000 0 d0 100 w 0 0 m 1000 1000 l S BT /T3 1000 Tf 1 0 0 1 0 0 Tm (a) Tj ET" + ) + plain = b"1000 0 d0 100 w 0 0 m 1000 1000 l S" + pdf = build_pdf( + [ + "<< /Type /Catalog /Pages 2 0 R >>", + "<< /Type /Pages /Kids [3 0 R] /Count 1 >>", + "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 200 200] " + "/Resources << /Font << /T3 5 0 R >> /ExtGState << /G1 << /CA 0.25 >> >> >> " + "/Contents 4 0 R >>", + content_stream( + "/G1 gs 0 G BT /T3 40 Tf 20 30 Td (a) Tj " + "1 0 0 1 80 30 Tm (b) Tj ET " + "0 0 1 RG 4 w 140 20 m 180 60 l S" + ), + "<< /Type /Font /Subtype /Type3 /FontBBox [0 0 1000 1000] " + "/FontMatrix [0.001 0 0 0.001 0 0] /CharProcs 6 0 R " + "/Encoding << /Type /Encoding /Differences [97 /a /b] >> " + "/FirstChar 97 /LastChar 98 /Widths [1000 1000] " + "/Resources << /Font << /T3 5 0 R >> >> >>", + "<< /a 7 0 R /b 8 0 R >>", + stream_object("", recursive), + stream_object("", plain), + ] + ) + image = render_page(pdf, scale=1.0).get_image().convert("RGB") + + # Eight quarter-opacity strokes: white * (3/4)^8, allowing 8-bit rounding. + assert image.getpixel((40, 149)) == pytest.approx((26, 26, 26), abs=3) + assert image.getpixel((100, 149)) == pytest.approx((191, 191, 191), abs=2) + assert image.getpixel((160, 159)) == pytest.approx((191, 191, 255), abs=2)