diff --git a/.agent-loop/initiatives/WS-ART-001-immutable-artifact-storage/STATUS.md b/.agent-loop/initiatives/WS-ART-001-immutable-artifact-storage/STATUS.md index 1b7e452a4..c2e04ab10 100644 --- a/.agent-loop/initiatives/WS-ART-001-immutable-artifact-storage/STATUS.md +++ b/.agent-loop/initiatives/WS-ART-001-immutable-artifact-storage/STATUS.md @@ -77,9 +77,12 @@ merge; repository-authored evidence alone is not authority. and adds bounded passive-PDF text extraction inside the existing isolated child. -03B3B3A is the active successor. It installs only the approved `defusedxml` -wheel and adds the shared bounded OPC/OOXML container security capability. It -does not activate a DOCX, PPTX, or XLSX extractor, AUTH, or sufficiency. +03B3B3A merged through PR #233. It installs only the approved `defusedxml` +wheel and adds the shared bounded OPC/OOXML container security capability. +03B3B3B is the active successor. Its bounded DOCX extraction and durable +omission-fact implementation passed focused tests and required internal review; +hosted CI, CodeRabbit, and human merge remain pending. PPTX, XLSX, image, AUTH, +and sufficiency work remain inactive. AUTH `WS-XINT-002-04B` follows the complete hidden split-03B series and activates only fixed-service binding and guide read. ART-03C then removes the diff --git a/.agent-loop/initiatives/WS-ART-001-immutable-artifact-storage/chunks/WS-ART-001-03B3B3B-docx-extractor.md b/.agent-loop/initiatives/WS-ART-001-immutable-artifact-storage/chunks/WS-ART-001-03B3B3B-docx-extractor.md index 8abc0461d..6f0b61a74 100644 --- a/.agent-loop/initiatives/WS-ART-001-immutable-artifact-storage/chunks/WS-ART-001-03B3B3B-docx-extractor.md +++ b/.agent-loop/initiatives/WS-ART-001-immutable-artifact-storage/chunks/WS-ART-001-03B3B3B-docx-extractor.md @@ -30,9 +30,15 @@ P2 ```text backend/app/modules/artifacts/guide_docx.py backend/app/modules/artifacts/guide_extraction.py +backend/app/modules/artifacts/guide_extraction_service.py backend/app/modules/artifacts/guide_extraction_worker.py +backend/scripts/run_test_lanes.py +backend/tests/test_artifact_architecture.py backend/tests/test_guide_docx.py +backend/tests/test_guide_extraction.py +backend/tests/test_guide_bindings.py backend/tests/fixtures/guide_docx/** +backend/tests/fixtures/guide_extraction_probe_worker.py docs/spec_artifact_storage_service.md .agent-loop/initiatives/WS-ART-001-immutable-artifact-storage/** ``` @@ -50,16 +56,39 @@ PPTX/XLSX behavior, package additions beyond approved lock, framework/AUTH/Celer comments, tracked-deletion text, and embedded objects are omitted and recorded in omission facts. Streaming stops with `limit_exceeded` before exceeding D42's 4 MiB canonical-output limit; no partial result is usable. +- Canonical DOCX output is compact sorted JSON with one `blocks` array. A + paragraph block is `{"type":"paragraph","text":"..."}`. A table block is + `{"type":"table","text":"..."}` with tab between cells, newline between + rows, newline between multiple paragraphs in a cell, and a newline on each + side of a nested table when adjacent cell content exists. Empty paragraphs, + rows, and cells remain explicit. `w:t` and visible field-result text are + retained; `w:tab` becomes tab and `w:br|w:cr` becomes newline. Hyperlink + display text stays in document order. Field instructions, hidden text, + comment markers, tracked deletions, drawings, pictures, and other passive + non-text body objects are omitted. Active embedded package content remains a + malformed OOXML rejection before DOCX extraction. Traversal deeper than 64 + nested document/container levels returns stable `malformed` evidence. +- Successful DOCX omission facts have the fixed bounded boolean schema + `truncated`, `omitted`, `headers`, `footers`, `comments`, + `tracked_deletions`, `embedded_objects`, `hidden_text`, and + `field_instructions`. The existing isolated-child result protocol carries these + facts; other formats retain the exact default + `{"truncated":false,"omitted":false}`. Persistence binds and replay-checks + omission facts with the canonical output. DOCX uses exact policy identity + `guide-extraction-v3`, so obsolete unsupported evidence cannot replay. - Import the parser only in the isolated child and prove deterministic output, malformed/unsafe input, exact output boundary, crash, timeout, cancellation, cleanup, approval-gate, and coverage behavior. +- Assign the focused DOCX test module to an existing canonical hosted semantic + lane without changing lane or coverage policy. ## Verification commands ```bash (cd backend && uv run ruff check app tests) -(cd backend && python scripts/check_guide_extractor_dependencies.py) -(cd backend && uv run pytest -q tests/test_guide_ooxml.py tests/test_guide_docx.py tests/test_guide_extraction.py --cov=app.modules.artifacts --cov-report=term-missing --cov-fail-under=90) +(cd backend && uv run python scripts/check_guide_extractor_dependencies.py) +(cd backend && uv run pytest -q tests/test_guide_ooxml.py tests/test_guide_docx.py tests/test_guide_extraction.py tests/test_guide_bindings.py tests/test_artifact_architecture.py) +(cd backend && uv run pytest -q tests/test_guide_docx.py --cov=app.modules.artifacts.guide_docx --cov-report=term-missing --cov-fail-under=90) (metadata_dir="$(mktemp -d)" && trap 'rm -rf "$metadata_dir"' EXIT && (cd backend && WORKSTREAM_TEST_ADMIN_DATABASE_URL=postgresql+asyncpg://workstream:workstream@localhost:5433/postgres .venv/bin/python scripts/run_isolated_tests.py --metadata-json "$metadata_dir/result.json" --timeout-seconds 12600 -- .venv/bin/python -m pytest -q --ignore=tests/test_isolated_database_runner.py --cov=app --cov-report=term-missing --cov-fail-under=78)) python3 scripts/check_stale_artifact_contracts.py python3 scripts/check_markdown_links.py diff --git a/.agent-loop/initiatives/WS-ART-001-immutable-artifact-storage/reviews/WS-ART-001-03B3B3B-external-review-response.md b/.agent-loop/initiatives/WS-ART-001-immutable-artifact-storage/reviews/WS-ART-001-03B3B3B-external-review-response.md new file mode 100644 index 000000000..f8695611c --- /dev/null +++ b/.agent-loop/initiatives/WS-ART-001-immutable-artifact-storage/reviews/WS-ART-001-03B3B3B-external-review-response.md @@ -0,0 +1,42 @@ +# External Review Response: WS-ART-001-03B3B3B + +## Comments addressed + +- GitHub Agent Gates flagged `worker protocol` as stale human-worker + vocabulary in the chunk contract and storage specification. Both references + now say `isolated-child result protocol`, and the exact stale authorization + documentation check passes locally. +- The first Backend gate reported `current_node_inventory_mismatch`. The DOCX + database-test parameter used generated ZIP bytes as its pytest ID, so ZIP + timestamps changed the collected node between the lane and independent + inventory passes. Stable explicit `json` and `docx` IDs now make repeated + collections identical. +- CodeRabbit reported four valid DOCX hardening/test gaps. The extractor now + returns stable `malformed/docx_nesting_limit` beyond 64 recursive levels, + preserves `w:sdt`-wrapped rows and cells, resolves validated OOXML part names + through their case-folded stored-name map, and proves the full nine-key + omission schema across the isolated-child JSON round trip. + +## Comments deferred + +None. + +## Human decisions needed + +None. + +## Commands rerun + +- `python3 scripts/check_stale_authorization_docs.py` +- `python3 scripts/check_markdown_links.py` +- `git diff --check` +- repeated `pytest --collect-only` for the affected binding module +- `ruff check tests/test_guide_bindings.py` +- `ruff check app tests scripts` +- focused DOCX/OOXML/extraction/architecture/lane suite (154 passed) +- DOCX coverage suite (12 passed, 93.24 percent) + +## Remaining risks + +Hosted Backend/Agent Gates and CodeRabbit completion remain required on the +repaired PR head before merge readiness. diff --git a/.agent-loop/initiatives/WS-ART-001-immutable-artifact-storage/reviews/WS-ART-001-03B3B3B-internal-review-evidence.md b/.agent-loop/initiatives/WS-ART-001-immutable-artifact-storage/reviews/WS-ART-001-03B3B3B-internal-review-evidence.md new file mode 100644 index 000000000..1e0e30471 --- /dev/null +++ b/.agent-loop/initiatives/WS-ART-001-immutable-artifact-storage/reviews/WS-ART-001-03B3B3B-internal-review-evidence.md @@ -0,0 +1,65 @@ +# Internal Review Evidence: WS-ART-001-03B3B3B + +Reviewed against trusted main: `2f441e99f1af` + +Reviewed at: `2026-07-31` + +## Candidate + +Hidden deterministic DOCX extraction on the merged shared OOXML boundary. The +candidate emits bounded canonical content and durable omission facts without +activating AUTH, guide sufficiency, PPTX/XLSX, or contributor submissions. + +## Deterministic Evidence + +- Ruff, approved extractor-dependency gate, stale artifact contracts, stale + wording, Markdown links, lane integrity, and `git diff --check`: PASS; +- focused DOCX, OOXML, extraction, architecture, and lane suite: 154 passed; +- focused DOCX adapter suite: 12 passed at 93.24 percent coverage; +- DB-backed binding/replay tests and repository-wide coverage remain delegated + to the hosted sharded Backend gate; no local full-suite run was used. + +## Reviewer Results + +| Reviewer | Result | Blocking findings | +|---|---|---| +| architecture | PASS | none | +| security/auth | PASS WITH LOW RISKS | none | +| QA/test | PASS WITH LOW RISKS | none | +| senior engineering | PASS WITH LOW RISKS | none | +| product/ops | PASS WITH LOW RISKS | none | +| reuse/dedup | PASS WITH LOW RISKS | none | +| CI integrity | PASS | none | +| test delta | PASS | none | +| docs | PASS | none | + +## Material Repairs + +- made body traversal omission-aware so block-level deletions and move-from + content cannot enter canonical output; +- covered direct, inherited character, inherited paragraph, and document- + default hidden-text semantics and recorded simple-field instructions; +- restored the worker-owned shared OOXML loader instead of adding a second + production validation path; +- made omission facts part of the bounded worker protocol, immutable persisted + evidence, and replay identity; +- corrected invalid-protocol tests so every assertion reaches its intended + validation branch; +- documented the exact canonical blocks and fixed omission schema. +- bounded recursive traversal, preserved content-control-wrapped rows/cells, + and resolved case-variant validated OOXML part names after external review. + +## Accepted Low Risks + +- Explicit false values on `w:vanish` are conservatively treated as hidden; + exact WordprocessingML visibility fidelity can be hardened later. +- The isolated child assembles a bounded block before the exact 4 MiB final + check; existing input, decompression, memory, CPU, wall-time, and parent + protocol limits prevent partial durable output. +- Parent and child deliberately repeat their small omission schemas so the + isolated protocol is independently validated; future formats should avoid + allowing those literals to proliferate. + +Valid findings addressed: yes + +Open sub-agent sessions: none diff --git a/.agent-loop/initiatives/WS-ART-001-immutable-artifact-storage/reviews/WS-ART-001-03B3B3B-pr-trust-bundle.md b/.agent-loop/initiatives/WS-ART-001-immutable-artifact-storage/reviews/WS-ART-001-03B3B3B-pr-trust-bundle.md new file mode 100644 index 000000000..2a1b311f5 --- /dev/null +++ b/.agent-loop/initiatives/WS-ART-001-immutable-artifact-storage/reviews/WS-ART-001-03B3B3B-pr-trust-bundle.md @@ -0,0 +1,108 @@ +# WS-ART-001-03B3B3B PR Trust Bundle + +## Chunk + +`WS-ART-001-03B3B3B` — DOCX Extractor (L1). + +## Goal and human-approved intent + +Add deterministic, bounded DOCX text and table extraction after exact DOCX +classification and the merged OOXML security boundary. Preserve original +verified guide bytes as authoritative. This hidden chunk must not activate +AUTH, invoke guide sufficiency, add other formats, or touch contributor ZIPs. + +## What changed and why + +- Added an isolated DOCX adapter that emits compact sorted JSON blocks in + document order with fixed paragraph, row, cell, and nested-table semantics. +- Added a fixed omission schema for headers, footers, comments, deletions, + passive objects, hidden text, and field instructions. +- Advanced DOCX evidence to `guide-extraction-v3` and bound omission facts to + immutable persistence and replay checks. +- Reused the existing worker-owned OOXML validator loader and kept both the + validator and DOCX adapter imports confined to the isolated child. +- Added focused tests, the existing hosted semantic lane assignment, and the + exact storage-service contract. + +## Design chosen and alternatives rejected + +The isolated worker composes the shared OOXML validator with the DOCX adapter +before descriptor-only seccomp. The adapter receives an explicit validator, +then parses only validated `word/document.xml` and bounded style facts. The +parent accepts only the exact result and omission schemas. Rejected alternatives +were upload-request parsing, direct provider access, a second OOXML validation +path, raw binary agent input, partial output, or generic document authority. + +## Scope control and product behavior + +Only hidden guide-source extraction changes. There is no route, authorization +activation, Celery continuation, sufficiency invocation, provider read, guide +binding, submission, checker, review, contribution, payment, or reputation +change. Unsupported and malformed content remains a bounded internal artifact +outcome, not a guide-insufficiency decision. + +## Acceptance criteria proof + +- Paragraphs, hyperlinks, tabs, breaks, tables, cells, nested tables, and empty + structures have exact canonical output tests. +- Headers, footers, comments, tracked deletion/move-from content, passive + objects, fields, direct/inherited style-hidden text, and document-default + hidden text are omitted and recorded. +- Active content and malformed XML fail through stable bounded outcomes. +- Exact output-limit behavior returns no partial usable result. +- The real isolated runner proves v3 policy, omission transport, and scratch + cleanup; architecture tests prove worker-only adapter/validator imports. +- Persistence tests bind canonical output and omission facts and reject obsolete + policy evidence as a replay target. + +## Tests and checks + +- Ruff and approved extractor-dependency gate — pass. +- Focused DOCX/OOXML/extraction/architecture/lane suite — 154 passed. +- DOCX module coverage — 93.24 percent (12 tests). +- Stale artifact contracts, stale wording, Markdown links, lane integrity, and + `git diff --check` — pass. +- Hosted Backend/Agent Gates retain DB-backed replay, repository-wide coverage, + and semantic-lane proof; no local full-suite run was used. + +## Test delta and CI integrity + +No test, assertion, lane, workflow, dependency rule, or coverage threshold was +removed, skipped, or weakened. Invalid result-shape fixtures carry valid +omission facts so they still reach their intended validation branches. The new +DOCX module joins the existing `shared_foundations` lane. + +## Reviewer results + +Architecture, CI integrity, test delta, and docs pass. Security, QA, senior +engineering, product/ops, and reuse/dedup pass with only documented low risks. +All blocking deletion traversal, hidden-style cascade, isolated-runner, shared- +loader reuse, protocol-test, and documentation findings were repaired and +re-reviewed. + +## External review + +The first Agent Gates run found two stale human-worker vocabulary matches; both +now use the exact isolated-child result-protocol wording and the local gate +passes. The first Backend run exposed a nondeterministic raw-ZIP pytest ID; +explicit stable format IDs now produce identical repeated collections. +CodeRabbit's first attempt was rate-limited without code findings and must be +retriggered. Its completed review then identified four valid gaps: recursive +depth, content-control-wrapped rows/cells, case-folded part resolution, and the +cross-process omission assertion. All four are repaired with focused regression +proof. Hosted checks remain required on the repaired PR head. + +## Remaining risks and follow-up + +DOCX has a large visibility model; this chunk intentionally supports a bounded +subset and treats `w:vanish` presence conservatively, including explicit false +values. Future formats should reuse the same isolated result protocol without +growing duplicated schemas. PPTX/XLSX, durable sufficiency continuation, and +AUTH activation remain separate later chunks. + +## Human review focus and merge ownership + +Review exact canonical ordering/separators, omission semantics, injected shared +OOXML validation, v3 replay identity, isolation, and absence of AUTH or +sufficiency activation. A human owns merge approval; the agent will not merge +this PR. diff --git a/backend/app/modules/artifacts/guide_docx.py b/backend/app/modules/artifacts/guide_docx.py new file mode 100644 index 000000000..76354bcf1 --- /dev/null +++ b/backend/app/modules/artifacts/guide_docx.py @@ -0,0 +1,382 @@ +"""Deterministic bounded DOCX extraction after shared OOXML validation.""" + +from __future__ import annotations + +import codecs +from dataclasses import dataclass +from io import BytesIO +import json +from xml.etree import ElementTree +import zipfile +from collections.abc import Callable + + +# Preload the ZIP filename codec before the isolated worker installs seccomp. +codecs.lookup("cp437") +MAXIMUM_OUTPUT_BYTES = 4 * 1024 * 1024 +MAXIMUM_NESTING_DEPTH = 64 +_WORD_NS = "http://schemas.openxmlformats.org/wordprocessingml/2006/main" +_BODY_TAG = f"{{{_WORD_NS}}}body" +_PARAGRAPH_TAG = f"{{{_WORD_NS}}}p" +_TABLE_TAG = f"{{{_WORD_NS}}}tbl" +_ROW_TAG = f"{{{_WORD_NS}}}tr" +_CELL_TAG = f"{{{_WORD_NS}}}tc" +_RUN_TAG = f"{{{_WORD_NS}}}r" +_RUN_PROPERTIES_TAG = f"{{{_WORD_NS}}}rPr" +_RUN_STYLE_TAG = f"{{{_WORD_NS}}}rStyle" +_PARAGRAPH_PROPERTIES_TAG = f"{{{_WORD_NS}}}pPr" +_PARAGRAPH_STYLE_TAG = f"{{{_WORD_NS}}}pStyle" +_STYLE_TAG = f"{{{_WORD_NS}}}style" +_BASED_ON_TAG = f"{{{_WORD_NS}}}basedOn" +_VALUE_ATTRIBUTE = f"{{{_WORD_NS}}}val" +_STYLE_ID_ATTRIBUTE = f"{{{_WORD_NS}}}styleId" +_STYLE_TYPE_ATTRIBUTE = f"{{{_WORD_NS}}}type" +_DOC_DEFAULTS_TAG = f"{{{_WORD_NS}}}docDefaults" +_RUN_PROPERTIES_DEFAULT_TAG = f"{{{_WORD_NS}}}rPrDefault" +_VANISH_TAG = f"{{{_WORD_NS}}}vanish" +_TEXT_TAG = f"{{{_WORD_NS}}}t" +_DELETION_TEXT_TAG = f"{{{_WORD_NS}}}delText" +_DELETION_TAG = f"{{{_WORD_NS}}}del" +_DELETION_CONTAINER_TAGS = frozenset({_DELETION_TAG, f"{{{_WORD_NS}}}moveFrom"}) +_TAB_TAG = f"{{{_WORD_NS}}}tab" +_BREAK_TAGS = frozenset({f"{{{_WORD_NS}}}br", f"{{{_WORD_NS}}}cr"}) +_FIELD_INSTRUCTION_TAG = f"{{{_WORD_NS}}}instrText" +_SIMPLE_FIELD_TAG = f"{{{_WORD_NS}}}fldSimple" +_FIELD_INSTRUCTION_ATTRIBUTE = f"{{{_WORD_NS}}}instr" +_COMMENT_TAGS = frozenset( + { + f"{{{_WORD_NS}}}commentRangeStart", + f"{{{_WORD_NS}}}commentRangeEnd", + f"{{{_WORD_NS}}}commentReference", + } +) +_EMBEDDED_BODY_TAGS = frozenset( + { + f"{{{_WORD_NS}}}drawing", + f"{{{_WORD_NS}}}object", + f"{{{_WORD_NS}}}pict", + } +) +_TRANSPARENT_CONTAINER_TAGS = frozenset( + { + f"{{{_WORD_NS}}}customXml", + f"{{{_WORD_NS}}}sdt", + f"{{{_WORD_NS}}}sdtContent", + } +) +_DOCX_OMISSION_KEYS = ( + "headers", + "footers", + "comments", + "tracked_deletions", + "embedded_objects", + "hidden_text", + "field_instructions", +) + + +@dataclass(frozen=True, slots=True) +class DocxExtraction: + """One canonical DOCX result and its fixed bounded omission facts.""" + + canonical_output: str + omission_facts: dict[str, bool] + + +@dataclass(frozen=True, slots=True) +class _HiddenStyleFacts: + default_hidden: bool + run_styles: frozenset[str] + paragraph_styles: frozenset[str] + + +class DocxExtractionFailure(Exception): + """Carry one bounded DOCX extraction outcome.""" + + def __init__(self, status: str, code: str) -> None: + super().__init__(code) + self.status = status + self.code = code + + +def _new_omissions(names: frozenset[str]) -> dict[str, bool]: + facts = { + "truncated": False, + "omitted": False, + "headers": any(name.startswith("word/header") and name.endswith(".xml") for name in names), + "footers": any(name.startswith("word/footer") and name.endswith(".xml") for name in names), + "comments": "word/comments.xml" in names, + "tracked_deletions": False, + "embedded_objects": False, + "hidden_text": False, + "field_instructions": False, + } + return facts + + +def _hidden_styles(styles: bytes | None) -> _HiddenStyleFacts: + if styles is None: + return _HiddenStyleFacts(False, frozenset(), frozenset()) + try: + root = ElementTree.fromstring(styles) + except ElementTree.ParseError as exc: + raise DocxExtractionFailure("malformed", "docx_invalid_styles_xml") from exc + hidden_by_type: dict[str, set[str]] = {"character": set(), "paragraph": set()} + parents_by_type: dict[str, dict[str, str]] = {"character": {}, "paragraph": {}} + for style in root.findall(_STYLE_TAG): + style_id = style.get(_STYLE_ID_ATTRIBUTE) + style_type = style.get(_STYLE_TYPE_ATTRIBUTE) + if not style_id or style_type not in hidden_by_type: + continue + properties = style.find(_RUN_PROPERTIES_TAG) + if properties is not None and properties.find(_VANISH_TAG) is not None: + hidden_by_type[style_type].add(style_id) + based_on = style.find(_BASED_ON_TAG) + parent_id = based_on.get(_VALUE_ATTRIBUTE) if based_on is not None else None + if parent_id: + parents_by_type[style_type][style_id] = parent_id + for style_type, hidden in hidden_by_type.items(): + changed = True + while changed: + inherited = { + style_id + for style_id, parent in parents_by_type[style_type].items() + if parent in hidden + } + changed = not inherited.issubset(hidden) + hidden.update(inherited) + defaults = root.find(_DOC_DEFAULTS_TAG) + run_defaults = defaults.find(_RUN_PROPERTIES_DEFAULT_TAG) if defaults is not None else None + default_properties = ( + run_defaults.find(_RUN_PROPERTIES_TAG) if run_defaults is not None else None + ) + return _HiddenStyleFacts( + default_hidden=( + default_properties is not None and default_properties.find(_VANISH_TAG) is not None + ), + run_styles=frozenset(hidden_by_type["character"]), + paragraph_styles=frozenset(hidden_by_type["paragraph"]), + ) + + +def _paragraph_text( + paragraph: ElementTree.Element, + omissions: dict[str, bool], + hidden_styles: _HiddenStyleFacts, + *, + depth: int = 0, +) -> str: + paragraph_properties = paragraph.find(_PARAGRAPH_PROPERTIES_TAG) + paragraph_style = ( + paragraph_properties.find(_PARAGRAPH_STYLE_TAG) + if paragraph_properties is not None + else None + ) + paragraph_style_id = ( + paragraph_style.get(_VALUE_ATTRIBUTE) if paragraph_style is not None else None + ) + if paragraph_style_id in hidden_styles.paragraph_styles: + omissions["hidden_text"] = True + return "" + values: list[str] = [] + + def walk(element: ElementTree.Element, current_depth: int) -> None: + if current_depth > MAXIMUM_NESTING_DEPTH: + raise DocxExtractionFailure("malformed", "docx_nesting_limit") + if element.tag in _DELETION_CONTAINER_TAGS: + omissions["tracked_deletions"] = True + return + if element.tag == _RUN_TAG: + properties = element.find(_RUN_PROPERTIES_TAG) + if properties is not None: + style = properties.find(_RUN_STYLE_TAG) + style_id = style.get(_VALUE_ATTRIBUTE) if style is not None else None + if ( + properties.find(_VANISH_TAG) is not None + or style_id in hidden_styles.run_styles + or hidden_styles.default_hidden + ): + omissions["hidden_text"] = True + return + elif hidden_styles.default_hidden: + omissions["hidden_text"] = True + return + if element.tag == _TEXT_TAG: + values.append(element.text or "") + return + if element.tag == _DELETION_TEXT_TAG: + omissions["tracked_deletions"] = True + return + if element.tag == _TAB_TAG: + values.append("\t") + return + if element.tag in _BREAK_TAGS: + values.append("\n") + return + if element.tag == _FIELD_INSTRUCTION_TAG: + omissions["field_instructions"] = True + return + if element.tag == _SIMPLE_FIELD_TAG and element.get(_FIELD_INSTRUCTION_ATTRIBUTE): + omissions["field_instructions"] = True + if element.tag in _COMMENT_TAGS: + omissions["comments"] = True + return + if element.tag in _EMBEDDED_BODY_TAGS: + omissions["embedded_objects"] = True + return + for child in element: + walk(child, current_depth + 1) + + walk(paragraph, depth) + return "".join(values) + + +def _contained_blocks( + container: ElementTree.Element, + omissions: dict[str, bool], + *, + depth: int = 0, +): + if depth > MAXIMUM_NESTING_DEPTH: + raise DocxExtractionFailure("malformed", "docx_nesting_limit") + for child in container: + if child.tag in _DELETION_CONTAINER_TAGS: + omissions["tracked_deletions"] = True + elif child.tag in {_PARAGRAPH_TAG, _TABLE_TAG}: + yield child + elif child.tag in _EMBEDDED_BODY_TAGS: + omissions["embedded_objects"] = True + else: + yield from _contained_blocks(child, omissions, depth=depth + 1) + + +def _contained_structures( + container: ElementTree.Element, + target_tag: str, + omissions: dict[str, bool], + *, + depth: int, +): + if depth > MAXIMUM_NESTING_DEPTH: + raise DocxExtractionFailure("malformed", "docx_nesting_limit") + for child in container: + if child.tag in _DELETION_CONTAINER_TAGS: + omissions["tracked_deletions"] = True + elif child.tag == target_tag: + yield child + elif child.tag in _EMBEDDED_BODY_TAGS: + omissions["embedded_objects"] = True + elif child.tag in _TRANSPARENT_CONTAINER_TAGS: + yield from _contained_structures( + child, + target_tag, + omissions, + depth=depth + 1, + ) + + +def _table_text( + table: ElementTree.Element, + omissions: dict[str, bool], + hidden_styles: _HiddenStyleFacts, + *, + depth: int = 0, +) -> str: + if depth > MAXIMUM_NESTING_DEPTH: + raise DocxExtractionFailure("malformed", "docx_nesting_limit") + rows: list[str] = [] + for row in _contained_structures(table, _ROW_TAG, omissions, depth=depth + 1): + cells: list[str] = [] + for cell in _contained_structures(row, _CELL_TAG, omissions, depth=depth + 1): + cell_parts: list[str] = [] + for child in _contained_blocks(cell, omissions, depth=depth + 1): + if child.tag == _PARAGRAPH_TAG: + cell_parts.append( + _paragraph_text( + child, + omissions, + hidden_styles, + depth=depth + 1, + ) + ) + elif child.tag == _TABLE_TAG: + cell_parts.append( + _table_text( + child, + omissions, + hidden_styles, + depth=depth + 1, + ) + ) + cells.append("\n".join(cell_parts)) + rows.append("\t".join(cells)) + return "\n".join(rows) + + +def _canonical_blocks( + body: ElementTree.Element, + omissions: dict[str, bool], + hidden_styles: _HiddenStyleFacts, + *, + maximum_output_bytes: int, +) -> str: + serialized: list[str] = [] + byte_count = len(b'{"blocks":[]}') + for child in _contained_blocks(body, omissions): + block: dict[str, str] | None = None + if child.tag == _PARAGRAPH_TAG: + block = { + "type": "paragraph", + "text": _paragraph_text(child, omissions, hidden_styles), + } + elif child.tag == _TABLE_TAG: + block = { + "type": "table", + "text": _table_text(child, omissions, hidden_styles, depth=1), + } + if block is None: + continue + encoded = json.dumps(block, sort_keys=True, separators=(",", ":"), ensure_ascii=False) + projected = byte_count + len(encoded.encode("utf-8")) + (1 if serialized else 0) + if projected > maximum_output_bytes: + raise DocxExtractionFailure("limit_exceeded", "output_limit") + serialized.append(encoded) + byte_count = projected + return '{"blocks":[' + ",".join(serialized) + "]}" + + +def extract_docx( + payload: bytes, + *, + validate_ooxml: Callable[[bytes], object], + maximum_output_bytes: int = MAXIMUM_OUTPUT_BYTES, +) -> DocxExtraction: + """Validate and extract one exact DOCX package without partial output.""" + try: + validate_ooxml(payload) + with zipfile.ZipFile(BytesIO(payload)) as archive: + stored = {info.filename.casefold(): info.filename for info in archive.infolist()} + names = frozenset(stored) + document = archive.read(stored["word/document.xml"]) + styles = ( + archive.read(stored["word/styles.xml"]) if "word/styles.xml" in stored else None + ) + except (KeyError, OSError, RuntimeError, ValueError, zipfile.BadZipFile) as exc: + raise DocxExtractionFailure("malformed", "docx_document_unavailable") from exc + try: + root = ElementTree.fromstring(document) + except ElementTree.ParseError as exc: + raise DocxExtractionFailure("malformed", "docx_invalid_document_xml") from exc + body = root.find(_BODY_TAG) + if body is None: + raise DocxExtractionFailure("malformed", "docx_body_missing") + omissions = _new_omissions(names) + hidden_styles = _hidden_styles(styles) + output = _canonical_blocks( + body, + omissions, + hidden_styles, + maximum_output_bytes=maximum_output_bytes, + ) + omissions["omitted"] = any(omissions[key] for key in _DOCX_OMISSION_KEYS) + return DocxExtraction(canonical_output=output, omission_facts=omissions) diff --git a/backend/app/modules/artifacts/guide_extraction.py b/backend/app/modules/artifacts/guide_extraction.py index 2b263af1b..6088239e8 100644 --- a/backend/app/modules/artifacts/guide_extraction.py +++ b/backend/app/modules/artifacts/guide_extraction.py @@ -15,12 +15,27 @@ EXTRACTION_POLICY_VERSION = "guide-extraction-v1" PDF_EXTRACTION_POLICY_VERSION = "guide-extraction-v2" +DOCX_EXTRACTION_POLICY_VERSION = "guide-extraction-v3" EXTRACTOR_VERSION = "1" MAXIMUM_INPUT_BYTES = 32 * 1024 * 1024 MAXIMUM_OUTPUT_BYTES = 4 * 1024 * 1024 MAXIMUM_PROTOCOL_BYTES = (MAXIMUM_OUTPUT_BYTES * 6) + 1024 WALL_TIMEOUT_SECONDS = 60 -_SUPPORTED = frozenset({"plain_text", "markdown", "json", "csv", "pdf"}) +_SUPPORTED = frozenset({"plain_text", "markdown", "json", "csv", "pdf", "docx"}) +_DEFAULT_OMISSION_FACTS = {"truncated": False, "omitted": False} +_DOCX_OMISSION_KEYS = frozenset( + { + "truncated", + "omitted", + "headers", + "footers", + "comments", + "tracked_deletions", + "embedded_objects", + "hidden_text", + "field_instructions", + } +) @dataclass(frozen=True, slots=True) @@ -31,6 +46,7 @@ class GuideExtractionResult: error_code: str | None canonical_output: str | None output_sha256: str | None + omission_facts: dict[str, bool] extractor_name: str extractor_version: str = EXTRACTOR_VERSION policy_version: str = EXTRACTION_POLICY_VERSION @@ -110,6 +126,7 @@ def extract( status = result["status"] error_code = result["error_code"] output = result["output"] + omission_facts = result["omission_facts"] except (KeyError, TypeError, ValueError, UnicodeDecodeError): return self._result(detected_format, "parser_failure", "invalid_executor_output", None) if status not in { @@ -122,6 +139,8 @@ def extract( return self._result(detected_format, "parser_failure", "invalid_executor_output", None) if output is not None and not isinstance(output, str): return self._result(detected_format, "parser_failure", "invalid_executor_output", None) + if not self._valid_omission_facts(detected_format, status, omission_facts): + return self._result(detected_format, "parser_failure", "invalid_executor_output", None) if isinstance(output, str) and len(output.encode("utf-8")) > MAXIMUM_OUTPUT_BYTES: return self._result(detected_format, "limit_exceeded", "output_limit", None) if (status == "extracted" and (error_code is not None or not isinstance(output, str))) or ( @@ -133,7 +152,26 @@ def extract( ) ): return self._result(detected_format, "parser_failure", "invalid_executor_output", None) - return self._result(detected_format, status, error_code, output) + return self._result( + detected_format, + status, + error_code, + output, + omission_facts=omission_facts, + ) + + @staticmethod + def _valid_omission_facts(detected_format: str, status: object, value: object) -> bool: + if not isinstance(value, dict) or not all( + isinstance(key, str) and isinstance(item, bool) for key, item in value.items() + ): + return False + if detected_format != "docx" or status != "extracted": + return value == _DEFAULT_OMISSION_FACTS + if frozenset(value) != _DOCX_OMISSION_KEYS or value["truncated"]: + return False + categories = _DOCX_OMISSION_KEYS - {"truncated", "omitted"} + return value["omitted"] == any(value[key] for key in categories) @staticmethod def _terminate(process: subprocess.Popen[bytes]) -> None: @@ -151,6 +189,8 @@ def _result( status: str, error_code: str | None, output: str | None, + *, + omission_facts: dict[str, bool] | None = None, ) -> GuideExtractionResult: if status != "extracted": output = None @@ -160,6 +200,7 @@ def _result( error_code=error_code, canonical_output=output, output_sha256=digest, + omission_facts=dict(omission_facts or _DEFAULT_OMISSION_FACTS), extractor_name=f"workstream.{detected_format}", policy_version=extraction_policy_version(detected_format), ) @@ -169,4 +210,6 @@ def extraction_policy_version(detected_format: str) -> str: """Return the policy identity that prevents obsolete format replay.""" if detected_format == "pdf": return PDF_EXTRACTION_POLICY_VERSION + if detected_format == "docx": + return DOCX_EXTRACTION_POLICY_VERSION return EXTRACTION_POLICY_VERSION diff --git a/backend/app/modules/artifacts/guide_extraction_service.py b/backend/app/modules/artifacts/guide_extraction_service.py index 8081528f9..1c5c9fb0e 100644 --- a/backend/app/modules/artifacts/guide_extraction_service.py +++ b/backend/app/modules/artifacts/guide_extraction_service.py @@ -205,7 +205,7 @@ async def record_cancellation() -> None: status="extracted", output_sha256=extracted.output_sha256, canonical_output=extracted.canonical_output, - omission_facts={"truncated": False, "omitted": False}, + omission_facts=extracted.omission_facts, ) .on_conflict_do_nothing(constraint="uq_guide_extracted_contents_identity") .returning(GuideSourceExtractedContent.id) @@ -230,6 +230,7 @@ async def record_cancellation() -> None: or content.source_byte_count != before.byte_count or content.output_sha256 != extracted.output_sha256 or content.canonical_output != extracted.canonical_output + or content.omission_facts != extracted.omission_facts ): raise GuideExtractionError("guide extraction result conflicts") usage = await session.scalar( diff --git a/backend/app/modules/artifacts/guide_extraction_worker.py b/backend/app/modules/artifacts/guide_extraction_worker.py index a0934625b..48e65e1e4 100644 --- a/backend/app/modules/artifacts/guide_extraction_worker.py +++ b/backend/app/modules/artifacts/guide_extraction_worker.py @@ -137,11 +137,16 @@ def _extract( payload: bytes, detected_format: str, pdf_extractor: Callable[[bytes], str] | None = None, -) -> str: + docx_extractor: Callable[[bytes], tuple[str, dict[str, bool]]] | None = None, +) -> str | tuple[str, dict[str, bool]]: if detected_format == "pdf": if pdf_extractor is None: raise ExtractionFailure("parser_failure", "parser_unavailable") return pdf_extractor(payload) + if detected_format == "docx": + if docx_extractor is None: + raise ExtractionFailure("parser_failure", "parser_unavailable") + return docx_extractor(payload) text = _decode_text(payload) if detected_format in {"plain_text", "markdown"}: return text @@ -201,28 +206,75 @@ def bounded_validate(payload: bytes, detected_format: str) -> object: return bounded_validate +def _load_docx_extractor( + validate_ooxml: Callable[[bytes, str], object], +) -> Callable[[bytes], tuple[str, dict[str, bool]]]: + """Load the DOCX adapter after limits but before descriptor-only seccomp.""" + from app.modules.artifacts.guide_docx import DocxExtractionFailure, extract_docx + + def bounded_extract(payload: bytes) -> tuple[str, dict[str, bool]]: + try: + extracted = extract_docx( + payload, + validate_ooxml=lambda exact_payload: validate_ooxml(exact_payload, "docx"), + ) + except DocxExtractionFailure as exc: + raise ExtractionFailure(exc.status, exc.code) from exc + return extracted.canonical_output, extracted.omission_facts + + return bounded_extract + + def main() -> int: """Apply resource/isolation controls and emit one bounded JSON result.""" detected_format = sys.argv[1] if len(sys.argv) == 2 else "" try: _install_limits() pdf_extractor = None + docx_extractor = None if detected_format == "pdf": pdf_extractor = _load_pdf_extractor() + elif detected_format == "docx": + docx_extractor = _load_docx_extractor(_load_ooxml_security()) _install_seccomp() payload = sys.stdin.buffer.read(_MAXIMUM_INPUT_BYTES + 1) if len(payload) > _MAXIMUM_INPUT_BYTES: raise ExtractionFailure("limit_exceeded", "input_limit") - output = _extract(payload, detected_format, pdf_extractor) + extracted = _extract(payload, detected_format, pdf_extractor, docx_extractor) + if isinstance(extracted, tuple): + output, omission_facts = extracted + else: + output = extracted + omission_facts = {"truncated": False, "omitted": False} if len(output.encode("utf-8")) > 4 * 1024 * 1024: raise ExtractionFailure("limit_exceeded", "output_limit") - result = {"status": "extracted", "error_code": None, "output": output} + result = { + "status": "extracted", + "error_code": None, + "output": output, + "omission_facts": omission_facts, + } except ExtractionFailure as exc: - result = {"status": exc.status, "error_code": exc.code, "output": None} + result = { + "status": exc.status, + "error_code": exc.code, + "output": None, + "omission_facts": {"truncated": False, "omitted": False}, + } except MemoryError: - result = {"status": "limit_exceeded", "error_code": "memory_limit", "output": None} + result = { + "status": "limit_exceeded", + "error_code": "memory_limit", + "output": None, + "omission_facts": {"truncated": False, "omitted": False}, + } except BaseException: - result = {"status": "parser_failure", "error_code": "parser_failure", "output": None} + result = { + "status": "parser_failure", + "error_code": "parser_failure", + "output": None, + "omission_facts": {"truncated": False, "omitted": False}, + } encoded = json.dumps(result, separators=(",", ":"), ensure_ascii=True).encode("utf-8") view = memoryview(encoded) while view: diff --git a/backend/scripts/run_test_lanes.py b/backend/scripts/run_test_lanes.py index cba4b8244..9649cd391 100644 --- a/backend/scripts/run_test_lanes.py +++ b/backend/scripts/run_test_lanes.py @@ -106,6 +106,7 @@ class TestLane: "tests/test_guide_bindings.py", "tests/test_guide_formats.py", "tests/test_guide_extraction.py", + "tests/test_guide_docx.py", "tests/test_guide_extractor_dependencies.py", "tests/test_guide_ooxml.py", "tests/test_guide_pdf.py", diff --git a/backend/tests/fixtures/guide_extraction_probe_worker.py b/backend/tests/fixtures/guide_extraction_probe_worker.py index a32e90ed9..3f3429b42 100644 --- a/backend/tests/fixtures/guide_extraction_probe_worker.py +++ b/backend/tests/fixtures/guide_extraction_probe_worker.py @@ -64,11 +64,26 @@ def main() -> int: os._exit(3) else: raise ExtractionFailure("parser_failure", "invalid_test_probe") - result = {"status": "extracted", "error_code": None, "output": output} + result = { + "status": "extracted", + "error_code": None, + "output": output, + "omission_facts": {"truncated": False, "omitted": False}, + } except ExtractionFailure as exc: - result = {"status": exc.status, "error_code": exc.code, "output": None} + result = { + "status": exc.status, + "error_code": exc.code, + "output": None, + "omission_facts": {"truncated": False, "omitted": False}, + } except MemoryError: - result = {"status": "limit_exceeded", "error_code": "memory_limit", "output": None} + result = { + "status": "limit_exceeded", + "error_code": "memory_limit", + "output": None, + "omission_facts": {"truncated": False, "omitted": False}, + } encoded = json.dumps(result, separators=(",", ":")).encode() os.write(1, encoded) return 0 diff --git a/backend/tests/test_artifact_architecture.py b/backend/tests/test_artifact_architecture.py index ab7daff6b..8e9ba6776 100644 --- a/backend/tests/test_artifact_architecture.py +++ b/backend/tests/test_artifact_architecture.py @@ -585,6 +585,23 @@ def test_pdf_parser_dependency_is_confined_to_the_format_adapter() -> None: assert adapter_importers == {"modules/artifacts/guide_extraction_worker.py"} +def test_docx_adapter_is_confined_to_the_isolated_worker() -> None: + adapter_importers: set[str] = set() + for path in APP_ROOT.rglob("*.py"): + relative = path.relative_to(APP_ROOT).as_posix() + for node in ast.walk(_tree(path)): + if isinstance(node, ast.Import) and any( + alias.name == "app.modules.artifacts.guide_docx" for alias in node.names + ): + adapter_importers.add(relative) + elif ( + isinstance(node, ast.ImportFrom) + and node.module == "app.modules.artifacts.guide_docx" + ): + adapter_importers.add(relative) + assert adapter_importers == {"modules/artifacts/guide_extraction_worker.py"} + + def test_ooxml_parser_dependency_and_adapter_are_confined_to_the_isolated_worker() -> None: dependency_importers: set[str] = set() adapter_importers: set[str] = set() @@ -594,9 +611,7 @@ def test_ooxml_parser_dependency_and_adapter_are_confined_to_the_isolated_worker if isinstance(node, ast.Import): if any(alias.name.split(".", 1)[0] == "defusedxml" for alias in node.names): dependency_importers.add(relative) - if any( - alias.name == "app.modules.artifacts.guide_ooxml" for alias in node.names - ): + if any(alias.name == "app.modules.artifacts.guide_ooxml" for alias in node.names): adapter_importers.add(relative) elif isinstance(node, ast.ImportFrom) and node.module is not None: if node.module.split(".", 1)[0] == "defusedxml": diff --git a/backend/tests/test_guide_bindings.py b/backend/tests/test_guide_bindings.py index ad95097db..76f92ef54 100644 --- a/backend/tests/test_guide_bindings.py +++ b/backend/tests/test_guide_bindings.py @@ -11,6 +11,7 @@ from types import SimpleNamespace from typing import Any from uuid import UUID, uuid4 +import zipfile import pytest from pypdf import PdfWriter @@ -166,6 +167,22 @@ async def _byte_stream(payload: bytes) -> AsyncIterator[bytes]: yield payload +def _docx_payload() -> bytes: + output = BytesIO() + document = ( + b'visible' + b'deleted' + b"" + ) + with zipfile.ZipFile(output, "w", zipfile.ZIP_DEFLATED) as archive: + archive.writestr("[Content_Types].xml", b"") + archive.writestr("_rels/.rels", b"") + archive.writestr("word/document.xml", document) + archive.writestr("word/header1.xml", b"
") + return output.getvalue() + + def _preparation( tmp_path: Path, **limit_changes: Any ) -> tuple[ArtifactPreparationService, ArtifactScratchManager]: @@ -448,8 +465,45 @@ async def _create_binding(factory, ids: dict[str, UUID]) -> UUID: @pytest.mark.asyncio @pytest.mark.postgres_schema_contract +@pytest.mark.parametrize( + ("payload", "media_type", "detected_format", "expected_output", "expected_omissions"), + [ + ( + b'{"z":2,"a":1}', + "application/json", + "json", + '{"a":1,"z":2}', + {"truncated": False, "omitted": False}, + ), + ( + _docx_payload(), + "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + "docx", + '{"blocks":[{"text":"visible","type":"paragraph"}]}', + { + "truncated": False, + "omitted": True, + "headers": True, + "footers": False, + "comments": False, + "tracked_deletions": True, + "embedded_objects": False, + "hidden_text": False, + "field_instructions": False, + }, + ), + ], + ids=("json", "docx"), +) async def test_extraction_publishes_deterministic_content_and_exact_usage( - isolated_database_env: str, tmp_path: Path, migration_lock + isolated_database_env: str, + tmp_path: Path, + migration_lock, + payload: bytes, + media_type: str, + detected_format: str, + expected_output: str, + expected_omissions: dict[str, bool], ) -> None: config = Config(str(Path(__file__).resolve().parents[1] / "alembic.ini")) config.set_main_option( @@ -457,7 +511,6 @@ async def test_extraction_publishes_deterministic_content_and_exact_usage( ) with migration_lock(): await asyncio.to_thread(command.downgrade, config, "0042_guide_extraction") - payload = b'{"z":2,"a":1}' digest = "sha256:" + hashlib.sha256(payload).hexdigest() engine = create_async_engine(isolated_database_env) factory = async_sessionmaker(engine, expire_on_commit=False) @@ -469,7 +522,7 @@ async def test_extraction_publishes_deterministic_content_and_exact_usage( session, sha256=digest, byte_count=len(payload), - media_type="application/json", + media_type=media_type, ) binding_id = await _create_binding(factory, ids) classification_id = uuid4() @@ -483,15 +536,15 @@ async def test_extraction_publishes_deterministic_content_and_exact_usage( setup_generation=1, sha256=digest, byte_count=len(payload), - media_type="application/json", - detected_format="json", + media_type=media_type, + detected_format=detected_format, status="classified", detector_name="workstream.guide_format", detector_version="1", classification_facts={}, ) ) - prepared = await preparation.prepare(_byte_stream(payload), media_type="application/json") + prepared = await preparation.prepare(_byte_stream(payload), media_type=media_type) request = GuideExtractionRequest( project_id=ids["project"], guide_id=ids["guide"], @@ -513,7 +566,8 @@ async def test_extraction_publishes_deterministic_content_and_exact_usage( GuideSourceExtractedContent, str(result.extracted_content_id) ) assert content is not None - assert content.canonical_output == '{"a":1,"z":2}' + assert content.canonical_output == expected_output + assert content.omission_facts == expected_omissions assert await session.scalar(select(func.count(GuideSourceExtractionAttempt.id))) == 1 assert await session.scalar(select(func.count(GuideSourceExtractionUsage.id))) == 1 async with factory() as session: @@ -778,14 +832,22 @@ async def test_successful_replay_requires_the_current_extraction_policy( @pytest.mark.asyncio -async def test_pdf_support_replaces_the_obsolete_policy_budget_without_replay( - isolated_database_env: str, tmp_path: Path +@pytest.mark.parametrize("detected_format", ["pdf", "docx"]) +async def test_new_format_support_replaces_obsolete_policy_budget_without_replay( + isolated_database_env: str, + tmp_path: Path, + detected_format: str, ) -> None: - writer = PdfWriter() - writer.add_blank_page(width=72, height=72) - stream = BytesIO() - writer.write(stream) - payload = stream.getvalue() + if detected_format == "pdf": + writer = PdfWriter() + writer.add_blank_page(width=72, height=72) + stream = BytesIO() + writer.write(stream) + payload = stream.getvalue() + media_type = "application/pdf" + else: + payload = _docx_payload() + media_type = "application/vnd.openxmlformats-officedocument.wordprocessingml.document" digest = "sha256:" + hashlib.sha256(payload).hexdigest() engine = create_async_engine(isolated_database_env) factory = async_sessionmaker(engine, expire_on_commit=False) @@ -794,7 +856,7 @@ async def test_pdf_support_replaces_the_obsolete_policy_budget_without_replay( try: async with factory() as session: ids = await _seed_binding_lineage( - session, sha256=digest, byte_count=len(payload), media_type="application/pdf" + session, sha256=digest, byte_count=len(payload), media_type=media_type ) binding_id = await _create_binding(factory, ids) classification_id = uuid4() @@ -808,8 +870,8 @@ async def test_pdf_support_replaces_the_obsolete_policy_budget_without_replay( setup_generation=1, sha256=digest, byte_count=len(payload), - media_type="application/pdf", - detected_format="pdf", + media_type=media_type, + detected_format=detected_format, status="classified", detector_name="workstream.guide_format", detector_version="1", @@ -824,8 +886,8 @@ async def test_pdf_support_replaces_the_obsolete_policy_budget_without_replay( content_id=str(ids["content"]), classification_id=str(classification_id), setup_generation=1, - detected_format="pdf", - extractor_name="workstream.pdf", + detected_format=detected_format, + extractor_name=f"workstream.{detected_format}", extractor_version="1", policy_version=EXTRACTION_POLICY_VERSION, attempt_number=1, @@ -863,11 +925,11 @@ async def test_pdf_support_replaces_the_obsolete_policy_budget_without_replay( async with factory() as session: budget = await session.get(GuideSourceExtractionRetryBudget, str(binding_id)) assert budget is not None - assert budget.policy_version == extraction_policy_version("pdf") + assert budget.policy_version == extraction_policy_version(detected_format) assert budget.claimed_slots == 1 preparation, scratch = _preparation(tmp_path) prepared = await preparation.prepare( - _byte_stream(payload), media_type="application/pdf" + _byte_stream(payload), media_type=media_type ) service = GuideExtractionService(factory, GuideExtractionRegistry()) extracted = await service.extract_prepared(request, prepared) @@ -888,7 +950,7 @@ async def test_pdf_support_replaces_the_obsolete_policy_budget_without_replay( ).all() assert [(attempt.policy_version, attempt.status) for attempt in attempts] == [ (EXTRACTION_POLICY_VERSION, "unsupported"), - (extraction_policy_version("pdf"), "extracted"), + (extraction_policy_version(detected_format), "extracted"), ] finally: if prepared is not None: diff --git a/backend/tests/test_guide_docx.py b/backend/tests/test_guide_docx.py new file mode 100644 index 000000000..450b374b4 --- /dev/null +++ b/backend/tests/test_guide_docx.py @@ -0,0 +1,350 @@ +"""Focused proofs for deterministic bounded DOCX extraction.""" + +from __future__ import annotations + +from io import BytesIO +import json +import zipfile + +import pytest + +from app.modules.artifacts.guide_docx import ( + DocxExtraction, + DocxExtractionFailure, + extract_docx as _extract_docx, +) +from app.modules.artifacts.guide_extraction import GuideExtractionRunner +from app.modules.artifacts.guide_ooxml import OoxmlSecurityFailure, validate_ooxml + + +_W = "http://schemas.openxmlformats.org/wordprocessingml/2006/main" + + +def _docx( + document: bytes, + *, + document_name: str = "word/document.xml", + additions: dict[str, bytes] | None = None, +) -> bytes: + members = { + "[Content_Types].xml": b"", + "_rels/.rels": b"", + document_name: document, + **(additions or {}), + } + output = BytesIO() + with zipfile.ZipFile(output, "w", zipfile.ZIP_DEFLATED) as archive: + for name, body in members.items(): + archive.writestr(name, body) + return output.getvalue() + + +def _document(body: str) -> bytes: + return f'{body}'.encode() + + +def extract_docx( + payload: bytes, + *, + maximum_output_bytes: int = 4 * 1024 * 1024, +) -> DocxExtraction: + def bounded_validate(exact_payload: bytes) -> object: + try: + return validate_ooxml(exact_payload, detected_format="docx") + except OoxmlSecurityFailure as exc: + raise DocxExtractionFailure(exc.status, exc.code) from exc + + return _extract_docx( + payload, + validate_ooxml=bounded_validate, + maximum_output_bytes=maximum_output_bytes, + ) + + +def test_extracts_paragraphs_tables_and_nested_tables_deterministically() -> None: + payload = _docx( + _document( + """ + Hello worldagain + + A + B + nested + tail + + + + """ + ) + ) + + result = extract_docx(payload) + + assert result.canonical_output == ( + '{"blocks":[{"text":"Hello \\tworld\\nagain","type":"paragraph"},' + '{"text":"A\\tB\\nnested\\ntail\\n\\t","type":"table"},' + '{"text":"","type":"paragraph"}]}' + ) + assert result.omission_facts == { + "truncated": False, + "omitted": False, + "headers": False, + "footers": False, + "comments": False, + "tracked_deletions": False, + "embedded_objects": False, + "hidden_text": False, + "field_instructions": False, + } + assert extract_docx(payload) == result + + +def test_records_every_docx_omission_without_exposing_omitted_text() -> None: + payload = _docx( + _document( + """ + + visible + deleted-secret + hidden-secret + field-secretfield result + + + """ + ), + additions={ + "word/header1.xml": b"
", + "word/footer1.xml": b"