diff --git a/.github/workflows/darwin-semantic-backlog.yml b/.github/workflows/darwin-semantic-backlog.yml new file mode 100644 index 00000000..f1cf16ec --- /dev/null +++ b/.github/workflows/darwin-semantic-backlog.yml @@ -0,0 +1,84 @@ +name: Darwin/Rachinsky 1864 semantic backlog + +on: + pull_request: + paths: + - ".github/workflows/darwin-semantic-backlog.yml" + - "scriptorium/darwin_semantic_backlog.py" + - "scriptorium/darwin_render_profile.py" + - "tests/test_darwin_semantic_backlog.py" + - "corpus/candidates/darwin-origin-species-rachinsky-1864-ru.md" + - "corpus/candidates/source-edition-traces/darwin-origin-species-rachinsky-1864-ru.render-surface.json" + - "corpus/candidates/source-edition-traces/darwin-origin-species-rachinsky-1864-ru.render-profile.json" + - "corpus/candidates/source-edition-traces/darwin-origin-species-rachinsky-1864-ru.semantic-backlog.json" + workflow_dispatch: + +permissions: + contents: read + +jobs: + verify: + name: Verify source-free unresolved-semantic backlog + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - name: Checkout pull-request head + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ github.event.pull_request.head.sha || github.sha }} + persist-credentials: false + + - name: Set up Python 3.13 + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: "3.13" + + - name: Run semantic backlog and render-profile regression tests + run: | + python -m unittest tests.test_darwin_semantic_backlog -v + python -m unittest tests.test_darwin_render_profile -v + + - name: Deterministically rebuild and compare backlog + run: | + set -euo pipefail + mkdir -p build/repro + python -m scriptorium.darwin_semantic_backlog \ + --profile corpus/candidates/source-edition-traces/darwin-origin-species-rachinsky-1864-ru.render-profile.json \ + --surface corpus/candidates/source-edition-traces/darwin-origin-species-rachinsky-1864-ru.render-surface.json \ + --output build/repro/darwin-origin-species-rachinsky-1864-ru.semantic-backlog.json + python - <<'PY' + import json + from pathlib import Path + expected = json.loads(Path("corpus/candidates/source-edition-traces/darwin-origin-species-rachinsky-1864-ru.semantic-backlog.json").read_text(encoding="utf-8")) + rebuilt = json.loads(Path("build/repro/darwin-origin-species-rachinsky-1864-ru.semantic-backlog.json").read_text(encoding="utf-8")) + assert rebuilt == expected + assert rebuilt["backlog_sha256"] == "4ec4f84566f5f3f8dccd1945906d1a782f4532547a13cadc8570cb5b682bdcb2" + assert rebuilt["unresolved_totals"] == { + "tag_shape_count": 5, + "tag_token_count": 518, + "template_shape_count": 38, + "template_invocation_count": 4583, + } + assert len(rebuilt["prioritized_items"]) == 43 + assert rebuilt["next_research_slice"]["name"] == "ё" + assert rebuilt["next_research_slice"]["count"] == 2227 + assert rebuilt["next_research_slice"]["semantic_status"] == "unresolved" + assert rebuilt["renderer_semantics_complete"] is False + assert rebuilt["renderer_implementation_ready"] is False + assert rebuilt["inter_page_composition_frozen"] is False + assert rebuilt["literary_body_count_and_digests_frozen"] is False + assert rebuilt["minimum_300k_proved"] is False + assert rebuilt["admitted_for_calibration"] is False + assert rebuilt["fantlab_source_edition_match"] == "unknown" + assert rebuilt["diagnostic_ready"] is False + assert rebuilt["m2_parity_admissible"] is False + print("SCRIPTORIUM_DARWIN_SEMANTIC_BACKLOG_SHA256=" + rebuilt["backlog_sha256"]) + PY + + - name: Upload source-free semantic backlog evidence + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: darwin-rachinsky-1864-semantic-backlog + path: build/repro/darwin-origin-species-rachinsky-1864-ru.semantic-backlog.json + if-no-files-found: error + retention-days: 7 diff --git a/corpus/candidates/darwin-origin-species-rachinsky-1864-ru-semantic-backlog.md b/corpus/candidates/darwin-origin-species-rachinsky-1864-ru-semantic-backlog.md new file mode 100644 index 00000000..e6b1cc37 --- /dev/null +++ b/corpus/candidates/darwin-origin-species-rachinsky-1864-ru-semantic-backlog.md @@ -0,0 +1,41 @@ +# Darwin/Rachinsky 1864 — unresolved rendering semantics + +This public source-free companion to `darwin-origin-species-rachinsky-1864-ru` exposes the remaining rendering-semantic work after the reviewed exact-388 Page surface and fail-closed render profile were frozen. + +No literary source text, Page wikitext, template argument values, rendered prose, OCR or scan bytes are stored here. Counts come only from the already frozen render profile SHA-256 `2ebcfea51282505c28824f54f6a1795bd7e87e2f958ac59f5c843e68c2066f3d`. + +## Remaining surface + +The profile still has **43 unresolved shapes**: + +| Research track | Shapes | Observed occurrences | +| --- | ---: | ---: | +| provider-template semantics | 37 | 4,579 | +| provider-reference semantics | 3 | 466 | +| provider-math semantics | 2 | 52 | +| inter-page semantics | 1 | 4 | + +Those totals exactly preserve the reviewed boundary: **5 unresolved tag shapes / 518 tag tokens** and **38 unresolved template shapes / 4,583 invocations**. + +## Mechanical next target + +The deterministic backlog ranks unresolved items by descending observed occurrence count with stable tie-breaking. The first research target is template **`ё`**, arity **0 positional / 0 named**, observed **2,227 times** (2,227 / 4,583 unresolved template invocations). + +This ranking is **not a semantics claim**. `ё` remains `semantic_status=unresolved`; a separate bounded unit must establish its provider behavior with independent evidence before the renderer profile may change. + +The next two rows are self-closing `` (**388**) and template `ВАР` with arity 2/0 (**388**). Reference/math behavior and the inter-page-sensitive `nop` track remain independently reviewable rather than being folded into one guessed renderer policy. + +## Frozen artifact + +Canonical source-free artifact: + +- [`source-edition-traces/darwin-origin-species-rachinsky-1864-ru.semantic-backlog.json`](source-edition-traces/darwin-origin-species-rachinsky-1864-ru.semantic-backlog.json) +- schema: `scriptorium-darwin-render-semantic-backlog-v1` +- backlog SHA-256: `4ec4f84566f5f3f8dccd1945906d1a782f4532547a13cadc8570cb5b682bdcb2` +- builder/validator: [`../../scriptorium/darwin_semantic_backlog.py`](../../scriptorium/darwin_semantic_backlog.py) + +The artifact is derived only after the existing render profile validates against its frozen source surface. Any profile/surface drift fails before backlog derivation. + +## Gates unchanged + +This slice does not implement rendering and does not advance corpus or FantLab parity gates. The following remain false/unknown: `renderer_semantics_complete`, `renderer_implementation_ready`, `inter_page_composition_frozen`, `literary_body_count_and_digests_frozen`, `minimum_300k_proved`, `admitted_for_calibration`, `diagnostic_ready`, `m2_parity_admissible`; `fantlab_source_edition_match=unknown`. M2 remains **0/5**. diff --git a/corpus/candidates/darwin-origin-species-rachinsky-1864-ru.md b/corpus/candidates/darwin-origin-species-rachinsky-1864-ru.md index 42ef51be..a1bb754f 100644 --- a/corpus/candidates/darwin-origin-species-rachinsky-1864-ru.md +++ b/corpus/candidates/darwin-origin-species-rachinsky-1864-ru.md @@ -81,12 +81,22 @@ Independent exact-head review re-read all settled CI on `ee97e29af7225131002b6e0 ## Fail-closed rendering decision profile -**SCRIP-CORPUS-045 / Draft PR #178** adds `scriptorium-darwin-page-render-profile-v1`, bound cryptographically to the reviewed exact-388 source-free surface freeze before any shape classification is accepted. The profile covers all **18 tag shapes / 5,164 tag tokens** and all **38 template shapes / 4,583 template invocations**. Its canonical profile SHA-256 is **`2ebcfea51282505c28824f54f6a1795bd7e87e2f958ac59f5c843e68c2066f3d`**. +**SCRIP-CORPUS-045 / merged PR #178** adds `scriptorium-darwin-page-render-profile-v1`, bound cryptographically to the reviewed exact-388 source-free surface freeze before any shape classification is accepted. The profile covers all **18 tag shapes / 5,164 tag tokens** and all **38 template shapes / 4,583 template invocations**. Its canonical profile SHA-256 is **`2ebcfea51282505c28824f54f6a1795bd7e87e2f958ac59f5c843e68c2066f3d`**. The decision table defines only conservative local markup handling: ordinary `big`, `div`, `small`, `sub` and `sup` wrappers preserve inner text; `` is excluded from transcluded content; ProofreadPage `pagequality` metadata is dropped. It intentionally leaves **5 tag shapes / 518 tokens** unresolved for provider semantics: `` open/close, `` open/close and self-closing ``. All **38 template shapes / 4,583 invocations** remain unresolved rather than inferring expansion from names; `nop` is additionally marked as inter-page-sensitive. This is a **frozen decision boundary, not a renderer**. `rendering_profile_frozen=true` coexists deliberately with `renderer_semantics_complete=false`, `renderer_implementation_ready=false`, `rendering_equivalence_claimed=false`, and `inter_page_composition_frozen=false`. Any changed count, new/missing shape, stale surface digest or wrong candidate identity fails closed. No Page wikitext, template argument values, rendered prose, OCR, DjVu/PDF bytes or literary text are committed. +## Source-free unresolved-semantic backlog + +**SCRIP-CORPUS-046 / Draft PR #180** derives a deterministic research backlog only after validating the frozen render-profile/source-surface boundary. The durable artifact preserves all **43 unresolved shapes**: **5 tag shapes / 518 tokens** and **38 template shapes / 4,583 invocations**. + +The backlog groups them into four independently reviewable tracks: provider-template (**37 shapes / 4,579 occurrences**), provider-reference (**3 / 466**), provider-math (**2 / 52**), and inter-page (**1 / 4**). Items are ranked mechanically by descending observed occurrence count with stable tie-breaking. The current next research target is template **`ё`**, arity **0 positional / 0 named**, observed **2,227 times**. That target remains explicitly `semantic_status=unresolved`; selection priority is not a semantics claim. + +Canonical backlog SHA-256 is **`4ec4f84566f5f3f8dccd1945906d1a782f4532547a13cadc8570cb5b682bdcb2`**. The source-free public companion is [`darwin-origin-species-rachinsky-1864-ru-semantic-backlog.md`](darwin-origin-species-rachinsky-1864-ru-semantic-backlog.md), and the machine-readable artifact is [`source-edition-traces/darwin-origin-species-rachinsky-1864-ru.semantic-backlog.json`](source-edition-traces/darwin-origin-species-rachinsky-1864-ru.semantic-backlog.json). + +This backlog does **not** implement rendering or advance any downstream gate. `renderer_semantics_complete=false`, `renderer_implementation_ready=false`, `inter_page_composition_frozen=false`, `literary_body_count_and_digests_frozen=false`, `minimum_300k_proved=false`, `admitted_for_calibration=false`, `diagnostic_ready=false`, `m2_parity_admissible=false`, and `fantlab_source_edition_match=unknown` remain mandatory. + ## Independent scan binary identity **SCRIP-CORPUS-037 / merged PR #162** independently retrieved the exact Wikimedia Commons DjVu backing this ProofreadPage family and streamed it through Scriptorium without writing the binary into the repository. The hosted capture observed **27,368,263 bytes** and SHA-1 `75ef508588194ae74874272ce290f3ec1043ea9b`, exactly matching the provider metadata that had already been frozen from the source surface. @@ -128,13 +138,15 @@ FantLab work **`work969964`** identifies Darwin's 1859 English monograph. The cu - [`source-edition-traces/darwin-origin-species-rachinsky-1864-ru.render-surface.json`](source-edition-traces/darwin-origin-species-rachinsky-1864-ru.render-surface.json) — deterministic source-free 388-Page construct inventory with no source payload. - [`source-edition-traces/darwin-origin-species-rachinsky-1864-ru.render-surface.source-graph.json`](source-edition-traces/darwin-origin-species-rachinsky-1864-ru.render-surface.source-graph.json) — provenance edges from frozen Page identities through literary composition and hosted markup audit to the durable inventory and frozen decision profile. - [`source-edition-traces/darwin-origin-species-rachinsky-1864-ru.render-profile.json`](source-edition-traces/darwin-origin-species-rachinsky-1864-ru.render-profile.json) — deterministic fail-closed source-free rendering decision profile with explicit unresolved semantics and closed downstream gates. +- [`source-edition-traces/darwin-origin-species-rachinsky-1864-ru.semantic-backlog.json`](source-edition-traces/darwin-origin-species-rachinsky-1864-ru.semantic-backlog.json) — deterministic source-free unresolved-semantic priority backlog; every listed shape remains unresolved. - [`source-edition-traces/darwin-origin-species-rachinsky-1864-ru.scan-identity.json`](source-edition-traces/darwin-origin-species-rachinsky-1864-ru.scan-identity.json) — independently verified source-free DjVu byte count, SHA-1 cross-check and SHA-256; scan bytes are not stored. - [`../../scriptorium/darwin_body_contract.py`](../../scriptorium/darwin_body_contract.py) — deterministic source-free 388/30 composition contract generator and fail-closed validator. - [`../../scriptorium/darwin_render_surface.py`](../../scriptorium/darwin_render_surface.py) — exact-revision identity-verifying hosted construct audit. - [`../../scriptorium/darwin_render_surface_freeze.py`](../../scriptorium/darwin_render_surface_freeze.py) — deterministic compact freeze reducer/validator. - [`../../scriptorium/darwin_render_profile.py`](../../scriptorium/darwin_render_profile.py) — deterministic fail-closed source-free decision-profile builder/validator. +- [`../../scriptorium/darwin_semantic_backlog.py`](../../scriptorium/darwin_semantic_backlog.py) — deterministic source-free unresolved-semantic backlog builder/validator. - [`../../scriptorium/darwin_scan_identity.py`](../../scriptorium/darwin_scan_identity.py) — streaming scan-identity capture/replay with provider cross-checks and no binary persistence. ## Next evidence -A later bounded unit should resolve and independently freeze the provider/template/reference/math semantics explicitly left unresolved by `scriptorium-darwin-page-render-profile-v1`; inter-page separator/composition semantics should remain a separately reviewable decision. Only after deterministic rendering exists should a separate identity/admission unit compute source-free counts/digests and independently prove the >=300,000-character threshold. FantLab parity remains a separate, stricter source-matching problem. +The mechanically selected next renderer-research slice is template `ё` (2,227 observed invocations), but its semantics remain unresolved. A later bounded unit should establish provider behavior with independent evidence before the render profile changes. Reference/math behavior and inter-page separator/composition semantics remain separate reviewable decisions. Only after deterministic rendering exists should a separate identity/admission unit compute source-free counts/digests and independently prove the >=300,000-character threshold. FantLab parity remains a separate, stricter source-matching problem. diff --git a/corpus/candidates/source-edition-traces/darwin-origin-species-rachinsky-1864-ru.semantic-backlog.json b/corpus/candidates/source-edition-traces/darwin-origin-species-rachinsky-1864-ru.semantic-backlog.json new file mode 100644 index 00000000..30084fcf --- /dev/null +++ b/corpus/candidates/source-edition-traces/darwin-origin-species-rachinsky-1864-ru.semantic-backlog.json @@ -0,0 +1 @@ +{"admitted_for_calibration":false,"backlog_sha256":"4ec4f84566f5f3f8dccd1945906d1a782f4532547a13cadc8570cb5b682bdcb2","candidate_id":"darwin-origin-species-rachinsky-1864-ru","diagnostic_ready":false,"fantlab_source_edition_match":"unknown","inter_page_composition_frozen":false,"literary_body_count_and_digests_frozen":false,"m2_parity_admissible":false,"minimum_300k_proved":false,"next_research_slice":{"count":2227,"name":"ё","named":0,"positional":0,"reason":"highest observed unresolved occurrence count; semantic meaning remains unresolved until independently evidenced","semantic_status":"unresolved","share_of_unresolved_template_invocations":{"denominator":4583,"numerator":2227},"source_kind":"template"},"prioritized_items":[{"count":2227,"handling":"provider_template_semantics_required","name":"ё","named":0,"positional":0,"priority_rank":1,"semantic_status":"unresolved","source_kind":"template","track":"provider_template"},{"count":388,"handling":"provider_reference_semantics_required","kind":"self_closing","name":"references","priority_rank":2,"semantic_status":"unresolved","source_kind":"tag","track":"provider_reference"},{"count":388,"handling":"provider_template_semantics_required","name":"ВАР","named":0,"positional":2,"priority_rank":3,"semantic_status":"unresolved","source_kind":"template","track":"provider_template"},{"count":259,"handling":"provider_template_semantics_required","name":"Перенос2","named":0,"positional":2,"priority_rank":4,"semantic_status":"unresolved","source_kind":"template","track":"provider_template"},{"count":255,"handling":"provider_template_semantics_required","name":"Lang","named":0,"positional":2,"priority_rank":5,"semantic_status":"unresolved","source_kind":"template","track":"provider_template"},{"count":253,"handling":"provider_template_semantics_required","name":"Перенос","named":0,"positional":2,"priority_rank":6,"semantic_status":"unresolved","source_kind":"template","track":"provider_template"},{"count":179,"handling":"provider_template_semantics_required","name":"lang","named":0,"positional":2,"priority_rank":7,"semantic_status":"unresolved","source_kind":"template","track":"provider_template"},{"count":135,"handling":"provider_template_semantics_required","name":"опечатка2","named":0,"positional":2,"priority_rank":8,"semantic_status":"unresolved","source_kind":"template","track":"provider_template"},{"count":122,"handling":"provider_template_semantics_required","name":"^","named":0,"positional":0,"priority_rank":9,"semantic_status":"unresolved","source_kind":"template","track":"provider_template"},{"count":108,"handling":"provider_template_semantics_required","name":"гравис","named":0,"positional":0,"priority_rank":10,"semantic_status":"unresolved","source_kind":"template","track":"provider_template"},{"count":99,"handling":"provider_template_semantics_required","name":"нп","named":0,"positional":0,"priority_rank":11,"semantic_status":"unresolved","source_kind":"template","track":"provider_template"},{"count":90,"handling":"provider_template_semantics_required","name":"^","named":0,"positional":1,"priority_rank":12,"semantic_status":"unresolved","source_kind":"template","track":"provider_template"},{"count":76,"handling":"provider_template_semantics_required","name":"мби","named":0,"positional":3,"priority_rank":13,"semantic_status":"unresolved","source_kind":"template","track":"provider_template"},{"count":62,"handling":"provider_template_semantics_required","name":"ц","named":0,"positional":1,"priority_rank":14,"semantic_status":"unresolved","source_kind":"template","track":"provider_template"},{"count":52,"handling":"provider_template_semantics_required","name":"sc","named":0,"positional":1,"priority_rank":15,"semantic_status":"unresolved","source_kind":"template","track":"provider_template"},{"count":46,"handling":"provider_template_semantics_required","name":"акут","named":0,"positional":0,"priority_rank":16,"semantic_status":"unresolved","source_kind":"template","track":"provider_template"},{"count":39,"handling":"provider_reference_semantics_required","kind":"close","name":"ref","priority_rank":17,"semantic_status":"unresolved","source_kind":"tag","track":"provider_reference"},{"count":39,"handling":"provider_reference_semantics_required","kind":"open","name":"ref","priority_rank":18,"semantic_status":"unresolved","source_kind":"tag","track":"provider_reference"},{"count":30,"handling":"provider_template_semantics_required","name":"roman","named":0,"positional":1,"priority_rank":19,"semantic_status":"unresolved","source_kind":"template","track":"provider_template"},{"count":28,"handling":"provider_template_semantics_required","name":"во","named":1,"positional":1,"priority_rank":20,"semantic_status":"unresolved","source_kind":"template","track":"provider_template"},{"count":26,"handling":"provider_math_semantics_required","kind":"close","name":"math","priority_rank":21,"semantic_status":"unresolved","source_kind":"tag","track":"provider_math"},{"count":26,"handling":"provider_math_semantics_required","kind":"open","name":"math","priority_rank":22,"semantic_status":"unresolved","source_kind":"tag","track":"provider_math"},{"count":22,"handling":"provider_template_semantics_required","name":"nobr","named":0,"positional":1,"priority_rank":23,"semantic_status":"unresolved","source_kind":"template","track":"provider_template"},{"count":21,"handling":"provider_template_semantics_required","name":"Градус","named":0,"positional":1,"priority_rank":24,"semantic_status":"unresolved","source_kind":"template","track":"provider_template"},{"count":17,"handling":"provider_template_semantics_required","name":"разделитель","named":0,"positional":1,"priority_rank":25,"semantic_status":"unresolved","source_kind":"template","track":"provider_template"},{"count":16,"handling":"provider_template_semantics_required","name":"дробь","named":0,"positional":2,"priority_rank":26,"semantic_status":"unresolved","source_kind":"template","track":"provider_template"},{"count":15,"handling":"provider_template_semantics_required","name":"ПримВТ","named":0,"positional":0,"priority_rank":27,"semantic_status":"unresolved","source_kind":"template","track":"provider_template"},{"count":14,"handling":"provider_template_semantics_required","name":"noindent","named":0,"positional":0,"priority_rank":28,"semantic_status":"unresolved","source_kind":"template","track":"provider_template"},{"count":14,"handling":"provider_template_semantics_required","name":"noindent","named":0,"positional":1,"priority_rank":29,"semantic_status":"unresolved","source_kind":"template","track":"provider_template"},{"count":13,"handling":"provider_template_semantics_required","name":"psp","named":0,"positional":0,"priority_rank":30,"semantic_status":"unresolved","source_kind":"template","track":"provider_template"},{"count":8,"handling":"provider_template_semantics_required","name":"razr2","named":0,"positional":1,"priority_rank":31,"semantic_status":"unresolved","source_kind":"template","track":"provider_template"},{"count":7,"handling":"provider_template_semantics_required","name":"fsp","named":0,"positional":0,"priority_rank":32,"semantic_status":"unresolved","source_kind":"template","track":"provider_template"},{"count":6,"handling":"provider_template_semantics_required","name":"dotted TOC","named":0,"positional":3,"priority_rank":33,"semantic_status":"unresolved","source_kind":"template","track":"provider_template"},{"count":4,"handling":"inter_page_semantics_required","name":"nop","named":0,"positional":0,"priority_rank":34,"semantic_status":"unresolved","source_kind":"template","track":"inter_page"},{"count":3,"handling":"provider_template_semantics_required","name":"опечатка","named":0,"positional":2,"priority_rank":35,"semantic_status":"unresolved","source_kind":"template","track":"provider_template"},{"count":2,"handling":"provider_template_semantics_required","name":"=","named":0,"positional":0,"priority_rank":36,"semantic_status":"unresolved","source_kind":"template","track":"provider_template"},{"count":2,"handling":"provider_template_semantics_required","name":"inline float","named":2,"positional":4,"priority_rank":37,"semantic_status":"unresolved","source_kind":"template","track":"provider_template"},{"count":2,"handling":"provider_template_semantics_required","name":"lang-en","named":0,"positional":1,"priority_rank":38,"semantic_status":"unresolved","source_kind":"template","track":"provider_template"},{"count":2,"handling":"provider_template_semantics_required","name":"right","named":0,"positional":1,"priority_rank":39,"semantic_status":"unresolved","source_kind":"template","track":"provider_template"},{"count":2,"handling":"provider_template_semantics_required","name":"ВАР2","named":0,"positional":2,"priority_rank":40,"semantic_status":"unresolved","source_kind":"template","track":"provider_template"},{"count":2,"handling":"provider_template_semantics_required","name":"Перенос","named":1,"positional":2,"priority_rank":41,"semantic_status":"unresolved","source_kind":"template","track":"provider_template"},{"count":1,"handling":"provider_template_semantics_required","name":"lang-la","named":0,"positional":1,"priority_rank":42,"semantic_status":"unresolved","source_kind":"template","track":"provider_template"},{"count":1,"handling":"provider_template_semantics_required","name":"Так в тексте","named":0,"positional":1,"priority_rank":43,"semantic_status":"unresolved","source_kind":"template","track":"provider_template"}],"renderer_implementation_ready":false,"renderer_semantics_complete":false,"resolution_tracks":[{"occurrence_count":4579,"shape_count":37,"track":"provider_template"},{"occurrence_count":466,"shape_count":3,"track":"provider_reference"},{"occurrence_count":52,"shape_count":2,"track":"provider_math"},{"occurrence_count":4,"shape_count":1,"track":"inter_page"}],"schema_version":"scriptorium-darwin-render-semantic-backlog-v1","source_profile":{"profile_sha256":"2ebcfea51282505c28824f54f6a1795bd7e87e2f958ac59f5c843e68c2066f3d","profile_version":"scriptorium-darwin-page-render-profile-v1"},"source_text_included":false,"status":"unresolved_semantics_prioritized_no_semantics_inferred","unresolved_totals":{"tag_shape_count":5,"tag_token_count":518,"template_invocation_count":4583,"template_shape_count":38}} diff --git a/project/CHANGELOG.d/SCRIP-CORPUS-046.md b/project/CHANGELOG.d/SCRIP-CORPUS-046.md new file mode 100644 index 00000000..5fa713e4 --- /dev/null +++ b/project/CHANGELOG.d/SCRIP-CORPUS-046.md @@ -0,0 +1,11 @@ +## SCRIP-CORPUS-046 — Darwin/Rachinsky unresolved-semantic priority backlog + +- Added `scriptorium-darwin-render-semantic-backlog-v1`, derived only after the independently reviewed render profile validates against its frozen exact-388 source surface. +- Preserved all unresolved rendering shapes without promoting semantics: 5 tag shapes / 518 tag tokens and 38 template shapes / 4,583 invocations. +- Grouped the unresolved surface into provider-template (37 shapes / 4,579 occurrences), provider-reference (3 / 466), provider-math (2 / 52), and inter-page (1 / 4) tracks. +- Made future renderer research selection deterministic: the highest-impact unresolved row is template `ё` with arity 0/0 and 2,227 observed invocations. This is a priority decision only; `semantic_status=unresolved` remains frozen. +- Added a canonical source-free backlog artifact with SHA-256 `4ec4f84566f5f3f8dccd1945906d1a782f4532547a13cadc8570cb5b682bdcb2`, regression coverage, deterministic rebuild CI, and a public companion page. +- Independent review COMMENT `5262708679` found two acceptance/recoverability blockers on the authored head: canonical state had not recorded SCRIP-CORPUS-046, and the canonical Darwin candidate page still described merged PR #178 as Draft while omitting the semantic backlog. +- Repaired both review blockers on Draft PR #180: `project/STATE_AND_QUEUE.md` now records SCRIP-CORPUS-046 / #179 / #180 as `REVIEW_PENDING`, and the canonical Darwin candidate page links/summarizes the source-free backlog and corrects PR #178 to merged status. +- Kept renderer semantics/implementation/equivalence, inter-page composition, literary-body identity, >=300k admission, FantLab source match, diagnostics, and M2 parity closed. M2 remains 0/5. +- Draft PR #180 remains intentionally unmerged after the recovery repair; the changed exact head requires fresh settled checks and a later independent review before Ready/merge. diff --git a/project/STATE_AND_QUEUE.md b/project/STATE_AND_QUEUE.md index 7eba316c..3910f857 100644 --- a/project/STATE_AND_QUEUE.md +++ b/project/STATE_AND_QUEUE.md @@ -1,23 +1,22 @@ # STATE_AND_QUEUE — Scriptorium -STATE_REVISION: 200 +STATE_REVISION: 201 PHASE: M1 — Deterministic FantLab surface -LAST_COMMITTED_RUN_AT: 2026-09-21T00:55:00Z -LAST_RESULT: SCRIP-CORPUS-045 / Issue #177 completed. Independent exact-head review read PR #178 at `b8238144caeb23256eae6a61a49e47b07c687f29` against unchanged base `87929007415ede118cc5700a1e34f2cb3cfc576f`, confirmed 11 commits ahead / 0 behind, 11 changed files, no inline review threads, and all 16 PR-triggered workflows settled `success`. Dedicated Darwin/Rachinsky render-profile run `35546120433` / job `106172226254` checked out the exact reviewed SHA and passed 7 render-profile tests, 5 provenance tests and deterministic profile rebuild. Artifact `10616812708` was independently downloaded and verified as a 1,914-byte source-free ZIP / SHA-256 `12f3be444a4565f7aee5413aef9183487a072bfe9de47ae6b6d27fec05474fbb`, containing only the 12,592-byte render-profile JSON whose canonical embedded SHA-256 recomputes to `2ebcfea51282505c28824f54f6a1795bd7e87e2f958ac59f5c843e68c2066f3d`. Pages run `35546120447` / job `106172226404` checked out the same SHA and passed the full 376-test standard-library suite, canonical site build and deterministic rebuild. Review was recorded as COMMENT `5262352606`, not self-approval; PR #178 was marked Ready and squash-merged as `5e3833e96fd5e28fe19d24c8e20798fa54eadca4`, closing Issue #177 completed. -LAST_VERIFIED_PROGRESS: Master now retains an independently reviewed source-free fail-closed rendering decision profile for the 1864 Rachinsky translation of Darwin's *On the Origin of Species*, cryptographically bound to the reviewed exact-388 markup-surface freeze. The profile classifies all 18 observed tag shapes / 5,164 tokens and all 38 template shapes / 4,583 invocations while leaving 5 math/reference tag shapes / 518 tokens and all template semantics unresolved; `nop` remains explicitly inter-page-sensitive. `rendering_profile_frozen=true` is intentionally distinct from `renderer_semantics_complete=false`, `renderer_implementation_ready=false`, `rendering_equivalence_claimed=false` and `inter_page_composition_frozen=false`. No Page wikitext, template argument values, OCR, rendered prose, scan bytes or literary text are retained. Literary-body counts/digests, >=300k admission, FantLab source match, diagnostic readiness and M2 parity remain closed; M2 remains 0/5. +LAST_COMMITTED_RUN_AT: 2026-09-21T03:47:52Z +LAST_RESULT: SCRIP-CORPUS-046 / Issue #179 remains in recovery after independent review COMMENT `5262708679` found two acceptance/recoverability blockers in Draft PR #180: canonical `project/STATE_AND_QUEUE.md` had not recorded the open unit, and the canonical Darwin/Rachinsky candidate page still described merged PR #178 as Draft while omitting the new semantic-backlog artifact. This recovery unit repaired both surfaces on the PR branch: the candidate page now links/summarizes the source-free backlog and keeps every renderer/body/>=300k/FantLab/diagnostic/M2 gate closed, while canonical state now records SCRIP-CORPUS-046 / #179 / #180 as `REVIEW_PENDING`. The PR remains Draft; because the head changed, fresh exact-head checks must settle and a later independent review must re-read the repaired head before any Ready/merge decision. +LAST_VERIFIED_PROGRESS: The Draft PR contains a deterministic source-free unresolved-semantic backlog derived only after validating the reviewed Darwin/Rachinsky render-profile/source-surface boundary. It preserves all 43 unresolved shapes: 5 tag shapes / 518 tag tokens and 38 template shapes / 4,583 invocations, grouped into provider-template (37 shapes / 4,579 occurrences), provider-reference (3 / 466), provider-math (2 / 52), and inter-page (1 / 4) tracks. The mechanically selected next research target is template `ё`, arity 0/0, observed 2,227 times, but it remains explicitly unresolved. Canonical backlog SHA-256 is `4ec4f84566f5f3f8dccd1945906d1a782f4532547a13cadc8570cb5b682bdcb2`. This is review-pending branch state, not merged capability. Renderer semantics/implementation/equivalence, inter-page composition, literary-body counts/digests, >=300k admission, FantLab source match, diagnostic readiness and M2 parity remain closed; M2 remains 0/5. ## Current unit ```text -UNIT_ID: SCRIP-CORPUS-045 -ISSUE: #177 (closed completed) -STATUS: COMPLETE -PR: #178 (squash-merged as 5e3833e96fd5e28fe19d24c8e20798fa54eadca4) -NEXT_ACTION: Resume normal-flow selection from the queue. A later bounded Darwin/Rachinsky unit may resolve and - independently freeze provider/template/reference/math semantics explicitly left unresolved by the - reviewed fail-closed profile; inter-page separator/composition semantics remain a separate subsequent - decision. Keep literary-body, >=300k admission, FantLab source-match, diagnostic and M2 gates closed - until their evidence is independently verified. +UNIT_ID: SCRIP-CORPUS-046 +ISSUE: #179 (open) +STATUS: REVIEW_PENDING +PR: #180 (Draft; recovery repair committed after review COMMENT 5262708679) +NEXT_ACTION: Re-read the final repaired PR #180 head after all exact-head workflows settle. Independently verify the + canonical candidate-page/state repairs plus the existing semantic-backlog implementation/artifact. + If no blocker remains, mark Ready and merge in that later judgement run. Do not infer template `ё`, + reference, math or inter-page semantics here; keep renderer/body/>=300k/FantLab/diagnostic/M2 gates closed. ``` ## Current milestone gate @@ -51,7 +50,7 @@ Detailed evidence lives in `corpus/candidates/`, `benchmarks/`, `project/run-rec - **The Twelve Chairs** — SCRIP-CORPUS-044 complete. Master retains the independently reviewed parent route graph, all 410 exact Page dependency identities, exact backing Commons PDF identity (77,978,350 bytes; SHA-1 `4ab6aa42c3517169e99c1177b6fb6412cfe9187d`; SHA-256 `5a82f8101f9c17dfafcf8b45dc9ed5a7cdfa12a3e88d18bd0f0987e4fcd51eb4`), four-gap audit/membership decision, exact-410 source-free markup inventory and fail-closed rendering profile. All 30 template shapes / 602 invocations and 410 `` remain semantically unresolved; inter-page composition, literary-body counts/digests, >=300k admission and FantLab source identity remain unresolved. The later 1938/1961 editorial family stays distinct; coauthorship prevents individual-author attribution without an explicit VOICE model. - **The Life of Klim Samgin** — SCRIP-CORPUS-020 complete. Four exact parent revisions and Part 2 dependency are replay-frozen; merged PRs #111–#124 freeze target-only `#lst`, inferred `poemx1`/Poem boundaries, structural surface and thirteen plain literary values; #126 merged fail-closed extraction; #127 merged canonical source-free per-part/composite literary-body identities; #128 reconciled structured provenance. Exact historical MediaWiki/Poem deployment and FantLab analyzer-input/source-edition identity remain unresolved. - **Children of Captain Grant — Beketova translation** — SCRIP-CORPUS-026 complete. `oldid=5304880` plus `scriptorium-beketova-captain-grant-wikisource-body-v1` freeze a 1,095,467-character / 2,040,240-byte literary body with raw and normalized SHA-256 `4931eba535a4ef3989ac905d895258f00db57029e488d98511db50a7ce4d1161`. Beketova / Detgiz 1955 / `az.lib.ru` bibliography and explicit source PD evidence are retained. It clears the >=300k general-corpus rule. No FantLab linguistic result/analyzer-input identity is established for this translation; M2 weight remains zero. -- **On the Origin of Species — Rachinsky translation** — SCRIP-CORPUS-045 complete. Master retains 418 exact Page revision identities, reviewed 388-literary/30-apparatus composition contract, exact-388 source-free markup inventory, independently reviewed backing Commons DjVu identity (27,368,263 bytes; SHA-1 `75ef508588194ae74874272ce290f3ec1043ea9b`; SHA-256 `7f3ae1aadd4a844b193783a0c350e23815a76e7bb14a92bae7fa1c077354a2c8`) and the independently reviewed fail-closed render decision profile SHA-256 `2ebcfea51282505c28824f54f6a1795bd7e87e2f958ac59f5c843e68c2066f3d`. The profile covers all 18 tag / 38 template shapes while leaving 5 math/reference tag shapes / 518 tokens and all 38 template shapes / 4,583 invocations unresolved. Renderer implementation/equivalence, inter-page composition, literary-body count/digests and >=300k admission remain unfrozen; `admitted_for_calibration=false`. FantLab work 969964 currently exposes Timiryazev rather than Rachinsky; source match/M2 remain unknown/zero. +- **On the Origin of Species — Rachinsky translation** — SCRIP-CORPUS-045 is independently reviewed and complete on master. The current SCRIP-CORPUS-046 Draft PR adds a review-pending source-free semantic backlog over that frozen profile. Master retains 418 exact Page revision identities, reviewed 388-literary/30-apparatus composition contract, exact-388 source-free markup inventory, independently reviewed backing Commons DjVu identity (27,368,263 bytes; SHA-1 `75ef508588194ae74874272ce290f3ec1043ea9b`; SHA-256 `7f3ae1aadd4a844b193783a0c350e23815a76e7bb14a92bae7fa1c077354a2c8`) and the independently reviewed fail-closed render decision profile SHA-256 `2ebcfea51282505c28824f54f6a1795bd7e87e2f958ac59f5c843e68c2066f3d`. The review-pending backlog preserves all 43 unresolved shapes and mechanically selects template `ё` (2,227 invocations) as the next research target without resolving its semantics. Renderer implementation/equivalence, inter-page composition, literary-body count/digests and >=300k admission remain unfrozen; `admitted_for_calibration=false`. FantLab work 969964 currently exposes Timiryazev rather than Rachinsky; source match/M2 remain unknown/zero. ## Deterministic / morphology findings @@ -72,7 +71,7 @@ Detailed evidence lives in `corpus/candidates/`, `benchmarks/`, `project/run-rec - Shining World exposes the mismatch between 34 advertised links and 19 existing pages as an explicit blocker. - Running on Waves exposes a frozen 363,819-character literary body while preserving the separate 1980 route and keeping FantLab source match/M2 closed. - The Twelve Chairs exposes the independently reviewed fail-closed exact-surface rendering decision profile and unresolved template/reference counts without claiming renderer/body equivalence. -- SCRIP-CORPUS-045 now exposes the independently reviewed Darwin/Rachinsky exact-388 fail-closed rendering decision profile and synchronized structured provenance, including unresolved 38 template shapes / 4,583 invocations and 5 math/reference tag shapes / 518 tokens, without claiming renderer/body equivalence or corpus admission. +- The Darwin/Rachinsky canonical candidate page now exposes the review-pending source-free unresolved-semantic backlog, its four research tracks, canonical digest, and mechanically selected `ё` target while explicitly preserving every renderer/body/>=300k/FantLab/diagnostic/M2 gate as closed. - Klim Samgin exposes the source graph, fail-closed extractor, canonical source-free per-part/composite literary-body identities and structured provenance; FantLab source match remains unknown. - Live Pages deployment remains disabled behind `SCRIPTORIUM_PAGES_DEPLOY_ENABLED=true` plus repository Pages administration. @@ -91,5 +90,5 @@ Detailed evidence lives in `corpus/candidates/`, `benchmarks/`, `project/run-rec 11. Klim Samgin is frozen at candidate-specific literary-body/provenance level, but exact historical MediaWiki-core/Poem deployment equivalence and FantLab analyzer-input bytes/edition remain unproven. 12. The Beketova *Children of Captain Grant* translation has a frozen >=300k literary body, but no FantLab linguistic result/analyzer-input identity for this translation; it cannot advance M2 yet. 13. The 1928 *Twelve Chairs* route has exact source-free dependencies/scan/gap/surface/profile evidence, but all 30 template shapes / 602 invocations and 410 `` tokens still require evidenced semantics; inter-page composition, body identity and FantLab source identity remain unresolved. -14. Darwin/Rachinsky 1864 now has reviewed source-free 388/30 composition, exact-388 markup inventory, backing scan identity and fail-closed decision profile. All 38 template shapes / 4,583 invocations plus math/reference semantics and inter-page composition still require evidence before deterministic body rendering; body count/digests and >=300k admission remain unresolved, and FantLab's current Russian translation surface is a different translation. -15. Pages live activation remains a repository-admin effect and is off. +14. Darwin/Rachinsky 1864 has reviewed source-free 388/30 composition, exact-388 markup inventory, backing scan identity and fail-closed decision profile. Draft PR #180 now also carries a deterministic unresolved-semantic backlog, but all 43 shapes remain unresolved and the repaired head still requires fresh exact-head CI settlement plus independent review before merge; body count/digests and >=300k admission remain unresolved, and FantLab's current Russian translation surface is a different translation. +15. Pages live activation remains a repository-admin effect and is off. \ No newline at end of file diff --git a/project/run-receipts/SCRIP-CORPUS-046.md b/project/run-receipts/SCRIP-CORPUS-046.md new file mode 100644 index 00000000..22e9975f --- /dev/null +++ b/project/run-receipts/SCRIP-CORPUS-046.md @@ -0,0 +1,41 @@ +# Run receipt — SCRIP-CORPUS-046 + +Status: `REVIEW_PENDING` + +Issue: #179 +Draft PR: #180 +Base at selection: `master@11a06cb619e53810cb7cf3daa104488b28262613` + +## Unit + +Freeze a deterministic source-free priority backlog for the unresolved Darwin/Rachinsky 1864 rendering-semantic surface already frozen by SCRIP-CORPUS-045. This unit prioritizes evidence work; it does not resolve template/reference/math/inter-page semantics or implement a renderer. + +## Produced + +- `scriptorium/darwin_semantic_backlog.py` — validates the existing render profile against its frozen surface, preserves every unresolved shape, groups them by evidence track, ranks them deterministically, and validates the derived backlog. +- `darwin-origin-species-rachinsky-1864-ru.semantic-backlog.json` — canonical source-free backlog, SHA-256 `4ec4f84566f5f3f8dccd1945906d1a782f4532547a13cadc8570cb5b682bdcb2`. +- `tests/test_darwin_semantic_backlog.py` — rebuild, coverage, deterministic-next-target, drift and gate-boundary regressions. +- `.github/workflows/darwin-semantic-backlog.yml` — exact-head deterministic rebuild/artifact verification. +- public companion page `corpus/candidates/darwin-origin-species-rachinsky-1864-ru-semantic-backlog.md`. +- canonical candidate page `corpus/candidates/darwin-origin-species-rachinsky-1864-ru.md` now links and summarizes the backlog while preserving closed downstream gates. +- `project/STATE_AND_QUEUE.md` now records SCRIP-CORPUS-046 / #179 / #180 as the current `REVIEW_PENDING` unit for stateless recovery. + +## Frozen findings + +The backlog preserves exactly 5 unresolved tag shapes / 518 tag tokens and 38 unresolved template shapes / 4,583 invocations. Tracks are provider-template 37 shapes / 4,579 occurrences, provider-reference 3 / 466, provider-math 2 / 52, and inter-page 1 / 4. + +The deterministic next research target is template `ё`, arity 0 positional / 0 named, with 2,227 observed invocations. Its `semantic_status` remains `unresolved`; the ranking is not evidence of its behavior. + +## Verification / repair history + +The first PR-triggered run on head `b69bce39a35297c305c8267868fecbb2577d7787` correctly failed because one regression expected the wrong existing validator error string (`profile digest drift` versus `Darwin/Rachinsky rendering profile drift`). The test expectation was repaired in commit `e62e4401807037afdeec2d0714bd5176e94da24b`; no production boundary was weakened. The authored exact-head handoff for `b1e7e63afcf293cc31d827ea9a526cec216223ac` recorded the dedicated backlog workflow, Pages workflow and frozen diagnostic as green. + +Independent review COMMENT `5262708679` on that exact authored head found two acceptance/recoverability blockers outside the backlog semantics: canonical state still pointed at completed SCRIP-CORPUS-045, and the canonical Darwin candidate page still called merged PR #178 Draft while omitting the new semantic-backlog artifact. This recovery unit repaired both surfaces. Because those commits changed the PR head, the repaired exact head must receive fresh settled checks and a later independent review; the recovery run does not self-approve or merge. + +## Gates + +No Page wikitext, template arguments, rendered prose, OCR, scan bytes or literary text are persisted. `renderer_semantics_complete=false`, `renderer_implementation_ready=false`, `inter_page_composition_frozen=false`, `literary_body_count_and_digests_frozen=false`, `minimum_300k_proved=false`, `admitted_for_calibration=false`, `fantlab_source_edition_match=unknown`, `diagnostic_ready=false`, and `m2_parity_admissible=false`. M2 remains 0/5. + +## Handoff + +PR #180 remains Draft and `REVIEW_PENDING`. A later independent wake must re-read the final repaired exact head, inspect all settled checks and the source-free artifact, verify the canonical candidate/state repair plus the backlog diff, and only then decide Ready/merge. If accepted, the next renderer-research slice should investigate template `ё` with independent provider/history evidence rather than inferring semantics from its name. diff --git a/scriptorium/darwin_semantic_backlog.py b/scriptorium/darwin_semantic_backlog.py new file mode 100644 index 00000000..b848d75e --- /dev/null +++ b/scriptorium/darwin_semantic_backlog.py @@ -0,0 +1,215 @@ +"""Prioritize unresolved Darwin/Rachinsky render semantics without inventing them. + +The reviewed render profile already freezes the exact unresolved tag/template surface. +This module reduces that source-free evidence to a deterministic research backlog. It +never reads Page wikitext and never promotes an unresolved rendering behavior to a +semantic claim. +""" + +from __future__ import annotations + +import argparse +from collections import Counter +from hashlib import sha256 +import json +from pathlib import Path +from typing import Mapping, Sequence + +from .darwin_render_profile import CANDIDATE_ID, PROFILE_VERSION, validate_profile + + +BACKLOG_VERSION = "scriptorium-darwin-render-semantic-backlog-v1" +BACKLOG_STATUS = "unresolved_semantics_prioritized_no_semantics_inferred" +_TRACK_BY_HANDLING = { + "provider_template_semantics_required": "provider_template", + "provider_reference_semantics_required": "provider_reference", + "provider_math_semantics_required": "provider_math", + "inter_page_semantics_required": "inter_page", +} +_TRACK_ORDER = ("provider_template", "provider_reference", "provider_math", "inter_page") + + +def _canonical_json(value: object) -> str: + return json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(",", ":")) + + +def _sha256_json(value: object) -> str: + return sha256(_canonical_json(value).encode("utf-8")).hexdigest() + + +def _priority_key(item: Mapping[str, object]) -> tuple[object, ...]: + return ( + -int(item["count"]), + str(item["source_kind"]), + str(item["name"]), + str(item.get("kind", "")), + int(item.get("positional", -1)), + int(item.get("named", -1)), + ) + + +def build_backlog(profile: Mapping[str, object], surface: Mapping[str, object]) -> dict[str, object]: + """Build a source-free deterministic backlog from an already frozen profile.""" + + validate_profile(profile, surface) + if profile.get("renderer_semantics_complete") is not False: + raise ValueError("Darwin semantic backlog requires incomplete renderer semantics") + + items: list[dict[str, object]] = [] + for source_kind, rules in (("tag", profile.get("tag_rules")), ("template", profile.get("template_rules"))): + if not isinstance(rules, list): + raise ValueError(f"Darwin render profile missing {source_kind} rules") + for raw in rules: + if not isinstance(raw, dict) or raw.get("semantic_status") != "unresolved": + continue + handling = raw.get("handling") + if handling not in _TRACK_BY_HANDLING: + raise ValueError(f"unclassified unresolved Darwin handling: {handling!r}") + item: dict[str, object] = { + "source_kind": source_kind, + "name": raw["name"], + "count": raw["count"], + "handling": handling, + "semantic_status": "unresolved", + "track": _TRACK_BY_HANDLING[str(handling)], + } + if source_kind == "tag": + item["kind"] = raw["kind"] + else: + item["positional"] = raw["positional"] + item["named"] = raw["named"] + items.append(item) + + items.sort(key=_priority_key) + for rank, item in enumerate(items, 1): + item["priority_rank"] = rank + + coverage = profile.get("coverage") + if not isinstance(coverage, dict): + raise ValueError("Darwin render profile missing coverage") + tag_items = [row for row in items if row["source_kind"] == "tag"] + template_items = [row for row in items if row["source_kind"] == "template"] + totals = { + "tag_shape_count": len(tag_items), + "tag_token_count": sum(int(row["count"]) for row in tag_items), + "template_shape_count": len(template_items), + "template_invocation_count": sum(int(row["count"]) for row in template_items), + } + expected = { + "tag_shape_count": int(coverage["unresolved_tag_shape_count"]), + "tag_token_count": int(coverage["unresolved_tag_token_count"]), + "template_shape_count": int(coverage["unresolved_template_shape_count"]), + "template_invocation_count": int(coverage["unresolved_template_invocation_count"]), + } + if totals != expected: + raise ValueError("Darwin unresolved semantic coverage drift") + if not items: + raise ValueError("Darwin semantic backlog unexpectedly empty") + + shape_counts: Counter[str] = Counter() + occurrence_counts: Counter[str] = Counter() + for item in items: + track = str(item["track"]) + shape_counts[track] += 1 + occurrence_counts[track] += int(item["count"]) + tracks = [ + { + "track": track, + "shape_count": shape_counts[track], + "occurrence_count": occurrence_counts[track], + } + for track in _TRACK_ORDER + ] + + first = items[0] + next_slice = { + key: first[key] + for key in ("source_kind", "name", "count") + } + if first["source_kind"] == "tag": + next_slice["kind"] = first["kind"] + else: + next_slice["positional"] = first["positional"] + next_slice["named"] = first["named"] + next_slice["share_of_unresolved_template_invocations"] = { + "numerator": first["count"], + "denominator": totals["template_invocation_count"], + } + next_slice["semantic_status"] = "unresolved" + next_slice["reason"] = ( + "highest observed unresolved occurrence count; semantic meaning remains unresolved " + "until independently evidenced" + ) + + manifest: dict[str, object] = { + "schema_version": BACKLOG_VERSION, + "candidate_id": CANDIDATE_ID, + "status": BACKLOG_STATUS, + "source_profile": { + "profile_version": PROFILE_VERSION, + "profile_sha256": profile["profile_sha256"], + }, + "source_text_included": False, + "unresolved_totals": totals, + "resolution_tracks": tracks, + "prioritized_items": items, + "next_research_slice": next_slice, + "renderer_semantics_complete": False, + "renderer_implementation_ready": False, + "inter_page_composition_frozen": False, + "literary_body_count_and_digests_frozen": False, + "minimum_300k_proved": False, + "admitted_for_calibration": False, + "fantlab_source_edition_match": "unknown", + "diagnostic_ready": False, + "m2_parity_admissible": False, + } + manifest["backlog_sha256"] = _sha256_json(manifest) + return manifest + + +def validate_backlog(manifest: Mapping[str, object], profile: Mapping[str, object], surface: Mapping[str, object]) -> None: + expected = build_backlog(profile, surface) + if dict(manifest) != expected: + raise ValueError("Darwin semantic backlog drift") + if manifest.get("source_text_included") is not False: + raise ValueError("Darwin semantic backlog must remain source-free") + for key in ( + "renderer_semantics_complete", + "renderer_implementation_ready", + "inter_page_composition_frozen", + "literary_body_count_and_digests_frozen", + "minimum_300k_proved", + "admitted_for_calibration", + "diagnostic_ready", + "m2_parity_admissible", + ): + if manifest.get(key) is not False: + raise ValueError(f"{key} must remain false in Darwin semantic backlog") + if manifest.get("fantlab_source_edition_match") != "unknown": + raise ValueError("FantLab source match must remain unknown") + + +def build_backlog_from_paths(profile_path: Path, surface_path: Path) -> dict[str, object]: + profile = json.loads(profile_path.read_text(encoding="utf-8")) + surface = json.loads(surface_path.read_text(encoding="utf-8")) + if not isinstance(profile, dict) or not isinstance(surface, dict): + raise ValueError("Darwin semantic backlog inputs must be JSON objects") + return build_backlog(profile, surface) + + +def main(argv: Sequence[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--profile", type=Path, required=True) + parser.add_argument("--surface", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args(argv) + manifest = build_backlog_from_paths(args.profile, args.surface) + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(manifest, ensure_ascii=False, indent=2, sort_keys=True) + "\n", encoding="utf-8") + print(manifest["backlog_sha256"]) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_darwin_semantic_backlog.py b/tests/test_darwin_semantic_backlog.py new file mode 100644 index 00000000..3e0454ba --- /dev/null +++ b/tests/test_darwin_semantic_backlog.py @@ -0,0 +1,89 @@ +from __future__ import annotations + +from copy import deepcopy +import json +from pathlib import Path +import unittest + +from scriptorium.darwin_semantic_backlog import build_backlog, build_backlog_from_paths, validate_backlog + + +ROOT = Path(__file__).resolve().parents[1] +SURFACE = ROOT / "corpus/candidates/source-edition-traces/darwin-origin-species-rachinsky-1864-ru.render-surface.json" +PROFILE = ROOT / "corpus/candidates/source-edition-traces/darwin-origin-species-rachinsky-1864-ru.render-profile.json" +BACKLOG = ROOT / "corpus/candidates/source-edition-traces/darwin-origin-species-rachinsky-1864-ru.semantic-backlog.json" + + +def load(path: Path) -> dict[str, object]: + value = json.loads(path.read_text(encoding="utf-8")) + assert isinstance(value, dict) + return value + + +class DarwinSemanticBacklogTests(unittest.TestCase): + def test_committed_backlog_rebuilds_exactly_from_reviewed_profile(self) -> None: + expected = build_backlog_from_paths(PROFILE, SURFACE) + committed = load(BACKLOG) + self.assertEqual(committed, expected) + validate_backlog(committed, load(PROFILE), load(SURFACE)) + + def test_all_unresolved_shapes_are_preserved_and_grouped(self) -> None: + backlog = build_backlog(load(PROFILE), load(SURFACE)) + self.assertEqual( + backlog["unresolved_totals"], + { + "tag_shape_count": 5, + "tag_token_count": 518, + "template_shape_count": 38, + "template_invocation_count": 4583, + }, + ) + self.assertEqual( + backlog["resolution_tracks"], + [ + {"track": "provider_template", "shape_count": 37, "occurrence_count": 4579}, + {"track": "provider_reference", "shape_count": 3, "occurrence_count": 466}, + {"track": "provider_math", "shape_count": 2, "occurrence_count": 52}, + {"track": "inter_page", "shape_count": 1, "occurrence_count": 4}, + ], + ) + items = backlog["prioritized_items"] + self.assertEqual(len(items), 43) + self.assertTrue(all(row["semantic_status"] == "unresolved" for row in items)) + + def test_next_slice_is_mechanical_not_a_semantics_claim(self) -> None: + backlog = build_backlog(load(PROFILE), load(SURFACE)) + next_slice = backlog["next_research_slice"] + self.assertEqual(next_slice["source_kind"], "template") + self.assertEqual(next_slice["name"], "ё") + self.assertEqual(next_slice["count"], 2227) + self.assertEqual(next_slice["share_of_unresolved_template_invocations"], {"numerator": 2227, "denominator": 4583}) + self.assertEqual(next_slice["semantic_status"], "unresolved") + self.assertEqual(backlog["prioritized_items"][0]["priority_rank"], 1) + + def test_profile_drift_fails_before_backlog_derivation(self) -> None: + profile = deepcopy(load(PROFILE)) + profile["template_rules"] = [dict(row) for row in profile["template_rules"]] + profile["template_rules"][-1]["count"] += 1 + with self.assertRaisesRegex(ValueError, "rendering profile drift"): + build_backlog(profile, load(SURFACE)) + + def test_backlog_keeps_source_and_downstream_gates_closed(self) -> None: + backlog = build_backlog(load(PROFILE), load(SURFACE)) + self.assertFalse(backlog["source_text_included"]) + for key in ( + "renderer_semantics_complete", + "renderer_implementation_ready", + "inter_page_composition_frozen", + "literary_body_count_and_digests_frozen", + "minimum_300k_proved", + "admitted_for_calibration", + "diagnostic_ready", + "m2_parity_admissible", + ): + self.assertFalse(backlog[key], key) + self.assertEqual(backlog["fantlab_source_edition_match"], "unknown") + + +if __name__ == "__main__": + unittest.main()