From c35c70fdd8010e00ad0fb7e47fec4baf11afea55 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 20 Jun 2026 07:43:56 +0000 Subject: [PATCH 1/3] contract(unicharset): transcode UNICHARSET property accessors, byte-parity 112/112 Add get_is{alpha,lower,upper,digit,punctuation} + get_isngram + dump_properties to UniCharSet, parsing the per-line hex property bitmask (unicharset.cpp:824) into a props: Vec and masking each flag exactly as set_is*(id, properties & MASK) (unicharset.cpp:888-892). Accessors mirror the C++ inline guard (unicharset.h:497+): an out-of-range id -> false (INVALID_UNICHAR_ID); is_ngram is always-false on the plain-table load path (unicharset.cpp:893). Byte-identical 112/112 vs tesseract's own get_is* on real eng.lstm-unicharset via a self-validating oracle: one harness dumps both the id<->unichar bijection (proven 112/112, E-CPP-PARITY-1) and the new properties; the bijection diffing 0 proves the 5.5.0-header / 5.3.4-lib layout is sound, making the property diff (also 0) trustworthy despite the version skew. Third leaf through PROBE-OGAR-ADAPTER-UNICHARSET. - +5 unicharset tests (15 total); clippy -D warnings + fmt clean (scoped -p) - examples/unicharset_dump.rs gains a `properties` mode (reproduces the parity diff, matching the committed-harness pattern of the sibling leaves) - incidental: rustfmt 1.9.0 normalized 2 pre-existing test-assert wraps in class_view.rs - board: EPIPHANIES E-CPP-PARITY-3; LATEST_STATE branch-work + D-UNICHARSET-PROPS Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_016b33swuXE23hKtqxsHu9p1 --- .claude/board/EPIPHANIES.md | 11 + .claude/board/LATEST_STATE.md | 4 + .../examples/unicharset_dump.rs | 30 ++- crates/lance-graph-contract/src/class_view.rs | 11 +- crates/lance-graph-contract/src/unicharset.rs | 210 +++++++++++++++++- 5 files changed, 250 insertions(+), 16 deletions(-) diff --git a/.claude/board/EPIPHANIES.md b/.claude/board/EPIPHANIES.md index 7e53431e..579812e8 100644 --- a/.claude/board/EPIPHANIES.md +++ b/.claude/board/EPIPHANIES.md @@ -73,6 +73,17 @@ The 5+3 council's headline was "five crates linked into one binary with ZERO run **Next edges:** Grid→NodeRow over a REAL Spain fixture (E1 acceptance gate); the kanban loop (D2: `LanceVersionScheduler` → `KanbanMove` → SoA write → Lance commit). Cross-ref: PR #555 INTEGRATION_PLAN D1; battle-test plan probes A1/B-series; STATUS_BOARD symbiont-golden-image-harness; `crates/symbiont/src/bridge.rs`. +## 2026-06-20 — E-CPP-PARITY-3 — the UNICHARSET property accessors (`get_is{alpha,lower,upper,digit,punctuation}`) are byte-identical to libtesseract; the third leaf through PROBE-OGAR-ADAPTER-UNICHARSET + +**Status:** FINDING (in-env, real trained data). `lance_graph_contract::unicharset::UniCharSet`'s property family dumps the `eng.lstm-unicharset` per-id category bits **byte-identical to tesseract's own `UNICHARSET::get_is*` accessors, 112/112** — every id's `(isalpha, islower, isupper, isdigit, ispunctuation)` tuple. This is the THIRD adapter surface proven after `UniCharSet` id↔unichar (E-CPP-PARITY-1) and the `UNICHAR` codec (E-CPP-PARITY-2): the core-first transcode now covers a content-store tier, a pure codec, AND a per-entry property decode. + +**The transcode.** Tesseract parses the second whitespace token of each line as a hex bitmask (`unicharset.cpp:824`, `stream >> std::hex >> properties`) and sets each flag via `set_is*(id, properties & MASK)` (`unicharset.cpp:888-892`); masks are `ISALPHA=0x1 ISLOWER=0x2 ISUPPER=0x4 ISDIGIT=0x8 ISPUNCTUATION=0x10` (`unicharset.cpp:41-45`). The Rust loader now parses + stores that byte (masked to the 5 meaningful bits) parallel to `reverse`, and the accessors mirror the C++ inline guard (`unicharset.h:497+`): out-of-range id (the `INVALID_UNICHAR_ID` sentinel) → `false`, else the stored bit. `is_ngram` is always-false on the plain-table load path (`unicharset.cpp:893`), faithfully reproduced. + +**The self-validating oracle (handles a real version skew honestly).** The in-env libtesseract is **5.3.4** with NO installed dev headers; the source tree is **5.5.0**. Compiling 5.5.0 headers against the 5.3.4 lib is an ABI hazard (the lib's `load_from_file` fills a struct my header-inlined accessors then read). Rather than assume it safe, the oracle (`/tmp/uniprops_oracle.cpp`, links `-ltesseract`) dumps BOTH halves: the id↔unichar **bijection** (for which E-CPP-PARITY-1 is a proven 112/112 Rust reference) AND the new properties. Step 1: the bijection half diffed **0 differences** vs the Rust `dump()` → the 5.5.0-header/5.3.4-lib object layout is proven sound for the fields read. Step 2 (now trustworthy): the property half diffed **0 differences** vs `dump_properties()`. A failing Step 1 would have invalidated Step 2; it passed, so the skew is empirically a non-issue here. (No encoding/threshold was tuned to force green — one input, one diff.) + +**Pattern confirmation (per E-CPP-KEYSTONE-1).** This leaf is exactly the predicted "repetition of a validated pattern": one new accessor family + one byte-parity `diff`, no new architecture, no Core gap (the property byte rides the existing content tier, the adapter holds no state). Consumed by `tesseract-core` as `CharSet::get_is*` with a consumer-boundary test; no re-implementation downstream. + +Cross-ref: `E-CPP-PARITY-1` (bijection 112/112), `E-CPP-PARITY-2` (UNICHAR 268/268), `E-CPP-KEYSTONE-1` (end-to-end dispatch), `.claude/knowledge/core-first-transcode-doctrine.md` (PROBE-OGAR-ADAPTER-UNICHARSET). Branch work (`claude/happy-hamilton-0azlw4`), lance-graph + tesseract-rs. --- diff --git a/.claude/board/LATEST_STATE.md b/.claude/board/LATEST_STATE.md index beba1e8e..4279d1d5 100644 --- a/.claude/board/LATEST_STATE.md +++ b/.claude/board/LATEST_STATE.md @@ -16,6 +16,8 @@ --- +> **2026-06-20 — branch work (`claude/happy-hamilton-0azlw4`)** — **UNICHARSET property accessors transcoded + byte-parity proven (E-CPP-PARITY-3), the third leaf through PROBE-OGAR-ADAPTER-UNICHARSET.** `lance_graph_contract::unicharset::UniCharSet` now parses the per-line hex property bitmask (`unicharset.cpp:824`) into a `props: Vec` and exposes `get_is{alpha,lower,upper,digit,punctuation}` + `get_isngram` + `dump_properties()`, mirroring the C++ inline accessors (`unicharset.h:497+`; out-of-range id → `false`, `INVALID_UNICHAR_ID` semantics). **Byte-identical 112/112** on real `eng.lstm-unicharset` vs tesseract's own `get_is*` via a **self-validating** oracle: the same harness dumps the id↔unichar bijection (proven 112/112 reference, E-CPP-PARITY-1) AND the properties — the bijection half diffing 0 proves the 5.5.0-header/5.3.4-lib layout is sound, making the property diff (also 0) trustworthy despite the version skew. Additive, zero-dep; +5 contract tests (15 unicharset total), clippy `-D warnings` + fmt clean. Consumed by `tesseract-core` as `CharSet::get_is*` (+1 consumer-boundary test, 4/4 green). Incidental: rustfmt-1.9.0 normalized two pre-existing test-assert wraps in `class_view.rs` (whitespace-only). No Core gap, no adapter state (per `E-CPP-KEYSTONE-1` "repetition of a validated pattern"). EPIPHANIES `E-CPP-PARITY-3`. +> > **2026-06-19 — IN PR (branch `claude/edge-distance-basin-node-epiphany`)** — **basin-IS-a-node: the substrate is a virtual tree of MailboxSoAs, navigated by pure key arithmetic.** New `graph::mailbox_scan::{members, memberof, BasinOf}` — one-to-many (`members` = direct children one HHTL tier down) / many-to-one (`memberof` = parent via `NiblePath::parent`, returns `BasinOf::Local(row)` or `BasinOf::Route(NiblePath)` when the parent lives in another shard — the HHTL prefix IS the route key, **no coarse-fingerprint table**; `None` only at the top tier). Realizes `E-BASIN-IS-A-NODE` with **no ownership restructure** — the tree is the radix trie of the keys, the SoA stays flat, the zero-copy/Lance-tombstone invariant is untouched; all navigation is **zero value decode** (F2-guarded). 16/16 mailbox_scan tests, clippy clean. **Probe (perturbation-sim `basin_placement_learning.rs`): field-perturbation placement learns the basin tree — green, mean tree-hop 1.00 vs 4.13 random (75.8 % tighter)**, promoting the one CONJECTURE in `E-BASIN-IS-A-NODE` to measured FINDING [G]. **Three epiphanies this arc:** `E-BASIN-IS-A-NODE` (basin=node; distance=hop=`node_distance(PrefixDepth)`; 4-ary fan-out = Morton tile pyramid = perturbation-learnable field), `E-FAMILY-NODE-IS-META-AWARENESS` (the parent node IS the coarse Walsh band of its subtree — meta-awareness is structural, not a column), `E-GUID-SELF-ROUTES-THE-BASIN-TREE` (HHTL-tier truncation of the GUID = every ancestor's route key; the GUID self-routes). **Capstone:** one 512 B key, read five ways — representation / ontology / compute (Morton pyramid) / learning / meta-awareness — four of the five are key-resident zero-decode. Builds on #544/#545/#548 (mailbox_scan facets) + `E-COARSE-QUANTIZER-IS-SCALE-FREE-ROUTER`. > > **2026-06-18 — branch work** — **OGAR → lance-graph-ontology wiring closed.** `OntologyRegistry::class_id_for_guid(&NodeGuid) -> Option` composes the canon GUID→NiblePath fold (`contract::hhtl::NiblePath::from_guid_prefix`) with the registry's `NiblePath ↔ entity_type` bijection — the single missing join an audit this session surfaced (both halves were built with **ZERO callers**). A node carrying a classid now resolves its ontology class → `RegistryClassView` (fields/labels/template/DOLCE). Round-trip test pins the `classid_lo ↔ entity_type` consistency the audit flagged; zero-fallback (unbound → None) + lossy-fold refusal (high classid u16 → None). Completes the third "classid → X" axis reachable from a GUID (read-mode ✅ ocr.rs, methods ✅ unicharset keystone, ontology-shape ✅ now); aligns with `E-ODOO-CORE-FIRST-STRUCTURAL` (Core-side resolution, no new predicate/type). 16 ontology tests green; `registry.rs` clippy-clean + fmt clean. EPIPHANIES `E-OGAR-ONTOLOGY-WIRED-1`. Pre-existing `lance-graph-ontology` clippy debt noted (`TD-ONTOLOGY-LINT`). @@ -106,6 +108,8 @@ > **2026-06-18 — ADDED (D-DO-ARM-1, the OGAR DO arm)**: `lance_graph_contract::action::{ActionState, StateGuard, ActionDef, ClassActions, actions_for, effective_actions, ActionInvocation}` — the Perdurant DO arm completing the OGAR IR (the action-axis sibling of `codegen_manifest`'s `MethodSig`/THINK). Both the 4-agent `sale_order` AR→DO probe (runtime-archaeologist) AND the merged cross-repo PR survey (ruff/OGAR/lance-graph/openproject/tesseract) agreed this was the ONE missing wire: the THINK arm (`classid → ClassView`, `has_function → MethodSig`) is converged + merged; the DO-arm `ActionInvocation`/`ActionDef` type was ABSENT. **`ActionDef`** (static, `const`-constructible, all `&'static`/`Copy`): `predicate` (= harvested `has_function` method), `object_class` (classid), `exec` (`ExecTarget` incl `SurrealQl`), `guard` (`StateGuard` = KausalSpec field==value), `required_role` (RBAC), `overrides` (OGAR `classid→ClassView` inheritance). **`ClassActions`+`actions_for`** (zero-fallback) mirror `ClassMethods`/`methods_for`. **`effective_actions(parent, child)`** = OGAR inheritance on the action axis (child overrides parent by predicate). **`ActionInvocation`** (dynamic, `Copy`): lifecycle `ActionState{Pending→Committed|Failed|Cancelled}` (sticky terminals), S2.5 `cycle` stamp, idempotency/trace keys, HLC `emitted_at_millis`. **`ActionInvocation::commit(def, actor, impact, now)`** is the gated egress — RBAC FIRST (`auth::ActorContext` must hold `required_role` or be admin → else `Failed`), THEN MUL impact (`mul::GateDecision`: `Flow→Committed`+stamped, `Hold→`Pending/escalate, `Block→Cancelled`). This IS "commit to the external consumer (odoo/openproject/woa/tesseract) after the cycle decides sound." Dispatched via `UnifiedStep`/`ExecTarget`, NOT a per-crate endpoint. Additive, zero-dep. +5 tests green. Consumer reference: `docs/OGAR_CONSUMER_API.md`. Branch `claude/soa-write-deinterlace-inc2`. +> **2026-06-20 — ADDED (D-UNICHARSET-PROPS, the property-accessor leaf)**: `lance_graph_contract::unicharset::UniCharSet` gained the character-category surface `get_isalpha` / `get_islower` / `get_isupper` / `get_isdigit` / `get_ispunctuation` / `get_isngram` + `dump_properties()`, backed by a new `props: Vec` parsed from the per-line hex bitmask (`unicharset.cpp:824`; masked to `ISALPHA=0x1 ISLOWER=0x2 ISUPPER=0x4 ISDIGIT=0x8 ISPUNCTUATION=0x10`). Accessors mirror the C++ inline guard (`unicharset.h:497+`): out-of-range id → `false` (`INVALID_UNICHAR_ID`); `get_isngram` is always-false on the plain-table load path (`unicharset.cpp:893`). **Byte-identical 112/112** vs tesseract's own `get_is*` on real `eng.lstm-unicharset` (self-validating oracle: bijection half cross-checks the 5.5.0-header/5.3.4-lib layout, then the property half diffs 0). Additive, zero-dep, behaviour-preserving on the existing id↔unichar bijection (lenient default-0 for a missing/!hex token). +5 tests (15 unicharset total). Consumed by `tesseract-core::CharSet::get_is*`. EPIPHANIES `E-CPP-PARITY-3`; the third leaf of `PROBE-OGAR-ADAPTER-UNICHARSET` (after D-UNICHARSET-1 + D-UNICHAR-1). Branch `claude/happy-hamilton-0azlw4`. + > **2026-06-18 — ADDED (D-UNICHARSET-KEYSTONE, classid → ClassView → adapter wiring)**: `lance_graph_contract::unicharset_adapter::{UniCharSetStore, UniCharCall, UniCharOut, DispatchError, invoke_unicharset}` — steps 2–3 of `PROBE-OGAR-ADAPTER-UNICHARSET`, the keystone composing the proven `UniCharSet` adapter through the OGAR Core's three movable parts. `invoke_unicharset(registry, store, classid, call)`: (1) **ClassView composition gate** — `codegen_manifest::methods_for(registry, classid)` must list the call's method (the harvested `has_function` manifest), else `MethodNotComposed` (zero-fallback: an unconfigured classid composes nothing); (2) **content-store tier** — `UniCharSetStore::unicharset(classid)`, a consumer-provided trait (dependency-inverted like `ClassView`/`PlannerContract`; the adapter holds NO state — `I-VSA-IDENTITIES`); (3) **adapter leaf** — routes to `UniCharSet::{id_to_unichar, unichar_to_id}`. DO-in (`UniCharCall`) / DO-out (`UniCharOut`, zero-copy borrow). **Byte-parity inherited** from `UniCharSet` (112/112); the keystone proves the dispatch path is faithful (the `NULL`→space edge survives it), the gate works, and there is **no Core gap** (the doctrine's iron guard holds — the variable-length bijection rides the content tier cleanly). NOT routed through the heavy `OrchestrationBridge` (cross-subsystem router); this is the adapter-invocation primitive a `UnifiedStep` calls. Additive, zero-dep. +5 tests; clippy `--all-targets -D warnings` + fmt clean. Completes the core-first doctrine END-TO-END for the unicharset leaf (`E-CPP-KEYSTONE-1`). > **2026-06-17 — ADDED (D-UNICHAR-1, SECOND byte-parity adapter)**: `lance_graph_contract::unichar::{utf8_step, utf8_to_utf32}` — the Tesseract `UNICHAR` UTF-8 codec that `UNICHARSET` sits on top of (`ccutil/unichar.cpp`). `utf8_step(lead) -> u8` is a `const fn` transcription of Tesseract's 256-entry lead-byte table (1/2/3/4 for legal leads, 0 for continuation bytes `0x80..=0xBF` + `0xF8..`); `utf8_to_utf32(bytes) -> Option>` mirrors `UNICHAR::UTF8ToUTF32` (lead-byte validation only, `None` on an illegal lead). **The second adapter through the transcode pipeline, byte-parity proven**: `examples/unichar_dump.rs` vs a libtesseract `UNICHAR` oracle is **268/268 identical** (256 EXHAUSTIVE `utf8_step` lead-byte values + 12 `utf8_to_utf32` corpus rows). Faithful-transcode note (the point of the exercise): Tesseract maps `0xC0`/`0xC1` to step 2 and decodes the overlong NUL `C0 80` to `[0]`; `core::str::from_utf8` REJECTS both, so a native-UTF-8 shortcut would silently diverge — mirroring the exact table is mandatory (`from_utf8_rejects_what_tesseract_accepts` test pins it). Additive, zero-dep, pure text (no leptonica). +8 tests + the `unichar_dump` example; 653 contract lib green; clippy `--all-targets -D warnings` clean. Sibling of D-UNICHARSET-1, same `PROBE-OGAR-ADAPTER-UNICHARSET` falsifier family (E-CPP-PARITY-2). diff --git a/crates/lance-graph-contract/examples/unicharset_dump.rs b/crates/lance-graph-contract/examples/unicharset_dump.rs index 468cbe57..ef875400 100644 --- a/crates/lance-graph-contract/examples/unicharset_dump.rs +++ b/crates/lance-graph-contract/examples/unicharset_dump.rs @@ -1,14 +1,23 @@ -//! Dump a `.unicharset`'s id→unichar table — the Rust side of the byte-parity -//! probe `PROBE-OGAR-ADAPTER-UNICHARSET`. +//! Dump a `.unicharset`'s id→unichar table (default) or its per-id property bits +//! (`properties` mode) — the Rust side of the byte-parity probe +//! `PROBE-OGAR-ADAPTER-UNICHARSET`. //! //! ```sh //! # on a box with libtesseract + libleptonica installed: //! combine_tessdata -u $(dpkg -L tesseract-ocr-eng | grep eng.traineddata) /tmp/eng. -//! # C++ oracle (links -lleptonica only to satisfy the linker; never calls it): -//! # g++ oracle.cpp -ltesseract -lleptonica -o oracle && ./oracle /tmp/eng.unicharset > /tmp/oracle.tsv +//! # C++ oracle (links -lleptonica only to satisfy the linker; never calls it). +//! # An oracle that prints BOTH modes self-validates the build: the bijection +//! # half is a proven 112/112 reference (E-CPP-PARITY-1), so a 0-diff there +//! # confirms the object layout before the property half (E-CPP-PARITY-3) is +//! # trusted — important when the source header and installed lib versions skew. +//! # g++ oracle.cpp -ltesseract -lleptonica -o oracle +//! # ./oracle /tmp/eng.unicharset bijection > /tmp/oracle.tsv +//! # ./oracle /tmp/eng.unicharset properties > /tmp/oracle_props.tsv //! # Rust side: -//! cargo run -p lance-graph-contract --example unicharset_dump -- /tmp/eng.unicharset > /tmp/rust.tsv -//! diff /tmp/oracle.tsv /tmp/rust.tsv # byte-identical => CONJECTURE -> FINDING +//! cargo run -p lance-graph-contract --example unicharset_dump -- /tmp/eng.unicharset > /tmp/rust.tsv +//! cargo run -p lance-graph-contract --example unicharset_dump -- /tmp/eng.unicharset properties > /tmp/rust_props.tsv +//! diff /tmp/oracle.tsv /tmp/rust.tsv && diff /tmp/oracle_props.tsv /tmp/rust_props.tsv +//! # both byte-identical => CONJECTURE -> FINDING //! ``` #![allow( @@ -23,12 +32,17 @@ use lance_graph_contract::unicharset::UniCharSet; fn main() -> ExitCode { let Some(path) = std::env::args().nth(1) else { - eprintln!("usage: unicharset_dump "); + eprintln!("usage: unicharset_dump [properties]"); return ExitCode::FAILURE; }; + let properties = std::env::args().nth(2).as_deref() == Some("properties"); match UniCharSet::load_from_file(Path::new(&path)) { Ok(unicharset) => { - print!("{}", unicharset.dump()); + if properties { + print!("{}", unicharset.dump_properties()); + } else { + print!("{}", unicharset.dump()); + } ExitCode::SUCCESS } Err(err) => { diff --git a/crates/lance-graph-contract/src/class_view.rs b/crates/lance-graph-contract/src/class_view.rs index 68f1f0c5..636fa5be 100644 --- a/crates/lance-graph-contract/src/class_view.rs +++ b/crates/lance-graph-contract/src/class_view.rs @@ -558,7 +558,11 @@ mod tests { fn compute_dag_topo_order_respects_dependencies() { // chain f0→f1→f2: f1 must come before f2; f0 is a leaf, not emitted. let order = compute_dag_topo_order(SAMPLE_DAG).expect("acyclic has an order"); - assert_eq!(order.len(), 2, "two targets (f1, f2); f0 is a read-only leaf"); + assert_eq!( + order.len(), + 2, + "two targets (f1, f2); f0 is a read-only leaf" + ); let pos1 = order.iter().position(|&t| t == 1).unwrap(); let pos2 = order.iter().position(|&t| t == 2).unwrap(); assert!(pos1 < pos2, "f1 recomputed before its dependent f2"); @@ -602,7 +606,10 @@ mod tests { ]; let order = compute_dag_topo_order(diamond).expect("acyclic"); let p = |t: u8| order.iter().position(|&x| x == t).unwrap(); - assert!(p(1) < p(3) && p(2) < p(3), "both precedents before the join"); + assert!( + p(1) < p(3) && p(2) < p(3), + "both precedents before the join" + ); assert_eq!(order.len(), 3); } diff --git a/crates/lance-graph-contract/src/unicharset.rs b/crates/lance-graph-contract/src/unicharset.rs index 2e765439..469a5fa7 100644 --- a/crates/lance-graph-contract/src/unicharset.rs +++ b/crates/lance-graph-contract/src/unicharset.rs @@ -21,14 +21,27 @@ //! # Format scope //! //! The `.unicharset` format is: line 1 = entry count `N`; then `N` lines, each -//! beginning with the unichar as its first whitespace-delimited token (the -//! remaining columns — properties / script / bounding boxes — do not affect the -//! id↔unichar bijection and are ignored). The line position (0-based, after the -//! count line) IS the unichar id. This is the `old_style_included_ == true` +//! beginning with the unichar as its first whitespace-delimited token, followed +//! by the **properties** as the second token (a hex bitmask), then script / +//! bounding boxes / case / direction columns. The line position (0-based, after +//! the count line) IS the unichar id. This is the `old_style_included_ == true` //! plain-table scope the adapter-shaper bounded; fragment/`CleanupString` //! normalization is a separate, later leaf. Any special-token edge case a real //! `eng.unicharset` reveals on first diff is refined then — this is built to the //! documented format, diff-pending. +//! +//! # Properties leaf +//! +//! The second token is a hex bitmask (tesseract `unicharset.cpp:824`, +//! `stream >> std::hex >> properties`) decoded by `set_is*(id, properties & MASK)` +//! at `unicharset.cpp:888-892`. The masks (`unicharset.cpp:41-45`) are +//! `ISALPHA=0x1 ISLOWER=0x2 ISUPPER=0x4 ISDIGIT=0x8 ISPUNCTUATION=0x10`. The +//! [`UniCharSet::get_isalpha`] family mirrors the C++ accessors +//! (`unicharset.h:497+`): an out-of-range id (the C++ `INVALID_UNICHAR_ID` +//! sentinel) returns `false`, else the stored bit. `is_ngram` is never set by +//! the plain-table loader (`unicharset.cpp:893` always `set_isngram(id, false)`) +//! so [`UniCharSet::get_isngram`] is always `false` for a file-loaded set. +//! [`UniCharSet::dump_properties`] is the byte-parity surface for these bits. use std::collections::HashMap; use std::path::Path; @@ -40,8 +53,26 @@ pub struct UniCharSet { reverse: Vec, /// unichar → id (the inverse of `reverse`). lookup: HashMap, + /// id → property bitmask (`ISALPHA|ISLOWER|ISUPPER|ISDIGIT|ISPUNCTUATION`), + /// parallel to `reverse`. Only the low 5 bits are meaningful; see the + /// `*_MASK` consts. + props: Vec, } +/// `isalpha` property bit (tesseract `unicharset.cpp:41`). +const ISALPHA_MASK: u8 = 0x1; +/// `islower` property bit (tesseract `unicharset.cpp:42`). +const ISLOWER_MASK: u8 = 0x2; +/// `isupper` property bit (tesseract `unicharset.cpp:43`). +const ISUPPER_MASK: u8 = 0x4; +/// `isdigit` property bit (tesseract `unicharset.cpp:44`). +const ISDIGIT_MASK: u8 = 0x8; +/// `ispunctuation` property bit (tesseract `unicharset.cpp:45`). +const ISPUNCTUATION_MASK: u8 = 0x10; +/// All meaningful property bits — the loader masks the parsed hex to these. +const PROPERTY_BITS: u8 = + ISALPHA_MASK | ISLOWER_MASK | ISUPPER_MASK | ISDIGIT_MASK | ISPUNCTUATION_MASK; + impl UniCharSet { /// Parse a `.unicharset` from its text contents. See the module docs for the /// format. Properties columns after the leading unichar token are ignored. @@ -63,6 +94,7 @@ impl UniCharSet { let mut reverse = Vec::with_capacity(count); let mut lookup = HashMap::with_capacity(count); + let mut props = Vec::with_capacity(count); for line in lines.take(count) { // The unichar is the first whitespace-delimited token; the id is the // entry's position. A unichar repeated in the file keeps its FIRST @@ -74,11 +106,26 @@ impl UniCharSet { // token), and load remaps `"NULL"` -> `" "` (tesseract // `unicharset.cpp:882`). The byte-parity probe surfaced this as the // sole id-0 diff against the C++ oracle. - let token = line.split_whitespace().next().unwrap_or(""); + let mut tokens = line.split_whitespace(); + let token = tokens.next().unwrap_or(""); let unichar = if token == "NULL" { " " } else { token }.to_string(); + // The second token is the property bitmask in hex (tesseract + // `unicharset.cpp:824`). Parse leniently — a missing/!hex token means + // "no properties" (0), matching this loader's documented tolerance for + // partial lines; a well-formed `eng.unicharset` always supplies it, so + // the byte-parity diff is unaffected. Mask to the 5 meaningful bits + // exactly as `set_is*(id, properties & MASK)` does downstream. + let properties = tokens + .next() + .and_then(|t| u32::from_str_radix(t, 16).ok()) + .unwrap_or(0); let id = u32::try_from(reverse.len()).map_err(|_| UniCharSetError::BadCount)?; lookup.entry(unichar.clone()).or_insert(id); reverse.push(unichar); + // `try_from` always succeeds here (the mask bounds the value to + // <= 0x1F); the fallback keeps the path total without `unwrap`. + let prop_byte = u8::try_from(properties & u32::from(PROPERTY_BITS)).unwrap_or(0); + props.push(prop_byte); } if reverse.len() != count { @@ -87,7 +134,11 @@ impl UniCharSet { found: reverse.len(), }); } - Ok(Self { reverse, lookup }) + Ok(Self { + reverse, + lookup, + props, + }) } /// Parse a `.unicharset` file from disk (a thin wrapper over @@ -123,6 +174,80 @@ impl UniCharSet { self.lookup.get(unichar).copied() } + /// Test property bit `mask` for `id`. An out-of-range `id` (the C++ + /// `INVALID_UNICHAR_ID` sentinel) returns `false`, mirroring the C++ + /// accessor guard at `unicharset.h:497+`. + fn has_property(&self, id: u32, mask: u8) -> bool { + self.props.get(id as usize).is_some_and(|&p| p & mask != 0) + } + + /// Whether `id` is alphabetic (`ISALPHA`). Out-of-range → `false`. + /// Mirrors `UNICHARSET::get_isalpha` (tesseract `unicharset.h`). + #[must_use] + pub fn get_isalpha(&self, id: u32) -> bool { + self.has_property(id, ISALPHA_MASK) + } + + /// Whether `id` is lower-case (`ISLOWER`). Out-of-range → `false`. + /// Mirrors `UNICHARSET::get_islower`. + #[must_use] + pub fn get_islower(&self, id: u32) -> bool { + self.has_property(id, ISLOWER_MASK) + } + + /// Whether `id` is upper-case (`ISUPPER`). Out-of-range → `false`. + /// Mirrors `UNICHARSET::get_isupper`. + #[must_use] + pub fn get_isupper(&self, id: u32) -> bool { + self.has_property(id, ISUPPER_MASK) + } + + /// Whether `id` is a digit (`ISDIGIT`). Out-of-range → `false`. + /// Mirrors `UNICHARSET::get_isdigit`. + #[must_use] + pub fn get_isdigit(&self, id: u32) -> bool { + self.has_property(id, ISDIGIT_MASK) + } + + /// Whether `id` is punctuation (`ISPUNCTUATION`). Out-of-range → `false`. + /// Mirrors `UNICHARSET::get_ispunctuation`. + #[must_use] + pub fn get_ispunctuation(&self, id: u32) -> bool { + self.has_property(id, ISPUNCTUATION_MASK) + } + + /// Whether `id` is an n-gram. The plain-table loader always clears this + /// (`unicharset.cpp:893`), so a file-loaded set returns `false` for every + /// id; this mirrors `UNICHARSET::get_isngram` for that load path. + #[must_use] + pub fn get_isngram(&self, _id: u32) -> bool { + false + } + + /// Render the id→properties table as + /// `"\t \n"` lines + /// (each flag `0`/`1`) — the exact shape the C++ property oracle prints, so + /// the byte-parity diff is `diff oracle_props.tsv rust_props.tsv`. + #[must_use] + pub fn dump_properties(&self) -> String { + let mut out = String::new(); + for id in 0..self.reverse.len() as u32 { + out.push_str(&id.to_string()); + out.push('\t'); + out.push(if self.get_isalpha(id) { '1' } else { '0' }); + out.push(' '); + out.push(if self.get_islower(id) { '1' } else { '0' }); + out.push(' '); + out.push(if self.get_isupper(id) { '1' } else { '0' }); + out.push(' '); + out.push(if self.get_isdigit(id) { '1' } else { '0' }); + out.push(' '); + out.push(if self.get_ispunctuation(id) { '1' } else { '0' }); + out.push('\n'); + } + out + } + /// Render the id→unichar table as `"\t\n"` lines — the exact /// shape the C++ oracle harness prints, so a byte-parity diff is /// `diff oracle_dump.tsv rust_dump.tsv`. @@ -224,6 +349,79 @@ cd 5 0,255,0,255,0,255,0,255,0,255 0 cd Left cd cd ); } + /// A sample exercising each property mask via the hex second column: + /// `0x3`=alpha+lower, `0x5`=alpha+upper, `0x8`=digit, `0x10`=punct, `0x1`=alpha. + const PROPS_SAMPLE: &str = "\ +5 +a 3 0,255,0,255,0,0,0,0,0,0 Latin 0 0 0 a +A 5 0,255,0,255,0,0,0,0,0,0 Latin 0 0 0 A +7 8 0,255,0,255,0,0,0,0,0,0 Common 0 0 0 7 +. 10 0,255,0,255,0,0,0,0,0,0 Common 0 0 0 . +x 1 0,255,0,255,0,0,0,0,0,0 Latin 0 0 0 x +"; + + #[test] + fn properties_decode_from_hex_column() { + let u = UniCharSet::load_from_str(PROPS_SAMPLE).expect("valid"); + // id 0 "a": 0x3 = ISALPHA | ISLOWER + assert!(u.get_isalpha(0) && u.get_islower(0)); + assert!(!u.get_isupper(0) && !u.get_isdigit(0) && !u.get_ispunctuation(0)); + // id 1 "A": 0x5 = ISALPHA | ISUPPER + assert!(u.get_isalpha(1) && u.get_isupper(1)); + assert!(!u.get_islower(1)); + // id 2 "7": 0x8 = ISDIGIT + assert!(u.get_isdigit(2)); + assert!(!u.get_isalpha(2) && !u.get_ispunctuation(2)); + // id 3 ".": 0x10 = ISPUNCTUATION + assert!(u.get_ispunctuation(3)); + assert!(!u.get_isalpha(3) && !u.get_isdigit(3)); + // id 4 "x": 0x1 = ISALPHA only + assert!(u.get_isalpha(4)); + assert!(!u.get_islower(4) && !u.get_isupper(4)); + } + + /// The C++ accessor guards `INVALID_UNICHAR_ID` → `false`; an out-of-range id + /// is the Rust analogue and must not panic. + #[test] + fn properties_out_of_range_is_false() { + let u = UniCharSet::load_from_str(PROPS_SAMPLE).expect("valid"); + assert!(!u.get_isalpha(99)); + assert!(!u.get_islower(99)); + assert!(!u.get_isupper(99)); + assert!(!u.get_isdigit(99)); + assert!(!u.get_ispunctuation(99)); + } + + /// The plain-table loader always clears `isngram` (`unicharset.cpp:893`). + #[test] + fn isngram_always_false() { + let u = UniCharSet::load_from_str(PROPS_SAMPLE).expect("valid"); + for id in 0..u.size() as u32 { + assert!(!u.get_isngram(id)); + } + assert!(!u.get_isngram(99)); + } + + #[test] + fn dump_properties_matches_oracle_shape() { + let u = UniCharSet::load_from_str(PROPS_SAMPLE).expect("valid"); + assert_eq!( + u.dump_properties(), + "0\t1 1 0 0 0\n1\t1 0 1 0 0\n2\t0 0 0 1 0\n3\t0 0 0 0 1\n4\t1 0 0 0 0\n" + ); + } + + /// A missing or non-hex properties token defaults to "no properties" (the + /// loader's documented tolerance for partial lines); the id↔unichar + /// bijection is unaffected. + #[test] + fn missing_properties_token_defaults_to_zero() { + let u = UniCharSet::load_from_str("2\na\nb 3\n").expect("valid"); + assert!(!u.get_isalpha(0)); // "a" has no second token -> 0 + assert!(u.get_isalpha(1) && u.get_islower(1)); // "b 3" -> 0x3 + assert_eq!(u.id_to_unichar(0), Some("a")); + } + #[test] fn errors_are_typed() { assert_eq!(UniCharSet::load_from_str(""), Err(UniCharSetError::Empty)); From 796f69bdcb31b00f0f5563a1b749b11b9c3db390 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 20 Jun 2026 07:57:58 +0000 Subject: [PATCH 2/3] contract(unicharset): transcode UNICHARSET script table, byte-parity 112/112 Add get_script + get_script_table_size + script_from_script_id + script_of + dump_script to UniCharSet, backed by an interned scripts: Vec and a parallel script_ids: Vec. The first leaf to transcode an interning side-table: add_script (unicharset.cpp:1063, insertion-order dedup) with null_script "NULL" seeded at sid 0 by the first unichar_insert (unicharset.cpp:680, so null_sid_ == 0 always); real scripts intern from 1 in first-seen id order. get_script mirrors unicharset.h:681 (out-of-range -> 0). The script name is the token immediately after the optional bbox/stats CSV (token[3] if token[2] has a comma, else token[2]) -- covers every column tier present on real eng data (id 0 is tier-5 no-CSV, others tier-1 CSV) without the full multi-tier parser, which the remaining columns will need (next leaf). Byte-identical 112/112 vs tesseract's own get_script on real eng.lstm-unicharset via the self-validating oracle (script mode; table = [NULL, Common, Latin] confirmed empirically before writing the Rust). Fourth leaf of PROBE-OGAR-ADAPTER-UNICHARSET. - +4 unicharset tests (19 total); clippy -D warnings + fmt clean (scoped -p) - examples/unicharset_dump.rs gains a `script` mode (reproduces the parity diff) - board: EPIPHANIES E-CPP-PARITY-4; LATEST_STATE branch-work + D-UNICHARSET-SCRIPT Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_016b33swuXE23hKtqxsHu9p1 --- .claude/board/EPIPHANIES.md | 16 ++ .claude/board/LATEST_STATE.md | 4 + .../examples/unicharset_dump.rs | 18 +- crates/lance-graph-contract/src/unicharset.rs | 156 ++++++++++++++++++ 4 files changed, 185 insertions(+), 9 deletions(-) diff --git a/.claude/board/EPIPHANIES.md b/.claude/board/EPIPHANIES.md index 579812e8..fd85dfda 100644 --- a/.claude/board/EPIPHANIES.md +++ b/.claude/board/EPIPHANIES.md @@ -73,6 +73,22 @@ The 5+3 council's headline was "five crates linked into one binary with ZERO run **Next edges:** Grid→NodeRow over a REAL Spain fixture (E1 acceptance gate); the kanban loop (D2: `LanceVersionScheduler` → `KanbanMove` → SoA write → Lance commit). Cross-ref: PR #555 INTEGRATION_PLAN D1; battle-test plan probes A1/B-series; STATUS_BOARD symbiont-golden-image-harness; `crates/symbiont/src/bridge.rs`. +## 2026-06-20 — E-CPP-PARITY-4 — the UNICHARSET script table (`get_script` + the interned `add_script` table) is byte-identical to libtesseract; the fourth leaf through PROBE-OGAR-ADAPTER-UNICHARSET, and the first to transcode an INTERNING side-table + +**Status:** FINDING (in-env, real trained data). `lance_graph_contract::unicharset::UniCharSet::get_script` dumps the `eng.lstm-unicharset` per-id script ids **byte-identical to tesseract's own `UNICHARSET::get_script`, 112/112** — verified by the same self-validating oracle (bijection half = proven 112/112 layout check, then `./uniprops_oracle … script` diffs 0). The fourth proven adapter surface (after id↔unichar, properties, and the UNICHAR codec). + +**What's new vs the property leaf — an interned side-table, not a per-id field.** `get_script(id)` returns an *index* into a script-name table built by `add_script` (insertion-order dedup, `unicharset.cpp:1063`). The non-obvious transcode detail the oracle confirmed empirically (rather than my deriving it wrong): `null_script` ("NULL") is **always** interned at id 0 — not because the file lists it first, but because `unichar_insert` calls `set_script(id, null_script)` for EVERY entry (`unicharset.cpp:680`) *before* the loop sets the real script, so the very first insert seeds "NULL"→0 and `null_sid_ == 0` is an invariant (`unicharset.cpp:949-950`). Real scripts then intern from 1 in first-seen id order. On real eng: table = `["NULL","Common","Latin"]`, and the space char (id 0, the `NULL`→space token) carries script "Common" (sid 1), not the null script. The Rust mirrors this by interning `NULL_SCRIPT` first every iteration (idempotent) then the parsed script. + +**Column-tier handling without the full multi-tier parser.** The script token's position varies (`unicharset.cpp:833-868` has 5 fallback tiers): on real eng, id 0 is tier-5 (`script other_case`, no bbox CSV) while most ids are tier-1/2 (full bbox+stats CSV). But script is ALWAYS the token immediately after the optional CSV, so `script = token[3] if token[2] contains ',' else token[2]` extracts it for every tier present — proven by the 112/112 diff across the mixed-tier real file. The remaining columns (other_case/mirror/direction/bbox/stats) DO need the full tier parser; that's the next leaf, deliberately scoped out here. + +**Methodology note (let the oracle teach the interning).** Rather than hand-derive the script numbering and risk an off-by-one on the null-seed, I extended the oracle with a `script` mode that prints the table to stderr (`sid 0=NULL, 1=Common, 2=Latin`) and the per-id ids to stdout, READ the truth, then wrote the Rust to match. This is the measure-before-assert discipline applied to a transcode: the falsifier defined the spec. + +**Pattern holds (E-CPP-KEYSTONE-1).** Still "repetition of a validated pattern": +1 accessor family (now with a backing intern table) + one `diff`, no new architecture, no Core gap (the table rides the content tier; the adapter holds no state). +4 contract tests (19 unicharset total); consumed by `tesseract-core::CharSet::{get_script,script_of}` (+1 boundary test, 5/5). Reproducible via the committed `examples/unicharset_dump.rs script`. + +Cross-ref: `E-CPP-PARITY-1/2/3` (the prior three leaves), `E-CPP-KEYSTONE-1` (end-to-end), `.claude/knowledge/core-first-transcode-doctrine.md`. Branch `claude/happy-hamilton-0azlw4`, lance-graph + tesseract-rs. + +--- + ## 2026-06-20 — E-CPP-PARITY-3 — the UNICHARSET property accessors (`get_is{alpha,lower,upper,digit,punctuation}`) are byte-identical to libtesseract; the third leaf through PROBE-OGAR-ADAPTER-UNICHARSET **Status:** FINDING (in-env, real trained data). `lance_graph_contract::unicharset::UniCharSet`'s property family dumps the `eng.lstm-unicharset` per-id category bits **byte-identical to tesseract's own `UNICHARSET::get_is*` accessors, 112/112** — every id's `(isalpha, islower, isupper, isdigit, ispunctuation)` tuple. This is the THIRD adapter surface proven after `UniCharSet` id↔unichar (E-CPP-PARITY-1) and the `UNICHAR` codec (E-CPP-PARITY-2): the core-first transcode now covers a content-store tier, a pure codec, AND a per-entry property decode. diff --git a/.claude/board/LATEST_STATE.md b/.claude/board/LATEST_STATE.md index 4279d1d5..c6defd60 100644 --- a/.claude/board/LATEST_STATE.md +++ b/.claude/board/LATEST_STATE.md @@ -16,6 +16,8 @@ --- +> **2026-06-20 — branch work (`claude/happy-hamilton-0azlw4`)** — **UNICHARSET script table transcoded + byte-parity proven (E-CPP-PARITY-4), the fourth leaf — first to transcode an INTERNING side-table.** `UniCharSet` now parses the per-line script name (the token after the optional bbox/stats CSV), interns it via an `add_script`-equivalent (`unicharset.cpp:1063`, insertion-order dedup) into `scripts: Vec` with `null_script` ("NULL") seeded at sid 0 (the `unichar_insert` set_script, `unicharset.cpp:680`; so `null_sid_ == 0` always), and stores `script_ids: Vec`. Exposes `get_script` / `get_script_table_size` / `script_from_script_id` / `script_of` / `dump_script`, mirroring `unicharset.h:681` (out-of-range → `null_sid_` 0). **Byte-identical 112/112** on real `eng.lstm-unicharset` vs tesseract's own `get_script` (same self-validating oracle, `script` mode; oracle table = `["NULL","Common","Latin"]` confirmed empirically before writing the Rust). Mixed-tier safe (eng id 0 is tier-5 no-CSV, others tier-1 CSV). Additive, zero-dep; +4 contract tests (19 unicharset total), clippy `-D warnings` + fmt clean; reproducible via `examples/unicharset_dump.rs script`. Consumed by `tesseract-core::CharSet::{get_script,script_of}` (+1 boundary test, 5/5). No Core gap. EPIPHANIES `E-CPP-PARITY-4`. Next leaf: the full column tier-parser (unlocks other_case/mirror/direction/bbox). +> > **2026-06-20 — branch work (`claude/happy-hamilton-0azlw4`)** — **UNICHARSET property accessors transcoded + byte-parity proven (E-CPP-PARITY-3), the third leaf through PROBE-OGAR-ADAPTER-UNICHARSET.** `lance_graph_contract::unicharset::UniCharSet` now parses the per-line hex property bitmask (`unicharset.cpp:824`) into a `props: Vec` and exposes `get_is{alpha,lower,upper,digit,punctuation}` + `get_isngram` + `dump_properties()`, mirroring the C++ inline accessors (`unicharset.h:497+`; out-of-range id → `false`, `INVALID_UNICHAR_ID` semantics). **Byte-identical 112/112** on real `eng.lstm-unicharset` vs tesseract's own `get_is*` via a **self-validating** oracle: the same harness dumps the id↔unichar bijection (proven 112/112 reference, E-CPP-PARITY-1) AND the properties — the bijection half diffing 0 proves the 5.5.0-header/5.3.4-lib layout is sound, making the property diff (also 0) trustworthy despite the version skew. Additive, zero-dep; +5 contract tests (15 unicharset total), clippy `-D warnings` + fmt clean. Consumed by `tesseract-core` as `CharSet::get_is*` (+1 consumer-boundary test, 4/4 green). Incidental: rustfmt-1.9.0 normalized two pre-existing test-assert wraps in `class_view.rs` (whitespace-only). No Core gap, no adapter state (per `E-CPP-KEYSTONE-1` "repetition of a validated pattern"). EPIPHANIES `E-CPP-PARITY-3`. > > **2026-06-19 — IN PR (branch `claude/edge-distance-basin-node-epiphany`)** — **basin-IS-a-node: the substrate is a virtual tree of MailboxSoAs, navigated by pure key arithmetic.** New `graph::mailbox_scan::{members, memberof, BasinOf}` — one-to-many (`members` = direct children one HHTL tier down) / many-to-one (`memberof` = parent via `NiblePath::parent`, returns `BasinOf::Local(row)` or `BasinOf::Route(NiblePath)` when the parent lives in another shard — the HHTL prefix IS the route key, **no coarse-fingerprint table**; `None` only at the top tier). Realizes `E-BASIN-IS-A-NODE` with **no ownership restructure** — the tree is the radix trie of the keys, the SoA stays flat, the zero-copy/Lance-tombstone invariant is untouched; all navigation is **zero value decode** (F2-guarded). 16/16 mailbox_scan tests, clippy clean. **Probe (perturbation-sim `basin_placement_learning.rs`): field-perturbation placement learns the basin tree — green, mean tree-hop 1.00 vs 4.13 random (75.8 % tighter)**, promoting the one CONJECTURE in `E-BASIN-IS-A-NODE` to measured FINDING [G]. **Three epiphanies this arc:** `E-BASIN-IS-A-NODE` (basin=node; distance=hop=`node_distance(PrefixDepth)`; 4-ary fan-out = Morton tile pyramid = perturbation-learnable field), `E-FAMILY-NODE-IS-META-AWARENESS` (the parent node IS the coarse Walsh band of its subtree — meta-awareness is structural, not a column), `E-GUID-SELF-ROUTES-THE-BASIN-TREE` (HHTL-tier truncation of the GUID = every ancestor's route key; the GUID self-routes). **Capstone:** one 512 B key, read five ways — representation / ontology / compute (Morton pyramid) / learning / meta-awareness — four of the five are key-resident zero-decode. Builds on #544/#545/#548 (mailbox_scan facets) + `E-COARSE-QUANTIZER-IS-SCALE-FREE-ROUTER`. @@ -108,6 +110,8 @@ > **2026-06-18 — ADDED (D-DO-ARM-1, the OGAR DO arm)**: `lance_graph_contract::action::{ActionState, StateGuard, ActionDef, ClassActions, actions_for, effective_actions, ActionInvocation}` — the Perdurant DO arm completing the OGAR IR (the action-axis sibling of `codegen_manifest`'s `MethodSig`/THINK). Both the 4-agent `sale_order` AR→DO probe (runtime-archaeologist) AND the merged cross-repo PR survey (ruff/OGAR/lance-graph/openproject/tesseract) agreed this was the ONE missing wire: the THINK arm (`classid → ClassView`, `has_function → MethodSig`) is converged + merged; the DO-arm `ActionInvocation`/`ActionDef` type was ABSENT. **`ActionDef`** (static, `const`-constructible, all `&'static`/`Copy`): `predicate` (= harvested `has_function` method), `object_class` (classid), `exec` (`ExecTarget` incl `SurrealQl`), `guard` (`StateGuard` = KausalSpec field==value), `required_role` (RBAC), `overrides` (OGAR `classid→ClassView` inheritance). **`ClassActions`+`actions_for`** (zero-fallback) mirror `ClassMethods`/`methods_for`. **`effective_actions(parent, child)`** = OGAR inheritance on the action axis (child overrides parent by predicate). **`ActionInvocation`** (dynamic, `Copy`): lifecycle `ActionState{Pending→Committed|Failed|Cancelled}` (sticky terminals), S2.5 `cycle` stamp, idempotency/trace keys, HLC `emitted_at_millis`. **`ActionInvocation::commit(def, actor, impact, now)`** is the gated egress — RBAC FIRST (`auth::ActorContext` must hold `required_role` or be admin → else `Failed`), THEN MUL impact (`mul::GateDecision`: `Flow→Committed`+stamped, `Hold→`Pending/escalate, `Block→Cancelled`). This IS "commit to the external consumer (odoo/openproject/woa/tesseract) after the cycle decides sound." Dispatched via `UnifiedStep`/`ExecTarget`, NOT a per-crate endpoint. Additive, zero-dep. +5 tests green. Consumer reference: `docs/OGAR_CONSUMER_API.md`. Branch `claude/soa-write-deinterlace-inc2`. +> **2026-06-20 — ADDED (D-UNICHARSET-SCRIPT, the script-table leaf)**: `lance_graph_contract::unicharset::UniCharSet` gained `get_script(id) -> i32` / `get_script_table_size()` / `script_from_script_id(sid) -> Option<&str>` / `script_of(id) -> Option<&str>` / `dump_script()`, backed by new `script_ids: Vec` + an interned `scripts: Vec`. The first leaf to transcode an **interning side-table** (`add_script`, `unicharset.cpp:1063`): `null_script` "NULL" seeded at sid 0 (the `unichar_insert` set_script, `unicharset.cpp:680` → `null_sid_ == 0`), real scripts intern from 1 in id order. Script name = token after the optional bbox/stats CSV (mixed-tier safe). Out-of-range → `null_sid_` 0 (`unicharset.h:681`). **Byte-identical 112/112** vs tesseract's own `get_script` on real `eng.lstm-unicharset` (self-validating oracle `script` mode; table `["NULL","Common","Latin"]`). Additive, zero-dep, behaviour-preserving on the bijection. +4 tests (19 unicharset total). Consumed by `tesseract-core::CharSet::{get_script,script_of}`. EPIPHANIES `E-CPP-PARITY-4`; fourth leaf of `PROBE-OGAR-ADAPTER-UNICHARSET`. Branch `claude/happy-hamilton-0azlw4`. + > **2026-06-20 — ADDED (D-UNICHARSET-PROPS, the property-accessor leaf)**: `lance_graph_contract::unicharset::UniCharSet` gained the character-category surface `get_isalpha` / `get_islower` / `get_isupper` / `get_isdigit` / `get_ispunctuation` / `get_isngram` + `dump_properties()`, backed by a new `props: Vec` parsed from the per-line hex bitmask (`unicharset.cpp:824`; masked to `ISALPHA=0x1 ISLOWER=0x2 ISUPPER=0x4 ISDIGIT=0x8 ISPUNCTUATION=0x10`). Accessors mirror the C++ inline guard (`unicharset.h:497+`): out-of-range id → `false` (`INVALID_UNICHAR_ID`); `get_isngram` is always-false on the plain-table load path (`unicharset.cpp:893`). **Byte-identical 112/112** vs tesseract's own `get_is*` on real `eng.lstm-unicharset` (self-validating oracle: bijection half cross-checks the 5.5.0-header/5.3.4-lib layout, then the property half diffs 0). Additive, zero-dep, behaviour-preserving on the existing id↔unichar bijection (lenient default-0 for a missing/!hex token). +5 tests (15 unicharset total). Consumed by `tesseract-core::CharSet::get_is*`. EPIPHANIES `E-CPP-PARITY-3`; the third leaf of `PROBE-OGAR-ADAPTER-UNICHARSET` (after D-UNICHARSET-1 + D-UNICHAR-1). Branch `claude/happy-hamilton-0azlw4`. > **2026-06-18 — ADDED (D-UNICHARSET-KEYSTONE, classid → ClassView → adapter wiring)**: `lance_graph_contract::unicharset_adapter::{UniCharSetStore, UniCharCall, UniCharOut, DispatchError, invoke_unicharset}` — steps 2–3 of `PROBE-OGAR-ADAPTER-UNICHARSET`, the keystone composing the proven `UniCharSet` adapter through the OGAR Core's three movable parts. `invoke_unicharset(registry, store, classid, call)`: (1) **ClassView composition gate** — `codegen_manifest::methods_for(registry, classid)` must list the call's method (the harvested `has_function` manifest), else `MethodNotComposed` (zero-fallback: an unconfigured classid composes nothing); (2) **content-store tier** — `UniCharSetStore::unicharset(classid)`, a consumer-provided trait (dependency-inverted like `ClassView`/`PlannerContract`; the adapter holds NO state — `I-VSA-IDENTITIES`); (3) **adapter leaf** — routes to `UniCharSet::{id_to_unichar, unichar_to_id}`. DO-in (`UniCharCall`) / DO-out (`UniCharOut`, zero-copy borrow). **Byte-parity inherited** from `UniCharSet` (112/112); the keystone proves the dispatch path is faithful (the `NULL`→space edge survives it), the gate works, and there is **no Core gap** (the doctrine's iron guard holds — the variable-length bijection rides the content tier cleanly). NOT routed through the heavy `OrchestrationBridge` (cross-subsystem router); this is the adapter-invocation primitive a `UnifiedStep` calls. Additive, zero-dep. +5 tests; clippy `--all-targets -D warnings` + fmt clean. Completes the core-first doctrine END-TO-END for the unicharset leaf (`E-CPP-KEYSTONE-1`). diff --git a/crates/lance-graph-contract/examples/unicharset_dump.rs b/crates/lance-graph-contract/examples/unicharset_dump.rs index ef875400..846e08f9 100644 --- a/crates/lance-graph-contract/examples/unicharset_dump.rs +++ b/crates/lance-graph-contract/examples/unicharset_dump.rs @@ -1,6 +1,6 @@ -//! Dump a `.unicharset`'s id→unichar table (default) or its per-id property bits -//! (`properties` mode) — the Rust side of the byte-parity probe -//! `PROBE-OGAR-ADAPTER-UNICHARSET`. +//! Dump a `.unicharset`'s id→unichar table (default), its per-id property bits +//! (`properties` mode), or its per-id script ids (`script` mode) — the Rust side +//! of the byte-parity probe `PROBE-OGAR-ADAPTER-UNICHARSET`. //! //! ```sh //! # on a box with libtesseract + libleptonica installed: @@ -32,16 +32,16 @@ use lance_graph_contract::unicharset::UniCharSet; fn main() -> ExitCode { let Some(path) = std::env::args().nth(1) else { - eprintln!("usage: unicharset_dump [properties]"); + eprintln!("usage: unicharset_dump [properties|script]"); return ExitCode::FAILURE; }; - let properties = std::env::args().nth(2).as_deref() == Some("properties"); + let mode = std::env::args().nth(2).unwrap_or_default(); match UniCharSet::load_from_file(Path::new(&path)) { Ok(unicharset) => { - if properties { - print!("{}", unicharset.dump_properties()); - } else { - print!("{}", unicharset.dump()); + match mode.as_str() { + "properties" => print!("{}", unicharset.dump_properties()), + "script" => print!("{}", unicharset.dump_script()), + _ => print!("{}", unicharset.dump()), } ExitCode::SUCCESS } diff --git a/crates/lance-graph-contract/src/unicharset.rs b/crates/lance-graph-contract/src/unicharset.rs index 469a5fa7..af9bdd61 100644 --- a/crates/lance-graph-contract/src/unicharset.rs +++ b/crates/lance-graph-contract/src/unicharset.rs @@ -42,6 +42,18 @@ //! the plain-table loader (`unicharset.cpp:893` always `set_isngram(id, false)`) //! so [`UniCharSet::get_isngram`] is always `false` for a file-loaded set. //! [`UniCharSet::dump_properties`] is the byte-parity surface for these bits. +//! +//! # Script leaf +//! +//! Each entry carries a script id — an index into an interned table built by +//! `add_script` (tesseract `unicharset.cpp:1063`, insertion-order dedup). +//! `null_script` ("NULL", `unicharset.cpp:82`) is seeded at id 0 by the first +//! `unichar_insert` (`unicharset.cpp:680`, before the real script is set), so +//! `null_sid_ == 0` always and real scripts follow in first-seen id order. +//! [`UniCharSet::get_script`] mirrors the C++ accessor (`unicharset.h:681`): +//! out-of-range id → `null_sid_` (0). The script name parsed per line is the +//! token after the optional bbox/stats CSV; [`UniCharSet::dump_script`] is the +//! byte-parity surface for the script ids. use std::collections::HashMap; use std::path::Path; @@ -57,6 +69,14 @@ pub struct UniCharSet { /// parallel to `reverse`. Only the low 5 bits are meaningful; see the /// `*_MASK` consts. props: Vec, + /// id → script id (index into `scripts`), parallel to `reverse`. The C++ + /// `unichars[id].properties.script_id` (tesseract `unicharset.cpp:894`). + script_ids: Vec, + /// The interned script-name table (`add_script`, tesseract + /// `unicharset.cpp:1063`). Index 0 is always `null_script` ("NULL"), seeded + /// by the first `unichar_insert` (`unicharset.cpp:680`); real scripts follow + /// in first-seen id order. `script_ids` indexes into this. + scripts: Vec, } /// `isalpha` property bit (tesseract `unicharset.cpp:41`). @@ -73,6 +93,23 @@ const ISPUNCTUATION_MASK: u8 = 0x10; const PROPERTY_BITS: u8 = ISALPHA_MASK | ISLOWER_MASK | ISUPPER_MASK | ISDIGIT_MASK | ISPUNCTUATION_MASK; +/// The null script name (tesseract `unicharset.cpp:82`), always interned at +/// script id 0 (`null_sid_ == 0`, `unicharset.cpp:949-950`). +const NULL_SCRIPT: &str = "NULL"; +/// The null script id — what `get_script` returns for an out-of-range id +/// (the C++ `null_sid_`, `unicharset.h:683`). +const NULL_SID: i32 = 0; + +/// Intern `name` into `scripts` (insertion-order dedup), returning its index — +/// the transcription of `UNICHARSET::add_script` (tesseract `unicharset.cpp:1063`). +fn intern_script(scripts: &mut Vec, name: &str) -> i32 { + let idx = scripts.iter().position(|s| s == name).unwrap_or_else(|| { + scripts.push(name.to_string()); + scripts.len() - 1 + }); + i32::try_from(idx).unwrap_or(NULL_SID) +} + impl UniCharSet { /// Parse a `.unicharset` from its text contents. See the module docs for the /// format. Properties columns after the leading unichar token are ignored. @@ -95,6 +132,8 @@ impl UniCharSet { let mut reverse = Vec::with_capacity(count); let mut lookup = HashMap::with_capacity(count); let mut props = Vec::with_capacity(count); + let mut script_ids = Vec::with_capacity(count); + let mut scripts: Vec = Vec::new(); for line in lines.take(count) { // The unichar is the first whitespace-delimited token; the id is the // entry's position. A unichar repeated in the file keeps its FIRST @@ -126,6 +165,20 @@ impl UniCharSet { // <= 0x1F); the fallback keeps the path total without `unwrap`. let prop_byte = u8::try_from(properties & u32::from(PROPERTY_BITS)).unwrap_or(0); props.push(prop_byte); + // Script (tesseract `unicharset.cpp:894`). `unichar_insert` seeds + // `null_script` at sid 0 (`unicharset.cpp:680`) BEFORE the real + // script is set, so intern "NULL" first every iteration (idempotent + // after the first) then the parsed script. The script token is the + // one right after the optional bbox/stats CSV: token[2] when it has + // no comma, else token[3] — covering every column tier present on + // real `eng` data; an absent token defaults to `null_script`. + intern_script(&mut scripts, NULL_SCRIPT); + let script_name = match tokens.next() { + Some(t) if t.contains(',') => tokens.next().unwrap_or(NULL_SCRIPT), + Some(t) => t, + None => NULL_SCRIPT, + }; + script_ids.push(intern_script(&mut scripts, script_name)); } if reverse.len() != count { @@ -138,6 +191,8 @@ impl UniCharSet { reverse, lookup, props, + script_ids, + scripts, }) } @@ -224,6 +279,44 @@ impl UniCharSet { false } + /// The script id of `id` — an index into the interned script table. Mirrors + /// `UNICHARSET::get_script` (tesseract `unicharset.h:681`): an out-of-range + /// id (the `INVALID_UNICHAR_ID` sentinel) returns the null script id + /// (`null_sid_ == 0`), else the stored `script_id`. + #[must_use] + pub fn get_script(&self, id: u32) -> i32 { + self.script_ids + .get(id as usize) + .copied() + .unwrap_or(NULL_SID) + } + + /// The number of interned scripts (always ≥ 1: index 0 is `null_script`). + /// Mirrors `UNICHARSET::get_script_table_size`. + #[must_use] + pub fn get_script_table_size(&self) -> usize { + self.scripts.len() + } + + /// The script name for a script id, or `None` if out of range. Mirrors + /// `UNICHARSET::get_script_from_script_id` (which returns `null_script` for + /// an out-of-range id; here the caller chooses the fallback via `Option`). + #[must_use] + pub fn script_from_script_id(&self, script_id: i32) -> Option<&str> { + usize::try_from(script_id) + .ok() + .and_then(|i| self.scripts.get(i)) + .map(String::as_str) + } + + /// The script name of unichar `id` (the composition of [`Self::get_script`] + /// and [`Self::script_from_script_id`]) — the OCR-facing "what script is this + /// character" query. Out-of-range → `Some("NULL")` (the null script). + #[must_use] + pub fn script_of(&self, id: u32) -> Option<&str> { + self.script_from_script_id(self.get_script(id)) + } + /// Render the id→properties table as /// `"\t \n"` lines /// (each flag `0`/`1`) — the exact shape the C++ property oracle prints, so @@ -248,6 +341,21 @@ impl UniCharSet { out } + /// Render the id→script-id table as `"\t\n"` lines — the exact + /// shape the C++ `get_script` oracle prints, so the byte-parity diff is + /// `diff oracle_script.tsv rust_script.tsv`. + #[must_use] + pub fn dump_script(&self) -> String { + let mut out = String::new(); + for id in 0..self.reverse.len() as u32 { + out.push_str(&id.to_string()); + out.push('\t'); + out.push_str(&self.get_script(id).to_string()); + out.push('\n'); + } + out + } + /// Render the id→unichar table as `"\t\n"` lines — the exact /// shape the C++ oracle harness prints, so a byte-parity diff is /// `diff oracle_dump.tsv rust_dump.tsv`. @@ -422,6 +530,54 @@ x 1 0,255,0,255,0,0,0,0,0,0 Latin 0 0 0 x assert_eq!(u.id_to_unichar(0), Some("a")); } + /// Mirrors the real `eng.lstm-unicharset` mixed-tier shape: id 0 has no bbox + /// CSV (script is token[2]); ids 1-3 do (script is token[3]). The interned + /// table is `["NULL", "Common", "Latin"]` — exactly the oracle's table. + const SCRIPT_SAMPLE: &str = "\ +4 +NULL 0 Common 0 +a 3 0,255,0,255,0,0,0,0,0,0 Latin 1 0 1 a +b 3 0,255,0,255,0,0,0,0,0,0 Latin 1 0 1 b +5 8 0,255,0,255,0,0,0,0,0,0 Common 2 0 2 5 +"; + + #[test] + fn scripts_intern_in_id_order_with_null_seeded_at_zero() { + let u = UniCharSet::load_from_str(SCRIPT_SAMPLE).expect("valid"); + // null_script seeded at 0; real scripts follow in first-seen id order. + assert_eq!(u.get_script_table_size(), 3); + assert_eq!(u.script_from_script_id(0), Some("NULL")); + assert_eq!(u.script_from_script_id(1), Some("Common")); // id 0's script + assert_eq!(u.script_from_script_id(2), Some("Latin")); // id 1's script + assert_eq!(u.script_from_script_id(3), None); // out of range + // per-id script ids (matches the libtesseract oracle on real eng shape) + assert_eq!(u.get_script(0), 1); // Common + assert_eq!(u.get_script(1), 2); // Latin (after the bbox CSV) + assert_eq!(u.get_script(2), 2); // Latin (deduped) + assert_eq!(u.get_script(3), 1); // Common (deduped) + } + + #[test] + fn script_out_of_range_is_null_sid() { + let u = UniCharSet::load_from_str(SCRIPT_SAMPLE).expect("valid"); + assert_eq!(u.get_script(99), 0); // null_sid_ + assert_eq!(u.script_of(99), Some("NULL")); // resolves the null script + } + + #[test] + fn script_of_resolves_name() { + let u = UniCharSet::load_from_str(SCRIPT_SAMPLE).expect("valid"); + assert_eq!(u.script_of(0), Some("Common")); + assert_eq!(u.script_of(1), Some("Latin")); + assert_eq!(u.script_of(3), Some("Common")); + } + + #[test] + fn dump_script_matches_oracle_shape() { + let u = UniCharSet::load_from_str(SCRIPT_SAMPLE).expect("valid"); + assert_eq!(u.dump_script(), "0\t1\n1\t2\n2\t2\n3\t1\n"); + } + #[test] fn errors_are_typed() { assert_eq!(UniCharSet::load_from_str(""), Err(UniCharSetError::Empty)); From c8cb95c6e4cd731f072f41598cfcc26e60243f89 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 20 Jun 2026 08:04:31 +0000 Subject: [PATCH 3/3] contract(unicharset): transcode UNICHARSET other_case (case pair), byte-parity 112/112 Add get_other_case + dump_other_case to UniCharSet, backed by other_cases: Vec. other_case is the case-paired unichar id ('C' -> 'c'), parsed as the token right after the script and clamped at load exactly as unicharset.cpp:901: a parsed value not less than size -- and the absent-column default (unicharset.cpp:813, = size) -- folds to the id itself. get_other_case mirrors unicharset.h:703 (out-of-range id -> INVALID_UNICHAR_ID -1). Byte-identical 112/112 vs tesseract's own get_other_case on real eng.lstm-unicharset (self-validating oracle, other_case mode; 60/112 self, 52 real pairs e.g. C->c). This is the last field cleanly reachable by token-offset; direction/mirror/bbox sit after other_case and need the full multi-tier column parser (next leaf). Fifth leaf of PROBE-OGAR-ADAPTER-UNICHARSET. - +4 unicharset tests (23 total); clippy -D warnings + fmt clean (scoped -p) - examples/unicharset_dump.rs gains an `other_case` mode (reproduces the diff) - board: EPIPHANIES E-CPP-PARITY-5; LATEST_STATE branch-work + D-UNICHARSET-OTHERCASE Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_016b33swuXE23hKtqxsHu9p1 --- .claude/board/EPIPHANIES.md | 14 +++ .claude/board/LATEST_STATE.md | 4 + .../examples/unicharset_dump.rs | 8 +- crates/lance-graph-contract/src/unicharset.rs | 95 +++++++++++++++++++ 4 files changed, 118 insertions(+), 3 deletions(-) diff --git a/.claude/board/EPIPHANIES.md b/.claude/board/EPIPHANIES.md index fd85dfda..402452f0 100644 --- a/.claude/board/EPIPHANIES.md +++ b/.claude/board/EPIPHANIES.md @@ -73,6 +73,20 @@ The 5+3 council's headline was "five crates linked into one binary with ZERO run **Next edges:** Grid→NodeRow over a REAL Spain fixture (E1 acceptance gate); the kanban loop (D2: `LanceVersionScheduler` → `KanbanMove` → SoA write → Lance commit). Cross-ref: PR #555 INTEGRATION_PLAN D1; battle-test plan probes A1/B-series; STATUS_BOARD symbiont-golden-image-harness; `crates/symbiont/src/bridge.rs`. +## 2026-06-20 — E-CPP-PARITY-5 — the UNICHARSET `other_case` (case-pair id, size-clamped) is byte-identical to libtesseract; the fifth leaf through PROBE-OGAR-ADAPTER-UNICHARSET + +**Status:** FINDING (in-env, real trained data). `lance_graph_contract::unicharset::UniCharSet::get_other_case` dumps the `eng.lstm-unicharset` per-id case-pair ids **byte-identical to tesseract's own `UNICHARSET::get_other_case`, 112/112** (same self-validating oracle, `other_case` mode). The fifth proven adapter surface; the second per-id integer field (after script id) and the first with a load-time **clamp**. + +**The transcode detail — a size-clamp with a size-valued default.** `other_case` is the case-paired unichar's id (`'C'` → the id of `'c'`), or the id itself when unpaired. Load clamps it (`unicharset.cpp:901`): `set_other_case(id, (other_case < size) ? other_case : id)` — a parsed value `>= size`, AND the absent-column default (`unicharset.cpp:813`, initialized to `size`), both fold to the id itself. So "no pair" and "out-of-range pair" collapse to self. `get_other_case` (`unicharset.h:703`) returns `INVALID_UNICHAR_ID` (-1) for an out-of-range id — distinct from the `null_sid_`-style 0 of `get_script`. The oracle confirmed 60/112 ids are self, 52 carry a real pair (e.g. id 3 `C`→87 `c`, id 4 `H`→97 `h`). + +**Column position without the full tier-parser (again).** `other_case` is the token immediately after the script token in every tier that carries it — so `other_case = token[4] if token[2] has a comma else token[3]` (one token past the script extractor), proven across eng's mixed tiers (tier-5 id 0 with no CSV: `NULL 0 Common 0` → other_case `0`; tier-1 ids with CSV). The remaining columns (direction/mirror/bbox/stats) sit *after* other_case and genuinely need the multi-tier fallback to place — that is the next, larger leaf. This is the last field cleanly reachable by token-offset alone. + +**Pattern holds (E-CPP-KEYSTONE-1).** +1 accessor + clamp + one `diff`, no new architecture, no Core gap. +4 contract tests (23 unicharset total); consumed by `tesseract-core::CharSet::get_other_case` (+1 boundary test, 6/6). Reproducible via the committed `examples/unicharset_dump.rs other_case`. Measure-before-assert held: the oracle's `other_case` dump defined the spec (incl. the 60-self / 52-pair split) before the Rust was written. + +Cross-ref: `E-CPP-PARITY-1/2/3/4` (the prior four leaves), `E-CPP-KEYSTONE-1`, `.claude/knowledge/core-first-transcode-doctrine.md`. Branch `claude/happy-hamilton-0azlw4`, lance-graph + tesseract-rs. + +--- + ## 2026-06-20 — E-CPP-PARITY-4 — the UNICHARSET script table (`get_script` + the interned `add_script` table) is byte-identical to libtesseract; the fourth leaf through PROBE-OGAR-ADAPTER-UNICHARSET, and the first to transcode an INTERNING side-table **Status:** FINDING (in-env, real trained data). `lance_graph_contract::unicharset::UniCharSet::get_script` dumps the `eng.lstm-unicharset` per-id script ids **byte-identical to tesseract's own `UNICHARSET::get_script`, 112/112** — verified by the same self-validating oracle (bijection half = proven 112/112 layout check, then `./uniprops_oracle … script` diffs 0). The fourth proven adapter surface (after id↔unichar, properties, and the UNICHAR codec). diff --git a/.claude/board/LATEST_STATE.md b/.claude/board/LATEST_STATE.md index c6defd60..988707a0 100644 --- a/.claude/board/LATEST_STATE.md +++ b/.claude/board/LATEST_STATE.md @@ -16,6 +16,8 @@ --- +> **2026-06-20 — branch work (`claude/happy-hamilton-0azlw4`)** — **UNICHARSET `other_case` transcoded + byte-parity proven (E-CPP-PARITY-5), the fifth leaf.** `UniCharSet` now parses the case-pair id (the token right after the script) into `other_cases: Vec`, applying the load-time clamp (`unicharset.cpp:901`: a value `>= size`, incl. the absent default, folds to the id itself). Exposes `get_other_case` + `dump_other_case`, mirroring `unicharset.h:703` (out-of-range id → `INVALID_UNICHAR_ID` -1). **Byte-identical 112/112** on real `eng.lstm-unicharset` vs tesseract's own `get_other_case` (self-validating oracle, `other_case` mode; 60/112 self, 52 real pairs, e.g. `C`→`c`). Last field cleanly reachable by token-offset; direction/mirror/bbox need the multi-tier parser (next, larger leaf). Additive, zero-dep; +4 contract tests (23 unicharset total), clippy `-D warnings` + fmt clean; reproducible via `examples/unicharset_dump.rs other_case`. Consumed by `tesseract-core::CharSet::get_other_case` (+1 boundary test, 6/6). No Core gap. EPIPHANIES `E-CPP-PARITY-5`. +> > **2026-06-20 — branch work (`claude/happy-hamilton-0azlw4`)** — **UNICHARSET script table transcoded + byte-parity proven (E-CPP-PARITY-4), the fourth leaf — first to transcode an INTERNING side-table.** `UniCharSet` now parses the per-line script name (the token after the optional bbox/stats CSV), interns it via an `add_script`-equivalent (`unicharset.cpp:1063`, insertion-order dedup) into `scripts: Vec` with `null_script` ("NULL") seeded at sid 0 (the `unichar_insert` set_script, `unicharset.cpp:680`; so `null_sid_ == 0` always), and stores `script_ids: Vec`. Exposes `get_script` / `get_script_table_size` / `script_from_script_id` / `script_of` / `dump_script`, mirroring `unicharset.h:681` (out-of-range → `null_sid_` 0). **Byte-identical 112/112** on real `eng.lstm-unicharset` vs tesseract's own `get_script` (same self-validating oracle, `script` mode; oracle table = `["NULL","Common","Latin"]` confirmed empirically before writing the Rust). Mixed-tier safe (eng id 0 is tier-5 no-CSV, others tier-1 CSV). Additive, zero-dep; +4 contract tests (19 unicharset total), clippy `-D warnings` + fmt clean; reproducible via `examples/unicharset_dump.rs script`. Consumed by `tesseract-core::CharSet::{get_script,script_of}` (+1 boundary test, 5/5). No Core gap. EPIPHANIES `E-CPP-PARITY-4`. Next leaf: the full column tier-parser (unlocks other_case/mirror/direction/bbox). > > **2026-06-20 — branch work (`claude/happy-hamilton-0azlw4`)** — **UNICHARSET property accessors transcoded + byte-parity proven (E-CPP-PARITY-3), the third leaf through PROBE-OGAR-ADAPTER-UNICHARSET.** `lance_graph_contract::unicharset::UniCharSet` now parses the per-line hex property bitmask (`unicharset.cpp:824`) into a `props: Vec` and exposes `get_is{alpha,lower,upper,digit,punctuation}` + `get_isngram` + `dump_properties()`, mirroring the C++ inline accessors (`unicharset.h:497+`; out-of-range id → `false`, `INVALID_UNICHAR_ID` semantics). **Byte-identical 112/112** on real `eng.lstm-unicharset` vs tesseract's own `get_is*` via a **self-validating** oracle: the same harness dumps the id↔unichar bijection (proven 112/112 reference, E-CPP-PARITY-1) AND the properties — the bijection half diffing 0 proves the 5.5.0-header/5.3.4-lib layout is sound, making the property diff (also 0) trustworthy despite the version skew. Additive, zero-dep; +5 contract tests (15 unicharset total), clippy `-D warnings` + fmt clean. Consumed by `tesseract-core` as `CharSet::get_is*` (+1 consumer-boundary test, 4/4 green). Incidental: rustfmt-1.9.0 normalized two pre-existing test-assert wraps in `class_view.rs` (whitespace-only). No Core gap, no adapter state (per `E-CPP-KEYSTONE-1` "repetition of a validated pattern"). EPIPHANIES `E-CPP-PARITY-3`. @@ -110,6 +112,8 @@ > **2026-06-18 — ADDED (D-DO-ARM-1, the OGAR DO arm)**: `lance_graph_contract::action::{ActionState, StateGuard, ActionDef, ClassActions, actions_for, effective_actions, ActionInvocation}` — the Perdurant DO arm completing the OGAR IR (the action-axis sibling of `codegen_manifest`'s `MethodSig`/THINK). Both the 4-agent `sale_order` AR→DO probe (runtime-archaeologist) AND the merged cross-repo PR survey (ruff/OGAR/lance-graph/openproject/tesseract) agreed this was the ONE missing wire: the THINK arm (`classid → ClassView`, `has_function → MethodSig`) is converged + merged; the DO-arm `ActionInvocation`/`ActionDef` type was ABSENT. **`ActionDef`** (static, `const`-constructible, all `&'static`/`Copy`): `predicate` (= harvested `has_function` method), `object_class` (classid), `exec` (`ExecTarget` incl `SurrealQl`), `guard` (`StateGuard` = KausalSpec field==value), `required_role` (RBAC), `overrides` (OGAR `classid→ClassView` inheritance). **`ClassActions`+`actions_for`** (zero-fallback) mirror `ClassMethods`/`methods_for`. **`effective_actions(parent, child)`** = OGAR inheritance on the action axis (child overrides parent by predicate). **`ActionInvocation`** (dynamic, `Copy`): lifecycle `ActionState{Pending→Committed|Failed|Cancelled}` (sticky terminals), S2.5 `cycle` stamp, idempotency/trace keys, HLC `emitted_at_millis`. **`ActionInvocation::commit(def, actor, impact, now)`** is the gated egress — RBAC FIRST (`auth::ActorContext` must hold `required_role` or be admin → else `Failed`), THEN MUL impact (`mul::GateDecision`: `Flow→Committed`+stamped, `Hold→`Pending/escalate, `Block→Cancelled`). This IS "commit to the external consumer (odoo/openproject/woa/tesseract) after the cycle decides sound." Dispatched via `UnifiedStep`/`ExecTarget`, NOT a per-crate endpoint. Additive, zero-dep. +5 tests green. Consumer reference: `docs/OGAR_CONSUMER_API.md`. Branch `claude/soa-write-deinterlace-inc2`. +> **2026-06-20 — ADDED (D-UNICHARSET-OTHERCASE, the case-pair leaf)**: `lance_graph_contract::unicharset::UniCharSet` gained `get_other_case(id) -> i32` + `dump_other_case()`, backed by `other_cases: Vec`. The case-paired unichar id (`'C'`→`'c'`), parsed as the token after the script and clamped at load (`unicharset.cpp:901`: a value `>= size`, and the absent default = size, fold to the id itself). Out-of-range id → `INVALID_UNICHAR_ID` -1 (`unicharset.h:703`). **Byte-identical 112/112** vs tesseract's own `get_other_case` on real `eng.lstm-unicharset` (self-validating oracle `other_case` mode; 60 self / 52 pairs). Additive, zero-dep. +4 tests (23 unicharset total). Consumed by `tesseract-core::CharSet::get_other_case`. EPIPHANIES `E-CPP-PARITY-5`; fifth leaf of `PROBE-OGAR-ADAPTER-UNICHARSET`; the last field reachable by token-offset (direction/mirror/bbox need the multi-tier parser). Branch `claude/happy-hamilton-0azlw4`. + > **2026-06-20 — ADDED (D-UNICHARSET-SCRIPT, the script-table leaf)**: `lance_graph_contract::unicharset::UniCharSet` gained `get_script(id) -> i32` / `get_script_table_size()` / `script_from_script_id(sid) -> Option<&str>` / `script_of(id) -> Option<&str>` / `dump_script()`, backed by new `script_ids: Vec` + an interned `scripts: Vec`. The first leaf to transcode an **interning side-table** (`add_script`, `unicharset.cpp:1063`): `null_script` "NULL" seeded at sid 0 (the `unichar_insert` set_script, `unicharset.cpp:680` → `null_sid_ == 0`), real scripts intern from 1 in id order. Script name = token after the optional bbox/stats CSV (mixed-tier safe). Out-of-range → `null_sid_` 0 (`unicharset.h:681`). **Byte-identical 112/112** vs tesseract's own `get_script` on real `eng.lstm-unicharset` (self-validating oracle `script` mode; table `["NULL","Common","Latin"]`). Additive, zero-dep, behaviour-preserving on the bijection. +4 tests (19 unicharset total). Consumed by `tesseract-core::CharSet::{get_script,script_of}`. EPIPHANIES `E-CPP-PARITY-4`; fourth leaf of `PROBE-OGAR-ADAPTER-UNICHARSET`. Branch `claude/happy-hamilton-0azlw4`. > **2026-06-20 — ADDED (D-UNICHARSET-PROPS, the property-accessor leaf)**: `lance_graph_contract::unicharset::UniCharSet` gained the character-category surface `get_isalpha` / `get_islower` / `get_isupper` / `get_isdigit` / `get_ispunctuation` / `get_isngram` + `dump_properties()`, backed by a new `props: Vec` parsed from the per-line hex bitmask (`unicharset.cpp:824`; masked to `ISALPHA=0x1 ISLOWER=0x2 ISUPPER=0x4 ISDIGIT=0x8 ISPUNCTUATION=0x10`). Accessors mirror the C++ inline guard (`unicharset.h:497+`): out-of-range id → `false` (`INVALID_UNICHAR_ID`); `get_isngram` is always-false on the plain-table load path (`unicharset.cpp:893`). **Byte-identical 112/112** vs tesseract's own `get_is*` on real `eng.lstm-unicharset` (self-validating oracle: bijection half cross-checks the 5.5.0-header/5.3.4-lib layout, then the property half diffs 0). Additive, zero-dep, behaviour-preserving on the existing id↔unichar bijection (lenient default-0 for a missing/!hex token). +5 tests (15 unicharset total). Consumed by `tesseract-core::CharSet::get_is*`. EPIPHANIES `E-CPP-PARITY-3`; the third leaf of `PROBE-OGAR-ADAPTER-UNICHARSET` (after D-UNICHARSET-1 + D-UNICHAR-1). Branch `claude/happy-hamilton-0azlw4`. diff --git a/crates/lance-graph-contract/examples/unicharset_dump.rs b/crates/lance-graph-contract/examples/unicharset_dump.rs index 846e08f9..04bd4d67 100644 --- a/crates/lance-graph-contract/examples/unicharset_dump.rs +++ b/crates/lance-graph-contract/examples/unicharset_dump.rs @@ -1,6 +1,7 @@ //! Dump a `.unicharset`'s id→unichar table (default), its per-id property bits -//! (`properties` mode), or its per-id script ids (`script` mode) — the Rust side -//! of the byte-parity probe `PROBE-OGAR-ADAPTER-UNICHARSET`. +//! (`properties` mode), its per-id script ids (`script` mode), or its per-id +//! case-pair ids (`other_case` mode) — the Rust side of the byte-parity probe +//! `PROBE-OGAR-ADAPTER-UNICHARSET`. //! //! ```sh //! # on a box with libtesseract + libleptonica installed: @@ -32,7 +33,7 @@ use lance_graph_contract::unicharset::UniCharSet; fn main() -> ExitCode { let Some(path) = std::env::args().nth(1) else { - eprintln!("usage: unicharset_dump [properties|script]"); + eprintln!("usage: unicharset_dump [properties|script|other_case]"); return ExitCode::FAILURE; }; let mode = std::env::args().nth(2).unwrap_or_default(); @@ -41,6 +42,7 @@ fn main() -> ExitCode { match mode.as_str() { "properties" => print!("{}", unicharset.dump_properties()), "script" => print!("{}", unicharset.dump_script()), + "other_case" => print!("{}", unicharset.dump_other_case()), _ => print!("{}", unicharset.dump()), } ExitCode::SUCCESS diff --git a/crates/lance-graph-contract/src/unicharset.rs b/crates/lance-graph-contract/src/unicharset.rs index af9bdd61..28932392 100644 --- a/crates/lance-graph-contract/src/unicharset.rs +++ b/crates/lance-graph-contract/src/unicharset.rs @@ -54,6 +54,16 @@ //! out-of-range id → `null_sid_` (0). The script name parsed per line is the //! token after the optional bbox/stats CSV; [`UniCharSet::dump_script`] is the //! byte-parity surface for the script ids. +//! +//! # Other-case leaf +//! +//! Each entry carries `other_case` — the id of its case-paired unichar (`'C'` → +//! `'c'`), or itself when there is no pair. It is the token immediately after +//! the script, clamped at load exactly as `unicharset.cpp:901`: a parsed value +//! not less than `size` (and the absent default, `unicharset.cpp:813`, = `size`) +//! folds to the id itself. [`UniCharSet::get_other_case`] mirrors the C++ +//! accessor (`unicharset.h:703`): out-of-range id → `INVALID_UNICHAR_ID` (-1). +//! [`UniCharSet::dump_other_case`] is the byte-parity surface. use std::collections::HashMap; use std::path::Path; @@ -77,6 +87,10 @@ pub struct UniCharSet { /// by the first `unichar_insert` (`unicharset.cpp:680`); real scripts follow /// in first-seen id order. `script_ids` indexes into this. scripts: Vec, + /// id → the case-paired unichar id (`other_case`), parallel to `reverse`. + /// Clamped at load: a parsed value `>= size` (incl. the default) becomes the + /// id itself (tesseract `unicharset.cpp:901`). + other_cases: Vec, } /// `isalpha` property bit (tesseract `unicharset.cpp:41`). @@ -99,6 +113,9 @@ const NULL_SCRIPT: &str = "NULL"; /// The null script id — what `get_script` returns for an out-of-range id /// (the C++ `null_sid_`, `unicharset.h:683`). const NULL_SID: i32 = 0; +/// The C++ `INVALID_UNICHAR_ID` sentinel — what id-returning accessors yield for +/// an out-of-range id (tesseract `unichar.h`; e.g. `get_other_case`). +const INVALID_UNICHAR_ID: i32 = -1; /// Intern `name` into `scripts` (insertion-order dedup), returning its index — /// the transcription of `UNICHARSET::add_script` (tesseract `unicharset.cpp:1063`). @@ -134,6 +151,8 @@ impl UniCharSet { let mut props = Vec::with_capacity(count); let mut script_ids = Vec::with_capacity(count); let mut scripts: Vec = Vec::new(); + let mut other_cases = Vec::with_capacity(count); + let count_i32 = i32::try_from(count).unwrap_or(i32::MAX); for line in lines.take(count) { // The unichar is the first whitespace-delimited token; the id is the // entry's position. A unichar repeated in the file keeps its FIRST @@ -179,6 +198,16 @@ impl UniCharSet { None => NULL_SCRIPT, }; script_ids.push(intern_script(&mut scripts, script_name)); + // other_case: the token right after the script (present in every + // column tier that carries it). Clamped exactly as + // `unicharset.cpp:901`: a parsed value not less than `size` — and the + // absent default (`unicharset.cpp:813`, = size) — folds to the id. + let id_i32 = i32::try_from(id).unwrap_or(INVALID_UNICHAR_ID); + let oc = tokens + .next() + .and_then(|t| t.parse::().ok()) + .unwrap_or(count_i32); + other_cases.push(if oc < count_i32 { oc } else { id_i32 }); } if reverse.len() != count { @@ -193,6 +222,7 @@ impl UniCharSet { props, script_ids, scripts, + other_cases, }) } @@ -317,6 +347,19 @@ impl UniCharSet { self.script_from_script_id(self.get_script(id)) } + /// The case-paired unichar id of `id` (`other_case`) — e.g. `'C'` → the id of + /// `'c'`, or the id itself when there is no pair. Mirrors + /// `UNICHARSET::get_other_case` (tesseract `unicharset.h:703`): an + /// out-of-range id (the `INVALID_UNICHAR_ID` sentinel) returns + /// `INVALID_UNICHAR_ID` (-1). + #[must_use] + pub fn get_other_case(&self, id: u32) -> i32 { + self.other_cases + .get(id as usize) + .copied() + .unwrap_or(INVALID_UNICHAR_ID) + } + /// Render the id→properties table as /// `"\t \n"` lines /// (each flag `0`/`1`) — the exact shape the C++ property oracle prints, so @@ -356,6 +399,21 @@ impl UniCharSet { out } + /// Render the id→other-case table as `"\t\n"` lines — the + /// exact shape the C++ `get_other_case` oracle prints, so the byte-parity + /// diff is `diff oracle_other_case.tsv rust_other_case.tsv`. + #[must_use] + pub fn dump_other_case(&self) -> String { + let mut out = String::new(); + for id in 0..self.reverse.len() as u32 { + out.push_str(&id.to_string()); + out.push('\t'); + out.push_str(&self.get_other_case(id).to_string()); + out.push('\n'); + } + out + } + /// Render the id→unichar table as `"\t\n"` lines — the exact /// shape the C++ oracle harness prints, so a byte-parity diff is /// `diff oracle_dump.tsv rust_dump.tsv`. @@ -578,6 +636,43 @@ b 3 0,255,0,255,0,0,0,0,0,0 Latin 1 0 1 b assert_eq!(u.dump_script(), "0\t1\n1\t2\n2\t2\n3\t1\n"); } + /// id 0 `C` pairs with id 1 `c` and vice-versa; id 2 `.` has an out-of-range + /// other_case (99 ≥ size 3) which clamps to itself (`unicharset.cpp:901`). + const OTHER_CASE_SAMPLE: &str = "\ +3 +C 5 0,255,0,255,0,0,0,0,0,0 Latin 1 0 0 C +c 3 0,255,0,255,0,0,0,0,0,0 Latin 0 0 1 c +. 10 0,255,0,255,0,0,0,0,0,0 Common 99 0 2 . +"; + + #[test] + fn other_case_decodes_and_clamps() { + let u = UniCharSet::load_from_str(OTHER_CASE_SAMPLE).expect("valid"); + assert_eq!(u.get_other_case(0), 1); // C -> c + assert_eq!(u.get_other_case(1), 0); // c -> C + assert_eq!(u.get_other_case(2), 2); // 99 >= size -> clamped to self + } + + #[test] + fn other_case_absent_defaults_to_self() { + // A line with no other_case token (script only): default = size, clamps + // to the id itself. + let u = UniCharSet::load_from_str("1\nx 1 Latin\n").expect("valid"); + assert_eq!(u.get_other_case(0), 0); + } + + #[test] + fn other_case_out_of_range_id_is_invalid() { + let u = UniCharSet::load_from_str(OTHER_CASE_SAMPLE).expect("valid"); + assert_eq!(u.get_other_case(99), -1); // INVALID_UNICHAR_ID + } + + #[test] + fn dump_other_case_matches_oracle_shape() { + let u = UniCharSet::load_from_str(OTHER_CASE_SAMPLE).expect("valid"); + assert_eq!(u.dump_other_case(), "0\t1\n1\t0\n2\t2\n"); + } + #[test] fn errors_are_typed() { assert_eq!(UniCharSet::load_from_str(""), Err(UniCharSetError::Empty));