diff --git a/research/packed-newton-membership/PERFORMANCE.md b/research/packed-newton-membership/PERFORMANCE.md new file mode 100644 index 0000000000..abacdf0744 --- /dev/null +++ b/research/packed-newton-membership/PERFORMANCE.md @@ -0,0 +1,73 @@ +# Local timing evidence + +This contribution was produced by Nick Petalas. Apache-2.0. + +The complete public membership call takes 45.35% less median time on one +accepted rank-nine fixture. This includes type and size validation, packing, +the full transform, unpacking and every divisibility check. It includes the +new 8192-bit component-limit check. It does not time only packed subtraction. + +The fixture is the positive encoding used by PR5374's existing benchmark at +coefficient width 1025 bits. Encoded input components reach 1034 bits. The +exact input SHA256 is +`151857dd471120344f0f7c0e8c5ab6542b8f7d815e5cbb56348eacd77f3c770b`. +The same arrays and positive endpoint are used on both sides. No result is +cached. Input encoding is outside the API timer on both sides. + +Five fresh processes per side ran 1000 calls each, alternating order between +trials. All 10,000 complete calls accepted, with zero errors. Separate +processes ran the complete inherited scientific replay five times per side; +all ten returned the same complete result hash and zero errors. + +| Timed work | Reference median, range in seconds | Packed median, range in seconds | +| --- | --- | --- | +| 1000 complete rank-nine API calls | 0.290105, 0.271077 to 0.366769 | 0.158548, 0.147782 to 0.162996 | +| Complete source scientific replay call | 0.907932, 0.901498 to 1.010443 | 0.859743, 0.850917 to 0.915044 | +| Whole fresh process for that replay | 0.948728, 0.938971 to 1.104224 | 0.899336, 0.890138 to 0.956943 | + +The API ranges do not overlap. Complete-replay timing is inconclusive: +its observed median falls by 5.31%, but the ranges overlap and the gap is +smaller than the observed variation. No end-to-end replay speedup is claimed. +Whole-process timing includes process startup, plan initialization, source +authentication, imports, output comparison and printing. None of these +measurements predicts a collector, tape implementation or supplier speedup. +The change is opt-in; the shared default command is unchanged. + +Environment: CPython 3.12.15, macOS 27.0.1, arm64, standard library only. +The shared host has 18 logical CPUs; the pre-measurement load averages were +2.99, 2.77 and 2.08. Other work was not stopped. Alternation and the reported +ranges expose the observed scheduling variation. Both implementations use +their ordinary interpreter configuration with no tuning knobs changed. + +## Limiter and end-to-end share + +An untimed cProfile run of the new API spent 0.2905 of +0.3898 profiled seconds cumulatively in validation and +packing, while the packed transform itself spent 0.0109 seconds. +Packing and validation now dominate. The old API's nested validation +generators and scalar transform account for most of its profile. The +improvement includes fusing validated input reduction into packing as well +as batching the arithmetic. + +A separate untimed full-replay profile spent 0.5868 of 3.1432 +profiled seconds cumulatively in the bounded membership predicate, about +18.7%. The unchanged Gaussian source validation alone spent +1.8896 seconds cumulatively. Profile durations are diagnostic and are +excluded from the timing table. The unchanged work limits the end-to-end gain. +The complete replay preserves its independent decoder, +subset checks, rejection tests and all 96 literal collector prefixes. + +## Repeat + +From the repository root: + +```sh +/Users/nick/git/integer-mult-bounds/.venv/bin/python3.12 -I -W error -B research/packed-newton-membership/benchmark.py +/Users/nick/git/integer-mult-bounds/.venv/bin/python3.12 -I -W error -B research/packed-newton-membership/benchmark.py --sample reference --profile +/Users/nick/git/integer-mult-bounds/.venv/bin/python3.12 -I -W error -B research/packed-newton-membership/benchmark.py --sample packed --profile +``` + +The benchmark prints every unrounded sample, per-sample error and work counts, +fixture/result hashes, environment, and hashes of its four controlling files. +It writes no files. Timing is not a registered correctness acceptance gate. +The ordinary registered replay reruns the exact preservation checks. diff --git a/research/packed-newton-membership/PROOF.md b/research/packed-newton-membership/PROOF.md new file mode 100644 index 0000000000..82fc2d3e99 --- /dev/null +++ b/research/packed-newton-membership/PROOF.md @@ -0,0 +1,109 @@ +# Guarded packed differences preserve Newton membership + +This contribution was produced by Nick Petalas. Apache-2.0. + +Claim type: scoped algorithm-preservation proof. Ali Romman's PR5374 supplies +bounded residues and Clear1nsight's PR3382 supplies the Gaussian Newton lattice. +Those mathematical predicates are premises, not new results here. + +## Contract + +Let `N=2^r`, `0<=r<=9`, and let `values` be a list of N Gaussian-integer +pairs with absolute component bit lengths at most 8192. Set `b=ceil(r/2)` +and `M=2^b`. The reference computes the Boolean difference coefficients, +modulo M in each component, and returns the first mask failing its existing +power-of-1+i divisibility test. The new implementation returns exactly the +same tuple `(mask, real_residue, imaginary_residue, b)`, or None. It preserves +the input. This includes negative components, r=0 and every value in the +stated domain. It rejects malformed inputs and inputs outside that domain. +It does not claim the reference's unbounded-rank API outside this domain. + +## Lane arithmetic + +Store real and imaginary residues as separate base-256 digits, in that +order at each mask. Each digit is in `[0,M-1]`, and `M<=32`. At a stage +for address bit k, select the upper masks, whose k-th bit is one. Let U +be the packed word restricted to their low b bits. Let L be the low b +bits of the corresponding lower masks, shifted by `16*2^k` bits into +the upper positions. Let G have digit 128 in each selected component lane +and digit zero elsewhere. Then one integer subtraction + + D = (U | G) - L + +has the intended independent digit values `128+u-l` at selected lanes and +zero elsewhere. Every selected digit is between 97 and 159, so no digit +borrows from another or carries into another. Unselected digits subtract +zero from zero. This is an equality in base 256, proved by independent +nonnegative digit differences. Masking D to the low b bits in selected lanes +therefore yields `(u-l) mod M`. The discarded guard has zero residue modulo +M. The lower lanes are copied unchanged from the previous word. + +`lower = upper >> shift` in the public plan selects precisely the lower +partner lanes. The implementation masks the word before shifting, which +keeps even intermediate positive integers within 16N bits. The shift never +moves a real digit into an imaginary digit or a different Boolean pair. + +This guard is necessary for the simple packed-subtraction scheme. +Without it, two destination bytes 0 minus two source bytes 1 give low-five-bit +residues `0x1e1f`, rather than the desired `0x1f1f`. With guards the masked +result is `((0x8080-0x0101)&0x1f1f)=0x1f1f`. + +## Stage induction and certificate + +The invariant after each processed address bit is equality, in every real +and imaginary component, with the reference partial Boolean difference +transform modulo M. Packing establishes it before the first stage. The lane +lemma proves one simultaneous stage, so induction proves every final +coefficient. It holds for all inputs in the domain, rather than only for the +finite fixtures. + +The unpacking uses the same increasing mask order and the same existing +divisibility predicate as PR5374. Equal complete coefficient arrays therefore +produce the same first failing mask, both identical residues and the same b. +For r=0 there are no stages, residues are zero modulo 1 and every valid +Gaussian input is accepted. All work uses newly allocated local objects; +none writes into the input list or its tuples. + +## Exact work and bounds + +The reference uses rN modular component subtractions. The packed transform +uses r packed integer subtractions, plus its masks, shifts, packing, +unpacking, validation and existing divisibility tests. At rank nine these +counts are 4608 and 9. These are different operation alphabets; their ratio +is not a bit-complexity or runtime speedup. The packed word has at most +16N bits, hence 8192 bits at rank nine, including unused and guard bits. +The private residue byte array has 2N bytes, hence 1024 bytes at rank nine. +The plan is immutable and depends only on rank and mask positions. No +coefficient, verdict, receipt or scientific result is cached. + +Packing and unpacking each scan N pairs; each transform stage operates on +O(N) machine bits. Input type checks and the 8192-bit limit run on every +call. Precomputed public plans for ranks zero through nine are built once +at module import. This cost is outside steady API timing and inside the +fresh-process end-to-end timing. Python object overhead, integer allocation +and physical tape costs are additional. No fixed-allocation or paid physical +scan bound follows from the bit counts. + +## Verification and limits + +The executable checker exhausts guarded subtraction on all residue digit +pairs for b=1 through 5, checks every small binary Gaussian array through +rank two and all rank-three real arrays over residues 0 through 3, compares +complete first-failure tuples and preserves inputs. A separate subset-sum +definition checks complete transformed arrays, including signed rank-nine +fixtures. Boundary and source-drift controls reject. + +An opt-in adapter runs the complete original PR5374 scientific replay with +only its member/obstruction globals replaced by this implementation. Its +independent source decoder, subset oracle, literal collector, source hashes, +rejection controls and recorded output comparison remain executed. The old +receipt's `rank9_modular_subtractions=4608` describes the original reference +algorithm, not instrumentation of the substituted transform. New operation +counts are reported separately. This is a preservation check for the new +implementation, not a scientific re-review of the predecessor result. + +No lattice theorem, sharp residue modulus, decoder, producer admission, +physical caller, native frame operation, owned erasure, precision/tape, +compiler or multiplication theorem is proved here. Selected kappa and all +five framework assumptions retain their status. No proof assistant or +independent human review is claimed. diff --git a/research/packed-newton-membership/README.md b/research/packed-newton-membership/README.md new file mode 100644 index 0000000000..e212d79358 --- /dev/null +++ b/research/packed-newton-membership/README.md @@ -0,0 +1,124 @@ +# Packed Gaussian Newton membership + +This contribution was produced by Nick Petalas. Apache-2.0. + +Claim type: scoped algorithm-preservation proof with finite corroboration +and local performance evidence. At rank nine, a guarded packed transform +uses nine integer subtractions in place of 4608 component subtractions. +These are different arithmetic operations; the count ratio is not a speedup. + +For lists of Gaussian integer pairs on ranks zero through nine, with absolute +component bit lengths at most 8192, [packed.py](packed.py) returns exactly +PR5374's membership decision and first failing coefficient tuple. It preserves +the input, including arbitrary signed components. [PROOF.md](PROOF.md) proves +borrow isolation and stage induction for every input in this domain. + +This builds on Ali Romman's [PR5374](https://github.com/CrocSwap/integer-mult-bounds/pull/5374) +bounded-residue algorithm and Clear1nsight's +[PR3382](https://github.com/CrocSwap/integer-mult-bounds/pull/3382) Gaussian +Newton lattice. Their lattice, sharp modulus and collector results remain +credited premises. No predecessor file or credit line changes. + +## Evidence + +The new checker passes 1364 guarded digit-pair cases, 65,812 small arrays, +160 signed and component-boundary fixtures, complete transformed-array +comparisons with independent subset sums, malformed-input/source controls, +and a fault injected into the production transform with its guards removed. +All checks use explicit exceptions and remain active under optimized Python. + +The opt-in adapter also executes the complete inherited scientific replay +with only the membership functions replaced. Its output is identical, +including all 96 literal collector prefixes. Source decoding, independent +oracles and failure controls remain executed. This is preservation evidence +for the new algorithm, not an audit deliverable for PR5374. + +[PERFORMANCE.md](PERFORMANCE.md) records five alternating fresh-process +trials per side. It reports the complete public API separately from the +complete scientific replay and whole-process costs. All timed work runs, +input/results match, and no errors occur. The benchmark prints raw samples +and controlling-file hashes; no timing threshold controls acceptance. + +## Reproduce + +From the repository root, using standard-library CPython 3.12: + +```sh +/Users/nick/git/integer-mult-bounds/.venv/bin/python3.12 -B scripts/research_package.py --package research/packed-newton-membership +/Users/nick/git/integer-mult-bounds/.venv/bin/python3.12 -I -W error -B research/packed-newton-membership/verify.py +/Users/nick/git/integer-mult-bounds/.venv/bin/python3.12 -I -W error -O -B research/packed-newton-membership/verify.py +/Users/nick/git/integer-mult-bounds/.venv/bin/python3.12 -B scripts/check_research_commit.py --package research/packed-newton-membership --check-source-history +``` + +The verifier prints `PASS_PACKED_NEWTON_MEMBERSHIP`, all exact counts and +hashes of its package files. It writes nothing. The generic launcher runs +both normal and optimized checks. For a different checkout, substitute its +CPython 3.12 interpreter path. Timing commands are in PERFORMANCE.md. + +[SOURCES.json](SOURCES.json) pins canonical base +`7c49fe32c55de02c2c873655c804b16e4c3fef0a` and every imported predecessor +input. [TASK.json](TASK.json) records the change from the broad replay-runtime +card to this exact public validation algorithm. The +[claim](https://github.com/CrocSwap/integer-mult-bounds/discussions/1166#discussioncomment-19380573) +states the original bounded target. No source or result cache is added. + +## Limits and credit + +The API has a declared finite rank and bit-width domain. It does not replace +the unbounded reference API outside that domain and emits no decoded wide +coefficients. No default caller is patched. Runtime numbers describe one +host and one fixture. A faster logical predicate does not price input +movement, physical validation, allocation, erasure or restoration. + +There is no new kappa, selected-frontier change, supplier admission, native +implementation, uniform compiler or complete multiplication theorem. All +five selected framework assumptions remain. No selected cold replay, Lean +or Docker run is claimed. Independent human and formal proof-kernel review +are unclaimed. + +Credit Ali Romman for PR5374's algorithm and replay; Chris R / Clear1nsight +for the PR3382 lattice and collector; ywz65, Arham Tahir, Tekalign / PR3025, +Ross Richardson and the retained word, wrapper and period-domain lineage. +Guarded packed arithmetic and Boolean difference transforms are standard +techniques. The new work is their concrete implementation, proof and +source-bound integration for this validator. No literature-priority claim +for the general techniques is made. + +## Open next questions + +1. Can a physical membership-only consumer benefit after input acquisition, + movement and restoration are fully charged? First name and price one + actual consumer. No physical scan or supplier contract is supplied here. +2. Can the unchanged input-validation and independent-decoder work in a + complete replay be reduced without losing diagnostics? First profile one + source-bound replacement. This contribution stops at the membership step. + +## Implementation bounds + +The new call graph has no recursion. Public input is capped at 512 pairs and +8192 bits per component. Plan construction has ten ranks, at most nine stages +and 1024 bytes per mask. A call has at most nine transform stages, 512 packing +iterations and 512 certificate tests. The independent subset oracle has at +most 512 squared iterations per fixture. The finite checker does not retain +its census; source replay has its existing finite bounds. New functions are +at most 60 lines. Exceptions enforce meaningful contracts under `-O`. + +Power of Ten rule 3 deviation: CPython allocates integers, byte arrays and +library objects while running. Fixed-capacity allocation is unavailable in +this runtime. The finite input limits bound live data; packed integers use at +most 8192 positive bits, and each source read is capped at 32 MiB and closed +with a context manager. The full source replay retains its predecessor's +managed-memory contract. A fixed-allocation port would be a separate task +if this predicate becomes a physical backend component. + +Rules 8 and 9 deviation: the preservation adapter imports a fixed finite set +of authenticated Python modules, and the benchmark selects between two fixed +functions. All targets are explicit and source-pinned. Module/path bindings +are restored after import; no shared predecessor module is modified. +This boundary is retained to execute the actual source replay. Source-drift +controls and the full replay check it. A static native integration would +need its own loader and memory contract. + +Warning-strict normal and optimized Python checks, development mode and +source compilation provide diagnostics. No external static analyzer is +available in the selected environment; no repository-wide settings change. diff --git a/research/packed-newton-membership/SOURCES.json b/research/packed-newton-membership/SOURCES.json new file mode 100644 index 0000000000..718ed30016 --- /dev/null +++ b/research/packed-newton-membership/SOURCES.json @@ -0,0 +1,78 @@ +{ + "schema_version": 1, + "base_commit": "7c49fe32c55de02c2c873655c804b16e4c3fef0a", + "files": [ + { + "path": "certificates/selected-result.json", + "usage": "provenance", + "git_blob": "d59b5035e7b4cbaa9d688a1d32dd0d6eb79c466a", + "sha256": "9b8d0eea95b3bf0dfcb7938a732a6439b87a2685dec2a5ab9e525b3b34debbfa" + }, + { + "path": "docs/research/agent-task-catalog.json", + "usage": "provenance", + "git_blob": "3404fbdd5e7529a0674eef26114b5071e4586cec", + "sha256": "ead6cdc9d0af6d58e8a49744d8ec8c7c15e13d2ba93c276c434bcf9b51b93a81" + }, + { + "path": "research/newton-bounded-residue-validator/PROOF.md", + "usage": "input", + "git_blob": "268bd6bbe330f3ef68835a0052183e8628f3e612", + "sha256": "7c2365a13736ff7cde04c053416fd678fe7ea7283fb18eb3f2b9ba3b5d6a6802" + }, + { + "path": "research/newton-bounded-residue-validator/membership.py", + "usage": "input", + "git_blob": "885bf6359b9dbaa149ac854098699c81697b929c", + "sha256": "21696a2d9763d067c3230cc84a2d9ec97b2afc38af53f5461463c592b1344d89" + }, + { + "path": "research/newton-bounded-residue-validator/verify.py", + "usage": "input", + "git_blob": "f8980cd28bef428a192276138a0a4a78017dd099", + "sha256": "9b8b9425ed944031e3848dad69c1504f0cb9eca097ed787b0f10bf172c5e7b22" + }, + { + "path": "research/newton-bounded-residue-validator/SOURCES.json", + "usage": "input", + "git_blob": "99cc6556ed4f674c448cddf26ec1d507e759c4bb", + "sha256": "4dca07519be2f2afc30e08447942eec9159bc80b59000f68850f132fcc96add2" + }, + { + "path": "research/newton-bounded-residue-validator/RESULT.json", + "usage": "input", + "git_blob": "82eb1c1b5ddd03a5be3ee3e201179ad518cf38fa", + "sha256": "42fd06e7658880be52d0bed8aadbc4cb73a113358a30eacb18a3bafc81e85784" + }, + { + "path": "research/p104-clifford-exactness/model.py", + "usage": "input", + "git_blob": "4dd24a2627db8239d062afc7888a39bf5cd0f478", + "sha256": "74c83367887dd885137b222155c86389fe0be9f1293a6f0e5fdb51d71f435fdd" + }, + { + "path": "research/p104-clifford-exactness/PROOF.md", + "usage": "input", + "git_blob": "10b3202c788b5b03188eed91181441dfd793ff7f", + "sha256": "1ca8c0e7e78f0f39de6cff7accc68ccd3901bfb50759bf7a2e2896bcf63bf081" + }, + { + "path": "research/p104-clifford-exactness/README.md", + "usage": "input", + "git_blob": "581edcfcbcc31cf78c02f9873c14094f9b37e1bf", + "sha256": "7374b6b360fa6755f68ee513467de1c0436a48a22a9584a3c27687e73a16a7db" + }, + { + "path": "research/p104-rounded-collector/inputs/WORD.json", + "usage": "input", + "git_blob": "93acaabc0a523789ef2f1b06f6eeeefed5551397", + "sha256": "cec1fb3fbd0067b500ad5119a62ba60685be718e769374d7ec8abdd2beaa7e50" + }, + { + "path": "research/p104-rounded-collector/inputs/NATIVE.json.gz", + "usage": "input", + "git_blob": "04f2877aa5444948b7ae37efd267745af82cb34d", + "sha256": "ab91c78835c3e10cf3b2cb8a5d4e0b7cfc3df1ae5a811298d27842f2193e1e3d" + } + ] +} diff --git a/research/packed-newton-membership/TASK.json b/research/packed-newton-membership/TASK.json new file mode 100644 index 0000000000..d62eb585fe --- /dev/null +++ b/research/packed-newton-membership/TASK.json @@ -0,0 +1,83 @@ +{ + "schema_version": 1, + "task_id": "replay-runtime", + "priority_revision": 8, + "base_commit": "7c49fe32c55de02c2c873655c804b16e4c3fef0a", + "original_card": { + "id": "replay-runtime", + "question": "Can one measured replay bottleneck be sped up without dropping coverage?", + "starting_points": [ + "scripts/run_isolated_tests.py", + "scripts/verify_replay_coverage.py", + "certificates/selected-result.json" + ], + "methods": [ + "Profile one isolated replay and optimize the measured bottleneck.", + "Design source-bound reuse with explicit dependency invalidation and negative controls." + ], + "acceptance_gate": "Report before/after wall time on the same fixture, retained checks and invalidation behavior.", + "first_step": { + "command": [ + "python3", + "-B", + "scripts/research_selected_baseline.py" + ], + "expected": "PASS_RESEARCH_BASELINE_BINDINGS_ONLY", + "inspect": "Resolve the current selected review, separate complex/ordinary BIT/admitted BIT/outer budgets, and compare the card against live handoffs before starting. Historical ceilings require exact source/model equivalence.", + "scope": "Source-binding orientation only; not a scientific replay or all-size proof. Materialize selected inputs if this checkout is sparse." + }, + "first_experiment": "Measure one unchanged fixture, preserve check coverage, and test one optimization with invalidation controls.", + "bottleneck_link": "Faster trustworthy replays increase review throughput and let mechanism candidates be checked sooner.", + "allowed_variations": [ + "Profile one stage or design one source-bound cache boundary." + ], + "scope_change_triggers": [ + "Skipping coverage or comparing different fixtures to claim speedup." + ], + "overlap_terms": [ + "replay runtime", + "cache invalidation", + "isolated tests" + ] + }, + "scope_status": "changed", + "actual_scope": "Guarded-byte Boolean differences for bounded Gaussian Newton membership, with exact first-failure preservation on ranks 0 through 9 and absolute component bit lengths at most 8192.", + "scope_change_reason": "The general replay-runtime card is narrowed to a new arithmetic algorithm for one public validator, with full source replay preservation and separately measured complete API and replay costs. No scientific result or decoded coefficient is cached.", + "bottleneck_link": "This is verifier engineering for the P104 entrance check, not a change to the binding complex supplier or its physical validation bill.", + "budget_minutes": 60, + "stop_rule": "Stop after the first verified packed implementation. Do not extend the rank domain, native collector, physical caller or kappa calculation.", + "outcome": "completed", + "evidence": "PROOF.md, packed.py, verify.py, PERFORMANCE.md and complete reference replay through an opt-in adapter.", + "author": "Nick Petalas", + "ai_assistance": "Independent human and formal proof-kernel review are unclaimed.", + "open_questions": [ + { + "question": "Can a physical membership-only consumer profit from guarded-lane batching after all input, movement and restoration costs?", + "why_it_matters": "The logical API improvement does not change the paid supplier bill by itself.", + "first_step": "Name an actual consumer whose only output is membership, and price input acquisition and the complete validation scan.", + "blocked_by": "No physical caller, allocation or tape implementation is supplied." + }, + { + "question": "Can source replay input validation and independent decoder work be reduced while retaining their complete diagnostics?", + "why_it_matters": "Those unchanged operations limit the end-to-end effect of the new transform.", + "first_step": "Profile a complete source replay and isolate one separately source-bound replacement.", + "blocked_by": "The present change intentionally replaces only the bounded membership algorithm." + } + ], + "claim_type": "scoped algorithm-preservation proof with finite certificate and local performance evidence", + "question_source": { + "pr": 5374, + "author": "aliromman", + "package": "research/newton-bounded-residue-validator", + "usage": "bounded-residue algorithm and replay inputs, unchanged" + }, + "coordination": { + "claim": "https://github.com/CrocSwap/integer-mult-bounds/discussions/1166#discussioncomment-19380573", + "overlap": "Live open PRs and active Discussions; account PRs; source-specific merged and main searches. Sparse cached inventory excludes many main notes; targeted git grep searches use origin/main. Unpublished work and arbitrary diffs remain outside coverage." + }, + "verification_environment": { + "python": "3.12.15", + "platform": "macOS-27.0.1-arm64-arm-64bit", + "dependencies": "standard library only" + } +} diff --git a/research/packed-newton-membership/VERIFY.json b/research/packed-newton-membership/VERIFY.json new file mode 100644 index 0000000000..342f475ab3 --- /dev/null +++ b/research/packed-newton-membership/VERIFY.json @@ -0,0 +1,45 @@ +{ + "schema_version": 1, + "status": "ready", + "python": "3.12", + "requirements": [], + "inputs": [ + "research/newton-bounded-residue-validator/PROOF.md", + "research/newton-bounded-residue-validator/membership.py", + "research/newton-bounded-residue-validator/verify.py", + "research/newton-bounded-residue-validator/SOURCES.json", + "research/newton-bounded-residue-validator/RESULT.json", + "research/p104-clifford-exactness/model.py", + "research/p104-clifford-exactness/PROOF.md", + "research/p104-clifford-exactness/README.md", + "research/p104-rounded-collector/inputs/WORD.json", + "research/p104-rounded-collector/inputs/NATIVE.json.gz" + ], + "commands": [ + { + "argv": [ + "{python}", + "-I", + "-W", + "error", + "-B", + "{package}/verify.py" + ], + "timeout_seconds": 120, + "checks": "Packed subtraction proof fixtures, exact first-failure preservation, complete original source replay and source integrity." + }, + { + "argv": [ + "{python}", + "-I", + "-W", + "error", + "-O", + "-B", + "{package}/verify.py" + ], + "timeout_seconds": 120, + "checks": "All checks with Python assertions disabled." + } + ] +} diff --git a/research/packed-newton-membership/benchmark.py b/research/packed-newton-membership/benchmark.py new file mode 100644 index 0000000000..21995bcaf6 --- /dev/null +++ b/research/packed-newton-membership/benchmark.py @@ -0,0 +1,114 @@ +"""Five alternating fresh-process trials; timing is not an acceptance gate. + +Produced by Nick Petalas. Apache-2.0. +""" +import argparse +import cProfile +from hashlib import sha256 +import json +from pathlib import Path +import platform +import pstats +from statistics import median +import subprocess +import sys +from time import perf_counter + +HERE = Path(__file__).resolve().parent +sys.path.insert(0, str(HERE)) +import packed +from support import need, replay_result, source_modules + + +def sample(side, workload, profile=False): + reference, replay = source_modules() + if workload == "replay": + start = perf_counter() + result = replay_result(replay, side == "packed") + return {"seconds": perf_counter() - start, "errors": 0, + "result_sha256": sha256(json.dumps(result, sort_keys=True).encode()).hexdigest()} + # Same positive encoding and coefficient width as the source benchmark. + old = replay.load_source() + coefficients = [((-1 if i & 1 else 1) * ((1 << 1024) + i * 17), + (-1 if i & 2 else 1) * ((1 << 1024) + i * 13)) for i in range(512)] + values = old.encode(coefficients) + function = packed.obstruction if side == "packed" else reference.obstruction + need(reference.obstruction(values) is None and packed.obstruction(values) is None, + "benchmark fixture must be accepted") + profiler = cProfile.Profile() if profile else None + if profiler is not None: + profiler.enable() + start = perf_counter() + for _ in range(1000): + need(function(values) is None, "timed validation failed") + seconds = perf_counter() - start + result = {"seconds": seconds, "calls": 1000, "errors": 0, + "input_sha256": sha256(json.dumps(values).encode()).hexdigest(), + "max_component_bits": max(x.bit_length() for z in values for x in z)} + if profiler is not None: + profiler.disable() + stats = pstats.Stats(profiler) + result["profile_total_seconds"] = stats.total_tt + result["profile_hotspots"] = [ + {"function": f"{Path(key[0]).name}:{key[1]}:{key[2]}", + "calls": row[1], "own_seconds": row[2], + "cumulative_seconds": row[3]} + for key, row in sorted(stats.stats.items(), key=lambda x: x[1][2], reverse=True)[:8]] + return result + + +def trials(): + rows = {workload: {side: [] for side in ("reference", "packed")} + for workload in ("api", "replay")} + for workload in ("api", "replay"): + for trial in range(5): + order = ("reference", "packed") if trial % 2 == 0 else ("packed", "reference") + for side in order: + argv = [sys.executable, "-I", "-W", "error", "-B", str(__file__), + "--sample", side, "--workload", workload] + start = perf_counter() + run = subprocess.run(argv, capture_output=True, text=True, timeout=120, check=True) + elapsed = perf_counter() - start + need(not run.stderr and len(run.stdout) < 16384, "unexpected sample output") + record = json.loads(run.stdout) + need(record["errors"] == 0, "sample errors") + record["process_seconds"] = elapsed + rows[workload][side].append(record) + return summarize(rows) + + +def summarize(rows): + summaries = {} + for workload, sides in rows.items(): + field = "result_sha256" if workload == "replay" else "input_sha256" + need(len({r[field] for side in sides.values() for r in side}) == 1, + "different fixtures or scientific outputs") + summary = {} + for side, records in sides.items(): + values = [r["seconds"] for r in records] + summary[side] = {"median_seconds": median(values), "min_seconds": min(values), + "max_seconds": max(values)} + before = summary["reference"]["median_seconds"] + after = summary["packed"]["median_seconds"] + summary["reduction_percent"] = 100 * (1 - after / before) + summaries[workload] = summary + return {"python": platform.python_version(), "platform": platform.platform(), + "trials_per_side": 5, "samples": rows, "summary": summaries, + "code_sha256": {name: sha256((HERE / name).read_bytes()).hexdigest() + for name in ("benchmark.py", "packed.py", "support.py", "SOURCES.json")}, + "scope": "Complete API calls and complete inherited scientific replay; no physical caller or kappa claim."} + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--sample", choices=("reference", "packed")) + parser.add_argument("--workload", choices=("api", "replay"), default="api") + parser.add_argument("--profile", action="store_true") + args = parser.parse_args() + need(not args.profile or args.sample is not None, "profile requires one sample") + result = sample(args.sample, args.workload, args.profile) if args.sample else trials() + print(json.dumps(result, sort_keys=True)) + + +if __name__ == "__main__": + main() diff --git a/research/packed-newton-membership/packed.py b/research/packed-newton-membership/packed.py new file mode 100644 index 0000000000..66c52cdf13 --- /dev/null +++ b/research/packed-newton-membership/packed.py @@ -0,0 +1,78 @@ +"""Guarded-byte Gaussian Newton membership, ranks zero through nine. + +Produced by Nick Petalas. Apache-2.0. +The lattice is Clear1nsight/PR3382; bounded residues are Ali Romman/PR5374. +""" + +MAX_RANK = 9 +MAX_COMPONENT_BITS = 8192 + + +def _plans(): + """Build public address masks, never coefficient-dependent tables.""" + plans = [] + for rank in range(MAX_RANK + 1): + size = 1 << rank + residue_mask = (1 << ((rank + 1) // 2)) - 1 + stages = [] + for bit in range(rank): + upper = bytes(residue_mask if (i // 2) & (1 << bit) else 0 + for i in range(2 * size)) + guard = bytes(128 if (i // 2) & (1 << bit) else 0 + for i in range(2 * size)) + upper = int.from_bytes(upper, "little") + guard = int.from_bytes(guard, "little") + shift = 16 * (1 << bit) + stages.append((upper, upper >> shift, guard, shift)) + plans.append(tuple(stages)) + return tuple(plans) + + +PLANS = _plans() + + +def _pack(values, residue_mask): + data = bytearray(2 * len(values)) + for index, value in enumerate(values): + if (type(value) is not tuple or len(value) != 2 or + type(value[0]) is not int or type(value[1]) is not int): + raise ValueError("Gaussian integer pairs required") + if max(value[0].bit_length(), value[1].bit_length()) > MAX_COMPONENT_BITS: + raise ValueError("component exceeds 8192-bit domain") + data[2 * index] = value[0] & residue_mask + data[2 * index + 1] = value[1] & residue_mask + return int.from_bytes(data, "little") + + +def _transform(word, rank): + for upper, lower, guard, shift in PLANS[rank]: + difference = ((word & upper) | guard) - ((word & lower) << shift) + word = (word & ~upper) | (difference & upper) + return word.to_bytes(2 * (1 << rank), "little") + + +def obstruction(values): + """Return the first failing (mask, real, imaginary, residue_bits), or None. + + Exact contract: a list of 2**rank integer pairs, 0 <= rank <= 9, + with absolute component bit lengths <= 8192. Input is never mutated. + """ + if (type(values) is not list or not values or + len(values) & (len(values) - 1) or len(values) > (1 << MAX_RANK)): + raise ValueError("binary array of length 1 through 512 required") + rank = len(values).bit_length() - 1 + bits = (rank + 1) // 2 + word = _pack(values, (1 << bits) - 1) + data = _transform(word, rank) + for mask in range(len(values)): + real, imag = data[2 * mask], data[2 * mask + 1] + weight = mask.bit_count() + half = weight // 2 + if ((real | imag) & ((1 << half) - 1) or + (weight & 1 and ((real >> half) ^ (imag >> half)) & 1)): + return mask, real, imag, bits + return None + + +def member(values): + return obstruction(values) is None diff --git a/research/packed-newton-membership/support.py b/research/packed-newton-membership/support.py new file mode 100644 index 0000000000..0bbac1bfa7 --- /dev/null +++ b/research/packed-newton-membership/support.py @@ -0,0 +1,72 @@ +"""Source authentication and the opt-in complete replay adapter. + +Produced by Nick Petalas. Apache-2.0. +""" +from hashlib import sha256 +import importlib.util +import json +from pathlib import Path +import sys + +HERE = Path(__file__).resolve().parent +ROOT = HERE.parents[1] +MAX_SOURCE_BYTES = 32 * 1024 * 1024 + + +def need(condition, message): + if not condition: + raise ValueError(message) + + +def checked_hash(path, expected): + need(path.is_file(), "missing source: " + str(path)) + with path.open("rb") as stream: + payload = stream.read(MAX_SOURCE_BYTES + 1) + need(len(payload) <= MAX_SOURCE_BYTES, "source size limit") + need(sha256(payload).hexdigest() == expected, "source changed: " + str(path)) + + +def authenticate(): + metadata = json.loads((HERE / "SOURCES.json").read_text()) + for row in metadata["files"]: + if row["usage"] == "input": + path = ROOT / row["path"] + checked_hash(path, row["sha256"]) + + +def load(name, path): + spec = importlib.util.spec_from_file_location(name, path) + need(spec is not None and spec.loader is not None, "missing module loader") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def source_modules(): + authenticate() + source = ROOT / "research/newton-bounded-residue-validator" + # The source verifier's literal import is bound to authenticated bytes. + reference = load("packed_reference", source / "membership.py") + previous = sys.modules.get("membership") + previous_path = list(sys.path) + sys.modules["membership"] = reference + try: + replay = load("packed_source_replay", source / "verify.py") + finally: + sys.path[:] = previous_path + if previous is None: + sys.modules.pop("membership", None) + else: + sys.modules["membership"] = previous + return reference, replay + + +def replay_result(replay, candidate): + import packed + if candidate: + replay.member = packed.member + replay.obstruction = packed.obstruction + result = replay.replay() + expected = json.loads((ROOT / "research/newton-bounded-residue-validator/RESULT.json").read_text()) + need(result == expected, "complete predecessor replay changed") + return result diff --git a/research/packed-newton-membership/verify.py b/research/packed-newton-membership/verify.py new file mode 100644 index 0000000000..12546b3a64 --- /dev/null +++ b/research/packed-newton-membership/verify.py @@ -0,0 +1,140 @@ +"""Exact bounded checks for the new packed membership algorithm. + +Produced by Nick Petalas. Apache-2.0. +""" +from hashlib import sha256 +from itertools import product +import json +from pathlib import Path +import sys + +HERE = Path(__file__).resolve().parent +sys.path.insert(0, str(HERE)) +import packed +from support import checked_hash, need, replay_result, source_modules + + +def subset_coefficients(values): + """Independent definition by subsets, with at most 512 masks/subsets.""" + rank = len(values).bit_length() - 1 + mask = (1 << ((rank + 1) // 2)) - 1 + result = [] + for target in range(len(values)): + real = imag = 0 + for source in range(target + 1): + if source & target == source: + sign = -1 if (target.bit_count() - source.bit_count()) & 1 else 1 + real += sign * values[source][0] + imag += sign * values[source][1] + result.extend((real & mask, imag & mask)) + return bytes(result) + + +def compare(values, reference, full=False): + before = list(values) + actual = packed.obstruction(values) + need(actual == reference.obstruction(values), "first-failure tuple changed") + need(values == before, "input mutated") + if full: + rank = len(values).bit_length() - 1 + mask = (1 << ((rank + 1) // 2)) - 1 + word = packed._pack(values, mask) + need(packed._transform(word, rank) == subset_coefficients(values), + "packed coefficients differ from independent subset sums") + + +def lane_checks(): + count = 0 + for bits in range(1, 6): + mask = (1 << bits) - 1 + for upper, lower in product(range(mask + 1), repeat=2): + # Two simultaneous destination bytes expose cross-lane borrowing. + values = upper | (upper << 8) + sources = lower | (lower << 8) + actual = ((values | 0x8080) - sources) & (mask | (mask << 8)) + expected = ((upper - lower) & mask) * 257 + need(actual == expected, "guarded subtraction failed") + count += 1 + need((-0x0101 & 0x1F1F) != 0x1F1F, "unguarded borrow control") + need(((0x8080 - 0x0101) & 0x1F1F) == 0x1F1F, "guard repair") + return count + + +def finite_checks(reference): + small = broad = 0 + for rank in (0, 1, 2): + for raw in product(range(2), repeat=2 * (1 << rank)): + compare(list(zip(raw[::2], raw[1::2])), reference, True) + small += 1 + for raw in product(range(4), repeat=8): + compare([(x, 0) for x in raw], reference) + small += 1 + for rank in range(10): + for seed in range(12): + values = [((i * i * 7 + seed * 11) % 67 - 33, + (i * 13 + seed * 17) % 71 - 35) for i in range(1 << rank)] + compare(values, reference, seed == 0) + broad += 1 + for width in (1, 127, 1024, 8191): + values = [(((-1) ** i) * ((1 << width) + i), + ((-1) ** (i // 2)) * ((1 << width) + 3 * i)) + for i in range(1 << rank)] + compare(values, reference) + broad += 1 + return small, broad + + +def failure_checks(): + malformed = (None, (), [], [(0, 0)] * 3, [(0, 0)] * 1024, + [[0, 0]], [(True, 0)], [(0, 1.0)], [(0,)], + [(1 << 8192, 0)], [(0, -(1 << 8192))]) + count = 0 + for values in malformed: + try: + packed.obstruction(values) + except ValueError: + count += 1 + else: + raise ValueError("invalid input accepted") + try: + checked_hash(HERE / "packed.py", "0" * 64) + except ValueError: + count += 1 + else: + raise ValueError("source drift not rejected") + need(packed.member([(0, 0)]) and packed.member([((1 << 8192) - 1, 0)]), + "rank-zero or component-limit acceptance") + plans = packed.PLANS + # The fault changes the actual production transform, then is restored. + packed.PLANS = tuple(tuple((u, l, 0, s) for u, l, g, s in stages) + for stages in plans) + try: + values = [(1, 0)] + [(0, 0)] * 7 + word = packed._pack(values, 3) + need(packed._transform(word, 3) != subset_coefficients(values), + "production guard-removal mutant survived") + count += 1 + finally: + packed.PLANS = plans + return count + + +def main(): + reference, replay = source_modules() + lanes = lane_checks() + small, broad = finite_checks(reference) + controls = failure_checks() + complete = replay_result(replay, True) + result = {"status": "PASS_PACKED_NEWTON_MEMBERSHIP", "lane_pairs": lanes, + "small_arrays": small, "broad_arrays": broad, + "rejection_controls": controls, "source_replay": complete, + "rank9_packed_subtractions": 9, + "rank9_reference_component_subtractions": 4608, + "rank9_packed_bits": 8192} + files = sorted(path for path in HERE.iterdir() if path.is_file()) + result["package_sha256"] = {p.name: sha256(p.read_bytes()).hexdigest() for p in files} + print(json.dumps(result, sort_keys=True)) + + +if __name__ == "__main__": + main()