diff --git a/Makefile b/Makefile index b8eaf2d..38a863f 100644 --- a/Makefile +++ b/Makefile @@ -16,7 +16,7 @@ EVIDENCE_ENV = \ TEMP="$(EVIDENCE_WORK)/tmp" .PHONY: \ - build check dependencies distribution-check evidence evidence-browser \ + build check complexity-profile dependencies distribution-check evidence evidence-browser \ evidence-check lint test typecheck lint: @@ -39,6 +39,9 @@ build: dependencies: @$(PYTHON) -m pip check +complexity-profile: + @PYTHONPATH=src $(PYTHON) scripts/profile_complexity.py --format text + distribution-check: @PYTHONDONTWRITEBYTECODE=1 $(PYTHON) scripts/attest_distribution.py \ --root "$(CURDIR)" \ diff --git a/README.md b/README.md index d5c0914..efa67b0 100644 --- a/README.md +++ b/README.md @@ -127,6 +127,44 @@ policy-satisfying share on an honest 0–100% scale. Counts and entropy describe uniform draw from this state space; they are not a password-strength score or an authentication guarantee. +## Differential dynamic-programming work evidence + +![Exact logical work counters for six fixed policy scenarios](docs/assets/dp-work-counts.svg) + +The profiler constructs the real `PasswordSpace` under six fixed public +policies and retains only deterministic logical counters. A context-managed +iterator wrapper counts every deficit vector yielded to the production loop; +`cProfile` independently records exact `_build_layers` and `_consume` call +counts. Separate polynomial-convolution and class-population oracles must agree +with layer occupancy and the final arbitrary-precision state-space total. + +The controlled comparison keeps length `24`, the same four class widths, and +the same total minimum `24`: balanced minima `(6, 6, 6, 6)` materialize +`31,213` cells and perform `124,848` transitions, while skewed minima +`(21, 1, 1, 1)` materialize `2,288` cells and perform `9,148` transitions. +This isolates the shape of the deficit state space instead of presenting a +wall-clock benchmark. + +![Observed layer occupancy for five accepted policies and one explicit rejection boundary](docs/assets/dp-layer-occupancy.svg) + +The near-budget eight-class case is accepted at `164,025 / 216,513` cells and +`1,312,192 / 1,732,104` transitions. Raising every minimum from `2` to `3` +pushes the conservative bounds to `2,162,688` cells and `17,301,504` +transitions; profiling observes zero product, layer-build, and consume calls +because rejection happens before enumeration. The maximum-length one-class +case remains accepted and reaches a `1,678`-bit count. That bit length describes +integer arithmetic magnitude, not allocated memory. + +![Real deterministic complexity profiler output](docs/assets/dp-complexity-cli.png) + +Run the concise profile with `make complexity-profile`. The complete +[canonical JSON receipt](docs/evidence/dp-complexity-profile.json) retains every +accepted layer, while the +[real command transcript](docs/evidence/dp-complexity-profile.txt) stays short +enough to review. The evidence makes no elapsed-time, throughput, RSS, +allocation, CPU, hardware, password-strength, or cross-machine performance +claim. It consumes no entropy and constructs or emits no password candidate. + ## Deterministic audit CLI The inspection commands never sample a password. They emit exact decimal diff --git a/docs/assets/dp-complexity-cli.png b/docs/assets/dp-complexity-cli.png new file mode 100644 index 0000000..69f9e0c Binary files /dev/null and b/docs/assets/dp-complexity-cli.png differ diff --git a/docs/assets/dp-layer-occupancy.svg b/docs/assets/dp-layer-occupancy.svg new file mode 100644 index 0000000..aaae9ca --- /dev/null +++ b/docs/assets/dp-layer-occupancy.svg @@ -0,0 +1,108 @@ + + Exact dynamic-programming layer occupancy + Five panels plot exact retained deficit-vector counts by remaining position. A sixth panel records a complexity-budget rejection and deliberately contains no curve. + + + + + + + + + + + + + + + Sparse DP occupancy, layer by exact layer + Every accepted curve is drawn directly from the profiled layer_occupancy array; the rejected policy has no invented series. + + Logical state counts only · no elapsed time · no RSS · no hardware performance claim + + + Default visible ASCII + L=20 · minima 1 / 1 / 1 / 1 · peak 16 states + states 16 + + + + + + remaining 0 · states 1 + remaining 20 · states 16 + occupied 304 · product 20 calls / 320 vectors · consume 1,212 + + + Balanced strict minima + L=24 · minima 6 / 6 / 6 / 6 · peak 2,401 states + states 2,401 + + + + + + remaining 0 · states 1 + remaining 24 · states 2,401 + occupied 31,213 · product 24 calls / 57,624 vectors · consume 124,848 + + + Skewed strict minima + L=24 · minima 21 / 1 / 1 / 1 · peak 176 states + states 176 + + + + + + remaining 0 · states 1 + remaining 24 · states 176 + occupied 2,288 · product 24 calls / 4,224 vectors · consume 9,148 + + + Eight-class budget edge + L=32 · minima 2 / 2 / 2 / 2 / 2 / 2 / 2 / 2 · peak 6,561 states + states 6,561 + + + + + + remaining 0 · states 1 + remaining 32 · states 6,561 + occupied 164,025 · product 32 calls / 209,952 vectors · consume 1,312,192 + + + Arbitrary-precision depth + L=256 · minima 256 · peak 257 states + states 257 + + + + + + remaining 0 · states 1 + remaining 256 · states 257 + occupied 33,153 · product 256 calls / 65,792 vectors · consume 33,152 + + + Fail-fast budget guard + L=32 · minima 3 / 3 / 3 / 3 / 3 / 3 / 3 / 3 + + NO CURVE · PRECHECK REJECTED + cell bound 2,162,688 / 250,000 · transition bound 17,301,504 / 2,000,000 + product 0 calls · 0 vectors · consume 0 + No layer table exists, so no occupancy series is drawn. + + Horizontal axis: remaining positions. Vertical axis: retained bounded-deficit vectors on an exact linear scale within each panel. + 5 observed curves · 1 fail-fast boundary + diff --git a/docs/assets/dp-work-counts.svg b/docs/assets/dp-work-counts.svg new file mode 100644 index 0000000..64ebd8d --- /dev/null +++ b/docs/assets/dp-work-counts.svg @@ -0,0 +1,91 @@ + + Deterministic dynamic-programming work counts + Six fixed password-policy cases compare exact logical occupied cells and transition calls with preflight bounds. The rejected case records no dynamic-programming enumeration. + + + + + + + + + + + + + + + Logical DP work, measured without a stopwatch + Five accepted policies expose actual table work; the sixth proves the preflight budget stops before enumeration. + + DETERMINISTIC COUNTER CONTRACT · cProfile CALL COUNTS + ITERATOR COUNTERS + Logical operations only · no elapsed time · no RSS · no hardware performance claim + OCCUPIED DP CELLS / RECTANGULAR BOUND + _CONSUME CALLS / TRANSITION BOUND + + + Default visible ASCII + L=20 · minima 1 / 1 / 1 / 1 + product 20 calls · 320 vectors · consume 1,212 + + 304 occupied / 336 bound + + 1,212 calls / 1,344 bound + + + Balanced strict minima + L=24 · minima 6 / 6 / 6 / 6 + product 24 calls · 57,624 vectors · consume 124,848 + + 31,213 occupied / 60,025 bound + + 124,848 calls / 240,100 bound + + + Skewed strict minima + L=24 · minima 21 / 1 / 1 / 1 + product 24 calls · 4,224 vectors · consume 9,148 + + 2,288 occupied / 4,400 bound + + 9,148 calls / 17,600 bound + + + Eight-class budget edge + L=32 · minima 2 / 2 / 2 / 2 / 2 / 2 / 2 / 2 + product 32 calls · 209,952 vectors · consume 1,312,192 + + 164,025 occupied / 216,513 bound + + 1,312,192 calls / 1,732,104 bound + + + Arbitrary-precision depth + L=256 · minima 256 + product 256 calls · 65,792 vectors · consume 33,152 + + 33,153 occupied / 66,049 bound + + 33,152 calls / 66,049 bound + + + Fail-fast budget guard + L=32 · minima 3 / 3 / 3 / 3 / 3 / 3 / 3 / 3 + product 0 calls · 0 vectors · consume 0 + + preflight stop · bound 2,162,688 / budget 250,000 + + preflight stop · bound 17,301,504 / budget 2,000,000 + + Bars encode exact observed-to-bound ratios; every raw integer remains printed beside its bar. + rejected case: product 0 · consume 0 + diff --git a/docs/assets/quality-gate.png b/docs/assets/quality-gate.png index 66e3e9e..f1e51d2 100644 Binary files a/docs/assets/quality-gate.png and b/docs/assets/quality-gate.png differ diff --git a/docs/evidence/dp-complexity-profile.json b/docs/evidence/dp-complexity-profile.json new file mode 100644 index 0000000..6865190 --- /dev/null +++ b/docs/evidence/dp-complexity-profile.json @@ -0,0 +1,715 @@ +{ + "cases": [ + { + "bounds": { + "dp_cells_budget": 250000, + "dp_cells_upper_bound": 336, + "dp_transitions_budget": 2000000, + "dp_transitions_upper_bound": 1344, + "state_vectors_upper_bound_per_layer": 16 + }, + "case_id": "default-visible-ascii-20", + "independent_oracles": { + "occupied_cells": 304, + "transitions": 1212, + "valid_state_space": "2585908648140078948280078326668093030400" + }, + "observed": { + "layer_occupancy": [ + 1, + 5, + 11, + 15, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16 + ], + "occupied_cells": 304, + "peak_count_bits": 132, + "transitions": 1212, + "valid_state_space": "2585908648140078948280078326668093030400" + }, + "outcome": "accepted", + "policy": { + "class_count": 4, + "class_minima": [ + 1, + 1, + 1, + 1 + ], + "class_widths": [ + 26, + 26, + 10, + 32 + ], + "length": 20 + }, + "profiled_calls": { + "build_layers": { + "primitive_calls": 1, + "total_calls": 1 + }, + "consume": { + "primitive_calls": 1212, + "total_calls": 1212 + } + }, + "work_counters": { + "consume_calls": 1212, + "product_calls": 20, + "product_vectors": 320 + } + }, + { + "bounds": { + "dp_cells_budget": 250000, + "dp_cells_upper_bound": 60025, + "dp_transitions_budget": 2000000, + "dp_transitions_upper_bound": 240100, + "state_vectors_upper_bound_per_layer": 2401 + }, + "case_id": "balanced-visible-ascii-24", + "independent_oracles": { + "occupied_cells": 31213, + "transitions": 124848, + "valid_state_space": "236567853874633254943290775707366457344000000" + }, + "observed": { + "layer_occupancy": [ + 1, + 5, + 15, + 35, + 70, + 126, + 210, + 326, + 475, + 655, + 861, + 1085, + 1316, + 1540, + 1746, + 1926, + 2075, + 2191, + 2275, + 2331, + 2366, + 2386, + 2396, + 2400, + 2401 + ], + "occupied_cells": 31213, + "peak_count_bits": 158, + "transitions": 124848, + "valid_state_space": "236567853874633254943290775707366457344000000" + }, + "outcome": "accepted", + "policy": { + "class_count": 4, + "class_minima": [ + 6, + 6, + 6, + 6 + ], + "class_widths": [ + 26, + 26, + 10, + 32 + ], + "length": 24 + }, + "profiled_calls": { + "build_layers": { + "primitive_calls": 1, + "total_calls": 1 + }, + "consume": { + "primitive_calls": 124848, + "total_calls": 124848 + } + }, + "work_counters": { + "consume_calls": 124848, + "product_calls": 24, + "product_vectors": 57624 + } + }, + { + "bounds": { + "dp_cells_budget": 250000, + "dp_cells_upper_bound": 4400, + "dp_transitions_budget": 2000000, + "dp_transitions_upper_bound": 17600, + "state_vectors_upper_bound_per_layer": 176 + }, + "case_id": "skewed-visible-ascii-24", + "independent_oracles": { + "occupied_cells": 2288, + "transitions": 9148, + "valid_state_space": "52351049460478077365858907291994030080" + }, + "observed": { + "layer_occupancy": [ + 1, + 5, + 12, + 20, + 28, + 36, + 44, + 52, + 60, + 68, + 76, + 84, + 92, + 100, + 108, + 116, + 124, + 132, + 140, + 148, + 156, + 164, + 171, + 175, + 176 + ], + "occupied_cells": 2288, + "peak_count_bits": 158, + "transitions": 9148, + "valid_state_space": "52351049460478077365858907291994030080" + }, + "outcome": "accepted", + "policy": { + "class_count": 4, + "class_minima": [ + 21, + 1, + 1, + 1 + ], + "class_widths": [ + 26, + 26, + 10, + 32 + ], + "length": 24 + }, + "profiled_calls": { + "build_layers": { + "primitive_calls": 1, + "total_calls": 1 + }, + "consume": { + "primitive_calls": 9148, + "total_calls": 9148 + } + }, + "work_counters": { + "consume_calls": 9148, + "product_calls": 24, + "product_vectors": 4224 + } + }, + { + "bounds": { + "dp_cells_budget": 250000, + "dp_cells_upper_bound": 216513, + "dp_transitions_budget": 2000000, + "dp_transitions_upper_bound": 1732104, + "state_vectors_upper_bound_per_layer": 6561 + }, + "case_id": "near-budget-eight-class-32", + "independent_oracles": { + "occupied_cells": 164025, + "transitions": 1312192, + "valid_state_space": "38257913344943146219110345600" + }, + "observed": { + "layer_occupancy": [ + 1, + 9, + 45, + 157, + 423, + 927, + 1711, + 2727, + 3834, + 4850, + 5634, + 6138, + 6404, + 6516, + 6552, + 6560, + 6561, + 6561, + 6561, + 6561, + 6561, + 6561, + 6561, + 6561, + 6561, + 6561, + 6561, + 6561, + 6561, + 6561, + 6561, + 6561, + 6561 + ], + "occupied_cells": 164025, + "peak_count_bits": 97, + "transitions": 1312192, + "valid_state_space": "38257913344943146219110345600" + }, + "outcome": "accepted", + "policy": { + "class_count": 8, + "class_minima": [ + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "class_widths": [ + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1 + ], + "length": 32 + }, + "profiled_calls": { + "build_layers": { + "primitive_calls": 1, + "total_calls": 1 + }, + "consume": { + "primitive_calls": 1312192, + "total_calls": 1312192 + } + }, + "work_counters": { + "consume_calls": 1312192, + "product_calls": 32, + "product_vectors": 209952 + } + }, + { + "bounds": { + "dp_cells_budget": 250000, + "dp_cells_upper_bound": 66049, + "dp_transitions_budget": 2000000, + "dp_transitions_upper_bound": 66049, + "state_vectors_upper_bound_per_layer": 257 + }, + "case_id": "arbitrary-precision-one-class-256", + "independent_oracles": { + "occupied_cells": 33153, + "transitions": 33152, + "valid_state_space": "13204760283529711086017975496827969102122578748228854932215768953608425957914283378793017392640590182002206404278763711059841035864423601625110254131892757565869495470338450089708080064566367418705500800588966751865304590703827220828902312168631896511588754032589566627857130891842176335593555479284390295575803372353530458476431465822620855640835321633973740736105593048630603713155210875181236521246988904838997921007024818852561511543790205251774474107664518880852848565042488233058496752224510521901056" + }, + "observed": { + "layer_occupancy": [ + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63, + 64, + 65, + 66, + 67, + 68, + 69, + 70, + 71, + 72, + 73, + 74, + 75, + 76, + 77, + 78, + 79, + 80, + 81, + 82, + 83, + 84, + 85, + 86, + 87, + 88, + 89, + 90, + 91, + 92, + 93, + 94, + 95, + 96, + 97, + 98, + 99, + 100, + 101, + 102, + 103, + 104, + 105, + 106, + 107, + 108, + 109, + 110, + 111, + 112, + 113, + 114, + 115, + 116, + 117, + 118, + 119, + 120, + 121, + 122, + 123, + 124, + 125, + 126, + 127, + 128, + 129, + 130, + 131, + 132, + 133, + 134, + 135, + 136, + 137, + 138, + 139, + 140, + 141, + 142, + 143, + 144, + 145, + 146, + 147, + 148, + 149, + 150, + 151, + 152, + 153, + 154, + 155, + 156, + 157, + 158, + 159, + 160, + 161, + 162, + 163, + 164, + 165, + 166, + 167, + 168, + 169, + 170, + 171, + 172, + 173, + 174, + 175, + 176, + 177, + 178, + 179, + 180, + 181, + 182, + 183, + 184, + 185, + 186, + 187, + 188, + 189, + 190, + 191, + 192, + 193, + 194, + 195, + 196, + 197, + 198, + 199, + 200, + 201, + 202, + 203, + 204, + 205, + 206, + 207, + 208, + 209, + 210, + 211, + 212, + 213, + 214, + 215, + 216, + 217, + 218, + 219, + 220, + 221, + 222, + 223, + 224, + 225, + 226, + 227, + 228, + 229, + 230, + 231, + 232, + 233, + 234, + 235, + 236, + 237, + 238, + 239, + 240, + 241, + 242, + 243, + 244, + 245, + 246, + 247, + 248, + 249, + 250, + 251, + 252, + 253, + 254, + 255, + 256, + 257 + ], + "occupied_cells": 33153, + "peak_count_bits": 1678, + "transitions": 33152, + "valid_state_space": "13204760283529711086017975496827969102122578748228854932215768953608425957914283378793017392640590182002206404278763711059841035864423601625110254131892757565869495470338450089708080064566367418705500800588966751865304590703827220828902312168631896511588754032589566627857130891842176335593555479284390295575803372353530458476431465822620855640835321633973740736105593048630603713155210875181236521246988904838997921007024818852561511543790205251774474107664518880852848565042488233058496752224510521901056" + }, + "outcome": "accepted", + "policy": { + "class_count": 1, + "class_minima": [ + 256 + ], + "class_widths": [ + 94 + ], + "length": 256 + }, + "profiled_calls": { + "build_layers": { + "primitive_calls": 1, + "total_calls": 1 + }, + "consume": { + "primitive_calls": 33152, + "total_calls": 33152 + } + }, + "work_counters": { + "consume_calls": 33152, + "product_calls": 256, + "product_vectors": 65792 + } + }, + { + "bounds": { + "dp_cells_budget": 250000, + "dp_cells_upper_bound": 2162688, + "dp_transitions_budget": 2000000, + "dp_transitions_upper_bound": 17301504, + "state_vectors_upper_bound_per_layer": 65536 + }, + "case_id": "rejected-eight-class-32", + "outcome": "rejected-before-enumeration", + "policy": { + "class_count": 8, + "class_minima": [ + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3 + ], + "class_widths": [ + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1 + ], + "length": 32 + }, + "profiled_calls": { + "build_layers": { + "primitive_calls": 0, + "total_calls": 0 + }, + "consume": { + "primitive_calls": 0, + "total_calls": 0 + } + }, + "rejection": "dynamic-programming-complexity-budget", + "work_counters": { + "consume_calls": 0, + "product_calls": 0, + "product_vectors": 0 + } + } + ], + "claim_boundaries": { + "candidate_output_included": false, + "entropy_consumed": false, + "hardware_performance_claimed": false, + "memory_usage_claimed": false, + "wall_clock_timing_included": false + }, + "counter_contract": "logical-dp-operations-v1", + "profiler": { + "engine": "cProfile", + "retained_fields": [ + "primitive_calls", + "total_calls" + ], + "selected_project_functions": [ + "PasswordSpace._build_layers", + "_consume" + ], + "timing_fields_retained": false + }, + "scenario_set": "deterministic-dp-work-v1", + "schema_version": 1 +} diff --git a/docs/evidence/dp-complexity-profile.txt b/docs/evidence/dp-complexity-profile.txt new file mode 100644 index 0000000..6108e09 --- /dev/null +++ b/docs/evidence/dp-complexity-profile.txt @@ -0,0 +1,15 @@ +$ PYTHONPATH=src python scripts/profile_complexity.py --format text +Password Policy State-Space Lab - deterministic DP work profile +schema_version: 1 +scenario_set: deterministic-dp-work-v1 +profiler: cProfile selected project call counts; timing discarded + +default-visible-ascii-20: accepted · cells 304/336 · transitions 1212/1344 · peak integer 132 bits +balanced-visible-ascii-24: accepted · cells 31213/60025 · transitions 124848/240100 · peak integer 158 bits +skewed-visible-ascii-24: accepted · cells 2288/4400 · transitions 9148/17600 · peak integer 158 bits +near-budget-eight-class-32: accepted · cells 164025/216513 · transitions 1312192/1732104 · peak integer 97 bits +arbitrary-precision-one-class-256: accepted · cells 33153/66049 · transitions 33152/66049 · peak integer 1678 bits +rejected-eight-class-32: rejected before enumeration · cells bound 2162688 · transitions bound 17301504 · build calls 0 · consume calls 0 + +Scope: deterministic work counts only; no elapsed-time, RSS, hardware, or speed claim. +Safety: no entropy consumed and no password candidate constructed or emitted. diff --git a/docs/evidence/manifest.json b/docs/evidence/manifest.json index bedea62..474c71a 100644 --- a/docs/evidence/manifest.json +++ b/docs/evidence/manifest.json @@ -48,16 +48,52 @@ "sha256": "37eeac7f88fd0d3dd844c1984a8d69cdaf707f7ae5f3229889cb6dafe0d6656d", "width": 1960 }, + { + "assertions": [ + "deterministic profiler output with selected cProfile call counts", + "timing, hardware, memory, entropy, and candidate output absent" + ], + "bytes": 104431, + "height": 706, + "media_type": "image/png", + "path": "docs/assets/dp-complexity-cli.png", + "sha256": "ed94a3aca71400fec0088c4bbab1f6b3afb1f76d3504d51fcde0c7d8e0c47776", + "width": 1600 + }, + { + "assertions": [ + "rendered from every observed accepted DP layer", + "rejected policy has no invented occupancy curve" + ], + "bytes": 13138, + "height": 1320, + "media_type": "image/svg+xml", + "path": "docs/assets/dp-layer-occupancy.svg", + "sha256": "2c4155a32eddc3a48fe6d2f3fe7a93c6f1a8910e45e2e93ad6cffc15dd4f168d", + "width": 1800 + }, + { + "assertions": [ + "exact product-vector, materialized-cell, and transition counts", + "balanced and skewed policies are directly comparable" + ], + "bytes": 9045, + "height": 1120, + "media_type": "image/svg+xml", + "path": "docs/assets/dp-work-counts.svg", + "sha256": "9acdd587d0a8464bc8952902fc3f92f00cc428527c185fc4d60cc03aaaff7d2e", + "width": 1800 + }, { "assertions": [ "rendered from normalized real gate transcript", "all commands exited zero" ], - "bytes": 146672, + "bytes": 146498, "height": 1276, "media_type": "image/png", "path": "docs/assets/quality-gate.png", - "sha256": "9c9e04e71175d5d90aaa8ffd242a02611b9f44a20dea3884f0756656a9080985", + "sha256": "48fa53a3b0fc7736433e304898164aa6ee2c9afdd38e81d0c7b392590a96d9db", "width": 1600 }, { @@ -177,6 +213,26 @@ "path": "docs/evidence/distribution-check.txt", "sha256": "94b61d45db43b334efe8f574c94d69931bdb35aa709bec1205e112913285a92e" }, + { + "assertions": [ + "canonical report from six real instrumented constructions", + "independent occupancy and exact-count oracles agree" + ], + "bytes": 14697, + "media_type": "application/json", + "path": "docs/evidence/dp-complexity-profile.json", + "sha256": "86d9c8fe43f8af462dc300f9174149db6ae89700e0895f814597e7e3e0102265" + }, + { + "assertions": [ + "real concise profiler command output", + "no timing, hardware, memory, entropy, or candidate claim" + ], + "bytes": 1114, + "media_type": "text/plain", + "path": "docs/evidence/dp-complexity-profile.txt", + "sha256": "abafbf17802b02deb570c5a43cf4dbb94148f748bf398b079f3d5f3e071f98b7" + }, { "assertions": [ "real normalized command output", @@ -185,7 +241,7 @@ "bytes": 1874, "media_type": "text/plain", "path": "docs/evidence/quality-gate.txt", - "sha256": "aec144f46e7385f6dc217ad9b6ac4ac877bb688cf0fcce7c557b74a3847b98b3" + "sha256": "e5e71ef16b8c36026ae9af49698b7e2c67d0619592e9975efac26f3870653f53" }, { "assertions": [ @@ -305,6 +361,23 @@ "architecture_ast_verified": true, "sampling_ast_verified": true }, + "dp_complexity_profile": { + "accepted_scenarios": 5, + "contains_candidate": false, + "counter_contract": "logical-dp-operations-v1", + "json_source_command": "PYTHONPATH=src python scripts/profile_complexity.py --format json", + "rejected_before_enumeration": 1, + "report_schema_version": 1, + "scenario_ids": [ + "default-visible-ascii-20", + "balanced-visible-ascii-24", + "skewed-visible-ascii-24", + "near-budget-eight-class-32", + "arbitrary-precision-one-class-256", + "rejected-eight-class-32" + ], + "text_source_command": "PYTHONPATH=src python scripts/profile_complexity.py --format text" + }, "quality_gate": { "all_passed": true, "commands": [ @@ -344,7 +417,7 @@ "python": "3.12.3", "waitress": "3.0.2" }, - "schema_version": 1, + "schema_version": 2, "source_files": [ { "path": "MANIFEST.in", @@ -352,7 +425,7 @@ }, { "path": "Makefile", - "sha256": "70f4fe061b50a4548ddd13f47247c2110fff7eccec33252d47dfde69468be2ea" + "sha256": "d6d1c0b532cc82a7a211013aba0821cdd90e9ab2d4b71342af54823a68227e20" }, { "path": "PACKAGE.md", @@ -360,7 +433,7 @@ }, { "path": "README.md", - "sha256": "405ede3a7f253df40b64180f804e4a228872132aaaf6ad5c282fcb65f4a2c42b" + "sha256": "2524d9e69d196ed23bb76be3659ebd690d17a3b4a7c974b02f1c7bab3bcde2e4" }, { "path": "app.py", @@ -376,7 +449,7 @@ }, { "path": "scripts/check_evidence.py", - "sha256": "eb72f02fd3fc50536aee670f0b83badffd1d9b13df5fa16d87c38a78be8cf881" + "sha256": "47b8739b7fde11642d88963d74b53aecfed96fc3e6ddc0cbb3dbf4770c7e7f4a" }, { "path": "scripts/distribution_contract.py", @@ -384,11 +457,15 @@ }, { "path": "scripts/evidence_rendering.py", - "sha256": "b313270e49c26a9eb8b966805df21b5c46ed30823794cb5272014efb9f100a92" + "sha256": "bc55294f4ad8e0c22e4059b302a6681d572cba066c79c979406f833f176eb380" }, { "path": "scripts/generate_evidence.py", - "sha256": "96e00dd2f4e48c6ffb20d7543ef02b687d3f94df2d259bc064d2b6f48ef3c534" + "sha256": "d4bedb3609f542683b2d2b28dc8194204537ea21708507f201257301db9fa033" + }, + { + "path": "scripts/profile_complexity.py", + "sha256": "aa77bdcadfb1f2d621eb88be986dda9d75872518d592e00d5c88f792cb78c214" }, { "path": "src/password_policy_lab/__init__.py", @@ -446,6 +523,14 @@ "path": "tests/test_cli.py", "sha256": "12e93f725e570849c588ec631968bf768dbde27121067a88617aef8cbc07ea54" }, + { + "path": "tests/test_complexity_profile.py", + "sha256": "7cb577e444eac0549398a79512d84dcea68528400cf0ef0b52285d60397b3245" + }, + { + "path": "tests/test_complexity_rendering.py", + "sha256": "6fe89cdf5bfeb61fba87662ce70dec7a7f41d24ed8d6d3295321dec651a62ca7" + }, { "path": "tests/test_counting.py", "sha256": "5f72e9acf6c22d35317549fd13eaf87a6a864d5ad88c1441bfa50850eefa6718" @@ -460,7 +545,11 @@ }, { "path": "tests/test_evidence.py", - "sha256": "8bc201dce3424e6456090e803fa0da4e83f625cb0ce2fc186f33ed255f12121c" + "sha256": "0f28070b81ff1e084d0eaa130b12f3cf61a949ca8d1c0372a8d5581aaa0c059c" + }, + { + "path": "tests/test_evidence_generation.py", + "sha256": "1f4e6869e12494e13f8f792d27be2ed6a0e6883250537fa4cc650bc77e4c80f9" }, { "path": "tests/test_inspection.py", diff --git a/docs/evidence/quality-gate.txt b/docs/evidence/quality-gate.txt index 99f8a56..270649a 100644 --- a/docs/evidence/quality-gate.txt +++ b/docs/evidence/quality-gate.txt @@ -2,15 +2,15 @@ $ python -m ruff check app.py scripts src tests All checks passed! $ python -m ruff format --check app.py scripts src tests -26 files already formatted +30 files already formatted $ MYPYPATH=src python -m mypy --strict app.py scripts src tests -Success: no issues found in 26 source files +Success: no issues found in 30 source files $ PYTHONPATH=src python -m pytest --cov=password_policy_lab --cov-branch --cov-report=term-missing -q -................................................................................................................ [ 42%] -................................................................................................................ [ 84%] -........................................ [100%] +................................................................................................................ [ 37%] +................................................................................................................ [ 74%] +............................................................................ [100%] ==================================================== tests coverage ==================================================== ___________________________________ coverage: platform linux, python 3.12.3-final-0 ____________________________________ @@ -20,7 +20,7 @@ TOTAL 526 0 130 0 100% 9 files skipped due to complete coverage. Required test coverage of 100.0% reached. Total coverage: 100.00% -264 passed +300 passed $ python scripts/attest_distribution.py distribution attestation: PASS (unofficial) diff --git a/scripts/check_evidence.py b/scripts/check_evidence.py index a91d4c0..dd18f1c 100644 --- a/scripts/check_evidence.py +++ b/scripts/check_evidence.py @@ -8,6 +8,7 @@ import csv import hashlib import html +import importlib.util import io import json import re @@ -15,8 +16,11 @@ import sys import zlib from collections.abc import Sequence +from functools import lru_cache +from math import prod from pathlib import Path, PurePosixPath -from typing import NoReturn, cast +from types import ModuleType +from typing import Any, NoReturn, cast from xml.etree import ElementTree from PIL import Image @@ -31,9 +35,38 @@ VISIBLE_ASCII_PROFILE, visible_ascii_policy, ) +from password_policy_lab.space import MAX_DP_CELLS, MAX_DP_TRANSITIONS MANIFEST_PATH = "docs/evidence/manifest.json" GENERATOR_PATH = "scripts/generate_evidence.py" +COMPLEXITY_JSON_PATH = "docs/evidence/dp-complexity-profile.json" +COMPLEXITY_TEXT_PATH = "docs/evidence/dp-complexity-profile.txt" +COMPLEXITY_JSON_COMMAND = ( + "PYTHONPATH=src python scripts/profile_complexity.py --format json" +) +COMPLEXITY_TEXT_COMMAND = ( + "PYTHONPATH=src python scripts/profile_complexity.py --format text" +) +COMPLEXITY_PNG_TITLE = "Deterministic DP work profile · six fixed policies" + + +def _load_local_module(name: str) -> ModuleType: + """Load one sibling evidence module without relying on caller sys.path.""" + + path = Path(__file__).with_name(f"{name}.py") + spec = importlib.util.spec_from_file_location( + f"password_policy_evidence_{name}", + path, + ) + if spec is None or spec.loader is None: + raise RuntimeError(f"could not load local evidence module: {name}") + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +complexity_profiler = cast(Any, _load_local_module("profile_complexity")) EXPECTED_ASSETS = frozenset( { @@ -41,6 +74,9 @@ "docs/assets/cli-inspect.png", "docs/assets/distribution-check.png", "docs/assets/distribution-contract.svg", + "docs/assets/dp-complexity-cli.png", + "docs/assets/dp-layer-occupancy.svg", + "docs/assets/dp-work-counts.svg", "docs/assets/quality-gate.png", "docs/assets/setup-workflow.svg", "docs/assets/state-space-sweep.png", @@ -56,6 +92,8 @@ "docs/evidence/cli-inspect.txt", "docs/evidence/distribution-attestation.json", "docs/evidence/distribution-check.txt", + COMPLEXITY_JSON_PATH, + COMPLEXITY_TEXT_PATH, "docs/evidence/quality-gate.txt", "docs/evidence/state-space-sweep.csv", } @@ -175,6 +213,36 @@ "dp_transitions_upper_bound", ) +COMPLEXITY_CASE_IDS = ( + "default-visible-ascii-20", + "balanced-visible-ascii-24", + "skewed-visible-ascii-24", + "near-budget-eight-class-32", + "arbitrary-precision-one-class-256", + "rejected-eight-class-32", +) +_COMPLEXITY_POLICIES: dict[str, tuple[int, tuple[int, ...], tuple[int, ...]]] = { + "default-visible-ascii-20": (20, (26, 26, 10, 32), (1, 1, 1, 1)), + "balanced-visible-ascii-24": (24, (26, 26, 10, 32), (6, 6, 6, 6)), + "skewed-visible-ascii-24": (24, (26, 26, 10, 32), (21, 1, 1, 1)), + "near-budget-eight-class-32": (32, (1,) * 8, (2,) * 8), + "arbitrary-precision-one-class-256": (256, (94,), (256,)), + "rejected-eight-class-32": (32, (1,) * 8, (3,) * 8), +} +_COMPLEXITY_ACCEPTED_WORK: dict[str, tuple[int, int, int, int, int]] = { + "default-visible-ascii-20": (304, 1_212, 20, 320, 1_212), + "balanced-visible-ascii-24": (31_213, 124_848, 24, 57_624, 124_848), + "skewed-visible-ascii-24": (2_288, 9_148, 24, 4_224, 9_148), + "near-budget-eight-class-32": ( + 164_025, + 1_312_192, + 32, + 209_952, + 1_312_192, + ), + "arbitrary-precision-one-class-256": (33_153, 33_152, 256, 65_792, 33_152), +} + _SHA256 = re.compile(r"[0-9a-f]{64}\Z") _EMAIL = re.compile( r"(?i)(? dict[str, tuple[int, int]]: return viewports +def _validate_complexity_envelope(value: object) -> None: + envelope = _mapping(value, "evidence.dp_complexity_profile") + _exact_keys( + envelope, + { + "accepted_scenarios", + "contains_candidate", + "counter_contract", + "json_source_command", + "rejected_before_enumeration", + "report_schema_version", + "scenario_ids", + "text_source_command", + }, + "evidence.dp_complexity_profile", + ) + scenario_ids = _validate_string_list( + envelope["scenario_ids"], + "evidence.dp_complexity_profile.scenario_ids", + ) + if scenario_ids != list(COMPLEXITY_CASE_IDS): + _fail("complexity envelope has the wrong fixed scenario order") + if ( + _integer( + envelope["accepted_scenarios"], + "evidence.dp_complexity_profile.accepted_scenarios", + ) + != 5 + or _integer( + envelope["rejected_before_enumeration"], + "evidence.dp_complexity_profile.rejected_before_enumeration", + ) + != 1 + or _integer( + envelope["report_schema_version"], + "evidence.dp_complexity_profile.report_schema_version", + ) + != 1 + ): + _fail("complexity envelope has the wrong scenario cardinality or schema") + if _boolean( + envelope["contains_candidate"], + "evidence.dp_complexity_profile.contains_candidate", + ): + _fail("complexity envelope must not claim candidate output") + if envelope["counter_contract"] != "logical-dp-operations-v1": + _fail("complexity envelope has the wrong counter contract") + if envelope["json_source_command"] != COMPLEXITY_JSON_COMMAND: + _fail("complexity envelope has the wrong JSON source command") + if envelope["text_source_command"] != COMPLEXITY_TEXT_COMMAND: + _fail("complexity envelope has the wrong text source command") + + def _validate_evidence_claims(value: object) -> None: evidence = _mapping(value, "evidence") _exact_keys( evidence, - {"cli_inspect", "diagrams", "quality_gate", "sweep"}, + { + "cli_inspect", + "diagrams", + "dp_complexity_profile", + "quality_gate", + "sweep", + }, "evidence", ) + _validate_complexity_envelope(evidence["dp_complexity_profile"]) + sweep = _mapping(evidence["sweep"], "evidence.sweep") _exact_keys( sweep, @@ -1178,6 +1307,470 @@ def _validate_raw_evidence(textual: dict[str, str]) -> None: _fail("quality-gate transcript omits ordered command headers") +def _integer_list(value: object, label: str, *, minimum: int = 0) -> list[int]: + return [ + _integer(item, f"{label}[{index}]", minimum=minimum) + for index, item in enumerate(_sequence(value, label)) + ] + + +def _validate_profiled_call_map(value: object, label: str) -> dict[str, int]: + calls = _mapping(value, label) + _exact_keys(calls, {"primitive_calls", "total_calls"}, label) + primitive = _integer(calls["primitive_calls"], f"{label}.primitive_calls") + total = _integer(calls["total_calls"], f"{label}.total_calls") + if primitive > total: + _fail(f"{label} has more primitive calls than total calls") + return {"primitive_calls": primitive, "total_calls": total} + + +@lru_cache(maxsize=1) +def _expected_complexity_evidence() -> tuple[dict[str, object], str, str]: + try: + report = cast(dict[str, object], complexity_profiler.build_profile()) + json_text = cast(str, complexity_profiler.profile_json(report)) + text = cast(str, complexity_profiler.profile_text(report)) + except Exception as error: + raise EvidenceValidationError( + "the deterministic complexity profiler could not rebuild its report" + ) from error + transcript = f"$ {COMPLEXITY_TEXT_COMMAND}\n{text}" + return report, json_text, transcript + + +def _complexity_transcript(report: dict[str, object]) -> str: + try: + text = cast(str, complexity_profiler.profile_text(report)) + except Exception as error: + raise EvidenceValidationError( + "the deterministic complexity text renderer rejected its report" + ) from error + return f"$ {COMPLEXITY_TEXT_COMMAND}\n{text}" + + +def _validate_complexity_profile( + document: dict[str, object], + json_text: str, + transcript: str, +) -> None: + """Validate exact logical DP work without accepting performance claims.""" + + _exact_keys( + document, + { + "cases", + "claim_boundaries", + "counter_contract", + "profiler", + "scenario_set", + "schema_version", + }, + "complexity profile", + ) + if _integer(document["schema_version"], "complexity profile.schema_version") != 1: + _fail("complexity profile schema version is not supported") + if document["scenario_set"] != "deterministic-dp-work-v1": + _fail("complexity profile has the wrong fixed scenario set") + if document["counter_contract"] != "logical-dp-operations-v1": + _fail("complexity profile has the wrong counter contract") + + profiler = _mapping(document["profiler"], "complexity profile.profiler") + _exact_keys( + profiler, + { + "engine", + "retained_fields", + "selected_project_functions", + "timing_fields_retained", + }, + "complexity profile.profiler", + ) + if profiler["engine"] != "cProfile": + _fail("complexity profile uses an unexpected call-count engine") + if _validate_string_list( + profiler["selected_project_functions"], + "complexity profile.profiler.selected_project_functions", + ) != ["PasswordSpace._build_layers", "_consume"]: + _fail("complexity profile selects the wrong project functions") + if _validate_string_list( + profiler["retained_fields"], + "complexity profile.profiler.retained_fields", + ) != ["primitive_calls", "total_calls"]: + _fail("complexity profile retained fields are not call-count only") + if _boolean( + profiler["timing_fields_retained"], + "complexity profile.profiler.timing_fields_retained", + ): + _fail("complexity profile must discard profiler timing fields") + + boundaries = _mapping( + document["claim_boundaries"], + "complexity profile.claim_boundaries", + ) + boundary_keys = { + "candidate_output_included", + "entropy_consumed", + "hardware_performance_claimed", + "memory_usage_claimed", + "wall_clock_timing_included", + } + _exact_keys(boundaries, boundary_keys, "complexity profile.claim_boundaries") + if any( + _boolean(boundaries[key], f"complexity profile.claim_boundaries.{key}") + for key in boundary_keys + ): + _fail("complexity profile claim boundaries must all remain false") + + cases = _sequence(document["cases"], "complexity profile.cases") + if len(cases) != len(COMPLEXITY_CASE_IDS): + _fail("complexity profile must contain exactly six fixed cases") + observed_ids: list[str] = [] + cases_by_id: dict[str, dict[str, object]] = {} + for index, raw_case in enumerate(cases): + label = f"complexity profile.cases[{index}]" + case = _mapping(raw_case, label) + case_id = _string(case.get("case_id"), f"{label}.case_id") + observed_ids.append(case_id) + cases_by_id[case_id] = case + if case_id not in _COMPLEXITY_POLICIES: + _fail(f"{label} is not in the fixed scenario set") + + accepted = case_id in _COMPLEXITY_ACCEPTED_WORK + expected_keys = { + "bounds", + "case_id", + "outcome", + "policy", + "profiled_calls", + "work_counters", + } + expected_keys |= ( + {"independent_oracles", "observed"} if accepted else {"rejection"} + ) + _exact_keys(case, expected_keys, label) + + policy = _mapping(case["policy"], f"{label}.policy") + _exact_keys( + policy, + {"class_count", "class_minima", "class_widths", "length"}, + f"{label}.policy", + ) + length = _integer(policy["length"], f"{label}.policy.length", minimum=1) + class_count = _integer( + policy["class_count"], + f"{label}.policy.class_count", + minimum=1, + ) + widths = _integer_list( + policy["class_widths"], + f"{label}.policy.class_widths", + minimum=1, + ) + minima = _integer_list( + policy["class_minima"], + f"{label}.policy.class_minima", + ) + expected_length, expected_widths, expected_minima = _COMPLEXITY_POLICIES[ + case_id + ] + if ( + length != expected_length + or class_count != len(widths) + or len(minima) != class_count + or tuple(widths) != expected_widths + or tuple(minima) != expected_minima + or sum(minima) > length + ): + _fail(f"{label} does not match its fixed public policy") + + bounds = _mapping(case["bounds"], f"{label}.bounds") + bound_keys = { + "dp_cells_budget", + "dp_cells_upper_bound", + "dp_transitions_budget", + "dp_transitions_upper_bound", + "state_vectors_upper_bound_per_layer", + } + _exact_keys(bounds, bound_keys, f"{label}.bounds") + vectors = prod(minimum + 1 for minimum in minima) + cells_upper = (length + 1) * vectors + transitions_upper = class_count * cells_upper + if ( + _integer( + bounds["state_vectors_upper_bound_per_layer"], + f"{label}.bounds.state_vectors_upper_bound_per_layer", + minimum=1, + ) + != vectors + or _integer( + bounds["dp_cells_upper_bound"], + f"{label}.bounds.dp_cells_upper_bound", + minimum=1, + ) + != cells_upper + or _integer( + bounds["dp_transitions_upper_bound"], + f"{label}.bounds.dp_transitions_upper_bound", + minimum=1, + ) + != transitions_upper + or bounds["dp_cells_budget"] != MAX_DP_CELLS + or bounds["dp_transitions_budget"] != MAX_DP_TRANSITIONS + ): + _fail(f"{label} bounds do not match the production admission contract") + + profiled = _mapping(case["profiled_calls"], f"{label}.profiled_calls") + _exact_keys( + profiled, + {"build_layers", "consume"}, + f"{label}.profiled_calls", + ) + build_calls = _validate_profiled_call_map( + profiled["build_layers"], + f"{label}.profiled_calls.build_layers", + ) + consume_calls = _validate_profiled_call_map( + profiled["consume"], + f"{label}.profiled_calls.consume", + ) + work = _mapping(case["work_counters"], f"{label}.work_counters") + _exact_keys( + work, + {"consume_calls", "product_calls", "product_vectors"}, + f"{label}.work_counters", + ) + product_calls = _integer( + work["product_calls"], + f"{label}.work_counters.product_calls", + ) + product_vectors = _integer( + work["product_vectors"], + f"{label}.work_counters.product_vectors", + ) + observed_consume_calls = _integer( + work["consume_calls"], + f"{label}.work_counters.consume_calls", + ) + + if not accepted: + if ( + case["outcome"] != "rejected-before-enumeration" + or case["rejection"] != "dynamic-programming-complexity-budget" + ): + _fail("rejected complexity case has the wrong failure contract") + if cells_upper <= MAX_DP_CELLS and transitions_upper <= MAX_DP_TRANSITIONS: + _fail("rejected complexity case is below both production budgets") + zero_calls = {"primitive_calls": 0, "total_calls": 0} + if ( + build_calls != zero_calls + or consume_calls != zero_calls + or product_calls != 0 + or product_vectors != 0 + or observed_consume_calls != 0 + ): + _fail("rejected complexity case performed forbidden enumeration") + continue + + if case["outcome"] != "accepted": + _fail(f"{label} unexpectedly records a rejected outcome") + if cells_upper > MAX_DP_CELLS or transitions_upper > MAX_DP_TRANSITIONS: + _fail(f"{label} was accepted above a production complexity budget") + observed = _mapping(case["observed"], f"{label}.observed") + _exact_keys( + observed, + { + "layer_occupancy", + "occupied_cells", + "peak_count_bits", + "transitions", + "valid_state_space", + }, + f"{label}.observed", + ) + occupancy = _integer_list( + observed["layer_occupancy"], + f"{label}.observed.layer_occupancy", + minimum=1, + ) + occupied = _integer( + observed["occupied_cells"], + f"{label}.observed.occupied_cells", + minimum=1, + ) + transitions = _integer( + observed["transitions"], + f"{label}.observed.transitions", + ) + peak_bits = _integer( + observed["peak_count_bits"], + f"{label}.observed.peak_count_bits", + minimum=1, + ) + valid = _string( + observed["valid_state_space"], + f"{label}.observed.valid_state_space", + ) + if re.fullmatch(r"[1-9][0-9]*", valid) is None: + _fail(f"{label}.observed.valid_state_space is not canonical decimal") + if ( + len(occupancy) != length + 1 + or occupancy[0] != 1 + or occupied != sum(occupancy) + or occupied > cells_upper + or transitions != class_count * (occupied - 1) + ): + _fail(f"{label} has inconsistent occupied cell or transition counts") + ( + expected_occupied, + expected_transitions, + expected_product_calls, + expected_vectors, + expected_consume, + ) = _COMPLEXITY_ACCEPTED_WORK[case_id] + if ( + occupied != expected_occupied + or transitions != expected_transitions + or product_calls != expected_product_calls + or product_vectors != expected_vectors + or observed_consume_calls != expected_consume + ): + _fail(f"{label} logical work counters changed from the fixed study") + if ( + build_calls != {"primitive_calls": 1, "total_calls": 1} + or consume_calls + != {"primitive_calls": transitions, "total_calls": transitions} + or product_calls != length + or product_vectors != length * vectors + or observed_consume_calls != transitions + ): + _fail(f"{label} profiler counters do not match the loop or call contract") + oracles = _mapping( + case["independent_oracles"], + f"{label}.independent_oracles", + ) + _exact_keys( + oracles, + {"occupied_cells", "transitions", "valid_state_space"}, + f"{label}.independent_oracles", + ) + oracle_occupied = _integer( + oracles["occupied_cells"], + f"{label}.independent_oracles.occupied_cells", + minimum=1, + ) + oracle_transitions = _integer( + oracles["transitions"], + f"{label}.independent_oracles.transitions", + ) + oracle_valid = _string( + oracles["valid_state_space"], + f"{label}.independent_oracles.valid_state_space", + ) + if ( + oracle_occupied != occupied + or oracle_transitions != transitions + or oracle_valid != valid + ): + _fail(f"{label} observed work disagrees with an independent oracle") + del peak_bits + + if observed_ids != list(COMPLEXITY_CASE_IDS) or len(cases_by_id) != len(cases): + _fail("complexity profile cases are duplicated or out of fixed order") + + balanced = cases_by_id["balanced-visible-ascii-24"] + skewed = cases_by_id["skewed-visible-ascii-24"] + balanced_policy = _mapping(balanced["policy"], "balanced policy") + skewed_policy = _mapping(skewed["policy"], "skewed policy") + balanced_observed = _mapping(balanced["observed"], "balanced observation") + skewed_observed = _mapping(skewed["observed"], "skewed observation") + if ( + balanced_policy["length"] != skewed_policy["length"] + or balanced_policy["class_widths"] != skewed_policy["class_widths"] + or sum(cast(list[int], balanced_policy["class_minima"])) + != sum(cast(list[int], skewed_policy["class_minima"])) + or _integer(balanced_observed["occupied_cells"], "balanced cells") + <= _integer(skewed_observed["occupied_cells"], "skewed cells") + or _integer(balanced_observed["transitions"], "balanced transitions") + <= _integer(skewed_observed["transitions"], "skewed transitions") + ): + _fail("complexity profile lost its balanced-versus-skewed differential") + + _validate_safe_text(json_text, "complexity profile JSON") + _validate_safe_text(transcript, "complexity profile transcript") + if ( + not transcript.endswith( + "Safety: no entropy consumed and no password candidate " + "constructed or emitted.\n" + ) + or "no elapsed-time, RSS, hardware, or speed claim" not in transcript + ): + _fail("complexity transcript omits its privacy or claim boundaries") + + expected_report, expected_json, expected_transcript = ( + _expected_complexity_evidence() + ) + if document != expected_report or json_text != expected_json: + _fail("complexity profile is stale against a fresh deterministic rebuild") + if transcript != expected_transcript: + _fail("complexity transcript is not the exact real text representation") + + +def _validate_complexity_renderings( + root: Path, + report: dict[str, object], + transcript: str, +) -> None: + """Require and byte-compare every pure complexity renderer.""" + + try: + rendering = _load_local_module("evidence_rendering") + except (ImportError, OSError, RuntimeError) as error: + raise EvidenceValidationError( + "complexity renderers could not be loaded" + ) from error + for renderer_name, relative in ( + ("render_dp_layer_occupancy_svg", "docs/assets/dp-layer-occupancy.svg"), + ("render_dp_work_counts_svg", "docs/assets/dp-work-counts.svg"), + ): + renderer = getattr(rendering, renderer_name, None) + if not callable(renderer): + _fail(f"required pure complexity renderer is missing: {renderer_name}") + try: + rendered = renderer(report) + except Exception as error: + raise EvidenceValidationError( + f"pure complexity renderer failed: {renderer_name}" + ) from error + if type(rendered) is str: + expected = rendered.encode("utf-8") + elif type(rendered) is bytes: + expected = rendered + else: + _fail( + f"pure complexity renderer returned an invalid value: {renderer_name}" + ) + actual = _read_bounded(root / relative, relative) + if actual != expected: + _fail(f"complexity visual is stale against its pure renderer: {relative}") + + png_renderer = getattr(rendering, "render_terminal_png_bytes", None) + if not callable(png_renderer): + _fail("required pure complexity PNG renderer is missing") + try: + rendered_png = png_renderer( + transcript=transcript, + title=COMPLEXITY_PNG_TITLE, + ) + except Exception as error: + raise EvidenceValidationError("pure complexity PNG renderer failed") from error + if type(rendered_png) is not bytes: + _fail("pure complexity PNG renderer returned an invalid value") + png_relative = "docs/assets/dp-complexity-cli.png" + actual_png = _read_bounded(root / png_relative, png_relative) + if actual_png != rendered_png: + _fail(f"complexity visual is stale against its pure renderer: {png_relative}") + + def _distribution_input_digest(root: Path) -> str: digest = hashlib.sha256() for relative in DISTRIBUTION_INPUTS: @@ -1834,8 +2427,8 @@ def validate_evidence(root: Path) -> None: }, "manifest", ) - if document["schema_version"] != 1: - _fail("manifest.schema_version must be 1") + if document["schema_version"] != 2: + _fail("manifest.schema_version must be 2") if document["generator"] != GENERATOR_PATH: _fail("manifest.generator is incorrect") _validate_safe_text(manifest_text, "manifest") @@ -1846,6 +2439,22 @@ def validate_evidence(root: Path) -> None: textual = _validate_artifacts(repository, document["artifacts"], viewports) for artifact, text in textual.items(): _validate_safe_text(text, artifact) + complexity_document, complexity_text = _load_json( + repository / COMPLEXITY_JSON_PATH, + label="complexity profile", + ) + if textual[COMPLEXITY_JSON_PATH] != complexity_text: + _fail("complexity profile changed between bounded reads") + _validate_complexity_profile( + complexity_document, + complexity_text, + textual[COMPLEXITY_TEXT_PATH], + ) + _validate_complexity_renderings( + repository, + complexity_document, + textual[COMPLEXITY_TEXT_PATH], + ) distribution_document, distribution_text = _load_json( repository / "docs/evidence/distribution-attestation.json", label="distribution attestation", diff --git a/scripts/evidence_rendering.py b/scripts/evidence_rendering.py index 567148d..8c0f3c4 100644 --- a/scripts/evidence_rendering.py +++ b/scripts/evidence_rendering.py @@ -11,8 +11,10 @@ from collections.abc import Sequence from dataclasses import dataclass from io import BytesIO +from itertools import pairwise +from math import prod from pathlib import Path -from typing import Any +from typing import Any, Literal, cast Image: Any = importlib.import_module("PIL.Image") ImageDraw: Any = importlib.import_module("PIL.ImageDraw") @@ -51,6 +53,443 @@ def frame_count(self) -> int: return len(self.frames) +@dataclass(frozen=True, slots=True) +class _DpCaseSpec: + identifier: str + title: str + length: int + class_widths: tuple[int, ...] + class_minima: tuple[int, ...] + outcome: Literal["accepted", "rejected-before-enumeration"] + + +@dataclass(frozen=True, slots=True) +class _DpProfileCase: + spec: _DpCaseSpec + cells_upper_bound: int + transitions_upper_bound: int + product_calls: int + product_vectors: int + consume_calls: int + layer_occupancy: tuple[int, ...] | None + occupied_cells: int + transitions: int + peak_count_bits: int | None + + +_DP_CASE_SPECS = ( + _DpCaseSpec( + "default-visible-ascii-20", + "Default visible ASCII", + 20, + (26, 26, 10, 32), + (1, 1, 1, 1), + "accepted", + ), + _DpCaseSpec( + "balanced-visible-ascii-24", + "Balanced strict minima", + 24, + (26, 26, 10, 32), + (6, 6, 6, 6), + "accepted", + ), + _DpCaseSpec( + "skewed-visible-ascii-24", + "Skewed strict minima", + 24, + (26, 26, 10, 32), + (21, 1, 1, 1), + "accepted", + ), + _DpCaseSpec( + "near-budget-eight-class-32", + "Eight-class budget edge", + 32, + (1, 1, 1, 1, 1, 1, 1, 1), + (2, 2, 2, 2, 2, 2, 2, 2), + "accepted", + ), + _DpCaseSpec( + "arbitrary-precision-one-class-256", + "Arbitrary-precision depth", + 256, + (94,), + (256,), + "accepted", + ), + _DpCaseSpec( + "rejected-eight-class-32", + "Fail-fast budget guard", + 32, + (1, 1, 1, 1, 1, 1, 1, 1), + (3, 3, 3, 3, 3, 3, 3, 3), + "rejected-before-enumeration", + ), +) + + +def _profile_mapping( + value: object, + *, + keys: set[str], + label: str, +) -> dict[str, object]: + if type(value) is not dict: + raise ValueError(f"invalid complexity profile {label}") + mapping = cast(dict[str, object], value) + if set(mapping) != keys: + raise ValueError(f"invalid complexity profile {label}") + return mapping + + +def _profile_list(value: object, *, label: str) -> list[object]: + if type(value) is not list: + raise ValueError(f"invalid complexity profile {label}") + return cast(list[object], value) + + +def _profile_integer( + value: object, + *, + label: str, + minimum: int = 0, +) -> int: + if type(value) is not int or value < minimum: + raise ValueError(f"invalid complexity profile {label}") + return value + + +def _profile_integer_tuple( + value: object, + *, + label: str, + minimum: int, +) -> tuple[int, ...]: + values = _profile_list(value, label=label) + return tuple( + _profile_integer(item, label=label, minimum=minimum) for item in values + ) + + +def _profile_calls(value: object, *, label: str) -> tuple[int, int]: + calls = _profile_mapping( + value, + keys={"primitive_calls", "total_calls"}, + label=label, + ) + primitive = _profile_integer( + calls["primitive_calls"], + label=f"{label}.primitive_calls", + ) + total = _profile_integer(calls["total_calls"], label=f"{label}.total_calls") + if primitive > total: + raise ValueError(f"invalid complexity profile {label}") + return primitive, total + + +def _profile_decimal(value: object, *, label: str) -> str: + if ( + type(value) is not str + or not value + or len(value) > 4096 + or not value.isascii() + or not value.isdecimal() + or (len(value) > 1 and value.startswith("0")) + or value == "0" + ): + raise ValueError(f"invalid complexity profile {label}") + return value + + +def _validated_dp_profile(report: object) -> tuple[_DpProfileCase, ...]: + document = _profile_mapping( + report, + keys={ + "cases", + "claim_boundaries", + "counter_contract", + "profiler", + "scenario_set", + "schema_version", + }, + label="document", + ) + if ( + type(document["schema_version"]) is not int + or document["schema_version"] != 1 + or document["scenario_set"] != "deterministic-dp-work-v1" + or document["counter_contract"] != "logical-dp-operations-v1" + ): + raise ValueError("invalid complexity profile identity") + + profiler = _profile_mapping( + document["profiler"], + keys={ + "engine", + "retained_fields", + "selected_project_functions", + "timing_fields_retained", + }, + label="profiler", + ) + if ( + profiler["engine"] != "cProfile" + or profiler["selected_project_functions"] + != ["PasswordSpace._build_layers", "_consume"] + or profiler["retained_fields"] != ["primitive_calls", "total_calls"] + or profiler["timing_fields_retained"] is not False + ): + raise ValueError("invalid complexity profile profiler contract") + + boundaries = _profile_mapping( + document["claim_boundaries"], + keys={ + "candidate_output_included", + "entropy_consumed", + "hardware_performance_claimed", + "memory_usage_claimed", + "wall_clock_timing_included", + }, + label="claim_boundaries", + ) + if any(value is not False for value in boundaries.values()): + raise ValueError("invalid complexity profile claim boundaries") + + raw_cases = _profile_list(document["cases"], label="cases") + if len(raw_cases) != len(_DP_CASE_SPECS): + raise ValueError("invalid complexity profile cases") + + validated: list[_DpProfileCase] = [] + for index, (raw_case, spec) in enumerate( + zip(raw_cases, _DP_CASE_SPECS, strict=True) + ): + common_keys = { + "bounds", + "case_id", + "outcome", + "policy", + "profiled_calls", + "work_counters", + } + case_keys = ( + common_keys | {"independent_oracles", "observed"} + if spec.outcome == "accepted" + else common_keys | {"rejection"} + ) + case = _profile_mapping( + raw_case, + keys=case_keys, + label=f"cases[{index}]", + ) + if case["case_id"] != spec.identifier or case["outcome"] != spec.outcome: + raise ValueError("invalid complexity profile case identity") + + policy = _profile_mapping( + case["policy"], + keys={"class_count", "class_minima", "class_widths", "length"}, + label=f"cases[{index}].policy", + ) + length = _profile_integer( + policy["length"], + label=f"cases[{index}].policy.length", + minimum=1, + ) + class_count = _profile_integer( + policy["class_count"], + label=f"cases[{index}].policy.class_count", + minimum=1, + ) + class_widths = _profile_integer_tuple( + policy["class_widths"], + label=f"cases[{index}].policy.class_widths", + minimum=1, + ) + class_minima = _profile_integer_tuple( + policy["class_minima"], + label=f"cases[{index}].policy.class_minima", + minimum=0, + ) + if ( + length != spec.length + or class_count != len(spec.class_widths) + or class_widths != spec.class_widths + or class_minima != spec.class_minima + or len(class_minima) != class_count + ): + raise ValueError("invalid complexity profile policy") + + vectors_upper_bound = prod(minimum + 1 for minimum in class_minima) + cells_upper_bound = (length + 1) * vectors_upper_bound + transitions_upper_bound = class_count * cells_upper_bound + bounds = _profile_mapping( + case["bounds"], + keys={ + "dp_cells_budget", + "dp_cells_upper_bound", + "dp_transitions_budget", + "dp_transitions_upper_bound", + "state_vectors_upper_bound_per_layer", + }, + label=f"cases[{index}].bounds", + ) + expected_bounds = { + "dp_cells_budget": 250_000, + "dp_cells_upper_bound": cells_upper_bound, + "dp_transitions_budget": 2_000_000, + "dp_transitions_upper_bound": transitions_upper_bound, + "state_vectors_upper_bound_per_layer": vectors_upper_bound, + } + if any(bounds[key] != value for key, value in expected_bounds.items()): + raise ValueError("invalid complexity profile bounds") + + profiled_calls = _profile_mapping( + case["profiled_calls"], + keys={"build_layers", "consume"}, + label=f"cases[{index}].profiled_calls", + ) + build_primitive, build_total = _profile_calls( + profiled_calls["build_layers"], + label=f"cases[{index}].profiled_calls.build_layers", + ) + consume_primitive, consume_total = _profile_calls( + profiled_calls["consume"], + label=f"cases[{index}].profiled_calls.consume", + ) + work = _profile_mapping( + case["work_counters"], + keys={"consume_calls", "product_calls", "product_vectors"}, + label=f"cases[{index}].work_counters", + ) + product_calls = _profile_integer( + work["product_calls"], + label=f"cases[{index}].work_counters.product_calls", + ) + product_vectors = _profile_integer( + work["product_vectors"], + label=f"cases[{index}].work_counters.product_vectors", + ) + work_consume_calls = _profile_integer( + work["consume_calls"], + label=f"cases[{index}].work_counters.consume_calls", + ) + + if spec.outcome == "rejected-before-enumeration": + if ( + case["rejection"] != "dynamic-programming-complexity-budget" + or cells_upper_bound <= 250_000 + or transitions_upper_bound <= 2_000_000 + or any( + value != 0 + for value in ( + build_primitive, + build_total, + consume_primitive, + consume_total, + product_calls, + product_vectors, + work_consume_calls, + ) + ) + ): + raise ValueError("invalid complexity profile rejection") + validated.append( + _DpProfileCase( + spec=spec, + cells_upper_bound=cells_upper_bound, + transitions_upper_bound=transitions_upper_bound, + product_calls=0, + product_vectors=0, + consume_calls=0, + layer_occupancy=None, + occupied_cells=0, + transitions=0, + peak_count_bits=None, + ) + ) + continue + + observed = _profile_mapping( + case["observed"], + keys={ + "layer_occupancy", + "occupied_cells", + "peak_count_bits", + "transitions", + "valid_state_space", + }, + label=f"cases[{index}].observed", + ) + layer_occupancy = _profile_integer_tuple( + observed["layer_occupancy"], + label=f"cases[{index}].observed.layer_occupancy", + minimum=1, + ) + occupied_cells = _profile_integer( + observed["occupied_cells"], + label=f"cases[{index}].observed.occupied_cells", + minimum=1, + ) + transitions = _profile_integer( + observed["transitions"], + label=f"cases[{index}].observed.transitions", + minimum=1, + ) + peak_count_bits = _profile_integer( + observed["peak_count_bits"], + label=f"cases[{index}].observed.peak_count_bits", + minimum=1, + ) + valid_state_space = _profile_decimal( + observed["valid_state_space"], + label=f"cases[{index}].observed.valid_state_space", + ) + independent = _profile_mapping( + case["independent_oracles"], + keys={"occupied_cells", "transitions", "valid_state_space"}, + label=f"cases[{index}].independent_oracles", + ) + if ( + len(layer_occupancy) != length + 1 + or layer_occupancy[0] != 1 + or layer_occupancy[-1] != vectors_upper_bound + or any( + current < previous for previous, current in pairwise(layer_occupancy) + ) + or any(value > vectors_upper_bound for value in layer_occupancy) + or sum(layer_occupancy) != occupied_cells + or transitions != class_count * (occupied_cells - 1) + or peak_count_bits != (sum(class_widths) ** length).bit_length() + or build_primitive != 1 + or build_total != 1 + or consume_primitive != transitions + or consume_total != transitions + or product_calls != length + or product_vectors != length * vectors_upper_bound + or work_consume_calls != transitions + or independent["occupied_cells"] != occupied_cells + or independent["transitions"] != transitions + or independent["valid_state_space"] != valid_state_space + ): + raise ValueError("invalid complexity profile accepted case") + validated.append( + _DpProfileCase( + spec=spec, + cells_upper_bound=cells_upper_bound, + transitions_upper_bound=transitions_upper_bound, + product_calls=product_calls, + product_vectors=product_vectors, + consume_calls=work_consume_calls, + layer_occupancy=layer_occupancy, + occupied_cells=occupied_cells, + transitions=transitions, + peak_count_bits=peak_count_bits, + ) + ) + + return tuple(validated) + + def _svg_document( *, title: str, @@ -557,6 +996,210 @@ def write_distribution_svg( ) +def _counter_bar(*, x: int, y: int, width: int, observed: int, bound: int) -> str: + if observed == 0: + return ( + f'' + ) + filled = max(2, (width * observed + bound // 2) // bound) + return ( + f'' + f'' + ) + + +def render_dp_work_counts_svg(report: object) -> str: + """Return exact logical DP work against the preflight upper bounds.""" + + cases = _validated_dp_profile(report) + rows: list[str] = [] + for index, case in enumerate(cases): + y = 300 + index * 120 + minima = " / ".join(str(value) for value in case.spec.class_minima) + aria = ( + f"{case.spec.title}; length {case.spec.length}; " + f"occupied cells {case.occupied_cells}; cell bound " + f"{case.cells_upper_bound}; transition calls {case.transitions}; " + f"transition bound {case.transitions_upper_bound}" + ) + if case.layer_occupancy is None: + cell_label = ( + f"preflight stop · bound {case.cells_upper_bound:,} / budget 250,000" + ) + transition_label = ( + f"preflight stop · bound {case.transitions_upper_bound:,} / " + "budget 2,000,000" + ) + work_label = "product 0 calls · 0 vectors · consume 0" + else: + cell_label = ( + f"{case.occupied_cells:,} occupied / {case.cells_upper_bound:,} bound" + ) + transition_label = ( + f"{case.transitions:,} calls / {case.transitions_upper_bound:,} bound" + ) + work_label = ( + f"product {case.product_calls:,} calls · " + f"{case.product_vectors:,} vectors · " + f"consume {case.consume_calls:,}" + ) + rows.append( + f''' + + {case.spec.title} + L={case.spec.length} · minima {minima} + {work_label} + {_counter_bar(x=470, y=y + 3, width=500, observed=case.occupied_cells, bound=case.cells_upper_bound)} + {cell_label} + {_counter_bar(x=1125, y=y + 3, width=500, observed=case.transitions, bound=case.transitions_upper_bound)} + {transition_label} + ''' + ) + + body = f''' Logical DP work, measured without a stopwatch + Five accepted policies expose actual table work; the sixth proves the preflight budget stops before enumeration. + + DETERMINISTIC COUNTER CONTRACT · cProfile CALL COUNTS + ITERATOR COUNTERS + Logical operations only · no elapsed time · no RSS · no hardware performance claim + OCCUPIED DP CELLS / RECTANGULAR BOUND + _CONSUME CALLS / TRANSITION BOUND +{"".join(rows)} + Bars encode exact observed-to-bound ratios; every raw integer remains printed beside its bar. + rejected case: product 0 · consume 0''' + return _svg_document( + title="Deterministic dynamic-programming work counts", + description=( + "Six fixed password-policy cases compare exact logical occupied " + "cells and transition calls with preflight bounds. The rejected " + "case records no dynamic-programming enumeration." + ), + width=1800, + height=1120, + body=body, + ) + + +def write_dp_work_counts_svg(report: object, path: Path) -> None: + """Write the deterministic logical-work SVG.""" + + path.write_text(render_dp_work_counts_svg(report), encoding="utf-8") + + +def _occupancy_points( + values: tuple[int, ...], + *, + x: int, + y: int, + width: int, + height: int, +) -> str: + last_index = len(values) - 1 + peak = max(values) + denominator = max(1, peak - 1) + return " ".join( + f"{x + index * width // last_index}," + f"{y + height - (value - 1) * height // denominator}" + for index, value in enumerate(values) + ) + + +def render_dp_layer_occupancy_svg(report: object) -> str: + """Return exact per-layer sparse-table occupancy for accepted cases.""" + + cases = _validated_dp_profile(report) + panels: list[str] = [] + for index, case in enumerate(cases): + column = index % 2 + row = index // 2 + x = 55 + column * 870 + y = 230 + row * 340 + minima = " / ".join(str(value) for value in case.spec.class_minima) + if case.layer_occupancy is None: + aria = ( + f"{case.spec.title}; rejected before enumeration; no layer " + "occupancy curve; product calls zero; consume calls zero" + ) + panels.append( + f''' + + {case.spec.title} + L={case.spec.length} · minima {minima} + + NO CURVE · PRECHECK REJECTED + cell bound {case.cells_upper_bound:,} / 250,000 · transition bound {case.transitions_upper_bound:,} / 2,000,000 + product 0 calls · 0 vectors · consume 0 + No layer table exists, so no occupancy series is drawn. + ''' + ) + continue + + values = case.layer_occupancy + peak = max(values) + plot_x = x + 48 + plot_y = y + 120 + plot_width = 724 + plot_height = 100 + points = _occupancy_points( + values, + x=plot_x, + y=plot_y, + width=plot_width, + height=plot_height, + ) + start_y = plot_y + plot_height + end_y = plot_y + aria = ( + f"{case.spec.title}; exact occupancy for layers zero through " + f"{case.spec.length}; total occupied cells {case.occupied_cells}; " + f"peak layer occupancy {peak}; transition calls {case.transitions}" + ) + panels.append( + f''' + + {case.spec.title} + L={case.spec.length} · minima {minima} · peak {peak:,} states + states {peak:,} + + + + + + remaining 0 · states 1 + remaining {case.spec.length} · states {peak:,} + occupied {case.occupied_cells:,} · product {case.product_calls:,} calls / {case.product_vectors:,} vectors · consume {case.consume_calls:,} + ''' + ) + + body = f''' Sparse DP occupancy, layer by exact layer + Every accepted curve is drawn directly from the profiled layer_occupancy array; the rejected policy has no invented series. + + Logical state counts only · no elapsed time · no RSS · no hardware performance claim +{"".join(panels)} + Horizontal axis: remaining positions. Vertical axis: retained bounded-deficit vectors on an exact linear scale within each panel. + 5 observed curves · 1 fail-fast boundary''' + return _svg_document( + title="Exact dynamic-programming layer occupancy", + description=( + "Five panels plot exact retained deficit-vector counts by " + "remaining position. A sixth panel records a complexity-budget " + "rejection and deliberately contains no curve." + ), + width=1800, + height=1320, + body=body, + ) + + +def write_dp_layer_occupancy_svg(report: object, path: Path) -> None: + """Write the deterministic layer-occupancy SVG.""" + + path.write_text(render_dp_layer_occupancy_svg(report), encoding="utf-8") + + def _wrapped_lines(transcript: str, width: int) -> list[str]: lines: list[str] = [] for line in transcript.rstrip("\n").splitlines(): @@ -577,8 +1220,8 @@ def _wrapped_lines(transcript: str, width: int) -> list[str]: return lines -def render_terminal_png(*, transcript: str, title: str, path: Path) -> None: - """Render a real, already-recorded transcript without terminal metadata.""" +def render_terminal_png_bytes(*, transcript: str, title: str) -> bytes: + """Return a deterministic PNG for a real, already-recorded transcript.""" width = 1600 padding = 54 @@ -618,7 +1261,15 @@ def render_terminal_png(*, transcript: str, title: str, path: Path) -> None: draw.text((padding, y), line, fill=color, font=body_font) y += line_height - image.save(path, format="PNG", optimize=True, compress_level=9) + buffer = BytesIO() + image.save(buffer, format="PNG", optimize=True, compress_level=9) + return buffer.getvalue() + + +def render_terminal_png(*, transcript: str, title: str, path: Path) -> None: + """Write a deterministic transcript PNG without terminal metadata.""" + + path.write_bytes(render_terminal_png_bytes(transcript=transcript, title=title)) def recompress_png(path: Path) -> None: diff --git a/scripts/generate_evidence.py b/scripts/generate_evidence.py index 46d377a..4bc678f 100644 --- a/scripts/generate_evidence.py +++ b/scripts/generate_evidence.py @@ -26,6 +26,7 @@ from urllib.parse import urlsplit import attest_distribution as distribution_attester +import profile_complexity as complexity_profiler from evidence_rendering import ( GOLD, LINE, @@ -37,6 +38,8 @@ render_terminal_png, write_architecture_svg, write_distribution_svg, + write_dp_layer_occupancy_svg, + write_dp_work_counts_svg, write_gif, write_sampling_svg, write_setup_svg, @@ -64,6 +67,12 @@ "password-policy-lab sweep --start-length 8 --end-length 32 --format csv" ) INSPECT_COMMAND = "password-policy-lab inspect --length 20 --format text" +PROFILE_JSON_COMMAND = ( + "PYTHONPATH=src python scripts/profile_complexity.py --format json" +) +PROFILE_TEXT_COMMAND = ( + "PYTHONPATH=src python scripts/profile_complexity.py --format text" +) QUALITY_COMMANDS = ( "python -m ruff check app.py scripts src tests", "python -m ruff format --check app.py scripts src tests", @@ -82,6 +91,9 @@ "docs/assets/cli-inspect.png", "docs/assets/distribution-check.png", "docs/assets/distribution-contract.svg", + "docs/assets/dp-complexity-cli.png", + "docs/assets/dp-layer-occupancy.svg", + "docs/assets/dp-work-counts.svg", "docs/assets/quality-gate.png", "docs/assets/setup-workflow.svg", "docs/assets/state-space-sweep.png", @@ -93,6 +105,8 @@ "docs/evidence/cli-inspect.txt", "docs/evidence/distribution-attestation.json", "docs/evidence/distribution-check.txt", + "docs/evidence/dp-complexity-profile.json", + "docs/evidence/dp-complexity-profile.txt", "docs/evidence/quality-gate.txt", "docs/evidence/state-space-sweep.csv", "docs/evidence/web-validation-reference.png", @@ -101,7 +115,12 @@ _ABSOLUTE_PATH = re.compile( r"(?\d+ passed" + r"(?:, \d+ (?:skipped|deselected|xfailed|xpassed|warnings?))*)" + r"(?: in \d+(?:\.\d+)?s(?: \(\d+:\d{2}:\d{2}\))?" + r"| \(\d+:\d{2}:\d{2}\))$" +) class _WaitressServer(Protocol): @@ -267,6 +286,50 @@ def _write_cli_evidence() -> list[_SweepRow]: return rows +def _write_complexity_evidence() -> dict[str, object]: + report = complexity_profiler.build_profile() + expected_json = complexity_profiler.profile_json(report) + expected_text = complexity_profiler.profile_text(report) + json_output = _run( + ( + str(PYTHON), + "scripts/profile_complexity.py", + "--format", + "json", + ) + ) + text_output = _run( + ( + str(PYTHON), + "scripts/profile_complexity.py", + "--format", + "text", + ) + ) + if json_output != expected_json or text_output != expected_text: + raise RuntimeError( + "complexity CLI output disagrees with the in-process profile" + ) + + (EVIDENCE_DIR / "dp-complexity-profile.json").write_text( + json_output, + encoding="utf-8", + ) + transcript = f"$ {PROFILE_TEXT_COMMAND}\n{text_output}" + (EVIDENCE_DIR / "dp-complexity-profile.txt").write_text( + transcript, + encoding="utf-8", + ) + render_terminal_png( + transcript=transcript, + title="Deterministic DP work profile · six fixed policies", + path=ASSET_DIR / "dp-complexity-cli.png", + ) + write_dp_work_counts_svg(report, ASSET_DIR / "dp-work-counts.svg") + write_dp_layer_occupancy_svg(report, ASSET_DIR / "dp-layer-occupancy.svg") + return report + + def _render_sweep_chart(rows: Sequence[_SweepRow]) -> None: matplotlib: Any = importlib.import_module("matplotlib") matplotlib.use("Agg") @@ -981,7 +1044,7 @@ def _write_distribution_evidence() -> None: def _normalize_quality_output(output: str) -> str: normalized = output.replace("\r\n", "\n").replace(str(ROOT), ".") - normalized = _PYTEST_DURATION.sub(r"\1", normalized) + normalized = _PYTEST_DURATION.sub(r"\g", normalized) if _ABSOLUTE_PATH.search(normalized): raise RuntimeError("quality output contains an absolute machine path") return normalized.rstrip() + "\n" @@ -1113,6 +1176,18 @@ def _artifact_assertions() -> dict[str, list[str]]: "rendered from measured distribution hashes and member counts", "claim boundaries remain explicit", ], + "docs/assets/dp-complexity-cli.png": [ + "deterministic profiler output with selected cProfile call counts", + "timing, hardware, memory, entropy, and candidate output absent", + ], + "docs/assets/dp-layer-occupancy.svg": [ + "rendered from every observed accepted DP layer", + "rejected policy has no invented occupancy curve", + ], + "docs/assets/dp-work-counts.svg": [ + "exact product-vector, materialized-cell, and transition counts", + "balanced and skewed policies are directly comparable", + ], "docs/assets/quality-gate.png": [ "rendered from normalized real gate transcript", "all commands exited zero", @@ -1159,6 +1234,14 @@ def _artifact_assertions() -> dict[str, list[str]]: "real normalized attestation output", "no password sampled", ], + "docs/evidence/dp-complexity-profile.json": [ + "canonical report from six real instrumented constructions", + "independent occupancy and exact-count oracles agree", + ], + "docs/evidence/dp-complexity-profile.txt": [ + "real concise profiler command output", + "no timing, hardware, memory, entropy, or candidate claim", + ], "docs/evidence/quality-gate.txt": [ "real normalized command output", "timing and absolute path absent", @@ -1232,7 +1315,9 @@ def _manifest( capture: _CaptureResult, architecture_verified: bool, sampling_verified: bool, + complexity_report: dict[str, object], ) -> dict[str, object]: + complexity_cases = cast(list[dict[str, object]], complexity_report["cases"]) return { "artifacts": _artifact_manifest(), "capture": { @@ -1251,6 +1336,21 @@ def _manifest( "architecture_ast_verified": architecture_verified, "sampling_ast_verified": sampling_verified, }, + "dp_complexity_profile": { + "accepted_scenarios": sum( + case["outcome"] == "accepted" for case in complexity_cases + ), + "contains_candidate": False, + "counter_contract": complexity_report["counter_contract"], + "json_source_command": PROFILE_JSON_COMMAND, + "rejected_before_enumeration": sum( + case["outcome"] == "rejected-before-enumeration" + for case in complexity_cases + ), + "report_schema_version": complexity_report["schema_version"], + "scenario_ids": [case["case_id"] for case in complexity_cases], + "text_source_command": PROFILE_TEXT_COMMAND, + }, "quality_gate": { "all_passed": True, "commands": list(QUALITY_COMMANDS), @@ -1270,7 +1370,7 @@ def _manifest( }, "generator": "scripts/generate_evidence.py", "runtime": _runtime_manifest(capture.chromium), - "schema_version": 1, + "schema_version": 2, "source_files": _source_manifest(), } @@ -1317,6 +1417,7 @@ def main() -> int: write_sampling_svg(ASSET_DIR / "uniform-sampling-flow.svg") write_setup_svg(ASSET_DIR / "setup-workflow.svg") _write_distribution_evidence() + complexity_report = _write_complexity_evidence() sweep_rows = _write_cli_evidence() _render_sweep_chart(sweep_rows) @@ -1328,6 +1429,7 @@ def main() -> int: capture=capture, architecture_verified=architecture_verified, sampling_verified=sampling_verified, + complexity_report=complexity_report, ) ) @@ -1339,6 +1441,7 @@ def main() -> int: capture=capture, architecture_verified=architecture_verified, sampling_verified=sampling_verified, + complexity_report=complexity_report, ) ) second_gate = _run_quality_gate() diff --git a/scripts/profile_complexity.py b/scripts/profile_complexity.py new file mode 100644 index 0000000..61410d0 --- /dev/null +++ b/scripts/profile_complexity.py @@ -0,0 +1,498 @@ +#!/usr/bin/env python3 +"""Profile deterministic dynamic-programming work without timing claims. + +The profiler deliberately retains only selected project-function call counts. +Wall-clock and CPU timings collected internally by :mod:`cProfile` are dropped +before a report is built. The report never samples or emits a password. +""" + +from __future__ import annotations + +import argparse +import cProfile +import csv +import io +import json +from collections.abc import Callable, Iterable, Iterator, Sequence +from contextlib import contextmanager +from dataclasses import dataclass +from math import comb, prod +from typing import Any, Literal, cast + +import password_policy_lab.space as space_module +from password_policy_lab.errors import PolicyComplexityError +from password_policy_lab.policy import CharacterClass, PasswordPolicy +from password_policy_lab.profiles import VISIBLE_ASCII_CLASS_SYMBOLS +from password_policy_lab.space import ( + MAX_DP_CELLS, + MAX_DP_TRANSITIONS, + PasswordSpace, +) + +PROFILE_SCHEMA_VERSION = 1 +SCENARIO_SET = "deterministic-dp-work-v1" + +_ALL_VISIBLE_ASCII = "".join(VISIBLE_ASCII_CLASS_SYMBOLS) + + +@dataclass(frozen=True, slots=True) +class ComplexityScenario: + """One fixed, public, candidate-free profiling policy.""" + + identifier: str + length: int + classes: tuple[tuple[str, str, int], ...] + + +@dataclass(slots=True) +class _ProductCounter: + calls: int = 0 + vectors: int = 0 + + +SCENARIOS = ( + ComplexityScenario( + identifier="default-visible-ascii-20", + length=20, + classes=tuple( + (name, symbols, 1) + for name, symbols in zip( + ("lower", "upper", "digits", "punctuation"), + VISIBLE_ASCII_CLASS_SYMBOLS, + strict=True, + ) + ), + ), + ComplexityScenario( + identifier="balanced-visible-ascii-24", + length=24, + classes=tuple( + (name, symbols, 6) + for name, symbols in zip( + ("lower", "upper", "digits", "punctuation"), + VISIBLE_ASCII_CLASS_SYMBOLS, + strict=True, + ) + ), + ), + ComplexityScenario( + identifier="skewed-visible-ascii-24", + length=24, + classes=tuple( + (name, symbols, minimum) + for name, symbols, minimum in zip( + ("lower", "upper", "digits", "punctuation"), + VISIBLE_ASCII_CLASS_SYMBOLS, + (21, 1, 1, 1), + strict=True, + ) + ), + ), + ComplexityScenario( + identifier="near-budget-eight-class-32", + length=32, + classes=tuple( + (f"class-{index}", symbol, 2) + for index, symbol in enumerate(_ALL_VISIBLE_ASCII[:8]) + ), + ), + ComplexityScenario( + identifier="arbitrary-precision-one-class-256", + length=256, + classes=(("visible-ascii", _ALL_VISIBLE_ASCII, 256),), + ), + ComplexityScenario( + identifier="rejected-eight-class-32", + length=32, + classes=tuple( + (f"class-{index}", symbol, 3) + for index, symbol in enumerate(_ALL_VISIBLE_ASCII[:8]) + ), + ), +) + +_CSV_COLUMNS = ( + "case_id", + "outcome", + "length", + "class_count", + "class_minima", + "state_vectors_upper_bound_per_layer", + "dp_cells_upper_bound", + "observed_occupied_cells", + "dp_transitions_upper_bound", + "observed_transitions", + "peak_count_bits", +) + + +def _policy(scenario: ComplexityScenario) -> PasswordPolicy: + return PasswordPolicy( + length=scenario.length, + classes=tuple( + CharacterClass(name, symbols, minimum) + for name, symbols, minimum in scenario.classes + ), + ) + + +def _bounds(policy: PasswordPolicy) -> dict[str, int]: + vectors = prod(minimum + 1 for minimum in policy.minima) + cells = (policy.length + 1) * vectors + return { + "state_vectors_upper_bound_per_layer": vectors, + "dp_cells_upper_bound": cells, + "dp_transitions_upper_bound": len(policy.classes) * cells, + "dp_cells_budget": MAX_DP_CELLS, + "dp_transitions_budget": MAX_DP_TRANSITIONS, + } + + +def _layer_occupancy_oracle(length: int, minima: tuple[int, ...]) -> tuple[int, ...]: + """Count bounded deficit vectors by an independent polynomial convolution.""" + + coefficients = [1] + for minimum in minima: + updated = [0] * (len(coefficients) + minimum) + for subtotal, coefficient in enumerate(coefficients): + for deficit in range(minimum + 1): + updated[subtotal + deficit] += coefficient + coefficients = updated + + return tuple( + sum(coefficients[: min(remaining, len(coefficients) - 1) + 1]) + for remaining in range(length + 1) + ) + + +def _state_space_oracle( + length: int, + class_widths: tuple[int, ...], + minima: tuple[int, ...], +) -> int: + """Count strings by interleaving class populations, not deficit-state DP.""" + + ways = [0] * (length + 1) + ways[0] = 1 + for width, minimum in zip(class_widths, minima, strict=True): + updated = [0] * (length + 1) + for used, existing in enumerate(ways): + if existing == 0: + continue + for amount in range(minimum, length - used + 1): + combined = used + amount + updated[combined] += existing * comb(combined, amount) * width**amount + ways = updated + return ways[length] + + +def _profiled_calls( + profiler: cProfile.Profile, + function: Callable[..., object], +) -> dict[str, int]: + code = function.__code__ + for raw_entry in cast(list[Any], profiler.getstats()): + if raw_entry.code is code: + total = int(raw_entry.callcount) + recursive = int(raw_entry.reccallcount) + return { + "primitive_calls": total - recursive, + "total_calls": total, + } + return {"primitive_calls": 0, "total_calls": 0} + + +@contextmanager +def _count_product_vectors(counter: _ProductCounter) -> Iterator[None]: + """Wrap the production iterator while preserving its exact yielded values.""" + + module = cast(Any, space_module) + original = cast(Callable[..., Iterator[tuple[int, ...]]], module.product) + + def counted_product(*iterables: Iterable[int]) -> Iterator[tuple[int, ...]]: + counter.calls += 1 + for vector in original(*iterables): + counter.vectors += 1 + yield vector + + module.product = counted_product + try: + yield + finally: + module.product = original + + +def profile_scenario(scenario: ComplexityScenario) -> dict[str, object]: + """Profile one construction and cross-check every retained work counter.""" + + if type(scenario) is not ComplexityScenario: + raise TypeError("scenario must be a ComplexityScenario") + + policy = _policy(scenario) + bounds = _bounds(policy) + expected_rejection = ( + bounds["dp_cells_upper_bound"] > MAX_DP_CELLS + or bounds["dp_transitions_upper_bound"] > MAX_DP_TRANSITIONS + ) + profiler = cProfile.Profile() + product_counter = _ProductCounter() + space: PasswordSpace | None = None + rejected = False + with _count_product_vectors(product_counter): + profiler.enable() + try: + space = PasswordSpace(policy) + except PolicyComplexityError: + rejected = True + finally: + profiler.disable() + + build_calls = _profiled_calls(profiler, PasswordSpace._build_layers) + consume_calls = _profiled_calls(profiler, space_module._consume) + common: dict[str, object] = { + "case_id": scenario.identifier, + "policy": { + "length": policy.length, + "class_count": len(policy.classes), + "class_widths": [len(item.symbols) for item in policy.classes], + "class_minima": list(policy.minima), + }, + "bounds": bounds, + "profiled_calls": { + "build_layers": build_calls, + "consume": consume_calls, + }, + "work_counters": { + "product_calls": product_counter.calls, + "product_vectors": product_counter.vectors, + "consume_calls": consume_calls["total_calls"], + }, + } + + if rejected: + if not expected_rejection: + raise RuntimeError("profiled policy was rejected below the declared budget") + if build_calls != { + "primitive_calls": 0, + "total_calls": 0, + } or consume_calls != { + "primitive_calls": 0, + "total_calls": 0, + }: + raise RuntimeError( + "rejected policy entered dynamic-programming enumeration" + ) + if product_counter != _ProductCounter(): + raise RuntimeError("rejected policy entered deficit-vector enumeration") + return { + **common, + "outcome": "rejected-before-enumeration", + "rejection": "dynamic-programming-complexity-budget", + } + + if expected_rejection or space is None: + raise RuntimeError("profiled policy did not enforce the declared budget") + + layers = cast(tuple[dict[tuple[int, ...], int], ...], cast(Any, space)._layers) + layer_occupancy = tuple(len(layer) for layer in layers) + oracle_occupancy = _layer_occupancy_oracle(policy.length, policy.minima) + occupied_cells = sum(layer_occupancy) + oracle_occupied_cells = sum(oracle_occupancy) + expected_product_calls = policy.length + expected_product_vectors = ( + policy.length * bounds["state_vectors_upper_bound_per_layer"] + ) + observed_transitions = consume_calls["primitive_calls"] + oracle_transitions = len(policy.classes) * (oracle_occupied_cells - 1) + widths = tuple(len(item.symbols) for item in policy.classes) + oracle_total = _state_space_oracle(policy.length, widths, policy.minima) + peak_count_bits = max( + value.bit_length() for layer in layers for value in layer.values() + ) + + if layer_occupancy != oracle_occupancy: + raise RuntimeError("profiled layer occupancy disagrees with the oracle") + if product_counter.calls != expected_product_calls: + raise RuntimeError("profiled product calls disagree with the loop contract") + if product_counter.vectors != expected_product_vectors: + raise RuntimeError("profiled product vectors disagree with the oracle") + if consume_calls != { + "primitive_calls": oracle_transitions, + "total_calls": oracle_transitions, + }: + raise RuntimeError("profiled transition calls disagree with the oracle") + if build_calls != {"primitive_calls": 1, "total_calls": 1}: + raise RuntimeError( + "profiled construction did not build exactly one layer stack" + ) + if space.total != oracle_total: + raise RuntimeError("profiled exact count disagrees with the class-count oracle") + + return { + **common, + "outcome": "accepted", + "observed": { + "layer_occupancy": list(layer_occupancy), + "occupied_cells": occupied_cells, + "transitions": observed_transitions, + "peak_count_bits": peak_count_bits, + "valid_state_space": str(space.total), + }, + "independent_oracles": { + "occupied_cells": oracle_occupied_cells, + "transitions": oracle_transitions, + "valid_state_space": str(oracle_total), + }, + } + + +def build_profile() -> dict[str, object]: + """Run the complete fixed scenario set and return a stable report.""" + + return { + "schema_version": PROFILE_SCHEMA_VERSION, + "scenario_set": SCENARIO_SET, + "counter_contract": "logical-dp-operations-v1", + "profiler": { + "engine": "cProfile", + "selected_project_functions": [ + "PasswordSpace._build_layers", + "_consume", + ], + "retained_fields": ["primitive_calls", "total_calls"], + "timing_fields_retained": False, + }, + "claim_boundaries": { + "candidate_output_included": False, + "entropy_consumed": False, + "hardware_performance_claimed": False, + "memory_usage_claimed": False, + "wall_clock_timing_included": False, + }, + "cases": [profile_scenario(scenario) for scenario in SCENARIOS], + } + + +def profile_json(report: dict[str, object]) -> str: + """Serialize a canonical human-readable JSON receipt.""" + + return json.dumps(report, ensure_ascii=True, indent=2, sort_keys=True) + "\n" + + +def _accepted_observation(case: dict[str, object]) -> dict[str, object] | None: + value = case.get("observed") + if value is None: + return None + return cast(dict[str, object], value) + + +def profile_csv(report: dict[str, object]) -> str: + """Serialize one deterministic summary row per scenario.""" + + output = io.StringIO(newline="") + writer = csv.DictWriter(output, fieldnames=_CSV_COLUMNS, lineterminator="\n") + writer.writeheader() + for raw_case in cast(list[dict[str, object]], report["cases"]): + policy = cast(dict[str, object], raw_case["policy"]) + bounds = cast(dict[str, object], raw_case["bounds"]) + observed = _accepted_observation(raw_case) + writer.writerow( + { + "case_id": raw_case["case_id"], + "outcome": raw_case["outcome"], + "length": policy["length"], + "class_count": policy["class_count"], + "class_minima": ";".join( + str(value) for value in cast(list[int], policy["class_minima"]) + ), + "state_vectors_upper_bound_per_layer": bounds[ + "state_vectors_upper_bound_per_layer" + ], + "dp_cells_upper_bound": bounds["dp_cells_upper_bound"], + "observed_occupied_cells": ( + observed["occupied_cells"] if observed is not None else 0 + ), + "dp_transitions_upper_bound": bounds["dp_transitions_upper_bound"], + "observed_transitions": ( + observed["transitions"] if observed is not None else 0 + ), + "peak_count_bits": ( + observed["peak_count_bits"] if observed is not None else 0 + ), + } + ) + return output.getvalue() + + +def profile_text(report: dict[str, object]) -> str: + """Render a concise transcript without candidate or timing data.""" + + lines = [ + "Password Policy State-Space Lab - deterministic DP work profile", + f"schema_version: {PROFILE_SCHEMA_VERSION}", + f"scenario_set: {SCENARIO_SET}", + "profiler: cProfile selected project call counts; timing discarded", + "", + ] + for raw_case in cast(list[dict[str, object]], report["cases"]): + bounds = cast(dict[str, object], raw_case["bounds"]) + observed = _accepted_observation(raw_case) + if observed is None: + lines.append( + f"{raw_case['case_id']}: rejected before enumeration · " + f"cells bound {bounds['dp_cells_upper_bound']} · " + f"transitions bound {bounds['dp_transitions_upper_bound']} · " + "build calls 0 · consume calls 0" + ) + else: + lines.append( + f"{raw_case['case_id']}: accepted · " + f"cells {observed['occupied_cells']}/" + f"{bounds['dp_cells_upper_bound']} · " + f"transitions {observed['transitions']}/" + f"{bounds['dp_transitions_upper_bound']} · " + f"peak integer {observed['peak_count_bits']} bits" + ) + lines.extend( + ( + "", + ( + "Scope: deterministic work counts only; no elapsed-time, RSS, " + "hardware, or speed claim." + ), + ( + "Safety: no entropy consumed and no password candidate " + "constructed or emitted." + ), + "", + ) + ) + return "\n".join(lines) + + +def main(argv: Sequence[str] | None = None) -> int: + """Run the fixed profile and write exactly one selected representation.""" + + parser = argparse.ArgumentParser( + description="Profile deterministic PasswordSpace construction work.", + allow_abbrev=False, + ) + parser.add_argument( + "--format", + choices=("json", "csv", "text"), + default="json", + ) + arguments = parser.parse_args(argv) + report = build_profile() + output_format = cast(Literal["json", "csv", "text"], arguments.format) + rendered = { + "json": profile_json, + "csv": profile_csv, + "text": profile_text, + }[output_format](report) + print(rendered, end="") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_complexity_profile.py b/tests/test_complexity_profile.py new file mode 100644 index 0000000..89308a4 --- /dev/null +++ b/tests/test_complexity_profile.py @@ -0,0 +1,205 @@ +from __future__ import annotations + +import importlib.util +import json +import secrets +import sys +from itertools import product +from pathlib import Path +from types import ModuleType +from typing import Any, cast + +import pytest + +import password_policy_lab.space as space_module +from password_policy_lab import CharacterClass, PasswordPolicy, PasswordSpace + + +def _load_profiler() -> ModuleType: + path = Path(__file__).resolve().parents[1] / "scripts/profile_complexity.py" + spec = importlib.util.spec_from_file_location("complexity_profile_for_tests", path) + if spec is None or spec.loader is None: + raise RuntimeError("could not load the complexity profiler") + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +profiler = cast(Any, _load_profiler()) + + +def _cases(report: dict[str, object]) -> dict[str, dict[str, object]]: + return { + cast(str, item["case_id"]): item + for item in cast(list[dict[str, object]], report["cases"]) + } + + +def test_profile_records_exact_sparse_work_and_fail_fast_boundary() -> None: + report = cast(dict[str, object], profiler.build_profile()) + cases = _cases(report) + + assert list(cases) == [scenario.identifier for scenario in profiler.SCENARIOS] + expected = { + "default-visible-ascii-20": (304, 336, 1_212), + "balanced-visible-ascii-24": (31_213, 60_025, 124_848), + "skewed-visible-ascii-24": (2_288, 4_400, 9_148), + "near-budget-eight-class-32": (164_025, 216_513, 1_312_192), + "arbitrary-precision-one-class-256": (33_153, 66_049, 33_152), + } + for identifier, (cells, upper_bound, transitions) in expected.items(): + case = cases[identifier] + observed = cast(dict[str, object], case["observed"]) + bounds = cast(dict[str, object], case["bounds"]) + calls = cast(dict[str, dict[str, int]], case["profiled_calls"]) + work = cast(dict[str, int], case["work_counters"]) + oracles = cast(dict[str, object], case["independent_oracles"]) + assert case["outcome"] == "accepted" + assert observed["occupied_cells"] == cells + assert bounds["dp_cells_upper_bound"] == upper_bound + assert observed["transitions"] == transitions + assert calls["build_layers"] == { + "primitive_calls": 1, + "total_calls": 1, + } + assert calls["consume"] == { + "primitive_calls": transitions, + "total_calls": transitions, + } + assert work == { + "product_calls": cast( + int, cast(dict[str, object], case["policy"])["length"] + ), + "product_vectors": cast( + int, cast(dict[str, object], case["policy"])["length"] + ) + * cast(int, bounds["state_vectors_upper_bound_per_layer"]), + "consume_calls": transitions, + } + assert oracles["occupied_cells"] == cells + assert oracles["transitions"] == transitions + assert oracles["valid_state_space"] == observed["valid_state_space"] + + arbitrary = cast( + dict[str, object], cases["arbitrary-precision-one-class-256"]["observed"] + ) + assert arbitrary["peak_count_bits"] == (94**256).bit_length() + + rejected = cases["rejected-eight-class-32"] + rejected_bounds = cast(dict[str, object], rejected["bounds"]) + assert rejected["outcome"] == "rejected-before-enumeration" + assert rejected["rejection"] == "dynamic-programming-complexity-budget" + assert rejected_bounds["dp_cells_upper_bound"] == 2_162_688 + assert rejected_bounds["dp_transitions_upper_bound"] == 17_301_504 + assert rejected["profiled_calls"] == { + "build_layers": {"primitive_calls": 0, "total_calls": 0}, + "consume": {"primitive_calls": 0, "total_calls": 0}, + } + assert rejected["work_counters"] == { + "product_calls": 0, + "product_vectors": 0, + "consume_calls": 0, + } + + +def test_independent_oracles_match_a_small_exhaustive_space() -> None: + policy = PasswordPolicy( + 4, + ( + CharacterClass("letters", "ab", 2), + CharacterClass("digits", "01", 1), + ), + ) + exhaustive = sum( + all( + sum(symbol in character_class.symbols for symbol in candidate) + >= character_class.minimum + for character_class in policy.classes + ) + for candidate in product(policy.alphabet, repeat=policy.length) + ) + oracle = profiler._state_space_oracle( + policy.length, + tuple(len(item.symbols) for item in policy.classes), + policy.minima, + ) + occupancy = profiler._layer_occupancy_oracle(policy.length, policy.minima) + space = PasswordSpace(policy) + + assert oracle == exhaustive == space.total + assert occupancy == tuple(len(layer) for layer in space._layers) + + +def test_rejected_profile_never_enters_product_or_consume( + monkeypatch: pytest.MonkeyPatch, +) -> None: + rejected = profiler.SCENARIOS[-1] + + def fail(*values: object) -> None: + raise AssertionError(f"enumeration must not start: {len(values)}") + + monkeypatch.setattr(space_module, "product", fail) + monkeypatch.setattr(space_module, "_consume", fail) + + case = profiler.profile_scenario(rejected) + + assert case["outcome"] == "rejected-before-enumeration" + + +def test_profile_never_uses_candidate_or_entropy_paths( + monkeypatch: pytest.MonkeyPatch, +) -> None: + def fail_entropy(upper_bound: int) -> int: + raise AssertionError(f"entropy must not be consumed: {upper_bound}") + + def fail_candidate(*values: object, **named: object) -> None: + raise AssertionError( + f"candidate path must not run: {len(values)} positional, {len(named)} named" + ) + + monkeypatch.setattr(secrets, "randbelow", fail_entropy) + for method in ("rank", "unrank", "sample_uniform"): + monkeypatch.setattr(PasswordSpace, method, fail_candidate) + + report = profiler.build_profile() + + assert ( + cast(dict[str, object], report["claim_boundaries"])["entropy_consumed"] is False + ) + + +def test_serializations_are_canonical_stable_and_candidate_free() -> None: + report = profiler.build_profile() + first_json = profiler.profile_json(report) + second_json = profiler.profile_json(profiler.build_profile()) + csv_text = profiler.profile_csv(report) + text = profiler.profile_text(report) + + assert first_json == second_json + assert first_json == ( + json.dumps(json.loads(first_json), ensure_ascii=True, indent=2, sort_keys=True) + + "\n" + ) + assert csv_text.splitlines()[0] == ",".join(profiler._CSV_COLUMNS) + assert len(csv_text.splitlines()) == len(profiler.SCENARIOS) + 1 + assert "timing discarded" in text + assert "no elapsed-time, RSS, hardware, or speed claim" in text + assert "no password candidate constructed or emitted" in text + assert "\N{EM DASH}" not in text + combined = first_json + csv_text + text + assert "/home/" not in combined + assert "generated_password" not in combined + assert 'wall_clock_timing_included": true' not in combined + + +@pytest.mark.parametrize("output_format", ("json", "csv", "text")) +def test_cli_emits_exact_selected_representation( + output_format: str, + capsys: pytest.CaptureFixture[str], +) -> None: + assert profiler.main(["--format", output_format]) == 0 + captured = capsys.readouterr() + assert captured.err == "" + assert captured.out + assert captured.out.endswith("\n") diff --git a/tests/test_complexity_rendering.py b/tests/test_complexity_rendering.py new file mode 100644 index 0000000..b50f5b1 --- /dev/null +++ b/tests/test_complexity_rendering.py @@ -0,0 +1,237 @@ +from __future__ import annotations + +import copy +import importlib.util +import sys +import xml.etree.ElementTree as ElementTree +from pathlib import Path +from types import ModuleType +from typing import Any, cast + +import pytest + + +def _load_script(name: str) -> ModuleType: + path = Path(__file__).resolve().parents[1] / "scripts" / f"{name}.py" + spec = importlib.util.spec_from_file_location(f"{name}_for_render_tests", path) + if spec is None or spec.loader is None: + raise RuntimeError(f"could not load {name}") + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +rendering = cast(Any, _load_script("evidence_rendering")) +profiler = cast(Any, _load_script("profile_complexity")) + + +def _svg(path: Path) -> tuple[ElementTree.Element, str]: + text = path.read_text(encoding="utf-8") + root = ElementTree.fromstring(text) + return root, text + + +def _local_name(element: ElementTree.Element) -> str: + return element.tag.rsplit("}", 1)[-1] + + +def _assert_accessible_local_svg( + root: ElementTree.Element, + *, + width: int, + height: int, +) -> None: + assert _local_name(root) == "svg" + assert root.attrib["width"] == str(width) + assert root.attrib["height"] == str(height) + assert root.attrib["viewBox"] == f"0 0 {width} {height}" + assert root.attrib["role"] == "img" + assert root.attrib["aria-labelledby"] == "title description" + assert sum(_local_name(item) == "title" for item in root) == 1 + assert sum(_local_name(item) == "desc" for item in root) == 1 + assert all( + _local_name(item).lower() not in {"foreignobject", "script"} + for item in root.iter() + ) + for item in root.iter(): + for key, value in item.attrib.items(): + if key.rsplit("}", 1)[-1].lower() == "href": + assert value.startswith("#") + assert "url(http" not in value.lower() + + +def _groups(root: ElementTree.Element, prefix: str) -> list[ElementTree.Element]: + return [ + item + for item in root.iter() + if _local_name(item) == "g" and item.attrib.get("id", "").startswith(prefix) + ] + + +def test_work_counts_svg_is_exact_accessible_and_deterministic( + tmp_path: Path, +) -> None: + report = profiler.build_profile() + before = copy.deepcopy(report) + first = tmp_path / "work-a.svg" + second = tmp_path / "work-b.svg" + rendered = rendering.render_dp_work_counts_svg(report) + + rendering.write_dp_work_counts_svg(report, first) + rendering.write_dp_work_counts_svg(report, second) + + root, text = _svg(first) + _assert_accessible_local_svg(root, width=1800, height=1120) + assert rendered == rendering.render_dp_work_counts_svg(report) + assert first.read_text(encoding="utf-8") == rendered + assert first.read_bytes() == second.read_bytes() + assert report == before + assert len(_groups(root, "dp-work-")) == 6 + assert all( + group.attrib.get("role") == "group" for group in _groups(root, "dp-work-") + ) + for expected in ( + "304 occupied / 336 bound", + "31,213 occupied / 60,025 bound", + "2,288 occupied / 4,400 bound", + "164,025 occupied / 216,513 bound", + "33,153 occupied / 66,049 bound", + "1,312,192 calls / 1,732,104 bound", + "bound 2,162,688 / budget 250,000", + "bound 17,301,504 / budget 2,000,000", + "product 0 calls · 0 vectors · consume 0", + ): + assert expected in text + assert "no elapsed time · no RSS · no hardware performance claim" in text + + +def test_layer_occupancy_svg_draws_only_five_real_series( + tmp_path: Path, +) -> None: + report = profiler.build_profile() + before = copy.deepcopy(report) + first = tmp_path / "layers-a.svg" + second = tmp_path / "layers-b.svg" + rendered = rendering.render_dp_layer_occupancy_svg(report) + + rendering.write_dp_layer_occupancy_svg(report, first) + rendering.write_dp_layer_occupancy_svg(report, second) + + root, text = _svg(first) + _assert_accessible_local_svg(root, width=1800, height=1320) + assert rendered == rendering.render_dp_layer_occupancy_svg(report) + assert first.read_text(encoding="utf-8") == rendered + assert first.read_bytes() == second.read_bytes() + assert report == before + groups = _groups(root, "dp-layer-") + assert len(groups) == 6 + accepted = groups[:-1] + rejected = groups[-1] + assert [ + sum( + _local_name(item) == "polyline" + and item.attrib.get("data-series") == "layer-occupancy" + for item in group.iter() + ) + for group in accepted + ] == [1, 1, 1, 1, 1] + assert not any(_local_name(item) == "polyline" for item in rejected.iter()) + assert "NO CURVE · PRECHECK REJECTED" in text + assert "No layer table exists, so no occupancy series is drawn." in text + for expected in ( + "occupied 304 · product 20 calls / 320 vectors · consume 1,212", + "occupied 31,213 · product 24 calls / 57,624 vectors · consume 124,848", + "occupied 2,288 · product 24 calls / 4,224 vectors · consume 9,148", + "occupied 164,025 · product 32 calls / 209,952 vectors · consume 1,312,192", + "occupied 33,153 · product 256 calls / 65,792 vectors · consume 33,152", + ): + assert expected in text + assert "no elapsed time · no RSS · no hardware performance claim" in text + + +def test_terminal_png_renderer_is_pure_and_matches_write_wrapper( + tmp_path: Path, +) -> None: + transcript = "$ profiler --format text\nASCII body · exact counters\n" + title = "Deterministic DP work profile · six fixed policies" + path = tmp_path / "profile.png" + + first = rendering.render_terminal_png_bytes( + transcript=transcript, + title=title, + ) + second = rendering.render_terminal_png_bytes( + transcript=transcript, + title=title, + ) + rendering.render_terminal_png( + transcript=transcript, + title=title, + path=path, + ) + + assert first == second == path.read_bytes() + assert first.startswith(b"\x89PNG\r\n\x1a\n") + + +def _mutated_report(mutation: str) -> dict[str, object]: + report = cast(dict[str, object], copy.deepcopy(profiler.build_profile())) + cases = cast(list[dict[str, object]], report["cases"]) + if mutation == "schema-bool": + report["schema_version"] = True + elif mutation == "case-order": + cases[0], cases[1] = cases[1], cases[0] + elif mutation == "missing-case": + cases.pop() + elif mutation == "timing-claim": + cast(dict[str, object], report["profiler"])["timing_fields_retained"] = True + elif mutation == "occupancy": + observed = cast(dict[str, object], cases[0]["observed"]) + occupancy = cast(list[int], observed["layer_occupancy"]) + occupancy[1] += 1 + elif mutation == "product-counter": + cast(dict[str, object], cases[0]["work_counters"])["product_vectors"] = 1 + elif mutation == "rejected-series": + cases[-1]["observed"] = { + "layer_occupancy": [1], + "occupied_cells": 1, + "peak_count_bits": 1, + "transitions": 1, + "valid_state_space": "1", + } + else: # pragma: no cover - test helper contract + raise AssertionError("unknown mutation") + return report + + +@pytest.mark.parametrize( + "mutation", + ( + "schema-bool", + "case-order", + "missing-case", + "timing-claim", + "occupancy", + "product-counter", + "rejected-series", + ), +) +@pytest.mark.parametrize( + "renderer", + ( + "write_dp_work_counts_svg", + "write_dp_layer_occupancy_svg", + ), +) +def test_complexity_renderers_fail_closed_before_writing( + mutation: str, + renderer: str, + tmp_path: Path, +) -> None: + path = tmp_path / f"{mutation}.svg" + + with pytest.raises(ValueError, match="invalid complexity profile"): + getattr(rendering, renderer)(_mutated_report(mutation), path) + + assert not path.exists() diff --git a/tests/test_evidence.py b/tests/test_evidence.py index 88cc4dc..c42d28b 100644 --- a/tests/test_evidence.py +++ b/tests/test_evidence.py @@ -16,7 +16,11 @@ class _Checker(Protocol): + COMPLEXITY_CASE_IDS: tuple[str, ...] + COMPLEXITY_JSON_PATH: str + COMPLEXITY_TEXT_PATH: str EvidenceValidationError: type[ValueError] + EXPECTED_ARTIFACTS: frozenset[str] MANIFEST_PATH: str SWEEP_COLUMNS: tuple[str, ...] @@ -44,7 +48,34 @@ def _expected_sweep(self) -> str: ... def _expected_inspection(self, report: StateSpaceInspection) -> str: ... - def _load_json(self, path: Path) -> tuple[dict[str, object], str]: ... + def _load_json( + self, + path: Path, + *, + label: str = "manifest", + ) -> tuple[dict[str, object], str]: ... + + def _expected_complexity_evidence( + self, + ) -> tuple[dict[str, object], str, str]: ... + + def _complexity_transcript(self, report: dict[str, object]) -> str: ... + + def _validate_complexity_envelope(self, value: object) -> None: ... + + def _validate_complexity_profile( + self, + document: dict[str, object], + json_text: str, + transcript: str, + ) -> None: ... + + def _validate_complexity_renderings( + self, + root: Path, + report: dict[str, object], + transcript: str, + ) -> None: ... def _distribution_input_digest(self, root: Path) -> str: ... @@ -267,6 +298,149 @@ def test_manifest_loader_rejects_duplicates_and_noncanonical_json( check_evidence._load_json(path) +def _canonical_json(document: dict[str, object]) -> str: + return json.dumps(document, ensure_ascii=True, indent=2, sort_keys=True) + "\n" + + +def test_complexity_profile_rebuild_has_exact_schema_and_invariants() -> None: + report, json_text, transcript = check_evidence._expected_complexity_evidence() + + check_evidence._validate_complexity_profile(report, json_text, transcript) + + cases = cast(list[dict[str, object]], report["cases"]) + assert [case["case_id"] for case in cases] == list( + check_evidence.COMPLEXITY_CASE_IDS + ) + assert len(cases) == 6 + assert sum(case["outcome"] == "accepted" for case in cases) == 5 + assert cases[-1]["outcome"] == "rejected-before-enumeration" + + +def test_complexity_envelope_is_exact_and_candidate_free() -> None: + envelope: dict[str, object] = { + "accepted_scenarios": 5, + "contains_candidate": False, + "counter_contract": "logical-dp-operations-v1", + "json_source_command": ( + "PYTHONPATH=src python scripts/profile_complexity.py --format json" + ), + "rejected_before_enumeration": 1, + "report_schema_version": 1, + "scenario_ids": list(check_evidence.COMPLEXITY_CASE_IDS), + "text_source_command": ( + "PYTHONPATH=src python scripts/profile_complexity.py --format text" + ), + } + + check_evidence._validate_complexity_envelope(envelope) + + stale = copy.deepcopy(envelope) + cast(list[str], stale["scenario_ids"]).reverse() + with pytest.raises( + check_evidence.EvidenceValidationError, + match="fixed scenario order", + ): + check_evidence._validate_complexity_envelope(stale) + + +def test_complexity_profile_rejects_observed_oracle_drift() -> None: + expected, _, _ = check_evidence._expected_complexity_evidence() + report = copy.deepcopy(expected) + first = cast(list[dict[str, object]], report["cases"])[0] + observed = cast(dict[str, object], first["observed"]) + observed["occupied_cells"] = cast(int, observed["occupied_cells"]) + 1 + + with pytest.raises( + check_evidence.EvidenceValidationError, + match="inconsistent occupied cell", + ): + check_evidence._validate_complexity_profile( + report, + _canonical_json(report), + check_evidence._complexity_transcript(report), + ) + + +def test_complexity_profile_rejects_work_after_budget_failure() -> None: + expected, _, _ = check_evidence._expected_complexity_evidence() + report = copy.deepcopy(expected) + rejected = cast(list[dict[str, object]], report["cases"])[-1] + work = cast(dict[str, object], rejected["work_counters"]) + work["product_vectors"] = 1 + + with pytest.raises( + check_evidence.EvidenceValidationError, + match="forbidden enumeration", + ): + check_evidence._validate_complexity_profile( + report, + _canonical_json(report), + check_evidence._complexity_transcript(report), + ) + + +def test_complexity_profile_rejects_performance_overclaim_or_extra_metric() -> None: + expected, _, _ = check_evidence._expected_complexity_evidence() + overclaim = copy.deepcopy(expected) + boundaries = cast(dict[str, object], overclaim["claim_boundaries"]) + boundaries["hardware_performance_claimed"] = True + with pytest.raises( + check_evidence.EvidenceValidationError, + match="claim boundaries", + ): + check_evidence._validate_complexity_profile( + overclaim, + _canonical_json(overclaim), + check_evidence._complexity_transcript(overclaim), + ) + + extra_metric = copy.deepcopy(expected) + extra_metric["elapsed_seconds"] = 0 + with pytest.raises( + check_evidence.EvidenceValidationError, + match="unexpected schema", + ): + check_evidence._validate_complexity_profile( + extra_metric, + _canonical_json(extra_metric), + check_evidence._complexity_transcript(extra_metric), + ) + + +def test_complexity_artifacts_are_part_of_the_exact_readme_inventory() -> None: + assert { + "docs/assets/dp-complexity-cli.png", + "docs/assets/dp-layer-occupancy.svg", + "docs/assets/dp-work-counts.svg", + check_evidence.COMPLEXITY_JSON_PATH, + check_evidence.COMPLEXITY_TEXT_PATH, + }.issubset(check_evidence.EXPECTED_ARTIFACTS) + + +def test_complexity_renderings_reject_a_mutated_png(tmp_path: Path) -> None: + root = Path(__file__).resolve().parents[1] + report, _, transcript = check_evidence._expected_complexity_evidence() + relatives = ( + "docs/assets/dp-complexity-cli.png", + "docs/assets/dp-layer-occupancy.svg", + "docs/assets/dp-work-counts.svg", + ) + for relative in relatives: + destination = tmp_path / relative + destination.parent.mkdir(parents=True, exist_ok=True) + destination.write_bytes((root / relative).read_bytes()) + + check_evidence._validate_complexity_renderings(tmp_path, report, transcript) + png = tmp_path / "docs/assets/dp-complexity-cli.png" + png.write_bytes(png.read_bytes() + b"mutated") + + with pytest.raises( + check_evidence.EvidenceValidationError, + match="stale against its pure renderer", + ): + check_evidence._validate_complexity_renderings(tmp_path, report, transcript) + + def test_distribution_attestation_is_source_bound_and_rejects_overclaim() -> None: root = Path(__file__).resolve().parents[1] path = root / "docs/evidence/distribution-attestation.json" @@ -382,11 +556,8 @@ def test_readme_validation_is_robust_when_readme_is_absent(tmp_path: Path) -> No check_evidence._validate_readme(tmp_path) -def test_checked_in_evidence_bundle_is_valid_when_generated() -> None: +def test_checked_in_evidence_bundle_is_valid() -> None: root = Path(__file__).resolve().parents[1] - if not (root / check_evidence.MANIFEST_PATH).exists(): - pytest.skip("evidence generator has not populated the bundle yet") - check_evidence.validate_evidence(root) diff --git a/tests/test_evidence_generation.py b/tests/test_evidence_generation.py new file mode 100644 index 0000000..02c42af --- /dev/null +++ b/tests/test_evidence_generation.py @@ -0,0 +1,50 @@ +from __future__ import annotations + +import importlib.util +import sys +from pathlib import Path +from typing import Protocol, cast + +import pytest + + +class _Generator(Protocol): + def _normalize_quality_output(self, output: str) -> str: ... + + +def _load_generator() -> _Generator: + scripts = Path(__file__).resolve().parents[1] / "scripts" + path = scripts / "generate_evidence.py" + spec = importlib.util.spec_from_file_location( + "password_policy_evidence_generator", + path, + ) + if spec is None or spec.loader is None: + raise RuntimeError("could not load the evidence generator") + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + sys.path.insert(0, str(scripts)) + try: + spec.loader.exec_module(module) + finally: + sys.path.pop(0) + return cast(_Generator, module) + + +generator = _load_generator() + + +@pytest.mark.parametrize( + ("raw", "expected"), + [ + ("294 passed in 130.25s\n", "294 passed\n"), + ("293 passed, 1 skipped in 131.04s\n", "293 passed, 1 skipped\n"), + ("294 passed (0:02:10)\n", "294 passed\n"), + ("297 passed in 129.08s (0:02:09)\n", "297 passed\n"), + ], +) +def test_quality_normalization_removes_both_pytest_duration_formats( + raw: str, + expected: str, +) -> None: + assert generator._normalize_quality_output(raw) == expected