From d2e3e7c0f3905f929998ac20d96147a2a89fcae9 Mon Sep 17 00:00:00 2001 From: Xiwei Pan Date: Sun, 19 Jul 2026 22:22:47 +0800 Subject: [PATCH 1/3] Implement hard-regime CnC experiment workflow --- .gitignore | 1 + benchmarks/cnc/README.md | 118 ++++ benchmarks/cnc/aggregate_hard_regime.py | 653 +++++++++++++++++++ benchmarks/cnc/calibrate_cc_difficulty.py | 35 +- benchmarks/cnc/calibrate_hard_regime.py | 284 ++++++++ benchmarks/cnc/conquer_parallel.py | 131 +++- benchmarks/cnc/contracts/hard-regime-v1.yaml | 113 ++++ benchmarks/cnc/hard_regime.py | 436 +++++++++++++ benchmarks/cnc/hard_regime_matrix.py | 538 +++++++++++++++ benchmarks/cnc/hpc/build_matrix.sh | 25 + benchmarks/cnc/hpc/calibrate.sh | 27 + benchmarks/cnc/hpc/materialize.sh | 20 + benchmarks/cnc/hpc/report.sh | 35 + benchmarks/cnc/hpc/run_cell.sh | 27 + benchmarks/cnc/hpc/smoke.sh | 19 + benchmarks/cnc/requirements.txt | 2 + benchmarks/cnc/run_hard_regime_cell.py | 620 ++++++++++++++++++ benchmarks/cnc/verify_hard_regime.py | 440 +++++++++++++ cnc-tools/Makefile | 45 +- cnc-tools/README.md | 8 +- src/bin/cnc_cuber.rs | 77 ++- src/cube.rs | 15 + tests/cnc_cuber_trace.rs | 82 +++ tests/test_aggregate_hard_regime.py | 91 +++ tests/test_calibrate_hard_regime.py | 47 ++ tests/test_conquer_parallel.py | 41 ++ tests/test_hard_regime.py | 73 +++ tests/test_hard_regime_matrix.py | 226 +++++++ tests/test_run_hard_regime_cell.py | 147 +++++ tests/test_verify_hard_regime.py | 133 ++++ 30 files changed, 4457 insertions(+), 52 deletions(-) create mode 100755 benchmarks/cnc/aggregate_hard_regime.py create mode 100755 benchmarks/cnc/calibrate_hard_regime.py create mode 100644 benchmarks/cnc/contracts/hard-regime-v1.yaml create mode 100755 benchmarks/cnc/hard_regime.py create mode 100755 benchmarks/cnc/hard_regime_matrix.py create mode 100755 benchmarks/cnc/hpc/build_matrix.sh create mode 100755 benchmarks/cnc/hpc/calibrate.sh create mode 100755 benchmarks/cnc/hpc/materialize.sh create mode 100755 benchmarks/cnc/hpc/report.sh create mode 100755 benchmarks/cnc/hpc/run_cell.sh create mode 100755 benchmarks/cnc/hpc/smoke.sh create mode 100644 benchmarks/cnc/requirements.txt create mode 100755 benchmarks/cnc/run_hard_regime_cell.py create mode 100755 benchmarks/cnc/verify_hard_regime.py create mode 100644 tests/test_aggregate_hard_regime.py create mode 100644 tests/test_calibrate_hard_regime.py create mode 100644 tests/test_hard_regime.py create mode 100644 tests/test_hard_regime_matrix.py create mode 100644 tests/test_run_hard_regime_cell.py create mode 100644 tests/test_verify_hard_regime.py diff --git a/.gitignore b/.gitignore index 6b744f9..fadbd5b 100644 --- a/.gitignore +++ b/.gitignore @@ -13,6 +13,7 @@ # Local research and experiment state. /.knowledge/ /experiments/ +/artifacts/ # Python caches. __pycache__/ diff --git a/benchmarks/cnc/README.md b/benchmarks/cnc/README.md index 203983a..d13ce3b 100644 --- a/benchmarks/cnc/README.md +++ b/benchmarks/cnc/README.md @@ -1,5 +1,123 @@ # Auditable Cube-and-Conquer measurements +## Issue #51 hard-UNSAT factoring study + +The frozen product-width study contract is +`contracts/hard-regime-v1.yaml`. Product width and factor-input width are +separate required fields: the declared pairs are 64/32, 72/36, and 80/40. +Targets are deterministic full-product-width primes inside the reachable +unsigned multiplier range and above the factor-input range, so each pinned +instance is UNSAT by construction. + +Generate and independently verify the 39 declared targets before materializing +the array-ripple CircuitSAT/CNF pairs: + +```bash +python3 -m benchmarks.cnc.hard_regime generate-targets \ + benchmarks/cnc/contracts/hard-regime-v1.yaml \ + --out artifacts/cnc-hard-regime/targets.jsonl +python3 -m benchmarks.cnc.hard_regime verify-targets \ + benchmarks/cnc/contracts/hard-regime-v1.yaml \ + artifacts/cnc-hard-regime/targets.jsonl +python3 -m benchmarks.cnc.hard_regime materialize \ + benchmarks/cnc/contracts/hard-regime-v1.yaml \ + --out-dir artifacts/cnc-hard-regime/instances +``` + +Raw run artifacts remain outside Git. The contract, checksummed artifact +manifest, aggregate statistics, primary table, and report are the reviewable +payload. + +Freeze a selector's three thresholds from exactly the three calibration +instances at one product width. This produces one shared threshold per band; +it never searches on a held-out instance: + +```bash +python3 -m benchmarks.cnc.calibrate_hard_regime \ + benchmarks/cnc/contracts/hard-regime-v1.yaml \ + artifacts/cnc-hard-regime/instances/manifest.jsonl \ + --product-width 64 --selector region \ + --cuber target/release/cnc_cuber \ + --out-dir artifacts/cnc-hard-regime/calibration/p64/region +``` + +After all six width/selector calibration locks exist, capture the exact tool +binaries and build the immutable 249-cell matrix (39 hardness rows plus 210 +held-out CnC rows): + +```bash +python3 -m benchmarks.cnc.hard_regime_matrix lock-toolchain \ + benchmarks/cnc/contracts/hard-regime-v1.yaml \ + --cuber target/release/cnc_cuber \ + --kissat cnc-tools/bin/kissat-8af8e56f174b778aef3aa45af9f739b2a5f492c2 \ + --march-cu cnc-tools/bin/march_cu-705b60c6491ef2b61988b3ce6ac674be1b90571d \ + --repository-revision "$(git rev-parse HEAD)" \ + --out artifacts/cnc-hard-regime/toolchain.json +python3 -m benchmarks.cnc.hard_regime_matrix build-matrix \ + benchmarks/cnc/contracts/hard-regime-v1.yaml \ + artifacts/cnc-hard-regime/instances/manifest.jsonl \ + --calibration-root artifacts/cnc-hard-regime/calibration \ + --toolchain artifacts/cnc-hard-regime/toolchain.json \ + --out artifacts/cnc-hard-regime/run-matrix.json +``` + +Every Slurm task executes exactly one immutable cell and writes +`cells//terminal.json` last. Re-running a completed cell is idempotent; +timeouts and errors are terminal outcomes rather than missing rows: + +```bash +python3 -m benchmarks.cnc.run_hard_regime_cell \ + benchmarks/cnc/contracts/hard-regime-v1.yaml \ + artifacts/cnc-hard-regime/run-matrix.json \ + artifacts/cnc-hard-regime/toolchain.json \ + --cell-id CELL_ID \ + --instance-root artifacts/cnc-hard-regime/instances \ + --output-root artifacts/cnc-hard-regime/runs +``` + +Verify either the pilot gate (all 39 monolithic rows plus the 72 pilot cells) +or the complete 249-cell table. The verifier reconstructs raw scheduling and +rejects missing cells, width confusion, split leakage, per-test thresholds, +mixed CNFs, malformed branch assignments/refutation reasons, and censored cubes +omitted from work/span accounting. Harness-error terminals remain visible and +are explicitly excluded from any claim that per-cube work/span reconstructed: + +```bash +python3 -m benchmarks.cnc.verify_hard_regime \ + benchmarks/cnc/contracts/hard-regime-v1.yaml \ + artifacts/cnc-hard-regime/instances/manifest.jsonl \ + --calibration-root artifacts/cnc-hard-regime/calibration \ + --toolchain artifacts/cnc-hard-regime/toolchain.json \ + --matrix artifacts/cnc-hard-regime/run-matrix.json \ + --runs-root artifacts/cnc-hard-regime/runs --scope pilot +``` + +Generate raw paired rows, log-log frontier-budget interpolation on common +support, deterministic instance bootstrap intervals, the primary table, and a +short report only after verification succeeds: + +```bash +python3 -m benchmarks.cnc.aggregate_hard_regime \ + benchmarks/cnc/contracts/hard-regime-v1.yaml \ + artifacts/cnc-hard-regime/run-matrix.json \ + --runs-root artifacts/cnc-hard-regime/runs --scope pilot \ + --out-dir artifacts/cnc-hard-regime/pilot-report +``` + +Pass `--aggregate artifacts/cnc-hard-regime/pilot-report/aggregate.json` to the +verifier for a final regeneration check that also rejects duplicate instance +IDs in any bootstrap group. + +## HPC2 execution + +The scripts in `hpc/` contain no hidden Slurm resource defaults. Supply every +partition, walltime, task/CPU count, memory limit, array range, working +directory, environment export, and stdout/stderr path explicitly to `sbatch`. +Use `BI51_SCRATCH_ROOT` on the HPC2 `jhspoolers` SSD for temporary per-cube CNF +files; persistent instances, raw results, and reports remain under +`BI51_ARTIFACT_ROOT` on HDD. Validate `smoke.sh` in the free `debug` partition +before materialization, calibration, or a run-cell array. + ## Current CnC comparison The primary comparison uses online stopping rules on both sides: diff --git a/benchmarks/cnc/aggregate_hard_regime.py b/benchmarks/cnc/aggregate_hard_regime.py new file mode 100755 index 0000000..824ef99 --- /dev/null +++ b/benchmarks/cnc/aggregate_hard_regime.py @@ -0,0 +1,653 @@ +#!/usr/bin/env python3 +"""Aggregate issue #51 terminal cells with held-out instances as the sample unit.""" + +from __future__ import annotations + +import argparse +import csv +import hashlib +import json +import math +import random +from pathlib import Path +from typing import Any + +from benchmarks.cnc.hard_regime import HardRegimeError, contract_sha256, load_contract +from benchmarks.cnc.verify_hard_regime import selected_cells +from benchmarks.pipeline.circuit import load_json, write_json, write_jsonl + + +class AggregateError(HardRegimeError): + """Terminal records cannot produce the preregistered instance-level analysis.""" + + +METRICS = { + "conquer_work_cpu_s": "Conquer work", + "conquer_span_s": "Maximum cube time", + "end_to_end_wall_s": "End-to-end wall time", + "measured_32_worker_makespan_s": "Measured 32-worker makespan", + "lpt_32_s": "LPT 32-worker makespan", + "lpt_128_s": "LPT 128-worker makespan", + "lpt_512_s": "LPT 512-worker makespan", +} + + +def metric_value(terminal: dict[str, Any], metric: str) -> float | None: + metrics = terminal.get("metrics") + if not isinstance(metrics, dict): + return None + if metric.startswith("lpt_"): + workers = metric.split("_")[1] + replay = metrics.get("lpt_makespan_by_workers_s") + value = replay.get(workers) if isinstance(replay, dict) else None + else: + value = metrics.get(metric) + if not isinstance(value, (int, float)) or isinstance(value, bool) or value <= 0: + return None + return float(value) + + +def geometric_mean(values: list[float]) -> float: + if not values or any(value <= 0 for value in values): + raise AggregateError("geometric mean requires positive observations") + return math.exp(sum(math.log(value) for value in values) / len(values)) + + +def percentile(values: list[float], probability: float) -> float: + ordered = sorted(values) + index = min(len(ordered) - 1, max(0, round(probability * (len(ordered) - 1)))) + return ordered[index] + + +def bootstrap_ci( + values: list[float], samples: int, confidence: float, seed: int, key: str +) -> list[float] | None: + if not values: + return None + key_seed = int(hashlib.sha256(key.encode()).hexdigest()[:16], 16) + rng = random.Random(seed ^ key_seed) + replicates = [ + geometric_mean([values[rng.randrange(len(values))] for _ in values]) + for _ in range(samples) + ] + alpha = (1.0 - confidence) / 2.0 + return [percentile(replicates, alpha), percentile(replicates, 1.0 - alpha)] + + +def interpolate_log(points: list[tuple[float, float]], target: float) -> float | None: + usable = sorted((x, y) for x, y in points if x > 0 and y > 0) + if not usable or target < usable[0][0] or target > usable[-1][0]: + return None + for x, y in usable: + if math.isclose(x, target, rel_tol=1e-12, abs_tol=0.0): + return y + for (left_x, left_y), (right_x, right_y) in zip(usable, usable[1:]): + if left_x <= target <= right_x: + if left_x == right_x: + return geometric_mean([left_y, right_y]) + fraction = (math.log(target) - math.log(left_x)) / ( + math.log(right_x) - math.log(left_x) + ) + return math.exp(math.log(left_y) + fraction * (math.log(right_y) - math.log(left_y))) + return None + + +def load_terminals( + matrix: dict[str, Any], runs_root: Path, scope: str +) -> dict[str, dict[str, Any]]: + terminals = {} + for cell in selected_cells(matrix, scope): + path = runs_root / "cells" / cell["cell_id"] / "terminal.json" + if not path.is_file(): + raise AggregateError(f"missing terminal cell {cell['cell_id']}") + terminal = load_json(path) + if terminal.get("cell_id") != cell["cell_id"]: + raise AggregateError(f"terminal identity mismatch for {cell['cell_id']}") + terminals[cell["cell_id"]] = terminal + return terminals + + +def paired_raw_observations( + matrix: dict[str, Any], terminals: dict[str, dict[str, Any]], metric: str +) -> list[dict[str, Any]]: + cells = { + (cell["instance_id"], cell["method"], cell["budget"]): cell + for cell in matrix["cells"] + if cell["split"] == "held_out" + } + observations = [] + for instance_id in sorted({key[0] for key in cells}): + for budget in ("low", "medium", "high"): + region = cells.get((instance_id, "region-cc", budget)) + blind = cells.get((instance_id, "structure-blind-cc", budget)) + if region is None or blind is None: + continue + region_terminal = terminals.get(region["cell_id"]) + blind_terminal = terminals.get(blind["cell_id"]) + if region_terminal is None or blind_terminal is None: + continue + region_value = metric_value(region_terminal, metric) + blind_value = metric_value(blind_terminal, metric) + complete = ( + region_terminal.get("state") == "complete" + and blind_terminal.get("state") == "complete" + and region_value is not None + and blind_value is not None + ) + observations.append( + { + "analysis": "raw-nominal-budget", + "instance_id": instance_id, + "product_width": region["product_width"], + "budget": budget, + "metric": metric, + "region_state": region_terminal.get("state"), + "blind_state": blind_terminal.get("state"), + "region_value": region_value, + "blind_value": blind_value, + "ratio": region_value / blind_value if complete else None, + "complete_pair": complete, + } + ) + return observations + + +def adjusted_observations( + matrix: dict[str, Any], terminals: dict[str, dict[str, Any]], metric: str +) -> list[dict[str, Any]]: + cells = { + (cell["instance_id"], cell["method"], cell["budget"]): cell + for cell in matrix["cells"] + if cell["split"] == "held_out" + } + observations = [] + declared_instances = sorted( + { + cell["instance_id"] + for cell in matrix["cells"] + if cell["split"] == "held_out" + and cell["method"] in {"region-cc", "structure-blind-cc"} + and cell["cell_id"] in terminals + } + ) + for instance_id in declared_instances: + series = {} + product_width = next( + cell["product_width"] + for cell in matrix["cells"] + if cell["instance_id"] == instance_id + ) + for method in ("region-cc", "structure-blind-cc"): + points = [] + states = [] + for budget in ("low", "medium", "high"): + cell = cells.get((instance_id, method, budget)) + if cell is None or cell["cell_id"] not in terminals: + continue + product_width = cell["product_width"] + terminal = terminals[cell["cell_id"]] + states.append(terminal.get("state")) + x = terminal.get("metrics", {}).get("frontier_size") + y = metric_value(terminal, metric) + if terminal.get("state") == "complete" and isinstance(x, (int, float)) and x > 0 and y is not None: + points.append((float(x), y)) + series[method] = {"points": points, "states": states} + region_points = series["region-cc"]["points"] + blind_points = series["structure-blind-cc"]["points"] + if not region_points or not blind_points: + support = None + adjustment_status = "incomplete-series" + else: + lower = max(min(x for x, _ in region_points), min(x for x, _ in blind_points)) + upper = min(max(x for x, _ in region_points), max(x for x, _ in blind_points)) + support = (lower, upper) if lower <= upper else None + adjustment_status = "common-support" if support is not None else "no-common-support" + grids = ( + {} + if support is None + else { + "overlap-low": support[0], + "overlap-mid": math.sqrt(support[0] * support[1]), + "overlap-high": support[1], + } + ) + for grid in ("overlap-low", "overlap-mid", "overlap-high"): + frontier_size = grids.get(grid) + region_value = ( + interpolate_log(region_points, frontier_size) + if frontier_size is not None + else None + ) + blind_value = ( + interpolate_log(blind_points, frontier_size) + if frontier_size is not None + else None + ) + complete = region_value is not None and blind_value is not None + observations.append( + { + "analysis": "frontier-budget-adjusted", + "instance_id": instance_id, + "product_width": product_width, + "grid": grid, + "common_frontier_size": frontier_size, + "metric": metric, + "adjustment_status": adjustment_status, + "region_states": series["region-cc"]["states"], + "blind_states": series["structure-blind-cc"]["states"], + "region_value": region_value, + "blind_value": blind_value, + "ratio": region_value / blind_value if complete else None, + "complete_pair": complete, + } + ) + return observations + + +def summarize_observations( + observations: list[dict[str, Any]], + analysis: str, + metric: str, + budget_key: str, + budget_value: str, + product_width: int | None, + bootstrap: dict[str, Any], +) -> dict[str, Any]: + selected = [ + row + for row in observations + if row["analysis"] == analysis + and row["metric"] == metric + and row.get(budget_key) == budget_value + and (product_width is None or row["product_width"] == product_width) + ] + complete = [row for row in selected if row["complete_pair"]] + ratios = [float(row["ratio"]) for row in complete] + instance_ids = [row["instance_id"] for row in complete] + declared_instance_ids = [row["instance_id"] for row in selected] + if len(instance_ids) != len(set(instance_ids)): + raise AggregateError("an aggregate group contains duplicate instance observations") + group = "overall" if product_width is None else f"p{product_width}" + key = f"{analysis}:{metric}:{budget_key}={budget_value}:{group}" + geometric_ratio = geometric_mean(ratios) if ratios else None + if metric == "conquer_work_cpu_s": + criterion = "ratio-no-worse-than-1.10" + meets_criterion = geometric_ratio is not None and geometric_ratio <= 1.10 + elif metric == "conquer_span_s": + criterion = "ratio-below-1.00" + meets_criterion = geometric_ratio is not None and geometric_ratio < 1.0 + else: + criterion = None + meets_criterion = None + return { + "analysis": analysis, + "metric": metric, + budget_key: budget_value, + "group": group, + "statistical_unit": "held-out-instance", + "declared_pairs": len(selected), + "complete_pairs": len(complete), + "censored_or_failed_pairs": len(selected) - len(complete), + "declared_instance_ids": declared_instance_ids, + "censored_or_failed_instance_ids": [ + row["instance_id"] for row in selected if not row["complete_pair"] + ], + "instance_ids": instance_ids, + "geometric_mean_ratio": geometric_ratio, + "region_win_rate": ( + sum(ratio < 1.0 for ratio in ratios) / len(ratios) if ratios else None + ), + "bootstrap_95_ci": bootstrap_ci( + ratios, + int(bootstrap["samples"]), + float(bootstrap["confidence"]), + int(bootstrap["seed"]), + key, + ), + "scientific_target": criterion, + "meets_scientific_target": meets_criterion, + } + + +def terminal_cell_rows( + matrix: dict[str, Any], terminals: dict[str, dict[str, Any]], scope: str +) -> list[dict[str, Any]]: + rows = [] + for cell in selected_cells(matrix, scope): + terminal = terminals[cell["cell_id"]] + rows.append( + { + "cell_id": cell["cell_id"], + "instance_id": cell["instance_id"], + "split": cell["split"], + "product_width": cell["product_width"], + "factor_input_width": cell["factor_input_width"], + "pilot": cell["pilot"], + "method": cell["method"], + "budget": cell["budget"], + "state": terminal["state"], + "verdict": terminal.get("verdict"), + "metrics": terminal.get("metrics"), + } + ) + return rows + + +def method_summaries(cell_rows: list[dict[str, Any]]) -> list[dict[str, Any]]: + summaries = [] + method_budgets = sorted({(row["method"], row["budget"]) for row in cell_rows}) + for method, budget in method_budgets: + for width in (None, 64, 72, 80): + rows = [ + row + for row in cell_rows + if row["method"] == method + and row["budget"] == budget + and row["split"] == "held_out" + and (width is None or row["product_width"] == width) + ] + if not rows: + continue + states: dict[str, int] = {} + for row in rows: + states[row["state"]] = states.get(row["state"], 0) + 1 + numeric = {} + metric_names = ( + "solver_wall_s", + "conquer_work_cpu_s", + "conquer_span_s", + "measured_32_worker_makespan_s", + "end_to_end_wall_s", + ) + for metric in metric_names: + values = [ + float(row["metrics"][metric]) + for row in rows + if row["state"] == "complete" + and isinstance(row.get("metrics"), dict) + and isinstance(row["metrics"].get(metric), (int, float)) + and row["metrics"][metric] > 0 + ] + numeric[metric] = { + "complete_values": len(values), + "geometric_mean": geometric_mean(values) if values else None, + } + summaries.append( + { + "method": method, + "budget": budget, + "group": "overall" if width is None else f"p{width}", + "declared_instances": len(rows), + "terminal_states": states, + "metrics": numeric, + } + ) + return summaries + + +def hardness_summary(matrix: dict[str, Any], terminals: dict[str, dict[str, Any]]) -> list[dict[str, Any]]: + rows = [] + for width in (64, 72, 80): + cells = [ + cell + for cell in matrix["cells"] + if cell["method"] == "monolithic-kissat" + and cell["split"] == "held_out" + and cell["product_width"] == width + ] + states = [terminals[cell["cell_id"]]["state"] for cell in cells] + timeouts = sum(state == "monolithic-timeout" for state in states) + rows.append( + { + "product_width": width, + "held_out_instances": len(cells), + "monolithic_timeouts": timeouts, + "cnc_regime": timeouts > len(cells) / 2, + } + ) + return rows + + +def pilot_gate( + matrix: dict[str, Any], terminals: dict[str, dict[str, Any]], hardness: list[dict[str, Any]] +) -> dict[str, Any]: + regime_widths = { + row["product_width"] for row in hardness if row["cnc_regime"] + } + frontier_methods = [] + for width in sorted(regime_widths): + for method in ("march-cu-dynamic", "region-cc", "structure-blind-cc"): + cells = [ + cell + for cell in matrix["cells"] + if cell.get("pilot") + and cell["product_width"] == width + and cell["method"] == method + ] + if cells and all( + terminals[cell["cell_id"]].get("stages", {}).get("cubing", {}).get("complete") + for cell in cells + ): + frontier_methods.append({"product_width": width, "method": method}) + artifact_reconstruction = all( + terminal.get("state") != "harness-error" for terminal in terminals.values() + ) + return { + "at_least_two_hard_widths": len(regime_widths) >= 2, + "hard_widths": sorted(regime_widths), + "complete_frontier_methods_in_regime": frontier_methods, + "at_least_one_complete_frontier_method": bool(frontier_methods), + "artifact_reconstruction": artifact_reconstruction, + "passed": ( + len(regime_widths) >= 2 + and bool(frontier_methods) + and artifact_reconstruction + ), + } + + +def write_primary_table(path: Path, summaries: list[dict[str, Any]]) -> None: + relevant = [ + row + for row in summaries + if row["analysis"] == "raw-nominal-budget" + and row["metric"] in {"conquer_work_cpu_s", "conquer_span_s"} + and row["group"] != "overall" + ] + lookup = { + (row["group"], row["budget"], row["metric"]): row for row in relevant + } + lines = [ + "| Product width | Budget | Complete pairs | Work ratio [95% CI] | Span ratio [95% CI] | Span win rate |", + "|---:|:---|---:|:---|:---|---:|", + ] + + def ratio(row: dict[str, Any] | None) -> str: + if row is None or row["geometric_mean_ratio"] is None: + return "NA" + interval = row["bootstrap_95_ci"] + return f"{row['geometric_mean_ratio']:.3f} [{interval[0]:.3f}, {interval[1]:.3f}]" + + for width in (64, 72, 80): + for budget in ("low", "medium", "high"): + work = lookup.get((f"p{width}", budget, "conquer_work_cpu_s")) + span = lookup.get((f"p{width}", budget, "conquer_span_s")) + complete = min( + work["complete_pairs"] if work else 0, + span["complete_pairs"] if span else 0, + ) + win = "NA" if span is None or span["region_win_rate"] is None else f"{span['region_win_rate']:.1%}" + lines.append( + f"| {width} | {budget} | {complete} | {ratio(work)} | {ratio(span)} | {win} |" + ) + path.write_text("\n".join(lines) + "\n", encoding="utf-8") + + +def write_raw_csv(path: Path, observations: list[dict[str, Any]]) -> None: + rows = [row for row in observations if row["analysis"] == "raw-nominal-budget"] + fields = [ + "instance_id", + "product_width", + "budget", + "metric", + "region_state", + "blind_state", + "region_value", + "blind_value", + "ratio", + "complete_pair", + ] + with path.open("w", newline="", encoding="utf-8") as stream: + writer = csv.DictWriter(stream, fieldnames=fields, extrasaction="ignore") + writer.writeheader() + writer.writerows(rows) + + +def write_report( + path: Path, + scope: str, + hardness: list[dict[str, Any]], + gate: dict[str, Any], + summaries: list[dict[str, Any]], + methods: list[dict[str, Any]], +) -> None: + complete_raw = sum( + row["complete_pairs"] + for row in summaries + if row["analysis"] == "raw-nominal-budget" and row["group"] == "overall" + ) + complete_adjusted = sum( + row["complete_pairs"] + for row in summaries + if row["analysis"] == "frontier-budget-adjusted" and row["group"] == "overall" + ) + lines = [ + "# Hard-regime CnC work/span report", + "", + f"Scope: `{scope}`. Statistical unit: held-out factoring instance.", + "", + "## Hardness gate", + "", + ] + lines.extend( + f"- Product width {row['product_width']}: {row['monolithic_timeouts']}/{row['held_out_instances']} monolithic timeouts; CnC regime = {row['cnc_regime']}." + for row in hardness + ) + lines.extend( + [ + "", + f"Pilot gate passed: **{gate['passed']}**.", + "", + "## Raw paired results", + "", + f"Complete overall instance-pairs across metrics/budgets: {complete_raw}. Censored and failed pairs remain in the machine-readable rows.", + "", + "## Frontier-budget-adjusted results", + "", + f"Complete interpolated overall instance-pairs across metrics/grid points: {complete_adjusted}. Interpolation is log-log within common support only; no extrapolation is used.", + "", + "## Finite-worker results", + "", + "LPT makespans are reported for 32, 128, and 512 workers. The directly measured 32-worker schedule remains separate from replayed LPT estimates.", + ] + ) + lines.extend(["", "## Method terminal overview", ""]) + for row in methods: + if row["group"] != "overall": + continue + end_to_end = row["metrics"]["end_to_end_wall_s"]["geometric_mean"] + end_text = "NA" if end_to_end is None else f"{end_to_end:.3f}s" + lines.append( + f"- `{row['method']}` / `{row['budget']}`: states={row['terminal_states']}; complete-case geometric mean end-to-end={end_text}." + ) + path.write_text("\n".join(lines) + "\n", encoding="utf-8") + + +def aggregate( + contract: dict[str, Any], + matrix: dict[str, Any], + terminals: dict[str, dict[str, Any]], + scope: str, +) -> dict[str, Any]: + cells = terminal_cell_rows(matrix, terminals, scope) + observations = [] + for metric in METRICS: + observations.extend(paired_raw_observations(matrix, terminals, metric)) + observations.extend(adjusted_observations(matrix, terminals, metric)) + bootstrap = contract["statistics"]["bootstrap"] + summaries = [] + for analysis, budget_key, budget_values in ( + ("raw-nominal-budget", "budget", ("low", "medium", "high")), + ( + "frontier-budget-adjusted", + "grid", + ("overlap-low", "overlap-mid", "overlap-high"), + ), + ): + for metric in METRICS: + for budget_value in budget_values: + for width in (None, 64, 72, 80): + summaries.append( + summarize_observations( + observations, + analysis, + metric, + budget_key, + budget_value, + width, + bootstrap, + ) + ) + hardness = hardness_summary(matrix, terminals) + gate = pilot_gate(matrix, terminals, hardness) + return { + "schema_version": 1, + "kind": "hard-regime-instance-level-aggregate", + "contract_sha256": contract_sha256(contract), + "scope": scope, + "statistical_unit": "held-out-instance", + "bootstrap": bootstrap, + "hardness": hardness, + "pilot_gate": gate, + "method_summaries": method_summaries(cells), + "summaries": summaries, + "terminal_cells": cells, + "observations": observations, + } + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("contract", type=Path) + parser.add_argument("matrix", type=Path) + parser.add_argument("--runs-root", type=Path, required=True) + parser.add_argument("--scope", choices=("pilot", "full"), default="full") + parser.add_argument("--out-dir", type=Path, required=True) + args = parser.parse_args() + try: + contract = load_contract(args.contract) + matrix = load_json(args.matrix) + terminals = load_terminals(matrix, args.runs_root, args.scope) + result = aggregate(contract, matrix, terminals, args.scope) + args.out_dir.mkdir(parents=True, exist_ok=True) + write_json(args.out_dir / "aggregate.json", result) + write_jsonl(args.out_dir / "terminal-cells.jsonl", result["terminal_cells"]) + write_jsonl(args.out_dir / "paired-observations.jsonl", result["observations"]) + write_raw_csv(args.out_dir / "raw-paired-results.csv", result["observations"]) + write_primary_table(args.out_dir / "primary-table.md", result["summaries"]) + write_report( + args.out_dir / "report.md", + args.scope, + result["hardness"], + result["pilot_gate"], + result["summaries"], + result["method_summaries"], + ) + except (AggregateError, OSError, ValueError, json.JSONDecodeError) as exc: + parser.error(str(exc)) + print(f"PASS aggregate: {args.out_dir / 'aggregate.json'}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/benchmarks/cnc/calibrate_cc_difficulty.py b/benchmarks/cnc/calibrate_cc_difficulty.py index a4a44e9..e37ef9e 100644 --- a/benchmarks/cnc/calibrate_cc_difficulty.py +++ b/benchmarks/cnc/calibrate_cc_difficulty.py @@ -45,7 +45,11 @@ def run_cuber( log: Path, max_rows: int, trace: Path | None = None, + selector: str = "region", + timeout_s: float | None = None, ) -> dict[str, int | float]: + if selector not in {"region", "structure-blind"}: + raise CalibrationError(f"unknown selector {selector!r}") command = [ str(cuber.resolve()), str(instance.resolve()), @@ -55,12 +59,28 @@ def run_cuber( str(cubes), "--max-rows", str(max_rows), + "--selector", + selector, ] if trace: command.extend(("--trace", str(trace))) before = resource.getrusage(resource.RUSAGE_CHILDREN) started = time.monotonic() - process = subprocess.run(command, capture_output=True, text=True, check=False) + try: + process = subprocess.run( + command, + capture_output=True, + text=True, + check=False, + timeout=timeout_s, + ) + except subprocess.TimeoutExpired as exc: + elapsed = time.monotonic() - started + stderr = exc.stderr.decode() if isinstance(exc.stderr, bytes) else exc.stderr or "" + log.write_text(stderr, encoding="utf-8") + raise CalibrationError( + f"cuber selector={selector} threshold={threshold} timed out after {elapsed:.3f}s" + ) from exc elapsed = time.monotonic() - started after = resource.getrusage(resource.RUSAGE_CHILDREN) log.write_text(process.stderr, encoding="utf-8") @@ -108,6 +128,8 @@ def calibrate( initial: int, maximum_threshold: int, max_rows: int, + selector: str = "region", + timeout_s: float | None = None, ) -> dict[str, Any]: if target <= 0 or minimum <= 0 or maximum < minimum or initial <= 0: raise CalibrationError("invalid task range or threshold") @@ -125,6 +147,8 @@ def probe(threshold: int) -> dict[str, int | float]: candidates / f"threshold-{threshold}.icnf", candidates / f"threshold-{threshold}.log", max_rows, + selector=selector, + timeout_s=timeout_s, ) return observed[threshold] @@ -152,6 +176,8 @@ def probe(threshold: int) -> dict[str, int | float]: out_dir / "final.log", max_rows, out_dir / "nodes.jsonl", + selector, + timeout_s, ) candidate = candidates / f"threshold-{threshold}.icnf" if sha256_file(candidate) != sha256_file(out_dir / "frontier.icnf"): @@ -159,6 +185,7 @@ def probe(threshold: int) -> dict[str, int | float]: record = { "schema_version": 1, "method": "classical-cc-difficulty-task-count-calibration", + "selector": selector, "formula": "D^2*(D+I)/N > threshold", "instance": str(instance), "instance_sha256": sha256_file(instance), @@ -191,6 +218,10 @@ def main() -> int: parser.add_argument("--initial-threshold", type=int, default=1024) parser.add_argument("--max-threshold", type=int, default=1 << 60) parser.add_argument("--max-rows", type=int, default=512) + parser.add_argument( + "--selector", choices=("region", "structure-blind"), default="region" + ) + parser.add_argument("--timeout-s", type=float) args = parser.parse_args() try: result = calibrate( @@ -203,6 +234,8 @@ def main() -> int: args.initial_threshold, args.max_threshold, args.max_rows, + args.selector, + args.timeout_s, ) except (CalibrationError, OSError) as exc: parser.error(str(exc)) diff --git a/benchmarks/cnc/calibrate_hard_regime.py b/benchmarks/cnc/calibrate_hard_regime.py new file mode 100755 index 0000000..59b9e5c --- /dev/null +++ b/benchmarks/cnc/calibrate_hard_regime.py @@ -0,0 +1,284 @@ +#!/usr/bin/env python3 +"""Freeze one CC threshold per product-width, selector, and frontier band.""" + +from __future__ import annotations + +import argparse +import json +import math +import statistics +from pathlib import Path +from typing import Any, Callable + +from benchmarks.cnc.calibrate_cc_difficulty import ( + CalibrationError, + run_cuber, + sha256_file, +) +from benchmarks.cnc.hard_regime import ( + HardRegimeError, + contract_sha256, + load_contract, +) +from benchmarks.pipeline.circuit import read_jsonl, write_json + + +def task_counts(row: dict[str, Any]) -> list[int]: + instances = row.get("instances") + if not isinstance(instances, list) or not instances: + raise CalibrationError("width-level response row has no instances") + counts = [] + for instance in instances: + if not isinstance(instance, dict) or not isinstance(instance.get("tasks"), int): + raise CalibrationError("width-level response has a malformed task count") + counts.append(instance["tasks"]) + return counts + + +def median_tasks(row: dict[str, Any]) -> float: + return float(statistics.median(task_counts(row))) + + +def calibration_loss(row: dict[str, Any], target: int) -> tuple[float, float]: + errors = [abs(math.log2(max(1, count) / target)) for count in task_counts(row)] + return float(statistics.median(errors)), max(errors) + + +def choose_width_response( + rows: list[dict[str, Any]], target: int, minimum: int, maximum: int +) -> dict[str, Any]: + if not rows: + raise CalibrationError("empty width-level cutoff response") + inside = [row for row in rows if minimum <= median_tasks(row) <= maximum] + if not inside: + raise CalibrationError( + f"no calibrated threshold reaches accepted task range [{minimum}, {maximum}]" + ) + return min( + inside, + key=lambda row: ( + *calibration_loss(row, target), + abs(median_tasks(row) - target), + int(row["threshold"]), + ), + ) + + +def calibration_inputs( + contract: dict[str, Any], manifest_path: Path, product_width: int +) -> list[dict[str, Any]]: + root = manifest_path.parent + digest = contract_sha256(contract) + selected = [] + for record in read_jsonl(manifest_path): + if record.get("product_width") != product_width or record.get("split") != "calibration": + continue + if record.get("factor_input_width") * 2 != product_width: + raise CalibrationError(f"{record.get('id')}: factor/product width confusion") + if record.get("architecture") != "array-ripple": + raise CalibrationError(f"{record.get('id')}: calibration must use array-ripple") + if record.get("contract_sha256") != digest: + raise CalibrationError(f"{record.get('id')}: contract hash mismatch") + relative = record.get("circuitsat") + if not isinstance(relative, str): + raise CalibrationError(f"{record.get('id')}: missing CircuitSAT path") + instance = (root / relative).resolve() + if sha256_file(instance) != record.get("circuitsat_sha256"): + raise CalibrationError(f"{record.get('id')}: CircuitSAT hash mismatch") + selected.append( + { + "id": record["id"], + "path": instance, + "sha256": record["circuitsat_sha256"], + } + ) + selected.sort(key=lambda row: row["id"]) + if len(selected) != 3: + raise CalibrationError( + f"product width {product_width} needs exactly 3 calibration instances" + ) + return selected + + +def calibrate_width( + *, + contract: dict[str, Any], + instances: list[dict[str, Any]], + product_width: int, + selector: str, + cuber: Path, + out_dir: Path, + initial_threshold: int = 1024, + maximum_threshold: int = 1 << 120, + probe_runner: Callable[..., dict[str, int | float]] = run_cuber, +) -> dict[str, Any]: + if selector not in {"region", "structure-blind"}: + raise CalibrationError(f"unsupported selector {selector!r}") + if len(instances) != 3: + raise CalibrationError("width-level calibration requires exactly three instances") + if initial_threshold <= 0 or maximum_threshold < initial_threshold: + raise CalibrationError("invalid calibration threshold range") + method = "region-cc" if selector == "region" else "structure-blind-cc" + max_rows = int(contract["methods"][method]["max_rows"]) + timeout_s = float(contract["limits_seconds"]["cubing"]) + out_dir.mkdir(parents=True, exist_ok=True) + observed: dict[int, dict[str, Any]] = {} + + def probe(threshold: int) -> dict[str, Any]: + if threshold in observed: + return observed[threshold] + threshold_dir = out_dir / "candidates" / f"threshold-{threshold}" + threshold_dir.mkdir(parents=True, exist_ok=True) + rows = [] + for instance in instances: + result = probe_runner( + cuber, + instance["path"], + threshold, + threshold_dir / f"{instance['id']}.icnf", + threshold_dir / f"{instance['id']}.log", + max_rows, + selector=selector, + timeout_s=timeout_s, + ) + rows.append( + { + "id": instance["id"], + "tasks": int(result["tasks"]), + "elapsed_s": float(result["elapsed_s"]), + "cpu_s": float(result["user_s"]) + float(result["system_s"]), + } + ) + observed[threshold] = {"threshold": threshold, "instances": rows} + return observed[threshold] + + bands = contract["frontier_bands"] + brackets: dict[str, tuple[int, int]] = {} + for band, spec in bands.items(): + target = int(spec["center_cubes"]) + lower, upper = 0, initial_threshold + probe(lower) + while median_tasks(probe(upper)) < target and upper < maximum_threshold: + lower, upper = upper, min(upper * 2, maximum_threshold) + if median_tasks(probe(upper)) < target: + raise CalibrationError( + f"maximum threshold did not reach {band} target {target}" + ) + while upper - lower > 1: + middle = (lower + upper) // 2 + if median_tasks(probe(middle)) >= target: + upper = middle + else: + lower = middle + brackets[band] = (lower, upper) + + response = sorted(observed.values(), key=lambda row: int(row["threshold"])) + selections = {} + for band, spec in bands.items(): + target = int(spec["center_cubes"]) + minimum = math.ceil(target * float(spec["accepted_ratio"][0])) + maximum = math.floor(target * float(spec["accepted_ratio"][1])) + selected = choose_width_response(response, target, minimum, maximum) + threshold = int(selected["threshold"]) + final_rows = [] + for instance in instances: + final_dir = out_dir / "selected" / band / instance["id"] + final_dir.mkdir(parents=True, exist_ok=True) + final = probe_runner( + cuber, + instance["path"], + threshold, + final_dir / "frontier.icnf", + final_dir / "cuber.log", + max_rows, + final_dir / "nodes.jsonl", + selector, + timeout_s, + ) + candidate = ( + out_dir + / "candidates" + / f"threshold-{threshold}" + / f"{instance['id']}.icnf" + ) + frontier = final_dir / "frontier.icnf" + if sha256_file(candidate) != sha256_file(frontier): + raise CalibrationError( + f"{instance['id']}: traced rerun changed selected frontier" + ) + final_rows.append( + { + "id": instance["id"], + "tasks": int(final["tasks"]), + "frontier_sha256": sha256_file(frontier), + "trace_sha256": sha256_file(final_dir / "nodes.jsonl"), + "cubing_elapsed_s": float(final["elapsed_s"]), + "cubing_cpu_s": float(final["user_s"]) + + float(final["system_s"]), + } + ) + selections[band] = { + "target_tasks": target, + "accepted_task_range": [minimum, maximum], + "selected_threshold": threshold, + "search_bracket": list(brackets[band]), + "median_tasks": median_tasks(selected), + "calibration_loss": calibration_loss(selected, target)[0], + "within_target_range": minimum <= median_tasks(selected) <= maximum, + "instances": final_rows, + } + + record = { + "schema_version": 1, + "kind": "width-level-cc-calibration-lock", + "contract_sha256": contract_sha256(contract), + "product_width": product_width, + "selector": selector, + "method": method, + "max_rows": max_rows, + "cuber_sha256": sha256_file(cuber), + "calibration_instances": [ + {"id": instance["id"], "sha256": instance["sha256"]} + for instance in instances + ], + "bands": selections, + "response": response, + } + write_json(out_dir / "calibration-lock.json", record) + return record + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("contract", type=Path) + parser.add_argument("manifest", type=Path) + parser.add_argument("--product-width", type=int, choices=(64, 72, 80), required=True) + parser.add_argument( + "--selector", choices=("region", "structure-blind"), required=True + ) + parser.add_argument("--cuber", type=Path, required=True) + parser.add_argument("--out-dir", type=Path, required=True) + parser.add_argument("--initial-threshold", type=int, default=1024) + parser.add_argument("--maximum-threshold", type=int, default=1 << 120) + args = parser.parse_args() + try: + contract = load_contract(args.contract) + instances = calibration_inputs(contract, args.manifest, args.product_width) + result = calibrate_width( + contract=contract, + instances=instances, + product_width=args.product_width, + selector=args.selector, + cuber=args.cuber, + out_dir=args.out_dir, + initial_threshold=args.initial_threshold, + maximum_threshold=args.maximum_threshold, + ) + except (CalibrationError, HardRegimeError, OSError, ValueError) as exc: + parser.error(str(exc)) + print(json.dumps(result, indent=2, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/benchmarks/cnc/conquer_parallel.py b/benchmarks/cnc/conquer_parallel.py index 68c68da..e119c1b 100644 --- a/benchmarks/cnc/conquer_parallel.py +++ b/benchmarks/cnc/conquer_parallel.py @@ -5,6 +5,7 @@ import argparse import concurrent.futures +import hashlib import json import math import os @@ -70,17 +71,17 @@ def percentile(values: list[float], quantile: float) -> float: return ordered[index] -def distribution(values: list[float]) -> dict[str, float]: +def distribution(values: list[float]) -> dict[str, float | None]: if not values: return { "total": 0.0, - "mean": math.nan, - "cv": math.nan, - "p50": math.nan, - "p95": math.nan, - "p99": math.nan, - "p99_over_p95": math.nan, - "max": math.nan, + "mean": None, + "cv": None, + "p50": None, + "p95": None, + "p99": None, + "p99_over_p95": None, + "max": None, } ordered = sorted(values) @@ -99,11 +100,11 @@ def ordered_percentile(quantile: float) -> float: return { "total": total, "mean": mean, - "cv": math.sqrt(variance) / mean if mean else math.nan, + "cv": math.sqrt(variance) / mean if mean else None, "p50": ordered_percentile(0.50), "p95": p95, "p99": p99, - "p99_over_p95": p99 / p95 if p95 > 0 else math.nan, + "p99_over_p95": p99 / p95 if p95 > 0 else None, "max": ordered[-1], } @@ -132,14 +133,26 @@ def _configure_worker( _TMPDIR = tmpdir -def _solve_cube(task: tuple[str, int, list[int]]) -> dict[str, Any]: - arm, index, cube = task +def _solve_cube(task: tuple[str, int, list[int], int]) -> dict[str, Any]: + arm, index, cube, released_ns = task header_fields = _BASE_HEADER.split() clause_count = int(header_fields[3]) + len(cube) header = b" ".join((*header_fields[:3], str(clause_count).encode())) + b"\n" units = b"".join(f"{literal} 0\n".encode() for literal in cube) payload = header + _BASE_BODY + units started_ns = time.monotonic_ns() + common = { + "schema_version": 1, + "arm": arm, + "cube_index": index, + "cube_literals": len(cube), + "cube_sha256": hashlib.sha256( + (" ".join(map(str, cube)) + " 0\n").encode() + ).hexdigest(), + "released_monotonic_ns": released_ns, + "started_monotonic_ns": started_ns, + "worker_pid": os.getpid(), + } temporary = tempfile.NamedTemporaryFile( prefix=f"cube-{arm}-{index}-", suffix=".cnf", dir=_TMPDIR, delete=False ) @@ -156,14 +169,12 @@ def _solve_cube(task: tuple[str, int, list[int]]) -> dict[str, Any]: check=False, ) elapsed_s = (time.monotonic_ns() - started_ns) / 1e9 + finished_ns = time.monotonic_ns() user_s, system_s = child_cpu_seconds(usage_before) decisions, conflicts = parse_stats(process.stdout) result = {10: "sat", 20: "unsat"}.get(process.returncode, "error") return { - "schema_version": 1, - "arm": arm, - "cube_index": index, - "cube_literals": len(cube), + **common, "result": result, "returncode": process.returncode, "elapsed_s": elapsed_s, @@ -172,23 +183,23 @@ def _solve_cube(task: tuple[str, int, list[int]]) -> dict[str, Any]: "decisions": decisions, "conflicts": conflicts, "censored": False, + "finished_monotonic_ns": finished_ns, "stderr_tail": process.stderr[-500:] if result == "error" else "", } except subprocess.TimeoutExpired: + finished_ns = time.monotonic_ns() user_s, system_s = child_cpu_seconds(usage_before) return { - "schema_version": 1, - "arm": arm, - "cube_index": index, - "cube_literals": len(cube), + **common, "result": "timeout", "returncode": None, - "elapsed_s": (time.monotonic_ns() - started_ns) / 1e9, + "elapsed_s": (finished_ns - started_ns) / 1e9, "user_s": user_s, "system_s": system_s, "decisions": None, "conflicts": None, "censored": True, + "finished_monotonic_ns": finished_ns, "stderr_tail": "", } finally: @@ -208,25 +219,51 @@ def summarize( decisions: list[float], conflicts: list[float], workers: int, + replay_workers: list[int], wall_s: float, + measured_makespan_s: float, ) -> dict[str, Any]: time_stats = distribution(durations) cpu_stats = distribution(cpu_durations) decision_stats = distribution(decisions) conflict_stats = distribution(conflicts) + result = ( + "sat" + if sat + else "error" + if errors + else "timeout" + if timeouts + else "unsat" + if unsat == cubes + else "incomplete" + ) + complete = result in {"sat", "unsat"} and completed == cubes + lpt_wall = {str(count): lpt_makespan(durations, count) for count in replay_workers} + lpt_cpu = { + str(count): lpt_makespan(cpu_durations, count) for count in replay_workers + } return { "cubes": cubes, + "terminal_records": completed + timeouts + errors, "completed": completed, "timeouts": timeouts, "errors": errors, "sat": sat, "unsat": unsat, + "result": result, + "complete": complete, + "censored": bool(timeouts), "total_solver_s": time_stats["total"], "total_cpu_s": cpu_stats["total"], "cpu_time": cpu_stats, "observed_parallel_wall_s": wall_s, + "measured_makespan_s": measured_makespan_s, "lpt_makespan_s": lpt_makespan(durations, workers), "cpu_lpt_makespan_s": lpt_makespan(cpu_durations, workers), + "lpt_makespan_by_workers_s": lpt_wall, + "cpu_lpt_makespan_by_workers_s": lpt_cpu, + "lpt_is_lower_bound": bool(timeouts or errors), "p50_s": time_stats["p50"], "p95_s": time_stats["p95"], "p99_s": time_stats["p99"], @@ -249,6 +286,7 @@ def run_arm( cubes: Iterator[list[int]], total_cubes: int, workers: int, + replay_workers: list[int], output: Path, worker_args: tuple[Any, ...], ) -> dict[str, Any]: @@ -259,6 +297,8 @@ def run_arm( counts = {"completed": 0, "timeouts": 0, "errors": 0, "sat": 0, "unsat": 0} progress_every = 10_000 if total_cubes > 10_000 else 200 started = time.monotonic() + earliest_release_ns: int | None = None + latest_collection_ns: int | None = None with output.open("w", encoding="utf-8", buffering=1) as stream: with concurrent.futures.ProcessPoolExecutor( max_workers=workers, @@ -273,7 +313,9 @@ def submit_one() -> bool: index, cube = next(indexed_cubes) except StopIteration: return False - pending.add(pool.submit(_solve_cube, (arm, index, cube))) + pending.add( + pool.submit(_solve_cube, (arm, index, cube, time.monotonic_ns())) + ) return True for _ in range(workers * 4): @@ -288,9 +330,27 @@ def submit_one() -> bool: for future in done: pending.remove(future) row = future.result() - row["completed_monotonic_ns"] = time.monotonic_ns() + row["collected_monotonic_ns"] = time.monotonic_ns() stream.write(json.dumps(row, sort_keys=True) + "\n") + row_released = int(row["released_monotonic_ns"]) + row_collected = int(row["collected_monotonic_ns"]) + earliest_release_ns = ( + row_released + if earliest_release_ns is None + else min(earliest_release_ns, row_released) + ) + latest_collection_ns = ( + row_collected + if latest_collection_ns is None + else max(latest_collection_ns, row_collected) + ) done_count += 1 + durations.append(float(row["elapsed_s"])) + cpu_durations.append(float(row["user_s"]) + float(row["system_s"])) + if row["decisions"] is not None: + decisions.append(float(row["decisions"])) + if row["conflicts"] is not None: + conflicts.append(float(row["conflicts"])) if row["censored"]: counts["timeouts"] += 1 elif row["result"] == "error": @@ -298,10 +358,6 @@ def submit_one() -> bool: else: counts["completed"] += 1 counts[row["result"]] += 1 - durations.append(float(row["elapsed_s"])) - cpu_durations.append(float(row["user_s"]) + float(row["system_s"])) - decisions.append(float(row["decisions"])) - conflicts.append(float(row["conflicts"])) submit_one() if done_count % progress_every == 0 or done_count == total_cubes: print(f"{arm}: {done_count}/{total_cubes}", flush=True) @@ -314,7 +370,13 @@ def submit_one() -> bool: decisions=decisions, conflicts=conflicts, workers=workers, + replay_workers=replay_workers, wall_s=time.monotonic() - started, + measured_makespan_s=( + 0.0 + if earliest_release_ns is None or latest_collection_ns is None + else (latest_collection_ns - earliest_release_ns) / 1e9 + ), **counts, ) @@ -325,12 +387,16 @@ def main() -> None: parser.add_argument("--arm", action="append", required=True, metavar="NAME=CUBES") parser.add_argument("--kissat", type=Path, required=True) parser.add_argument("--workers", type=int, required=True) + parser.add_argument("--lpt-workers", type=int, action="append", default=[]) parser.add_argument("--timeout-s", type=float, default=600.0) parser.add_argument("--out-dir", type=Path, required=True) parser.add_argument("--tmp-dir", type=Path) args = parser.parse_args() - if args.workers < 1 or args.timeout_s <= 0: - parser.error("workers and timeout-s must be positive") + if args.workers < 1 or args.timeout_s <= 0 or any( + count < 1 for count in args.lpt_workers + ): + parser.error("workers, lpt-workers, and timeout-s must be positive") + replay_workers = sorted(set(args.lpt_workers or [args.workers])) variables, clauses, body = parse_cnf(args.cnf.read_bytes()) args.out_dir.mkdir(parents=True, exist_ok=True) @@ -357,21 +423,24 @@ def main() -> None: read_cubes(cube_file), total_cubes, args.workers, + replay_workers, args.out_dir / f"{arm}.jsonl", worker_args, ) bundle = { "schema_version": 1, "workers": args.workers, + "lpt_workers": replay_workers, "timeout_s": args.timeout_s, "cnf": str(args.cnf), "kissat": str(args.kissat), "arms": summaries, } (args.out_dir / "summary.json").write_text( - json.dumps(bundle, indent=2, sort_keys=True) + "\n", encoding="utf-8" + json.dumps(bundle, indent=2, sort_keys=True, allow_nan=False) + "\n", + encoding="utf-8", ) - print(json.dumps(bundle, indent=2, sort_keys=True)) + print(json.dumps(bundle, indent=2, sort_keys=True, allow_nan=False)) if __name__ == "__main__": diff --git a/benchmarks/cnc/contracts/hard-regime-v1.yaml b/benchmarks/cnc/contracts/hard-regime-v1.yaml new file mode 100644 index 0000000..07603a0 --- /dev/null +++ b/benchmarks/cnc/contracts/hard-regime-v1.yaml @@ -0,0 +1,113 @@ +schema_version: 1 +study_id: array-ripple-hard-unsat-cnc-v1 +issue: 51 +architecture: array-ripple +semantic_task: unsigned-factoring +expected_outcome: unsat + +widths: + - product_width: 64 + factor_input_width: 32 + calibration: + count: 3 + seed: 202607196401 + held_out: + count: 10 + seed: 202607196402 + - product_width: 72 + factor_input_width: 36 + calibration: + count: 3 + seed: 202607197201 + held_out: + count: 10 + seed: 202607197202 + - product_width: 80 + factor_input_width: 40 + calibration: + count: 3 + seed: 202607198001 + held_out: + count: 10 + seed: 202607198002 + +frontier_bands: + low: + center_cubes: 16384 + accepted_ratio: [0.75, 1.25] + medium: + center_cubes: 65536 + accepted_ratio: [0.75, 1.25] + high: + center_cubes: 262144 + accepted_ratio: [0.75, 1.25] + +methods: + monolithic-kissat: + budgets: ["none"] + input: global-tseitin-cnf + march-cu-dynamic: + budgets: [dynamic-default] + input: global-tseitin-cnf + cutoff_policy: upstream-default-dynamic + region-cc: + budgets: [low, medium, high] + input: structured-circuitsat + selector: region + max_rows: 512 + cutoff_policy: calibration-frozen-cc-difficulty + structure-blind-cc: + budgets: [low, medium, high] + input: structured-circuitsat + selector: structure-blind + max_rows: 512 + cutoff_policy: calibration-frozen-cc-difficulty + +limits_seconds: + monolithic: 600 + cubing: 7200 + per_cube_conquer: 1800 + +scheduling: + measured_workers: 32 + lpt_replay_workers: [32, 128, 512] + completion_policy: complete-all-cubes + +calibration: + unit: product-width-method-budget + selection_loss: median-absolute-log2-frontier-error + held_out_recalibration: forbidden + center_revision_policy: calibration-only-before-held-out + +statistics: + unit: held-out-instance + paired_geometric_mean: true + bootstrap: + samples: 10000 + confidence: 0.95 + seed: 2026071951 + budget_adjustment: + x: log2-frontier-size + y: log-metric + interpolation: piecewise-linear + extrapolation: forbidden + common_grid: overlap-low-geometric-mid-overlap-high + censored_policy: report-separately-no-finite-imputation + +artifact_policy: + raw_directory: artifacts/cnc-hard-regime + git_payload: + - contract + - sha256-manifest + - aggregate-statistics + - primary-table + - report + tool_identity: version-source-revision-command-and-executable-sha256 + +tool_sources: + kissat: + repository: https://github.com/arminbiere/kissat.git + revision: 8af8e56f174b778aef3aa45af9f739b2a5f492c2 + march_cu: + repository: https://github.com/marijnheule/CnC.git + revision: 705b60c6491ef2b61988b3ce6ac674be1b90571d diff --git a/benchmarks/cnc/hard_regime.py b/benchmarks/cnc/hard_regime.py new file mode 100755 index 0000000..78365c3 --- /dev/null +++ b/benchmarks/cnc/hard_regime.py @@ -0,0 +1,436 @@ +#!/usr/bin/env python3 +"""Generate and audit the hard-UNSAT factoring study declared by issue #51.""" + +from __future__ import annotations + +import argparse +import json +import random +import time +from collections import Counter +from pathlib import Path +from typing import Any, Iterable + +import sympy +import yaml + +from benchmarks.pipeline.circuit import ( + canonical_bytes, + pin_port_values, + read_jsonl, + sha256_bytes, + sha256_file, + write_json, + write_jsonl, +) +from benchmarks.pipeline.cnf import encode_validated_circuit +from benchmarks.pipeline.multipliers import generate_multiplier + + +MILLER_RABIN_BASES = (2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37) +EXPECTED_WIDTHS = ((64, 32), (72, 36), (80, 40)) +EXPECTED_METHOD_BUDGETS = { + "monolithic-kissat": ("none",), + "march-cu-dynamic": ("dynamic-default",), + "region-cc": ("low", "medium", "high"), + "structure-blind-cc": ("low", "medium", "high"), +} +EXPECTED_BANDS = {"low": 1 << 14, "medium": 1 << 16, "high": 1 << 18} +EXPECTED_TOOL_REVISIONS = { + "kissat": "8af8e56f174b778aef3aa45af9f739b2a5f492c2", + "march_cu": "705b60c6491ef2b61988b3ce6ac674be1b90571d", +} + + +class HardRegimeError(ValueError): + """The study contract or one of its artifacts violates the frozen protocol.""" + + +def mapping(value: object, label: str) -> dict[str, Any]: + if not isinstance(value, dict): + raise HardRegimeError(f"{label} must be an object") + return value + + +def positive_int(value: object, label: str) -> int: + if not isinstance(value, int) or isinstance(value, bool) or value <= 0: + raise HardRegimeError(f"{label} must be a positive integer") + return value + + +def load_contract(path: Path) -> dict[str, Any]: + try: + value = yaml.safe_load(path.read_text(encoding="utf-8")) + except (OSError, yaml.YAMLError) as exc: + raise HardRegimeError(f"cannot read contract {path}: {exc}") from exc + contract = mapping(value, "contract") + validate_contract(contract) + return contract + + +def contract_sha256(contract: dict[str, Any]) -> str: + return sha256_bytes(canonical_bytes(contract)) + + +def validate_contract(contract: dict[str, Any]) -> None: + if contract.get("schema_version") != 1: + raise HardRegimeError("contract schema_version must be 1") + if contract.get("study_id") != "array-ripple-hard-unsat-cnc-v1": + raise HardRegimeError("unexpected study_id") + if contract.get("issue") != 51: + raise HardRegimeError("contract must identify issue 51") + if contract.get("architecture") != "array-ripple": + raise HardRegimeError("hard-regime primary table must use array-ripple") + if contract.get("expected_outcome") != "unsat": + raise HardRegimeError("hard-regime targets must declare UNSAT") + + widths = contract.get("widths") + if not isinstance(widths, list): + raise HardRegimeError("widths must be a list") + actual_widths = [] + seeds = [] + for index, raw in enumerate(widths): + width = mapping(raw, f"widths[{index}]") + product = positive_int(width.get("product_width"), "product_width") + factor = positive_int(width.get("factor_input_width"), "factor_input_width") + if product != 2 * factor: + raise HardRegimeError( + f"product width {product} must be twice factor-input width {factor}" + ) + actual_widths.append((product, factor)) + for split, expected_count in (("calibration", 3), ("held_out", 10)): + spec = mapping(width.get(split), f"width {product} {split}") + count = positive_int(spec.get("count"), f"width {product} {split}.count") + seed = positive_int(spec.get("seed"), f"width {product} {split}.seed") + if count != expected_count: + raise HardRegimeError( + f"width {product} {split} count must be {expected_count}" + ) + seeds.append(seed) + if tuple(actual_widths) != EXPECTED_WIDTHS: + raise HardRegimeError( + "width ladder must be product/factor pairs 64/32, 72/36, and 80/40" + ) + duplicates = sorted(seed for seed, count in Counter(seeds).items() if count > 1) + if duplicates: + raise HardRegimeError(f"calibration and held-out seeds overlap: {duplicates}") + + bands = mapping(contract.get("frontier_bands"), "frontier_bands") + actual_bands = {} + for name, raw_spec in bands.items(): + spec = mapping(raw_spec, f"frontier band {name}") + actual_bands[name] = positive_int( + spec.get("center_cubes"), f"frontier band {name}.center_cubes" + ) + ratio = spec.get("accepted_ratio") + if ratio != [0.75, 1.25]: + raise HardRegimeError( + f"frontier band {name}.accepted_ratio must be [0.75, 1.25]" + ) + if actual_bands != EXPECTED_BANDS: + raise HardRegimeError("frontier bands must be centered at 2^14, 2^16, and 2^18") + + methods = mapping(contract.get("methods"), "methods") + if set(methods) != set(EXPECTED_METHOD_BUDGETS): + raise HardRegimeError("method set does not match the frozen issue #51 matrix") + for name, expected in EXPECTED_METHOD_BUDGETS.items(): + budgets = mapping(methods[name], f"method {name}").get("budgets") + if not isinstance(budgets, list) or tuple(budgets) != expected: + raise HardRegimeError(f"method {name} budgets must be {list(expected)}") + if methods["march-cu-dynamic"].get("cutoff_policy") != "upstream-default-dynamic": + raise HardRegimeError("march_cu must retain its upstream default dynamic cutoff") + for name in ("region-cc", "structure-blind-cc"): + if methods[name].get("max_rows") != 512: + raise HardRegimeError(f"method {name} max_rows must be 512") + + limits = mapping(contract.get("limits_seconds"), "limits_seconds") + expected_limits = {"monolithic": 600, "cubing": 7200, "per_cube_conquer": 1800} + if limits != expected_limits: + raise HardRegimeError(f"limits_seconds must be {expected_limits}") + scheduling = mapping(contract.get("scheduling"), "scheduling") + if scheduling.get("measured_workers") != 32: + raise HardRegimeError("measured worker count must be 32") + if scheduling.get("lpt_replay_workers") != [32, 128, 512]: + raise HardRegimeError("LPT replay worker counts must be 32, 128, and 512") + statistics = mapping(contract.get("statistics"), "statistics") + if statistics.get("unit") != "held-out-instance": + raise HardRegimeError("statistical unit must be the held-out instance") + adjustment = mapping(statistics.get("budget_adjustment"), "budget_adjustment") + if adjustment.get("common_grid") != "overlap-low-geometric-mid-overlap-high": + raise HardRegimeError("budget adjustment common grid is not preregistered") + calibration = mapping(contract.get("calibration"), "calibration") + if calibration.get("held_out_recalibration") != "forbidden": + raise HardRegimeError("held-out threshold tuning must be forbidden") + tool_sources = mapping(contract.get("tool_sources"), "tool_sources") + if set(tool_sources) != set(EXPECTED_TOOL_REVISIONS): + raise HardRegimeError("tool_sources must pin Kissat and march_cu") + for name, revision in EXPECTED_TOOL_REVISIONS.items(): + source = mapping(tool_sources[name], f"tool source {name}") + if source.get("revision") != revision: + raise HardRegimeError(f"tool source {name} revision is not frozen") + + +def strong_miller_rabin(value: int, bases: Iterable[int] = MILLER_RABIN_BASES) -> bool: + """Return whether ``value`` passes the declared independent strong-MR checks.""" + + if value < 2: + return False + small = tuple(bases) + for prime in small: + if value % prime == 0: + return value == prime + odd = value - 1 + power = 0 + while odd % 2 == 0: + odd //= 2 + power += 1 + for base in small: + if base >= value: + continue + witness = pow(base, odd, value) + if witness in (1, value - 1): + continue + for _ in range(power - 1): + witness = witness * witness % value + if witness == value - 1: + break + else: + return False + return True + + +def _targets_for_split( + product_width: int, + factor_width: int, + split: str, + count: int, + seed: int, +) -> list[dict[str, Any]]: + rng = random.Random(seed) + lower = 1 << (product_width - 1) + factor_max = (1 << factor_width) - 1 + reachable_max = factor_max * factor_max + records = [] + seen: set[int] = set() + while len(records) < count: + candidate = rng.randrange(lower, reachable_max + 1) | 1 + target = int(sympy.nextprime(candidate - 1)) + if target > reachable_max or target in seen: + continue + seen.add(target) + index = len(records) + split_label = "cal" if split == "calibration" else "test" + range_checks = { + "full_product_width": target.bit_length() == product_width, + "above_factor_input_range": target > factor_max, + "within_reachable_product_range": target <= reachable_max, + } + mr_passed = strong_miller_rabin(target) + sympy_passed = bool(sympy.isprime(target)) + if not all(range_checks.values()) or not mr_passed or not sympy_passed: + raise HardRegimeError("deterministic target generation produced an invalid prime") + records.append( + { + "schema_version": 1, + "id": f"prime-p{product_width}-{split_label}-{index:02d}", + "generator": "deterministic-prime-hard-unsat-v1", + "architecture": "array-ripple", + "semantic_task": "unsigned-factoring", + "expected_outcome": "unsat", + "product_width": product_width, + "factor_input_width": factor_width, + "split": split, + "split_index": index, + "seed": seed, + "target": target, + "primality_checks": { + "sympy_isprime": { + "result": sympy_passed, + "version": sympy.__version__, + }, + "strong_miller_rabin": { + "bases": list(MILLER_RABIN_BASES), + "result": mr_passed, + }, + }, + "range_checks": range_checks, + "unsat_argument": "prime-target-above-unsigned-factor-input-range", + } + ) + return records + + +def target_records(contract: dict[str, Any]) -> list[dict[str, Any]]: + validate_contract(contract) + records = [] + for width in contract["widths"]: + for split in ("calibration", "held_out"): + spec = width[split] + records.extend( + _targets_for_split( + width["product_width"], + width["factor_input_width"], + split, + spec["count"], + spec["seed"], + ) + ) + return records + + +def verify_target_records( + contract: dict[str, Any], records: list[dict[str, Any]] +) -> list[str]: + validate_contract(contract) + expected = target_records(contract) + if records != expected: + raise HardRegimeError( + "target records do not byte-semantically match deterministic regeneration" + ) + ids = [record["id"] for record in records] + if len(ids) != len(set(ids)): + raise HardRegimeError("target records contain duplicate instance IDs") + split_targets: dict[str, set[int]] = {"calibration": set(), "held_out": set()} + for record in records: + product_width = positive_int(record.get("product_width"), "record product_width") + factor_width = positive_int( + record.get("factor_input_width"), "record factor_input_width" + ) + target = positive_int(record.get("target"), "record target") + if product_width != 2 * factor_width: + raise HardRegimeError(f"{record.get('id')}: factor/product width confusion") + factor_max = (1 << factor_width) - 1 + if target.bit_length() != product_width: + raise HardRegimeError(f"{record.get('id')}: target does not use product width") + if target <= factor_max or target > factor_max * factor_max: + raise HardRegimeError(f"{record.get('id')}: target violates factor/product range") + if not sympy.isprime(target) or not strong_miller_rabin(target): + raise HardRegimeError(f"{record.get('id')}: target fails primality verification") + split = record.get("split") + if split not in split_targets: + raise HardRegimeError(f"{record.get('id')}: invalid split") + split_targets[split].add(target) + overlap = split_targets["calibration"] & split_targets["held_out"] + if overlap: + raise HardRegimeError(f"calibration and held-out targets overlap: {sorted(overlap)}") + return [ + f"PASS contract: {contract_sha256(contract)}", + f"PASS targets: {len(records)} deterministic prime UNSAT instances", + "PASS widths: product/factor pairs are 64/32, 72/36, and 80/40", + "PASS split: calibration and held-out seeds and targets are disjoint", + "PASS primality: SymPy and independent strong Miller-Rabin checks agree", + ] + + +def materialize_instances( + contract: dict[str, Any], records: list[dict[str, Any]], out_dir: Path +) -> list[dict[str, Any]]: + verify_target_records(contract, records) + out_dir.mkdir(parents=True, exist_ok=True) + write_jsonl(out_dir / "targets.jsonl", records) + contract_digest = contract_sha256(contract) + circuits: dict[int, tuple[dict[str, Any], Path]] = {} + manifest = [] + for record in records: + factor_width = record["factor_input_width"] + if factor_width not in circuits: + raw = generate_multiplier(factor_width, "array-ripple") + raw_path = out_dir / "raw" / f"array-ripple-f{factor_width}.json" + write_json(raw_path, raw) + circuits[factor_width] = (raw, raw_path) + raw, raw_path = circuits[factor_width] + pinned = pin_port_values(raw, {"product": record["target"]}) + benchmark = pinned.setdefault("metadata", {}).setdefault("benchmark", {}) + benchmark.update( + { + **record, + "contract_sha256": contract_digest, + "family": "hard-unsat-prime-factoring", + } + ) + instance_dir = out_dir / "instances" / f"p{record['product_width']}" / record["id"] + circuitsat = instance_dir / f"{record['id']}.circuitsat.json" + cnf = instance_dir / f"{record['id']}.cnf" + metadata = instance_dir / f"{record['id']}.meta.json" + write_json(circuitsat, pinned) + encoding_started = time.perf_counter() + encoding_cpu_started = time.process_time() + encode_validated_circuit(pinned).write_dimacs(cnf) + encoding_wall_s = time.perf_counter() - encoding_started + encoding_cpu_s = time.process_time() - encoding_cpu_started + meta = { + **record, + "contract_sha256": contract_digest, + "raw_circuit": str(raw_path.relative_to(out_dir)), + "raw_circuit_sha256": sha256_file(raw_path), + "circuitsat": str(circuitsat.relative_to(out_dir)), + "circuitsat_sha256": sha256_file(circuitsat), + "cnf": str(cnf.relative_to(out_dir)), + "cnf_sha256": sha256_file(cnf), + "encoding_wall_s": encoding_wall_s, + "encoding_cpu_s": encoding_cpu_s, + } + write_json(metadata, meta) + manifest.append({**meta, "metadata": str(metadata.relative_to(out_dir))}) + write_jsonl(out_dir / "manifest.jsonl", manifest) + return manifest + + +def command_validate_contract(args: argparse.Namespace) -> None: + contract = load_contract(args.contract) + print(f"PASS contract: {contract_sha256(contract)}") + + +def command_generate_targets(args: argparse.Namespace) -> None: + contract = load_contract(args.contract) + records = target_records(contract) + write_jsonl(args.out, records) + for line in verify_target_records(contract, records): + print(line) + + +def command_verify_targets(args: argparse.Namespace) -> None: + contract = load_contract(args.contract) + for line in verify_target_records(contract, read_jsonl(args.targets)): + print(line) + + +def command_materialize(args: argparse.Namespace) -> None: + contract = load_contract(args.contract) + records = target_records(contract) + manifest = materialize_instances(contract, records, args.out_dir) + print(f"PASS materialization: {len(manifest)} CircuitSAT/CNF pairs") + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + commands = parser.add_subparsers(dest="command", required=True) + command = commands.add_parser("validate-contract") + command.add_argument("contract", type=Path) + command.set_defaults(handler=command_validate_contract) + command = commands.add_parser("generate-targets") + command.add_argument("contract", type=Path) + command.add_argument("--out", type=Path, required=True) + command.set_defaults(handler=command_generate_targets) + command = commands.add_parser("verify-targets") + command.add_argument("contract", type=Path) + command.add_argument("targets", type=Path) + command.set_defaults(handler=command_verify_targets) + command = commands.add_parser("materialize") + command.add_argument("contract", type=Path) + command.add_argument("--out-dir", type=Path, required=True) + command.set_defaults(handler=command_materialize) + return parser + + +def main() -> int: + args = build_parser().parse_args() + try: + args.handler(args) + except (HardRegimeError, OSError, ValueError, json.JSONDecodeError) as exc: + raise SystemExit(f"FAIL: {exc}") from exc + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/benchmarks/cnc/hard_regime_matrix.py b/benchmarks/cnc/hard_regime_matrix.py new file mode 100755 index 0000000..b237081 --- /dev/null +++ b/benchmarks/cnc/hard_regime_matrix.py @@ -0,0 +1,538 @@ +#!/usr/bin/env python3 +"""Lock the issue #51 toolchain and materialize its complete run matrix.""" + +from __future__ import annotations + +import argparse +import json +import math +import os +import subprocess +from collections import Counter +from pathlib import Path +from typing import Any + +from benchmarks.cnc.calibrate_hard_regime import ( + CalibrationError, + calibration_loss, + choose_width_response, + median_tasks, +) + +from benchmarks.cnc.hard_regime import ( + HardRegimeError, + contract_sha256, + load_contract, + mapping, + target_records, +) +from benchmarks.pipeline.circuit import ( + canonical_bytes, + load_json, + read_jsonl, + sha256_bytes, + sha256_file, + write_json, +) + + +class MatrixError(HardRegimeError): + """The tool lock, instance manifest, calibration lock, or matrix is invalid.""" + + +def executable_record( + path: Path, + source_revision: str, + version_args: list[str], + *, + allow_nonzero: bool = False, + required_basename: str | None = None, +) -> dict[str, Any]: + resolved = path.resolve() + if not resolved.is_file() or not os.access(resolved, os.X_OK): + raise MatrixError(f"tool is not executable: {resolved}") + if required_basename is not None and resolved.name != required_basename: + raise MatrixError( + f"tool {resolved} is not the revision-qualified binary {required_basename}" + ) + process = subprocess.run( + [str(resolved), *version_args], capture_output=True, text=True, check=False + ) + if process.returncode and not allow_nonzero: + raise MatrixError(f"version command failed for {resolved}: {process.stderr[-500:]}") + output = (process.stdout + process.stderr).strip() + if not output: + raise MatrixError(f"version command produced no identity output for {resolved}") + return { + "path": str(resolved), + "executable_sha256": sha256_file(resolved), + "source_revision": source_revision, + "version_command": [str(resolved), *version_args], + "version_output": output[:4000], + "version_output_sha256": sha256_bytes(output.encode()), + } + + +def lock_toolchain( + contract: dict[str, Any], + cuber: Path, + kissat: Path, + march_cu: Path, + repository_revision: str, +) -> dict[str, Any]: + if not repository_revision or len(repository_revision) < 7: + raise MatrixError("repository revision is required for cnc_cuber provenance") + sources = contract["tool_sources"] + record = { + "schema_version": 1, + "kind": "hard-regime-toolchain-lock", + "contract_sha256": contract_sha256(contract), + "tools": { + "cnc_cuber": executable_record(cuber, repository_revision, ["--help"]), + "kissat": executable_record( + kissat, + sources["kissat"]["revision"], + ["--version"], + required_basename=f"kissat-{sources['kissat']['revision']}", + ), + "march_cu": executable_record( + march_cu, + sources["march_cu"]["revision"], + [], + allow_nonzero=True, + required_basename=f"march_cu-{sources['march_cu']['revision']}", + ), + }, + } + record["toolchain_sha256"] = sha256_bytes(canonical_bytes(record)) + return record + + +def verify_toolchain( + contract: dict[str, Any], toolchain: dict[str, Any], *, check_paths: bool +) -> None: + if toolchain.get("schema_version") != 1 or toolchain.get("kind") != "hard-regime-toolchain-lock": + raise MatrixError("unsupported toolchain lock") + if toolchain.get("contract_sha256") != contract_sha256(contract): + raise MatrixError("toolchain lock uses a different contract") + tools = mapping(toolchain.get("tools"), "toolchain.tools") + if set(tools) != {"cnc_cuber", "kissat", "march_cu"}: + raise MatrixError("toolchain must contain cnc_cuber, kissat, and march_cu") + expected_revisions = { + "kissat": contract["tool_sources"]["kissat"]["revision"], + "march_cu": contract["tool_sources"]["march_cu"]["revision"], + } + for name, raw in tools.items(): + spec = mapping(raw, f"toolchain tool {name}") + digest = spec.get("executable_sha256") + if not isinstance(digest, str) or len(digest) != 64: + raise MatrixError(f"toolchain tool {name} has no executable SHA-256") + if name in expected_revisions and spec.get("source_revision") != expected_revisions[name]: + raise MatrixError(f"toolchain tool {name} source revision mismatch") + if check_paths: + path = Path(str(spec.get("path"))).resolve() + if sha256_file(path) != digest: + raise MatrixError(f"toolchain tool {name} executable hash mismatch") + unsigned = {key: value for key, value in toolchain.items() if key != "toolchain_sha256"} + if toolchain.get("toolchain_sha256") != sha256_bytes(canonical_bytes(unsigned)): + raise MatrixError("toolchain lock fingerprint mismatch") + + +def validate_instance_manifest( + contract: dict[str, Any], manifest: list[dict[str, Any]] +) -> None: + expected = {record["id"]: record for record in target_records(contract)} + if len(manifest) != len(expected): + raise MatrixError(f"instance manifest needs {len(expected)} records") + seen = set() + digest = contract_sha256(contract) + for record in manifest: + instance_id = record.get("id") + if instance_id in seen: + raise MatrixError(f"duplicate manifest instance {instance_id!r}") + seen.add(instance_id) + if instance_id not in expected: + raise MatrixError(f"undeclared manifest instance {instance_id!r}") + target = expected[instance_id] + for field in ( + "target", + "product_width", + "factor_input_width", + "split", + "split_index", + "seed", + "architecture", + "expected_outcome", + ): + if record.get(field) != target[field]: + raise MatrixError(f"{instance_id}: manifest field {field} differs from target lock") + if record.get("contract_sha256") != digest: + raise MatrixError(f"{instance_id}: manifest contract hash mismatch") + if record["product_width"] != 2 * record["factor_input_width"]: + raise MatrixError(f"{instance_id}: factor/product width confusion") + for artifact in ("circuitsat", "cnf"): + if not isinstance(record.get(artifact), str) or not isinstance( + record.get(f"{artifact}_sha256"), str + ): + raise MatrixError(f"{instance_id}: incomplete {artifact} provenance") + for metric in ("encoding_wall_s", "encoding_cpu_s"): + value = record.get(metric) + if not isinstance(value, (int, float)) or isinstance(value, bool) or value < 0: + raise MatrixError(f"{instance_id}: invalid {metric}") + if seen != set(expected): + raise MatrixError("instance manifest does not cover the declared target set") + + +def load_calibration_lock( + contract: dict[str, Any], + calibration_root: Path, + product_width: int, + selector: str, + toolchain: dict[str, Any], + manifest: list[dict[str, Any]], +) -> dict[str, Any]: + path = calibration_root / f"p{product_width}" / selector / "calibration-lock.json" + lock = load_json(path) + if lock.get("schema_version") != 1 or lock.get("kind") != "width-level-cc-calibration-lock": + raise MatrixError(f"{path}: unsupported calibration lock") + for field, value in { + "contract_sha256": contract_sha256(contract), + "product_width": product_width, + "selector": selector, + "cuber_sha256": toolchain["tools"]["cnc_cuber"]["executable_sha256"], + }.items(): + if lock.get(field) != value: + raise MatrixError(f"{path}: calibration {field} mismatch") + expected_instances = sorted( + [ + {"id": record["id"], "sha256": record["circuitsat_sha256"]} + for record in manifest + if record["product_width"] == product_width + and record["split"] == "calibration" + ], + key=lambda item: item["id"], + ) + actual_instances = lock.get("calibration_instances") + if not isinstance(actual_instances, list) or not all( + isinstance(item, dict) for item in actual_instances + ): + raise MatrixError(f"{path}: calibration instance provenance is malformed") + actual_instances = sorted(actual_instances, key=lambda item: str(item.get("id"))) + if actual_instances != expected_instances: + raise MatrixError(f"{path}: calibration instances are not the frozen split") + method = "region-cc" if selector == "region" else "structure-blind-cc" + if lock.get("method") != method: + raise MatrixError(f"{path}: calibration method mismatch") + if lock.get("max_rows") != contract["methods"][method]["max_rows"]: + raise MatrixError(f"{path}: calibration max_rows mismatch") + + response = lock.get("response") + if not isinstance(response, list) or not response: + raise MatrixError(f"{path}: calibration response is empty") + expected_ids = [item["id"] for item in expected_instances] + response_by_threshold = {} + for row in response: + if not isinstance(row, dict): + raise MatrixError(f"{path}: malformed calibration response row") + threshold = row.get("threshold") + if not isinstance(threshold, int) or isinstance(threshold, bool) or threshold < 0: + raise MatrixError(f"{path}: malformed calibration response threshold") + if threshold in response_by_threshold: + raise MatrixError(f"{path}: duplicate calibration response threshold {threshold}") + instances = row.get("instances") + response_ids = ( + [item.get("id") for item in instances] + if isinstance(instances, list) + and all(isinstance(item, dict) for item in instances) + else [] + ) + if not all(isinstance(instance_id, str) for instance_id in response_ids) or sorted( + response_ids + ) != expected_ids: + raise MatrixError(f"{path}: response threshold {threshold} uses the wrong instances") + for item in instances: + if not isinstance(item, dict): + raise MatrixError(f"{path}: malformed response instance") + tasks = item.get("tasks") + if not isinstance(tasks, int) or isinstance(tasks, bool) or tasks < 0: + raise MatrixError(f"{path}: response task count is invalid") + for field in ("elapsed_s", "cpu_s"): + value = item.get(field) + if ( + not isinstance(value, (int, float)) + or isinstance(value, bool) + or not math.isfinite(float(value)) + or value < 0 + ): + raise MatrixError(f"{path}: response {field} is invalid") + response_by_threshold[threshold] = row + + bands = mapping(lock.get("bands"), f"{path} bands") + if set(bands) != set(contract["frontier_bands"]): + raise MatrixError(f"{path}: calibration bands are incomplete") + for band, spec in bands.items(): + if not isinstance(spec, dict): + raise MatrixError(f"{path}: band {band} is malformed") + contract_band = contract["frontier_bands"][band] + target = int(contract_band["center_cubes"]) + minimum = math.ceil(target * float(contract_band["accepted_ratio"][0])) + maximum = math.floor(target * float(contract_band["accepted_ratio"][1])) + threshold = spec.get("selected_threshold") + if not isinstance(threshold, int) or threshold < 0: + raise MatrixError(f"{path}: band {band} has no frozen threshold") + if spec.get("target_tasks") != target: + raise MatrixError(f"{path}: band {band} target differs from contract") + if spec.get("accepted_task_range") != [minimum, maximum]: + raise MatrixError(f"{path}: band {band} accepted range differs from contract") + bracket = spec.get("search_bracket") + if ( + not isinstance(bracket, list) + or len(bracket) != 2 + or not all(isinstance(value, int) and value >= 0 for value in bracket) + or bracket[0] >= bracket[1] + or any(value not in response_by_threshold for value in bracket) + ): + raise MatrixError(f"{path}: band {band} search bracket is invalid") + try: + selected = choose_width_response(response, target, minimum, maximum) + except CalibrationError as error: + raise MatrixError(f"{path}: band {band} has no acceptable calibration") from error + if threshold != selected["threshold"] or threshold not in response_by_threshold: + raise MatrixError(f"{path}: band {band} threshold is not selected from the response") + expected_median = median_tasks(selected) + expected_loss = calibration_loss(selected, target)[0] + recorded_median = spec.get("median_tasks") + if not isinstance(recorded_median, (int, float)) or isinstance( + recorded_median, bool + ) or not math.isclose( + float(recorded_median), expected_median, rel_tol=1e-12, abs_tol=1e-12 + ): + raise MatrixError(f"{path}: band {band} median task count is inconsistent") + recorded_loss = spec.get("calibration_loss") + if not isinstance(recorded_loss, (int, float)) or isinstance( + recorded_loss, bool + ) or not math.isclose( + float(recorded_loss), expected_loss, rel_tol=1e-12, abs_tol=1e-12 + ): + raise MatrixError(f"{path}: band {band} selection loss is inconsistent") + if spec.get("within_target_range") is not True: + raise MatrixError(f"{path}: band {band} is outside its accepted task range") + final_instances = spec.get("instances") + final_ids = ( + [item.get("id") for item in final_instances] + if isinstance(final_instances, list) + and all(isinstance(item, dict) for item in final_instances) + else [] + ) + if not all(isinstance(instance_id, str) for instance_id in final_ids) or sorted( + final_ids + ) != expected_ids: + raise MatrixError(f"{path}: band {band} final rerun uses the wrong instances") + response_tasks = { + item["id"]: item["tasks"] for item in response_by_threshold[threshold]["instances"] + } + for item in final_instances: + if not isinstance(item, dict) or item.get("tasks") != response_tasks.get(item.get("id")): + raise MatrixError(f"{path}: band {band} final task count changed on rerun") + for field in ("frontier_sha256", "trace_sha256"): + digest = item.get(field) + if not isinstance(digest, str) or len(digest) != 64: + raise MatrixError(f"{path}: band {band} has invalid {field}") + for field in ("cubing_elapsed_s", "cubing_cpu_s"): + value = item.get(field) + if ( + not isinstance(value, (int, float)) + or isinstance(value, bool) + or not math.isfinite(float(value)) + or value < 0 + ): + raise MatrixError(f"{path}: band {band} has invalid {field}") + return lock + + +def cell_id(instance: str, method: str, budget: str) -> str: + return f"{instance}__{method}__{budget}" + + +def build_matrix( + contract: dict[str, Any], + manifest: list[dict[str, Any]], + calibration_root: Path, + toolchain: dict[str, Any], +) -> dict[str, Any]: + validate_instance_manifest(contract, manifest) + verify_toolchain(contract, toolchain, check_paths=False) + locks = { + (width, selector): load_calibration_lock( + contract, + calibration_root, + width, + selector, + toolchain, + manifest, + ) + for width in (64, 72, 80) + for selector in ("region", "structure-blind") + } + cells = [] + for instance in sorted(manifest, key=lambda record: record["id"]): + common = { + "schema_version": 1, + "instance_id": instance["id"], + "split": instance["split"], + "split_index": instance["split_index"], + "product_width": instance["product_width"], + "factor_input_width": instance["factor_input_width"], + "expected_outcome": "unsat", + "circuitsat": instance["circuitsat"], + "circuitsat_sha256": instance["circuitsat_sha256"], + "global_cnf": instance["cnf"], + "global_cnf_sha256": instance["cnf_sha256"], + "encoding_wall_s": instance["encoding_wall_s"], + "encoding_cpu_s": instance["encoding_cpu_s"], + "pilot": instance["split"] == "held_out" and instance["split_index"] < 3, + } + + cells.append( + { + **common, + "cell_id": cell_id(instance["id"], "monolithic-kissat", "none"), + "method": "monolithic-kissat", + "budget": "none", + "required_cpus": 1, + "time_limit_s": contract["limits_seconds"]["monolithic"], + } + ) + if instance["split"] != "held_out": + continue + cells.append( + { + **common, + "cell_id": cell_id(instance["id"], "march-cu-dynamic", "dynamic-default"), + "method": "march-cu-dynamic", + "budget": "dynamic-default", + "required_cpus": contract["scheduling"]["measured_workers"], + "cubing_time_limit_s": contract["limits_seconds"]["cubing"], + "per_cube_time_limit_s": contract["limits_seconds"]["per_cube_conquer"], + "cutoff_policy": "upstream-default-dynamic", + } + ) + for method, selector in ( + ("region-cc", "region"), + ("structure-blind-cc", "structure-blind"), + ): + lock = locks[(instance["product_width"], selector)] + for budget in ("low", "medium", "high"): + cells.append( + { + **common, + "cell_id": cell_id(instance["id"], method, budget), + "method": method, + "budget": budget, + "required_cpus": contract["scheduling"]["measured_workers"], + "cubing_time_limit_s": contract["limits_seconds"]["cubing"], + "per_cube_time_limit_s": contract["limits_seconds"]["per_cube_conquer"], + "selector": selector, + "max_rows": contract["methods"][method]["max_rows"], + "cc_threshold": lock["bands"][budget]["selected_threshold"], + "calibration_lock": str( + Path(f"p{instance['product_width']}") + / selector + / "calibration-lock.json" + ), + } + ) + counts = Counter(cell["method"] for cell in cells) + expected_counts = { + "monolithic-kissat": 39, + "march-cu-dynamic": 30, + "region-cc": 90, + "structure-blind-cc": 90, + } + if dict(counts) != expected_counts: + raise MatrixError(f"internal run-matrix count mismatch: {dict(counts)}") + if len({cell["cell_id"] for cell in cells}) != len(cells): + raise MatrixError("run matrix contains duplicate cell IDs") + return { + "schema_version": 1, + "kind": "hard-regime-run-matrix", + "contract_sha256": contract_sha256(contract), + "toolchain_sha256": toolchain["toolchain_sha256"], + "cell_counts": expected_counts, + "cells": cells, + } + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + commands = parser.add_subparsers(dest="command", required=True) + command = commands.add_parser("lock-toolchain") + command.add_argument("contract", type=Path) + command.add_argument("--cuber", type=Path, required=True) + command.add_argument("--kissat", type=Path, required=True) + command.add_argument("--march-cu", type=Path, required=True) + command.add_argument("--repository-revision", required=True) + command.add_argument("--out", type=Path, required=True) + command = commands.add_parser("build-matrix") + command.add_argument("contract", type=Path) + command.add_argument("manifest", type=Path) + command.add_argument("--calibration-root", type=Path, required=True) + command.add_argument("--toolchain", type=Path, required=True) + command.add_argument("--out", type=Path, required=True) + command = commands.add_parser("list-cells") + command.add_argument("matrix", type=Path) + command.add_argument( + "--set", + choices=("monolithic", "pilot-cnc", "full-cnc", "remaining-cnc"), + required=True, + ) + command.add_argument("--out", type=Path, required=True) + args = parser.parse_args() + try: + if args.command == "list-cells": + matrix = load_json(args.matrix) + cells = matrix.get("cells") + if not isinstance(cells, list): + raise MatrixError("run matrix has no cells") + selected = [] + for cell in cells: + monolithic = cell.get("method") == "monolithic-kissat" + include = { + "monolithic": monolithic, + "pilot-cnc": not monolithic and cell.get("pilot") is True, + "full-cnc": not monolithic, + "remaining-cnc": not monolithic and cell.get("pilot") is not True, + }[args.set] + if include: + selected.append(cell["cell_id"]) + args.out.parent.mkdir(parents=True, exist_ok=True) + args.out.write_text("".join(f"{cell_id}\n" for cell_id in selected), encoding="utf-8") + print(f"PASS cell-list: {len(selected)} cells -> {args.out}") + return 0 + contract = load_contract(args.contract) + if args.command == "lock-toolchain": + result = lock_toolchain( + contract, + args.cuber, + args.kissat, + args.march_cu, + args.repository_revision, + ) + elif args.command == "build-matrix": + result = build_matrix( + contract, + read_jsonl(args.manifest), + args.calibration_root, + load_json(args.toolchain), + ) + else: + raise MatrixError(f"unsupported command {args.command}") + write_json(args.out, result) + except (MatrixError, HardRegimeError, OSError, ValueError, json.JSONDecodeError) as exc: + parser.error(str(exc)) + print(f"PASS {result['kind']}: {args.out}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/benchmarks/cnc/hpc/build_matrix.sh b/benchmarks/cnc/hpc/build_matrix.sh new file mode 100755 index 0000000..66e5544 --- /dev/null +++ b/benchmarks/cnc/hpc/build_matrix.sh @@ -0,0 +1,25 @@ +#!/bin/bash +set -euo pipefail + +: "${BI51_REPO:?set BI51_REPO to the repository checkout}" +: "${BI51_ARTIFACT_ROOT:?set BI51_ARTIFACT_ROOT to the hard-regime artifact root}" + +cd "$BI51_REPO" +source .venv/bin/activate + +python -m benchmarks.cnc.hard_regime_matrix build-matrix \ + benchmarks/cnc/contracts/hard-regime-v1.yaml \ + "$BI51_ARTIFACT_ROOT/instances/manifest.jsonl" \ + --calibration-root "$BI51_ARTIFACT_ROOT/calibration" \ + --toolchain "$BI51_ARTIFACT_ROOT/toolchain.json" \ + --out "$BI51_ARTIFACT_ROOT/run-matrix.json" + +python -m benchmarks.cnc.hard_regime_matrix list-cells \ + "$BI51_ARTIFACT_ROOT/run-matrix.json" --set monolithic \ + --out "$BI51_ARTIFACT_ROOT/monolithic-cells.txt" +python -m benchmarks.cnc.hard_regime_matrix list-cells \ + "$BI51_ARTIFACT_ROOT/run-matrix.json" --set pilot-cnc \ + --out "$BI51_ARTIFACT_ROOT/pilot-cnc-cells.txt" +python -m benchmarks.cnc.hard_regime_matrix list-cells \ + "$BI51_ARTIFACT_ROOT/run-matrix.json" --set remaining-cnc \ + --out "$BI51_ARTIFACT_ROOT/remaining-cnc-cells.txt" diff --git a/benchmarks/cnc/hpc/calibrate.sh b/benchmarks/cnc/hpc/calibrate.sh new file mode 100755 index 0000000..ad5719d --- /dev/null +++ b/benchmarks/cnc/hpc/calibrate.sh @@ -0,0 +1,27 @@ +#!/bin/bash +set -euo pipefail + +: "${BI51_REPO:?set BI51_REPO to the repository checkout}" +: "${BI51_ARTIFACT_ROOT:?set BI51_ARTIFACT_ROOT to the hard-regime artifact root}" +: "${SLURM_ARRAY_TASK_ID:?calibration must run as a six-element Slurm array}" + +case "$SLURM_ARRAY_TASK_ID" in + 0) PRODUCT_WIDTH=64; SELECTOR=region ;; + 1) PRODUCT_WIDTH=64; SELECTOR=structure-blind ;; + 2) PRODUCT_WIDTH=72; SELECTOR=region ;; + 3) PRODUCT_WIDTH=72; SELECTOR=structure-blind ;; + 4) PRODUCT_WIDTH=80; SELECTOR=region ;; + 5) PRODUCT_WIDTH=80; SELECTOR=structure-blind ;; + *) echo "invalid calibration array index: $SLURM_ARRAY_TASK_ID" >&2; exit 2 ;; +esac + +cd "$BI51_REPO" +source .venv/bin/activate + +python -m benchmarks.cnc.calibrate_hard_regime \ + benchmarks/cnc/contracts/hard-regime-v1.yaml \ + "$BI51_ARTIFACT_ROOT/instances/manifest.jsonl" \ + --product-width "$PRODUCT_WIDTH" \ + --selector "$SELECTOR" \ + --cuber target/release/cnc_cuber \ + --out-dir "$BI51_ARTIFACT_ROOT/calibration/p${PRODUCT_WIDTH}/${SELECTOR}" diff --git a/benchmarks/cnc/hpc/materialize.sh b/benchmarks/cnc/hpc/materialize.sh new file mode 100755 index 0000000..225fe87 --- /dev/null +++ b/benchmarks/cnc/hpc/materialize.sh @@ -0,0 +1,20 @@ +#!/bin/bash +set -euo pipefail + +: "${BI51_REPO:?set BI51_REPO to the repository checkout}" +: "${BI51_ARTIFACT_ROOT:?set BI51_ARTIFACT_ROOT to the hard-regime artifact root}" + +cd "$BI51_REPO" +source .venv/bin/activate + +python -m benchmarks.cnc.hard_regime materialize \ + benchmarks/cnc/contracts/hard-regime-v1.yaml \ + --out-dir "$BI51_ARTIFACT_ROOT/instances" + +python -m benchmarks.cnc.hard_regime_matrix lock-toolchain \ + benchmarks/cnc/contracts/hard-regime-v1.yaml \ + --cuber target/release/cnc_cuber \ + --kissat cnc-tools/bin/kissat-8af8e56f174b778aef3aa45af9f739b2a5f492c2 \ + --march-cu cnc-tools/bin/march_cu-705b60c6491ef2b61988b3ce6ac674be1b90571d \ + --repository-revision "$(git rev-parse HEAD)" \ + --out "$BI51_ARTIFACT_ROOT/toolchain.json" diff --git a/benchmarks/cnc/hpc/report.sh b/benchmarks/cnc/hpc/report.sh new file mode 100755 index 0000000..d75c58d --- /dev/null +++ b/benchmarks/cnc/hpc/report.sh @@ -0,0 +1,35 @@ +#!/bin/bash +set -euo pipefail + +: "${BI51_REPO:?set BI51_REPO to the repository checkout}" +: "${BI51_ARTIFACT_ROOT:?set BI51_ARTIFACT_ROOT to the hard-regime artifact root}" +: "${BI51_SCOPE:?set BI51_SCOPE to pilot or full}" + +cd "$BI51_REPO" +source .venv/bin/activate + +REPORT_ROOT="$BI51_ARTIFACT_ROOT/${BI51_SCOPE}-report" + +python -m benchmarks.cnc.verify_hard_regime \ + benchmarks/cnc/contracts/hard-regime-v1.yaml \ + "$BI51_ARTIFACT_ROOT/instances/manifest.jsonl" \ + --calibration-root "$BI51_ARTIFACT_ROOT/calibration" \ + --toolchain "$BI51_ARTIFACT_ROOT/toolchain.json" \ + --matrix "$BI51_ARTIFACT_ROOT/run-matrix.json" \ + --runs-root "$BI51_ARTIFACT_ROOT/runs" \ + --scope "$BI51_SCOPE" + +python -m benchmarks.cnc.aggregate_hard_regime \ + benchmarks/cnc/contracts/hard-regime-v1.yaml \ + "$BI51_ARTIFACT_ROOT/run-matrix.json" \ + --runs-root "$BI51_ARTIFACT_ROOT/runs" \ + --scope "$BI51_SCOPE" --out-dir "$REPORT_ROOT" + +python -m benchmarks.cnc.verify_hard_regime \ + benchmarks/cnc/contracts/hard-regime-v1.yaml \ + "$BI51_ARTIFACT_ROOT/instances/manifest.jsonl" \ + --calibration-root "$BI51_ARTIFACT_ROOT/calibration" \ + --toolchain "$BI51_ARTIFACT_ROOT/toolchain.json" \ + --matrix "$BI51_ARTIFACT_ROOT/run-matrix.json" \ + --runs-root "$BI51_ARTIFACT_ROOT/runs" \ + --scope "$BI51_SCOPE" --aggregate "$REPORT_ROOT/aggregate.json" diff --git a/benchmarks/cnc/hpc/run_cell.sh b/benchmarks/cnc/hpc/run_cell.sh new file mode 100755 index 0000000..e2ec860 --- /dev/null +++ b/benchmarks/cnc/hpc/run_cell.sh @@ -0,0 +1,27 @@ +#!/bin/bash +set -euo pipefail + +: "${BI51_REPO:?set BI51_REPO to the repository checkout}" +: "${BI51_ARTIFACT_ROOT:?set BI51_ARTIFACT_ROOT to the hard-regime artifact root}" +: "${BI51_CELL_LIST:?set BI51_CELL_LIST to an immutable cell-list text file}" +: "${BI51_SCRATCH_ROOT:?set BI51_SCRATCH_ROOT to an HPC2 SSD scratch directory}" +: "${SLURM_ARRAY_TASK_ID:?cell execution must run as a Slurm array}" + +LINE_NUMBER=$((SLURM_ARRAY_TASK_ID + 1)) +CELL_ID=$(sed -n "${LINE_NUMBER}p" "$BI51_CELL_LIST") +if [[ -z "$CELL_ID" ]]; then + echo "no cell at array index $SLURM_ARRAY_TASK_ID" >&2 + exit 2 +fi + +cd "$BI51_REPO" +source .venv/bin/activate + +python -m benchmarks.cnc.run_hard_regime_cell \ + benchmarks/cnc/contracts/hard-regime-v1.yaml \ + "$BI51_ARTIFACT_ROOT/run-matrix.json" \ + "$BI51_ARTIFACT_ROOT/toolchain.json" \ + --cell-id "$CELL_ID" \ + --instance-root "$BI51_ARTIFACT_ROOT/instances" \ + --output-root "$BI51_ARTIFACT_ROOT/runs" \ + --temp-root "$BI51_SCRATCH_ROOT" diff --git a/benchmarks/cnc/hpc/smoke.sh b/benchmarks/cnc/hpc/smoke.sh new file mode 100755 index 0000000..fda312b --- /dev/null +++ b/benchmarks/cnc/hpc/smoke.sh @@ -0,0 +1,19 @@ +#!/bin/bash +set -euo pipefail + +: "${BI51_REPO:?set BI51_REPO to the repository checkout}" +: "${BI51_ARTIFACT_ROOT:?set BI51_ARTIFACT_ROOT to the hard-regime artifact root}" + +cd "$BI51_REPO" +source .venv/bin/activate + +python -m benchmarks.cnc.hard_regime validate-contract \ + benchmarks/cnc/contracts/hard-regime-v1.yaml +python -m pytest tests/test_hard_regime.py tests/test_calibrate_hard_regime.py \ + tests/test_hard_regime_matrix.py tests/test_run_hard_regime_cell.py \ + tests/test_verify_hard_regime.py tests/test_aggregate_hard_regime.py -q +cargo test --test cnc_cuber_trace +cargo build --release --bin cnc_cuber +python -m benchmarks.cnc.hard_regime generate-targets \ + benchmarks/cnc/contracts/hard-regime-v1.yaml \ + --out "$BI51_ARTIFACT_ROOT/smoke-targets.jsonl" diff --git a/benchmarks/cnc/requirements.txt b/benchmarks/cnc/requirements.txt new file mode 100644 index 0000000..c99f4fb --- /dev/null +++ b/benchmarks/cnc/requirements.txt @@ -0,0 +1,2 @@ +PyYAML==6.0.3 +sympy==1.14.0 diff --git a/benchmarks/cnc/run_hard_regime_cell.py b/benchmarks/cnc/run_hard_regime_cell.py new file mode 100755 index 0000000..3256d36 --- /dev/null +++ b/benchmarks/cnc/run_hard_regime_cell.py @@ -0,0 +1,620 @@ +#!/usr/bin/env python3 +"""Execute one frozen hard-regime run-matrix cell and write a terminal record.""" + +from __future__ import annotations + +import argparse +import json +import os +import re +import resource +import socket +import subprocess +import time +from collections import defaultdict +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +from benchmarks.cnc.conquer_parallel import ( + parse_cnf, + parse_stats, + read_cubes, + run_arm, +) +from benchmarks.cnc.hard_regime import HardRegimeError, contract_sha256, load_contract +from benchmarks.cnc.hard_regime_matrix import MatrixError, verify_toolchain +from benchmarks.pipeline.circuit import ( + canonical_bytes, + load_json, + read_jsonl, + sha256_bytes, + sha256_file, + write_json, +) + + +_CUBER_STATS = re.compile(r"status=OK cubes=(\d+)") + + +class CellError(HardRegimeError): + """A frozen cell cannot be executed or its output is inconsistent.""" + + +def utc_now() -> str: + return datetime.now(timezone.utc).isoformat() + + +def _as_text(value: str | bytes | None) -> str: + if value is None: + return "" + return value.decode(errors="replace") if isinstance(value, bytes) else value + + +def run_process( + command: list[str], timeout_s: float, stdout_path: Path, stderr_path: Path +) -> dict[str, Any]: + stdout_path.parent.mkdir(parents=True, exist_ok=True) + before = resource.getrusage(resource.RUSAGE_CHILDREN) + started = time.monotonic() + started_utc = utc_now() + try: + process = subprocess.run( + command, + capture_output=True, + text=True, + timeout=timeout_s, + check=False, + ) + stdout = process.stdout + stderr = process.stderr + returncode = process.returncode + state = "finished" + except subprocess.TimeoutExpired as exc: + stdout = _as_text(exc.stdout) + stderr = _as_text(exc.stderr) + returncode = None + state = "timeout" + elapsed_s = time.monotonic() - started + after = resource.getrusage(resource.RUSAGE_CHILDREN) + stdout_path.write_text(stdout, encoding="utf-8") + stderr_path.write_text(stderr, encoding="utf-8") + return { + "command": command, + "state": state, + "returncode": returncode, + "started_utc": started_utc, + "finished_utc": utc_now(), + "wall_s": elapsed_s, + "user_s": after.ru_utime - before.ru_utime, + "system_s": after.ru_stime - before.ru_stime, + "cpu_s": (after.ru_utime - before.ru_utime) + + (after.ru_stime - before.ru_stime), + "stdout": stdout_path.name, + "stdout_sha256": sha256_file(stdout_path), + "stderr": stderr_path.name, + "stderr_sha256": sha256_file(stderr_path), + } + + +def verify_region_trace(frontier: Path, trace: Path) -> dict[str, int]: + cubes = list(read_cubes(frontier)) + nodes = read_jsonl(trace) + if not nodes: + raise CellError("region trace is empty") + children: dict[int, list[dict[str, Any]]] = defaultdict(list) + cutoff_literals = [] + refutation_counts: dict[str, int] = defaultdict(int) + variable_count = None + for expected_id, node in enumerate(nodes): + if node.get("node_id") != expected_id: + raise CellError("region trace node IDs are not contiguous") + literals = node.get("literals") + if not isinstance(literals, list) or not all( + isinstance(literal, int) and not isinstance(literal, bool) and literal != 0 + for literal in literals + ): + raise CellError(f"region trace node {expected_id} has invalid literals") + if len({abs(literal) for literal in literals}) != len(literals): + raise CellError(f"region trace node {expected_id} repeats a decision variable") + sigma_dec = node.get("sigma_dec") + sigma_all = node.get("sigma_all") + freevars = node.get("freevars") + if ( + sigma_dec != len(literals) + or not isinstance(sigma_all, int) + or isinstance(sigma_all, bool) + or not isinstance(freevars, int) + or isinstance(freevars, bool) + or sigma_all < sigma_dec + or freevars < 0 + ): + raise CellError(f"region trace node {expected_id} has invalid assignment counts") + node_variables = sigma_all + freevars + if variable_count is None: + variable_count = node_variables + elif node_variables != variable_count: + raise CellError("region trace changes the declared variable count") + + parent = node.get("parent_id") + if expected_id == 0: + if parent is not None or node.get("child_index") is not None or node.get("depth") != 0: + raise CellError("region trace has an invalid root") + else: + if not isinstance(parent, int) or parent < 0 or parent >= expected_id: + raise CellError(f"region trace node {expected_id} has an invalid parent") + if node.get("depth") != nodes[parent].get("depth") + 1: + raise CellError(f"region trace node {expected_id} has an invalid depth") + children[parent].append(node) + kind = node.get("kind") + if kind not in {"branch", "cutoff", "refuted", "sat"}: + raise CellError(f"region trace node {expected_id} has invalid kind {kind!r}") + reason = node.get("refutation_reason") + if kind == "refuted": + if reason not in { + "root-propagation-contradiction", + "selector-no-feasible-config", + "branch-propagation-contradiction", + }: + raise CellError(f"region refuted node {expected_id} has no closure reason") + refutation_counts[reason] += 1 + elif reason is not None: + raise CellError(f"region non-refuted node {expected_id} has a closure reason") + if kind == "sat": + raise CellError("region trace contains a SAT leaf for an expected-UNSAT target") + if kind == "cutoff": + cutoff_literals.append(literals) + for node in nodes: + node_id = node["node_id"] + actual_children = children.get(node_id, []) + if node["kind"] == "branch": + variables = node.get("rule_variables") + clauses = node.get("rule_clauses") + if ( + not isinstance(variables, list) + or not variables + or len(variables) > 64 + or not all(isinstance(variable, int) and variable > 0 for variable in variables) + or len(set(variables)) != len(variables) + or any(variable in {abs(literal) for literal in node["literals"]} for variable in variables) + ): + raise CellError(f"region branch node {node_id} has invalid rule variables") + if not isinstance(clauses, list) or not clauses or len(actual_children) != len(clauses): + raise CellError(f"region branch node {node_id} has incomplete children") + indices = sorted(child.get("child_index") for child in actual_children) + if indices != list(range(len(clauses))): + raise CellError(f"region branch node {node_id} has invalid child indices") + by_index = {child["child_index"]: child for child in actual_children} + variable_mask = (1 << len(variables)) - 1 + for index, clause in enumerate(clauses): + if not isinstance(clause, dict): + raise CellError(f"region branch node {node_id} has a malformed rule clause") + mask = clause.get("mask") + value = clause.get("value") + if ( + not isinstance(mask, int) + or isinstance(mask, bool) + or not isinstance(value, int) + or isinstance(value, bool) + or mask <= 0 + or mask & ~variable_mask + or value & ~mask + ): + raise CellError(f"region branch node {node_id} has an invalid rule clause") + suffix = [ + variable if (value >> bit) & 1 else -variable + for bit, variable in enumerate(variables) + if (mask >> bit) & 1 + ] + if by_index[index].get("literals") != node["literals"] + suffix: + raise CellError( + f"region child {by_index[index]['node_id']} does not implement rule clause {index}" + ) + elif actual_children: + raise CellError(f"region leaf node {node_id} unexpectedly has children") + elif node.get("rule_clauses") not in ([], None): + raise CellError(f"region leaf node {node_id} unexpectedly has rule clauses") + if cutoff_literals != cubes: + raise CellError("region trace cutoff leaves do not reproduce frontier bytes") + return { + "nodes": len(nodes), + "branches": sum(node["kind"] == "branch" for node in nodes), + "cutoffs": len(cutoff_literals), + "refuted": sum(node["kind"] == "refuted" for node in nodes), + "sat_leaves": sum(node["kind"] == "sat" for node in nodes), + "root_refutations": refutation_counts["root-propagation-contradiction"], + "selector_refutations": refutation_counts["selector-no-feasible-config"], + "branch_refutations": refutation_counts["branch-propagation-contradiction"], + } + + +def artifact(path: Path, root: Path) -> dict[str, Any]: + return { + "path": str(path.relative_to(root)), + "sha256": sha256_file(path), + "bytes": path.stat().st_size, + } + + +def select_cell(matrix: dict[str, Any], cell_id: str) -> dict[str, Any]: + if matrix.get("schema_version") != 1 or matrix.get("kind") != "hard-regime-run-matrix": + raise CellError("unsupported run matrix") + matches = [cell for cell in matrix.get("cells", []) if cell.get("cell_id") == cell_id] + if len(matches) != 1: + raise CellError(f"run matrix contains {len(matches)} matches for {cell_id!r}") + return matches[0] + + +def resolve_input(root: Path, relative: object, expected_sha256: object) -> Path: + if not isinstance(relative, str) or not isinstance(expected_sha256, str): + raise CellError("cell input provenance is incomplete") + path = (root / relative).resolve() + resolved_root = root.resolve() + if path != resolved_root and resolved_root not in path.parents: + raise CellError(f"cell input escapes instance root: {relative}") + if sha256_file(path) != expected_sha256: + raise CellError(f"cell input hash mismatch: {relative}") + return path + + +def slurm_context() -> dict[str, str]: + names = ( + "SLURM_JOB_ID", + "SLURM_ARRAY_JOB_ID", + "SLURM_ARRAY_TASK_ID", + "SLURM_JOB_PARTITION", + "SLURM_CPUS_PER_TASK", + "SLURM_NTASKS", + "SLURM_MEM_PER_NODE", + "SLURM_TIMELIMIT", + ) + return {name: os.environ[name] for name in names if name in os.environ} + + +def base_terminal( + contract: dict[str, Any], + matrix_path: Path, + matrix: dict[str, Any], + toolchain: dict[str, Any], + cell: dict[str, Any], +) -> dict[str, Any]: + return { + "schema_version": 1, + "kind": "hard-regime-terminal-cell", + "contract_sha256": contract_sha256(contract), + "matrix_sha256": sha256_file(matrix_path), + "toolchain_sha256": toolchain["toolchain_sha256"], + "cell_id": cell["cell_id"], + "cell_sha256": sha256_bytes(canonical_bytes(cell)), + "instance_id": cell["instance_id"], + "split": cell["split"], + "product_width": cell["product_width"], + "factor_input_width": cell["factor_input_width"], + "method": cell["method"], + "budget": cell["budget"], + "expected_outcome": cell["expected_outcome"], + "input_artifacts": { + "circuitsat": { + "path": cell["circuitsat"], + "sha256": cell["circuitsat_sha256"], + }, + "global_cnf": { + "path": cell["global_cnf"], + "sha256": cell["global_cnf_sha256"], + }, + }, + "host": socket.gethostname(), + "slurm": slurm_context(), + "started_utc": utc_now(), + } + + +def run_monolithic( + cell: dict[str, Any], cnf: Path, kissat: Path, cell_dir: Path +) -> dict[str, Any]: + stage = run_process( + [str(kissat), "--statistics", "--relaxed", str(cnf)], + float(cell["time_limit_s"]), + cell_dir / "monolithic.stdout", + cell_dir / "monolithic.stderr", + ) + stdout = (cell_dir / "monolithic.stdout").read_text(encoding="utf-8") + decisions, conflicts = parse_stats(stdout) + verdict = {10: "sat", 20: "unsat"}.get(stage["returncode"]) + state = ( + "complete" + if verdict == "unsat" + else "wrong-answer" + if verdict == "sat" + else "monolithic-timeout" + if stage["state"] == "timeout" + else "monolithic-error" + ) + return { + "state": state, + "verdict": verdict, + "stages": {"monolithic": stage}, + "metrics": { + "encoding_wall_s": cell["encoding_wall_s"], + "encoding_cpu_s": cell["encoding_cpu_s"], + "solver_wall_s": stage["wall_s"], + "solver_cpu_s": stage["cpu_s"], + "decisions": decisions, + "conflicts": conflicts, + "end_to_end_wall_s": cell["encoding_wall_s"] + stage["wall_s"], + "end_to_end_cpu_s": cell["encoding_cpu_s"] + stage["cpu_s"], + "censored": stage["state"] == "timeout", + }, + } + + +def run_cubing( + cell: dict[str, Any], + circuitsat: Path, + cnf: Path, + tools: dict[str, Any], + cell_dir: Path, +) -> tuple[dict[str, Any], Path, Path | None]: + frontier = cell_dir / "frontier.icnf" + trace: Path | None = None + if cell["method"] == "march-cu-dynamic": + command = [tools["march_cu"]["path"], str(cnf), "-o", str(frontier)] + else: + trace = cell_dir / "nodes.jsonl" + command = [ + tools["cnc_cuber"]["path"], + str(circuitsat), + "--cc-threshold", + str(cell["cc_threshold"]), + "-o", + str(frontier), + "--selector", + cell["selector"], + "--max-rows", + str(cell["max_rows"]), + "--trace", + str(trace), + ] + stage = run_process( + command, + float(cell["cubing_time_limit_s"]), + cell_dir / "cubing.stdout", + cell_dir / "cubing.stderr", + ) + stage["complete"] = stage["state"] == "finished" and stage["returncode"] == 0 + if not stage["complete"]: + return stage, frontier, trace + if not frontier.is_file(): + raise CellError("cuber reported success without a frontier") + cubes = sum(1 for _ in read_cubes(frontier)) + stage["frontier_size"] = cubes + stage["frontier_bytes"] = frontier.stat().st_size + stage["frontier_sha256"] = sha256_file(frontier) + if trace is not None: + trace_summary = verify_region_trace(frontier, trace) + stderr = (cell_dir / "cubing.stderr").read_text(encoding="utf-8") + match = _CUBER_STATS.search(stderr) + if not match or int(match.group(1)) != cubes: + raise CellError("region cuber log/frontier task counts disagree") + stage["completeness"] = { + "complete": True, + "evidence": "verified-branch-assignments-and-refutation-reasons", + **trace_summary, + } + else: + stage["completeness"] = { + "complete": True, + "evidence": "upstream-march-cu-successful-partition-output", + } + return stage, frontier, trace + + +def run_cnc( + contract: dict[str, Any], + cell: dict[str, Any], + circuitsat: Path, + cnf: Path, + tools: dict[str, Any], + cell_dir: Path, + temp_root: Path | None, +) -> dict[str, Any]: + cubing, frontier, trace = run_cubing(cell, circuitsat, cnf, tools, cell_dir) + artifacts = {} + if frontier.is_file(): + artifacts["frontier"] = artifact(frontier, cell_dir) + if trace is not None and trace.is_file(): + artifacts["trace"] = artifact(trace, cell_dir) + if not cubing["complete"]: + state = "cubing-timeout" if cubing["state"] == "timeout" else "cubing-error" + return { + "state": state, + "verdict": None, + "stages": {"cubing": cubing}, + "artifacts": artifacts, + "metrics": { + "encoding_wall_s": cell["encoding_wall_s"], + "encoding_cpu_s": cell["encoding_cpu_s"], + "cubing_wall_s": cubing["wall_s"], + "cubing_cpu_s": cubing["cpu_s"], + "end_to_end_wall_s": cell["encoding_wall_s"] + cubing["wall_s"], + "end_to_end_cpu_s": cell["encoding_cpu_s"] + cubing["cpu_s"], + "censored": cubing["state"] == "timeout", + }, + } + + cnf_data = cnf.read_bytes() + variables, clauses, body = parse_cnf(cnf_data) + raw_results = cell_dir / "cube-results.jsonl" + temp_dir = ( + temp_root / cell["cell_id"] if temp_root is not None else cell_dir / "tmp" + ) + temp_dir.mkdir(parents=True, exist_ok=True) + workers = int(contract["scheduling"]["measured_workers"]) + replay_workers = list(contract["scheduling"]["lpt_replay_workers"]) + total_cubes = sum(1 for _ in read_cubes(frontier)) + conquer = run_arm( + cell["cell_id"], + read_cubes(frontier), + total_cubes, + workers, + replay_workers, + raw_results, + ( + variables, + clauses, + body, + tools["kissat"]["path"], + float(cell["per_cube_time_limit_s"]), + str(temp_dir), + ), + ) + artifacts["cube_results"] = artifact(raw_results, cell_dir) + state = ( + "complete" + if conquer["complete"] and conquer["result"] == "unsat" + else "wrong-answer" + if conquer["result"] == "sat" + else "conquer-timeout" + if conquer["timeouts"] + else "conquer-error" + ) + end_wall = ( + float(cell["encoding_wall_s"]) + + float(cubing["wall_s"]) + + float(conquer["measured_makespan_s"]) + ) + end_cpu = ( + float(cell["encoding_cpu_s"]) + + float(cubing["cpu_s"]) + + float(conquer["total_cpu_s"]) + ) + return { + "state": state, + "verdict": conquer["result"] if conquer["complete"] else None, + "stages": {"cubing": cubing, "conquer": conquer}, + "artifacts": artifacts, + "metrics": { + "encoding_wall_s": cell["encoding_wall_s"], + "encoding_cpu_s": cell["encoding_cpu_s"], + "cubing_wall_s": cubing["wall_s"], + "cubing_cpu_s": cubing["cpu_s"], + "frontier_size": cubing["frontier_size"], + "frontier_bytes": cubing["frontier_bytes"], + "conquer_work_cpu_s": conquer["total_cpu_s"], + "conquer_span_s": conquer["max_s"], + "p99_s": conquer["p99_s"], + "maximum_conflicts": conquer["conflicts_max"], + "timeout_count": conquer["timeouts"], + "measured_32_worker_makespan_s": conquer["measured_makespan_s"], + "lpt_makespan_by_workers_s": conquer["lpt_makespan_by_workers_s"], + "end_to_end_wall_s": end_wall, + "end_to_end_cpu_s": end_cpu, + "censored": conquer["censored"], + }, + } + + +def atomic_terminal(path: Path, value: dict[str, Any]) -> None: + temporary = path.with_suffix(".tmp") + write_json(temporary, value) + os.replace(temporary, path) + + +def run_cell( + contract: dict[str, Any], + matrix_path: Path, + matrix: dict[str, Any], + toolchain: dict[str, Any], + cell: dict[str, Any], + instance_root: Path, + output_root: Path, + temp_root: Path | None = None, +) -> dict[str, Any]: + if matrix.get("contract_sha256") != contract_sha256(contract): + raise CellError("run matrix contract hash mismatch") + if matrix.get("toolchain_sha256") != toolchain.get("toolchain_sha256"): + raise CellError("run matrix toolchain hash mismatch") + verify_toolchain(contract, toolchain, check_paths=True) + required_cpus = int(cell["required_cpus"]) + allocated = os.environ.get("SLURM_CPUS_PER_TASK") + if allocated is not None and int(allocated) < required_cpus: + raise CellError( + f"cell needs {required_cpus} CPUs but SLURM_CPUS_PER_TASK={allocated}" + ) + cell_dir = output_root / "cells" / cell["cell_id"] + cell_dir.mkdir(parents=True, exist_ok=True) + terminal_path = cell_dir / "terminal.json" + base = base_terminal(contract, matrix_path, matrix, toolchain, cell) + if terminal_path.is_file(): + existing = load_json(terminal_path) + if existing.get("cell_sha256") != base["cell_sha256"]: + raise CellError("existing terminal record belongs to a different cell lock") + return existing + + try: + circuitsat = resolve_input( + instance_root, cell["circuitsat"], cell["circuitsat_sha256"] + ) + cnf = resolve_input( + instance_root, cell["global_cnf"], cell["global_cnf_sha256"] + ) + tools = toolchain["tools"] + if cell["method"] == "monolithic-kissat": + result = run_monolithic(cell, cnf, Path(tools["kissat"]["path"]), cell_dir) + else: + result = run_cnc( + contract, cell, circuitsat, cnf, tools, cell_dir, temp_root + ) + terminal = { + **base, + **result, + "finished_utc": utc_now(), + } + except Exception as exc: + terminal = { + **base, + "state": "harness-error", + "verdict": None, + "error": f"{type(exc).__name__}: {exc}", + "finished_utc": utc_now(), + } + atomic_terminal(terminal_path, terminal) + raise + atomic_terminal(terminal_path, terminal) + return terminal + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("contract", type=Path) + parser.add_argument("matrix", type=Path) + parser.add_argument("toolchain", type=Path) + parser.add_argument("--cell-id", required=True) + parser.add_argument("--instance-root", type=Path, required=True) + parser.add_argument("--output-root", type=Path, required=True) + parser.add_argument("--temp-root", type=Path) + args = parser.parse_args() + try: + contract = load_contract(args.contract) + matrix = load_json(args.matrix) + toolchain = load_json(args.toolchain) + cell = select_cell(matrix, args.cell_id) + terminal = run_cell( + contract, + args.matrix, + matrix, + toolchain, + cell, + args.instance_root, + args.output_root, + args.temp_root, + ) + except (CellError, MatrixError, OSError, ValueError, json.JSONDecodeError) as exc: + print(f"FAIL cell {args.cell_id}: {exc}") + return 2 + print(f"TERMINAL {terminal['cell_id']}: {terminal['state']}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/benchmarks/cnc/verify_hard_regime.py b/benchmarks/cnc/verify_hard_regime.py new file mode 100755 index 0000000..aae42ef --- /dev/null +++ b/benchmarks/cnc/verify_hard_regime.py @@ -0,0 +1,440 @@ +#!/usr/bin/env python3 +"""Verify issue #51 contracts, calibration locks, terminal cells, and accounting.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import math +from collections import Counter +from pathlib import Path +from typing import Any + +from benchmarks.cnc.conquer_parallel import distribution, lpt_makespan, read_cubes +from benchmarks.cnc.hard_regime import HardRegimeError, contract_sha256, load_contract +from benchmarks.cnc.hard_regime_matrix import ( + MatrixError, + build_matrix, + validate_instance_manifest, + verify_toolchain, +) +from benchmarks.cnc.run_hard_regime_cell import verify_region_trace +from benchmarks.pipeline.circuit import ( + canonical_bytes, + load_json, + read_jsonl, + sha256_bytes, + sha256_file, +) + + +class VerificationError(HardRegimeError): + """The hard-regime evidence bundle is incomplete or inconsistent.""" + + +def close(actual: object, expected: float, label: str) -> None: + if not isinstance(actual, (int, float)) or isinstance(actual, bool): + raise VerificationError(f"{label} is not numeric") + if math.isnan(expected): + if not math.isnan(float(actual)): + raise VerificationError(f"{label} should be NaN") + elif not math.isclose(float(actual), expected, rel_tol=1e-8, abs_tol=1e-8): + raise VerificationError(f"{label} mismatch: {actual} != {expected}") + + +def close_optional(actual: object, expected: float | None, label: str) -> None: + if expected is None: + if actual is not None: + raise VerificationError(f"{label} should be null") + return + close(actual, expected, label) + + +def checked_relative(root: Path, spec: object, label: str) -> Path: + if not isinstance(spec, dict): + raise VerificationError(f"{label} artifact spec is missing") + relative = spec.get("path") + expected = spec.get("sha256") + if not isinstance(relative, str) or not isinstance(expected, str): + raise VerificationError(f"{label} artifact provenance is incomplete") + root = root.resolve() + path = (root / relative).resolve() + if path != root and root not in path.parents: + raise VerificationError(f"{label} artifact escapes its cell directory") + if sha256_file(path) != expected: + raise VerificationError(f"{label} artifact SHA-256 mismatch") + if spec.get("bytes") != path.stat().st_size: + raise VerificationError(f"{label} artifact byte count mismatch") + return path + + +def verify_stage_logs(cell_dir: Path, stage: dict[str, Any], label: str) -> None: + for stream in ("stdout", "stderr"): + relative = stage.get(stream) + expected = stage.get(f"{stream}_sha256") + if not isinstance(relative, str) or not isinstance(expected, str): + raise VerificationError(f"{label} stage has incomplete {stream} provenance") + path = cell_dir / relative + if sha256_file(path) != expected: + raise VerificationError(f"{label} stage {stream} hash mismatch") + + +def verify_conquer_records( + frontier: Path, + results: Path, + summary: dict[str, Any], + workers: int, + replay_workers: list[int], +) -> None: + cubes = list(read_cubes(frontier)) + rows = read_jsonl(results) + if len(rows) != len(cubes) or summary.get("cubes") != len(cubes): + raise VerificationError("conquer records do not cover every frontier cube") + by_index = {} + intervals = [] + per_worker: dict[int, list[tuple[int, int]]] = {} + for row in rows: + index = row.get("cube_index") + if not isinstance(index, int) or index < 0 or index >= len(cubes) or index in by_index: + raise VerificationError("conquer records contain invalid or duplicate cube indices") + by_index[index] = row + expected_cube_hash = hashlib.sha256( + (" ".join(map(str, cubes[index])) + " 0\n").encode() + ).hexdigest() + if row.get("cube_sha256") != expected_cube_hash: + raise VerificationError(f"cube {index} assumption hash mismatch") + released = row.get("released_monotonic_ns") + started = row.get("started_monotonic_ns") + finished = row.get("finished_monotonic_ns") + collected = row.get("collected_monotonic_ns") + worker = row.get("worker_pid") + if not all(isinstance(value, int) for value in (released, started, finished, collected, worker)): + raise VerificationError(f"cube {index} has malformed scheduling events") + if not released <= started <= finished <= collected: + raise VerificationError(f"cube {index} has illegal scheduling event order") + intervals.append((started, 1)) + intervals.append((finished, -1)) + per_worker.setdefault(worker, []).append((started, finished)) + if set(by_index) != set(range(len(cubes))): + raise VerificationError("conquer cube indices are not exhaustive") + active = maximum = 0 + for _, delta in sorted(intervals, key=lambda event: (event[0], event[1])): + active += delta + if active < 0: + raise VerificationError("conquer schedule has a finish before its start") + maximum = max(maximum, active) + if active or maximum > workers: + raise VerificationError("conquer schedule exceeds the declared worker count") + for worker, assigned in per_worker.items(): + assigned.sort() + if any(left[1] > right[0] for left, right in zip(assigned, assigned[1:])): + raise VerificationError(f"worker {worker} has overlapping cube assignments") + + ordered = [by_index[index] for index in range(len(cubes))] + durations = [float(row["elapsed_s"]) for row in ordered] + cpu = [float(row["user_s"]) + float(row["system_s"]) for row in ordered] + conflicts = [float(row["conflicts"]) for row in ordered if row.get("conflicts") is not None] + counts = { + "timeouts": sum(bool(row.get("censored")) for row in ordered), + "errors": sum(row.get("result") == "error" for row in ordered), + "sat": sum(row.get("result") == "sat" for row in ordered), + "unsat": sum(row.get("result") == "unsat" for row in ordered), + } + counts["completed"] = counts["sat"] + counts["unsat"] + for name, value in counts.items(): + if summary.get(name) != value: + raise VerificationError(f"conquer {name} count does not reconstruct") + if summary.get("terminal_records") != len(ordered): + raise VerificationError("conquer terminal-record count does not reconstruct") + time_stats = distribution(durations) + conflict_stats = distribution(conflicts) + close(summary.get("total_solver_s"), sum(durations), "conquer total solver work") + close(summary.get("total_cpu_s"), sum(cpu), "conquer total CPU work") + close(summary.get("max_s"), time_stats["max"], "conquer span") + close(summary.get("p99_s"), time_stats["p99"], "conquer p99") + close_optional( + summary.get("conflicts_max"), + conflict_stats["max"], + "conquer maximum conflicts", + ) + measured = ( + 0.0 + if not rows + else ( + max(int(row["collected_monotonic_ns"]) for row in rows) + - min(int(row["released_monotonic_ns"]) for row in rows) + ) + / 1e9 + ) + close(summary.get("measured_makespan_s"), measured, "measured makespan") + expected_lpt = {str(count): lpt_makespan(durations, count) for count in replay_workers} + actual_lpt = summary.get("lpt_makespan_by_workers_s") + if not isinstance(actual_lpt, dict) or set(actual_lpt) != set(expected_lpt): + raise VerificationError("LPT replay worker set differs from the contract") + for count, value in expected_lpt.items(): + close(actual_lpt[count], value, f"LPT {count}-worker makespan") + if summary.get("lpt_is_lower_bound") != bool(counts["timeouts"] or counts["errors"]): + raise VerificationError("LPT censoring designation is incorrect") + + +def verify_terminal( + contract: dict[str, Any], + matrix_path: Path, + matrix: dict[str, Any], + toolchain: dict[str, Any], + cell: dict[str, Any], + terminal: dict[str, Any], + cell_dir: Path, +) -> str: + expected = { + "kind": "hard-regime-terminal-cell", + "contract_sha256": contract_sha256(contract), + "matrix_sha256": sha256_file(matrix_path), + "toolchain_sha256": toolchain["toolchain_sha256"], + "cell_id": cell["cell_id"], + "cell_sha256": sha256_bytes(canonical_bytes(cell)), + "instance_id": cell["instance_id"], + "method": cell["method"], + "budget": cell["budget"], + "product_width": cell["product_width"], + "factor_input_width": cell["factor_input_width"], + } + for field, value in expected.items(): + if terminal.get(field) != value: + raise VerificationError(f"{cell['cell_id']}: terminal {field} mismatch") + if terminal.get("state") not in { + "complete", + "monolithic-timeout", + "monolithic-error", + "cubing-timeout", + "cubing-error", + "conquer-timeout", + "conquer-error", + "wrong-answer", + "harness-error", + }: + raise VerificationError(f"{cell['cell_id']}: unknown terminal state") + if terminal.get("state") == "wrong-answer": + raise VerificationError(f"{cell['cell_id']}: solver returned SAT for a prime target") + inputs = terminal.get("input_artifacts") + if not isinstance(inputs, dict): + raise VerificationError(f"{cell['cell_id']}: terminal input hashes are missing") + if inputs.get("global_cnf", {}).get("sha256") != cell["global_cnf_sha256"]: + raise VerificationError(f"{cell['cell_id']}: mixed conquer encoding") + if inputs.get("circuitsat", {}).get("sha256") != cell["circuitsat_sha256"]: + raise VerificationError(f"{cell['cell_id']}: CircuitSAT input hash mismatch") + + stages = terminal.get("stages", {}) + metrics = terminal.get("metrics", {}) + artifacts = terminal.get("artifacts", {}) + if terminal["state"] == "harness-error": + if not isinstance(terminal.get("error"), str) or not terminal["error"]: + raise VerificationError(f"{cell['cell_id']}: harness error has no diagnostic") + return "harness-terminal-only" + if cell["method"] == "monolithic-kissat": + stage = stages.get("monolithic") + if not isinstance(stage, dict): + raise VerificationError(f"{cell['cell_id']}: missing monolithic stage") + verify_stage_logs(cell_dir, stage, "monolithic") + close( + metrics.get("end_to_end_wall_s"), + float(cell["encoding_wall_s"]) + float(stage["wall_s"]), + "monolithic end-to-end wall", + ) + close( + metrics.get("end_to_end_cpu_s"), + float(cell["encoding_cpu_s"]) + float(stage["cpu_s"]), + "monolithic end-to-end CPU", + ) + return "monolithic-stage-reconstructed" + + cubing = stages.get("cubing") + if not isinstance(cubing, dict): + raise VerificationError(f"{cell['cell_id']}: missing cubing stage") + verify_stage_logs(cell_dir, cubing, "cubing") + if not cubing.get("complete"): + if terminal["state"] not in {"cubing-timeout", "cubing-error"}: + raise VerificationError(f"{cell['cell_id']}: incomplete cubing has wrong terminal state") + return "cubing-stage-reconstructed" + frontier = checked_relative(cell_dir, artifacts.get("frontier"), "frontier") + if cubing.get("frontier_size") != sum(1 for _ in read_cubes(frontier)): + raise VerificationError(f"{cell['cell_id']}: frontier size mismatch") + if cell["method"] in {"region-cc", "structure-blind-cc"}: + trace = checked_relative(cell_dir, artifacts.get("trace"), "trace") + verify_region_trace(frontier, trace) + conquer = stages.get("conquer") + if not isinstance(conquer, dict): + if terminal["state"] not in {"cubing-timeout", "cubing-error"}: + raise VerificationError(f"{cell['cell_id']}: completed frontier has no conquer records") + return "cubing-stage-reconstructed" + results = checked_relative(cell_dir, artifacts.get("cube_results"), "cube results") + verify_conquer_records( + frontier, + results, + conquer, + int(contract["scheduling"]["measured_workers"]), + list(contract["scheduling"]["lpt_replay_workers"]), + ) + close(metrics.get("conquer_work_cpu_s"), float(conquer["total_cpu_s"]), "conquer work") + close(metrics.get("conquer_span_s"), float(conquer["max_s"]), "conquer span") + expected_wall = ( + float(cell["encoding_wall_s"]) + + float(cubing["wall_s"]) + + float(conquer["measured_makespan_s"]) + ) + close(metrics.get("end_to_end_wall_s"), expected_wall, "CnC end-to-end wall") + return "conquer-records-reconstructed" + + +def selected_cells(matrix: dict[str, Any], scope: str) -> list[dict[str, Any]]: + cells = matrix["cells"] + if scope == "full": + return cells + return [ + cell + for cell in cells + if cell["method"] == "monolithic-kissat" or cell.get("pilot") is True + ] + + +def verify_bundle( + contract_path: Path, + manifest_path: Path, + calibration_root: Path, + toolchain_path: Path, + matrix_path: Path, + runs_root: Path, + scope: str, + aggregate_path: Path | None = None, +) -> list[str]: + contract = load_contract(contract_path) + manifest = read_jsonl(manifest_path) + validate_instance_manifest(contract, manifest) + toolchain = load_json(toolchain_path) + verify_toolchain(contract, toolchain, check_paths=False) + matrix = load_json(matrix_path) + regenerated = build_matrix(contract, manifest, calibration_root, toolchain) + if matrix != regenerated: + raise VerificationError("run matrix does not regenerate from frozen inputs") + cells = selected_cells(matrix, scope) + missing = [] + reconstruction = Counter() + for cell in cells: + cell_dir = runs_root / "cells" / cell["cell_id"] + terminal_path = cell_dir / "terminal.json" + if not terminal_path.is_file(): + missing.append(cell["cell_id"]) + continue + reconstruction[ + verify_terminal( + contract, + matrix_path, + matrix, + toolchain, + cell, + load_json(terminal_path), + cell_dir, + ) + ] += 1 + if missing: + raise VerificationError( + f"missing terminal cells ({len(missing)}): {', '.join(missing[:3])}" + ) + thresholds = {} + for cell in matrix["cells"]: + if "cc_threshold" not in cell: + continue + key = (cell["product_width"], cell["method"], cell["budget"]) + previous = thresholds.setdefault(key, cell["cc_threshold"]) + if previous != cell["cc_threshold"]: + raise VerificationError("held-out cells contain per-instance threshold tuning") + messages = [ + f"PASS contract: {contract_sha256(contract)}", + "PASS widths/splits: factor-product semantics and calibration holdout are frozen", + "PASS toolchain/encoding: source revisions, executables, and global CNFs are fixed", + f"PASS completeness: {len(cells)} {scope} cells have terminal records", + ( + "PASS reconstruction: " + f"{reconstruction['conquer-records-reconstructed']} conquer cells reconstruct " + "per-cube work, span, scheduling, LPT replay, and hashes; " + f"{reconstruction['monolithic-stage-reconstructed'] + reconstruction['cubing-stage-reconstructed']} " + "stage-terminal cells reconstruct their available logs/metrics; " + f"{reconstruction['harness-terminal-only']} harness errors are explicitly " + "terminal-only and are not claimed as work/span reconstruction" + ), + "PASS tuning: every width/method/budget uses one calibration-frozen threshold", + ] + if aggregate_path is not None: + from benchmarks.cnc.aggregate_hard_regime import aggregate, load_terminals + + reported = load_json(aggregate_path) + regenerated_aggregate = aggregate( + contract, matrix, load_terminals(matrix, runs_root, scope), scope + ) + if reported != regenerated_aggregate: + raise VerificationError( + "aggregate does not regenerate from instance-level terminal records" + ) + if reported.get("statistical_unit") != "held-out-instance": + raise VerificationError("aggregate uses cube-level pseudoreplication") + for summary in reported.get("summaries", []): + declared_ids = summary.get("declared_instance_ids") + instance_ids = summary.get("instance_ids") + if not isinstance(declared_ids, list) or len(declared_ids) != len( + set(declared_ids) + ): + raise VerificationError( + "aggregate summary duplicates a declared held-out instance" + ) + if summary.get("declared_pairs") != len(declared_ids): + raise VerificationError( + "aggregate declared pair count does not match instance IDs" + ) + if not isinstance(instance_ids, list) or len(instance_ids) != len( + set(instance_ids) + ): + raise VerificationError( + "aggregate summary duplicates a held-out instance" + ) + if summary.get("complete_pairs") != len(instance_ids): + raise VerificationError( + "aggregate pair count does not match instance IDs" + ) + messages.append( + "PASS statistics: paired ratios and bootstrap samples use held-out instances" + ) + return messages + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("contract", type=Path) + parser.add_argument("manifest", type=Path) + parser.add_argument("--calibration-root", type=Path, required=True) + parser.add_argument("--toolchain", type=Path, required=True) + parser.add_argument("--matrix", type=Path, required=True) + parser.add_argument("--runs-root", type=Path, required=True) + parser.add_argument("--scope", choices=("pilot", "full"), default="full") + parser.add_argument("--aggregate", type=Path) + args = parser.parse_args() + try: + messages = verify_bundle( + args.contract, + args.manifest, + args.calibration_root, + args.toolchain, + args.matrix, + args.runs_root, + args.scope, + args.aggregate, + ) + except (VerificationError, MatrixError, OSError, ValueError, json.JSONDecodeError) as exc: + print(f"FAIL: {exc}") + return 1 + print("\n".join(messages)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/cnc-tools/Makefile b/cnc-tools/Makefile index 8bf447d..f1016d2 100644 --- a/cnc-tools/Makefile +++ b/cnc-tools/Makefile @@ -5,12 +5,10 @@ # make clean remove temporary build trees (keeps bin/ + proofix/) # make distclean remove bin/ and proofix/ too (full reset) # -# Idempotent: a tool already present in bin/ is left untouched (delete it, or -# run `make distclean`, to force a rebuild). Each tool is built from upstream -# source into a scratch dir, the binary copied into bin/, and the source tree -# removed — so only the ~2 MB of binaries (self-contained: libc++/libSystem -# only) and the small proofix/ Python clone remain. The whole directory is -# git-ignored via .git/info/exclude. +# Pinned tools are built into revision-qualified paths and the convenience +# names are refreshed from those exact binaries on every make invocation. A +# revision change therefore creates a new prerequisite instead of silently +# accepting a stale bin/kissat or bin/march_cu. Source trees remain temporary. BIN := $(CURDIR)/bin BUILD := $(CURDIR)/.build @@ -20,7 +18,14 @@ KISSAT_REPO := https://github.com/arminbiere/kissat.git CNC_REPO := https://github.com/marijnheule/CnC.git PROOFIX_REPO := https://github.com/zaxioms0/proofix.git -.PHONY: all check clean distclean +# Frozen source revisions for the issue #51 hard-regime table. Update these +# only by revising the experiment contract before any held-out run. +KISSAT_REV := 8af8e56f174b778aef3aa45af9f739b2a5f492c2 +CNC_REV := 705b60c6491ef2b61988b3ce6ac674be1b90571d +KISSAT_PINNED := $(BIN)/kissat-$(KISSAT_REV) +CNC_PINNED := $(BIN)/march_cu-$(CNC_REV) + +.PHONY: all check clean distclean FORCE bin/kissat bin/march_cu all: $(BIN)/cadical $(BIN)/kissat $(BIN)/march_cu proofix @rm -rf $(BUILD) @@ -28,6 +33,12 @@ all: $(BIN)/cadical $(BIN)/kissat $(BIN)/march_cu proofix @echo "cnc-tools ready. Put the binaries on PATH:" @echo " export PATH=$(BIN):\$$PATH" +# Relative aliases make explicit targets behave like the default build even +# though the canonical recipes use absolute paths. +bin/kissat: $(BIN)/kissat + +bin/march_cu: $(BIN)/march_cu + $(BIN): mkdir -p $(BIN) @@ -40,17 +51,25 @@ $(BIN)/cadical: | $(BIN) rm -rf $(BUILD)/cadical # kissat — the fixed conquer solver for every cuber (fairness). -$(BIN)/kissat: | $(BIN) +$(BIN)/kissat: $(KISSAT_PINNED) FORCE | $(BIN) + @cmp -s $< $@ || cp $< $@ + +$(KISSAT_PINNED): | $(BIN) rm -rf $(BUILD)/kissat - git clone --depth 1 $(KISSAT_REPO) $(BUILD)/kissat + git clone $(KISSAT_REPO) $(BUILD)/kissat + cd $(BUILD)/kissat && git checkout --detach $(KISSAT_REV) cd $(BUILD)/kissat && ./configure && $(MAKE) cp $(BUILD)/kissat/build/kissat $@ rm -rf $(BUILD)/kissat # march_cu — the lookahead cuber baseline (bundled in marijnheule/CnC). -$(BIN)/march_cu: | $(BIN) +$(BIN)/march_cu: $(CNC_PINNED) FORCE | $(BIN) + @cmp -s $< $@ || cp $< $@ + +$(CNC_PINNED): | $(BIN) rm -rf $(BUILD)/CnC - git clone --depth 1 $(CNC_REPO) $(BUILD)/CnC + git clone $(CNC_REPO) $(BUILD)/CnC + cd $(BUILD)/CnC && git checkout --detach $(CNC_REV) cd $(BUILD)/CnC/march_cu && $(MAKE) cp $(BUILD)/CnC/march_cu/march_cu $@ rm -rf $(BUILD)/CnC @@ -61,8 +80,8 @@ proofix: check: @printf "cadical "; $(BIN)/cadical --version - @printf "kissat "; $(BIN)/kissat --version - @printf "march_cu "; $(BIN)/march_cu 2>&1 | head -1 + @printf "kissat "; $(KISSAT_PINNED) --version + @printf "march_cu "; $(CNC_PINNED) 2>&1 | head -1 @test -f proofix/proofix.py && echo "proofix present (proofix/proofix.py)" || echo "proofix MISSING" clean: diff --git a/cnc-tools/README.md b/cnc-tools/README.md index 04f55c8..89f0c4e 100644 --- a/cnc-tools/README.md +++ b/cnc-tools/README.md @@ -8,11 +8,15 @@ once it is on `PATH`: export PATH=cnc-tools/bin:$PATH ``` +`make` also keeps revision-qualified experiment binaries such as +`kissat-8af8e56f...` and `march_cu-705b60c...`. Hard-regime toolchain locks must +name those qualified paths; the bare names are convenience copies only. + | Tool | Role | Version | Source/build policy | |---|---|---|---| | `cadical` | conquer solver + DRAT emitter | record `--version` and executable hash | build with the pinned Makefile target | -| `kissat` | fixed conquer solver | record `--version` and executable hash | build with the pinned Makefile target | -| `march_cu` | external lookahead cuber | record executable hash | build upstream source with the Makefile target | +| `kissat` | fixed conquer solver | `8af8e56f174b778aef3aa45af9f739b2a5f492c2`; also record `--version` and executable hash | build with the pinned Makefile target | +| `march_cu` | external lookahead cuber | `705b60c6491ef2b61988b3ce6ac674be1b90571d`; also record executable hash | build upstream source with the Makefile target | | Proofix | optional proof-prefix cuber | SAT 2025 | pinned clone in `proofix/` | ## Primary cube generation diff --git a/src/bin/cnc_cuber.rs b/src/bin/cnc_cuber.rs index 94caab7..9bfa5ee 100644 --- a/src/bin/cnc_cuber.rs +++ b/src/bin/cnc_cuber.rs @@ -13,7 +13,7 @@ use boolean_inference::adapter::BranchSolver; use boolean_inference::circuit::network_from_circuit_sat; use boolean_inference::cube::{ generate_cubes_with_cutoff, generate_cubes_with_cutoff_trace, CubeCutoff, CubeNodeKind, - CubeNodeTrace, + CubeNodeTrace, CubeRefutationReason, }; use boolean_inference::dimacs::network_from_dimacs; use boolean_inference::measure::Measure; @@ -23,12 +23,38 @@ use boolean_inference::selector::Selector; use optimal_branching_core::GreedyMerge; const USAGE: &str = "usage: cnc_cuber (-n | --cc-threshold ) -o \ - [--max-rows ] [--trace ]"; + [--selector ] [--max-rows ] [--trace ]"; + +#[derive(Clone, Copy, Debug)] +enum SelectorKind { + Region, + StructureBlind, +} + +impl SelectorKind { + fn parse(value: &str) -> Result { + match value { + "region" => Ok(Self::Region), + "structure-blind" => Ok(Self::StructureBlind), + _ => Err(format!( + "invalid --selector value: {value}; expected region or structure-blind" + )), + } + } + + fn label(self) -> &'static str { + match self { + Self::Region => "region", + Self::StructureBlind => "structure-blind", + } + } +} struct Args { input: PathBuf, output: PathBuf, cutoff: CubeCutoff, + selector: SelectorKind, max_rows: usize, trace: Option, } @@ -52,6 +78,7 @@ fn parse_args() -> Result { let mut cutoff_vars = None; let mut cc_threshold = None; let mut max_rows = 512usize; + let mut selector = SelectorKind::Region; let mut trace = None; let mut i = 0usize; @@ -78,6 +105,9 @@ fn parse_args() -> Result { } "-o" => output = Some(take_value(&raw, &mut i, "-o")?), "--trace" => trace = Some(take_value(&raw, &mut i, "--trace")?), + "--selector" => { + selector = SelectorKind::parse(&take_value(&raw, &mut i, "--selector")?)?; + } "--max-rows" => { let value = take_value(&raw, &mut i, "--max-rows")?; max_rows = value @@ -108,6 +138,7 @@ fn parse_args() -> Result { input: PathBuf::from(input.ok_or_else(|| "missing input instance".to_string())?), output: PathBuf::from(output.ok_or_else(|| "missing -o output".to_string())?), cutoff, + selector, max_rows, trace: trace.map(PathBuf::from), })) @@ -146,6 +177,14 @@ fn node_kind(kind: CubeNodeKind) -> &'static str { } } +fn refutation_reason(reason: CubeRefutationReason) -> &'static str { + match reason { + CubeRefutationReason::RootPropagation => "root-propagation-contradiction", + CubeRefutationReason::SelectorNoFeasibleConfig => "selector-no-feasible-config", + CubeRefutationReason::BranchPropagation => "branch-propagation-contradiction", + } +} + fn write_trace_node( writer: &mut dyn Write, node: CubeNodeTrace, @@ -180,6 +219,7 @@ fn write_trace_node( "child_index": node.child_index, "depth": node.depth, "kind": node_kind(node.kind), + "refutation_reason": node.refutation_reason.map(refutation_reason), "literals": literals, "sigma_dec": node.sigma_dec, "sigma_all": node.sigma_all, @@ -225,6 +265,29 @@ fn run(args: Args) -> Result<(), String> { let mut problem = match TnProblem::from_network(network) { Ok(problem) => problem, Err(_) => { + if let Some(trace_writer) = trace_writer.as_mut() { + write_trace_node( + trace_writer, + CubeNodeTrace { + node_id: 0, + parent_id: None, + child_index: None, + depth: 0, + kind: CubeNodeKind::Refuted, + refutation_reason: Some(CubeRefutationReason::RootPropagation), + decisions: Vec::new(), + sigma_dec: 0, + sigma_all: 0, + freevars: nvars, + variables: Vec::new(), + clauses: Vec::new(), + }, + &new_to_orig, + )?; + trace_writer + .flush() + .map_err(|error| format!("flush trace: {error}"))?; + } writer.flush().map_err(|e| format!("flush output: {e}"))?; eprintln!( "status=UNSAT_AT_ROOT cubes=0 refuted=1 sat_leaves=0 cutoff={:?}", @@ -238,8 +301,11 @@ fn run(args: Args) -> Result<(), String> { let mut emitted = 0usize; let mut min_remaining = usize::MAX; let mut max_remaining = 0usize; - let selector = Selector::MostOccurrence { - max_rows: args.max_rows, + let selector = match args.selector { + SelectorKind::Region => Selector::MostOccurrence { + max_rows: args.max_rows, + }, + SelectorKind::StructureBlind => Selector::BinaryOccurrence, }; let solver = BranchSolver::Greedy(GreedyMerge); let mut emit = |cube: boolean_inference::cube::Cube| { @@ -312,7 +378,7 @@ fn run(args: Args) -> Result<(), String> { }; eprintln!( "status=OK cubes={} refuted={} sat_leaves={} visited={} cutoff={:?} \ - root_unfixed={} remaining_range={} max_rows={}", + root_unfixed={} remaining_range={} selector={} max_rows={}", stats.cubes, stats.refuted, stats.sat_leaves, @@ -320,6 +386,7 @@ fn run(args: Args) -> Result<(), String> { args.cutoff, root_unfixed, remaining_range, + args.selector.label(), args.max_rows ); if emitted != stats.cubes { diff --git a/src/cube.rs b/src/cube.rs index 4745a7d..be8bf41 100644 --- a/src/cube.rs +++ b/src/cube.rs @@ -61,6 +61,14 @@ pub enum CubeNodeKind { Sat, } +/// Why a traced leaf is known to be closed without a conquer cube. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum CubeRefutationReason { + RootPropagation, + SelectorNoFeasibleConfig, + BranchPropagation, +} + /// One branching clause in the bit encoding over `CubeNodeTrace::variables`. #[derive(Clone, Debug, PartialEq, Eq)] pub struct TraceClause { @@ -78,6 +86,7 @@ pub struct CubeNodeTrace { pub child_index: Option, pub depth: usize, pub kind: CubeNodeKind, + pub refutation_reason: Option, pub decisions: Vec<(usize, bool)>, pub sigma_dec: usize, pub sigma_all: usize, @@ -287,6 +296,7 @@ where child_index: None, depth: 0, kind: CubeNodeKind::Refuted, + refutation_reason: Some(CubeRefutationReason::RootPropagation), decisions: Vec::new(), sigma_dec: 0, sigma_all: 0, @@ -384,6 +394,7 @@ where child_index, depth, kind: CubeNodeKind::Cutoff, + refutation_reason: None, decisions: decisions.clone(), sigma_dec, sigma_all, @@ -417,6 +428,7 @@ where child_index, depth, kind: CubeNodeKind::Sat, + refutation_reason: None, decisions: decisions.clone(), sigma_dec, sigma_all, @@ -459,6 +471,7 @@ where child_index, depth, kind: CubeNodeKind::Refuted, + refutation_reason: Some(CubeRefutationReason::SelectorNoFeasibleConfig), decisions: decisions.clone(), sigma_dec, sigma_all, @@ -496,6 +509,7 @@ where child_index, depth, kind: CubeNodeKind::Branch, + refutation_reason: None, decisions: decisions.clone(), sigma_dec, sigma_all, @@ -538,6 +552,7 @@ where child_index: Some(branch_index), depth: depth + 1, kind: CubeNodeKind::Refuted, + refutation_reason: Some(CubeRefutationReason::BranchPropagation), decisions: decisions.clone(), sigma_dec: decisions.len(), sigma_all: doms.len() - branch_freevars, diff --git a/tests/cnc_cuber_trace.rs b/tests/cnc_cuber_trace.rs index 9964c26..fdf2829 100644 --- a/tests/cnc_cuber_trace.rs +++ b/tests/cnc_cuber_trace.rs @@ -65,6 +65,9 @@ fn trace_flag_preserves_cubes_and_writes_original_variable_ids() { assert!(!records.is_empty()); assert_eq!(records[0]["node_id"], 0); assert!(records[0]["parent_id"].is_null()); + assert!(records + .iter() + .all(|record| { record["kind"] == "refuted" || record["refutation_reason"].is_null() })); assert_eq!( records .iter() @@ -83,3 +86,82 @@ fn trace_flag_preserves_cubes_and_writes_original_variable_ids() { fs::remove_dir_all(dir).expect("remove temp directory"); } + +#[test] +fn structure_blind_selector_is_auditable_binary_control() { + let dir = temp_dir(); + fs::create_dir_all(&dir).expect("create temp directory"); + let input = dir.join("input.cnf"); + let cubes = dir.join("blind.cubes"); + let trace = dir.join("blind.jsonl"); + fs::write(&input, "p cnf 3 4\n1 2 0\n-1 -2 0\n2 3 0\n-2 -3 0\n").expect("write CNF"); + + let run = Command::new(env!("CARGO_BIN_EXE_cnc_cuber")) + .args([ + input.as_os_str(), + "-n".as_ref(), + "3".as_ref(), + "-o".as_ref(), + cubes.as_os_str(), + "--selector".as_ref(), + "structure-blind".as_ref(), + "--trace".as_ref(), + trace.as_os_str(), + ]) + .output() + .expect("run structure-blind cuber"); + assert!(run.status.success(), "{:?}", run); + let stderr = String::from_utf8(run.stderr).expect("UTF-8 stderr"); + assert!(stderr.contains("selector=structure-blind"), "{stderr}"); + + let records: Vec = fs::read_to_string(&trace) + .unwrap() + .lines() + .map(|line| serde_json::from_str(line).expect("valid trace JSON")) + .collect(); + for branch in records.iter().filter(|record| record["kind"] == "branch") { + assert_eq!(branch["rule_variables"].as_array().unwrap().len(), 1); + let clauses = branch["rule_clauses"].as_array().unwrap(); + assert_eq!(clauses.len(), 2); + assert_eq!(clauses[0]["mask"], 1); + assert_eq!(clauses[0]["value"], 0); + assert_eq!(clauses[1]["mask"], 1); + assert_eq!(clauses[1]["value"], 1); + } + + fs::remove_dir_all(dir).expect("remove temp directory"); +} + +#[test] +fn root_refutation_trace_records_a_semantic_closure_reason() { + let dir = temp_dir(); + fs::create_dir_all(&dir).expect("create temp directory"); + let input = dir.join("root-unsat.cnf"); + let cubes = dir.join("root-unsat.cubes"); + let trace = dir.join("root-unsat.jsonl"); + fs::write(&input, "p cnf 1 2\n1 0\n-1 0\n").expect("write CNF"); + + let run = Command::new(env!("CARGO_BIN_EXE_cnc_cuber")) + .args([ + input.as_os_str(), + "-n".as_ref(), + "1".as_ref(), + "-o".as_ref(), + cubes.as_os_str(), + "--trace".as_ref(), + trace.as_os_str(), + ]) + .output() + .expect("run root-UNSAT cuber"); + assert!(run.status.success(), "{:?}", run); + assert!(fs::read_to_string(&cubes).unwrap().is_empty()); + let record: serde_json::Value = + serde_json::from_str(fs::read_to_string(&trace).unwrap().trim()).unwrap(); + assert_eq!(record["kind"], "refuted"); + assert_eq!( + record["refutation_reason"], + "root-propagation-contradiction" + ); + + fs::remove_dir_all(dir).expect("remove temp directory"); +} diff --git a/tests/test_aggregate_hard_regime.py b/tests/test_aggregate_hard_regime.py new file mode 100644 index 0000000..81c015e --- /dev/null +++ b/tests/test_aggregate_hard_regime.py @@ -0,0 +1,91 @@ +import unittest + +from benchmarks.cnc.aggregate_hard_regime import ( + adjusted_observations, + bootstrap_ci, + geometric_mean, + interpolate_log, + summarize_observations, +) + + +class HardRegimeAggregateTests(unittest.TestCase): + def test_log_interpolation_has_no_extrapolation(self): + points = [(100.0, 10.0), (400.0, 40.0)] + self.assertAlmostEqual(interpolate_log(points, 200.0), 20.0) + self.assertIsNone(interpolate_log(points, 50.0)) + self.assertIsNone(interpolate_log(points, 800.0)) + + def test_geometric_mean_and_bootstrap_are_deterministic(self): + self.assertAlmostEqual(geometric_mean([0.5, 2.0]), 1.0) + first = bootstrap_ci([0.8, 1.0, 1.2], 100, 0.95, 51, "work:low") + second = bootstrap_ci([0.8, 1.0, 1.2], 100, 0.95, 51, "work:low") + self.assertEqual(first, second) + + def test_summary_counts_instances_not_cubes(self): + observations = [ + { + "analysis": "raw-nominal-budget", + "metric": "conquer_work_cpu_s", + "budget": "low", + "product_width": 64, + "instance_id": "a", + "complete_pair": True, + "ratio": 0.8, + }, + { + "analysis": "raw-nominal-budget", + "metric": "conquer_work_cpu_s", + "budget": "low", + "product_width": 64, + "instance_id": "b", + "complete_pair": False, + "ratio": None, + }, + ] + summary = summarize_observations( + observations, + "raw-nominal-budget", + "conquer_work_cpu_s", + "budget", + "low", + 64, + {"samples": 100, "confidence": 0.95, "seed": 51}, + ) + self.assertEqual(summary["declared_pairs"], 2) + self.assertEqual(summary["complete_pairs"], 1) + self.assertEqual(summary["instance_ids"], ["a"]) + self.assertAlmostEqual(summary["geometric_mean_ratio"], 0.8) + + def test_adjusted_rows_retain_instances_without_common_support(self): + cells = [] + terminals = {} + for method in ("region-cc", "structure-blind-cc"): + for budget in ("low", "medium", "high"): + cell_id = f"instance__{method}__{budget}" + cells.append( + { + "cell_id": cell_id, + "instance_id": "instance", + "split": "held_out", + "product_width": 64, + "method": method, + "budget": budget, + } + ) + terminals[cell_id] = { + "state": "conquer-timeout", + "metrics": {"frontier_size": 100}, + } + rows = adjusted_observations( + {"cells": cells}, terminals, "conquer_work_cpu_s" + ) + self.assertEqual(len(rows), 3) + self.assertEqual({row["instance_id"] for row in rows}, {"instance"}) + self.assertTrue(all(row["adjustment_status"] == "incomplete-series" for row in rows)) + self.assertTrue(all(row["complete_pair"] is False for row in rows)) + self.assertTrue(all(row["common_frontier_size"] is None for row in rows)) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_calibrate_hard_regime.py b/tests/test_calibrate_hard_regime.py new file mode 100644 index 0000000..16365c0 --- /dev/null +++ b/tests/test_calibrate_hard_regime.py @@ -0,0 +1,47 @@ +import unittest + +from benchmarks.cnc.calibrate_cc_difficulty import CalibrationError +from benchmarks.cnc.calibrate_hard_regime import ( + calibration_loss, + choose_width_response, + median_tasks, +) + + +def row(threshold, *counts): + return { + "threshold": threshold, + "instances": [ + {"id": f"cal-{index}", "tasks": count} + for index, count in enumerate(counts) + ], + } + + +class HardRegimeCalibrationTests(unittest.TestCase): + def test_width_selection_uses_all_three_calibration_instances(self): + response = [ + row(100, 300, 400, 500), + row(200, 480, 520, 560), + row(300, 500, 700, 4000), + ] + selected = choose_width_response(response, 512, 384, 640) + self.assertEqual(selected["threshold"], 200) + self.assertEqual(median_tasks(selected), 520) + + def test_loss_penalizes_cross_instance_mismatch(self): + balanced = row(100, 480, 512, 544) + skewed = row(200, 128, 512, 2048) + self.assertLess(calibration_loss(balanced, 512), calibration_loss(skewed, 512)) + + def test_empty_response_fails_closed(self): + with self.assertRaises(CalibrationError): + choose_width_response([], 512, 384, 640) + + def test_response_without_an_in_band_threshold_fails_closed(self): + with self.assertRaisesRegex(CalibrationError, "accepted task range"): + choose_width_response([row(100, 10, 20, 30)], 512, 384, 640) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_conquer_parallel.py b/tests/test_conquer_parallel.py index 685f5ac..6397c20 100644 --- a/tests/test_conquer_parallel.py +++ b/tests/test_conquer_parallel.py @@ -61,4 +61,45 @@ def test_parallel_conquer_streams_cubes_and_reports_conflict_distribution( assert result["total_conflicts"] == 15 assert result["conflicts_p50"] == 3 assert result["conflicts_p99_over_p95"] == 1 + assert result["complete"] is True + assert result["result"] == "unsat" + assert result["terminal_records"] == 5 + assert result["lpt_makespan_by_workers_s"]["2"] > 0 + raw = [json.loads(line) for line in (out_dir / "test.jsonl").read_text().splitlines()] + assert all(row["released_monotonic_ns"] <= row["started_monotonic_ns"] for row in raw) + assert all(row["started_monotonic_ns"] <= row["finished_monotonic_ns"] for row in raw) + assert all(row["worker_pid"] > 0 for row in raw) + expected_measured = ( + max(row["collected_monotonic_ns"] for row in raw) + - min(row["released_monotonic_ns"] for row in raw) + ) / 1e9 + assert result["measured_makespan_s"] == expected_measured + assert result["observed_parallel_wall_s"] >= result["measured_makespan_s"] assert len((out_dir / "test.jsonl").read_text().splitlines()) == 5 + + +def test_censored_cubes_remain_in_work_and_lpt_accounting(): + result = conquer_parallel.summarize( + cubes=2, + completed=1, + timeouts=1, + errors=0, + sat=0, + unsat=1, + durations=[1.0, 5.0], + cpu_durations=[0.5, 4.5], + decisions=[7.0], + conflicts=[3.0], + workers=2, + replay_workers=[2, 4], + wall_s=5.2, + measured_makespan_s=5.0, + ) + assert result["result"] == "timeout" + assert result["complete"] is False + assert result["censored"] is True + assert result["total_solver_s"] == 6.0 + assert result["total_cpu_s"] == 5.0 + assert result["lpt_is_lower_bound"] is True + assert result["lpt_makespan_by_workers_s"] == {"2": 5.0, "4": 5.0} + assert result["conflicts_max"] == 3.0 diff --git a/tests/test_hard_regime.py b/tests/test_hard_regime.py new file mode 100644 index 0000000..d4a209d --- /dev/null +++ b/tests/test_hard_regime.py @@ -0,0 +1,73 @@ +import copy +import unittest +from pathlib import Path + +from benchmarks.cnc.hard_regime import ( + EXPECTED_METHOD_BUDGETS, + HardRegimeError, + load_contract, + strong_miller_rabin, + target_records, + validate_contract, + verify_target_records, +) + + +ROOT = Path(__file__).resolve().parents[1] +CONTRACT = ROOT / "benchmarks/cnc/contracts/hard-regime-v1.yaml" + + +class HardRegimeTests(unittest.TestCase): + def test_contract_freezes_issue_51_matrix(self): + contract = load_contract(CONTRACT) + self.assertEqual( + { + name: tuple(spec["budgets"]) + for name, spec in contract["methods"].items() + }, + EXPECTED_METHOD_BUDGETS, + ) + self.assertEqual(contract["limits_seconds"]["per_cube_conquer"], 1800) + self.assertEqual(contract["statistics"]["unit"], "held-out-instance") + + def test_targets_are_deterministic_prime_in_range_and_disjoint(self): + contract = load_contract(CONTRACT) + first = target_records(contract) + second = target_records(contract) + self.assertEqual(first, second) + self.assertEqual(len(first), 39) + self.assertTrue(all(record["expected_outcome"] == "unsat" for record in first)) + self.assertEqual(len({record["id"] for record in first}), 39) + messages = verify_target_records(contract, first) + self.assertTrue(any("39 deterministic prime" in message for message in messages)) + + def test_contract_rejects_width_confusion_and_split_seed_overlap(self): + contract = load_contract(CONTRACT) + confused = copy.deepcopy(contract) + confused["widths"][0]["factor_input_width"] = 64 + with self.assertRaisesRegex(HardRegimeError, "twice factor-input width"): + validate_contract(confused) + + overlapping = copy.deepcopy(contract) + overlapping["widths"][0]["held_out"]["seed"] = overlapping["widths"][0][ + "calibration" + ]["seed"] + with self.assertRaisesRegex(HardRegimeError, "seeds overlap"): + validate_contract(overlapping) + + def test_contract_rejects_march_budget_tuning(self): + contract = load_contract(CONTRACT) + contract["methods"]["march-cu-dynamic"]["budgets"] = [ + "low", + "medium", + "high", + ] + with self.assertRaisesRegex(HardRegimeError, "march-cu-dynamic budgets"): + validate_contract(contract) + + def test_independent_primality_check_rejects_composites(self): + self.assertTrue(strong_miller_rabin(37)) + self.assertFalse(strong_miller_rabin(39)) + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_hard_regime_matrix.py b/tests/test_hard_regime_matrix.py new file mode 100644 index 0000000..dfb4246 --- /dev/null +++ b/tests/test_hard_regime_matrix.py @@ -0,0 +1,226 @@ +import tempfile +import unittest +from collections import Counter +from pathlib import Path + +from benchmarks.cnc.hard_regime import contract_sha256, load_contract, target_records +from benchmarks.cnc.hard_regime_matrix import MatrixError, build_matrix, lock_toolchain +from benchmarks.pipeline.circuit import canonical_bytes, sha256_bytes, write_json + + +ROOT = Path(__file__).resolve().parents[1] +CONTRACT_PATH = ROOT / "benchmarks/cnc/contracts/hard-regime-v1.yaml" + + +def fake_manifest(contract): + digest = contract_sha256(contract) + return [ + { + **record, + "contract_sha256": digest, + "circuitsat": f"instances/{record['id']}.json", + "circuitsat_sha256": f"{index + 1:064x}", + "cnf": f"instances/{record['id']}.cnf", + "cnf_sha256": f"{index + 100:064x}", + "encoding_wall_s": 1.0, + "encoding_cpu_s": 0.9, + } + for index, record in enumerate(target_records(contract)) + ] + + +def fake_toolchain(contract): + record = { + "schema_version": 1, + "kind": "hard-regime-toolchain-lock", + "contract_sha256": contract_sha256(contract), + "tools": { + "cnc_cuber": { + "path": "/tools/cnc_cuber", + "executable_sha256": "a" * 64, + "source_revision": "repo-revision", + }, + "kissat": { + "path": "/tools/kissat", + "executable_sha256": "b" * 64, + "source_revision": contract["tool_sources"]["kissat"]["revision"], + }, + "march_cu": { + "path": "/tools/march_cu", + "executable_sha256": "c" * 64, + "source_revision": contract["tool_sources"]["march_cu"]["revision"], + }, + }, + } + record["toolchain_sha256"] = sha256_bytes(canonical_bytes(record)) + return record + + +def write_calibration_locks(root, contract, manifest, cuber_sha): + for width in (64, 72, 80): + calibration = [ + {"id": record["id"], "sha256": record["circuitsat_sha256"]} + for record in manifest + if record["product_width"] == width and record["split"] == "calibration" + ] + for selector in ("region", "structure-blind"): + method = "region-cc" if selector == "region" else "structure-blind-cc" + responses = [] + bands = {} + for index, (name, spec) in enumerate(contract["frontier_bands"].items(), 1): + threshold = width * 1000 + index + target = spec["center_cubes"] + response_instances = [ + {"id": item["id"], "tasks": target, "elapsed_s": 1.0, "cpu_s": 0.9} + for item in calibration + ] + responses.append( + {"threshold": threshold, "instances": response_instances} + ) + minimum = int(target * spec["accepted_ratio"][0]) + maximum = int(target * spec["accepted_ratio"][1]) + bands[name] = { + "target_tasks": target, + "accepted_task_range": [minimum, maximum], + "selected_threshold": threshold, + "search_bracket": ( + [threshold, threshold + 1] + if index == 1 + else [threshold - 1, threshold] + ), + "median_tasks": float(target), + "calibration_loss": 0.0, + "within_target_range": True, + "instances": [ + { + **item, + "frontier_sha256": f"{index:064x}", + "trace_sha256": f"{index + 10:064x}", + "cubing_elapsed_s": 1.0, + "cubing_cpu_s": 0.9, + } + for item in response_instances + ], + } + write_json( + root / f"p{width}" / selector / "calibration-lock.json", + { + "schema_version": 1, + "kind": "width-level-cc-calibration-lock", + "contract_sha256": contract_sha256(contract), + "product_width": width, + "selector": selector, + "method": method, + "max_rows": contract["methods"][method]["max_rows"], + "cuber_sha256": cuber_sha, + "calibration_instances": calibration, + "bands": bands, + "response": responses, + }, + ) + + +class HardRegimeMatrixTests(unittest.TestCase): + def test_matrix_declares_every_required_cell_once(self): + contract = load_contract(CONTRACT_PATH) + manifest = fake_manifest(contract) + tools = fake_toolchain(contract) + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + write_calibration_locks( + root, contract, manifest, tools["tools"]["cnc_cuber"]["executable_sha256"] + ) + matrix = build_matrix(contract, manifest, root, tools) + counts = Counter(cell["method"] for cell in matrix["cells"]) + self.assertEqual( + counts, + { + "monolithic-kissat": 39, + "march-cu-dynamic": 30, + "region-cc": 90, + "structure-blind-cc": 90, + }, + ) + self.assertEqual(sum(cell["pilot"] for cell in matrix["cells"]), 72) + held_out = [cell for cell in matrix["cells"] if cell["method"] == "region-cc"] + thresholds = { + (cell["product_width"], cell["budget"]): cell["cc_threshold"] + for cell in held_out + } + self.assertEqual(len(thresholds), 9) + for cell in held_out: + self.assertEqual( + cell["cc_threshold"], + thresholds[(cell["product_width"], cell["budget"])], + ) + + def test_matrix_rejects_width_confusion(self): + contract = load_contract(CONTRACT_PATH) + manifest = fake_manifest(contract) + manifest[0]["factor_input_width"] = manifest[0]["product_width"] + with self.assertRaisesRegex(MatrixError, "factor_input_width differs"): + with tempfile.TemporaryDirectory() as directory: + build_matrix(contract, manifest, Path(directory), fake_toolchain(contract)) + + def test_matrix_rejects_calibration_test_leakage(self): + contract = load_contract(CONTRACT_PATH) + manifest = fake_manifest(contract) + tools = fake_toolchain(contract) + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + write_calibration_locks( + root, contract, manifest, tools["tools"]["cnc_cuber"]["executable_sha256"] + ) + path = root / "p64/region/calibration-lock.json" + lock = __import__("json").loads(path.read_text()) + held_out = next( + record + for record in manifest + if record["product_width"] == 64 and record["split"] == "held_out" + ) + lock["calibration_instances"][0]["id"] = held_out["id"] + write_json(path, lock) + with self.assertRaisesRegex(MatrixError, "not the frozen split"): + build_matrix(contract, manifest, root, tools) + + def test_matrix_recomputes_the_selected_calibration_threshold(self): + contract = load_contract(CONTRACT_PATH) + manifest = fake_manifest(contract) + tools = fake_toolchain(contract) + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + write_calibration_locks( + root, contract, manifest, tools["tools"]["cnc_cuber"]["executable_sha256"] + ) + path = root / "p64/region/calibration-lock.json" + lock = __import__("json").loads(path.read_text()) + lock["bands"]["low"]["selected_threshold"] += 999 + write_json(path, lock) + with self.assertRaisesRegex(MatrixError, "not selected from the response"): + build_matrix(contract, manifest, root, tools) + + def test_toolchain_lock_requires_revision_qualified_upstream_binaries(self): + contract = load_contract(CONTRACT_PATH) + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + cuber = root / "cnc_cuber" + kissat_revision = contract["tool_sources"]["kissat"]["revision"] + march_revision = contract["tool_sources"]["march_cu"]["revision"] + kissat = root / f"kissat-{kissat_revision}" + march = root / f"march_cu-{march_revision}" + for path in (cuber, kissat, march): + path.write_text("#!/bin/sh\necho identity\n", encoding="utf-8") + path.chmod(0o755) + record = lock_toolchain(contract, cuber, kissat, march, "abcdef0123456789") + self.assertEqual( + record["tools"]["kissat"]["source_revision"], kissat_revision + ) + generic = root / "kissat" + generic.write_text("#!/bin/sh\necho stale\n", encoding="utf-8") + generic.chmod(0o755) + with self.assertRaisesRegex(MatrixError, "revision-qualified binary"): + lock_toolchain(contract, cuber, generic, march, "abcdef0123456789") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_run_hard_regime_cell.py b/tests/test_run_hard_regime_cell.py new file mode 100644 index 0000000..d6e792f --- /dev/null +++ b/tests/test_run_hard_regime_cell.py @@ -0,0 +1,147 @@ +import json +import sys +import tempfile +import unittest +from pathlib import Path + +from benchmarks.cnc.run_hard_regime_cell import CellError, run_process, verify_region_trace + + +class HardRegimeCellTests(unittest.TestCase): + def test_region_trace_reconstructs_complete_frontier(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + frontier = root / "frontier.icnf" + trace = root / "nodes.jsonl" + frontier.write_text("a 1 0\na -1 0\n", encoding="utf-8") + rows = [ + { + "node_id": 0, + "parent_id": None, + "child_index": None, + "depth": 0, + "kind": "branch", + "refutation_reason": None, + "literals": [], + "sigma_dec": 0, + "sigma_all": 0, + "freevars": 1, + "rule_variables": [1], + "rule_clauses": [ + {"mask": 1, "value": 1}, + {"mask": 1, "value": 0}, + ], + }, + { + "node_id": 1, + "parent_id": 0, + "child_index": 0, + "depth": 1, + "kind": "cutoff", + "refutation_reason": None, + "literals": [1], + "sigma_dec": 1, + "sigma_all": 1, + "freevars": 0, + "rule_variables": [], + "rule_clauses": [], + }, + { + "node_id": 2, + "parent_id": 0, + "child_index": 1, + "depth": 1, + "kind": "cutoff", + "refutation_reason": None, + "literals": [-1], + "sigma_dec": 1, + "sigma_all": 1, + "freevars": 0, + "rule_variables": [], + "rule_clauses": [], + }, + ] + trace.write_text( + "".join(json.dumps(row) + "\n" for row in rows), encoding="utf-8" + ) + summary = verify_region_trace(frontier, trace) + self.assertEqual( + summary, + { + "nodes": 3, + "branches": 1, + "cutoffs": 2, + "refuted": 0, + "sat_leaves": 0, + "root_refutations": 0, + "selector_refutations": 0, + "branch_refutations": 0, + }, + ) + + rows.pop() + trace.write_text( + "".join(json.dumps(row) + "\n" for row in rows), encoding="utf-8" + ) + with self.assertRaisesRegex(CellError, "incomplete children"): + verify_region_trace(frontier, trace) + + def test_region_trace_rejects_child_that_does_not_apply_its_rule(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + frontier = root / "frontier.icnf" + trace = root / "nodes.jsonl" + frontier.write_text("a -1 0\n", encoding="utf-8") + rows = [ + { + "node_id": 0, + "parent_id": None, + "child_index": None, + "depth": 0, + "kind": "branch", + "refutation_reason": None, + "literals": [], + "sigma_dec": 0, + "sigma_all": 0, + "freevars": 1, + "rule_variables": [1], + "rule_clauses": [{"mask": 1, "value": 1}], + }, + { + "node_id": 1, + "parent_id": 0, + "child_index": 0, + "depth": 1, + "kind": "cutoff", + "refutation_reason": None, + "literals": [-1], + "sigma_dec": 1, + "sigma_all": 1, + "freevars": 0, + "rule_variables": [], + "rule_clauses": [], + }, + ] + trace.write_text( + "".join(json.dumps(row) + "\n" for row in rows), encoding="utf-8" + ) + with self.assertRaisesRegex(CellError, "does not implement rule clause"): + verify_region_trace(frontier, trace) + + def test_process_timeout_is_a_terminal_stage(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + result = run_process( + [sys.executable, "-c", "import time; time.sleep(1)"], + 0.01, + root / "stdout", + root / "stderr", + ) + self.assertEqual(result["state"], "timeout") + self.assertIsNone(result["returncode"]) + self.assertGreaterEqual(result["wall_s"], 0.01) + self.assertEqual(len(result["stdout_sha256"]), 64) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_verify_hard_regime.py b/tests/test_verify_hard_regime.py new file mode 100644 index 0000000..5fc10db --- /dev/null +++ b/tests/test_verify_hard_regime.py @@ -0,0 +1,133 @@ +import hashlib +import json +import tempfile +import unittest +from pathlib import Path + +from benchmarks.cnc.conquer_parallel import summarize +from benchmarks.cnc.hard_regime import contract_sha256, load_contract +from benchmarks.cnc.verify_hard_regime import ( + VerificationError, + verify_conquer_records, + verify_terminal, +) +from benchmarks.pipeline.circuit import canonical_bytes, sha256_bytes, sha256_file + + +ROOT = Path(__file__).resolve().parents[1] +CONTRACT_PATH = ROOT / "benchmarks/cnc/contracts/hard-regime-v1.yaml" + + +class HardRegimeVerifierTests(unittest.TestCase): + def make_records(self, root: Path): + frontier = root / "frontier.icnf" + results = root / "results.jsonl" + frontier.write_text("a 1 0\na -1 0\n", encoding="utf-8") + rows = [] + for index, literal in enumerate((1, -1)): + started = 100 + index * 20 + rows.append( + { + "cube_index": index, + "cube_sha256": hashlib.sha256(f"{literal} 0\n".encode()).hexdigest(), + "released_monotonic_ns": started - 5, + "started_monotonic_ns": started, + "finished_monotonic_ns": started + 10, + "collected_monotonic_ns": started + 15, + "worker_pid": index + 1, + "elapsed_s": 1.0 + index, + "user_s": 0.5 + index, + "system_s": 0.25, + "decisions": 4, + "conflicts": 3 + index, + "result": "unsat", + "censored": False, + } + ) + results.write_text("".join(json.dumps(row) + "\n" for row in rows), encoding="utf-8") + summary = summarize( + cubes=2, + completed=2, + timeouts=0, + errors=0, + sat=0, + unsat=2, + durations=[1.0, 2.0], + cpu_durations=[0.75, 1.75], + decisions=[4.0, 4.0], + conflicts=[3.0, 4.0], + workers=2, + replay_workers=[2, 4], + wall_s=2.1, + measured_makespan_s=40 / 1e9, + ) + return frontier, results, summary, rows + + def test_per_cube_records_reconstruct_work_and_scheduling(self): + with tempfile.TemporaryDirectory() as directory: + frontier, results, summary, _ = self.make_records(Path(directory)) + verify_conquer_records(frontier, results, summary, 2, [2, 4]) + + def test_duplicate_cube_record_is_rejected(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + frontier, results, summary, rows = self.make_records(root) + rows[1]["cube_index"] = 0 + results.write_text( + "".join(json.dumps(row) + "\n" for row in rows), encoding="utf-8" + ) + with self.assertRaisesRegex(VerificationError, "duplicate cube indices"): + verify_conquer_records(frontier, results, summary, 2, [2, 4]) + + def test_harness_error_is_terminal_only_not_reconstruction(self): + contract = load_contract(CONTRACT_PATH) + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + matrix_path = root / "matrix.json" + matrix_path.write_text("{}\n", encoding="utf-8") + toolchain = {"toolchain_sha256": "toolchain"} + cell = { + "cell_id": "cell", + "instance_id": "instance", + "method": "region-cc", + "budget": "low", + "product_width": 64, + "factor_input_width": 32, + "global_cnf_sha256": "a" * 64, + "circuitsat_sha256": "b" * 64, + } + terminal = { + "kind": "hard-regime-terminal-cell", + "contract_sha256": contract_sha256(contract), + "matrix_sha256": sha256_file(matrix_path), + "toolchain_sha256": "toolchain", + "cell_id": "cell", + "cell_sha256": sha256_bytes(canonical_bytes(cell)), + "instance_id": "instance", + "method": "region-cc", + "budget": "low", + "product_width": 64, + "factor_input_width": 32, + "state": "harness-error", + "error": "CellError: input hash mismatch", + "input_artifacts": { + "global_cnf": {"sha256": "a" * 64}, + "circuitsat": {"sha256": "b" * 64}, + }, + } + self.assertEqual( + verify_terminal( + contract, + matrix_path, + {}, + toolchain, + cell, + terminal, + root, + ), + "harness-terminal-only", + ) + + +if __name__ == "__main__": + unittest.main() From 13caefcb1aa70e2a44121ff71c3a133db0df69d2 Mon Sep 17 00:00:00 2001 From: Xiwei Pan Date: Mon, 20 Jul 2026 01:51:35 +0800 Subject: [PATCH 2/3] Make hard-regime calibration resumable --- benchmarks/cnc/README.md | 6 + benchmarks/cnc/calibrate_hard_regime.py | 233 ++++++++++++++++++++---- benchmarks/cnc/hard_regime_matrix.py | 18 ++ benchmarks/cnc/run_hard_regime_cell.py | 12 +- benchmarks/pipeline/circuit.py | 17 ++ tests/test_calibrate_hard_regime.py | 188 ++++++++++++++++++- tests/test_hard_regime_matrix.py | 81 ++++++++ 7 files changed, 514 insertions(+), 41 deletions(-) diff --git a/benchmarks/cnc/README.md b/benchmarks/cnc/README.md index d13ce3b..a949d54 100644 --- a/benchmarks/cnc/README.md +++ b/benchmarks/cnc/README.md @@ -41,6 +41,12 @@ python3 -m benchmarks.cnc.calibrate_hard_regime \ --out-dir artifacts/cnc-hard-regime/calibration/p64/region ``` +Hard-regime threshold search starts at `1` and doubles before bisection for +both selectors. Every completed instance probe writes a provenance- and +hash-checked `*.probe.json` checkpoint next to its frontier and log. Re-running +the same calibration validates and reuses complete probes, while malformed, +tampered, or mismatched checkpoints fail closed. + After all six width/selector calibration locks exist, capture the exact tool binaries and build the immutable 249-cell matrix (39 hardness rows plus 210 held-out CnC rows): diff --git a/benchmarks/cnc/calibrate_hard_regime.py b/benchmarks/cnc/calibrate_hard_regime.py index 59b9e5c..f45795a 100755 --- a/benchmarks/cnc/calibrate_hard_regime.py +++ b/benchmarks/cnc/calibrate_hard_regime.py @@ -4,9 +4,11 @@ from __future__ import annotations import argparse +import hashlib import json import math import statistics +from dataclasses import dataclass from pathlib import Path from typing import Any, Callable @@ -20,7 +22,156 @@ contract_sha256, load_contract, ) -from benchmarks.pipeline.circuit import read_jsonl, write_json +from benchmarks.pipeline.circuit import atomic_write_json, read_jsonl, write_json + + +PROBE_CHECKPOINT_SCHEMA_VERSION = 1 + + +@dataclass(frozen=True) +class ProbeContext: + probe_runner: Callable[..., dict[str, int | float]] + cuber: Path + cuber_sha256: str + contract_digest: str + product_width: int + selector: str + max_rows: int + timeout_s: float + + +def _validated_probe_result( + value: dict[str, int | float], threshold: int +) -> dict[str, int | float]: + if value.get("threshold") != threshold: + raise CalibrationError("calibration probe threshold mismatch") + tasks = value.get("tasks") + if not isinstance(tasks, int) or isinstance(tasks, bool) or tasks < 0: + raise CalibrationError("calibration probe has an invalid task count") + result: dict[str, int | float] = {"threshold": threshold, "tasks": tasks} + for field in ("elapsed_s", "user_s", "system_s"): + metric = value.get(field) + if ( + not isinstance(metric, (int, float)) + or isinstance(metric, bool) + or not math.isfinite(float(metric)) + or metric < 0 + ): + raise CalibrationError(f"calibration probe has invalid {field}") + result[field] = float(metric) + return result + + +def _frontier_summary(path: Path) -> tuple[str, int]: + digest = hashlib.sha256() + tasks = 0 + with path.open("rb") as stream: + for line in stream: + digest.update(line) + if line.startswith(b"a "): + tasks += 1 + return digest.hexdigest(), tasks + + +def _probe_artifacts( + frontier: Path, log: Path, trace: Path | None +) -> tuple[dict[str, str | None], int]: + try: + frontier_digest, tasks = _frontier_summary(frontier) + artifacts: dict[str, str | None] = { + "frontier_sha256": frontier_digest, + "log_sha256": sha256_file(log), + "trace_sha256": None, + } + if trace is not None: + artifacts["trace_sha256"] = sha256_file(trace) + return artifacts, tasks + except OSError as exc: + raise CalibrationError(f"calibration probe artifact is missing: {exc}") from exc + + +def run_or_resume_probe( + *, + context: ProbeContext, + instance: Path, + instance_id: str, + instance_sha256: str, + threshold: int, + phase: str, + frontier: Path, + log: Path, + checkpoint: Path, + trace: Path | None = None, +) -> dict[str, int | float | str | None]: + identity = { + "contract_sha256": context.contract_digest, + "product_width": context.product_width, + "selector": context.selector, + "max_rows": context.max_rows, + "threshold": threshold, + "phase": phase, + "cuber_sha256": context.cuber_sha256, + "instance_id": instance_id, + "instance_sha256": instance_sha256, + } + try: + checkpoint_text = checkpoint.read_text(encoding="utf-8") + except FileNotFoundError: + checkpoint_text = None + except OSError as exc: + raise CalibrationError(f"invalid calibration checkpoint {checkpoint}: {exc}") from exc + if checkpoint_text is not None: + try: + record = json.loads(checkpoint_text) + except json.JSONDecodeError as exc: + raise CalibrationError(f"invalid calibration checkpoint {checkpoint}: {exc}") from exc + if not isinstance(record, dict): + raise CalibrationError(f"invalid calibration checkpoint {checkpoint}") + if ( + record.get("schema_version") != PROBE_CHECKPOINT_SCHEMA_VERSION + or record.get("kind") != "hard-regime-calibration-probe" + or record.get("identity") != identity + ): + raise CalibrationError(f"calibration checkpoint provenance mismatch: {checkpoint}") + artifacts, tasks = _probe_artifacts(frontier, log, trace) + if record.get("artifacts") != artifacts: + raise CalibrationError(f"calibration checkpoint artifact hash mismatch: {checkpoint}") + result_value = record.get("result") + if not isinstance(result_value, dict): + raise CalibrationError(f"calibration checkpoint result is malformed: {checkpoint}") + result = _validated_probe_result(result_value, threshold) + if tasks != result["tasks"]: + raise CalibrationError(f"calibration checkpoint task count mismatch: {checkpoint}") + return {**result, **artifacts} + + result = _validated_probe_result( + context.probe_runner( + context.cuber, + instance, + threshold, + frontier, + log, + context.max_rows, + trace=trace, + selector=context.selector, + timeout_s=context.timeout_s, + ), + threshold, + ) + artifacts, tasks = _probe_artifacts(frontier, log, trace) + if tasks != result["tasks"]: + raise CalibrationError("reported and emitted calibration task counts differ") + atomic_write_json( + checkpoint, + { + "schema_version": PROBE_CHECKPOINT_SCHEMA_VERSION, + "kind": "hard-regime-calibration-probe", + "identity": identity, + "artifacts": artifacts, + "result": result, + }, + ) + return {**result, **artifacts} def task_counts(row: dict[str, Any]) -> list[int]: @@ -108,7 +259,7 @@ def calibrate_width( selector: str, cuber: Path, out_dir: Path, - initial_threshold: int = 1024, + initial_threshold: int = 1, maximum_threshold: int = 1 << 120, probe_runner: Callable[..., dict[str, int | float]] = run_cuber, ) -> dict[str, Any]: @@ -116,11 +267,25 @@ def calibrate_width( raise CalibrationError(f"unsupported selector {selector!r}") if len(instances) != 3: raise CalibrationError("width-level calibration requires exactly three instances") - if initial_threshold <= 0 or maximum_threshold < initial_threshold: - raise CalibrationError("invalid calibration threshold range") + if initial_threshold != 1 or maximum_threshold <= initial_threshold: + raise CalibrationError( + "calibration threshold search must start at 1 and allow a larger maximum" + ) method = "region-cc" if selector == "region" else "structure-blind-cc" max_rows = int(contract["methods"][method]["max_rows"]) timeout_s = float(contract["limits_seconds"]["cubing"]) + contract_digest = contract_sha256(contract) + cuber_digest = sha256_file(cuber) + probe_context = ProbeContext( + probe_runner=probe_runner, + cuber=cuber, + cuber_sha256=cuber_digest, + contract_digest=contract_digest, + product_width=product_width, + selector=selector, + max_rows=max_rows, + timeout_s=timeout_s, + ) out_dir.mkdir(parents=True, exist_ok=True) observed: dict[int, dict[str, Any]] = {} @@ -131,15 +296,16 @@ def probe(threshold: int) -> dict[str, Any]: threshold_dir.mkdir(parents=True, exist_ok=True) rows = [] for instance in instances: - result = probe_runner( - cuber, - instance["path"], - threshold, - threshold_dir / f"{instance['id']}.icnf", - threshold_dir / f"{instance['id']}.log", - max_rows, - selector=selector, - timeout_s=timeout_s, + result = run_or_resume_probe( + context=probe_context, + instance=instance["path"], + instance_id=instance["id"], + instance_sha256=instance["sha256"], + threshold=threshold, + phase="candidate", + frontier=threshold_dir / f"{instance['id']}.icnf", + log=threshold_dir / f"{instance['id']}.log", + checkpoint=threshold_dir / f"{instance['id']}.probe.json", ) rows.append( { @@ -156,7 +322,7 @@ def probe(threshold: int) -> dict[str, Any]: brackets: dict[str, tuple[int, int]] = {} for band, spec in bands.items(): target = int(spec["center_cubes"]) - lower, upper = 0, initial_threshold + lower, upper = initial_threshold, initial_threshold * 2 probe(lower) while median_tasks(probe(upper)) < target and upper < maximum_threshold: lower, upper = upper, min(upper * 2, maximum_threshold) @@ -184,16 +350,17 @@ def probe(threshold: int) -> dict[str, Any]: for instance in instances: final_dir = out_dir / "selected" / band / instance["id"] final_dir.mkdir(parents=True, exist_ok=True) - final = probe_runner( - cuber, - instance["path"], - threshold, - final_dir / "frontier.icnf", - final_dir / "cuber.log", - max_rows, - final_dir / "nodes.jsonl", - selector, - timeout_s, + final = run_or_resume_probe( + context=probe_context, + instance=instance["path"], + instance_id=instance["id"], + instance_sha256=instance["sha256"], + threshold=threshold, + phase=f"selected-{band}", + frontier=final_dir / "frontier.icnf", + log=final_dir / "cuber.log", + checkpoint=final_dir / "probe.json", + trace=final_dir / "nodes.jsonl", ) candidate = ( out_dir @@ -201,8 +368,7 @@ def probe(threshold: int) -> dict[str, Any]: / f"threshold-{threshold}" / f"{instance['id']}.icnf" ) - frontier = final_dir / "frontier.icnf" - if sha256_file(candidate) != sha256_file(frontier): + if sha256_file(candidate) != final["frontier_sha256"]: raise CalibrationError( f"{instance['id']}: traced rerun changed selected frontier" ) @@ -210,8 +376,8 @@ def probe(threshold: int) -> dict[str, Any]: { "id": instance["id"], "tasks": int(final["tasks"]), - "frontier_sha256": sha256_file(frontier), - "trace_sha256": sha256_file(final_dir / "nodes.jsonl"), + "frontier_sha256": final["frontier_sha256"], + "trace_sha256": final["trace_sha256"], "cubing_elapsed_s": float(final["elapsed_s"]), "cubing_cpu_s": float(final["user_s"]) + float(final["system_s"]), @@ -231,12 +397,17 @@ def probe(threshold: int) -> dict[str, Any]: record = { "schema_version": 1, "kind": "width-level-cc-calibration-lock", - "contract_sha256": contract_sha256(contract), + "contract_sha256": contract_digest, "product_width": product_width, "selector": selector, "method": method, "max_rows": max_rows, - "cuber_sha256": sha256_file(cuber), + "cuber_sha256": cuber_digest, + "search": { + "initial_threshold": initial_threshold, + "maximum_threshold": maximum_threshold, + "probe_checkpoint_schema_version": PROBE_CHECKPOINT_SCHEMA_VERSION, + }, "calibration_instances": [ {"id": instance["id"], "sha256": instance["sha256"]} for instance in instances @@ -258,7 +429,7 @@ def main() -> int: ) parser.add_argument("--cuber", type=Path, required=True) parser.add_argument("--out-dir", type=Path, required=True) - parser.add_argument("--initial-threshold", type=int, default=1024) + parser.add_argument("--initial-threshold", type=int, default=1) parser.add_argument("--maximum-threshold", type=int, default=1 << 120) args = parser.parse_args() try: diff --git a/benchmarks/cnc/hard_regime_matrix.py b/benchmarks/cnc/hard_regime_matrix.py index b237081..30c9336 100755 --- a/benchmarks/cnc/hard_regime_matrix.py +++ b/benchmarks/cnc/hard_regime_matrix.py @@ -226,6 +226,24 @@ def load_calibration_lock( if lock.get("max_rows") != contract["methods"][method]["max_rows"]: raise MatrixError(f"{path}: calibration max_rows mismatch") + search = lock.get("search") + if not isinstance(search, dict): + raise MatrixError(f"{path}: calibration search provenance is missing") + initial_threshold = search.get("initial_threshold") + maximum_threshold = search.get("maximum_threshold") + if ( + not isinstance(initial_threshold, int) + or isinstance(initial_threshold, bool) + or initial_threshold != 1 + or not isinstance(maximum_threshold, int) + or isinstance(maximum_threshold, bool) + or maximum_threshold <= initial_threshold + or not isinstance(search.get("probe_checkpoint_schema_version"), int) + or isinstance(search.get("probe_checkpoint_schema_version"), bool) + or search["probe_checkpoint_schema_version"] <= 0 + ): + raise MatrixError(f"{path}: calibration search provenance is malformed") + response = lock.get("response") if not isinstance(response, list) or not response: raise MatrixError(f"{path}: calibration response is empty") diff --git a/benchmarks/cnc/run_hard_regime_cell.py b/benchmarks/cnc/run_hard_regime_cell.py index 3256d36..197145f 100755 --- a/benchmarks/cnc/run_hard_regime_cell.py +++ b/benchmarks/cnc/run_hard_regime_cell.py @@ -25,12 +25,12 @@ from benchmarks.cnc.hard_regime import HardRegimeError, contract_sha256, load_contract from benchmarks.cnc.hard_regime_matrix import MatrixError, verify_toolchain from benchmarks.pipeline.circuit import ( + atomic_write_json, canonical_bytes, load_json, read_jsonl, sha256_bytes, sha256_file, - write_json, ) @@ -514,12 +514,6 @@ def run_cnc( } -def atomic_terminal(path: Path, value: dict[str, Any]) -> None: - temporary = path.with_suffix(".tmp") - write_json(temporary, value) - os.replace(temporary, path) - - def run_cell( contract: dict[str, Any], matrix_path: Path, @@ -578,9 +572,9 @@ def run_cell( "error": f"{type(exc).__name__}: {exc}", "finished_utc": utc_now(), } - atomic_terminal(terminal_path, terminal) + atomic_write_json(terminal_path, terminal) raise - atomic_terminal(terminal_path, terminal) + atomic_write_json(terminal_path, terminal) return terminal diff --git a/benchmarks/pipeline/circuit.py b/benchmarks/pipeline/circuit.py index d8d26e8..f8310f2 100644 --- a/benchmarks/pipeline/circuit.py +++ b/benchmarks/pipeline/circuit.py @@ -5,6 +5,8 @@ import copy import hashlib import json +import os +import tempfile from collections import defaultdict, deque from collections.abc import Iterable from pathlib import Path @@ -89,6 +91,21 @@ def write_json(path: Path, value: Any) -> None: ) +def atomic_write_json(path: Path, value: Any) -> None: + """Atomically replace *path* with a JSON document via a unique sibling file.""" + path.parent.mkdir(parents=True, exist_ok=True) + descriptor, temporary_name = tempfile.mkstemp( + dir=path.parent, prefix=f".{path.name}.", suffix=".tmp" + ) + os.close(descriptor) + temporary = Path(temporary_name) + try: + write_json(temporary, value) + os.replace(temporary, path) + finally: + temporary.unlink(missing_ok=True) + + def write_jsonl(path: Path, values: Iterable[Any]) -> None: path.parent.mkdir(parents=True, exist_ok=True) with path.open("w", encoding="utf-8") as stream: diff --git a/tests/test_calibrate_hard_regime.py b/tests/test_calibrate_hard_regime.py index 16365c0..fefe279 100644 --- a/tests/test_calibrate_hard_regime.py +++ b/tests/test_calibrate_hard_regime.py @@ -1,10 +1,17 @@ +import inspect import unittest +from dataclasses import replace +from pathlib import Path +from tempfile import TemporaryDirectory -from benchmarks.cnc.calibrate_cc_difficulty import CalibrationError +from benchmarks.cnc.calibrate_cc_difficulty import CalibrationError, sha256_file from benchmarks.cnc.calibrate_hard_regime import ( + calibrate_width, calibration_loss, choose_width_response, median_tasks, + ProbeContext, + run_or_resume_probe, ) @@ -19,6 +26,185 @@ def row(threshold, *counts): class HardRegimeCalibrationTests(unittest.TestCase): + def probe_fixture(self, root: Path, *, phase: str = "candidate"): + cuber = root / "cnc_cuber" + cuber.write_text("fake cuber\n", encoding="utf-8") + instance = root / "instance.json" + instance.write_text("{}\n", encoding="utf-8") + frontier = root / "frontier.icnf" + log = root / "cuber.log" + checkpoint = root / "probe.json" + trace = root / "nodes.jsonl" if phase != "candidate" else None + calls = [] + + def runner( + cuber_path, + instance_path, + threshold, + frontier_path, + log_path, + max_rows, + *, + trace, + selector, + timeout_s, + ): + calls.append((cuber_path, instance_path, threshold, selector, timeout_s)) + frontier_path.write_text("a 1 0\n", encoding="utf-8") + log_path.write_text("status=OK cubes=1\n", encoding="utf-8") + if trace is not None: + trace.write_text('{"node":0}\n', encoding="utf-8") + return { + "threshold": threshold, + "tasks": 1, + "elapsed_s": 2.0, + "user_s": 1.5, + "system_s": 0.25, + } + + arguments = { + "context": ProbeContext( + probe_runner=runner, + cuber=cuber, + cuber_sha256=sha256_file(cuber), + contract_digest="a" * 64, + product_width=64, + selector="region", + max_rows=512, + timeout_s=7200.0, + ), + "instance": instance, + "instance_id": "cal-0", + "instance_sha256": sha256_file(instance), + "threshold": 1, + "phase": phase, + "frontier": frontier, + "log": log, + "checkpoint": checkpoint, + "trace": trace, + } + return arguments, calls + + def test_completed_probe_checkpoint_is_reused(self): + with TemporaryDirectory() as directory: + arguments, calls = self.probe_fixture(Path(directory)) + first = run_or_resume_probe(**arguments) + + def unexpected_runner(*args, **kwargs): + self.fail("valid checkpoint should avoid rerunning the cuber") + + arguments["context"] = replace( + arguments["context"], probe_runner=unexpected_runner + ) + second = run_or_resume_probe(**arguments) + self.assertEqual(first, second) + self.assertEqual(len(calls), 1) + + def test_checkpoint_rejects_tampered_frontier(self): + with TemporaryDirectory() as directory: + arguments, _ = self.probe_fixture(Path(directory)) + run_or_resume_probe(**arguments) + arguments["frontier"].write_text("a 1 0\na 2 0\n", encoding="utf-8") + with self.assertRaisesRegex(CalibrationError, "artifact hash mismatch"): + run_or_resume_probe(**arguments) + + def test_checkpoint_rejects_changed_provenance(self): + with TemporaryDirectory() as directory: + arguments, _ = self.probe_fixture(Path(directory)) + run_or_resume_probe(**arguments) + arguments["context"] = replace( + arguments["context"], selector="structure-blind" + ) + with self.assertRaisesRegex(CalibrationError, "provenance mismatch"): + run_or_resume_probe(**arguments) + + def test_selected_checkpoint_requires_trace(self): + with TemporaryDirectory() as directory: + arguments, _ = self.probe_fixture(Path(directory), phase="selected-low") + run_or_resume_probe(**arguments) + arguments["trace"].unlink() + with self.assertRaisesRegex(CalibrationError, "artifact is missing"): + run_or_resume_probe(**arguments) + + def test_hard_regime_search_starts_at_one(self): + default = inspect.signature(calibrate_width).parameters["initial_threshold"].default + self.assertEqual(default, 1) + + def test_hard_regime_rejects_a_different_search_start(self): + with self.assertRaisesRegex(CalibrationError, "must start at 1"): + calibrate_width( + contract={"methods": {}}, + instances=[{}, {}, {}], + product_width=64, + selector="region", + cuber=Path("cuber"), + out_dir=Path("out"), + initial_threshold=1024, + ) + + def test_hard_regime_search_probes_one_then_two_and_never_zero(self): + with TemporaryDirectory() as directory: + root = Path(directory) + cuber = root / "cnc_cuber" + cuber.write_text("fake cuber\n", encoding="utf-8") + instances = [] + for index in range(3): + path = root / f"cal-{index}.json" + path.write_text("{}\n", encoding="utf-8") + instances.append( + {"id": f"cal-{index}", "path": path, "sha256": sha256_file(path)} + ) + candidate_thresholds = [] + + def runner( + cuber_path, + instance_path, + threshold, + frontier_path, + log_path, + max_rows, + *, + trace, + selector, + timeout_s, + ): + if trace is None: + candidate_thresholds.append(threshold) + frontier_path.write_text( + "".join(f"a {index + 1} 0\n" for index in range(threshold)), + encoding="utf-8", + ) + log_path.write_text("status=OK\n", encoding="utf-8") + if trace is not None: + trace.write_text('{"node":0}\n', encoding="utf-8") + return { + "threshold": threshold, + "tasks": threshold, + "elapsed_s": 1.0, + "user_s": 0.5, + "system_s": 0.25, + } + + calibrate_width( + contract={ + "methods": {"region-cc": {"max_rows": 512}}, + "limits_seconds": {"cubing": 60}, + "frontier_bands": { + "low": {"center_cubes": 2, "accepted_ratio": [0.5, 1.5]} + }, + }, + instances=instances, + product_width=64, + selector="region", + cuber=cuber, + out_dir=root / "calibration", + maximum_threshold=8, + probe_runner=runner, + ) + + self.assertEqual(candidate_thresholds, [1, 1, 1, 2, 2, 2]) + self.assertNotIn(0, candidate_thresholds) + def test_width_selection_uses_all_three_calibration_instances(self): response = [ row(100, 300, 400, 500), diff --git a/tests/test_hard_regime_matrix.py b/tests/test_hard_regime_matrix.py index dfb4246..22ebc87 100644 --- a/tests/test_hard_regime_matrix.py +++ b/tests/test_hard_regime_matrix.py @@ -113,6 +113,11 @@ def write_calibration_locks(root, contract, manifest, cuber_sha): "method": method, "max_rows": contract["methods"][method]["max_rows"], "cuber_sha256": cuber_sha, + "search": { + "initial_threshold": 1, + "maximum_threshold": 1 << 120, + "probe_checkpoint_schema_version": 1, + }, "calibration_instances": calibration, "bands": bands, "response": responses, @@ -199,6 +204,82 @@ def test_matrix_recomputes_the_selected_calibration_threshold(self): with self.assertRaisesRegex(MatrixError, "not selected from the response"): build_matrix(contract, manifest, root, tools) + def test_matrix_rejects_missing_calibration_search_provenance(self): + contract = load_contract(CONTRACT_PATH) + manifest = fake_manifest(contract) + tools = fake_toolchain(contract) + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + write_calibration_locks( + root, + contract, + manifest, + tools["tools"]["cnc_cuber"]["executable_sha256"], + ) + path = root / "p64/region/calibration-lock.json" + lock = __import__("json").loads(path.read_text()) + del lock["search"] + write_json(path, lock) + with self.assertRaisesRegex(MatrixError, "search provenance is missing"): + build_matrix(contract, manifest, root, tools) + + def test_matrix_requires_calibration_search_to_start_at_one(self): + contract = load_contract(CONTRACT_PATH) + manifest = fake_manifest(contract) + tools = fake_toolchain(contract) + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + write_calibration_locks( + root, + contract, + manifest, + tools["tools"]["cnc_cuber"]["executable_sha256"], + ) + path = root / "p64/region/calibration-lock.json" + lock = __import__("json").loads(path.read_text()) + lock["search"]["initial_threshold"] = 1024 + write_json(path, lock) + with self.assertRaisesRegex(MatrixError, "search provenance is malformed"): + build_matrix(contract, manifest, root, tools) + + def test_matrix_accepts_newer_positive_probe_checkpoint_schema(self): + contract = load_contract(CONTRACT_PATH) + manifest = fake_manifest(contract) + tools = fake_toolchain(contract) + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + write_calibration_locks( + root, + contract, + manifest, + tools["tools"]["cnc_cuber"]["executable_sha256"], + ) + for path in root.glob("p*/*/calibration-lock.json"): + lock = __import__("json").loads(path.read_text()) + lock["search"]["probe_checkpoint_schema_version"] = 2 + write_json(path, lock) + matrix = build_matrix(contract, manifest, root, tools) + self.assertEqual(matrix["kind"], "hard-regime-run-matrix") + + def test_matrix_rejects_a_search_without_room_to_expand(self): + contract = load_contract(CONTRACT_PATH) + manifest = fake_manifest(contract) + tools = fake_toolchain(contract) + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + write_calibration_locks( + root, + contract, + manifest, + tools["tools"]["cnc_cuber"]["executable_sha256"], + ) + path = root / "p64/region/calibration-lock.json" + lock = __import__("json").loads(path.read_text()) + lock["search"]["maximum_threshold"] = 1 + write_json(path, lock) + with self.assertRaisesRegex(MatrixError, "search provenance is malformed"): + build_matrix(contract, manifest, root, tools) + def test_toolchain_lock_requires_revision_qualified_upstream_binaries(self): contract = load_contract(CONTRACT_PATH) with tempfile.TemporaryDirectory() as directory: From a11a8cc063d12baa49f42e152b2f086a277490ef Mon Sep 17 00:00:00 2001 From: Xiwei Pan Date: Wed, 22 Jul 2026 13:48:44 +0800 Subject: [PATCH 3/3] Add minimal factoring Cube-and-Conquer workflow --- benchmarks/cnc/README.md | 216 ++---- benchmarks/cnc/aggregate_hard_regime.py | 653 ------------------- benchmarks/cnc/calibrate_cc_difficulty.py | 247 ------- benchmarks/cnc/calibrate_hard_regime.py | 455 ------------- benchmarks/cnc/conquer_parallel.py | 119 +--- benchmarks/cnc/contracts/hard-regime-v1.yaml | 113 ---- benchmarks/cnc/cubing.py | 173 +++++ benchmarks/cnc/factoring.py | 287 ++++++++ benchmarks/cnc/hard_regime.py | 436 ------------- benchmarks/cnc/hard_regime_matrix.py | 556 ---------------- benchmarks/cnc/hpc/build_matrix.sh | 25 - benchmarks/cnc/hpc/calibrate.sh | 27 - benchmarks/cnc/hpc/materialize.sh | 20 - benchmarks/cnc/hpc/report.sh | 35 - benchmarks/cnc/hpc/run_cell.sh | 27 - benchmarks/cnc/hpc/smoke.sh | 19 - benchmarks/cnc/requirements.txt | 2 - benchmarks/cnc/run_hard_regime_cell.py | 614 ----------------- benchmarks/cnc/solve.py | 174 +++++ benchmarks/cnc/verify_hard_regime.py | 440 ------------- benchmarks/cnc/verify_measurements.py | 460 ------------- benchmarks/pipeline/multipliers.py | 15 +- src/bin/cnc_cuber.rs | 199 ++++-- src/conquer.rs | 313 +++++++++ src/lib.rs | 1 + tests/cnc_streaming.rs | 151 +++++ tests/test_aggregate_hard_regime.py | 91 --- tests/test_calibrate_cc_difficulty.py | 36 - tests/test_calibrate_hard_regime.py | 233 ------- tests/test_cnc_measurements.py | 100 --- tests/test_conquer_parallel.py | 125 ++-- tests/test_cubing.py | 58 ++ tests/test_factoring_corpus.py | 45 ++ tests/test_hard_regime.py | 73 --- tests/test_hard_regime_matrix.py | 307 --------- tests/test_run_hard_regime_cell.py | 147 ----- tests/test_solve.py | 24 + tests/test_verify_hard_regime.py | 133 ---- 38 files changed, 1549 insertions(+), 5600 deletions(-) delete mode 100755 benchmarks/cnc/aggregate_hard_regime.py delete mode 100644 benchmarks/cnc/calibrate_cc_difficulty.py delete mode 100755 benchmarks/cnc/calibrate_hard_regime.py delete mode 100644 benchmarks/cnc/contracts/hard-regime-v1.yaml create mode 100644 benchmarks/cnc/cubing.py create mode 100644 benchmarks/cnc/factoring.py delete mode 100755 benchmarks/cnc/hard_regime.py delete mode 100755 benchmarks/cnc/hard_regime_matrix.py delete mode 100755 benchmarks/cnc/hpc/build_matrix.sh delete mode 100755 benchmarks/cnc/hpc/calibrate.sh delete mode 100755 benchmarks/cnc/hpc/materialize.sh delete mode 100755 benchmarks/cnc/hpc/report.sh delete mode 100755 benchmarks/cnc/hpc/run_cell.sh delete mode 100755 benchmarks/cnc/hpc/smoke.sh delete mode 100644 benchmarks/cnc/requirements.txt delete mode 100755 benchmarks/cnc/run_hard_regime_cell.py create mode 100644 benchmarks/cnc/solve.py delete mode 100755 benchmarks/cnc/verify_hard_regime.py delete mode 100644 benchmarks/cnc/verify_measurements.py create mode 100644 src/conquer.rs create mode 100644 tests/cnc_streaming.rs delete mode 100644 tests/test_aggregate_hard_regime.py delete mode 100644 tests/test_calibrate_cc_difficulty.py delete mode 100644 tests/test_calibrate_hard_regime.py delete mode 100644 tests/test_cnc_measurements.py create mode 100644 tests/test_cubing.py create mode 100644 tests/test_factoring_corpus.py delete mode 100644 tests/test_hard_regime.py delete mode 100644 tests/test_hard_regime_matrix.py delete mode 100644 tests/test_run_hard_regime_cell.py create mode 100644 tests/test_solve.py delete mode 100644 tests/test_verify_hard_regime.py diff --git a/benchmarks/cnc/README.md b/benchmarks/cnc/README.md index a949d54..e220adf 100644 --- a/benchmarks/cnc/README.md +++ b/benchmarks/cnc/README.md @@ -1,168 +1,76 @@ -# Auditable Cube-and-Conquer measurements - -## Issue #51 hard-UNSAT factoring study - -The frozen product-width study contract is -`contracts/hard-regime-v1.yaml`. Product width and factor-input width are -separate required fields: the declared pairs are 64/32, 72/36, and 80/40. -Targets are deterministic full-product-width primes inside the reachable -unsigned multiplier range and above the factor-input range, so each pinned -instance is UNSAT by construction. - -Generate and independently verify the 39 declared targets before materializing -the array-ripple CircuitSAT/CNF pairs: - -```bash -python3 -m benchmarks.cnc.hard_regime generate-targets \ - benchmarks/cnc/contracts/hard-regime-v1.yaml \ - --out artifacts/cnc-hard-regime/targets.jsonl -python3 -m benchmarks.cnc.hard_regime verify-targets \ - benchmarks/cnc/contracts/hard-regime-v1.yaml \ - artifacts/cnc-hard-regime/targets.jsonl -python3 -m benchmarks.cnc.hard_regime materialize \ - benchmarks/cnc/contracts/hard-regime-v1.yaml \ - --out-dir artifacts/cnc-hard-regime/instances -``` +# Factoring Cube-and-Conquer + +This directory contains only the operations needed for factoring runs: + +1. generate paired SAT/UNSAT `n×n` factoring instances in CircuitSAT and CNF; +2. solve a CNF directly with Kissat; +3. cube a CNF with `march_cu`, then solve the cubes in parallel with Kissat; +4. use the Rust solver either to export a complete frontier or to stream cubes + directly into a bounded parallel Kissat pool; +5. solve a frozen frontier in Python for reproducible paper measurements. -Raw run artifacts remain outside Git. The contract, checksummed artifact -manifest, aggregate statistics, primary table, and report are the reviewable -payload. - -Freeze a selector's three thresholds from exactly the three calibration -instances at one product width. This produces one shared threshold per band; -it never searches on a held-out instance: - -```bash -python3 -m benchmarks.cnc.calibrate_hard_regime \ - benchmarks/cnc/contracts/hard-regime-v1.yaml \ - artifacts/cnc-hard-regime/instances/manifest.jsonl \ - --product-width 64 --selector region \ - --cuber target/release/cnc_cuber \ - --out-dir artifacts/cnc-hard-regime/calibration/p64/region +## Generate instances + +```sh +PYTHONPATH=. python3 -m benchmarks.cnc.factoring \ + --width 26 --width 28 --count 10 \ + --out-dir artifacts/factoring ``` -Hard-regime threshold search starts at `1` and doubles before bisection for -both selectors. Every completed instance probe writes a provenance- and -hash-checked `*.probe.json` checkpoint next to its frontier and log. Re-running -the same calibration validates and reuses complete probes, while malformed, -tampered, or mismatched checkpoints fail closed. - -After all six width/selector calibration locks exist, capture the exact tool -binaries and build the immutable 249-cell matrix (39 hardness rows plus 210 -held-out CnC rows): - -```bash -python3 -m benchmarks.cnc.hard_regime_matrix lock-toolchain \ - benchmarks/cnc/contracts/hard-regime-v1.yaml \ - --cuber target/release/cnc_cuber \ - --kissat cnc-tools/bin/kissat-8af8e56f174b778aef3aa45af9f739b2a5f492c2 \ - --march-cu cnc-tools/bin/march_cu-705b60c6491ef2b61988b3ce6ac674be1b90571d \ - --repository-revision "$(git rev-parse HEAD)" \ - --out artifacts/cnc-hard-regime/toolchain.json -python3 -m benchmarks.cnc.hard_regime_matrix build-matrix \ - benchmarks/cnc/contracts/hard-regime-v1.yaml \ - artifacts/cnc-hard-regime/instances/manifest.jsonl \ - --calibration-root artifacts/cnc-hard-regime/calibration \ - --toolchain artifacts/cnc-hard-regime/toolchain.json \ - --out artifacts/cnc-hard-regime/run-matrix.json +Each manifest row points to `instance.circuitsat.json` and `instance.cnf` with +matching SAT/UNSAT metadata and hashes. + +## Direct Kissat + +```sh +PYTHONPATH=. python3 -m benchmarks.cnc.solve INSTANCE.cnf \ + --kissat cnc-tools/bin/kissat \ + --timeout-s 600 --out-dir artifacts/direct ``` -Every Slurm task executes exactly one immutable cell and writes -`cells//terminal.json` last. Re-running a completed cell is idempotent; -timeouts and errors are terminal outcomes rather than missing rows: - -```bash -python3 -m benchmarks.cnc.run_hard_regime_cell \ - benchmarks/cnc/contracts/hard-regime-v1.yaml \ - artifacts/cnc-hard-regime/run-matrix.json \ - artifacts/cnc-hard-regime/toolchain.json \ - --cell-id CELL_ID \ - --instance-root artifacts/cnc-hard-regime/instances \ - --output-root artifacts/cnc-hard-regime/runs +## march_cu then parallel Kissat + +```sh +PYTHONPATH=. python3 -m benchmarks.cnc.cubing march INSTANCE.cnf \ + --march-cu cnc-tools/bin/march_cu \ + --kissat cnc-tools/bin/kissat --workers 32 \ + --out-dir artifacts/march ``` -Verify either the pilot gate (all 39 monolithic rows plus the 72 pilot cells) -or the complete 249-cell table. The verifier reconstructs raw scheduling and -rejects missing cells, width confusion, split leakage, per-test thresholds, -mixed CNFs, malformed branch assignments/refutation reasons, and censored cubes -omitted from work/span accounting. Harness-error terminals remain visible and -are explicitly excluded from any claim that per-cube work/span reconstructed: - -```bash -python3 -m benchmarks.cnc.verify_hard_regime \ - benchmarks/cnc/contracts/hard-regime-v1.yaml \ - artifacts/cnc-hard-regime/instances/manifest.jsonl \ - --calibration-root artifacts/cnc-hard-regime/calibration \ - --toolchain artifacts/cnc-hard-regime/toolchain.json \ - --matrix artifacts/cnc-hard-regime/run-matrix.json \ - --runs-root artifacts/cnc-hard-regime/runs --scope pilot +Pass `--remaining-vars N` to override `march_cu`'s dynamic cutoff. + +## Rust solver: export all cubes + +```sh +cargo build --release --bin cnc_cuber +target/release/cnc_cuber INSTANCE.circuitsat.json \ + --cc-threshold 65536 -o artifacts/project/frontier.icnf ``` -Generate raw paired rows, log-log frontier-budget interpolation on common -support, deterministic instance bootstrap intervals, the primary table, and a -short report only after verification succeeds: +This mode finishes the whole cubing traversal and never starts Kissat. -```bash -python3 -m benchmarks.cnc.aggregate_hard_regime \ - benchmarks/cnc/contracts/hard-regime-v1.yaml \ - artifacts/cnc-hard-regime/run-matrix.json \ - --runs-root artifacts/cnc-hard-regime/runs --scope pilot \ - --out-dir artifacts/cnc-hard-regime/pilot-report +## Rust solver: streaming Cube-and-Conquer + +```sh +target/release/cnc_cuber INSTANCE.circuitsat.json \ + --cc-threshold 65536 \ + --solve-cnf INSTANCE.cnf \ + --kissat cnc-tools/bin/kissat --workers 32 ``` -Pass `--aggregate artifacts/cnc-hard-regime/pilot-report/aggregate.json` to the -verifier for a final regeneration check that also rejects duplicate instance -IDs in any bootstrap group. - -## HPC2 execution - -The scripts in `hpc/` contain no hidden Slurm resource defaults. Supply every -partition, walltime, task/CPU count, memory limit, array range, working -directory, environment export, and stdout/stderr path explicitly to `sbatch`. -Use `BI51_SCRATCH_ROOT` on the HPC2 `jhspoolers` SSD for temporary per-cube CNF -files; persistent instances, raw results, and reports remain under -`BI51_ARTIFACT_ROOT` on HDD. Validate `smoke.sh` in the free `debug` partition -before materialization, calibration, or a run-cell array. - -## Current CnC comparison - -The primary comparison uses online stopping rules on both sides: - -- Region cubing reads structured CircuitSAT and stops at classical CC - difficulty `D^2(D+I)/N > threshold`. Calibrate the threshold by frontier - count only with `calibrate_cc_difficulty.py`. -- `march_cu` reads a globally encoded CNF and uses its upstream default dynamic - cutoff unchanged. Do not pass `-d`, `-n`, `-e`, or `-f` in the primary arm. -- Both frontiers are conquered by the same solver and resource policy. Report - preprocessing, cubing, and conquer work/span separately and end to end. - -Machine-specific Slurm wrappers are intentionally not versioned. Build the -frontiers with `cnc_cuber`, `calibrate_cc_difficulty.py`, and upstream -`march_cu`, then run both arms through `conquer_parallel.py`. Rejected -static/product cutoff workflows are not maintained. - -Each measurement bundle is a directory containing a hash-linked `bundle.json`, -the input DIMACS file, a frontier JSONL, a monotonic event JSONL, per-cube raw -result JSONL, and a SAT witness when applicable. The verifier treats those raw -records as authoritative and independently reconstructs: - -- complete, non-overlapping frontier coverage; -- every cube's solved, cancelled, timed-out, or never-started lifecycle; -- cubing wall/CPU time, conquer CPU work and scheduled makespan; -- orchestration and end-to-end wall time; -- maximum worker concurrency and the aggregate verdict; -- input, tool, and executable provenance plus SAT witness validity. - -Run the positive fixture and the generated negative controls from the -repository root: - -```bash -python3 benchmarks/cnc/verify_measurements.py \ - --bundle tests/fixtures/cnc/measurement-valid - -python3 -m unittest tests/test_cnc_measurements.py +The Rust cuber submits each open leaf immediately to a bounded worker pool. +Exit codes follow SAT conventions: `10` for SAT and `20` for UNSAT. + +## Analyze a frozen frontier + +```sh +PYTHONPATH=. python3 -m benchmarks.cnc.cubing frontier \ + INSTANCE.cnf artifacts/project/frontier.icnf \ + --kissat cnc-tools/bin/kissat --workers 32 \ + --out-dir artifacts/project-analysis ``` -The exhaustive frontier representation is intended for small audit fixtures. -Large production runs should add a branching-tree certificate before using -this format beyond a tractable number of frontier variables. +The Python path never invokes the project cuber. It consumes a complete, +frozen frontier and records per-cube timing, decisions, conflicts, and aggregate +statistics. March and project frontiers can therefore use the same analysis +backend. diff --git a/benchmarks/cnc/aggregate_hard_regime.py b/benchmarks/cnc/aggregate_hard_regime.py deleted file mode 100755 index 824ef99..0000000 --- a/benchmarks/cnc/aggregate_hard_regime.py +++ /dev/null @@ -1,653 +0,0 @@ -#!/usr/bin/env python3 -"""Aggregate issue #51 terminal cells with held-out instances as the sample unit.""" - -from __future__ import annotations - -import argparse -import csv -import hashlib -import json -import math -import random -from pathlib import Path -from typing import Any - -from benchmarks.cnc.hard_regime import HardRegimeError, contract_sha256, load_contract -from benchmarks.cnc.verify_hard_regime import selected_cells -from benchmarks.pipeline.circuit import load_json, write_json, write_jsonl - - -class AggregateError(HardRegimeError): - """Terminal records cannot produce the preregistered instance-level analysis.""" - - -METRICS = { - "conquer_work_cpu_s": "Conquer work", - "conquer_span_s": "Maximum cube time", - "end_to_end_wall_s": "End-to-end wall time", - "measured_32_worker_makespan_s": "Measured 32-worker makespan", - "lpt_32_s": "LPT 32-worker makespan", - "lpt_128_s": "LPT 128-worker makespan", - "lpt_512_s": "LPT 512-worker makespan", -} - - -def metric_value(terminal: dict[str, Any], metric: str) -> float | None: - metrics = terminal.get("metrics") - if not isinstance(metrics, dict): - return None - if metric.startswith("lpt_"): - workers = metric.split("_")[1] - replay = metrics.get("lpt_makespan_by_workers_s") - value = replay.get(workers) if isinstance(replay, dict) else None - else: - value = metrics.get(metric) - if not isinstance(value, (int, float)) or isinstance(value, bool) or value <= 0: - return None - return float(value) - - -def geometric_mean(values: list[float]) -> float: - if not values or any(value <= 0 for value in values): - raise AggregateError("geometric mean requires positive observations") - return math.exp(sum(math.log(value) for value in values) / len(values)) - - -def percentile(values: list[float], probability: float) -> float: - ordered = sorted(values) - index = min(len(ordered) - 1, max(0, round(probability * (len(ordered) - 1)))) - return ordered[index] - - -def bootstrap_ci( - values: list[float], samples: int, confidence: float, seed: int, key: str -) -> list[float] | None: - if not values: - return None - key_seed = int(hashlib.sha256(key.encode()).hexdigest()[:16], 16) - rng = random.Random(seed ^ key_seed) - replicates = [ - geometric_mean([values[rng.randrange(len(values))] for _ in values]) - for _ in range(samples) - ] - alpha = (1.0 - confidence) / 2.0 - return [percentile(replicates, alpha), percentile(replicates, 1.0 - alpha)] - - -def interpolate_log(points: list[tuple[float, float]], target: float) -> float | None: - usable = sorted((x, y) for x, y in points if x > 0 and y > 0) - if not usable or target < usable[0][0] or target > usable[-1][0]: - return None - for x, y in usable: - if math.isclose(x, target, rel_tol=1e-12, abs_tol=0.0): - return y - for (left_x, left_y), (right_x, right_y) in zip(usable, usable[1:]): - if left_x <= target <= right_x: - if left_x == right_x: - return geometric_mean([left_y, right_y]) - fraction = (math.log(target) - math.log(left_x)) / ( - math.log(right_x) - math.log(left_x) - ) - return math.exp(math.log(left_y) + fraction * (math.log(right_y) - math.log(left_y))) - return None - - -def load_terminals( - matrix: dict[str, Any], runs_root: Path, scope: str -) -> dict[str, dict[str, Any]]: - terminals = {} - for cell in selected_cells(matrix, scope): - path = runs_root / "cells" / cell["cell_id"] / "terminal.json" - if not path.is_file(): - raise AggregateError(f"missing terminal cell {cell['cell_id']}") - terminal = load_json(path) - if terminal.get("cell_id") != cell["cell_id"]: - raise AggregateError(f"terminal identity mismatch for {cell['cell_id']}") - terminals[cell["cell_id"]] = terminal - return terminals - - -def paired_raw_observations( - matrix: dict[str, Any], terminals: dict[str, dict[str, Any]], metric: str -) -> list[dict[str, Any]]: - cells = { - (cell["instance_id"], cell["method"], cell["budget"]): cell - for cell in matrix["cells"] - if cell["split"] == "held_out" - } - observations = [] - for instance_id in sorted({key[0] for key in cells}): - for budget in ("low", "medium", "high"): - region = cells.get((instance_id, "region-cc", budget)) - blind = cells.get((instance_id, "structure-blind-cc", budget)) - if region is None or blind is None: - continue - region_terminal = terminals.get(region["cell_id"]) - blind_terminal = terminals.get(blind["cell_id"]) - if region_terminal is None or blind_terminal is None: - continue - region_value = metric_value(region_terminal, metric) - blind_value = metric_value(blind_terminal, metric) - complete = ( - region_terminal.get("state") == "complete" - and blind_terminal.get("state") == "complete" - and region_value is not None - and blind_value is not None - ) - observations.append( - { - "analysis": "raw-nominal-budget", - "instance_id": instance_id, - "product_width": region["product_width"], - "budget": budget, - "metric": metric, - "region_state": region_terminal.get("state"), - "blind_state": blind_terminal.get("state"), - "region_value": region_value, - "blind_value": blind_value, - "ratio": region_value / blind_value if complete else None, - "complete_pair": complete, - } - ) - return observations - - -def adjusted_observations( - matrix: dict[str, Any], terminals: dict[str, dict[str, Any]], metric: str -) -> list[dict[str, Any]]: - cells = { - (cell["instance_id"], cell["method"], cell["budget"]): cell - for cell in matrix["cells"] - if cell["split"] == "held_out" - } - observations = [] - declared_instances = sorted( - { - cell["instance_id"] - for cell in matrix["cells"] - if cell["split"] == "held_out" - and cell["method"] in {"region-cc", "structure-blind-cc"} - and cell["cell_id"] in terminals - } - ) - for instance_id in declared_instances: - series = {} - product_width = next( - cell["product_width"] - for cell in matrix["cells"] - if cell["instance_id"] == instance_id - ) - for method in ("region-cc", "structure-blind-cc"): - points = [] - states = [] - for budget in ("low", "medium", "high"): - cell = cells.get((instance_id, method, budget)) - if cell is None or cell["cell_id"] not in terminals: - continue - product_width = cell["product_width"] - terminal = terminals[cell["cell_id"]] - states.append(terminal.get("state")) - x = terminal.get("metrics", {}).get("frontier_size") - y = metric_value(terminal, metric) - if terminal.get("state") == "complete" and isinstance(x, (int, float)) and x > 0 and y is not None: - points.append((float(x), y)) - series[method] = {"points": points, "states": states} - region_points = series["region-cc"]["points"] - blind_points = series["structure-blind-cc"]["points"] - if not region_points or not blind_points: - support = None - adjustment_status = "incomplete-series" - else: - lower = max(min(x for x, _ in region_points), min(x for x, _ in blind_points)) - upper = min(max(x for x, _ in region_points), max(x for x, _ in blind_points)) - support = (lower, upper) if lower <= upper else None - adjustment_status = "common-support" if support is not None else "no-common-support" - grids = ( - {} - if support is None - else { - "overlap-low": support[0], - "overlap-mid": math.sqrt(support[0] * support[1]), - "overlap-high": support[1], - } - ) - for grid in ("overlap-low", "overlap-mid", "overlap-high"): - frontier_size = grids.get(grid) - region_value = ( - interpolate_log(region_points, frontier_size) - if frontier_size is not None - else None - ) - blind_value = ( - interpolate_log(blind_points, frontier_size) - if frontier_size is not None - else None - ) - complete = region_value is not None and blind_value is not None - observations.append( - { - "analysis": "frontier-budget-adjusted", - "instance_id": instance_id, - "product_width": product_width, - "grid": grid, - "common_frontier_size": frontier_size, - "metric": metric, - "adjustment_status": adjustment_status, - "region_states": series["region-cc"]["states"], - "blind_states": series["structure-blind-cc"]["states"], - "region_value": region_value, - "blind_value": blind_value, - "ratio": region_value / blind_value if complete else None, - "complete_pair": complete, - } - ) - return observations - - -def summarize_observations( - observations: list[dict[str, Any]], - analysis: str, - metric: str, - budget_key: str, - budget_value: str, - product_width: int | None, - bootstrap: dict[str, Any], -) -> dict[str, Any]: - selected = [ - row - for row in observations - if row["analysis"] == analysis - and row["metric"] == metric - and row.get(budget_key) == budget_value - and (product_width is None or row["product_width"] == product_width) - ] - complete = [row for row in selected if row["complete_pair"]] - ratios = [float(row["ratio"]) for row in complete] - instance_ids = [row["instance_id"] for row in complete] - declared_instance_ids = [row["instance_id"] for row in selected] - if len(instance_ids) != len(set(instance_ids)): - raise AggregateError("an aggregate group contains duplicate instance observations") - group = "overall" if product_width is None else f"p{product_width}" - key = f"{analysis}:{metric}:{budget_key}={budget_value}:{group}" - geometric_ratio = geometric_mean(ratios) if ratios else None - if metric == "conquer_work_cpu_s": - criterion = "ratio-no-worse-than-1.10" - meets_criterion = geometric_ratio is not None and geometric_ratio <= 1.10 - elif metric == "conquer_span_s": - criterion = "ratio-below-1.00" - meets_criterion = geometric_ratio is not None and geometric_ratio < 1.0 - else: - criterion = None - meets_criterion = None - return { - "analysis": analysis, - "metric": metric, - budget_key: budget_value, - "group": group, - "statistical_unit": "held-out-instance", - "declared_pairs": len(selected), - "complete_pairs": len(complete), - "censored_or_failed_pairs": len(selected) - len(complete), - "declared_instance_ids": declared_instance_ids, - "censored_or_failed_instance_ids": [ - row["instance_id"] for row in selected if not row["complete_pair"] - ], - "instance_ids": instance_ids, - "geometric_mean_ratio": geometric_ratio, - "region_win_rate": ( - sum(ratio < 1.0 for ratio in ratios) / len(ratios) if ratios else None - ), - "bootstrap_95_ci": bootstrap_ci( - ratios, - int(bootstrap["samples"]), - float(bootstrap["confidence"]), - int(bootstrap["seed"]), - key, - ), - "scientific_target": criterion, - "meets_scientific_target": meets_criterion, - } - - -def terminal_cell_rows( - matrix: dict[str, Any], terminals: dict[str, dict[str, Any]], scope: str -) -> list[dict[str, Any]]: - rows = [] - for cell in selected_cells(matrix, scope): - terminal = terminals[cell["cell_id"]] - rows.append( - { - "cell_id": cell["cell_id"], - "instance_id": cell["instance_id"], - "split": cell["split"], - "product_width": cell["product_width"], - "factor_input_width": cell["factor_input_width"], - "pilot": cell["pilot"], - "method": cell["method"], - "budget": cell["budget"], - "state": terminal["state"], - "verdict": terminal.get("verdict"), - "metrics": terminal.get("metrics"), - } - ) - return rows - - -def method_summaries(cell_rows: list[dict[str, Any]]) -> list[dict[str, Any]]: - summaries = [] - method_budgets = sorted({(row["method"], row["budget"]) for row in cell_rows}) - for method, budget in method_budgets: - for width in (None, 64, 72, 80): - rows = [ - row - for row in cell_rows - if row["method"] == method - and row["budget"] == budget - and row["split"] == "held_out" - and (width is None or row["product_width"] == width) - ] - if not rows: - continue - states: dict[str, int] = {} - for row in rows: - states[row["state"]] = states.get(row["state"], 0) + 1 - numeric = {} - metric_names = ( - "solver_wall_s", - "conquer_work_cpu_s", - "conquer_span_s", - "measured_32_worker_makespan_s", - "end_to_end_wall_s", - ) - for metric in metric_names: - values = [ - float(row["metrics"][metric]) - for row in rows - if row["state"] == "complete" - and isinstance(row.get("metrics"), dict) - and isinstance(row["metrics"].get(metric), (int, float)) - and row["metrics"][metric] > 0 - ] - numeric[metric] = { - "complete_values": len(values), - "geometric_mean": geometric_mean(values) if values else None, - } - summaries.append( - { - "method": method, - "budget": budget, - "group": "overall" if width is None else f"p{width}", - "declared_instances": len(rows), - "terminal_states": states, - "metrics": numeric, - } - ) - return summaries - - -def hardness_summary(matrix: dict[str, Any], terminals: dict[str, dict[str, Any]]) -> list[dict[str, Any]]: - rows = [] - for width in (64, 72, 80): - cells = [ - cell - for cell in matrix["cells"] - if cell["method"] == "monolithic-kissat" - and cell["split"] == "held_out" - and cell["product_width"] == width - ] - states = [terminals[cell["cell_id"]]["state"] for cell in cells] - timeouts = sum(state == "monolithic-timeout" for state in states) - rows.append( - { - "product_width": width, - "held_out_instances": len(cells), - "monolithic_timeouts": timeouts, - "cnc_regime": timeouts > len(cells) / 2, - } - ) - return rows - - -def pilot_gate( - matrix: dict[str, Any], terminals: dict[str, dict[str, Any]], hardness: list[dict[str, Any]] -) -> dict[str, Any]: - regime_widths = { - row["product_width"] for row in hardness if row["cnc_regime"] - } - frontier_methods = [] - for width in sorted(regime_widths): - for method in ("march-cu-dynamic", "region-cc", "structure-blind-cc"): - cells = [ - cell - for cell in matrix["cells"] - if cell.get("pilot") - and cell["product_width"] == width - and cell["method"] == method - ] - if cells and all( - terminals[cell["cell_id"]].get("stages", {}).get("cubing", {}).get("complete") - for cell in cells - ): - frontier_methods.append({"product_width": width, "method": method}) - artifact_reconstruction = all( - terminal.get("state") != "harness-error" for terminal in terminals.values() - ) - return { - "at_least_two_hard_widths": len(regime_widths) >= 2, - "hard_widths": sorted(regime_widths), - "complete_frontier_methods_in_regime": frontier_methods, - "at_least_one_complete_frontier_method": bool(frontier_methods), - "artifact_reconstruction": artifact_reconstruction, - "passed": ( - len(regime_widths) >= 2 - and bool(frontier_methods) - and artifact_reconstruction - ), - } - - -def write_primary_table(path: Path, summaries: list[dict[str, Any]]) -> None: - relevant = [ - row - for row in summaries - if row["analysis"] == "raw-nominal-budget" - and row["metric"] in {"conquer_work_cpu_s", "conquer_span_s"} - and row["group"] != "overall" - ] - lookup = { - (row["group"], row["budget"], row["metric"]): row for row in relevant - } - lines = [ - "| Product width | Budget | Complete pairs | Work ratio [95% CI] | Span ratio [95% CI] | Span win rate |", - "|---:|:---|---:|:---|:---|---:|", - ] - - def ratio(row: dict[str, Any] | None) -> str: - if row is None or row["geometric_mean_ratio"] is None: - return "NA" - interval = row["bootstrap_95_ci"] - return f"{row['geometric_mean_ratio']:.3f} [{interval[0]:.3f}, {interval[1]:.3f}]" - - for width in (64, 72, 80): - for budget in ("low", "medium", "high"): - work = lookup.get((f"p{width}", budget, "conquer_work_cpu_s")) - span = lookup.get((f"p{width}", budget, "conquer_span_s")) - complete = min( - work["complete_pairs"] if work else 0, - span["complete_pairs"] if span else 0, - ) - win = "NA" if span is None or span["region_win_rate"] is None else f"{span['region_win_rate']:.1%}" - lines.append( - f"| {width} | {budget} | {complete} | {ratio(work)} | {ratio(span)} | {win} |" - ) - path.write_text("\n".join(lines) + "\n", encoding="utf-8") - - -def write_raw_csv(path: Path, observations: list[dict[str, Any]]) -> None: - rows = [row for row in observations if row["analysis"] == "raw-nominal-budget"] - fields = [ - "instance_id", - "product_width", - "budget", - "metric", - "region_state", - "blind_state", - "region_value", - "blind_value", - "ratio", - "complete_pair", - ] - with path.open("w", newline="", encoding="utf-8") as stream: - writer = csv.DictWriter(stream, fieldnames=fields, extrasaction="ignore") - writer.writeheader() - writer.writerows(rows) - - -def write_report( - path: Path, - scope: str, - hardness: list[dict[str, Any]], - gate: dict[str, Any], - summaries: list[dict[str, Any]], - methods: list[dict[str, Any]], -) -> None: - complete_raw = sum( - row["complete_pairs"] - for row in summaries - if row["analysis"] == "raw-nominal-budget" and row["group"] == "overall" - ) - complete_adjusted = sum( - row["complete_pairs"] - for row in summaries - if row["analysis"] == "frontier-budget-adjusted" and row["group"] == "overall" - ) - lines = [ - "# Hard-regime CnC work/span report", - "", - f"Scope: `{scope}`. Statistical unit: held-out factoring instance.", - "", - "## Hardness gate", - "", - ] - lines.extend( - f"- Product width {row['product_width']}: {row['monolithic_timeouts']}/{row['held_out_instances']} monolithic timeouts; CnC regime = {row['cnc_regime']}." - for row in hardness - ) - lines.extend( - [ - "", - f"Pilot gate passed: **{gate['passed']}**.", - "", - "## Raw paired results", - "", - f"Complete overall instance-pairs across metrics/budgets: {complete_raw}. Censored and failed pairs remain in the machine-readable rows.", - "", - "## Frontier-budget-adjusted results", - "", - f"Complete interpolated overall instance-pairs across metrics/grid points: {complete_adjusted}. Interpolation is log-log within common support only; no extrapolation is used.", - "", - "## Finite-worker results", - "", - "LPT makespans are reported for 32, 128, and 512 workers. The directly measured 32-worker schedule remains separate from replayed LPT estimates.", - ] - ) - lines.extend(["", "## Method terminal overview", ""]) - for row in methods: - if row["group"] != "overall": - continue - end_to_end = row["metrics"]["end_to_end_wall_s"]["geometric_mean"] - end_text = "NA" if end_to_end is None else f"{end_to_end:.3f}s" - lines.append( - f"- `{row['method']}` / `{row['budget']}`: states={row['terminal_states']}; complete-case geometric mean end-to-end={end_text}." - ) - path.write_text("\n".join(lines) + "\n", encoding="utf-8") - - -def aggregate( - contract: dict[str, Any], - matrix: dict[str, Any], - terminals: dict[str, dict[str, Any]], - scope: str, -) -> dict[str, Any]: - cells = terminal_cell_rows(matrix, terminals, scope) - observations = [] - for metric in METRICS: - observations.extend(paired_raw_observations(matrix, terminals, metric)) - observations.extend(adjusted_observations(matrix, terminals, metric)) - bootstrap = contract["statistics"]["bootstrap"] - summaries = [] - for analysis, budget_key, budget_values in ( - ("raw-nominal-budget", "budget", ("low", "medium", "high")), - ( - "frontier-budget-adjusted", - "grid", - ("overlap-low", "overlap-mid", "overlap-high"), - ), - ): - for metric in METRICS: - for budget_value in budget_values: - for width in (None, 64, 72, 80): - summaries.append( - summarize_observations( - observations, - analysis, - metric, - budget_key, - budget_value, - width, - bootstrap, - ) - ) - hardness = hardness_summary(matrix, terminals) - gate = pilot_gate(matrix, terminals, hardness) - return { - "schema_version": 1, - "kind": "hard-regime-instance-level-aggregate", - "contract_sha256": contract_sha256(contract), - "scope": scope, - "statistical_unit": "held-out-instance", - "bootstrap": bootstrap, - "hardness": hardness, - "pilot_gate": gate, - "method_summaries": method_summaries(cells), - "summaries": summaries, - "terminal_cells": cells, - "observations": observations, - } - - -def main() -> int: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("contract", type=Path) - parser.add_argument("matrix", type=Path) - parser.add_argument("--runs-root", type=Path, required=True) - parser.add_argument("--scope", choices=("pilot", "full"), default="full") - parser.add_argument("--out-dir", type=Path, required=True) - args = parser.parse_args() - try: - contract = load_contract(args.contract) - matrix = load_json(args.matrix) - terminals = load_terminals(matrix, args.runs_root, args.scope) - result = aggregate(contract, matrix, terminals, args.scope) - args.out_dir.mkdir(parents=True, exist_ok=True) - write_json(args.out_dir / "aggregate.json", result) - write_jsonl(args.out_dir / "terminal-cells.jsonl", result["terminal_cells"]) - write_jsonl(args.out_dir / "paired-observations.jsonl", result["observations"]) - write_raw_csv(args.out_dir / "raw-paired-results.csv", result["observations"]) - write_primary_table(args.out_dir / "primary-table.md", result["summaries"]) - write_report( - args.out_dir / "report.md", - args.scope, - result["hardness"], - result["pilot_gate"], - result["summaries"], - result["method_summaries"], - ) - except (AggregateError, OSError, ValueError, json.JSONDecodeError) as exc: - parser.error(str(exc)) - print(f"PASS aggregate: {args.out_dir / 'aggregate.json'}") - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/benchmarks/cnc/calibrate_cc_difficulty.py b/benchmarks/cnc/calibrate_cc_difficulty.py deleted file mode 100644 index e37ef9e..0000000 --- a/benchmarks/cnc/calibrate_cc_difficulty.py +++ /dev/null @@ -1,247 +0,0 @@ -#!/usr/bin/env python3 -"""Calibrate the classical online CC difficulty cutoff by emitted task count.""" - -from __future__ import annotations - -import argparse -import hashlib -import json -import math -import re -import resource -import subprocess -import time -from pathlib import Path -from typing import Any - - -_STATS = re.compile(r"status=OK cubes=(\d+).*cutoff=CcDifficulty\((\d+)\)") - - -class CalibrationError(ValueError): - pass - - -def sha256_file(path: Path) -> str: - digest = hashlib.sha256() - with path.open("rb") as stream: - while chunk := stream.read(1024 * 1024): - digest.update(chunk) - return digest.hexdigest() - - -def task_lines(path: Path) -> int: - return sum( - line.startswith("a ") or line == "a 0\n" - for line in path.read_text(encoding="utf-8").splitlines(keepends=True) - ) - - -def run_cuber( - cuber: Path, - instance: Path, - threshold: int, - cubes: Path, - log: Path, - max_rows: int, - trace: Path | None = None, - selector: str = "region", - timeout_s: float | None = None, -) -> dict[str, int | float]: - if selector not in {"region", "structure-blind"}: - raise CalibrationError(f"unknown selector {selector!r}") - command = [ - str(cuber.resolve()), - str(instance.resolve()), - "--cc-threshold", - str(threshold), - "-o", - str(cubes), - "--max-rows", - str(max_rows), - "--selector", - selector, - ] - if trace: - command.extend(("--trace", str(trace))) - before = resource.getrusage(resource.RUSAGE_CHILDREN) - started = time.monotonic() - try: - process = subprocess.run( - command, - capture_output=True, - text=True, - check=False, - timeout=timeout_s, - ) - except subprocess.TimeoutExpired as exc: - elapsed = time.monotonic() - started - stderr = exc.stderr.decode() if isinstance(exc.stderr, bytes) else exc.stderr or "" - log.write_text(stderr, encoding="utf-8") - raise CalibrationError( - f"cuber selector={selector} threshold={threshold} timed out after {elapsed:.3f}s" - ) from exc - elapsed = time.monotonic() - started - after = resource.getrusage(resource.RUSAGE_CHILDREN) - log.write_text(process.stderr, encoding="utf-8") - match = _STATS.search(process.stderr) - if process.returncode or not match or int(match.group(2)) != threshold: - raise CalibrationError( - f"cuber threshold={threshold} failed: {process.stderr[-500:]}" - ) - tasks = int(match.group(1)) - if task_lines(cubes) != tasks: - raise CalibrationError("reported and emitted task counts differ") - return { - "threshold": threshold, - "tasks": tasks, - "elapsed_s": elapsed, - "user_s": after.ru_utime - before.ru_utime, - "system_s": after.ru_stime - before.ru_stime, - } - - -def choose( - rows: list[dict[str, int | float]], target: int, minimum: int, maximum: int -) -> dict[str, int | float]: - inside = [row for row in rows if minimum <= int(row["tasks"]) <= maximum] - pool = inside or rows - if not pool: - raise CalibrationError("empty cutoff response") - return min( - pool, - key=lambda row: ( - abs(math.log2(max(1, int(row["tasks"])) / target)), - abs(int(row["tasks"]) - target), - int(row["threshold"]), - ), - ) - - -def calibrate( - instance: Path, - cuber: Path, - out_dir: Path, - target: int, - minimum: int, - maximum: int, - initial: int, - maximum_threshold: int, - max_rows: int, - selector: str = "region", - timeout_s: float | None = None, -) -> dict[str, Any]: - if target <= 0 or minimum <= 0 or maximum < minimum or initial <= 0: - raise CalibrationError("invalid task range or threshold") - out_dir.mkdir(parents=True, exist_ok=True) - candidates = out_dir / "candidates" - candidates.mkdir(exist_ok=True) - observed: dict[int, dict[str, int | float]] = {} - - def probe(threshold: int) -> dict[str, int | float]: - if threshold not in observed: - observed[threshold] = run_cuber( - cuber, - instance, - threshold, - candidates / f"threshold-{threshold}.icnf", - candidates / f"threshold-{threshold}.log", - max_rows, - selector=selector, - timeout_s=timeout_s, - ) - return observed[threshold] - - lower, upper = 0, initial - probe(lower) - while int(probe(upper)["tasks"]) < target and upper < maximum_threshold: - lower, upper = upper, min(upper * 2, maximum_threshold) - if int(probe(upper)["tasks"]) < target: - raise CalibrationError("maximum threshold did not reach target task count") - while upper - lower > 1: - middle = (lower + upper) // 2 - if int(probe(middle)["tasks"]) >= target: - upper = middle - else: - lower = middle - - response = sorted(observed.values(), key=lambda row: int(row["threshold"])) - selected = choose(response, target, minimum, maximum) - threshold = int(selected["threshold"]) - final = run_cuber( - cuber, - instance, - threshold, - out_dir / "frontier.icnf", - out_dir / "final.log", - max_rows, - out_dir / "nodes.jsonl", - selector, - timeout_s, - ) - candidate = candidates / f"threshold-{threshold}.icnf" - if sha256_file(candidate) != sha256_file(out_dir / "frontier.icnf"): - raise CalibrationError("traced rerun changed frontier bytes") - record = { - "schema_version": 1, - "method": "classical-cc-difficulty-task-count-calibration", - "selector": selector, - "formula": "D^2*(D+I)/N > threshold", - "instance": str(instance), - "instance_sha256": sha256_file(instance), - "cuber_sha256": sha256_file(cuber), - "target_tasks": target, - "accepted_task_range": [minimum, maximum], - "selected_threshold": threshold, - "tasks": int(final["tasks"]), - "cubing_elapsed_s": float(final["elapsed_s"]), - "cubing_cpu_s": float(final["user_s"]) + float(final["system_s"]), - "frontier_sha256": sha256_file(out_dir / "frontier.icnf"), - "trace_sha256": sha256_file(out_dir / "nodes.jsonl"), - "within_target_range": minimum <= int(final["tasks"]) <= maximum, - "response": response, - } - (out_dir / "selection.json").write_text( - json.dumps(record, indent=2, sort_keys=True) + "\n", encoding="utf-8" - ) - return record - - -def main() -> int: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("instance", type=Path) - parser.add_argument("--cuber", type=Path, required=True) - parser.add_argument("--out-dir", type=Path, required=True) - parser.add_argument("--target", type=int, default=512) - parser.add_argument("--min-tasks", type=int, default=384) - parser.add_argument("--max-tasks", type=int, default=640) - parser.add_argument("--initial-threshold", type=int, default=1024) - parser.add_argument("--max-threshold", type=int, default=1 << 60) - parser.add_argument("--max-rows", type=int, default=512) - parser.add_argument( - "--selector", choices=("region", "structure-blind"), default="region" - ) - parser.add_argument("--timeout-s", type=float) - args = parser.parse_args() - try: - result = calibrate( - args.instance, - args.cuber, - args.out_dir, - args.target, - args.min_tasks, - args.max_tasks, - args.initial_threshold, - args.max_threshold, - args.max_rows, - args.selector, - args.timeout_s, - ) - except (CalibrationError, OSError) as exc: - parser.error(str(exc)) - print(json.dumps(result, indent=2, sort_keys=True)) - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/benchmarks/cnc/calibrate_hard_regime.py b/benchmarks/cnc/calibrate_hard_regime.py deleted file mode 100755 index f45795a..0000000 --- a/benchmarks/cnc/calibrate_hard_regime.py +++ /dev/null @@ -1,455 +0,0 @@ -#!/usr/bin/env python3 -"""Freeze one CC threshold per product-width, selector, and frontier band.""" - -from __future__ import annotations - -import argparse -import hashlib -import json -import math -import statistics -from dataclasses import dataclass -from pathlib import Path -from typing import Any, Callable - -from benchmarks.cnc.calibrate_cc_difficulty import ( - CalibrationError, - run_cuber, - sha256_file, -) -from benchmarks.cnc.hard_regime import ( - HardRegimeError, - contract_sha256, - load_contract, -) -from benchmarks.pipeline.circuit import atomic_write_json, read_jsonl, write_json - - -PROBE_CHECKPOINT_SCHEMA_VERSION = 1 - - -@dataclass(frozen=True) -class ProbeContext: - probe_runner: Callable[..., dict[str, int | float]] - cuber: Path - cuber_sha256: str - contract_digest: str - product_width: int - selector: str - max_rows: int - timeout_s: float - - -def _validated_probe_result( - value: dict[str, int | float], threshold: int -) -> dict[str, int | float]: - if value.get("threshold") != threshold: - raise CalibrationError("calibration probe threshold mismatch") - tasks = value.get("tasks") - if not isinstance(tasks, int) or isinstance(tasks, bool) or tasks < 0: - raise CalibrationError("calibration probe has an invalid task count") - result: dict[str, int | float] = {"threshold": threshold, "tasks": tasks} - for field in ("elapsed_s", "user_s", "system_s"): - metric = value.get(field) - if ( - not isinstance(metric, (int, float)) - or isinstance(metric, bool) - or not math.isfinite(float(metric)) - or metric < 0 - ): - raise CalibrationError(f"calibration probe has invalid {field}") - result[field] = float(metric) - return result - - -def _frontier_summary(path: Path) -> tuple[str, int]: - digest = hashlib.sha256() - tasks = 0 - with path.open("rb") as stream: - for line in stream: - digest.update(line) - if line.startswith(b"a "): - tasks += 1 - return digest.hexdigest(), tasks - - -def _probe_artifacts( - frontier: Path, log: Path, trace: Path | None -) -> tuple[dict[str, str | None], int]: - try: - frontier_digest, tasks = _frontier_summary(frontier) - artifacts: dict[str, str | None] = { - "frontier_sha256": frontier_digest, - "log_sha256": sha256_file(log), - "trace_sha256": None, - } - if trace is not None: - artifacts["trace_sha256"] = sha256_file(trace) - return artifacts, tasks - except OSError as exc: - raise CalibrationError(f"calibration probe artifact is missing: {exc}") from exc - - -def run_or_resume_probe( - *, - context: ProbeContext, - instance: Path, - instance_id: str, - instance_sha256: str, - threshold: int, - phase: str, - frontier: Path, - log: Path, - checkpoint: Path, - trace: Path | None = None, -) -> dict[str, int | float | str | None]: - identity = { - "contract_sha256": context.contract_digest, - "product_width": context.product_width, - "selector": context.selector, - "max_rows": context.max_rows, - "threshold": threshold, - "phase": phase, - "cuber_sha256": context.cuber_sha256, - "instance_id": instance_id, - "instance_sha256": instance_sha256, - } - try: - checkpoint_text = checkpoint.read_text(encoding="utf-8") - except FileNotFoundError: - checkpoint_text = None - except OSError as exc: - raise CalibrationError(f"invalid calibration checkpoint {checkpoint}: {exc}") from exc - if checkpoint_text is not None: - try: - record = json.loads(checkpoint_text) - except json.JSONDecodeError as exc: - raise CalibrationError(f"invalid calibration checkpoint {checkpoint}: {exc}") from exc - if not isinstance(record, dict): - raise CalibrationError(f"invalid calibration checkpoint {checkpoint}") - if ( - record.get("schema_version") != PROBE_CHECKPOINT_SCHEMA_VERSION - or record.get("kind") != "hard-regime-calibration-probe" - or record.get("identity") != identity - ): - raise CalibrationError(f"calibration checkpoint provenance mismatch: {checkpoint}") - artifacts, tasks = _probe_artifacts(frontier, log, trace) - if record.get("artifacts") != artifacts: - raise CalibrationError(f"calibration checkpoint artifact hash mismatch: {checkpoint}") - result_value = record.get("result") - if not isinstance(result_value, dict): - raise CalibrationError(f"calibration checkpoint result is malformed: {checkpoint}") - result = _validated_probe_result(result_value, threshold) - if tasks != result["tasks"]: - raise CalibrationError(f"calibration checkpoint task count mismatch: {checkpoint}") - return {**result, **artifacts} - - result = _validated_probe_result( - context.probe_runner( - context.cuber, - instance, - threshold, - frontier, - log, - context.max_rows, - trace=trace, - selector=context.selector, - timeout_s=context.timeout_s, - ), - threshold, - ) - artifacts, tasks = _probe_artifacts(frontier, log, trace) - if tasks != result["tasks"]: - raise CalibrationError("reported and emitted calibration task counts differ") - atomic_write_json( - checkpoint, - { - "schema_version": PROBE_CHECKPOINT_SCHEMA_VERSION, - "kind": "hard-regime-calibration-probe", - "identity": identity, - "artifacts": artifacts, - "result": result, - }, - ) - return {**result, **artifacts} - - -def task_counts(row: dict[str, Any]) -> list[int]: - instances = row.get("instances") - if not isinstance(instances, list) or not instances: - raise CalibrationError("width-level response row has no instances") - counts = [] - for instance in instances: - if not isinstance(instance, dict) or not isinstance(instance.get("tasks"), int): - raise CalibrationError("width-level response has a malformed task count") - counts.append(instance["tasks"]) - return counts - - -def median_tasks(row: dict[str, Any]) -> float: - return float(statistics.median(task_counts(row))) - - -def calibration_loss(row: dict[str, Any], target: int) -> tuple[float, float]: - errors = [abs(math.log2(max(1, count) / target)) for count in task_counts(row)] - return float(statistics.median(errors)), max(errors) - - -def choose_width_response( - rows: list[dict[str, Any]], target: int, minimum: int, maximum: int -) -> dict[str, Any]: - if not rows: - raise CalibrationError("empty width-level cutoff response") - inside = [row for row in rows if minimum <= median_tasks(row) <= maximum] - if not inside: - raise CalibrationError( - f"no calibrated threshold reaches accepted task range [{minimum}, {maximum}]" - ) - return min( - inside, - key=lambda row: ( - *calibration_loss(row, target), - abs(median_tasks(row) - target), - int(row["threshold"]), - ), - ) - - -def calibration_inputs( - contract: dict[str, Any], manifest_path: Path, product_width: int -) -> list[dict[str, Any]]: - root = manifest_path.parent - digest = contract_sha256(contract) - selected = [] - for record in read_jsonl(manifest_path): - if record.get("product_width") != product_width or record.get("split") != "calibration": - continue - if record.get("factor_input_width") * 2 != product_width: - raise CalibrationError(f"{record.get('id')}: factor/product width confusion") - if record.get("architecture") != "array-ripple": - raise CalibrationError(f"{record.get('id')}: calibration must use array-ripple") - if record.get("contract_sha256") != digest: - raise CalibrationError(f"{record.get('id')}: contract hash mismatch") - relative = record.get("circuitsat") - if not isinstance(relative, str): - raise CalibrationError(f"{record.get('id')}: missing CircuitSAT path") - instance = (root / relative).resolve() - if sha256_file(instance) != record.get("circuitsat_sha256"): - raise CalibrationError(f"{record.get('id')}: CircuitSAT hash mismatch") - selected.append( - { - "id": record["id"], - "path": instance, - "sha256": record["circuitsat_sha256"], - } - ) - selected.sort(key=lambda row: row["id"]) - if len(selected) != 3: - raise CalibrationError( - f"product width {product_width} needs exactly 3 calibration instances" - ) - return selected - - -def calibrate_width( - *, - contract: dict[str, Any], - instances: list[dict[str, Any]], - product_width: int, - selector: str, - cuber: Path, - out_dir: Path, - initial_threshold: int = 1, - maximum_threshold: int = 1 << 120, - probe_runner: Callable[..., dict[str, int | float]] = run_cuber, -) -> dict[str, Any]: - if selector not in {"region", "structure-blind"}: - raise CalibrationError(f"unsupported selector {selector!r}") - if len(instances) != 3: - raise CalibrationError("width-level calibration requires exactly three instances") - if initial_threshold != 1 or maximum_threshold <= initial_threshold: - raise CalibrationError( - "calibration threshold search must start at 1 and allow a larger maximum" - ) - method = "region-cc" if selector == "region" else "structure-blind-cc" - max_rows = int(contract["methods"][method]["max_rows"]) - timeout_s = float(contract["limits_seconds"]["cubing"]) - contract_digest = contract_sha256(contract) - cuber_digest = sha256_file(cuber) - probe_context = ProbeContext( - probe_runner=probe_runner, - cuber=cuber, - cuber_sha256=cuber_digest, - contract_digest=contract_digest, - product_width=product_width, - selector=selector, - max_rows=max_rows, - timeout_s=timeout_s, - ) - out_dir.mkdir(parents=True, exist_ok=True) - observed: dict[int, dict[str, Any]] = {} - - def probe(threshold: int) -> dict[str, Any]: - if threshold in observed: - return observed[threshold] - threshold_dir = out_dir / "candidates" / f"threshold-{threshold}" - threshold_dir.mkdir(parents=True, exist_ok=True) - rows = [] - for instance in instances: - result = run_or_resume_probe( - context=probe_context, - instance=instance["path"], - instance_id=instance["id"], - instance_sha256=instance["sha256"], - threshold=threshold, - phase="candidate", - frontier=threshold_dir / f"{instance['id']}.icnf", - log=threshold_dir / f"{instance['id']}.log", - checkpoint=threshold_dir / f"{instance['id']}.probe.json", - ) - rows.append( - { - "id": instance["id"], - "tasks": int(result["tasks"]), - "elapsed_s": float(result["elapsed_s"]), - "cpu_s": float(result["user_s"]) + float(result["system_s"]), - } - ) - observed[threshold] = {"threshold": threshold, "instances": rows} - return observed[threshold] - - bands = contract["frontier_bands"] - brackets: dict[str, tuple[int, int]] = {} - for band, spec in bands.items(): - target = int(spec["center_cubes"]) - lower, upper = initial_threshold, initial_threshold * 2 - probe(lower) - while median_tasks(probe(upper)) < target and upper < maximum_threshold: - lower, upper = upper, min(upper * 2, maximum_threshold) - if median_tasks(probe(upper)) < target: - raise CalibrationError( - f"maximum threshold did not reach {band} target {target}" - ) - while upper - lower > 1: - middle = (lower + upper) // 2 - if median_tasks(probe(middle)) >= target: - upper = middle - else: - lower = middle - brackets[band] = (lower, upper) - - response = sorted(observed.values(), key=lambda row: int(row["threshold"])) - selections = {} - for band, spec in bands.items(): - target = int(spec["center_cubes"]) - minimum = math.ceil(target * float(spec["accepted_ratio"][0])) - maximum = math.floor(target * float(spec["accepted_ratio"][1])) - selected = choose_width_response(response, target, minimum, maximum) - threshold = int(selected["threshold"]) - final_rows = [] - for instance in instances: - final_dir = out_dir / "selected" / band / instance["id"] - final_dir.mkdir(parents=True, exist_ok=True) - final = run_or_resume_probe( - context=probe_context, - instance=instance["path"], - instance_id=instance["id"], - instance_sha256=instance["sha256"], - threshold=threshold, - phase=f"selected-{band}", - frontier=final_dir / "frontier.icnf", - log=final_dir / "cuber.log", - checkpoint=final_dir / "probe.json", - trace=final_dir / "nodes.jsonl", - ) - candidate = ( - out_dir - / "candidates" - / f"threshold-{threshold}" - / f"{instance['id']}.icnf" - ) - if sha256_file(candidate) != final["frontier_sha256"]: - raise CalibrationError( - f"{instance['id']}: traced rerun changed selected frontier" - ) - final_rows.append( - { - "id": instance["id"], - "tasks": int(final["tasks"]), - "frontier_sha256": final["frontier_sha256"], - "trace_sha256": final["trace_sha256"], - "cubing_elapsed_s": float(final["elapsed_s"]), - "cubing_cpu_s": float(final["user_s"]) - + float(final["system_s"]), - } - ) - selections[band] = { - "target_tasks": target, - "accepted_task_range": [minimum, maximum], - "selected_threshold": threshold, - "search_bracket": list(brackets[band]), - "median_tasks": median_tasks(selected), - "calibration_loss": calibration_loss(selected, target)[0], - "within_target_range": minimum <= median_tasks(selected) <= maximum, - "instances": final_rows, - } - - record = { - "schema_version": 1, - "kind": "width-level-cc-calibration-lock", - "contract_sha256": contract_digest, - "product_width": product_width, - "selector": selector, - "method": method, - "max_rows": max_rows, - "cuber_sha256": cuber_digest, - "search": { - "initial_threshold": initial_threshold, - "maximum_threshold": maximum_threshold, - "probe_checkpoint_schema_version": PROBE_CHECKPOINT_SCHEMA_VERSION, - }, - "calibration_instances": [ - {"id": instance["id"], "sha256": instance["sha256"]} - for instance in instances - ], - "bands": selections, - "response": response, - } - write_json(out_dir / "calibration-lock.json", record) - return record - - -def main() -> int: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("contract", type=Path) - parser.add_argument("manifest", type=Path) - parser.add_argument("--product-width", type=int, choices=(64, 72, 80), required=True) - parser.add_argument( - "--selector", choices=("region", "structure-blind"), required=True - ) - parser.add_argument("--cuber", type=Path, required=True) - parser.add_argument("--out-dir", type=Path, required=True) - parser.add_argument("--initial-threshold", type=int, default=1) - parser.add_argument("--maximum-threshold", type=int, default=1 << 120) - args = parser.parse_args() - try: - contract = load_contract(args.contract) - instances = calibration_inputs(contract, args.manifest, args.product_width) - result = calibrate_width( - contract=contract, - instances=instances, - product_width=args.product_width, - selector=args.selector, - cuber=args.cuber, - out_dir=args.out_dir, - initial_threshold=args.initial_threshold, - maximum_threshold=args.maximum_threshold, - ) - except (CalibrationError, HardRegimeError, OSError, ValueError) as exc: - parser.error(str(exc)) - print(json.dumps(result, indent=2, sort_keys=True)) - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/benchmarks/cnc/conquer_parallel.py b/benchmarks/cnc/conquer_parallel.py index e119c1b..1c55add 100644 --- a/benchmarks/cnc/conquer_parallel.py +++ b/benchmarks/cnc/conquer_parallel.py @@ -3,7 +3,6 @@ from __future__ import annotations -import argparse import concurrent.futures import hashlib import json @@ -21,7 +20,8 @@ _CNF_HEADER = re.compile(rb"^p cnf (\d+) (\d+)\s*$", re.MULTILINE) _STAT = re.compile(r"^c\s+(decisions|conflicts):\s+(\d+)", re.MULTILINE) -_BASE_HEADER: bytes +_BASE_VARIABLES: int +_BASE_CLAUSES: int _BASE_BODY: bytes _KISSAT: str _TIMEOUT_S: float @@ -37,7 +37,7 @@ def parse_cnf(data: bytes) -> tuple[int, int, bytes]: return variables, clauses, body -def read_cubes(path: Path) -> Iterator[list[int]]: +def read_cubes(path: Path, variables: int | None = None) -> Iterator[list[int]]: with path.open(encoding="utf-8") as stream: for lineno, line in enumerate(stream, 1): fields = line.split() @@ -48,6 +48,12 @@ def read_cubes(path: Path) -> Iterator[list[int]]: literals = [int(field) for field in fields[1:-1]] if any(literal == 0 for literal in literals): raise ValueError(f"{path}:{lineno}: embedded zero literal") + if variables is not None and any(abs(literal) > variables for literal in literals): + raise ValueError(f"{path}:{lineno}: literal exceeds CNF variable range") + if len(set(literals)) != len(literals): + raise ValueError(f"{path}:{lineno}: duplicate literal") + if any(-literal in literals for literal in literals): + raise ValueError(f"{path}:{lineno}: contradictory literals") yield literals @@ -63,14 +69,6 @@ def child_cpu_seconds(before: resource.struct_rusage) -> tuple[float, float]: return after.ru_utime - before.ru_utime, after.ru_stime - before.ru_stime -def percentile(values: list[float], quantile: float) -> float: - if not values: - return math.nan - ordered = sorted(values) - index = min(len(ordered) - 1, max(0, round(quantile * (len(ordered) - 1)))) - return ordered[index] - - def distribution(values: list[float]) -> dict[str, float | None]: if not values: return { @@ -125,8 +123,9 @@ def _configure_worker( timeout_s: float, tmpdir: str | None, ) -> None: - global _BASE_HEADER, _BASE_BODY, _KISSAT, _TIMEOUT_S, _TMPDIR - _BASE_HEADER = f"p cnf {variables} {clauses}".encode() + global _BASE_VARIABLES, _BASE_CLAUSES, _BASE_BODY, _KISSAT, _TIMEOUT_S, _TMPDIR + _BASE_VARIABLES = variables + _BASE_CLAUSES = clauses _BASE_BODY = body _KISSAT = kissat _TIMEOUT_S = timeout_s @@ -135,11 +134,6 @@ def _configure_worker( def _solve_cube(task: tuple[str, int, list[int], int]) -> dict[str, Any]: arm, index, cube, released_ns = task - header_fields = _BASE_HEADER.split() - clause_count = int(header_fields[3]) + len(cube) - header = b" ".join((*header_fields[:3], str(clause_count).encode())) + b"\n" - units = b"".join(f"{literal} 0\n".encode() for literal in cube) - payload = header + _BASE_BODY + units started_ns = time.monotonic_ns() common = { "schema_version": 1, @@ -158,14 +152,19 @@ def _solve_cube(task: tuple[str, int, list[int], int]) -> dict[str, Any]: ) try: with temporary: - temporary.write(payload) + temporary.write( + f"p cnf {_BASE_VARIABLES} {_BASE_CLAUSES + len(cube)}\n".encode() + ) + temporary.write(_BASE_BODY) + for literal in cube: + temporary.write(f"{literal} 0\n".encode()) usage_before = resource.getrusage(resource.RUSAGE_CHILDREN) try: process = subprocess.run( [_KISSAT, "--statistics", "--relaxed", temporary.name], capture_output=True, text=True, - timeout=_TIMEOUT_S, + timeout=None if _TIMEOUT_S == 0 else _TIMEOUT_S, check=False, ) elapsed_s = (time.monotonic_ns() - started_ns) / 1e9 @@ -222,6 +221,7 @@ def summarize( replay_workers: list[int], wall_s: float, measured_makespan_s: float, + not_started: int = 0, ) -> dict[str, Any]: time_stats = distribution(durations) cpu_stats = distribution(cpu_durations) @@ -238,7 +238,7 @@ def summarize( if unsat == cubes else "incomplete" ) - complete = result in {"sat", "unsat"} and completed == cubes + complete = result == "sat" or (result == "unsat" and completed == cubes) lpt_wall = {str(count): lpt_makespan(durations, count) for count in replay_workers} lpt_cpu = { str(count): lpt_makespan(cpu_durations, count) for count in replay_workers @@ -249,6 +249,7 @@ def summarize( "completed": completed, "timeouts": timeouts, "errors": errors, + "not_started": not_started, "sat": sat, "unsat": unsat, "result": result, @@ -263,7 +264,7 @@ def summarize( "cpu_lpt_makespan_s": lpt_makespan(cpu_durations, workers), "lpt_makespan_by_workers_s": lpt_wall, "cpu_lpt_makespan_by_workers_s": lpt_cpu, - "lpt_is_lower_bound": bool(timeouts or errors), + "lpt_is_lower_bound": bool(timeouts or errors or not_started), "p50_s": time_stats["p50"], "p95_s": time_stats["p95"], "p99_s": time_stats["p99"], @@ -318,7 +319,7 @@ def submit_one() -> bool: ) return True - for _ in range(workers * 4): + for _ in range(workers): if not submit_one(): break @@ -358,10 +359,11 @@ def submit_one() -> bool: else: counts["completed"] += 1 counts[row["result"]] += 1 - submit_one() + if not counts["sat"]: + submit_one() if done_count % progress_every == 0 or done_count == total_cubes: print(f"{arm}: {done_count}/{total_cubes}", flush=True) - if done_count != total_cubes: + if done_count != total_cubes and not counts["sat"]: raise RuntimeError(f"{arm}: expected {total_cubes} cubes, completed {done_count}") return summarize( cubes=total_cubes, @@ -377,71 +379,6 @@ def submit_one() -> bool: if earliest_release_ns is None or latest_collection_ns is None else (latest_collection_ns - earliest_release_ns) / 1e9 ), + not_started=total_cubes - done_count, **counts, ) - - -def main() -> None: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("cnf", type=Path) - parser.add_argument("--arm", action="append", required=True, metavar="NAME=CUBES") - parser.add_argument("--kissat", type=Path, required=True) - parser.add_argument("--workers", type=int, required=True) - parser.add_argument("--lpt-workers", type=int, action="append", default=[]) - parser.add_argument("--timeout-s", type=float, default=600.0) - parser.add_argument("--out-dir", type=Path, required=True) - parser.add_argument("--tmp-dir", type=Path) - args = parser.parse_args() - if args.workers < 1 or args.timeout_s <= 0 or any( - count < 1 for count in args.lpt_workers - ): - parser.error("workers, lpt-workers, and timeout-s must be positive") - replay_workers = sorted(set(args.lpt_workers or [args.workers])) - - variables, clauses, body = parse_cnf(args.cnf.read_bytes()) - args.out_dir.mkdir(parents=True, exist_ok=True) - if args.tmp_dir: - args.tmp_dir.mkdir(parents=True, exist_ok=True) - worker_args = ( - variables, - clauses, - body, - str(args.kissat.resolve()), - args.timeout_s, - str(args.tmp_dir.resolve()) if args.tmp_dir else None, - ) - summaries: dict[str, Any] = {} - for spec in args.arm: - try: - arm, cube_path = spec.split("=", 1) - except ValueError as error: - raise SystemExit(f"invalid --arm {spec!r}; expected NAME=PATH") from error - cube_file = Path(cube_path) - total_cubes = sum(1 for _ in read_cubes(cube_file)) - summaries[arm] = run_arm( - arm, - read_cubes(cube_file), - total_cubes, - args.workers, - replay_workers, - args.out_dir / f"{arm}.jsonl", - worker_args, - ) - bundle = { - "schema_version": 1, - "workers": args.workers, - "lpt_workers": replay_workers, - "timeout_s": args.timeout_s, - "cnf": str(args.cnf), - "kissat": str(args.kissat), - "arms": summaries, - } - (args.out_dir / "summary.json").write_text( - json.dumps(bundle, indent=2, sort_keys=True, allow_nan=False) + "\n", - encoding="utf-8", - ) - print(json.dumps(bundle, indent=2, sort_keys=True, allow_nan=False)) - - -if __name__ == "__main__": - main() diff --git a/benchmarks/cnc/contracts/hard-regime-v1.yaml b/benchmarks/cnc/contracts/hard-regime-v1.yaml deleted file mode 100644 index 07603a0..0000000 --- a/benchmarks/cnc/contracts/hard-regime-v1.yaml +++ /dev/null @@ -1,113 +0,0 @@ -schema_version: 1 -study_id: array-ripple-hard-unsat-cnc-v1 -issue: 51 -architecture: array-ripple -semantic_task: unsigned-factoring -expected_outcome: unsat - -widths: - - product_width: 64 - factor_input_width: 32 - calibration: - count: 3 - seed: 202607196401 - held_out: - count: 10 - seed: 202607196402 - - product_width: 72 - factor_input_width: 36 - calibration: - count: 3 - seed: 202607197201 - held_out: - count: 10 - seed: 202607197202 - - product_width: 80 - factor_input_width: 40 - calibration: - count: 3 - seed: 202607198001 - held_out: - count: 10 - seed: 202607198002 - -frontier_bands: - low: - center_cubes: 16384 - accepted_ratio: [0.75, 1.25] - medium: - center_cubes: 65536 - accepted_ratio: [0.75, 1.25] - high: - center_cubes: 262144 - accepted_ratio: [0.75, 1.25] - -methods: - monolithic-kissat: - budgets: ["none"] - input: global-tseitin-cnf - march-cu-dynamic: - budgets: [dynamic-default] - input: global-tseitin-cnf - cutoff_policy: upstream-default-dynamic - region-cc: - budgets: [low, medium, high] - input: structured-circuitsat - selector: region - max_rows: 512 - cutoff_policy: calibration-frozen-cc-difficulty - structure-blind-cc: - budgets: [low, medium, high] - input: structured-circuitsat - selector: structure-blind - max_rows: 512 - cutoff_policy: calibration-frozen-cc-difficulty - -limits_seconds: - monolithic: 600 - cubing: 7200 - per_cube_conquer: 1800 - -scheduling: - measured_workers: 32 - lpt_replay_workers: [32, 128, 512] - completion_policy: complete-all-cubes - -calibration: - unit: product-width-method-budget - selection_loss: median-absolute-log2-frontier-error - held_out_recalibration: forbidden - center_revision_policy: calibration-only-before-held-out - -statistics: - unit: held-out-instance - paired_geometric_mean: true - bootstrap: - samples: 10000 - confidence: 0.95 - seed: 2026071951 - budget_adjustment: - x: log2-frontier-size - y: log-metric - interpolation: piecewise-linear - extrapolation: forbidden - common_grid: overlap-low-geometric-mid-overlap-high - censored_policy: report-separately-no-finite-imputation - -artifact_policy: - raw_directory: artifacts/cnc-hard-regime - git_payload: - - contract - - sha256-manifest - - aggregate-statistics - - primary-table - - report - tool_identity: version-source-revision-command-and-executable-sha256 - -tool_sources: - kissat: - repository: https://github.com/arminbiere/kissat.git - revision: 8af8e56f174b778aef3aa45af9f739b2a5f492c2 - march_cu: - repository: https://github.com/marijnheule/CnC.git - revision: 705b60c6491ef2b61988b3ce6ac674be1b90571d diff --git a/benchmarks/cnc/cubing.py b/benchmarks/cnc/cubing.py new file mode 100644 index 0000000..83d156b --- /dev/null +++ b/benchmarks/cnc/cubing.py @@ -0,0 +1,173 @@ +#!/usr/bin/env python3 +"""Run March or analyze a frozen cube frontier with parallel Kissat.""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path +from typing import Any + +from benchmarks.cnc.conquer_parallel import read_cubes +from benchmarks.cnc.solve import conquer_frontier, run_process +from benchmarks.pipeline.circuit import sha256_file, write_json + + +def run_cuber( + command: list[str], + frontier: Path, + *, + timeout_s: float, + out_dir: Path, +) -> dict[str, Any]: + frontier.parent.mkdir(parents=True, exist_ok=True) + process = run_process( + command, + timeout_s=timeout_s, + out_dir=out_dir, + label="cubing", + ) + if process["timed_out"]: + raise TimeoutError("cuber timed out") + if process["returncode"] != 0: + raise RuntimeError(f"cuber exited with status {process['returncode']}") + if not frontier.is_file(): + raise RuntimeError("cuber reported success without producing a frontier") + cubes = sum(1 for _ in read_cubes(frontier)) + record = { + "command": command, + "returncode": process["returncode"], + "wall_s": process["wall_s"], + "cubes": cubes, + "frontier": str(frontier.resolve()), + "frontier_sha256": sha256_file(frontier), + "frontier_bytes": frontier.stat().st_size, + } + write_json(out_dir / "cubing.json", record) + return record + + +def march_then_conquer( + cnf: Path, + march_cu: Path, + kissat: Path, + *, + workers: int, + cube_timeout_s: float, + cubing_timeout_s: float, + out_dir: Path, + remaining_vars: int | None = None, + tmp_dir: Path | None = None, +) -> dict[str, Any]: + frontier = out_dir / "frontier.icnf" + command = [str(march_cu), str(cnf)] + if remaining_vars is not None: + command.extend(["-n", str(remaining_vars)]) + command.extend(["-o", str(frontier)]) + cubing = run_cuber( + command, + frontier, + timeout_s=cubing_timeout_s, + out_dir=out_dir, + ) + conquer = conquer_frontier( + cnf, + frontier, + kissat, + workers=workers, + timeout_s=cube_timeout_s, + out_dir=out_dir / "conquer", + tmp_dir=tmp_dir, + total_cubes=cubing["cubes"], + ) + record = {"schema_version": 1, "mode": "march-cu", "cubing": cubing, "conquer": conquer} + write_json(out_dir / "summary.json", record) + return record + + +def frozen_then_conquer( + cnf: Path, + frontier: Path, + kissat: Path, + *, + workers: int, + cube_timeout_s: float, + out_dir: Path, + tmp_dir: Path | None = None, +) -> dict[str, Any]: + cubes = sum(1 for _ in read_cubes(frontier)) + conquer = conquer_frontier( + cnf, + frontier, + kissat, + workers=workers, + timeout_s=cube_timeout_s, + out_dir=out_dir / "conquer", + tmp_dir=tmp_dir, + total_cubes=cubes, + ) + record = { + "schema_version": 1, + "mode": "frozen-frontier", + "cnf": str(cnf.resolve()), + "cnf_sha256": sha256_file(cnf), + "frontier": str(frontier.resolve()), + "frontier_sha256": sha256_file(frontier), + "cubes": cubes, + "conquer": conquer, + } + write_json(out_dir / "summary.json", record) + return record + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + subparsers = parser.add_subparsers(dest="mode", required=True) + march = subparsers.add_parser("march") + march.add_argument("cnf", type=Path) + march.add_argument("--march-cu", type=Path, required=True) + march.add_argument("--remaining-vars", type=int) + frozen = subparsers.add_parser("frontier") + frozen.add_argument("cnf", type=Path) + frozen.add_argument("frontier", type=Path) + for command in (march, frozen): + command.add_argument("--kissat", type=Path, required=True) + command.add_argument("--workers", type=int, required=True) + command.add_argument("--cube-timeout-s", type=float, default=600.0) + command.add_argument("--out-dir", type=Path, required=True) + command.add_argument("--tmp-dir", type=Path) + march.add_argument("--cubing-timeout-s", type=float, default=3600.0) + args = parser.parse_args() + if ( + args.workers < 1 + or args.cube_timeout_s < 0 + or getattr(args, "cubing_timeout_s", 0) < 0 + ): + parser.error("workers must be positive and timeouts non-negative") + if args.mode == "march": + record = march_then_conquer( + args.cnf, + args.march_cu, + args.kissat, + workers=args.workers, + cube_timeout_s=args.cube_timeout_s, + cubing_timeout_s=args.cubing_timeout_s, + out_dir=args.out_dir, + remaining_vars=args.remaining_vars, + tmp_dir=args.tmp_dir, + ) + else: + record = frozen_then_conquer( + args.cnf, + args.frontier, + args.kissat, + workers=args.workers, + cube_timeout_s=args.cube_timeout_s, + out_dir=args.out_dir, + tmp_dir=args.tmp_dir, + ) + print(json.dumps(record, indent=2, sort_keys=True, allow_nan=False)) + + +if __name__ == "__main__": + main() diff --git a/benchmarks/cnc/factoring.py b/benchmarks/cnc/factoring.py new file mode 100644 index 0000000..d4386c7 --- /dev/null +++ b/benchmarks/cnc/factoring.py @@ -0,0 +1,287 @@ +#!/usr/bin/env python3 +"""Generate n×n SAT/UNSAT factoring instances as CircuitSAT and DIMACS CNF.""" + +from __future__ import annotations + +import argparse +from dataclasses import asdict, dataclass +from pathlib import Path +from typing import Literal + +from benchmarks.pipeline.circuit import ( + pin_port_values, + sha256_file, + write_json, + write_jsonl, +) +from benchmarks.pipeline.cnf import encode_validated_circuit +from benchmarks.pipeline.multipliers import ( + DETERMINISTIC_MILLER_RABIN_LIMIT, + generate_multiplier, + is_prime, + records as factoring_records, +) + +Outcome = Literal["sat", "unsat"] +SAT_SEED_BASE = 20260709 +UNSAT_MATCH_CANDIDATES = 64 + + +@dataclass(frozen=True, slots=True) +class FactoringTarget: + instance_id: str + factor_bits: int + expected_outcome: Outcome + target: int + target_bits: int + seed: int + sequence_index: int + paired_sat_id: str | None + + +def sat_targets( + width: int, + count: int, + *, + seed_base: int = SAT_SEED_BASE, + instance_prefix: str = "factoring", +) -> tuple[list[FactoringTarget], list[dict]]: + if not 2 <= width <= 39 or count < 1: + raise ValueError("factor width must be between 2 and 39; count must be positive") + if width <= 12: + prime_count = sum( + is_prime(candidate) + for candidate in range(1 << (width - 1), 1 << width) + ) + distinct_products = prime_count * (prime_count + 1) // 2 + if count > distinct_products: + raise ValueError( + f"width {width} has only {distinct_products} distinct " + "balanced semiprime products" + ) + targets = [] + oracles = [] + for public, oracle in factoring_records([width], count, seed_base): + target = public["target"] + index = public["sequence_index"] + instance_id = f"{instance_prefix}-n{width}-sat-{index:02d}" + targets.append( + FactoringTarget( + instance_id=instance_id, + factor_bits=width, + expected_outcome="sat", + target=target, + target_bits=target.bit_length(), + seed=public["seed"], + sequence_index=index, + paired_sat_id=None, + ) + ) + oracles.append( + { + "instance_id": instance_id, + "expected_outcome": "sat", + "left_factor": oracle["left_factor"], + "right_factor": oracle["right_factor"], + } + ) + return targets, oracles + + +def unsat_targets( + width: int, + sat: list[FactoringTarget], + *, + instance_prefix: str = "factoring", +) -> tuple[list[FactoringTarget], list[dict]]: + factor_max = (1 << width) - 1 + reachable_max = factor_max * factor_max + targets = [] + oracles = [] + seen: set[int] = set() + for index, paired in enumerate(sat): + candidates = [] + delta = 2 + lower = 1 << (paired.target_bits - 1) + upper = min(1 << paired.target_bits, reachable_max + 1) + max_delta = max(paired.target - lower, upper - 1 - paired.target) + while len(candidates) < UNSAT_MATCH_CANDIDATES and delta <= max_delta: + for target in (paired.target - delta, paired.target + delta): + if ( + lower <= target < upper + and target > factor_max + and target not in seen + and target < DETERMINISTIC_MILLER_RABIN_LIMIT + and is_prime(target) + ): + candidates.append(target) + if len(candidates) == UNSAT_MATCH_CANDIDATES: + break + delta += 2 + if not candidates: + raise ValueError( + f"{paired.instance_id}: no distinct prime UNSAT target in range" + ) + target = min( + candidates, + key=lambda value: ( + (value ^ paired.target).bit_count(), + abs(value - paired.target), + value, + ), + ) + seen.add(target) + instance_id = f"{instance_prefix}-n{width}-unsat-{index:02d}" + targets.append( + FactoringTarget( + instance_id=instance_id, + factor_bits=width, + expected_outcome="unsat", + target=target, + target_bits=target.bit_length(), + seed=paired.seed, + sequence_index=index, + paired_sat_id=paired.instance_id, + ) + ) + oracles.append( + { + "instance_id": instance_id, + "expected_outcome": "unsat", + "target_is_prime": True, + "target_exceeds_max_factor": True, + "target_within_multiplier_range": True, + "paired_sat_id": paired.instance_id, + } + ) + return targets, oracles + + +def validate_targets(targets: list[FactoringTarget], oracles: list[dict]) -> None: + by_id = {target.instance_id: target for target in targets} + oracle_ids = [oracle.get("instance_id") for oracle in oracles] + if len(by_id) != len(targets) or len(oracle_ids) != len(set(oracle_ids)): + raise ValueError("duplicate factoring target or oracle id") + if set(oracle_ids) != set(by_id): + raise ValueError("factoring targets and oracles are not one-to-one") + for oracle in oracles: + target = by_id[oracle["instance_id"]] + if target.target_bits != target.target.bit_length(): + raise ValueError(f"{target.instance_id}: stale target bit length") + if target.expected_outcome == "sat": + left = int(oracle["left_factor"]) + right = int(oracle["right_factor"]) + if ( + target.paired_sat_id is not None + or left.bit_length() != target.factor_bits + or right.bit_length() != target.factor_bits + or left * right != target.target + ): + raise ValueError(f"{target.instance_id}: invalid SAT oracle") + elif target.expected_outcome == "unsat": + factor_max = (1 << target.factor_bits) - 1 + if ( + target.paired_sat_id not in by_id + or target.target <= factor_max + or target.target > factor_max * factor_max + or target.target >= DETERMINISTIC_MILLER_RABIN_LIMIT + or not is_prime(target.target) + ): + raise ValueError(f"{target.instance_id}: invalid UNSAT oracle") + else: + raise ValueError(f"{target.instance_id}: invalid expected outcome") + + +def materialize( + widths: list[int], + count: int, + out_dir: Path, + *, + seed_base: int = SAT_SEED_BASE, + instance_prefix: str = "factoring", +) -> list[dict]: + if not widths or len(widths) != len(set(widths)): + raise ValueError("factor widths must be non-empty and unique") + all_targets: list[FactoringTarget] = [] + all_oracles: list[dict] = [] + for width in widths: + sat, sat_oracles = sat_targets( + width, + count, + seed_base=seed_base, + instance_prefix=instance_prefix, + ) + unsat, unsat_oracles = unsat_targets( + width, + sat, + instance_prefix=instance_prefix, + ) + all_targets.extend((*sat, *unsat)) + all_oracles.extend((*sat_oracles, *unsat_oracles)) + validate_targets(all_targets, all_oracles) + + raw_by_width = { + width: generate_multiplier(width, "array-ripple") for width in widths + } + manifest = [] + for target in all_targets: + instance_dir = ( + out_dir + / f"n{target.factor_bits}" + / target.expected_outcome + / f"{target.sequence_index:02d}" + ) + circuit_path = instance_dir / "instance.circuitsat.json" + cnf_path = instance_dir / "instance.cnf" + metadata_path = instance_dir / "instance.meta.json" + circuit = pin_port_values( + raw_by_width[target.factor_bits], {"product": target.target} + ) + circuit.setdefault("metadata", {})["factoring"] = { + **asdict(target), + } + write_json(circuit_path, circuit) + encode_validated_circuit(circuit).write_dimacs(cnf_path) + metadata = { + **asdict(target), + "circuit": str(circuit_path.relative_to(out_dir)), + "circuit_sha256": sha256_file(circuit_path), + "cnf": str(cnf_path.relative_to(out_dir)), + "cnf_sha256": sha256_file(cnf_path), + } + write_json(metadata_path, metadata) + manifest.append( + { + **metadata, + "metadata": str(metadata_path.relative_to(out_dir)), + } + ) + write_jsonl(out_dir / "manifest.jsonl", manifest) + write_jsonl(out_dir / "oracles.jsonl", all_oracles) + return manifest + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--width", type=int, action="append", required=True) + parser.add_argument("--count", type=int, default=10) + parser.add_argument("--seed-base", type=int, default=SAT_SEED_BASE) + parser.add_argument("--instance-prefix", default="factoring") + parser.add_argument("--out-dir", type=Path, required=True) + args = parser.parse_args() + widths = sorted(args.width) + try: + manifest = materialize( + widths, + args.count, + args.out_dir, + seed_base=args.seed_base, + instance_prefix=args.instance_prefix, + ) + except ValueError as error: + parser.error(str(error)) + print(f"wrote {len(manifest)} CircuitSAT/CNF instance pairs to {args.out_dir}") + + +if __name__ == "__main__": + main() diff --git a/benchmarks/cnc/hard_regime.py b/benchmarks/cnc/hard_regime.py deleted file mode 100755 index 78365c3..0000000 --- a/benchmarks/cnc/hard_regime.py +++ /dev/null @@ -1,436 +0,0 @@ -#!/usr/bin/env python3 -"""Generate and audit the hard-UNSAT factoring study declared by issue #51.""" - -from __future__ import annotations - -import argparse -import json -import random -import time -from collections import Counter -from pathlib import Path -from typing import Any, Iterable - -import sympy -import yaml - -from benchmarks.pipeline.circuit import ( - canonical_bytes, - pin_port_values, - read_jsonl, - sha256_bytes, - sha256_file, - write_json, - write_jsonl, -) -from benchmarks.pipeline.cnf import encode_validated_circuit -from benchmarks.pipeline.multipliers import generate_multiplier - - -MILLER_RABIN_BASES = (2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37) -EXPECTED_WIDTHS = ((64, 32), (72, 36), (80, 40)) -EXPECTED_METHOD_BUDGETS = { - "monolithic-kissat": ("none",), - "march-cu-dynamic": ("dynamic-default",), - "region-cc": ("low", "medium", "high"), - "structure-blind-cc": ("low", "medium", "high"), -} -EXPECTED_BANDS = {"low": 1 << 14, "medium": 1 << 16, "high": 1 << 18} -EXPECTED_TOOL_REVISIONS = { - "kissat": "8af8e56f174b778aef3aa45af9f739b2a5f492c2", - "march_cu": "705b60c6491ef2b61988b3ce6ac674be1b90571d", -} - - -class HardRegimeError(ValueError): - """The study contract or one of its artifacts violates the frozen protocol.""" - - -def mapping(value: object, label: str) -> dict[str, Any]: - if not isinstance(value, dict): - raise HardRegimeError(f"{label} must be an object") - return value - - -def positive_int(value: object, label: str) -> int: - if not isinstance(value, int) or isinstance(value, bool) or value <= 0: - raise HardRegimeError(f"{label} must be a positive integer") - return value - - -def load_contract(path: Path) -> dict[str, Any]: - try: - value = yaml.safe_load(path.read_text(encoding="utf-8")) - except (OSError, yaml.YAMLError) as exc: - raise HardRegimeError(f"cannot read contract {path}: {exc}") from exc - contract = mapping(value, "contract") - validate_contract(contract) - return contract - - -def contract_sha256(contract: dict[str, Any]) -> str: - return sha256_bytes(canonical_bytes(contract)) - - -def validate_contract(contract: dict[str, Any]) -> None: - if contract.get("schema_version") != 1: - raise HardRegimeError("contract schema_version must be 1") - if contract.get("study_id") != "array-ripple-hard-unsat-cnc-v1": - raise HardRegimeError("unexpected study_id") - if contract.get("issue") != 51: - raise HardRegimeError("contract must identify issue 51") - if contract.get("architecture") != "array-ripple": - raise HardRegimeError("hard-regime primary table must use array-ripple") - if contract.get("expected_outcome") != "unsat": - raise HardRegimeError("hard-regime targets must declare UNSAT") - - widths = contract.get("widths") - if not isinstance(widths, list): - raise HardRegimeError("widths must be a list") - actual_widths = [] - seeds = [] - for index, raw in enumerate(widths): - width = mapping(raw, f"widths[{index}]") - product = positive_int(width.get("product_width"), "product_width") - factor = positive_int(width.get("factor_input_width"), "factor_input_width") - if product != 2 * factor: - raise HardRegimeError( - f"product width {product} must be twice factor-input width {factor}" - ) - actual_widths.append((product, factor)) - for split, expected_count in (("calibration", 3), ("held_out", 10)): - spec = mapping(width.get(split), f"width {product} {split}") - count = positive_int(spec.get("count"), f"width {product} {split}.count") - seed = positive_int(spec.get("seed"), f"width {product} {split}.seed") - if count != expected_count: - raise HardRegimeError( - f"width {product} {split} count must be {expected_count}" - ) - seeds.append(seed) - if tuple(actual_widths) != EXPECTED_WIDTHS: - raise HardRegimeError( - "width ladder must be product/factor pairs 64/32, 72/36, and 80/40" - ) - duplicates = sorted(seed for seed, count in Counter(seeds).items() if count > 1) - if duplicates: - raise HardRegimeError(f"calibration and held-out seeds overlap: {duplicates}") - - bands = mapping(contract.get("frontier_bands"), "frontier_bands") - actual_bands = {} - for name, raw_spec in bands.items(): - spec = mapping(raw_spec, f"frontier band {name}") - actual_bands[name] = positive_int( - spec.get("center_cubes"), f"frontier band {name}.center_cubes" - ) - ratio = spec.get("accepted_ratio") - if ratio != [0.75, 1.25]: - raise HardRegimeError( - f"frontier band {name}.accepted_ratio must be [0.75, 1.25]" - ) - if actual_bands != EXPECTED_BANDS: - raise HardRegimeError("frontier bands must be centered at 2^14, 2^16, and 2^18") - - methods = mapping(contract.get("methods"), "methods") - if set(methods) != set(EXPECTED_METHOD_BUDGETS): - raise HardRegimeError("method set does not match the frozen issue #51 matrix") - for name, expected in EXPECTED_METHOD_BUDGETS.items(): - budgets = mapping(methods[name], f"method {name}").get("budgets") - if not isinstance(budgets, list) or tuple(budgets) != expected: - raise HardRegimeError(f"method {name} budgets must be {list(expected)}") - if methods["march-cu-dynamic"].get("cutoff_policy") != "upstream-default-dynamic": - raise HardRegimeError("march_cu must retain its upstream default dynamic cutoff") - for name in ("region-cc", "structure-blind-cc"): - if methods[name].get("max_rows") != 512: - raise HardRegimeError(f"method {name} max_rows must be 512") - - limits = mapping(contract.get("limits_seconds"), "limits_seconds") - expected_limits = {"monolithic": 600, "cubing": 7200, "per_cube_conquer": 1800} - if limits != expected_limits: - raise HardRegimeError(f"limits_seconds must be {expected_limits}") - scheduling = mapping(contract.get("scheduling"), "scheduling") - if scheduling.get("measured_workers") != 32: - raise HardRegimeError("measured worker count must be 32") - if scheduling.get("lpt_replay_workers") != [32, 128, 512]: - raise HardRegimeError("LPT replay worker counts must be 32, 128, and 512") - statistics = mapping(contract.get("statistics"), "statistics") - if statistics.get("unit") != "held-out-instance": - raise HardRegimeError("statistical unit must be the held-out instance") - adjustment = mapping(statistics.get("budget_adjustment"), "budget_adjustment") - if adjustment.get("common_grid") != "overlap-low-geometric-mid-overlap-high": - raise HardRegimeError("budget adjustment common grid is not preregistered") - calibration = mapping(contract.get("calibration"), "calibration") - if calibration.get("held_out_recalibration") != "forbidden": - raise HardRegimeError("held-out threshold tuning must be forbidden") - tool_sources = mapping(contract.get("tool_sources"), "tool_sources") - if set(tool_sources) != set(EXPECTED_TOOL_REVISIONS): - raise HardRegimeError("tool_sources must pin Kissat and march_cu") - for name, revision in EXPECTED_TOOL_REVISIONS.items(): - source = mapping(tool_sources[name], f"tool source {name}") - if source.get("revision") != revision: - raise HardRegimeError(f"tool source {name} revision is not frozen") - - -def strong_miller_rabin(value: int, bases: Iterable[int] = MILLER_RABIN_BASES) -> bool: - """Return whether ``value`` passes the declared independent strong-MR checks.""" - - if value < 2: - return False - small = tuple(bases) - for prime in small: - if value % prime == 0: - return value == prime - odd = value - 1 - power = 0 - while odd % 2 == 0: - odd //= 2 - power += 1 - for base in small: - if base >= value: - continue - witness = pow(base, odd, value) - if witness in (1, value - 1): - continue - for _ in range(power - 1): - witness = witness * witness % value - if witness == value - 1: - break - else: - return False - return True - - -def _targets_for_split( - product_width: int, - factor_width: int, - split: str, - count: int, - seed: int, -) -> list[dict[str, Any]]: - rng = random.Random(seed) - lower = 1 << (product_width - 1) - factor_max = (1 << factor_width) - 1 - reachable_max = factor_max * factor_max - records = [] - seen: set[int] = set() - while len(records) < count: - candidate = rng.randrange(lower, reachable_max + 1) | 1 - target = int(sympy.nextprime(candidate - 1)) - if target > reachable_max or target in seen: - continue - seen.add(target) - index = len(records) - split_label = "cal" if split == "calibration" else "test" - range_checks = { - "full_product_width": target.bit_length() == product_width, - "above_factor_input_range": target > factor_max, - "within_reachable_product_range": target <= reachable_max, - } - mr_passed = strong_miller_rabin(target) - sympy_passed = bool(sympy.isprime(target)) - if not all(range_checks.values()) or not mr_passed or not sympy_passed: - raise HardRegimeError("deterministic target generation produced an invalid prime") - records.append( - { - "schema_version": 1, - "id": f"prime-p{product_width}-{split_label}-{index:02d}", - "generator": "deterministic-prime-hard-unsat-v1", - "architecture": "array-ripple", - "semantic_task": "unsigned-factoring", - "expected_outcome": "unsat", - "product_width": product_width, - "factor_input_width": factor_width, - "split": split, - "split_index": index, - "seed": seed, - "target": target, - "primality_checks": { - "sympy_isprime": { - "result": sympy_passed, - "version": sympy.__version__, - }, - "strong_miller_rabin": { - "bases": list(MILLER_RABIN_BASES), - "result": mr_passed, - }, - }, - "range_checks": range_checks, - "unsat_argument": "prime-target-above-unsigned-factor-input-range", - } - ) - return records - - -def target_records(contract: dict[str, Any]) -> list[dict[str, Any]]: - validate_contract(contract) - records = [] - for width in contract["widths"]: - for split in ("calibration", "held_out"): - spec = width[split] - records.extend( - _targets_for_split( - width["product_width"], - width["factor_input_width"], - split, - spec["count"], - spec["seed"], - ) - ) - return records - - -def verify_target_records( - contract: dict[str, Any], records: list[dict[str, Any]] -) -> list[str]: - validate_contract(contract) - expected = target_records(contract) - if records != expected: - raise HardRegimeError( - "target records do not byte-semantically match deterministic regeneration" - ) - ids = [record["id"] for record in records] - if len(ids) != len(set(ids)): - raise HardRegimeError("target records contain duplicate instance IDs") - split_targets: dict[str, set[int]] = {"calibration": set(), "held_out": set()} - for record in records: - product_width = positive_int(record.get("product_width"), "record product_width") - factor_width = positive_int( - record.get("factor_input_width"), "record factor_input_width" - ) - target = positive_int(record.get("target"), "record target") - if product_width != 2 * factor_width: - raise HardRegimeError(f"{record.get('id')}: factor/product width confusion") - factor_max = (1 << factor_width) - 1 - if target.bit_length() != product_width: - raise HardRegimeError(f"{record.get('id')}: target does not use product width") - if target <= factor_max or target > factor_max * factor_max: - raise HardRegimeError(f"{record.get('id')}: target violates factor/product range") - if not sympy.isprime(target) or not strong_miller_rabin(target): - raise HardRegimeError(f"{record.get('id')}: target fails primality verification") - split = record.get("split") - if split not in split_targets: - raise HardRegimeError(f"{record.get('id')}: invalid split") - split_targets[split].add(target) - overlap = split_targets["calibration"] & split_targets["held_out"] - if overlap: - raise HardRegimeError(f"calibration and held-out targets overlap: {sorted(overlap)}") - return [ - f"PASS contract: {contract_sha256(contract)}", - f"PASS targets: {len(records)} deterministic prime UNSAT instances", - "PASS widths: product/factor pairs are 64/32, 72/36, and 80/40", - "PASS split: calibration and held-out seeds and targets are disjoint", - "PASS primality: SymPy and independent strong Miller-Rabin checks agree", - ] - - -def materialize_instances( - contract: dict[str, Any], records: list[dict[str, Any]], out_dir: Path -) -> list[dict[str, Any]]: - verify_target_records(contract, records) - out_dir.mkdir(parents=True, exist_ok=True) - write_jsonl(out_dir / "targets.jsonl", records) - contract_digest = contract_sha256(contract) - circuits: dict[int, tuple[dict[str, Any], Path]] = {} - manifest = [] - for record in records: - factor_width = record["factor_input_width"] - if factor_width not in circuits: - raw = generate_multiplier(factor_width, "array-ripple") - raw_path = out_dir / "raw" / f"array-ripple-f{factor_width}.json" - write_json(raw_path, raw) - circuits[factor_width] = (raw, raw_path) - raw, raw_path = circuits[factor_width] - pinned = pin_port_values(raw, {"product": record["target"]}) - benchmark = pinned.setdefault("metadata", {}).setdefault("benchmark", {}) - benchmark.update( - { - **record, - "contract_sha256": contract_digest, - "family": "hard-unsat-prime-factoring", - } - ) - instance_dir = out_dir / "instances" / f"p{record['product_width']}" / record["id"] - circuitsat = instance_dir / f"{record['id']}.circuitsat.json" - cnf = instance_dir / f"{record['id']}.cnf" - metadata = instance_dir / f"{record['id']}.meta.json" - write_json(circuitsat, pinned) - encoding_started = time.perf_counter() - encoding_cpu_started = time.process_time() - encode_validated_circuit(pinned).write_dimacs(cnf) - encoding_wall_s = time.perf_counter() - encoding_started - encoding_cpu_s = time.process_time() - encoding_cpu_started - meta = { - **record, - "contract_sha256": contract_digest, - "raw_circuit": str(raw_path.relative_to(out_dir)), - "raw_circuit_sha256": sha256_file(raw_path), - "circuitsat": str(circuitsat.relative_to(out_dir)), - "circuitsat_sha256": sha256_file(circuitsat), - "cnf": str(cnf.relative_to(out_dir)), - "cnf_sha256": sha256_file(cnf), - "encoding_wall_s": encoding_wall_s, - "encoding_cpu_s": encoding_cpu_s, - } - write_json(metadata, meta) - manifest.append({**meta, "metadata": str(metadata.relative_to(out_dir))}) - write_jsonl(out_dir / "manifest.jsonl", manifest) - return manifest - - -def command_validate_contract(args: argparse.Namespace) -> None: - contract = load_contract(args.contract) - print(f"PASS contract: {contract_sha256(contract)}") - - -def command_generate_targets(args: argparse.Namespace) -> None: - contract = load_contract(args.contract) - records = target_records(contract) - write_jsonl(args.out, records) - for line in verify_target_records(contract, records): - print(line) - - -def command_verify_targets(args: argparse.Namespace) -> None: - contract = load_contract(args.contract) - for line in verify_target_records(contract, read_jsonl(args.targets)): - print(line) - - -def command_materialize(args: argparse.Namespace) -> None: - contract = load_contract(args.contract) - records = target_records(contract) - manifest = materialize_instances(contract, records, args.out_dir) - print(f"PASS materialization: {len(manifest)} CircuitSAT/CNF pairs") - - -def build_parser() -> argparse.ArgumentParser: - parser = argparse.ArgumentParser(description=__doc__) - commands = parser.add_subparsers(dest="command", required=True) - command = commands.add_parser("validate-contract") - command.add_argument("contract", type=Path) - command.set_defaults(handler=command_validate_contract) - command = commands.add_parser("generate-targets") - command.add_argument("contract", type=Path) - command.add_argument("--out", type=Path, required=True) - command.set_defaults(handler=command_generate_targets) - command = commands.add_parser("verify-targets") - command.add_argument("contract", type=Path) - command.add_argument("targets", type=Path) - command.set_defaults(handler=command_verify_targets) - command = commands.add_parser("materialize") - command.add_argument("contract", type=Path) - command.add_argument("--out-dir", type=Path, required=True) - command.set_defaults(handler=command_materialize) - return parser - - -def main() -> int: - args = build_parser().parse_args() - try: - args.handler(args) - except (HardRegimeError, OSError, ValueError, json.JSONDecodeError) as exc: - raise SystemExit(f"FAIL: {exc}") from exc - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/benchmarks/cnc/hard_regime_matrix.py b/benchmarks/cnc/hard_regime_matrix.py deleted file mode 100755 index 30c9336..0000000 --- a/benchmarks/cnc/hard_regime_matrix.py +++ /dev/null @@ -1,556 +0,0 @@ -#!/usr/bin/env python3 -"""Lock the issue #51 toolchain and materialize its complete run matrix.""" - -from __future__ import annotations - -import argparse -import json -import math -import os -import subprocess -from collections import Counter -from pathlib import Path -from typing import Any - -from benchmarks.cnc.calibrate_hard_regime import ( - CalibrationError, - calibration_loss, - choose_width_response, - median_tasks, -) - -from benchmarks.cnc.hard_regime import ( - HardRegimeError, - contract_sha256, - load_contract, - mapping, - target_records, -) -from benchmarks.pipeline.circuit import ( - canonical_bytes, - load_json, - read_jsonl, - sha256_bytes, - sha256_file, - write_json, -) - - -class MatrixError(HardRegimeError): - """The tool lock, instance manifest, calibration lock, or matrix is invalid.""" - - -def executable_record( - path: Path, - source_revision: str, - version_args: list[str], - *, - allow_nonzero: bool = False, - required_basename: str | None = None, -) -> dict[str, Any]: - resolved = path.resolve() - if not resolved.is_file() or not os.access(resolved, os.X_OK): - raise MatrixError(f"tool is not executable: {resolved}") - if required_basename is not None and resolved.name != required_basename: - raise MatrixError( - f"tool {resolved} is not the revision-qualified binary {required_basename}" - ) - process = subprocess.run( - [str(resolved), *version_args], capture_output=True, text=True, check=False - ) - if process.returncode and not allow_nonzero: - raise MatrixError(f"version command failed for {resolved}: {process.stderr[-500:]}") - output = (process.stdout + process.stderr).strip() - if not output: - raise MatrixError(f"version command produced no identity output for {resolved}") - return { - "path": str(resolved), - "executable_sha256": sha256_file(resolved), - "source_revision": source_revision, - "version_command": [str(resolved), *version_args], - "version_output": output[:4000], - "version_output_sha256": sha256_bytes(output.encode()), - } - - -def lock_toolchain( - contract: dict[str, Any], - cuber: Path, - kissat: Path, - march_cu: Path, - repository_revision: str, -) -> dict[str, Any]: - if not repository_revision or len(repository_revision) < 7: - raise MatrixError("repository revision is required for cnc_cuber provenance") - sources = contract["tool_sources"] - record = { - "schema_version": 1, - "kind": "hard-regime-toolchain-lock", - "contract_sha256": contract_sha256(contract), - "tools": { - "cnc_cuber": executable_record(cuber, repository_revision, ["--help"]), - "kissat": executable_record( - kissat, - sources["kissat"]["revision"], - ["--version"], - required_basename=f"kissat-{sources['kissat']['revision']}", - ), - "march_cu": executable_record( - march_cu, - sources["march_cu"]["revision"], - [], - allow_nonzero=True, - required_basename=f"march_cu-{sources['march_cu']['revision']}", - ), - }, - } - record["toolchain_sha256"] = sha256_bytes(canonical_bytes(record)) - return record - - -def verify_toolchain( - contract: dict[str, Any], toolchain: dict[str, Any], *, check_paths: bool -) -> None: - if toolchain.get("schema_version") != 1 or toolchain.get("kind") != "hard-regime-toolchain-lock": - raise MatrixError("unsupported toolchain lock") - if toolchain.get("contract_sha256") != contract_sha256(contract): - raise MatrixError("toolchain lock uses a different contract") - tools = mapping(toolchain.get("tools"), "toolchain.tools") - if set(tools) != {"cnc_cuber", "kissat", "march_cu"}: - raise MatrixError("toolchain must contain cnc_cuber, kissat, and march_cu") - expected_revisions = { - "kissat": contract["tool_sources"]["kissat"]["revision"], - "march_cu": contract["tool_sources"]["march_cu"]["revision"], - } - for name, raw in tools.items(): - spec = mapping(raw, f"toolchain tool {name}") - digest = spec.get("executable_sha256") - if not isinstance(digest, str) or len(digest) != 64: - raise MatrixError(f"toolchain tool {name} has no executable SHA-256") - if name in expected_revisions and spec.get("source_revision") != expected_revisions[name]: - raise MatrixError(f"toolchain tool {name} source revision mismatch") - if check_paths: - path = Path(str(spec.get("path"))).resolve() - if sha256_file(path) != digest: - raise MatrixError(f"toolchain tool {name} executable hash mismatch") - unsigned = {key: value for key, value in toolchain.items() if key != "toolchain_sha256"} - if toolchain.get("toolchain_sha256") != sha256_bytes(canonical_bytes(unsigned)): - raise MatrixError("toolchain lock fingerprint mismatch") - - -def validate_instance_manifest( - contract: dict[str, Any], manifest: list[dict[str, Any]] -) -> None: - expected = {record["id"]: record for record in target_records(contract)} - if len(manifest) != len(expected): - raise MatrixError(f"instance manifest needs {len(expected)} records") - seen = set() - digest = contract_sha256(contract) - for record in manifest: - instance_id = record.get("id") - if instance_id in seen: - raise MatrixError(f"duplicate manifest instance {instance_id!r}") - seen.add(instance_id) - if instance_id not in expected: - raise MatrixError(f"undeclared manifest instance {instance_id!r}") - target = expected[instance_id] - for field in ( - "target", - "product_width", - "factor_input_width", - "split", - "split_index", - "seed", - "architecture", - "expected_outcome", - ): - if record.get(field) != target[field]: - raise MatrixError(f"{instance_id}: manifest field {field} differs from target lock") - if record.get("contract_sha256") != digest: - raise MatrixError(f"{instance_id}: manifest contract hash mismatch") - if record["product_width"] != 2 * record["factor_input_width"]: - raise MatrixError(f"{instance_id}: factor/product width confusion") - for artifact in ("circuitsat", "cnf"): - if not isinstance(record.get(artifact), str) or not isinstance( - record.get(f"{artifact}_sha256"), str - ): - raise MatrixError(f"{instance_id}: incomplete {artifact} provenance") - for metric in ("encoding_wall_s", "encoding_cpu_s"): - value = record.get(metric) - if not isinstance(value, (int, float)) or isinstance(value, bool) or value < 0: - raise MatrixError(f"{instance_id}: invalid {metric}") - if seen != set(expected): - raise MatrixError("instance manifest does not cover the declared target set") - - -def load_calibration_lock( - contract: dict[str, Any], - calibration_root: Path, - product_width: int, - selector: str, - toolchain: dict[str, Any], - manifest: list[dict[str, Any]], -) -> dict[str, Any]: - path = calibration_root / f"p{product_width}" / selector / "calibration-lock.json" - lock = load_json(path) - if lock.get("schema_version") != 1 or lock.get("kind") != "width-level-cc-calibration-lock": - raise MatrixError(f"{path}: unsupported calibration lock") - for field, value in { - "contract_sha256": contract_sha256(contract), - "product_width": product_width, - "selector": selector, - "cuber_sha256": toolchain["tools"]["cnc_cuber"]["executable_sha256"], - }.items(): - if lock.get(field) != value: - raise MatrixError(f"{path}: calibration {field} mismatch") - expected_instances = sorted( - [ - {"id": record["id"], "sha256": record["circuitsat_sha256"]} - for record in manifest - if record["product_width"] == product_width - and record["split"] == "calibration" - ], - key=lambda item: item["id"], - ) - actual_instances = lock.get("calibration_instances") - if not isinstance(actual_instances, list) or not all( - isinstance(item, dict) for item in actual_instances - ): - raise MatrixError(f"{path}: calibration instance provenance is malformed") - actual_instances = sorted(actual_instances, key=lambda item: str(item.get("id"))) - if actual_instances != expected_instances: - raise MatrixError(f"{path}: calibration instances are not the frozen split") - method = "region-cc" if selector == "region" else "structure-blind-cc" - if lock.get("method") != method: - raise MatrixError(f"{path}: calibration method mismatch") - if lock.get("max_rows") != contract["methods"][method]["max_rows"]: - raise MatrixError(f"{path}: calibration max_rows mismatch") - - search = lock.get("search") - if not isinstance(search, dict): - raise MatrixError(f"{path}: calibration search provenance is missing") - initial_threshold = search.get("initial_threshold") - maximum_threshold = search.get("maximum_threshold") - if ( - not isinstance(initial_threshold, int) - or isinstance(initial_threshold, bool) - or initial_threshold != 1 - or not isinstance(maximum_threshold, int) - or isinstance(maximum_threshold, bool) - or maximum_threshold <= initial_threshold - or not isinstance(search.get("probe_checkpoint_schema_version"), int) - or isinstance(search.get("probe_checkpoint_schema_version"), bool) - or search["probe_checkpoint_schema_version"] <= 0 - ): - raise MatrixError(f"{path}: calibration search provenance is malformed") - - response = lock.get("response") - if not isinstance(response, list) or not response: - raise MatrixError(f"{path}: calibration response is empty") - expected_ids = [item["id"] for item in expected_instances] - response_by_threshold = {} - for row in response: - if not isinstance(row, dict): - raise MatrixError(f"{path}: malformed calibration response row") - threshold = row.get("threshold") - if not isinstance(threshold, int) or isinstance(threshold, bool) or threshold < 0: - raise MatrixError(f"{path}: malformed calibration response threshold") - if threshold in response_by_threshold: - raise MatrixError(f"{path}: duplicate calibration response threshold {threshold}") - instances = row.get("instances") - response_ids = ( - [item.get("id") for item in instances] - if isinstance(instances, list) - and all(isinstance(item, dict) for item in instances) - else [] - ) - if not all(isinstance(instance_id, str) for instance_id in response_ids) or sorted( - response_ids - ) != expected_ids: - raise MatrixError(f"{path}: response threshold {threshold} uses the wrong instances") - for item in instances: - if not isinstance(item, dict): - raise MatrixError(f"{path}: malformed response instance") - tasks = item.get("tasks") - if not isinstance(tasks, int) or isinstance(tasks, bool) or tasks < 0: - raise MatrixError(f"{path}: response task count is invalid") - for field in ("elapsed_s", "cpu_s"): - value = item.get(field) - if ( - not isinstance(value, (int, float)) - or isinstance(value, bool) - or not math.isfinite(float(value)) - or value < 0 - ): - raise MatrixError(f"{path}: response {field} is invalid") - response_by_threshold[threshold] = row - - bands = mapping(lock.get("bands"), f"{path} bands") - if set(bands) != set(contract["frontier_bands"]): - raise MatrixError(f"{path}: calibration bands are incomplete") - for band, spec in bands.items(): - if not isinstance(spec, dict): - raise MatrixError(f"{path}: band {band} is malformed") - contract_band = contract["frontier_bands"][band] - target = int(contract_band["center_cubes"]) - minimum = math.ceil(target * float(contract_band["accepted_ratio"][0])) - maximum = math.floor(target * float(contract_band["accepted_ratio"][1])) - threshold = spec.get("selected_threshold") - if not isinstance(threshold, int) or threshold < 0: - raise MatrixError(f"{path}: band {band} has no frozen threshold") - if spec.get("target_tasks") != target: - raise MatrixError(f"{path}: band {band} target differs from contract") - if spec.get("accepted_task_range") != [minimum, maximum]: - raise MatrixError(f"{path}: band {band} accepted range differs from contract") - bracket = spec.get("search_bracket") - if ( - not isinstance(bracket, list) - or len(bracket) != 2 - or not all(isinstance(value, int) and value >= 0 for value in bracket) - or bracket[0] >= bracket[1] - or any(value not in response_by_threshold for value in bracket) - ): - raise MatrixError(f"{path}: band {band} search bracket is invalid") - try: - selected = choose_width_response(response, target, minimum, maximum) - except CalibrationError as error: - raise MatrixError(f"{path}: band {band} has no acceptable calibration") from error - if threshold != selected["threshold"] or threshold not in response_by_threshold: - raise MatrixError(f"{path}: band {band} threshold is not selected from the response") - expected_median = median_tasks(selected) - expected_loss = calibration_loss(selected, target)[0] - recorded_median = spec.get("median_tasks") - if not isinstance(recorded_median, (int, float)) or isinstance( - recorded_median, bool - ) or not math.isclose( - float(recorded_median), expected_median, rel_tol=1e-12, abs_tol=1e-12 - ): - raise MatrixError(f"{path}: band {band} median task count is inconsistent") - recorded_loss = spec.get("calibration_loss") - if not isinstance(recorded_loss, (int, float)) or isinstance( - recorded_loss, bool - ) or not math.isclose( - float(recorded_loss), expected_loss, rel_tol=1e-12, abs_tol=1e-12 - ): - raise MatrixError(f"{path}: band {band} selection loss is inconsistent") - if spec.get("within_target_range") is not True: - raise MatrixError(f"{path}: band {band} is outside its accepted task range") - final_instances = spec.get("instances") - final_ids = ( - [item.get("id") for item in final_instances] - if isinstance(final_instances, list) - and all(isinstance(item, dict) for item in final_instances) - else [] - ) - if not all(isinstance(instance_id, str) for instance_id in final_ids) or sorted( - final_ids - ) != expected_ids: - raise MatrixError(f"{path}: band {band} final rerun uses the wrong instances") - response_tasks = { - item["id"]: item["tasks"] for item in response_by_threshold[threshold]["instances"] - } - for item in final_instances: - if not isinstance(item, dict) or item.get("tasks") != response_tasks.get(item.get("id")): - raise MatrixError(f"{path}: band {band} final task count changed on rerun") - for field in ("frontier_sha256", "trace_sha256"): - digest = item.get(field) - if not isinstance(digest, str) or len(digest) != 64: - raise MatrixError(f"{path}: band {band} has invalid {field}") - for field in ("cubing_elapsed_s", "cubing_cpu_s"): - value = item.get(field) - if ( - not isinstance(value, (int, float)) - or isinstance(value, bool) - or not math.isfinite(float(value)) - or value < 0 - ): - raise MatrixError(f"{path}: band {band} has invalid {field}") - return lock - - -def cell_id(instance: str, method: str, budget: str) -> str: - return f"{instance}__{method}__{budget}" - - -def build_matrix( - contract: dict[str, Any], - manifest: list[dict[str, Any]], - calibration_root: Path, - toolchain: dict[str, Any], -) -> dict[str, Any]: - validate_instance_manifest(contract, manifest) - verify_toolchain(contract, toolchain, check_paths=False) - locks = { - (width, selector): load_calibration_lock( - contract, - calibration_root, - width, - selector, - toolchain, - manifest, - ) - for width in (64, 72, 80) - for selector in ("region", "structure-blind") - } - cells = [] - for instance in sorted(manifest, key=lambda record: record["id"]): - common = { - "schema_version": 1, - "instance_id": instance["id"], - "split": instance["split"], - "split_index": instance["split_index"], - "product_width": instance["product_width"], - "factor_input_width": instance["factor_input_width"], - "expected_outcome": "unsat", - "circuitsat": instance["circuitsat"], - "circuitsat_sha256": instance["circuitsat_sha256"], - "global_cnf": instance["cnf"], - "global_cnf_sha256": instance["cnf_sha256"], - "encoding_wall_s": instance["encoding_wall_s"], - "encoding_cpu_s": instance["encoding_cpu_s"], - "pilot": instance["split"] == "held_out" and instance["split_index"] < 3, - } - - cells.append( - { - **common, - "cell_id": cell_id(instance["id"], "monolithic-kissat", "none"), - "method": "monolithic-kissat", - "budget": "none", - "required_cpus": 1, - "time_limit_s": contract["limits_seconds"]["monolithic"], - } - ) - if instance["split"] != "held_out": - continue - cells.append( - { - **common, - "cell_id": cell_id(instance["id"], "march-cu-dynamic", "dynamic-default"), - "method": "march-cu-dynamic", - "budget": "dynamic-default", - "required_cpus": contract["scheduling"]["measured_workers"], - "cubing_time_limit_s": contract["limits_seconds"]["cubing"], - "per_cube_time_limit_s": contract["limits_seconds"]["per_cube_conquer"], - "cutoff_policy": "upstream-default-dynamic", - } - ) - for method, selector in ( - ("region-cc", "region"), - ("structure-blind-cc", "structure-blind"), - ): - lock = locks[(instance["product_width"], selector)] - for budget in ("low", "medium", "high"): - cells.append( - { - **common, - "cell_id": cell_id(instance["id"], method, budget), - "method": method, - "budget": budget, - "required_cpus": contract["scheduling"]["measured_workers"], - "cubing_time_limit_s": contract["limits_seconds"]["cubing"], - "per_cube_time_limit_s": contract["limits_seconds"]["per_cube_conquer"], - "selector": selector, - "max_rows": contract["methods"][method]["max_rows"], - "cc_threshold": lock["bands"][budget]["selected_threshold"], - "calibration_lock": str( - Path(f"p{instance['product_width']}") - / selector - / "calibration-lock.json" - ), - } - ) - counts = Counter(cell["method"] for cell in cells) - expected_counts = { - "monolithic-kissat": 39, - "march-cu-dynamic": 30, - "region-cc": 90, - "structure-blind-cc": 90, - } - if dict(counts) != expected_counts: - raise MatrixError(f"internal run-matrix count mismatch: {dict(counts)}") - if len({cell["cell_id"] for cell in cells}) != len(cells): - raise MatrixError("run matrix contains duplicate cell IDs") - return { - "schema_version": 1, - "kind": "hard-regime-run-matrix", - "contract_sha256": contract_sha256(contract), - "toolchain_sha256": toolchain["toolchain_sha256"], - "cell_counts": expected_counts, - "cells": cells, - } - - -def main() -> int: - parser = argparse.ArgumentParser(description=__doc__) - commands = parser.add_subparsers(dest="command", required=True) - command = commands.add_parser("lock-toolchain") - command.add_argument("contract", type=Path) - command.add_argument("--cuber", type=Path, required=True) - command.add_argument("--kissat", type=Path, required=True) - command.add_argument("--march-cu", type=Path, required=True) - command.add_argument("--repository-revision", required=True) - command.add_argument("--out", type=Path, required=True) - command = commands.add_parser("build-matrix") - command.add_argument("contract", type=Path) - command.add_argument("manifest", type=Path) - command.add_argument("--calibration-root", type=Path, required=True) - command.add_argument("--toolchain", type=Path, required=True) - command.add_argument("--out", type=Path, required=True) - command = commands.add_parser("list-cells") - command.add_argument("matrix", type=Path) - command.add_argument( - "--set", - choices=("monolithic", "pilot-cnc", "full-cnc", "remaining-cnc"), - required=True, - ) - command.add_argument("--out", type=Path, required=True) - args = parser.parse_args() - try: - if args.command == "list-cells": - matrix = load_json(args.matrix) - cells = matrix.get("cells") - if not isinstance(cells, list): - raise MatrixError("run matrix has no cells") - selected = [] - for cell in cells: - monolithic = cell.get("method") == "monolithic-kissat" - include = { - "monolithic": monolithic, - "pilot-cnc": not monolithic and cell.get("pilot") is True, - "full-cnc": not monolithic, - "remaining-cnc": not monolithic and cell.get("pilot") is not True, - }[args.set] - if include: - selected.append(cell["cell_id"]) - args.out.parent.mkdir(parents=True, exist_ok=True) - args.out.write_text("".join(f"{cell_id}\n" for cell_id in selected), encoding="utf-8") - print(f"PASS cell-list: {len(selected)} cells -> {args.out}") - return 0 - contract = load_contract(args.contract) - if args.command == "lock-toolchain": - result = lock_toolchain( - contract, - args.cuber, - args.kissat, - args.march_cu, - args.repository_revision, - ) - elif args.command == "build-matrix": - result = build_matrix( - contract, - read_jsonl(args.manifest), - args.calibration_root, - load_json(args.toolchain), - ) - else: - raise MatrixError(f"unsupported command {args.command}") - write_json(args.out, result) - except (MatrixError, HardRegimeError, OSError, ValueError, json.JSONDecodeError) as exc: - parser.error(str(exc)) - print(f"PASS {result['kind']}: {args.out}") - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/benchmarks/cnc/hpc/build_matrix.sh b/benchmarks/cnc/hpc/build_matrix.sh deleted file mode 100755 index 66e5544..0000000 --- a/benchmarks/cnc/hpc/build_matrix.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/bin/bash -set -euo pipefail - -: "${BI51_REPO:?set BI51_REPO to the repository checkout}" -: "${BI51_ARTIFACT_ROOT:?set BI51_ARTIFACT_ROOT to the hard-regime artifact root}" - -cd "$BI51_REPO" -source .venv/bin/activate - -python -m benchmarks.cnc.hard_regime_matrix build-matrix \ - benchmarks/cnc/contracts/hard-regime-v1.yaml \ - "$BI51_ARTIFACT_ROOT/instances/manifest.jsonl" \ - --calibration-root "$BI51_ARTIFACT_ROOT/calibration" \ - --toolchain "$BI51_ARTIFACT_ROOT/toolchain.json" \ - --out "$BI51_ARTIFACT_ROOT/run-matrix.json" - -python -m benchmarks.cnc.hard_regime_matrix list-cells \ - "$BI51_ARTIFACT_ROOT/run-matrix.json" --set monolithic \ - --out "$BI51_ARTIFACT_ROOT/monolithic-cells.txt" -python -m benchmarks.cnc.hard_regime_matrix list-cells \ - "$BI51_ARTIFACT_ROOT/run-matrix.json" --set pilot-cnc \ - --out "$BI51_ARTIFACT_ROOT/pilot-cnc-cells.txt" -python -m benchmarks.cnc.hard_regime_matrix list-cells \ - "$BI51_ARTIFACT_ROOT/run-matrix.json" --set remaining-cnc \ - --out "$BI51_ARTIFACT_ROOT/remaining-cnc-cells.txt" diff --git a/benchmarks/cnc/hpc/calibrate.sh b/benchmarks/cnc/hpc/calibrate.sh deleted file mode 100755 index ad5719d..0000000 --- a/benchmarks/cnc/hpc/calibrate.sh +++ /dev/null @@ -1,27 +0,0 @@ -#!/bin/bash -set -euo pipefail - -: "${BI51_REPO:?set BI51_REPO to the repository checkout}" -: "${BI51_ARTIFACT_ROOT:?set BI51_ARTIFACT_ROOT to the hard-regime artifact root}" -: "${SLURM_ARRAY_TASK_ID:?calibration must run as a six-element Slurm array}" - -case "$SLURM_ARRAY_TASK_ID" in - 0) PRODUCT_WIDTH=64; SELECTOR=region ;; - 1) PRODUCT_WIDTH=64; SELECTOR=structure-blind ;; - 2) PRODUCT_WIDTH=72; SELECTOR=region ;; - 3) PRODUCT_WIDTH=72; SELECTOR=structure-blind ;; - 4) PRODUCT_WIDTH=80; SELECTOR=region ;; - 5) PRODUCT_WIDTH=80; SELECTOR=structure-blind ;; - *) echo "invalid calibration array index: $SLURM_ARRAY_TASK_ID" >&2; exit 2 ;; -esac - -cd "$BI51_REPO" -source .venv/bin/activate - -python -m benchmarks.cnc.calibrate_hard_regime \ - benchmarks/cnc/contracts/hard-regime-v1.yaml \ - "$BI51_ARTIFACT_ROOT/instances/manifest.jsonl" \ - --product-width "$PRODUCT_WIDTH" \ - --selector "$SELECTOR" \ - --cuber target/release/cnc_cuber \ - --out-dir "$BI51_ARTIFACT_ROOT/calibration/p${PRODUCT_WIDTH}/${SELECTOR}" diff --git a/benchmarks/cnc/hpc/materialize.sh b/benchmarks/cnc/hpc/materialize.sh deleted file mode 100755 index 225fe87..0000000 --- a/benchmarks/cnc/hpc/materialize.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/bin/bash -set -euo pipefail - -: "${BI51_REPO:?set BI51_REPO to the repository checkout}" -: "${BI51_ARTIFACT_ROOT:?set BI51_ARTIFACT_ROOT to the hard-regime artifact root}" - -cd "$BI51_REPO" -source .venv/bin/activate - -python -m benchmarks.cnc.hard_regime materialize \ - benchmarks/cnc/contracts/hard-regime-v1.yaml \ - --out-dir "$BI51_ARTIFACT_ROOT/instances" - -python -m benchmarks.cnc.hard_regime_matrix lock-toolchain \ - benchmarks/cnc/contracts/hard-regime-v1.yaml \ - --cuber target/release/cnc_cuber \ - --kissat cnc-tools/bin/kissat-8af8e56f174b778aef3aa45af9f739b2a5f492c2 \ - --march-cu cnc-tools/bin/march_cu-705b60c6491ef2b61988b3ce6ac674be1b90571d \ - --repository-revision "$(git rev-parse HEAD)" \ - --out "$BI51_ARTIFACT_ROOT/toolchain.json" diff --git a/benchmarks/cnc/hpc/report.sh b/benchmarks/cnc/hpc/report.sh deleted file mode 100755 index d75c58d..0000000 --- a/benchmarks/cnc/hpc/report.sh +++ /dev/null @@ -1,35 +0,0 @@ -#!/bin/bash -set -euo pipefail - -: "${BI51_REPO:?set BI51_REPO to the repository checkout}" -: "${BI51_ARTIFACT_ROOT:?set BI51_ARTIFACT_ROOT to the hard-regime artifact root}" -: "${BI51_SCOPE:?set BI51_SCOPE to pilot or full}" - -cd "$BI51_REPO" -source .venv/bin/activate - -REPORT_ROOT="$BI51_ARTIFACT_ROOT/${BI51_SCOPE}-report" - -python -m benchmarks.cnc.verify_hard_regime \ - benchmarks/cnc/contracts/hard-regime-v1.yaml \ - "$BI51_ARTIFACT_ROOT/instances/manifest.jsonl" \ - --calibration-root "$BI51_ARTIFACT_ROOT/calibration" \ - --toolchain "$BI51_ARTIFACT_ROOT/toolchain.json" \ - --matrix "$BI51_ARTIFACT_ROOT/run-matrix.json" \ - --runs-root "$BI51_ARTIFACT_ROOT/runs" \ - --scope "$BI51_SCOPE" - -python -m benchmarks.cnc.aggregate_hard_regime \ - benchmarks/cnc/contracts/hard-regime-v1.yaml \ - "$BI51_ARTIFACT_ROOT/run-matrix.json" \ - --runs-root "$BI51_ARTIFACT_ROOT/runs" \ - --scope "$BI51_SCOPE" --out-dir "$REPORT_ROOT" - -python -m benchmarks.cnc.verify_hard_regime \ - benchmarks/cnc/contracts/hard-regime-v1.yaml \ - "$BI51_ARTIFACT_ROOT/instances/manifest.jsonl" \ - --calibration-root "$BI51_ARTIFACT_ROOT/calibration" \ - --toolchain "$BI51_ARTIFACT_ROOT/toolchain.json" \ - --matrix "$BI51_ARTIFACT_ROOT/run-matrix.json" \ - --runs-root "$BI51_ARTIFACT_ROOT/runs" \ - --scope "$BI51_SCOPE" --aggregate "$REPORT_ROOT/aggregate.json" diff --git a/benchmarks/cnc/hpc/run_cell.sh b/benchmarks/cnc/hpc/run_cell.sh deleted file mode 100755 index e2ec860..0000000 --- a/benchmarks/cnc/hpc/run_cell.sh +++ /dev/null @@ -1,27 +0,0 @@ -#!/bin/bash -set -euo pipefail - -: "${BI51_REPO:?set BI51_REPO to the repository checkout}" -: "${BI51_ARTIFACT_ROOT:?set BI51_ARTIFACT_ROOT to the hard-regime artifact root}" -: "${BI51_CELL_LIST:?set BI51_CELL_LIST to an immutable cell-list text file}" -: "${BI51_SCRATCH_ROOT:?set BI51_SCRATCH_ROOT to an HPC2 SSD scratch directory}" -: "${SLURM_ARRAY_TASK_ID:?cell execution must run as a Slurm array}" - -LINE_NUMBER=$((SLURM_ARRAY_TASK_ID + 1)) -CELL_ID=$(sed -n "${LINE_NUMBER}p" "$BI51_CELL_LIST") -if [[ -z "$CELL_ID" ]]; then - echo "no cell at array index $SLURM_ARRAY_TASK_ID" >&2 - exit 2 -fi - -cd "$BI51_REPO" -source .venv/bin/activate - -python -m benchmarks.cnc.run_hard_regime_cell \ - benchmarks/cnc/contracts/hard-regime-v1.yaml \ - "$BI51_ARTIFACT_ROOT/run-matrix.json" \ - "$BI51_ARTIFACT_ROOT/toolchain.json" \ - --cell-id "$CELL_ID" \ - --instance-root "$BI51_ARTIFACT_ROOT/instances" \ - --output-root "$BI51_ARTIFACT_ROOT/runs" \ - --temp-root "$BI51_SCRATCH_ROOT" diff --git a/benchmarks/cnc/hpc/smoke.sh b/benchmarks/cnc/hpc/smoke.sh deleted file mode 100755 index fda312b..0000000 --- a/benchmarks/cnc/hpc/smoke.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/bash -set -euo pipefail - -: "${BI51_REPO:?set BI51_REPO to the repository checkout}" -: "${BI51_ARTIFACT_ROOT:?set BI51_ARTIFACT_ROOT to the hard-regime artifact root}" - -cd "$BI51_REPO" -source .venv/bin/activate - -python -m benchmarks.cnc.hard_regime validate-contract \ - benchmarks/cnc/contracts/hard-regime-v1.yaml -python -m pytest tests/test_hard_regime.py tests/test_calibrate_hard_regime.py \ - tests/test_hard_regime_matrix.py tests/test_run_hard_regime_cell.py \ - tests/test_verify_hard_regime.py tests/test_aggregate_hard_regime.py -q -cargo test --test cnc_cuber_trace -cargo build --release --bin cnc_cuber -python -m benchmarks.cnc.hard_regime generate-targets \ - benchmarks/cnc/contracts/hard-regime-v1.yaml \ - --out "$BI51_ARTIFACT_ROOT/smoke-targets.jsonl" diff --git a/benchmarks/cnc/requirements.txt b/benchmarks/cnc/requirements.txt deleted file mode 100644 index c99f4fb..0000000 --- a/benchmarks/cnc/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -PyYAML==6.0.3 -sympy==1.14.0 diff --git a/benchmarks/cnc/run_hard_regime_cell.py b/benchmarks/cnc/run_hard_regime_cell.py deleted file mode 100755 index 197145f..0000000 --- a/benchmarks/cnc/run_hard_regime_cell.py +++ /dev/null @@ -1,614 +0,0 @@ -#!/usr/bin/env python3 -"""Execute one frozen hard-regime run-matrix cell and write a terminal record.""" - -from __future__ import annotations - -import argparse -import json -import os -import re -import resource -import socket -import subprocess -import time -from collections import defaultdict -from datetime import datetime, timezone -from pathlib import Path -from typing import Any - -from benchmarks.cnc.conquer_parallel import ( - parse_cnf, - parse_stats, - read_cubes, - run_arm, -) -from benchmarks.cnc.hard_regime import HardRegimeError, contract_sha256, load_contract -from benchmarks.cnc.hard_regime_matrix import MatrixError, verify_toolchain -from benchmarks.pipeline.circuit import ( - atomic_write_json, - canonical_bytes, - load_json, - read_jsonl, - sha256_bytes, - sha256_file, -) - - -_CUBER_STATS = re.compile(r"status=OK cubes=(\d+)") - - -class CellError(HardRegimeError): - """A frozen cell cannot be executed or its output is inconsistent.""" - - -def utc_now() -> str: - return datetime.now(timezone.utc).isoformat() - - -def _as_text(value: str | bytes | None) -> str: - if value is None: - return "" - return value.decode(errors="replace") if isinstance(value, bytes) else value - - -def run_process( - command: list[str], timeout_s: float, stdout_path: Path, stderr_path: Path -) -> dict[str, Any]: - stdout_path.parent.mkdir(parents=True, exist_ok=True) - before = resource.getrusage(resource.RUSAGE_CHILDREN) - started = time.monotonic() - started_utc = utc_now() - try: - process = subprocess.run( - command, - capture_output=True, - text=True, - timeout=timeout_s, - check=False, - ) - stdout = process.stdout - stderr = process.stderr - returncode = process.returncode - state = "finished" - except subprocess.TimeoutExpired as exc: - stdout = _as_text(exc.stdout) - stderr = _as_text(exc.stderr) - returncode = None - state = "timeout" - elapsed_s = time.monotonic() - started - after = resource.getrusage(resource.RUSAGE_CHILDREN) - stdout_path.write_text(stdout, encoding="utf-8") - stderr_path.write_text(stderr, encoding="utf-8") - return { - "command": command, - "state": state, - "returncode": returncode, - "started_utc": started_utc, - "finished_utc": utc_now(), - "wall_s": elapsed_s, - "user_s": after.ru_utime - before.ru_utime, - "system_s": after.ru_stime - before.ru_stime, - "cpu_s": (after.ru_utime - before.ru_utime) - + (after.ru_stime - before.ru_stime), - "stdout": stdout_path.name, - "stdout_sha256": sha256_file(stdout_path), - "stderr": stderr_path.name, - "stderr_sha256": sha256_file(stderr_path), - } - - -def verify_region_trace(frontier: Path, trace: Path) -> dict[str, int]: - cubes = list(read_cubes(frontier)) - nodes = read_jsonl(trace) - if not nodes: - raise CellError("region trace is empty") - children: dict[int, list[dict[str, Any]]] = defaultdict(list) - cutoff_literals = [] - refutation_counts: dict[str, int] = defaultdict(int) - variable_count = None - for expected_id, node in enumerate(nodes): - if node.get("node_id") != expected_id: - raise CellError("region trace node IDs are not contiguous") - literals = node.get("literals") - if not isinstance(literals, list) or not all( - isinstance(literal, int) and not isinstance(literal, bool) and literal != 0 - for literal in literals - ): - raise CellError(f"region trace node {expected_id} has invalid literals") - if len({abs(literal) for literal in literals}) != len(literals): - raise CellError(f"region trace node {expected_id} repeats a decision variable") - sigma_dec = node.get("sigma_dec") - sigma_all = node.get("sigma_all") - freevars = node.get("freevars") - if ( - sigma_dec != len(literals) - or not isinstance(sigma_all, int) - or isinstance(sigma_all, bool) - or not isinstance(freevars, int) - or isinstance(freevars, bool) - or sigma_all < sigma_dec - or freevars < 0 - ): - raise CellError(f"region trace node {expected_id} has invalid assignment counts") - node_variables = sigma_all + freevars - if variable_count is None: - variable_count = node_variables - elif node_variables != variable_count: - raise CellError("region trace changes the declared variable count") - - parent = node.get("parent_id") - if expected_id == 0: - if parent is not None or node.get("child_index") is not None or node.get("depth") != 0: - raise CellError("region trace has an invalid root") - else: - if not isinstance(parent, int) or parent < 0 or parent >= expected_id: - raise CellError(f"region trace node {expected_id} has an invalid parent") - if node.get("depth") != nodes[parent].get("depth") + 1: - raise CellError(f"region trace node {expected_id} has an invalid depth") - children[parent].append(node) - kind = node.get("kind") - if kind not in {"branch", "cutoff", "refuted", "sat"}: - raise CellError(f"region trace node {expected_id} has invalid kind {kind!r}") - reason = node.get("refutation_reason") - if kind == "refuted": - if reason not in { - "root-propagation-contradiction", - "selector-no-feasible-config", - "branch-propagation-contradiction", - }: - raise CellError(f"region refuted node {expected_id} has no closure reason") - refutation_counts[reason] += 1 - elif reason is not None: - raise CellError(f"region non-refuted node {expected_id} has a closure reason") - if kind == "sat": - raise CellError("region trace contains a SAT leaf for an expected-UNSAT target") - if kind == "cutoff": - cutoff_literals.append(literals) - for node in nodes: - node_id = node["node_id"] - actual_children = children.get(node_id, []) - if node["kind"] == "branch": - variables = node.get("rule_variables") - clauses = node.get("rule_clauses") - if ( - not isinstance(variables, list) - or not variables - or len(variables) > 64 - or not all(isinstance(variable, int) and variable > 0 for variable in variables) - or len(set(variables)) != len(variables) - or any(variable in {abs(literal) for literal in node["literals"]} for variable in variables) - ): - raise CellError(f"region branch node {node_id} has invalid rule variables") - if not isinstance(clauses, list) or not clauses or len(actual_children) != len(clauses): - raise CellError(f"region branch node {node_id} has incomplete children") - indices = sorted(child.get("child_index") for child in actual_children) - if indices != list(range(len(clauses))): - raise CellError(f"region branch node {node_id} has invalid child indices") - by_index = {child["child_index"]: child for child in actual_children} - variable_mask = (1 << len(variables)) - 1 - for index, clause in enumerate(clauses): - if not isinstance(clause, dict): - raise CellError(f"region branch node {node_id} has a malformed rule clause") - mask = clause.get("mask") - value = clause.get("value") - if ( - not isinstance(mask, int) - or isinstance(mask, bool) - or not isinstance(value, int) - or isinstance(value, bool) - or mask <= 0 - or mask & ~variable_mask - or value & ~mask - ): - raise CellError(f"region branch node {node_id} has an invalid rule clause") - suffix = [ - variable if (value >> bit) & 1 else -variable - for bit, variable in enumerate(variables) - if (mask >> bit) & 1 - ] - if by_index[index].get("literals") != node["literals"] + suffix: - raise CellError( - f"region child {by_index[index]['node_id']} does not implement rule clause {index}" - ) - elif actual_children: - raise CellError(f"region leaf node {node_id} unexpectedly has children") - elif node.get("rule_clauses") not in ([], None): - raise CellError(f"region leaf node {node_id} unexpectedly has rule clauses") - if cutoff_literals != cubes: - raise CellError("region trace cutoff leaves do not reproduce frontier bytes") - return { - "nodes": len(nodes), - "branches": sum(node["kind"] == "branch" for node in nodes), - "cutoffs": len(cutoff_literals), - "refuted": sum(node["kind"] == "refuted" for node in nodes), - "sat_leaves": sum(node["kind"] == "sat" for node in nodes), - "root_refutations": refutation_counts["root-propagation-contradiction"], - "selector_refutations": refutation_counts["selector-no-feasible-config"], - "branch_refutations": refutation_counts["branch-propagation-contradiction"], - } - - -def artifact(path: Path, root: Path) -> dict[str, Any]: - return { - "path": str(path.relative_to(root)), - "sha256": sha256_file(path), - "bytes": path.stat().st_size, - } - - -def select_cell(matrix: dict[str, Any], cell_id: str) -> dict[str, Any]: - if matrix.get("schema_version") != 1 or matrix.get("kind") != "hard-regime-run-matrix": - raise CellError("unsupported run matrix") - matches = [cell for cell in matrix.get("cells", []) if cell.get("cell_id") == cell_id] - if len(matches) != 1: - raise CellError(f"run matrix contains {len(matches)} matches for {cell_id!r}") - return matches[0] - - -def resolve_input(root: Path, relative: object, expected_sha256: object) -> Path: - if not isinstance(relative, str) or not isinstance(expected_sha256, str): - raise CellError("cell input provenance is incomplete") - path = (root / relative).resolve() - resolved_root = root.resolve() - if path != resolved_root and resolved_root not in path.parents: - raise CellError(f"cell input escapes instance root: {relative}") - if sha256_file(path) != expected_sha256: - raise CellError(f"cell input hash mismatch: {relative}") - return path - - -def slurm_context() -> dict[str, str]: - names = ( - "SLURM_JOB_ID", - "SLURM_ARRAY_JOB_ID", - "SLURM_ARRAY_TASK_ID", - "SLURM_JOB_PARTITION", - "SLURM_CPUS_PER_TASK", - "SLURM_NTASKS", - "SLURM_MEM_PER_NODE", - "SLURM_TIMELIMIT", - ) - return {name: os.environ[name] for name in names if name in os.environ} - - -def base_terminal( - contract: dict[str, Any], - matrix_path: Path, - matrix: dict[str, Any], - toolchain: dict[str, Any], - cell: dict[str, Any], -) -> dict[str, Any]: - return { - "schema_version": 1, - "kind": "hard-regime-terminal-cell", - "contract_sha256": contract_sha256(contract), - "matrix_sha256": sha256_file(matrix_path), - "toolchain_sha256": toolchain["toolchain_sha256"], - "cell_id": cell["cell_id"], - "cell_sha256": sha256_bytes(canonical_bytes(cell)), - "instance_id": cell["instance_id"], - "split": cell["split"], - "product_width": cell["product_width"], - "factor_input_width": cell["factor_input_width"], - "method": cell["method"], - "budget": cell["budget"], - "expected_outcome": cell["expected_outcome"], - "input_artifacts": { - "circuitsat": { - "path": cell["circuitsat"], - "sha256": cell["circuitsat_sha256"], - }, - "global_cnf": { - "path": cell["global_cnf"], - "sha256": cell["global_cnf_sha256"], - }, - }, - "host": socket.gethostname(), - "slurm": slurm_context(), - "started_utc": utc_now(), - } - - -def run_monolithic( - cell: dict[str, Any], cnf: Path, kissat: Path, cell_dir: Path -) -> dict[str, Any]: - stage = run_process( - [str(kissat), "--statistics", "--relaxed", str(cnf)], - float(cell["time_limit_s"]), - cell_dir / "monolithic.stdout", - cell_dir / "monolithic.stderr", - ) - stdout = (cell_dir / "monolithic.stdout").read_text(encoding="utf-8") - decisions, conflicts = parse_stats(stdout) - verdict = {10: "sat", 20: "unsat"}.get(stage["returncode"]) - state = ( - "complete" - if verdict == "unsat" - else "wrong-answer" - if verdict == "sat" - else "monolithic-timeout" - if stage["state"] == "timeout" - else "monolithic-error" - ) - return { - "state": state, - "verdict": verdict, - "stages": {"monolithic": stage}, - "metrics": { - "encoding_wall_s": cell["encoding_wall_s"], - "encoding_cpu_s": cell["encoding_cpu_s"], - "solver_wall_s": stage["wall_s"], - "solver_cpu_s": stage["cpu_s"], - "decisions": decisions, - "conflicts": conflicts, - "end_to_end_wall_s": cell["encoding_wall_s"] + stage["wall_s"], - "end_to_end_cpu_s": cell["encoding_cpu_s"] + stage["cpu_s"], - "censored": stage["state"] == "timeout", - }, - } - - -def run_cubing( - cell: dict[str, Any], - circuitsat: Path, - cnf: Path, - tools: dict[str, Any], - cell_dir: Path, -) -> tuple[dict[str, Any], Path, Path | None]: - frontier = cell_dir / "frontier.icnf" - trace: Path | None = None - if cell["method"] == "march-cu-dynamic": - command = [tools["march_cu"]["path"], str(cnf), "-o", str(frontier)] - else: - trace = cell_dir / "nodes.jsonl" - command = [ - tools["cnc_cuber"]["path"], - str(circuitsat), - "--cc-threshold", - str(cell["cc_threshold"]), - "-o", - str(frontier), - "--selector", - cell["selector"], - "--max-rows", - str(cell["max_rows"]), - "--trace", - str(trace), - ] - stage = run_process( - command, - float(cell["cubing_time_limit_s"]), - cell_dir / "cubing.stdout", - cell_dir / "cubing.stderr", - ) - stage["complete"] = stage["state"] == "finished" and stage["returncode"] == 0 - if not stage["complete"]: - return stage, frontier, trace - if not frontier.is_file(): - raise CellError("cuber reported success without a frontier") - cubes = sum(1 for _ in read_cubes(frontier)) - stage["frontier_size"] = cubes - stage["frontier_bytes"] = frontier.stat().st_size - stage["frontier_sha256"] = sha256_file(frontier) - if trace is not None: - trace_summary = verify_region_trace(frontier, trace) - stderr = (cell_dir / "cubing.stderr").read_text(encoding="utf-8") - match = _CUBER_STATS.search(stderr) - if not match or int(match.group(1)) != cubes: - raise CellError("region cuber log/frontier task counts disagree") - stage["completeness"] = { - "complete": True, - "evidence": "verified-branch-assignments-and-refutation-reasons", - **trace_summary, - } - else: - stage["completeness"] = { - "complete": True, - "evidence": "upstream-march-cu-successful-partition-output", - } - return stage, frontier, trace - - -def run_cnc( - contract: dict[str, Any], - cell: dict[str, Any], - circuitsat: Path, - cnf: Path, - tools: dict[str, Any], - cell_dir: Path, - temp_root: Path | None, -) -> dict[str, Any]: - cubing, frontier, trace = run_cubing(cell, circuitsat, cnf, tools, cell_dir) - artifacts = {} - if frontier.is_file(): - artifacts["frontier"] = artifact(frontier, cell_dir) - if trace is not None and trace.is_file(): - artifacts["trace"] = artifact(trace, cell_dir) - if not cubing["complete"]: - state = "cubing-timeout" if cubing["state"] == "timeout" else "cubing-error" - return { - "state": state, - "verdict": None, - "stages": {"cubing": cubing}, - "artifacts": artifacts, - "metrics": { - "encoding_wall_s": cell["encoding_wall_s"], - "encoding_cpu_s": cell["encoding_cpu_s"], - "cubing_wall_s": cubing["wall_s"], - "cubing_cpu_s": cubing["cpu_s"], - "end_to_end_wall_s": cell["encoding_wall_s"] + cubing["wall_s"], - "end_to_end_cpu_s": cell["encoding_cpu_s"] + cubing["cpu_s"], - "censored": cubing["state"] == "timeout", - }, - } - - cnf_data = cnf.read_bytes() - variables, clauses, body = parse_cnf(cnf_data) - raw_results = cell_dir / "cube-results.jsonl" - temp_dir = ( - temp_root / cell["cell_id"] if temp_root is not None else cell_dir / "tmp" - ) - temp_dir.mkdir(parents=True, exist_ok=True) - workers = int(contract["scheduling"]["measured_workers"]) - replay_workers = list(contract["scheduling"]["lpt_replay_workers"]) - total_cubes = sum(1 for _ in read_cubes(frontier)) - conquer = run_arm( - cell["cell_id"], - read_cubes(frontier), - total_cubes, - workers, - replay_workers, - raw_results, - ( - variables, - clauses, - body, - tools["kissat"]["path"], - float(cell["per_cube_time_limit_s"]), - str(temp_dir), - ), - ) - artifacts["cube_results"] = artifact(raw_results, cell_dir) - state = ( - "complete" - if conquer["complete"] and conquer["result"] == "unsat" - else "wrong-answer" - if conquer["result"] == "sat" - else "conquer-timeout" - if conquer["timeouts"] - else "conquer-error" - ) - end_wall = ( - float(cell["encoding_wall_s"]) - + float(cubing["wall_s"]) - + float(conquer["measured_makespan_s"]) - ) - end_cpu = ( - float(cell["encoding_cpu_s"]) - + float(cubing["cpu_s"]) - + float(conquer["total_cpu_s"]) - ) - return { - "state": state, - "verdict": conquer["result"] if conquer["complete"] else None, - "stages": {"cubing": cubing, "conquer": conquer}, - "artifacts": artifacts, - "metrics": { - "encoding_wall_s": cell["encoding_wall_s"], - "encoding_cpu_s": cell["encoding_cpu_s"], - "cubing_wall_s": cubing["wall_s"], - "cubing_cpu_s": cubing["cpu_s"], - "frontier_size": cubing["frontier_size"], - "frontier_bytes": cubing["frontier_bytes"], - "conquer_work_cpu_s": conquer["total_cpu_s"], - "conquer_span_s": conquer["max_s"], - "p99_s": conquer["p99_s"], - "maximum_conflicts": conquer["conflicts_max"], - "timeout_count": conquer["timeouts"], - "measured_32_worker_makespan_s": conquer["measured_makespan_s"], - "lpt_makespan_by_workers_s": conquer["lpt_makespan_by_workers_s"], - "end_to_end_wall_s": end_wall, - "end_to_end_cpu_s": end_cpu, - "censored": conquer["censored"], - }, - } - - -def run_cell( - contract: dict[str, Any], - matrix_path: Path, - matrix: dict[str, Any], - toolchain: dict[str, Any], - cell: dict[str, Any], - instance_root: Path, - output_root: Path, - temp_root: Path | None = None, -) -> dict[str, Any]: - if matrix.get("contract_sha256") != contract_sha256(contract): - raise CellError("run matrix contract hash mismatch") - if matrix.get("toolchain_sha256") != toolchain.get("toolchain_sha256"): - raise CellError("run matrix toolchain hash mismatch") - verify_toolchain(contract, toolchain, check_paths=True) - required_cpus = int(cell["required_cpus"]) - allocated = os.environ.get("SLURM_CPUS_PER_TASK") - if allocated is not None and int(allocated) < required_cpus: - raise CellError( - f"cell needs {required_cpus} CPUs but SLURM_CPUS_PER_TASK={allocated}" - ) - cell_dir = output_root / "cells" / cell["cell_id"] - cell_dir.mkdir(parents=True, exist_ok=True) - terminal_path = cell_dir / "terminal.json" - base = base_terminal(contract, matrix_path, matrix, toolchain, cell) - if terminal_path.is_file(): - existing = load_json(terminal_path) - if existing.get("cell_sha256") != base["cell_sha256"]: - raise CellError("existing terminal record belongs to a different cell lock") - return existing - - try: - circuitsat = resolve_input( - instance_root, cell["circuitsat"], cell["circuitsat_sha256"] - ) - cnf = resolve_input( - instance_root, cell["global_cnf"], cell["global_cnf_sha256"] - ) - tools = toolchain["tools"] - if cell["method"] == "monolithic-kissat": - result = run_monolithic(cell, cnf, Path(tools["kissat"]["path"]), cell_dir) - else: - result = run_cnc( - contract, cell, circuitsat, cnf, tools, cell_dir, temp_root - ) - terminal = { - **base, - **result, - "finished_utc": utc_now(), - } - except Exception as exc: - terminal = { - **base, - "state": "harness-error", - "verdict": None, - "error": f"{type(exc).__name__}: {exc}", - "finished_utc": utc_now(), - } - atomic_write_json(terminal_path, terminal) - raise - atomic_write_json(terminal_path, terminal) - return terminal - - -def main() -> int: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("contract", type=Path) - parser.add_argument("matrix", type=Path) - parser.add_argument("toolchain", type=Path) - parser.add_argument("--cell-id", required=True) - parser.add_argument("--instance-root", type=Path, required=True) - parser.add_argument("--output-root", type=Path, required=True) - parser.add_argument("--temp-root", type=Path) - args = parser.parse_args() - try: - contract = load_contract(args.contract) - matrix = load_json(args.matrix) - toolchain = load_json(args.toolchain) - cell = select_cell(matrix, args.cell_id) - terminal = run_cell( - contract, - args.matrix, - matrix, - toolchain, - cell, - args.instance_root, - args.output_root, - args.temp_root, - ) - except (CellError, MatrixError, OSError, ValueError, json.JSONDecodeError) as exc: - print(f"FAIL cell {args.cell_id}: {exc}") - return 2 - print(f"TERMINAL {terminal['cell_id']}: {terminal['state']}") - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/benchmarks/cnc/solve.py b/benchmarks/cnc/solve.py new file mode 100644 index 0000000..1ab3a3f --- /dev/null +++ b/benchmarks/cnc/solve.py @@ -0,0 +1,174 @@ +#!/usr/bin/env python3 +"""Solve factoring CNFs directly with Kissat or conquer a cube frontier.""" + +from __future__ import annotations + +import argparse +import json +import resource +import subprocess +import time +from pathlib import Path +from typing import Any + +from benchmarks.cnc.conquer_parallel import ( + parse_cnf, + parse_stats, + read_cubes, + run_arm, +) +from benchmarks.pipeline.circuit import write_json + + +def _text(value: str | bytes | None) -> str: + if value is None: + return "" + return value.decode(errors="replace") if isinstance(value, bytes) else value + + +def run_process( + command: list[str], + *, + timeout_s: float, + out_dir: Path, + label: str, +) -> dict[str, Any]: + out_dir.mkdir(parents=True, exist_ok=True) + before = resource.getrusage(resource.RUSAGE_CHILDREN) + started = time.monotonic() + try: + process = subprocess.run( + command, + capture_output=True, + text=True, + timeout=None if timeout_s == 0 else timeout_s, + check=False, + ) + stdout = process.stdout + stderr = process.stderr + returncode = process.returncode + timed_out = False + except subprocess.TimeoutExpired as error: + stdout = _text(error.stdout) + stderr = _text(error.stderr) + returncode = None + timed_out = True + after = resource.getrusage(resource.RUSAGE_CHILDREN) + (out_dir / f"{label}.stdout").write_text(stdout, encoding="utf-8") + (out_dir / f"{label}.stderr").write_text(stderr, encoding="utf-8") + return { + "command": command, + "stdout": stdout, + "stderr": stderr, + "returncode": returncode, + "timed_out": timed_out, + "wall_s": time.monotonic() - started, + "user_s": after.ru_utime - before.ru_utime, + "system_s": after.ru_stime - before.ru_stime, + } + + +def run_kissat( + cnf: Path, + kissat: Path, + *, + timeout_s: float, + out_dir: Path, +) -> dict[str, Any]: + process = run_process( + [str(kissat), "--statistics", "--relaxed", str(cnf)], + timeout_s=timeout_s, + out_dir=out_dir, + label="kissat", + ) + decisions, conflicts = parse_stats(process["stdout"]) + result = ( + "timeout" + if process["timed_out"] + else {10: "sat", 20: "unsat"}.get(process["returncode"], "error") + ) + record = { + "schema_version": 1, + "mode": "direct-kissat", + "result": result, + "returncode": process["returncode"], + "timed_out": process["timed_out"], + "wall_s": process["wall_s"], + "user_s": process["user_s"], + "system_s": process["system_s"], + "decisions": decisions, + "conflicts": conflicts, + "cnf": str(cnf.resolve()), + "kissat": str(kissat.resolve()), + } + write_json(out_dir / "summary.json", record) + return record + + +def conquer_frontier( + cnf: Path, + frontier: Path, + kissat: Path, + *, + workers: int, + timeout_s: float, + out_dir: Path, + tmp_dir: Path | None = None, + total_cubes: int | None = None, +) -> dict[str, Any]: + variables, clauses, body = parse_cnf(cnf.read_bytes()) + out_dir.mkdir(parents=True, exist_ok=True) + if tmp_dir: + tmp_dir.mkdir(parents=True, exist_ok=True) + if total_cubes is None: + total_cubes = sum(1 for _ in read_cubes(frontier, variables)) + summary = run_arm( + "factoring", + read_cubes(frontier, variables), + total_cubes, + workers, + [workers], + out_dir / "cubes.jsonl", + ( + variables, + clauses, + body, + str(kissat.resolve()), + timeout_s, + str(tmp_dir.resolve()) if tmp_dir else None, + ), + ) + record = { + "schema_version": 1, + "mode": "parallel-conquer", + "cnf": str(cnf.resolve()), + "frontier": str(frontier.resolve()), + "kissat": str(kissat.resolve()), + "workers": workers, + "per_cube_timeout_s": timeout_s, + **summary, + } + write_json(out_dir / "summary.json", record) + return record + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("cnf", type=Path) + parser.add_argument("--kissat", type=Path, required=True) + parser.add_argument("--timeout-s", type=float, default=600.0) + parser.add_argument("--out-dir", type=Path, required=True) + args = parser.parse_args() + if args.timeout_s < 0: + parser.error("timeout must be non-negative") + record = run_kissat( + args.cnf, + args.kissat, + timeout_s=args.timeout_s, + out_dir=args.out_dir, + ) + print(json.dumps(record, indent=2, sort_keys=True, allow_nan=False)) + + +if __name__ == "__main__": + main() diff --git a/benchmarks/cnc/verify_hard_regime.py b/benchmarks/cnc/verify_hard_regime.py deleted file mode 100755 index aae42ef..0000000 --- a/benchmarks/cnc/verify_hard_regime.py +++ /dev/null @@ -1,440 +0,0 @@ -#!/usr/bin/env python3 -"""Verify issue #51 contracts, calibration locks, terminal cells, and accounting.""" - -from __future__ import annotations - -import argparse -import hashlib -import json -import math -from collections import Counter -from pathlib import Path -from typing import Any - -from benchmarks.cnc.conquer_parallel import distribution, lpt_makespan, read_cubes -from benchmarks.cnc.hard_regime import HardRegimeError, contract_sha256, load_contract -from benchmarks.cnc.hard_regime_matrix import ( - MatrixError, - build_matrix, - validate_instance_manifest, - verify_toolchain, -) -from benchmarks.cnc.run_hard_regime_cell import verify_region_trace -from benchmarks.pipeline.circuit import ( - canonical_bytes, - load_json, - read_jsonl, - sha256_bytes, - sha256_file, -) - - -class VerificationError(HardRegimeError): - """The hard-regime evidence bundle is incomplete or inconsistent.""" - - -def close(actual: object, expected: float, label: str) -> None: - if not isinstance(actual, (int, float)) or isinstance(actual, bool): - raise VerificationError(f"{label} is not numeric") - if math.isnan(expected): - if not math.isnan(float(actual)): - raise VerificationError(f"{label} should be NaN") - elif not math.isclose(float(actual), expected, rel_tol=1e-8, abs_tol=1e-8): - raise VerificationError(f"{label} mismatch: {actual} != {expected}") - - -def close_optional(actual: object, expected: float | None, label: str) -> None: - if expected is None: - if actual is not None: - raise VerificationError(f"{label} should be null") - return - close(actual, expected, label) - - -def checked_relative(root: Path, spec: object, label: str) -> Path: - if not isinstance(spec, dict): - raise VerificationError(f"{label} artifact spec is missing") - relative = spec.get("path") - expected = spec.get("sha256") - if not isinstance(relative, str) or not isinstance(expected, str): - raise VerificationError(f"{label} artifact provenance is incomplete") - root = root.resolve() - path = (root / relative).resolve() - if path != root and root not in path.parents: - raise VerificationError(f"{label} artifact escapes its cell directory") - if sha256_file(path) != expected: - raise VerificationError(f"{label} artifact SHA-256 mismatch") - if spec.get("bytes") != path.stat().st_size: - raise VerificationError(f"{label} artifact byte count mismatch") - return path - - -def verify_stage_logs(cell_dir: Path, stage: dict[str, Any], label: str) -> None: - for stream in ("stdout", "stderr"): - relative = stage.get(stream) - expected = stage.get(f"{stream}_sha256") - if not isinstance(relative, str) or not isinstance(expected, str): - raise VerificationError(f"{label} stage has incomplete {stream} provenance") - path = cell_dir / relative - if sha256_file(path) != expected: - raise VerificationError(f"{label} stage {stream} hash mismatch") - - -def verify_conquer_records( - frontier: Path, - results: Path, - summary: dict[str, Any], - workers: int, - replay_workers: list[int], -) -> None: - cubes = list(read_cubes(frontier)) - rows = read_jsonl(results) - if len(rows) != len(cubes) or summary.get("cubes") != len(cubes): - raise VerificationError("conquer records do not cover every frontier cube") - by_index = {} - intervals = [] - per_worker: dict[int, list[tuple[int, int]]] = {} - for row in rows: - index = row.get("cube_index") - if not isinstance(index, int) or index < 0 or index >= len(cubes) or index in by_index: - raise VerificationError("conquer records contain invalid or duplicate cube indices") - by_index[index] = row - expected_cube_hash = hashlib.sha256( - (" ".join(map(str, cubes[index])) + " 0\n").encode() - ).hexdigest() - if row.get("cube_sha256") != expected_cube_hash: - raise VerificationError(f"cube {index} assumption hash mismatch") - released = row.get("released_monotonic_ns") - started = row.get("started_monotonic_ns") - finished = row.get("finished_monotonic_ns") - collected = row.get("collected_monotonic_ns") - worker = row.get("worker_pid") - if not all(isinstance(value, int) for value in (released, started, finished, collected, worker)): - raise VerificationError(f"cube {index} has malformed scheduling events") - if not released <= started <= finished <= collected: - raise VerificationError(f"cube {index} has illegal scheduling event order") - intervals.append((started, 1)) - intervals.append((finished, -1)) - per_worker.setdefault(worker, []).append((started, finished)) - if set(by_index) != set(range(len(cubes))): - raise VerificationError("conquer cube indices are not exhaustive") - active = maximum = 0 - for _, delta in sorted(intervals, key=lambda event: (event[0], event[1])): - active += delta - if active < 0: - raise VerificationError("conquer schedule has a finish before its start") - maximum = max(maximum, active) - if active or maximum > workers: - raise VerificationError("conquer schedule exceeds the declared worker count") - for worker, assigned in per_worker.items(): - assigned.sort() - if any(left[1] > right[0] for left, right in zip(assigned, assigned[1:])): - raise VerificationError(f"worker {worker} has overlapping cube assignments") - - ordered = [by_index[index] for index in range(len(cubes))] - durations = [float(row["elapsed_s"]) for row in ordered] - cpu = [float(row["user_s"]) + float(row["system_s"]) for row in ordered] - conflicts = [float(row["conflicts"]) for row in ordered if row.get("conflicts") is not None] - counts = { - "timeouts": sum(bool(row.get("censored")) for row in ordered), - "errors": sum(row.get("result") == "error" for row in ordered), - "sat": sum(row.get("result") == "sat" for row in ordered), - "unsat": sum(row.get("result") == "unsat" for row in ordered), - } - counts["completed"] = counts["sat"] + counts["unsat"] - for name, value in counts.items(): - if summary.get(name) != value: - raise VerificationError(f"conquer {name} count does not reconstruct") - if summary.get("terminal_records") != len(ordered): - raise VerificationError("conquer terminal-record count does not reconstruct") - time_stats = distribution(durations) - conflict_stats = distribution(conflicts) - close(summary.get("total_solver_s"), sum(durations), "conquer total solver work") - close(summary.get("total_cpu_s"), sum(cpu), "conquer total CPU work") - close(summary.get("max_s"), time_stats["max"], "conquer span") - close(summary.get("p99_s"), time_stats["p99"], "conquer p99") - close_optional( - summary.get("conflicts_max"), - conflict_stats["max"], - "conquer maximum conflicts", - ) - measured = ( - 0.0 - if not rows - else ( - max(int(row["collected_monotonic_ns"]) for row in rows) - - min(int(row["released_monotonic_ns"]) for row in rows) - ) - / 1e9 - ) - close(summary.get("measured_makespan_s"), measured, "measured makespan") - expected_lpt = {str(count): lpt_makespan(durations, count) for count in replay_workers} - actual_lpt = summary.get("lpt_makespan_by_workers_s") - if not isinstance(actual_lpt, dict) or set(actual_lpt) != set(expected_lpt): - raise VerificationError("LPT replay worker set differs from the contract") - for count, value in expected_lpt.items(): - close(actual_lpt[count], value, f"LPT {count}-worker makespan") - if summary.get("lpt_is_lower_bound") != bool(counts["timeouts"] or counts["errors"]): - raise VerificationError("LPT censoring designation is incorrect") - - -def verify_terminal( - contract: dict[str, Any], - matrix_path: Path, - matrix: dict[str, Any], - toolchain: dict[str, Any], - cell: dict[str, Any], - terminal: dict[str, Any], - cell_dir: Path, -) -> str: - expected = { - "kind": "hard-regime-terminal-cell", - "contract_sha256": contract_sha256(contract), - "matrix_sha256": sha256_file(matrix_path), - "toolchain_sha256": toolchain["toolchain_sha256"], - "cell_id": cell["cell_id"], - "cell_sha256": sha256_bytes(canonical_bytes(cell)), - "instance_id": cell["instance_id"], - "method": cell["method"], - "budget": cell["budget"], - "product_width": cell["product_width"], - "factor_input_width": cell["factor_input_width"], - } - for field, value in expected.items(): - if terminal.get(field) != value: - raise VerificationError(f"{cell['cell_id']}: terminal {field} mismatch") - if terminal.get("state") not in { - "complete", - "monolithic-timeout", - "monolithic-error", - "cubing-timeout", - "cubing-error", - "conquer-timeout", - "conquer-error", - "wrong-answer", - "harness-error", - }: - raise VerificationError(f"{cell['cell_id']}: unknown terminal state") - if terminal.get("state") == "wrong-answer": - raise VerificationError(f"{cell['cell_id']}: solver returned SAT for a prime target") - inputs = terminal.get("input_artifacts") - if not isinstance(inputs, dict): - raise VerificationError(f"{cell['cell_id']}: terminal input hashes are missing") - if inputs.get("global_cnf", {}).get("sha256") != cell["global_cnf_sha256"]: - raise VerificationError(f"{cell['cell_id']}: mixed conquer encoding") - if inputs.get("circuitsat", {}).get("sha256") != cell["circuitsat_sha256"]: - raise VerificationError(f"{cell['cell_id']}: CircuitSAT input hash mismatch") - - stages = terminal.get("stages", {}) - metrics = terminal.get("metrics", {}) - artifacts = terminal.get("artifacts", {}) - if terminal["state"] == "harness-error": - if not isinstance(terminal.get("error"), str) or not terminal["error"]: - raise VerificationError(f"{cell['cell_id']}: harness error has no diagnostic") - return "harness-terminal-only" - if cell["method"] == "monolithic-kissat": - stage = stages.get("monolithic") - if not isinstance(stage, dict): - raise VerificationError(f"{cell['cell_id']}: missing monolithic stage") - verify_stage_logs(cell_dir, stage, "monolithic") - close( - metrics.get("end_to_end_wall_s"), - float(cell["encoding_wall_s"]) + float(stage["wall_s"]), - "monolithic end-to-end wall", - ) - close( - metrics.get("end_to_end_cpu_s"), - float(cell["encoding_cpu_s"]) + float(stage["cpu_s"]), - "monolithic end-to-end CPU", - ) - return "monolithic-stage-reconstructed" - - cubing = stages.get("cubing") - if not isinstance(cubing, dict): - raise VerificationError(f"{cell['cell_id']}: missing cubing stage") - verify_stage_logs(cell_dir, cubing, "cubing") - if not cubing.get("complete"): - if terminal["state"] not in {"cubing-timeout", "cubing-error"}: - raise VerificationError(f"{cell['cell_id']}: incomplete cubing has wrong terminal state") - return "cubing-stage-reconstructed" - frontier = checked_relative(cell_dir, artifacts.get("frontier"), "frontier") - if cubing.get("frontier_size") != sum(1 for _ in read_cubes(frontier)): - raise VerificationError(f"{cell['cell_id']}: frontier size mismatch") - if cell["method"] in {"region-cc", "structure-blind-cc"}: - trace = checked_relative(cell_dir, artifacts.get("trace"), "trace") - verify_region_trace(frontier, trace) - conquer = stages.get("conquer") - if not isinstance(conquer, dict): - if terminal["state"] not in {"cubing-timeout", "cubing-error"}: - raise VerificationError(f"{cell['cell_id']}: completed frontier has no conquer records") - return "cubing-stage-reconstructed" - results = checked_relative(cell_dir, artifacts.get("cube_results"), "cube results") - verify_conquer_records( - frontier, - results, - conquer, - int(contract["scheduling"]["measured_workers"]), - list(contract["scheduling"]["lpt_replay_workers"]), - ) - close(metrics.get("conquer_work_cpu_s"), float(conquer["total_cpu_s"]), "conquer work") - close(metrics.get("conquer_span_s"), float(conquer["max_s"]), "conquer span") - expected_wall = ( - float(cell["encoding_wall_s"]) - + float(cubing["wall_s"]) - + float(conquer["measured_makespan_s"]) - ) - close(metrics.get("end_to_end_wall_s"), expected_wall, "CnC end-to-end wall") - return "conquer-records-reconstructed" - - -def selected_cells(matrix: dict[str, Any], scope: str) -> list[dict[str, Any]]: - cells = matrix["cells"] - if scope == "full": - return cells - return [ - cell - for cell in cells - if cell["method"] == "monolithic-kissat" or cell.get("pilot") is True - ] - - -def verify_bundle( - contract_path: Path, - manifest_path: Path, - calibration_root: Path, - toolchain_path: Path, - matrix_path: Path, - runs_root: Path, - scope: str, - aggregate_path: Path | None = None, -) -> list[str]: - contract = load_contract(contract_path) - manifest = read_jsonl(manifest_path) - validate_instance_manifest(contract, manifest) - toolchain = load_json(toolchain_path) - verify_toolchain(contract, toolchain, check_paths=False) - matrix = load_json(matrix_path) - regenerated = build_matrix(contract, manifest, calibration_root, toolchain) - if matrix != regenerated: - raise VerificationError("run matrix does not regenerate from frozen inputs") - cells = selected_cells(matrix, scope) - missing = [] - reconstruction = Counter() - for cell in cells: - cell_dir = runs_root / "cells" / cell["cell_id"] - terminal_path = cell_dir / "terminal.json" - if not terminal_path.is_file(): - missing.append(cell["cell_id"]) - continue - reconstruction[ - verify_terminal( - contract, - matrix_path, - matrix, - toolchain, - cell, - load_json(terminal_path), - cell_dir, - ) - ] += 1 - if missing: - raise VerificationError( - f"missing terminal cells ({len(missing)}): {', '.join(missing[:3])}" - ) - thresholds = {} - for cell in matrix["cells"]: - if "cc_threshold" not in cell: - continue - key = (cell["product_width"], cell["method"], cell["budget"]) - previous = thresholds.setdefault(key, cell["cc_threshold"]) - if previous != cell["cc_threshold"]: - raise VerificationError("held-out cells contain per-instance threshold tuning") - messages = [ - f"PASS contract: {contract_sha256(contract)}", - "PASS widths/splits: factor-product semantics and calibration holdout are frozen", - "PASS toolchain/encoding: source revisions, executables, and global CNFs are fixed", - f"PASS completeness: {len(cells)} {scope} cells have terminal records", - ( - "PASS reconstruction: " - f"{reconstruction['conquer-records-reconstructed']} conquer cells reconstruct " - "per-cube work, span, scheduling, LPT replay, and hashes; " - f"{reconstruction['monolithic-stage-reconstructed'] + reconstruction['cubing-stage-reconstructed']} " - "stage-terminal cells reconstruct their available logs/metrics; " - f"{reconstruction['harness-terminal-only']} harness errors are explicitly " - "terminal-only and are not claimed as work/span reconstruction" - ), - "PASS tuning: every width/method/budget uses one calibration-frozen threshold", - ] - if aggregate_path is not None: - from benchmarks.cnc.aggregate_hard_regime import aggregate, load_terminals - - reported = load_json(aggregate_path) - regenerated_aggregate = aggregate( - contract, matrix, load_terminals(matrix, runs_root, scope), scope - ) - if reported != regenerated_aggregate: - raise VerificationError( - "aggregate does not regenerate from instance-level terminal records" - ) - if reported.get("statistical_unit") != "held-out-instance": - raise VerificationError("aggregate uses cube-level pseudoreplication") - for summary in reported.get("summaries", []): - declared_ids = summary.get("declared_instance_ids") - instance_ids = summary.get("instance_ids") - if not isinstance(declared_ids, list) or len(declared_ids) != len( - set(declared_ids) - ): - raise VerificationError( - "aggregate summary duplicates a declared held-out instance" - ) - if summary.get("declared_pairs") != len(declared_ids): - raise VerificationError( - "aggregate declared pair count does not match instance IDs" - ) - if not isinstance(instance_ids, list) or len(instance_ids) != len( - set(instance_ids) - ): - raise VerificationError( - "aggregate summary duplicates a held-out instance" - ) - if summary.get("complete_pairs") != len(instance_ids): - raise VerificationError( - "aggregate pair count does not match instance IDs" - ) - messages.append( - "PASS statistics: paired ratios and bootstrap samples use held-out instances" - ) - return messages - - -def main() -> int: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("contract", type=Path) - parser.add_argument("manifest", type=Path) - parser.add_argument("--calibration-root", type=Path, required=True) - parser.add_argument("--toolchain", type=Path, required=True) - parser.add_argument("--matrix", type=Path, required=True) - parser.add_argument("--runs-root", type=Path, required=True) - parser.add_argument("--scope", choices=("pilot", "full"), default="full") - parser.add_argument("--aggregate", type=Path) - args = parser.parse_args() - try: - messages = verify_bundle( - args.contract, - args.manifest, - args.calibration_root, - args.toolchain, - args.matrix, - args.runs_root, - args.scope, - args.aggregate, - ) - except (VerificationError, MatrixError, OSError, ValueError, json.JSONDecodeError) as exc: - print(f"FAIL: {exc}") - return 1 - print("\n".join(messages)) - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/benchmarks/cnc/verify_measurements.py b/benchmarks/cnc/verify_measurements.py deleted file mode 100644 index 47932db..0000000 --- a/benchmarks/cnc/verify_measurements.py +++ /dev/null @@ -1,460 +0,0 @@ -#!/usr/bin/env python3 -"""Verify a self-contained Cube-and-Conquer measurement evidence bundle.""" - -from __future__ import annotations - -import argparse -import hashlib -import itertools -import json -import re -from collections import defaultdict -from pathlib import Path - - -class BundleError(ValueError): - """The bundle is incomplete, inconsistent, or unauditable.""" - - -def load_json(path: Path) -> dict: - try: - value = json.loads(path.read_text(encoding="utf-8")) - except (OSError, json.JSONDecodeError) as exc: - raise BundleError(f"cannot read {path}: {exc}") from exc - if not isinstance(value, dict): - raise BundleError(f"{path}: expected a JSON object") - return value - - -def read_jsonl(path: Path) -> list[dict]: - records = [] - for line_number, line in enumerate( - path.read_text(encoding="utf-8").splitlines(), 1 - ): - if not line.strip(): - continue - value = json.loads(line) - if not isinstance(value, dict): - raise BundleError(f"{path}:{line_number}: expected a JSON object") - records.append(value) - return records - - -def bundle_path(root: Path, relative: object) -> Path: - if not isinstance(relative, str) or not relative: - raise BundleError("bundle artifact path must be a non-empty string") - root = root.resolve() - path = (root / relative).resolve() - if path != root and root not in path.parents: - raise BundleError(f"bundle artifact escapes its root: {relative}") - return path - - -def sha256_file(path: Path) -> str: - digest = hashlib.sha256() - with path.open("rb") as stream: - while chunk := stream.read(1024 * 1024): - digest.update(chunk) - return digest.hexdigest() - - -def checked_artifact(root: Path, spec: object, label: str) -> Path: - if not isinstance(spec, dict): - raise BundleError(f"manifest.{label} must be an object") - path = bundle_path(root, spec.get("path")) - expected = spec.get("sha256") - if not isinstance(expected, str) or not re.fullmatch(r"[0-9a-f]{64}", expected): - raise BundleError(f"manifest.{label}.sha256 must be a lowercase SHA-256") - if sha256_file(path) != expected: - raise BundleError(f"{label} SHA-256 mismatch") - return path - - -def parse_dimacs(path: Path) -> tuple[int, list[list[int]]]: - variables = None - clauses: list[list[int]] = [] - pending: list[int] = [] - for line in path.read_text(encoding="utf-8").splitlines(): - stripped = line.strip() - if not stripped or stripped.startswith("c"): - continue - if stripped.startswith("p "): - fields = stripped.split() - if len(fields) != 4 or fields[:2] != ["p", "cnf"]: - raise BundleError(f"{path}: malformed DIMACS header") - variables = int(fields[2]) - declared_clauses = int(fields[3]) - continue - for token in stripped.split(): - literal = int(token) - if literal == 0: - clauses.append(pending) - pending = [] - else: - pending.append(literal) - if variables is None or pending or len(clauses) != declared_clauses: - raise BundleError(f"{path}: malformed DIMACS body") - if any(abs(literal) > variables for clause in clauses for literal in clause): - raise BundleError(f"{path}: literal exceeds declared variable count") - return variables, clauses - - -def parse_cube(record: dict, variables: list[int]) -> tuple[str, dict[int, bool]]: - cube_id = record.get("cube_id") - literals = record.get("literals") - if not isinstance(cube_id, str) or not cube_id or not isinstance(literals, list): - raise BundleError("every frontier record needs cube_id and literals") - assignment: dict[int, bool] = {} - for literal in literals: - if not isinstance(literal, int) or literal == 0 or abs(literal) not in variables: - raise BundleError(f"cube {cube_id}: invalid literal {literal!r}") - variable = abs(literal) - value = literal > 0 - if variable in assignment and assignment[variable] != value: - raise BundleError(f"cube {cube_id}: contradictory literal for {variable}") - assignment[variable] = value - return cube_id, assignment - - -def verify_frontier(records: list[dict], variables: list[int]) -> tuple[int, int]: - cubes: list[tuple[str, dict[int, bool]]] = [] - ids: set[str] = set() - for record in records: - cube_id, cube = parse_cube(record, variables) - if cube_id in ids: - raise BundleError(f"duplicate cube id {cube_id!r}") - ids.add(cube_id) - cubes.append((cube_id, cube)) - total = 1 << len(variables) - covered = 0 - for bits in itertools.product((False, True), repeat=len(variables)): - assignment = dict(zip(variables, bits, strict=True)) - owners = [ - cube_id - for cube_id, cube in cubes - if all(assignment[var] == value for var, value in cube.items()) - ] - label = "".join("1" if bit else "0" for bit in bits) - if not owners: - raise BundleError(f"frontier: assignment {label} is uncovered") - if len(owners) > 1: - raise BundleError( - f"frontier: assignment {label} is covered by {len(owners)} cubes" - ) - covered += 1 - return covered, total - - -def solver_verdict(raw_output: object) -> str | None: - if not isinstance(raw_output, str): - raise BundleError("raw_solver_output must be a string") - statuses = { - line.strip().upper() - for line in raw_output.splitlines() - if line.strip().upper().startswith("S ") - } - mapping = { - "S SATISFIABLE": "sat", - "S UNSATISFIABLE": "unsat", - "S UNKNOWN": "unknown", - } - parsed = {mapping[status] for status in statuses if status in mapping} - if len(parsed) > 1: - raise BundleError("raw solver output contains contradictory verdicts") - return next(iter(parsed), None) - - -def number(value: float) -> str: - return str(int(value)) if value.is_integer() else str(value) - - -def verify_events( - events: list[dict], results: list[dict], cube_ids: set[str], worker_count: int -) -> dict[str, float]: - if [event.get("seq") for event in events] != list(range(len(events))): - raise BundleError("events must have contiguous sequence numbers from zero") - times = [event.get("monotonic_seconds") for event in events] - if not all(isinstance(value, (int, float)) for value in times): - raise BundleError("every event needs a numeric monotonic_seconds") - if any(left > right for left, right in zip(times, times[1:])): - raise BundleError("event timestamps are not monotonic") - - by_kind: dict[str, list[dict]] = defaultdict(list) - for event in events: - kind = event.get("event") - if not isinstance(kind, str): - raise BundleError("every event needs an event name") - by_kind[kind].append(event) - for marker in ("run_started", "cubing_started", "cubing_finished", "run_finished"): - if len(by_kind[marker]) != 1: - raise BundleError(f"event log needs exactly one {marker}") - - result_by_id: dict[str, dict] = {} - for record in results: - cube_id = record.get("cube_id") - termination = record.get("termination") - verdict = record.get("verdict") - cpu_seconds = record.get("cpu_seconds") - if not isinstance(cube_id, str) or cube_id in result_by_id: - raise BundleError(f"duplicate or malformed result cube id {cube_id!r}") - if termination not in {"solved", "cancelled", "timed_out", "never_started"}: - raise BundleError(f"cube {cube_id}: invalid termination state") - if verdict not in {"sat", "unsat", "unknown", None}: - raise BundleError(f"cube {cube_id}: invalid verdict") - if not isinstance(cpu_seconds, (int, float)) or cpu_seconds < 0: - raise BundleError(f"cube {cube_id}: invalid cpu_seconds") - parsed = solver_verdict(record.get("raw_solver_output")) - if parsed != verdict: - raise BundleError(f"cube {cube_id}: raw output disagrees with verdict") - if termination == "solved" and verdict not in {"sat", "unsat"}: - raise BundleError(f"cube {cube_id}: solved cube needs SAT or UNSAT verdict") - if termination != "solved" and verdict not in {"unknown", None}: - raise BundleError(f"cube {cube_id}: non-solved cube has definitive verdict") - result_by_id[cube_id] = record - if set(result_by_id) != cube_ids: - missing = sorted(cube_ids - set(result_by_id)) - extra = sorted(set(result_by_id) - cube_ids) - raise BundleError(f"result coverage mismatch: missing={missing}, extra={extra}") - - started: dict[str, dict] = {} - terminal: dict[str, dict] = {} - intervals = [] - for event in events: - if event["event"] not in {"cube_started", "cube_terminal"}: - continue - cube_id = event.get("cube_id") - worker = event.get("worker") - if cube_id not in cube_ids or not isinstance(worker, int) or not 0 <= worker < worker_count: - raise BundleError("cube event has invalid cube_id or worker") - target = started if event["event"] == "cube_started" else terminal - if cube_id in target: - raise BundleError(f"cube {cube_id}: duplicate {event['event']}") - target[cube_id] = event - - for cube_id, result in result_by_id.items(): - termination = result["termination"] - if termination == "never_started": - if cube_id in started or cube_id in terminal: - raise BundleError(f"cube {cube_id}: never-started cube has events") - continue - if cube_id not in started or cube_id not in terminal: - raise BundleError(f"cube {cube_id}: missing lifecycle event") - begin = started[cube_id] - end = terminal[cube_id] - if begin["worker"] != end["worker"] or end.get("termination") != termination: - raise BundleError(f"cube {cube_id}: inconsistent terminal event") - if begin["monotonic_seconds"] >= end["monotonic_seconds"]: - raise BundleError(f"cube {cube_id}: terminal event must follow start") - intervals.append( - ( - begin["monotonic_seconds"], - end["monotonic_seconds"], - cube_id, - begin["worker"], - ) - ) - - by_worker: dict[int, list[tuple[float, float, str]]] = defaultdict(list) - for begin, end, cube_id, worker in intervals: - by_worker[worker].append((begin, end, cube_id)) - for worker, work in by_worker.items(): - previous_end = None - for begin, end, cube_id in sorted(work): - if previous_end is not None and begin < previous_end: - raise BundleError( - f"worker {worker} runs overlapping cube {cube_id}" - ) - previous_end = end - - points = [] - for begin, end, _, _ in intervals: - points.extend(((begin, 1), (end, -1))) - active = maximum = 0 - for _, delta in sorted(points, key=lambda item: (item[0], item[1])): - active += delta - if active < 0: - raise BundleError("worker interval accounting became negative") - maximum = max(maximum, active) - if maximum > worker_count: - raise BundleError( - f"observed concurrency {maximum} exceeds worker limit {worker_count}" - ) - - run_start = float(by_kind["run_started"][0]["monotonic_seconds"]) - run_end = float(by_kind["run_finished"][0]["monotonic_seconds"]) - cube_start = float(by_kind["cubing_started"][0]["monotonic_seconds"]) - cube_end = float(by_kind["cubing_finished"][0]["monotonic_seconds"]) - cpu_start = by_kind["cubing_started"][0].get("process_cpu_seconds") - cpu_end = by_kind["cubing_finished"][0].get("process_cpu_seconds") - if not isinstance(cpu_start, (int, float)) or not isinstance(cpu_end, (int, float)): - raise BundleError("cubing events need process_cpu_seconds counters") - if not run_start <= cube_start <= cube_end <= run_end or cpu_end < cpu_start: - raise BundleError("run or cubing counters are not properly nested") - conquer_start = min((begin for begin, _, _, _ in intervals), default=cube_end) - conquer_end = max((end for _, end, _, _ in intervals), default=cube_end) - if conquer_start < cube_end or conquer_end > run_end: - raise BundleError("conquer intervals fall outside the scheduled run phase") - metrics = { - "cubing_wall": cube_end - cube_start, - "cubing_cpu": float(cpu_end - cpu_start), - "conquer_cpu": float(sum(record["cpu_seconds"] for record in results)), - "conquer_makespan": float(conquer_end - conquer_start), - "end_to_end_wall": run_end - run_start, - } - metrics["orchestration"] = ( - metrics["end_to_end_wall"] - - metrics["cubing_wall"] - - metrics["conquer_makespan"] - ) - if any(value < 0 for value in metrics.values()): - raise BundleError("derived accounting contains a negative duration") - metrics["maximum_concurrency"] = float(maximum) - return metrics - - -def verify_model( - clauses: list[list[int]], model: list[int], variables: int -) -> dict[int, bool]: - assignment: dict[int, bool] = {} - for literal in model: - if not isinstance(literal, int) or literal == 0 or abs(literal) > variables: - raise BundleError(f"witness contains invalid literal {literal!r}") - variable = abs(literal) - value = literal > 0 - if variable in assignment and assignment[variable] != value: - raise BundleError(f"witness contradicts variable {variable}") - assignment[variable] = value - missing = set(range(1, variables + 1)) - assignment.keys() - if missing: - raise BundleError(f"witness omits variables: {sorted(missing)}") - if not all( - any(assignment[abs(literal)] == (literal > 0) for literal in clause) - for clause in clauses - ): - raise BundleError("returned model does not satisfy the input") - return assignment - - -def verify(bundle: Path) -> list[str]: - manifest = load_json(bundle / "bundle.json") - if manifest.get("format_version") != 1: - raise BundleError("unsupported bundle format_version") - worker_count = manifest.get("worker_count") - if not isinstance(worker_count, int) or worker_count < 1: - raise BundleError("worker_count must be positive") - for key in ("scheduler", "termination_policy"): - if not isinstance(manifest.get(key), str) or not manifest[key]: - raise BundleError(f"manifest needs {key}") - provenance = manifest.get("provenance") - if not isinstance(provenance, dict): - raise BundleError("manifest needs provenance") - for tool in ("cuber", "conquer"): - record = provenance.get(tool) - if not isinstance(record, dict) or not all( - isinstance(record.get(key), str) and record[key] - for key in ("id", "version", "path", "executable_sha256") - ): - raise BundleError(f"provenance needs complete {tool} identity") - if not re.fullmatch(r"[0-9a-f]{64}", record["executable_sha256"]): - raise BundleError(f"{tool} executable_sha256 is malformed") - executable = bundle_path(bundle, record["path"]) - if sha256_file(executable) != record["executable_sha256"]: - raise BundleError(f"{tool} executable SHA-256 mismatch") - - input_path = checked_artifact(bundle, manifest.get("input"), "input") - frontier_path = checked_artifact(bundle, manifest.get("frontier"), "frontier") - events_path = checked_artifact(bundle, manifest.get("events"), "events") - results_path = checked_artifact(bundle, manifest.get("results"), "results") - variables, clauses = parse_dimacs(input_path) - frontier_spec = manifest["frontier"] - frontier_variables = frontier_spec.get("variables") - if frontier_spec.get("mode") != "exhaustive" or not isinstance( - frontier_variables, list - ): - raise BundleError("frontier must declare exhaustive mode and variables") - if frontier_variables != list(range(1, variables + 1)): - raise BundleError("exhaustive frontier variables must match the input") - frontier = read_jsonl(frontier_path) - covered, total = verify_frontier(frontier, frontier_variables) - results = read_jsonl(results_path) - events = read_jsonl(events_path) - cube_ids = {record["cube_id"] for record in frontier} - metrics = verify_events(events, results, cube_ids, worker_count) - - declared = manifest.get("accounting") - if not isinstance(declared, dict): - raise BundleError("manifest needs declared accounting") - for key in ( - "cubing_wall", - "cubing_cpu", - "conquer_cpu", - "conquer_makespan", - "orchestration", - "end_to_end_wall", - ): - if declared.get(key) != metrics[key]: - raise BundleError( - f"accounting mismatch for {key}: declared={declared.get(key)!r}, " - f"derived={metrics[key]!r}" - ) - - verdicts = {record["verdict"] for record in results} - if "sat" in verdicts: - aggregate = "sat" - elif all(record["termination"] == "solved" for record in results) and verdicts == {"unsat"}: - aggregate = "unsat" - else: - aggregate = "unknown" - if manifest.get("aggregate_verdict") != aggregate: - raise BundleError("aggregate verdict disagrees with cube records") - verdict_message = "all cube outcomes justify the aggregate verdict" - if aggregate == "sat": - witness_path = checked_artifact(bundle, manifest.get("witness"), "witness") - witness = load_json(witness_path).get("model") - if not isinstance(witness, list): - raise BundleError("witness needs a model list") - model = verify_model(clauses, witness, variables) - model_owner = next( - ( - cube_id - for cube_id, cube in ( - parse_cube(record, frontier_variables) for record in frontier - ) - if all(model[variable] == value for variable, value in cube.items()) - ), - None, - ) - sat_cubes = { - record["cube_id"] for record in results if record["verdict"] == "sat" - } - if model_owner not in sat_cubes: - raise BundleError("returned model does not belong to a SAT cube") - verdict_message = "returned model satisfies the input" - - return [ - f"PASS frontier: {covered}/{total} assignments are covered exactly once", - f"PASS verdict: {verdict_message}", - "PASS accounting: " - f"cubing_wall={number(metrics['cubing_wall'])} " - f"conquer_cpu={number(metrics['conquer_cpu'])} " - f"conquer_makespan={number(metrics['conquer_makespan'])} " - f"end_to_end_wall={number(metrics['end_to_end_wall'])}", - f"PASS workers: observed concurrency does not exceed {worker_count}", - ] - - -def main() -> int: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--bundle", type=Path, required=True) - args = parser.parse_args() - try: - messages = verify(args.bundle) - except (BundleError, OSError, json.JSONDecodeError, ValueError) as exc: - print(f"FAIL {exc}") - return 1 - print("\n".join(messages)) - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/benchmarks/pipeline/multipliers.py b/benchmarks/pipeline/multipliers.py index 87c248a..c8c4927 100644 --- a/benchmarks/pipeline/multipliers.py +++ b/benchmarks/pipeline/multipliers.py @@ -16,6 +16,9 @@ var, ) +DETERMINISTIC_MILLER_RABIN_LIMIT = 318_665_857_834_031_151_167_461 + + def is_prime(value: int) -> bool: if value < 2: return False @@ -28,8 +31,16 @@ def is_prime(value: int) -> bool: while odd % 2 == 0: odd //= 2 power += 1 - # Deterministic for unsigned 64-bit integers; still a strong fixed-base test above it. - for base in (2, 325, 9375, 28178, 450775, 9780504, 1795265022): + if value < (1 << 64): + bases = (2, 325, 9375, 28178, 450775, 9780504, 1795265022) + else: + # Deterministic below DETERMINISTIC_MILLER_RABIN_LIMIT. This covers + # every product of two factors up to 39 bits, including the 34-bit + # scale-extension corpus. Above the bound it remains a strong + # fixed-base probable-prime test, so scientific corpus generators + # must reject values outside the certified range. + bases = small + for base in bases: if base % value == 0: continue witness = pow(base, odd, value) diff --git a/src/bin/cnc_cuber.rs b/src/bin/cnc_cuber.rs index 9bfa5ee..0ec18b6 100644 --- a/src/bin/cnc_cuber.rs +++ b/src/bin/cnc_cuber.rs @@ -1,4 +1,4 @@ -//! Generate Cube-and-Conquer assumptions with the current Rust region cuber. +//! Export a complete cube frontier or stream it into parallel Kissat workers. //! //! The primary stopping rule is the classical online Cube-and-Conquer //! difficulty cutoff (`--cc-threshold`). A march-compatible remaining-variable @@ -11,6 +11,7 @@ use std::path::{Path, PathBuf}; use boolean_inference::adapter::BranchSolver; use boolean_inference::circuit::network_from_circuit_sat; +use boolean_inference::conquer::{ConquerResult, StreamingConquer}; use boolean_inference::cube::{ generate_cubes_with_cutoff, generate_cubes_with_cutoff_trace, CubeCutoff, CubeNodeKind, CubeNodeTrace, CubeRefutationReason, @@ -22,8 +23,11 @@ use boolean_inference::problem::TnProblem; use boolean_inference::selector::Selector; use optimal_branching_core::GreedyMerge; -const USAGE: &str = "usage: cnc_cuber (-n | --cc-threshold ) -o \ +const USAGE: &str = + "usage: cnc_cuber (-n | --cc-threshold ) \ + (-o | --solve-cnf --kissat --workers ) \ [--selector ] [--max-rows ] [--trace ]"; +const SOLVED: &str = "streaming-conquer-found-sat"; #[derive(Clone, Copy, Debug)] enum SelectorKind { @@ -52,7 +56,10 @@ impl SelectorKind { struct Args { input: PathBuf, - output: PathBuf, + output: Option, + solve_cnf: Option, + kissat: Option, + workers: Option, cutoff: CubeCutoff, selector: SelectorKind, max_rows: usize, @@ -75,6 +82,9 @@ fn parse_args() -> Result { let raw: Vec = std::env::args().skip(1).collect(); let mut input = None; let mut output = None; + let mut solve_cnf = None; + let mut kissat = None; + let mut workers = None; let mut cutoff_vars = None; let mut cc_threshold = None; let mut max_rows = 512usize; @@ -104,6 +114,18 @@ fn parse_args() -> Result { ); } "-o" => output = Some(take_value(&raw, &mut i, "-o")?), + "--solve-cnf" => solve_cnf = Some(take_value(&raw, &mut i, "--solve-cnf")?), + "--kissat" => kissat = Some(take_value(&raw, &mut i, "--kissat")?), + "--workers" => { + let value = take_value(&raw, &mut i, "--workers")?; + let count = value + .parse::() + .map_err(|_| format!("invalid --workers value: {value}"))?; + if count == 0 { + return Err("--workers must be greater than zero".into()); + } + workers = Some(count); + } "--trace" => trace = Some(take_value(&raw, &mut i, "--trace")?), "--selector" => { selector = SelectorKind::parse(&take_value(&raw, &mut i, "--selector")?)?; @@ -134,9 +156,17 @@ fn parse_args() -> Result { return Err("-n and --cc-threshold are mutually exclusive".to_string()) } }; + match (&output, &solve_cnf, &kissat, workers) { + (Some(_), None, None, None) => {} + (None, Some(_), Some(_), Some(_)) => {} + _ => return Err("select either -o, or all of --solve-cnf/--kissat/--workers".to_string()), + } Ok(Command::Run(Args { input: PathBuf::from(input.ok_or_else(|| "missing input instance".to_string())?), - output: PathBuf::from(output.ok_or_else(|| "missing -o output".to_string())?), + output: output.map(PathBuf::from), + solve_cnf: solve_cnf.map(PathBuf::from), + kissat: kissat.map(PathBuf::from), + workers, cutoff, selector, max_rows, @@ -234,8 +264,12 @@ fn write_trace_node( .map_err(|error| format!("write trace: {error}")) } -fn run(args: Args) -> Result<(), String> { - if args.trace.as_deref() == Some(args.output.as_path()) { +fn run(args: Args) -> Result { + if args + .trace + .as_ref() + .is_some_and(|trace| args.output.as_ref() == Some(trace)) + { return Err("--trace must differ from the cube output path".into()); } let network = load_network(&args.input)?; @@ -253,7 +287,16 @@ fn run(args: Args) -> Result<(), String> { return Err("constraint network has an incomplete variable map".into()); } - let mut writer = output_writer(&args.output)?; + let mut writer: Box = match &args.output { + Some(output) => output_writer(output)?, + None => Box::new(io::sink()), + }; + let mut conquer = match (&args.solve_cnf, &args.kissat, args.workers) { + (Some(cnf), Some(kissat), Some(workers)) => { + Some(StreamingConquer::start(cnf, kissat, workers).map_err(|error| error.to_string())?) + } + _ => None, + }; let mut trace_writer = match &args.trace { Some(path) => { Some(BufWriter::new(File::create(path).map_err(|error| { @@ -293,7 +336,13 @@ fn run(args: Args) -> Result<(), String> { "status=UNSAT_AT_ROOT cubes=0 refuted=1 sat_leaves=0 cutoff={:?}", args.cutoff ); - return Ok(()); + if let Some(conquer) = conquer.take() { + let summary = conquer.finish(true).map_err(|error| error.to_string())?; + debug_assert_eq!(summary.result, ConquerResult::Unsat); + println!("s UNSATISFIABLE"); + return Ok(20); + } + return Ok(0); } }; let root_unfixed = problem.count_unfixed(); @@ -309,10 +358,18 @@ fn run(args: Args) -> Result<(), String> { }; let solver = BranchSolver::Greedy(GreedyMerge); let mut emit = |cube: boolean_inference::cube::Cube| { - if cube.refuted || cube.sat { + if cube.refuted { return Ok(()); } + let leaf_sat = cube.sat; + if leaf_sat { + if let Some(conquer) = conquer.as_ref() { + conquer.mark_sat(); + return Err(SOLVED.to_string()); + } + } + let remaining = nvars - cube.sigma_all; let stopped = match args.cutoff { CubeCutoff::RemainingVars(n) => remaining < n.get(), @@ -321,31 +378,44 @@ fn run(args: Args) -> Result<(), String> { > threshold * (nvars as u128) } }; - if !stopped { + if !leaf_sat && !stopped { return Err(format!( "internal cutoff error: emitted cube does not satisfy {:?}", args.cutoff )); } - writer - .write_all(b"a") - .map_err(|e| format!("write output: {e}"))?; + let mut literals = Vec::with_capacity(cube.decisions.len()); for &(compressed, value) in &cube.decisions { let literal = (new_to_orig[compressed] + 1) as i64; let literal = if value { literal } else { -literal }; - write!(writer, " {literal}").map_err(|e| format!("write output: {e}"))?; + literals.push(literal); + } + if let Some(conquer) = conquer.as_ref() { + if !conquer + .submit(literals) + .map_err(|error| error.to_string())? + { + return Err(SOLVED.to_string()); + } + } else { + writer + .write_all(b"a") + .map_err(|e| format!("write output: {e}"))?; + for literal in literals { + write!(writer, " {literal}").map_err(|e| format!("write output: {e}"))?; + } + writer + .write_all(b" 0\n") + .map_err(|e| format!("write output: {e}"))?; } - writer - .write_all(b" 0\n") - .map_err(|e| format!("write output: {e}"))?; emitted += 1; min_remaining = min_remaining.min(remaining); max_remaining = max_remaining.max(remaining); Ok(()) }; - let stats = match trace_writer.as_mut() { + let generated = match trace_writer.as_mut() { Some(trace_writer) => generate_cubes_with_cutoff_trace( &mut problem, selector, @@ -363,7 +433,18 @@ fn run(args: Args) -> Result<(), String> { args.cutoff, &mut emit, ), - }?; + }; + let stopped_on_sat = matches!(&generated, Err(error) if error == SOLVED); + let stats = match generated { + Ok(stats) => Some(stats), + Err(error) if error == SOLVED => None, + Err(error) => { + if let Some(conquer) = conquer.take() { + let _ = conquer.finish(false); + } + return Err(error); + } + }; writer.flush().map_err(|e| format!("flush output: {e}"))?; if let Some(trace_writer) = trace_writer.as_mut() { trace_writer @@ -376,38 +457,74 @@ fn run(args: Args) -> Result<(), String> { } else { format!("{min_remaining}..={max_remaining}") }; - eprintln!( - "status=OK cubes={} refuted={} sat_leaves={} visited={} cutoff={:?} \ - root_unfixed={} remaining_range={} selector={} max_rows={}", - stats.cubes, - stats.refuted, - stats.sat_leaves, - stats.visited, - args.cutoff, - root_unfixed, - remaining_range, - args.selector.label(), - args.max_rows - ); - if emitted != stats.cubes { - return Err(format!( - "internal accounting error: wrote {emitted} cubes, expected {}", - stats.cubes - )); + if let Some(stats) = stats { + eprintln!( + "status=OK cubes={} refuted={} sat_leaves={} visited={} cutoff={:?} \ + root_unfixed={} remaining_range={} selector={} max_rows={}", + stats.cubes, + stats.refuted, + stats.sat_leaves, + stats.visited, + args.cutoff, + root_unfixed, + remaining_range, + args.selector.label(), + args.max_rows + ); + let expected = stats.cubes + stats.sat_leaves; + if emitted != expected { + return Err(format!( + "internal accounting error: wrote {emitted} cubes, expected {}", + expected + )); + } + } else { + eprintln!( + "status=SAT_EARLY cubes_submitted={} cutoff={:?} selector={}", + emitted, + args.cutoff, + args.selector.label() + ); + } + if let Some(conquer) = conquer.take() { + let summary = conquer + .finish(!stopped_on_sat) + .map_err(|error| error.to_string())?; + eprintln!( + "streaming submitted={} sat={} unsat={} errors={}", + summary.submitted, summary.sat, summary.unsat, summary.errors + ); + return match summary.result { + ConquerResult::Sat => { + if let Some(witness) = summary.witness { + print!("{witness}"); + } else { + println!("s SATISFIABLE"); + } + Ok(10) + } + ConquerResult::Unsat => { + println!("s UNSATISFIABLE"); + Ok(20) + } + ConquerResult::Incomplete => Err("streaming conquer was incomplete".into()), + }; } - Ok(()) + Ok(0) } fn main() { match parse_args() { Ok(Command::Help) => println!("{USAGE}"), - Ok(Command::Run(args)) => { - if let Err(message) = run(args) { + Ok(Command::Run(args)) => match run(args) { + Ok(0) => {} + Ok(code) => std::process::exit(code), + Err(message) => { eprintln!("error: {message}"); eprintln!("{USAGE}"); std::process::exit(2); } - } + }, Err(message) => { eprintln!("error: {message}"); eprintln!("{USAGE}"); diff --git a/src/conquer.rs b/src/conquer.rs new file mode 100644 index 0000000..09d2eda --- /dev/null +++ b/src/conquer.rs @@ -0,0 +1,313 @@ +//! Bounded streaming conquer pool for Cube-and-Conquer frontiers. + +use std::fs; +use std::io::{BufRead, BufReader, Write}; +use std::path::{Path, PathBuf}; +use std::process::{Command, Stdio}; +use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; +use std::sync::{mpsc, Arc, Mutex}; +use std::thread::{self, JoinHandle}; + +#[derive(Debug, thiserror::Error)] +pub enum ConquerError { + #[error("read CNF {path}: {source}")] + ReadCnf { + path: PathBuf, + source: std::io::Error, + }, + #[error("CNF has no valid 'p cnf ' header")] + MissingHeader, + #[error("start Kissat: {0}")] + StartKissat(std::io::Error), + #[error("write CNF to Kissat: {0}")] + WriteKissat(std::io::Error), + #[error("read Kissat output: {0}")] + ReadKissat(std::io::Error), + #[error("streaming conquer worker disconnected")] + Disconnected, + #[error("streaming conquer worker panicked")] + WorkerPanicked, +} + +#[derive(Clone)] +struct CnfTemplate { + variables: usize, + clauses: usize, + body: Arc>, +} + +impl CnfTemplate { + fn read(path: &Path) -> Result { + let text = fs::read_to_string(path).map_err(|source| ConquerError::ReadCnf { + path: path.to_owned(), + source, + })?; + let mut header = None; + let mut body = Vec::with_capacity(text.len()); + for line in text.split_inclusive('\n') { + let fields: Vec<_> = line.split_whitespace().collect(); + if fields.first() == Some(&"p") { + if fields.len() != 4 || fields[1] != "cnf" { + return Err(ConquerError::MissingHeader); + } + let variables = fields[2].parse().map_err(|_| ConquerError::MissingHeader)?; + let clauses = fields[3].parse().map_err(|_| ConquerError::MissingHeader)?; + if header.replace((variables, clauses)).is_some() { + return Err(ConquerError::MissingHeader); + } + } else { + body.extend_from_slice(line.as_bytes()); + } + } + let (variables, clauses) = header.ok_or(ConquerError::MissingHeader)?; + Ok(Self { + variables, + clauses, + body: Arc::new(body), + }) + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ConquerResult { + Sat, + Unsat, + Incomplete, +} + +#[derive(Debug)] +pub struct ConquerSummary { + pub result: ConquerResult, + pub submitted: usize, + pub sat: usize, + pub unsat: usize, + pub errors: usize, + pub witness: Option, +} + +struct Shared { + stopped: AtomicBool, + submitted: AtomicUsize, + sat: AtomicUsize, + unsat: AtomicUsize, + errors: AtomicUsize, + witness: Mutex>, +} + +impl Shared { + fn new() -> Self { + Self { + stopped: AtomicBool::new(false), + submitted: AtomicUsize::new(0), + sat: AtomicUsize::new(0), + unsat: AtomicUsize::new(0), + errors: AtomicUsize::new(0), + witness: Mutex::new(None), + } + } +} + +/// A fixed-size Kissat pool fed directly by the cuber's leaf callback. +pub struct StreamingConquer { + sender: Option>>, + shared: Arc, + workers: Vec>, +} + +impl StreamingConquer { + pub fn start(cnf: &Path, kissat: &Path, workers: usize) -> Result { + assert!(workers > 0, "workers must be positive"); + let template = CnfTemplate::read(cnf)?; + let shared = Arc::new(Shared::new()); + let (sender, receiver) = mpsc::sync_channel(workers); + let receiver = Arc::new(Mutex::new(receiver)); + let mut handles = Vec::with_capacity(workers); + for _ in 0..workers { + let receiver = Arc::clone(&receiver); + let shared = Arc::clone(&shared); + let template = template.clone(); + let kissat = kissat.to_owned(); + handles.push(thread::spawn(move || { + worker_loop(receiver, shared, template, kissat) + })); + } + Ok(Self { + sender: Some(sender), + shared, + workers: handles, + }) + } + + /// Submit one open cube. Returns `false` once another cube has proved SAT. + pub fn submit(&self, cube: Vec) -> Result { + if self.shared.stopped.load(Ordering::Acquire) { + return Ok(false); + } + let sent = self + .sender + .as_ref() + .ok_or(ConquerError::Disconnected)? + .send(cube); + if sent.is_err() { + return if self.shared.stopped.load(Ordering::Acquire) { + Ok(false) + } else { + Err(ConquerError::Disconnected) + }; + } + self.shared.submitted.fetch_add(1, Ordering::Relaxed); + Ok(!self.shared.stopped.load(Ordering::Acquire)) + } + + /// Record a satisfying leaf found by the cuber itself. + pub fn mark_sat(&self) { + self.shared.sat.fetch_add(1, Ordering::Relaxed); + self.shared.stopped.store(true, Ordering::Release); + } + + pub fn finish(mut self, cubing_complete: bool) -> Result { + self.sender.take(); + for worker in self.workers.drain(..) { + worker.join().map_err(|_| ConquerError::WorkerPanicked)?; + } + let submitted = self.shared.submitted.load(Ordering::Relaxed); + let sat = self.shared.sat.load(Ordering::Relaxed); + let unsat = self.shared.unsat.load(Ordering::Relaxed); + let errors = self.shared.errors.load(Ordering::Relaxed); + let result = if sat > 0 { + ConquerResult::Sat + } else if cubing_complete && errors == 0 && unsat == submitted { + ConquerResult::Unsat + } else { + ConquerResult::Incomplete + }; + let witness = self.shared.witness.lock().expect("witness lock").take(); + Ok(ConquerSummary { + result, + submitted, + sat, + unsat, + errors, + witness, + }) + } +} + +fn worker_loop( + receiver: Arc>>>, + shared: Arc, + template: CnfTemplate, + kissat: PathBuf, +) { + loop { + if shared.stopped.load(Ordering::Acquire) { + break; + } + let cube = match receiver.lock().expect("cube receiver lock").recv() { + Ok(cube) => cube, + Err(_) => break, + }; + if shared.stopped.load(Ordering::Acquire) { + break; + } + match solve_cube(&template, &kissat, &shared, &cube) { + Ok(CubeResult::Sat(output)) => { + shared.sat.fetch_add(1, Ordering::Relaxed); + *shared.witness.lock().expect("witness lock") = Some(output); + shared.stopped.store(true, Ordering::Release); + } + Ok(CubeResult::Unsat) => { + shared.unsat.fetch_add(1, Ordering::Relaxed); + } + Ok(CubeResult::Cancelled) => {} + Ok(CubeResult::Unknown) | Err(_) => { + shared.errors.fetch_add(1, Ordering::Relaxed); + } + } + } +} + +enum CubeResult { + Sat(String), + Unsat, + Cancelled, + Unknown, +} + +fn solve_cube( + template: &CnfTemplate, + kissat: &Path, + shared: &Shared, + cube: &[i64], +) -> Result { + if cube + .iter() + .any(|literal| literal.unsigned_abs() as usize > template.variables) + { + return Ok(CubeResult::Unknown); + } + let mut child = Command::new(kissat) + .arg("--relaxed") + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::null()) + .spawn() + .map_err(ConquerError::StartKissat)?; + let stdout = child.stdout.take().expect("piped Kissat stdout"); + let reader = thread::spawn(move || read_solution(BufReader::new(stdout))); + let write_result = (|| { + let mut stdin = child.stdin.take().expect("piped Kissat stdin"); + writeln!( + stdin, + "p cnf {} {}", + template.variables, + template.clauses + cube.len() + ) + .and_then(|_| stdin.write_all(&template.body)) + .map_err(ConquerError::WriteKissat)?; + if !template.body.ends_with(b"\n") { + stdin.write_all(b"\n").map_err(ConquerError::WriteKissat)?; + } + for literal in cube { + writeln!(stdin, "{literal} 0").map_err(ConquerError::WriteKissat)?; + } + Ok(()) + })(); + if let Err(error) = write_result { + let _ = child.kill(); + let _ = child.wait(); + let _ = reader.join(); + return Err(error); + } + + let status = loop { + if shared.stopped.load(Ordering::Acquire) { + let _ = child.kill(); + let _ = child.wait(); + reader.join().expect("Kissat output reader panicked")?; + return Ok(CubeResult::Cancelled); + } + if let Some(status) = child.try_wait().map_err(ConquerError::StartKissat)? { + break status; + } + thread::sleep(std::time::Duration::from_millis(5)); + }; + let solution = reader.join().expect("Kissat output reader panicked")?; + match status.code() { + Some(10) => Ok(CubeResult::Sat(solution)), + Some(20) => Ok(CubeResult::Unsat), + _ => Ok(CubeResult::Unknown), + } +} + +fn read_solution(reader: impl BufRead) -> Result { + let mut solution = String::new(); + for line in reader.lines() { + let line = line.map_err(ConquerError::ReadKissat)?; + if line.starts_with("s ") || line.starts_with("v ") { + solution.push_str(&line); + solution.push('\n'); + } + } + Ok(solution) +} diff --git a/src/lib.rs b/src/lib.rs index e41c58c..981c463 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -2,6 +2,7 @@ pub mod adapter; pub mod api; pub mod canonicalize; pub mod circuit; +pub mod conquer; pub mod contract; pub mod ct; pub mod cube; diff --git a/tests/cnc_streaming.rs b/tests/cnc_streaming.rs new file mode 100644 index 0000000..6c81b9d --- /dev/null +++ b/tests/cnc_streaming.rs @@ -0,0 +1,151 @@ +#![cfg(unix)] + +use std::fs; +use std::os::unix::fs::PermissionsExt; +use std::path::{Path, PathBuf}; +use std::process::Command; +use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; + +use boolean_inference::conquer::{ConquerResult, StreamingConquer}; + +fn temp_dir() -> PathBuf { + let nonce = SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("clock after epoch") + .as_nanos(); + std::env::temp_dir().join(format!( + "boolean-inference-streaming-{}-{nonce}", + std::process::id() + )) +} + +fn fake_kissat(path: &Path, result: &str, code: i32) { + fs::write( + path, + format!( + "#!/bin/sh\n\ + [ \"$#\" -eq 1 ] && [ \"$1\" = --relaxed ] || exit 3\n\ + input=$(cat)\n\ + case \"$input\" in *\"p cnf 2 \"*) ;; *) exit 4 ;; esac\n\ + echo 'c output that streaming mode must discard'\n\ + echo 's {result}'\n\ + exit {code}\n" + ), + ) + .expect("write fake Kissat"); + let mut permissions = fs::metadata(path).unwrap().permissions(); + permissions.set_mode(0o755); + fs::set_permissions(path, permissions).unwrap(); +} + +fn run_streaming(dir: &Path, kissat: &Path) -> std::process::Output { + let cnf = dir.join("input.cnf"); + fs::write(&cnf, "p cnf 2 2\n1 2 0\n-1 -2 0\n").expect("write CNF"); + Command::new(env!("CARGO_BIN_EXE_cnc_cuber")) + .args([ + cnf.as_os_str(), + "-n".as_ref(), + "2".as_ref(), + "--solve-cnf".as_ref(), + cnf.as_os_str(), + "--kissat".as_ref(), + kissat.as_os_str(), + "--workers".as_ref(), + "2".as_ref(), + ]) + .output() + .expect("run streaming solver") +} + +#[test] +fn streaming_mode_reports_unsat_after_all_open_cubes_close() { + let dir = temp_dir(); + fs::create_dir_all(&dir).unwrap(); + let kissat = dir.join("kissat-unsat"); + fake_kissat(&kissat, "UNSATISFIABLE", 20); + + let output = run_streaming(&dir, &kissat); + + assert_eq!(output.status.code(), Some(20), "{output:?}"); + assert!(String::from_utf8_lossy(&output.stdout).contains("s UNSATISFIABLE")); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!(stderr.contains("submitted=1"), "{stderr}"); + assert!(stderr.contains("unsat=1"), "{stderr}"); + fs::remove_dir_all(dir).unwrap(); +} + +#[test] +fn streaming_mode_stops_after_a_sat_cube() { + let dir = temp_dir(); + fs::create_dir_all(&dir).unwrap(); + let kissat = dir.join("kissat-sat"); + fake_kissat(&kissat, "SATISFIABLE", 10); + + let output = run_streaming(&dir, &kissat); + + assert_eq!(output.status.code(), Some(10), "{output:?}"); + let stdout = String::from_utf8_lossy(&output.stdout); + assert!(stdout.contains("s SATISFIABLE")); + assert!(!stdout.contains("output that streaming mode must discard")); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!(stderr.contains("sat=1"), "{stderr}"); + fs::remove_dir_all(dir).unwrap(); +} + +#[test] +fn streaming_mode_kills_an_inflight_solver_after_sat() { + let dir = temp_dir(); + fs::create_dir_all(&dir).unwrap(); + let cnf = dir.join("input.cnf"); + fs::write(&cnf, "p cnf 2 0\n").unwrap(); + let started = dir.join("slow-started"); + let kissat = dir.join("kissat-race"); + fs::write( + &kissat, + format!( + "#!/bin/sh\n\ + input=$(cat)\n\ + case \"$input\" in\n\ + *'-1 0'*) while [ ! -e '{}' ]; do sleep 0.01; done; echo 's SATISFIABLE'; exit 10 ;;\n\ + *) touch '{}'; exec sleep 30 ;;\n\ + esac\n", + started.display(), + started.display(), + ), + ) + .unwrap(); + let mut permissions = fs::metadata(&kissat).unwrap().permissions(); + permissions.set_mode(0o755); + fs::set_permissions(&kissat, permissions).unwrap(); + + let conquer = StreamingConquer::start(&cnf, &kissat, 2).unwrap(); + assert!(conquer.submit(vec![1]).unwrap()); + assert!(conquer.submit(vec![-1]).unwrap()); + let before = Instant::now(); + let summary = conquer.finish(true).unwrap(); + + assert_eq!(summary.result, ConquerResult::Sat); + assert_eq!(summary.sat, 1); + assert!(started.exists(), "slow Kissat never started"); + assert!( + before.elapsed() < Duration::from_secs(10), + "in-flight Kissat was not killed: {:?}", + before.elapsed() + ); + fs::remove_dir_all(dir).unwrap(); +} + +#[test] +fn streaming_mode_never_claims_unsat_after_a_worker_error() { + let dir = temp_dir(); + fs::create_dir_all(&dir).unwrap(); + let kissat = dir.join("kissat-error"); + fake_kissat(&kissat, "UNKNOWN", 1); + + let output = run_streaming(&dir, &kissat); + + assert_eq!(output.status.code(), Some(2), "{output:?}"); + assert!(!String::from_utf8_lossy(&output.stdout).contains("UNSATISFIABLE")); + assert!(String::from_utf8_lossy(&output.stderr).contains("incomplete")); + fs::remove_dir_all(dir).unwrap(); +} diff --git a/tests/test_aggregate_hard_regime.py b/tests/test_aggregate_hard_regime.py deleted file mode 100644 index 81c015e..0000000 --- a/tests/test_aggregate_hard_regime.py +++ /dev/null @@ -1,91 +0,0 @@ -import unittest - -from benchmarks.cnc.aggregate_hard_regime import ( - adjusted_observations, - bootstrap_ci, - geometric_mean, - interpolate_log, - summarize_observations, -) - - -class HardRegimeAggregateTests(unittest.TestCase): - def test_log_interpolation_has_no_extrapolation(self): - points = [(100.0, 10.0), (400.0, 40.0)] - self.assertAlmostEqual(interpolate_log(points, 200.0), 20.0) - self.assertIsNone(interpolate_log(points, 50.0)) - self.assertIsNone(interpolate_log(points, 800.0)) - - def test_geometric_mean_and_bootstrap_are_deterministic(self): - self.assertAlmostEqual(geometric_mean([0.5, 2.0]), 1.0) - first = bootstrap_ci([0.8, 1.0, 1.2], 100, 0.95, 51, "work:low") - second = bootstrap_ci([0.8, 1.0, 1.2], 100, 0.95, 51, "work:low") - self.assertEqual(first, second) - - def test_summary_counts_instances_not_cubes(self): - observations = [ - { - "analysis": "raw-nominal-budget", - "metric": "conquer_work_cpu_s", - "budget": "low", - "product_width": 64, - "instance_id": "a", - "complete_pair": True, - "ratio": 0.8, - }, - { - "analysis": "raw-nominal-budget", - "metric": "conquer_work_cpu_s", - "budget": "low", - "product_width": 64, - "instance_id": "b", - "complete_pair": False, - "ratio": None, - }, - ] - summary = summarize_observations( - observations, - "raw-nominal-budget", - "conquer_work_cpu_s", - "budget", - "low", - 64, - {"samples": 100, "confidence": 0.95, "seed": 51}, - ) - self.assertEqual(summary["declared_pairs"], 2) - self.assertEqual(summary["complete_pairs"], 1) - self.assertEqual(summary["instance_ids"], ["a"]) - self.assertAlmostEqual(summary["geometric_mean_ratio"], 0.8) - - def test_adjusted_rows_retain_instances_without_common_support(self): - cells = [] - terminals = {} - for method in ("region-cc", "structure-blind-cc"): - for budget in ("low", "medium", "high"): - cell_id = f"instance__{method}__{budget}" - cells.append( - { - "cell_id": cell_id, - "instance_id": "instance", - "split": "held_out", - "product_width": 64, - "method": method, - "budget": budget, - } - ) - terminals[cell_id] = { - "state": "conquer-timeout", - "metrics": {"frontier_size": 100}, - } - rows = adjusted_observations( - {"cells": cells}, terminals, "conquer_work_cpu_s" - ) - self.assertEqual(len(rows), 3) - self.assertEqual({row["instance_id"] for row in rows}, {"instance"}) - self.assertTrue(all(row["adjustment_status"] == "incomplete-series" for row in rows)) - self.assertTrue(all(row["complete_pair"] is False for row in rows)) - self.assertTrue(all(row["common_frontier_size"] is None for row in rows)) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_calibrate_cc_difficulty.py b/tests/test_calibrate_cc_difficulty.py deleted file mode 100644 index 40c20cf..0000000 --- a/tests/test_calibrate_cc_difficulty.py +++ /dev/null @@ -1,36 +0,0 @@ -import unittest - -from benchmarks.cnc.calibrate_cc_difficulty import CalibrationError, choose - - -class CalibrateCcDifficultyTests(unittest.TestCase): - def test_choose_prefers_closest_count_in_accepted_range(self): - rows = [ - {"threshold": 100, "tasks": 300}, - {"threshold": 200, "tasks": 480}, - {"threshold": 300, "tasks": 540}, - {"threshold": 400, "tasks": 900}, - ] - self.assertEqual(choose(rows, 512, 384, 640)["threshold"], 300) - - def test_choose_falls_back_when_range_is_skipped(self): - rows = [ - {"threshold": 100, "tasks": 100}, - {"threshold": 200, "tasks": 1000}, - ] - self.assertEqual(choose(rows, 512, 384, 640)["threshold"], 200) - - def test_tie_breaks_toward_lower_threshold(self): - rows = [ - {"threshold": 200, "tasks": 512}, - {"threshold": 100, "tasks": 512}, - ] - self.assertEqual(choose(rows, 512, 1, 2000)["threshold"], 100) - - def test_empty_response_fails_closed(self): - with self.assertRaises(CalibrationError): - choose([], 512, 384, 640) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_calibrate_hard_regime.py b/tests/test_calibrate_hard_regime.py deleted file mode 100644 index fefe279..0000000 --- a/tests/test_calibrate_hard_regime.py +++ /dev/null @@ -1,233 +0,0 @@ -import inspect -import unittest -from dataclasses import replace -from pathlib import Path -from tempfile import TemporaryDirectory - -from benchmarks.cnc.calibrate_cc_difficulty import CalibrationError, sha256_file -from benchmarks.cnc.calibrate_hard_regime import ( - calibrate_width, - calibration_loss, - choose_width_response, - median_tasks, - ProbeContext, - run_or_resume_probe, -) - - -def row(threshold, *counts): - return { - "threshold": threshold, - "instances": [ - {"id": f"cal-{index}", "tasks": count} - for index, count in enumerate(counts) - ], - } - - -class HardRegimeCalibrationTests(unittest.TestCase): - def probe_fixture(self, root: Path, *, phase: str = "candidate"): - cuber = root / "cnc_cuber" - cuber.write_text("fake cuber\n", encoding="utf-8") - instance = root / "instance.json" - instance.write_text("{}\n", encoding="utf-8") - frontier = root / "frontier.icnf" - log = root / "cuber.log" - checkpoint = root / "probe.json" - trace = root / "nodes.jsonl" if phase != "candidate" else None - calls = [] - - def runner( - cuber_path, - instance_path, - threshold, - frontier_path, - log_path, - max_rows, - *, - trace, - selector, - timeout_s, - ): - calls.append((cuber_path, instance_path, threshold, selector, timeout_s)) - frontier_path.write_text("a 1 0\n", encoding="utf-8") - log_path.write_text("status=OK cubes=1\n", encoding="utf-8") - if trace is not None: - trace.write_text('{"node":0}\n', encoding="utf-8") - return { - "threshold": threshold, - "tasks": 1, - "elapsed_s": 2.0, - "user_s": 1.5, - "system_s": 0.25, - } - - arguments = { - "context": ProbeContext( - probe_runner=runner, - cuber=cuber, - cuber_sha256=sha256_file(cuber), - contract_digest="a" * 64, - product_width=64, - selector="region", - max_rows=512, - timeout_s=7200.0, - ), - "instance": instance, - "instance_id": "cal-0", - "instance_sha256": sha256_file(instance), - "threshold": 1, - "phase": phase, - "frontier": frontier, - "log": log, - "checkpoint": checkpoint, - "trace": trace, - } - return arguments, calls - - def test_completed_probe_checkpoint_is_reused(self): - with TemporaryDirectory() as directory: - arguments, calls = self.probe_fixture(Path(directory)) - first = run_or_resume_probe(**arguments) - - def unexpected_runner(*args, **kwargs): - self.fail("valid checkpoint should avoid rerunning the cuber") - - arguments["context"] = replace( - arguments["context"], probe_runner=unexpected_runner - ) - second = run_or_resume_probe(**arguments) - self.assertEqual(first, second) - self.assertEqual(len(calls), 1) - - def test_checkpoint_rejects_tampered_frontier(self): - with TemporaryDirectory() as directory: - arguments, _ = self.probe_fixture(Path(directory)) - run_or_resume_probe(**arguments) - arguments["frontier"].write_text("a 1 0\na 2 0\n", encoding="utf-8") - with self.assertRaisesRegex(CalibrationError, "artifact hash mismatch"): - run_or_resume_probe(**arguments) - - def test_checkpoint_rejects_changed_provenance(self): - with TemporaryDirectory() as directory: - arguments, _ = self.probe_fixture(Path(directory)) - run_or_resume_probe(**arguments) - arguments["context"] = replace( - arguments["context"], selector="structure-blind" - ) - with self.assertRaisesRegex(CalibrationError, "provenance mismatch"): - run_or_resume_probe(**arguments) - - def test_selected_checkpoint_requires_trace(self): - with TemporaryDirectory() as directory: - arguments, _ = self.probe_fixture(Path(directory), phase="selected-low") - run_or_resume_probe(**arguments) - arguments["trace"].unlink() - with self.assertRaisesRegex(CalibrationError, "artifact is missing"): - run_or_resume_probe(**arguments) - - def test_hard_regime_search_starts_at_one(self): - default = inspect.signature(calibrate_width).parameters["initial_threshold"].default - self.assertEqual(default, 1) - - def test_hard_regime_rejects_a_different_search_start(self): - with self.assertRaisesRegex(CalibrationError, "must start at 1"): - calibrate_width( - contract={"methods": {}}, - instances=[{}, {}, {}], - product_width=64, - selector="region", - cuber=Path("cuber"), - out_dir=Path("out"), - initial_threshold=1024, - ) - - def test_hard_regime_search_probes_one_then_two_and_never_zero(self): - with TemporaryDirectory() as directory: - root = Path(directory) - cuber = root / "cnc_cuber" - cuber.write_text("fake cuber\n", encoding="utf-8") - instances = [] - for index in range(3): - path = root / f"cal-{index}.json" - path.write_text("{}\n", encoding="utf-8") - instances.append( - {"id": f"cal-{index}", "path": path, "sha256": sha256_file(path)} - ) - candidate_thresholds = [] - - def runner( - cuber_path, - instance_path, - threshold, - frontier_path, - log_path, - max_rows, - *, - trace, - selector, - timeout_s, - ): - if trace is None: - candidate_thresholds.append(threshold) - frontier_path.write_text( - "".join(f"a {index + 1} 0\n" for index in range(threshold)), - encoding="utf-8", - ) - log_path.write_text("status=OK\n", encoding="utf-8") - if trace is not None: - trace.write_text('{"node":0}\n', encoding="utf-8") - return { - "threshold": threshold, - "tasks": threshold, - "elapsed_s": 1.0, - "user_s": 0.5, - "system_s": 0.25, - } - - calibrate_width( - contract={ - "methods": {"region-cc": {"max_rows": 512}}, - "limits_seconds": {"cubing": 60}, - "frontier_bands": { - "low": {"center_cubes": 2, "accepted_ratio": [0.5, 1.5]} - }, - }, - instances=instances, - product_width=64, - selector="region", - cuber=cuber, - out_dir=root / "calibration", - maximum_threshold=8, - probe_runner=runner, - ) - - self.assertEqual(candidate_thresholds, [1, 1, 1, 2, 2, 2]) - self.assertNotIn(0, candidate_thresholds) - - def test_width_selection_uses_all_three_calibration_instances(self): - response = [ - row(100, 300, 400, 500), - row(200, 480, 520, 560), - row(300, 500, 700, 4000), - ] - selected = choose_width_response(response, 512, 384, 640) - self.assertEqual(selected["threshold"], 200) - self.assertEqual(median_tasks(selected), 520) - - def test_loss_penalizes_cross_instance_mismatch(self): - balanced = row(100, 480, 512, 544) - skewed = row(200, 128, 512, 2048) - self.assertLess(calibration_loss(balanced, 512), calibration_loss(skewed, 512)) - - def test_empty_response_fails_closed(self): - with self.assertRaises(CalibrationError): - choose_width_response([], 512, 384, 640) - - def test_response_without_an_in_band_threshold_fails_closed(self): - with self.assertRaisesRegex(CalibrationError, "accepted task range"): - choose_width_response([row(100, 10, 20, 30)], 512, 384, 640) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_cnc_measurements.py b/tests/test_cnc_measurements.py deleted file mode 100644 index d097138..0000000 --- a/tests/test_cnc_measurements.py +++ /dev/null @@ -1,100 +0,0 @@ -import hashlib -import json -import shutil -import subprocess -import tempfile -import unittest -from pathlib import Path - - -ROOT = Path(__file__).resolve().parents[1] -VERIFIER = ROOT / "benchmarks" / "cnc" / "verify_measurements.py" -FIXTURES = ROOT / "tests" / "fixtures" / "cnc" - - -class CncMeasurementsTest(unittest.TestCase): - def run_bundle(self, path: Path) -> subprocess.CompletedProcess[str]: - return subprocess.run( - ["python3", str(VERIFIER), "--bundle", str(path)], - cwd=ROOT, - text=True, - capture_output=True, - check=False, - ) - - def run_fixture(self, name: str) -> subprocess.CompletedProcess[str]: - return self.run_bundle(FIXTURES / name) - - def test_valid_bundle_recomputes_issue_41_metrics(self): - result = self.run_fixture("measurement-valid") - self.assertEqual(result.returncode, 0, result.stdout + result.stderr) - self.assertEqual( - result.stdout.splitlines(), - [ - "PASS frontier: 8/8 assignments are covered exactly once", - "PASS verdict: returned model satisfies the input", - "PASS accounting: cubing_wall=2 conquer_cpu=9 " - "conquer_makespan=5 end_to_end_wall=7", - "PASS workers: observed concurrency does not exceed 2", - ], - ) - - def test_missing_cube_fails_with_uncovered_assignment(self): - with tempfile.TemporaryDirectory() as directory: - bundle = Path(directory) / "bundle" - shutil.copytree(FIXTURES / "measurement-valid", bundle) - frontier_path = bundle / "frontier.jsonl" - frontier = "\n".join(frontier_path.read_text().splitlines()[1:]) + "\n" - frontier_path.write_text(frontier, encoding="utf-8") - manifest_path = bundle / "bundle.json" - manifest = json.loads(manifest_path.read_text(encoding="utf-8")) - manifest["frontier"]["sha256"] = hashlib.sha256( - frontier.encode("utf-8") - ).hexdigest() - manifest_path.write_text(json.dumps(manifest), encoding="utf-8") - result = self.run_bundle(bundle) - self.assertNotEqual(result.returncode, 0) - self.assertIn("FAIL frontier: assignment 000 is uncovered", result.stdout) - - def test_accounting_is_recomputed_instead_of_trusted(self): - with tempfile.TemporaryDirectory() as directory: - bundle = Path(directory) / "bundle" - shutil.copytree(FIXTURES / "measurement-valid", bundle) - manifest_path = bundle / "bundle.json" - manifest = json.loads(manifest_path.read_text(encoding="utf-8")) - manifest["accounting"]["conquer_cpu"] = 8.0 - manifest_path.write_text(json.dumps(manifest), encoding="utf-8") - result = self.run_bundle(bundle) - self.assertNotEqual(result.returncode, 0) - self.assertIn("accounting mismatch for conquer_cpu", result.stdout) - - def test_worker_identity_cannot_hide_overlapping_work(self): - with tempfile.TemporaryDirectory() as directory: - bundle = Path(directory) / "bundle" - shutil.copytree(FIXTURES / "measurement-valid", bundle) - events_path = bundle / "events.jsonl" - events = [ - json.loads(line) - for line in events_path.read_text(encoding="utf-8").splitlines() - ] - for event in events: - if event.get("cube_id") == "010": - event["worker"] = 1 - encoded = "".join( - json.dumps(event, sort_keys=True, separators=(",", ":")) + "\n" - for event in events - ) - events_path.write_text(encoded, encoding="utf-8") - manifest_path = bundle / "bundle.json" - manifest = json.loads(manifest_path.read_text(encoding="utf-8")) - manifest["events"]["sha256"] = hashlib.sha256( - encoded.encode("utf-8") - ).hexdigest() - manifest_path.write_text(json.dumps(manifest), encoding="utf-8") - result = self.run_bundle(bundle) - self.assertNotEqual(result.returncode, 0) - self.assertIn("worker 1 runs overlapping cube 010", result.stdout) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_conquer_parallel.py b/tests/test_conquer_parallel.py index 6397c20..5d43e9b 100644 --- a/tests/test_conquer_parallel.py +++ b/tests/test_conquer_parallel.py @@ -1,84 +1,86 @@ import concurrent.futures -import json -import sys + +import pytest from benchmarks.cnc import conquer_parallel +from benchmarks.cnc.solve import conquer_frontier -def test_parallel_conquer_streams_cubes_and_reports_conflict_distribution( - tmp_path, monkeypatch -): +def test_parallel_conquer_reports_unsat_distribution(tmp_path, monkeypatch): cnf = tmp_path / "base.cnf" - cnf.write_text("p cnf 2 1\n1 2 0\n") + cnf.write_text("p cnf 2 1\n1 2 0\n", encoding="utf-8") cubes = tmp_path / "cubes.icnf" - cubes.write_text("".join(f"a {literal} 0\n" for literal in [1, -1, 2, -2, 1])) - fake_kissat = tmp_path / "kissat" - fake_kissat.write_text( + cubes.write_text("".join(f"a {literal} 0\n" for literal in [1, -1, 2, -2])) + kissat = tmp_path / "kissat" + kissat.write_text( "#!/bin/sh\n" - "case \" $* \" in *\" --statistics \"*) ;; *) exit 2 ;; esac\n" "echo 'c decisions: 7'\n" "echo 'c conflicts: 3'\n" - "exit 20\n" + "exit 20\n", + encoding="utf-8", ) - fake_kissat.chmod(0o755) - out_dir = tmp_path / "out" - temp_dir = tmp_path / "tmp" - + kissat.chmod(0o755) monkeypatch.setattr( concurrent.futures, "ProcessPoolExecutor", concurrent.futures.ThreadPoolExecutor, ) + + result = conquer_frontier( + cnf, + cubes, + kissat, + workers=2, + timeout_s=5, + out_dir=tmp_path / "out", + tmp_dir=tmp_path / "tmp", + ) + + assert result["result"] == "unsat" + assert result["complete"] is True + assert result["completed"] == 4 + assert result["total_decisions"] == 28 + assert result["total_conflicts"] == 12 + assert result["not_started"] == 0 + + +def test_parallel_conquer_stops_submitting_after_sat(tmp_path, monkeypatch): + cnf = tmp_path / "base.cnf" + cnf.write_text("p cnf 1 0\n", encoding="utf-8") + cubes = tmp_path / "cubes.icnf" + cubes.write_text("a 1 0\n" + "a -1 0\n" * 9, encoding="utf-8") + kissat = tmp_path / "kissat" + kissat.write_text("#!/bin/sh\nexit 10\n", encoding="utf-8") + kissat.chmod(0o755) monkeypatch.setattr( - sys, - "argv", - [ - "conquer_parallel.py", - str(cnf), - "--arm", - f"test={cubes}", - "--kissat", - str(fake_kissat), - "--workers", - "2", - "--timeout-s", - "5", - "--tmp-dir", - str(temp_dir), - "--out-dir", - str(out_dir), - ], + concurrent.futures, + "ProcessPoolExecutor", + concurrent.futures.ThreadPoolExecutor, ) - conquer_parallel.main() - summary = json.loads((out_dir / "summary.json").read_text()) - result = summary["arms"]["test"] - assert result["cubes"] == 5 - assert result["completed"] == 5 - assert result["errors"] == 0 - assert result["timeouts"] == 0 - assert result["total_decisions"] == 35 - assert result["total_conflicts"] == 15 - assert result["conflicts_p50"] == 3 - assert result["conflicts_p99_over_p95"] == 1 + result = conquer_frontier( + cnf, + cubes, + kissat, + workers=1, + timeout_s=5, + out_dir=tmp_path / "out", + ) + + assert result["result"] == "sat" assert result["complete"] is True - assert result["result"] == "unsat" - assert result["terminal_records"] == 5 - assert result["lpt_makespan_by_workers_s"]["2"] > 0 - raw = [json.loads(line) for line in (out_dir / "test.jsonl").read_text().splitlines()] - assert all(row["released_monotonic_ns"] <= row["started_monotonic_ns"] for row in raw) - assert all(row["started_monotonic_ns"] <= row["finished_monotonic_ns"] for row in raw) - assert all(row["worker_pid"] > 0 for row in raw) - expected_measured = ( - max(row["collected_monotonic_ns"] for row in raw) - - min(row["released_monotonic_ns"] for row in raw) - ) / 1e9 - assert result["measured_makespan_s"] == expected_measured - assert result["observed_parallel_wall_s"] >= result["measured_makespan_s"] - assert len((out_dir / "test.jsonl").read_text().splitlines()) == 5 + assert result["completed"] == 1 + assert result["not_started"] == 9 + + +def test_frontier_literals_are_checked_against_cnf_variables(tmp_path): + frontier = tmp_path / "bad.icnf" + frontier.write_text("a 3 0\n", encoding="utf-8") + with pytest.raises(ValueError, match="variable range"): + list(conquer_parallel.read_cubes(frontier, variables=2)) -def test_censored_cubes_remain_in_work_and_lpt_accounting(): +def test_censored_cubes_remain_in_work_accounting(): result = conquer_parallel.summarize( cubes=2, completed=1, @@ -91,7 +93,7 @@ def test_censored_cubes_remain_in_work_and_lpt_accounting(): decisions=[7.0], conflicts=[3.0], workers=2, - replay_workers=[2, 4], + replay_workers=[2], wall_s=5.2, measured_makespan_s=5.0, ) @@ -99,7 +101,4 @@ def test_censored_cubes_remain_in_work_and_lpt_accounting(): assert result["complete"] is False assert result["censored"] is True assert result["total_solver_s"] == 6.0 - assert result["total_cpu_s"] == 5.0 assert result["lpt_is_lower_bound"] is True - assert result["lpt_makespan_by_workers_s"] == {"2": 5.0, "4": 5.0} - assert result["conflicts_max"] == 3.0 diff --git a/tests/test_cubing.py b/tests/test_cubing.py new file mode 100644 index 0000000..d30ff59 --- /dev/null +++ b/tests/test_cubing.py @@ -0,0 +1,58 @@ +from pathlib import Path + +from benchmarks.cnc import cubing + + +def successful_cuber(command): + frontier = Path(command[command.index("-o") + 1]) + frontier.parent.mkdir(parents=True, exist_ok=True) + frontier.write_text("a 1 0\na -1 0\n", encoding="utf-8") + + +def fake_run_process(command, **kwargs): + successful_cuber(command) + return {"returncode": 0, "timed_out": False, "wall_s": 0.1} + + +def fake_conquer(*args, **kwargs): + return {"result": "unsat", "complete": True, "cubes": 2} + + +def test_march_pipeline_uses_cnf_then_shared_conquer(tmp_path, monkeypatch): + monkeypatch.setattr(cubing, "run_process", fake_run_process) + monkeypatch.setattr(cubing, "conquer_frontier", fake_conquer) + + record = cubing.march_then_conquer( + tmp_path / "instance.cnf", + tmp_path / "march_cu", + tmp_path / "kissat", + workers=4, + cube_timeout_s=5, + cubing_timeout_s=5, + remaining_vars=20, + out_dir=tmp_path / "run", + ) + + assert record["mode"] == "march-cu" + assert record["cubing"]["cubes"] == 2 + assert record["cubing"]["command"][1].endswith("instance.cnf") + assert record["cubing"]["command"][-4:-2] == ["-n", "20"] + + +def test_frozen_frontier_goes_directly_to_shared_conquer(tmp_path, monkeypatch): + monkeypatch.setattr(cubing, "conquer_frontier", fake_conquer) + (tmp_path / "instance.cnf").write_text("p cnf 1 0\n", encoding="utf-8") + (tmp_path / "frontier.icnf").write_text("a 1 0\na -1 0\n", encoding="utf-8") + + record = cubing.frozen_then_conquer( + tmp_path / "instance.cnf", + tmp_path / "frontier.icnf", + tmp_path / "kissat", + workers=4, + cube_timeout_s=5, + out_dir=tmp_path / "run", + ) + + assert record["mode"] == "frozen-frontier" + assert record["cubes"] == 2 + assert record["frontier"].endswith("frontier.icnf") diff --git a/tests/test_factoring_corpus.py b/tests/test_factoring_corpus.py new file mode 100644 index 0000000..9f0675a --- /dev/null +++ b/tests/test_factoring_corpus.py @@ -0,0 +1,45 @@ +import json + +import pytest + +from benchmarks.cnc.factoring import materialize, sat_targets, unsat_targets +from benchmarks.pipeline.circuit import load_json, read_jsonl, sha256_file +from benchmarks.pipeline.multipliers import is_prime + + +def test_sat_and_unsat_targets_are_deterministic_and_paired(): + first_sat, _ = sat_targets(12, 3) + second_sat, _ = sat_targets(12, 3) + unsat, oracles = unsat_targets(12, first_sat) + + assert first_sat == second_sat + assert all(target.paired_sat_id == sat.instance_id for target, sat in zip(unsat, first_sat)) + assert all(is_prime(target.target) for target in unsat) + assert all(oracle["target_exceeds_max_factor"] for oracle in oracles) + + +def test_tiny_width_rejects_more_distinct_instances_than_exist(): + with pytest.raises(ValueError, match="only 3 distinct"): + sat_targets(4, 4) + + +def test_materialize_writes_matching_circuit_and_cnf(tmp_path): + manifest = materialize([4], 1, tmp_path) + + assert len(manifest) == 2 + assert {row["expected_outcome"] for row in manifest} == {"sat", "unsat"} + for row in manifest: + circuit_path = tmp_path / row["circuit"] + cnf_path = tmp_path / row["cnf"] + circuit = load_json(circuit_path) + assert circuit["metadata"]["factoring"]["target"] == row["target"] + assert circuit["metadata"]["pinned_outputs"]["product"] == row["target"] + assert cnf_path.read_text(encoding="utf-8").startswith("c var ") + assert "\np cnf " in cnf_path.read_text(encoding="utf-8") + assert sha256_file(circuit_path) == row["circuit_sha256"] + assert sha256_file(cnf_path) == row["cnf_sha256"] + + assert read_jsonl(tmp_path / "manifest.jsonl") == manifest + oracles = read_jsonl(tmp_path / "oracles.jsonl") + assert len(oracles) == 2 + assert json.loads((tmp_path / manifest[0]["metadata"]).read_text())["target"] == manifest[0]["target"] diff --git a/tests/test_hard_regime.py b/tests/test_hard_regime.py deleted file mode 100644 index d4a209d..0000000 --- a/tests/test_hard_regime.py +++ /dev/null @@ -1,73 +0,0 @@ -import copy -import unittest -from pathlib import Path - -from benchmarks.cnc.hard_regime import ( - EXPECTED_METHOD_BUDGETS, - HardRegimeError, - load_contract, - strong_miller_rabin, - target_records, - validate_contract, - verify_target_records, -) - - -ROOT = Path(__file__).resolve().parents[1] -CONTRACT = ROOT / "benchmarks/cnc/contracts/hard-regime-v1.yaml" - - -class HardRegimeTests(unittest.TestCase): - def test_contract_freezes_issue_51_matrix(self): - contract = load_contract(CONTRACT) - self.assertEqual( - { - name: tuple(spec["budgets"]) - for name, spec in contract["methods"].items() - }, - EXPECTED_METHOD_BUDGETS, - ) - self.assertEqual(contract["limits_seconds"]["per_cube_conquer"], 1800) - self.assertEqual(contract["statistics"]["unit"], "held-out-instance") - - def test_targets_are_deterministic_prime_in_range_and_disjoint(self): - contract = load_contract(CONTRACT) - first = target_records(contract) - second = target_records(contract) - self.assertEqual(first, second) - self.assertEqual(len(first), 39) - self.assertTrue(all(record["expected_outcome"] == "unsat" for record in first)) - self.assertEqual(len({record["id"] for record in first}), 39) - messages = verify_target_records(contract, first) - self.assertTrue(any("39 deterministic prime" in message for message in messages)) - - def test_contract_rejects_width_confusion_and_split_seed_overlap(self): - contract = load_contract(CONTRACT) - confused = copy.deepcopy(contract) - confused["widths"][0]["factor_input_width"] = 64 - with self.assertRaisesRegex(HardRegimeError, "twice factor-input width"): - validate_contract(confused) - - overlapping = copy.deepcopy(contract) - overlapping["widths"][0]["held_out"]["seed"] = overlapping["widths"][0][ - "calibration" - ]["seed"] - with self.assertRaisesRegex(HardRegimeError, "seeds overlap"): - validate_contract(overlapping) - - def test_contract_rejects_march_budget_tuning(self): - contract = load_contract(CONTRACT) - contract["methods"]["march-cu-dynamic"]["budgets"] = [ - "low", - "medium", - "high", - ] - with self.assertRaisesRegex(HardRegimeError, "march-cu-dynamic budgets"): - validate_contract(contract) - - def test_independent_primality_check_rejects_composites(self): - self.assertTrue(strong_miller_rabin(37)) - self.assertFalse(strong_miller_rabin(39)) - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_hard_regime_matrix.py b/tests/test_hard_regime_matrix.py deleted file mode 100644 index 22ebc87..0000000 --- a/tests/test_hard_regime_matrix.py +++ /dev/null @@ -1,307 +0,0 @@ -import tempfile -import unittest -from collections import Counter -from pathlib import Path - -from benchmarks.cnc.hard_regime import contract_sha256, load_contract, target_records -from benchmarks.cnc.hard_regime_matrix import MatrixError, build_matrix, lock_toolchain -from benchmarks.pipeline.circuit import canonical_bytes, sha256_bytes, write_json - - -ROOT = Path(__file__).resolve().parents[1] -CONTRACT_PATH = ROOT / "benchmarks/cnc/contracts/hard-regime-v1.yaml" - - -def fake_manifest(contract): - digest = contract_sha256(contract) - return [ - { - **record, - "contract_sha256": digest, - "circuitsat": f"instances/{record['id']}.json", - "circuitsat_sha256": f"{index + 1:064x}", - "cnf": f"instances/{record['id']}.cnf", - "cnf_sha256": f"{index + 100:064x}", - "encoding_wall_s": 1.0, - "encoding_cpu_s": 0.9, - } - for index, record in enumerate(target_records(contract)) - ] - - -def fake_toolchain(contract): - record = { - "schema_version": 1, - "kind": "hard-regime-toolchain-lock", - "contract_sha256": contract_sha256(contract), - "tools": { - "cnc_cuber": { - "path": "/tools/cnc_cuber", - "executable_sha256": "a" * 64, - "source_revision": "repo-revision", - }, - "kissat": { - "path": "/tools/kissat", - "executable_sha256": "b" * 64, - "source_revision": contract["tool_sources"]["kissat"]["revision"], - }, - "march_cu": { - "path": "/tools/march_cu", - "executable_sha256": "c" * 64, - "source_revision": contract["tool_sources"]["march_cu"]["revision"], - }, - }, - } - record["toolchain_sha256"] = sha256_bytes(canonical_bytes(record)) - return record - - -def write_calibration_locks(root, contract, manifest, cuber_sha): - for width in (64, 72, 80): - calibration = [ - {"id": record["id"], "sha256": record["circuitsat_sha256"]} - for record in manifest - if record["product_width"] == width and record["split"] == "calibration" - ] - for selector in ("region", "structure-blind"): - method = "region-cc" if selector == "region" else "structure-blind-cc" - responses = [] - bands = {} - for index, (name, spec) in enumerate(contract["frontier_bands"].items(), 1): - threshold = width * 1000 + index - target = spec["center_cubes"] - response_instances = [ - {"id": item["id"], "tasks": target, "elapsed_s": 1.0, "cpu_s": 0.9} - for item in calibration - ] - responses.append( - {"threshold": threshold, "instances": response_instances} - ) - minimum = int(target * spec["accepted_ratio"][0]) - maximum = int(target * spec["accepted_ratio"][1]) - bands[name] = { - "target_tasks": target, - "accepted_task_range": [minimum, maximum], - "selected_threshold": threshold, - "search_bracket": ( - [threshold, threshold + 1] - if index == 1 - else [threshold - 1, threshold] - ), - "median_tasks": float(target), - "calibration_loss": 0.0, - "within_target_range": True, - "instances": [ - { - **item, - "frontier_sha256": f"{index:064x}", - "trace_sha256": f"{index + 10:064x}", - "cubing_elapsed_s": 1.0, - "cubing_cpu_s": 0.9, - } - for item in response_instances - ], - } - write_json( - root / f"p{width}" / selector / "calibration-lock.json", - { - "schema_version": 1, - "kind": "width-level-cc-calibration-lock", - "contract_sha256": contract_sha256(contract), - "product_width": width, - "selector": selector, - "method": method, - "max_rows": contract["methods"][method]["max_rows"], - "cuber_sha256": cuber_sha, - "search": { - "initial_threshold": 1, - "maximum_threshold": 1 << 120, - "probe_checkpoint_schema_version": 1, - }, - "calibration_instances": calibration, - "bands": bands, - "response": responses, - }, - ) - - -class HardRegimeMatrixTests(unittest.TestCase): - def test_matrix_declares_every_required_cell_once(self): - contract = load_contract(CONTRACT_PATH) - manifest = fake_manifest(contract) - tools = fake_toolchain(contract) - with tempfile.TemporaryDirectory() as directory: - root = Path(directory) - write_calibration_locks( - root, contract, manifest, tools["tools"]["cnc_cuber"]["executable_sha256"] - ) - matrix = build_matrix(contract, manifest, root, tools) - counts = Counter(cell["method"] for cell in matrix["cells"]) - self.assertEqual( - counts, - { - "monolithic-kissat": 39, - "march-cu-dynamic": 30, - "region-cc": 90, - "structure-blind-cc": 90, - }, - ) - self.assertEqual(sum(cell["pilot"] for cell in matrix["cells"]), 72) - held_out = [cell for cell in matrix["cells"] if cell["method"] == "region-cc"] - thresholds = { - (cell["product_width"], cell["budget"]): cell["cc_threshold"] - for cell in held_out - } - self.assertEqual(len(thresholds), 9) - for cell in held_out: - self.assertEqual( - cell["cc_threshold"], - thresholds[(cell["product_width"], cell["budget"])], - ) - - def test_matrix_rejects_width_confusion(self): - contract = load_contract(CONTRACT_PATH) - manifest = fake_manifest(contract) - manifest[0]["factor_input_width"] = manifest[0]["product_width"] - with self.assertRaisesRegex(MatrixError, "factor_input_width differs"): - with tempfile.TemporaryDirectory() as directory: - build_matrix(contract, manifest, Path(directory), fake_toolchain(contract)) - - def test_matrix_rejects_calibration_test_leakage(self): - contract = load_contract(CONTRACT_PATH) - manifest = fake_manifest(contract) - tools = fake_toolchain(contract) - with tempfile.TemporaryDirectory() as directory: - root = Path(directory) - write_calibration_locks( - root, contract, manifest, tools["tools"]["cnc_cuber"]["executable_sha256"] - ) - path = root / "p64/region/calibration-lock.json" - lock = __import__("json").loads(path.read_text()) - held_out = next( - record - for record in manifest - if record["product_width"] == 64 and record["split"] == "held_out" - ) - lock["calibration_instances"][0]["id"] = held_out["id"] - write_json(path, lock) - with self.assertRaisesRegex(MatrixError, "not the frozen split"): - build_matrix(contract, manifest, root, tools) - - def test_matrix_recomputes_the_selected_calibration_threshold(self): - contract = load_contract(CONTRACT_PATH) - manifest = fake_manifest(contract) - tools = fake_toolchain(contract) - with tempfile.TemporaryDirectory() as directory: - root = Path(directory) - write_calibration_locks( - root, contract, manifest, tools["tools"]["cnc_cuber"]["executable_sha256"] - ) - path = root / "p64/region/calibration-lock.json" - lock = __import__("json").loads(path.read_text()) - lock["bands"]["low"]["selected_threshold"] += 999 - write_json(path, lock) - with self.assertRaisesRegex(MatrixError, "not selected from the response"): - build_matrix(contract, manifest, root, tools) - - def test_matrix_rejects_missing_calibration_search_provenance(self): - contract = load_contract(CONTRACT_PATH) - manifest = fake_manifest(contract) - tools = fake_toolchain(contract) - with tempfile.TemporaryDirectory() as directory: - root = Path(directory) - write_calibration_locks( - root, - contract, - manifest, - tools["tools"]["cnc_cuber"]["executable_sha256"], - ) - path = root / "p64/region/calibration-lock.json" - lock = __import__("json").loads(path.read_text()) - del lock["search"] - write_json(path, lock) - with self.assertRaisesRegex(MatrixError, "search provenance is missing"): - build_matrix(contract, manifest, root, tools) - - def test_matrix_requires_calibration_search_to_start_at_one(self): - contract = load_contract(CONTRACT_PATH) - manifest = fake_manifest(contract) - tools = fake_toolchain(contract) - with tempfile.TemporaryDirectory() as directory: - root = Path(directory) - write_calibration_locks( - root, - contract, - manifest, - tools["tools"]["cnc_cuber"]["executable_sha256"], - ) - path = root / "p64/region/calibration-lock.json" - lock = __import__("json").loads(path.read_text()) - lock["search"]["initial_threshold"] = 1024 - write_json(path, lock) - with self.assertRaisesRegex(MatrixError, "search provenance is malformed"): - build_matrix(contract, manifest, root, tools) - - def test_matrix_accepts_newer_positive_probe_checkpoint_schema(self): - contract = load_contract(CONTRACT_PATH) - manifest = fake_manifest(contract) - tools = fake_toolchain(contract) - with tempfile.TemporaryDirectory() as directory: - root = Path(directory) - write_calibration_locks( - root, - contract, - manifest, - tools["tools"]["cnc_cuber"]["executable_sha256"], - ) - for path in root.glob("p*/*/calibration-lock.json"): - lock = __import__("json").loads(path.read_text()) - lock["search"]["probe_checkpoint_schema_version"] = 2 - write_json(path, lock) - matrix = build_matrix(contract, manifest, root, tools) - self.assertEqual(matrix["kind"], "hard-regime-run-matrix") - - def test_matrix_rejects_a_search_without_room_to_expand(self): - contract = load_contract(CONTRACT_PATH) - manifest = fake_manifest(contract) - tools = fake_toolchain(contract) - with tempfile.TemporaryDirectory() as directory: - root = Path(directory) - write_calibration_locks( - root, - contract, - manifest, - tools["tools"]["cnc_cuber"]["executable_sha256"], - ) - path = root / "p64/region/calibration-lock.json" - lock = __import__("json").loads(path.read_text()) - lock["search"]["maximum_threshold"] = 1 - write_json(path, lock) - with self.assertRaisesRegex(MatrixError, "search provenance is malformed"): - build_matrix(contract, manifest, root, tools) - - def test_toolchain_lock_requires_revision_qualified_upstream_binaries(self): - contract = load_contract(CONTRACT_PATH) - with tempfile.TemporaryDirectory() as directory: - root = Path(directory) - cuber = root / "cnc_cuber" - kissat_revision = contract["tool_sources"]["kissat"]["revision"] - march_revision = contract["tool_sources"]["march_cu"]["revision"] - kissat = root / f"kissat-{kissat_revision}" - march = root / f"march_cu-{march_revision}" - for path in (cuber, kissat, march): - path.write_text("#!/bin/sh\necho identity\n", encoding="utf-8") - path.chmod(0o755) - record = lock_toolchain(contract, cuber, kissat, march, "abcdef0123456789") - self.assertEqual( - record["tools"]["kissat"]["source_revision"], kissat_revision - ) - generic = root / "kissat" - generic.write_text("#!/bin/sh\necho stale\n", encoding="utf-8") - generic.chmod(0o755) - with self.assertRaisesRegex(MatrixError, "revision-qualified binary"): - lock_toolchain(contract, cuber, generic, march, "abcdef0123456789") - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_run_hard_regime_cell.py b/tests/test_run_hard_regime_cell.py deleted file mode 100644 index d6e792f..0000000 --- a/tests/test_run_hard_regime_cell.py +++ /dev/null @@ -1,147 +0,0 @@ -import json -import sys -import tempfile -import unittest -from pathlib import Path - -from benchmarks.cnc.run_hard_regime_cell import CellError, run_process, verify_region_trace - - -class HardRegimeCellTests(unittest.TestCase): - def test_region_trace_reconstructs_complete_frontier(self): - with tempfile.TemporaryDirectory() as directory: - root = Path(directory) - frontier = root / "frontier.icnf" - trace = root / "nodes.jsonl" - frontier.write_text("a 1 0\na -1 0\n", encoding="utf-8") - rows = [ - { - "node_id": 0, - "parent_id": None, - "child_index": None, - "depth": 0, - "kind": "branch", - "refutation_reason": None, - "literals": [], - "sigma_dec": 0, - "sigma_all": 0, - "freevars": 1, - "rule_variables": [1], - "rule_clauses": [ - {"mask": 1, "value": 1}, - {"mask": 1, "value": 0}, - ], - }, - { - "node_id": 1, - "parent_id": 0, - "child_index": 0, - "depth": 1, - "kind": "cutoff", - "refutation_reason": None, - "literals": [1], - "sigma_dec": 1, - "sigma_all": 1, - "freevars": 0, - "rule_variables": [], - "rule_clauses": [], - }, - { - "node_id": 2, - "parent_id": 0, - "child_index": 1, - "depth": 1, - "kind": "cutoff", - "refutation_reason": None, - "literals": [-1], - "sigma_dec": 1, - "sigma_all": 1, - "freevars": 0, - "rule_variables": [], - "rule_clauses": [], - }, - ] - trace.write_text( - "".join(json.dumps(row) + "\n" for row in rows), encoding="utf-8" - ) - summary = verify_region_trace(frontier, trace) - self.assertEqual( - summary, - { - "nodes": 3, - "branches": 1, - "cutoffs": 2, - "refuted": 0, - "sat_leaves": 0, - "root_refutations": 0, - "selector_refutations": 0, - "branch_refutations": 0, - }, - ) - - rows.pop() - trace.write_text( - "".join(json.dumps(row) + "\n" for row in rows), encoding="utf-8" - ) - with self.assertRaisesRegex(CellError, "incomplete children"): - verify_region_trace(frontier, trace) - - def test_region_trace_rejects_child_that_does_not_apply_its_rule(self): - with tempfile.TemporaryDirectory() as directory: - root = Path(directory) - frontier = root / "frontier.icnf" - trace = root / "nodes.jsonl" - frontier.write_text("a -1 0\n", encoding="utf-8") - rows = [ - { - "node_id": 0, - "parent_id": None, - "child_index": None, - "depth": 0, - "kind": "branch", - "refutation_reason": None, - "literals": [], - "sigma_dec": 0, - "sigma_all": 0, - "freevars": 1, - "rule_variables": [1], - "rule_clauses": [{"mask": 1, "value": 1}], - }, - { - "node_id": 1, - "parent_id": 0, - "child_index": 0, - "depth": 1, - "kind": "cutoff", - "refutation_reason": None, - "literals": [-1], - "sigma_dec": 1, - "sigma_all": 1, - "freevars": 0, - "rule_variables": [], - "rule_clauses": [], - }, - ] - trace.write_text( - "".join(json.dumps(row) + "\n" for row in rows), encoding="utf-8" - ) - with self.assertRaisesRegex(CellError, "does not implement rule clause"): - verify_region_trace(frontier, trace) - - def test_process_timeout_is_a_terminal_stage(self): - with tempfile.TemporaryDirectory() as directory: - root = Path(directory) - result = run_process( - [sys.executable, "-c", "import time; time.sleep(1)"], - 0.01, - root / "stdout", - root / "stderr", - ) - self.assertEqual(result["state"], "timeout") - self.assertIsNone(result["returncode"]) - self.assertGreaterEqual(result["wall_s"], 0.01) - self.assertEqual(len(result["stdout_sha256"]), 64) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_solve.py b/tests/test_solve.py new file mode 100644 index 0000000..8c68ba8 --- /dev/null +++ b/tests/test_solve.py @@ -0,0 +1,24 @@ +import json + +from benchmarks.cnc.solve import run_kissat + + +def test_run_kissat_records_direct_result(tmp_path): + cnf = tmp_path / "instance.cnf" + cnf.write_text("p cnf 1 1\n1 0\n", encoding="utf-8") + kissat = tmp_path / "kissat" + kissat.write_text( + "#!/bin/sh\n" + "echo 'c decisions: 7'\n" + "echo 'c conflicts: 3'\n" + "exit 10\n", + encoding="utf-8", + ) + kissat.chmod(0o755) + + record = run_kissat(cnf, kissat, timeout_s=5, out_dir=tmp_path / "run") + + assert record["result"] == "sat" + assert record["decisions"] == 7 + assert record["conflicts"] == 3 + assert json.loads((tmp_path / "run/summary.json").read_text()) == record diff --git a/tests/test_verify_hard_regime.py b/tests/test_verify_hard_regime.py deleted file mode 100644 index 5fc10db..0000000 --- a/tests/test_verify_hard_regime.py +++ /dev/null @@ -1,133 +0,0 @@ -import hashlib -import json -import tempfile -import unittest -from pathlib import Path - -from benchmarks.cnc.conquer_parallel import summarize -from benchmarks.cnc.hard_regime import contract_sha256, load_contract -from benchmarks.cnc.verify_hard_regime import ( - VerificationError, - verify_conquer_records, - verify_terminal, -) -from benchmarks.pipeline.circuit import canonical_bytes, sha256_bytes, sha256_file - - -ROOT = Path(__file__).resolve().parents[1] -CONTRACT_PATH = ROOT / "benchmarks/cnc/contracts/hard-regime-v1.yaml" - - -class HardRegimeVerifierTests(unittest.TestCase): - def make_records(self, root: Path): - frontier = root / "frontier.icnf" - results = root / "results.jsonl" - frontier.write_text("a 1 0\na -1 0\n", encoding="utf-8") - rows = [] - for index, literal in enumerate((1, -1)): - started = 100 + index * 20 - rows.append( - { - "cube_index": index, - "cube_sha256": hashlib.sha256(f"{literal} 0\n".encode()).hexdigest(), - "released_monotonic_ns": started - 5, - "started_monotonic_ns": started, - "finished_monotonic_ns": started + 10, - "collected_monotonic_ns": started + 15, - "worker_pid": index + 1, - "elapsed_s": 1.0 + index, - "user_s": 0.5 + index, - "system_s": 0.25, - "decisions": 4, - "conflicts": 3 + index, - "result": "unsat", - "censored": False, - } - ) - results.write_text("".join(json.dumps(row) + "\n" for row in rows), encoding="utf-8") - summary = summarize( - cubes=2, - completed=2, - timeouts=0, - errors=0, - sat=0, - unsat=2, - durations=[1.0, 2.0], - cpu_durations=[0.75, 1.75], - decisions=[4.0, 4.0], - conflicts=[3.0, 4.0], - workers=2, - replay_workers=[2, 4], - wall_s=2.1, - measured_makespan_s=40 / 1e9, - ) - return frontier, results, summary, rows - - def test_per_cube_records_reconstruct_work_and_scheduling(self): - with tempfile.TemporaryDirectory() as directory: - frontier, results, summary, _ = self.make_records(Path(directory)) - verify_conquer_records(frontier, results, summary, 2, [2, 4]) - - def test_duplicate_cube_record_is_rejected(self): - with tempfile.TemporaryDirectory() as directory: - root = Path(directory) - frontier, results, summary, rows = self.make_records(root) - rows[1]["cube_index"] = 0 - results.write_text( - "".join(json.dumps(row) + "\n" for row in rows), encoding="utf-8" - ) - with self.assertRaisesRegex(VerificationError, "duplicate cube indices"): - verify_conquer_records(frontier, results, summary, 2, [2, 4]) - - def test_harness_error_is_terminal_only_not_reconstruction(self): - contract = load_contract(CONTRACT_PATH) - with tempfile.TemporaryDirectory() as directory: - root = Path(directory) - matrix_path = root / "matrix.json" - matrix_path.write_text("{}\n", encoding="utf-8") - toolchain = {"toolchain_sha256": "toolchain"} - cell = { - "cell_id": "cell", - "instance_id": "instance", - "method": "region-cc", - "budget": "low", - "product_width": 64, - "factor_input_width": 32, - "global_cnf_sha256": "a" * 64, - "circuitsat_sha256": "b" * 64, - } - terminal = { - "kind": "hard-regime-terminal-cell", - "contract_sha256": contract_sha256(contract), - "matrix_sha256": sha256_file(matrix_path), - "toolchain_sha256": "toolchain", - "cell_id": "cell", - "cell_sha256": sha256_bytes(canonical_bytes(cell)), - "instance_id": "instance", - "method": "region-cc", - "budget": "low", - "product_width": 64, - "factor_input_width": 32, - "state": "harness-error", - "error": "CellError: input hash mismatch", - "input_artifacts": { - "global_cnf": {"sha256": "a" * 64}, - "circuitsat": {"sha256": "b" * 64}, - }, - } - self.assertEqual( - verify_terminal( - contract, - matrix_path, - {}, - toolchain, - cell, - terminal, - root, - ), - "harness-terminal-only", - ) - - -if __name__ == "__main__": - unittest.main()