From 1dd76f73461da5cfdd17911127f470eaf8a4662a Mon Sep 17 00:00:00 2001 From: sneakocom <192013763+sneakocom@users.noreply.github.com> Date: Sun, 30 Aug 2026 19:00:26 +0000 Subject: [PATCH] Prepare EXP-001 live authorization preflight --- PROGRAM_STATUS.md | 6 +- experiments/exp-001/README.md | 7 + .../exp-001/live-preflight-v1/README.md | 29 + .../live-preflight-v1/model-catalogue.json | 79 ++ .../exp-001/live-preflight-v1/preflight.py | 948 ++++++++++++++++++ program/PRIORITY.md | 8 +- .../evidence/exp-001-live-preflight/README.md | 34 + .../model-catalogue.json | 1 + .../pricing-preflight.json | 1 + .../qualification-report.json | 1 + .../exp-001-live-preflight/value-receipt.json | 1 + program/experiments.json | 49 +- program/registry.json | 20 +- tests/test_exp001_live_preflight.py | 205 ++++ tests/test_validate_program.py | 61 ++ tools/validate_program.py | 97 +- 16 files changed, 1522 insertions(+), 25 deletions(-) create mode 100644 experiments/exp-001/live-preflight-v1/README.md create mode 100644 experiments/exp-001/live-preflight-v1/model-catalogue.json create mode 100755 experiments/exp-001/live-preflight-v1/preflight.py create mode 100644 program/evidence/exp-001-live-preflight/README.md create mode 100644 program/evidence/exp-001-live-preflight/model-catalogue.json create mode 100644 program/evidence/exp-001-live-preflight/pricing-preflight.json create mode 100644 program/evidence/exp-001-live-preflight/qualification-report.json create mode 100644 program/evidence/exp-001-live-preflight/value-receipt.json create mode 100644 tests/test_exp001_live_preflight.py diff --git a/PROGRAM_STATUS.md b/PROGRAM_STATUS.md index 3a66676..bb9ba09 100644 --- a/PROGRAM_STATUS.md +++ b/PROGRAM_STATUS.md @@ -43,7 +43,7 @@ Program state totals: active 4; waiting 16; complete 0. | 15 | [agent-recovery-policy](https://github.com/opsle/agent-recovery-policy) | concept | `1b733a111e26` | `THEORY` | [none; placeholder source directory only](https://github.com/opsle/agent-recovery-policy/blob/1b733a111e26e0a409fee3b96f627048531daefe/THEORY.md); placeholder only; no automated tests | No shared failure schema, attempt ledger, route evaluator, or comparative fixture set. | After decision evidence and route schemas stabilize, define same-failure convergence on synthetic failures. | `agent-routing-policy`, `agent-state-ledger`, `decision-evidence-protocol` | waiting | | 16 | [ephemeral-agent-workers](https://github.com/opsle/ephemeral-agent-workers) | concept | `ad96fcfdfac0` | `THEORY` | [none; placeholder source directory only](https://github.com/opsle/ephemeral-agent-workers/blob/ad96fcfdfac06d340b5e96d369634980cee78ef4/THEORY.md); placeholder only; no automated tests | Portable authority, claim, and handoff contracts are not ready; no safe synthetic containment harness exists. | Wait for prerequisite contracts, then define a fake worker adapter and destruction receipt without infrastructure changes. | `agent-execution-authorization`, `agent-resource-claims`, `verifiable-agent-handoff` | waiting | | 17 | [gearbox](https://github.com/opsle/gearbox) | concept | `f3fab9f292cf` | `PROTOTYPED` | [provider-free Python reference core with strict authority-policy admission, exact deterministic argv execution, content-addressed staged helper context, injected one-shot helper transport, passive process waiting, compact results, raw-artifact accounting, fail-closed budgets, and Visible Value receipts](https://github.com/opsle/gearbox/blob/f3fab9f292cf4eabd7200615d444f98881f57d55/src/opsle_gearbox/core.py); 19 of 19 provider-free automated tests passed locally, in PR #1 CI, and in final-main CI; ruff, shellcheck, actionlint, gitleaks, wheel build, receipt validation, and public raw-locator/hash checks passed | A production-quality bounded helper transport, independently verified isolation and termination, full Context Firewall integration, and a frozen comparative benchmark remain missing. | Freeze a provider-free deterministic-versus-direct baseline and helper-transport conformance corpus before considering any live model/provider run. | `context-firewall`, `decision-evidence-protocol`, `agent-trajectory-profiler`, `agent-routing-policy`, `agent-execution-authorization` | waiting | -| 18 | [research](https://github.com/opsle/research) | program infrastructure | `9ee43197880c` | `PROTOTYPED` | [authoritative 20-repository ledger, machine-readable 17-concept theory registry, canonical theory map, normative Visible Value controls, and provider-free EXP-001 benchmark, launch, and one-block coordinator freezes with six content-addressed tasks, deterministic oracle, four arm contracts, sealed blinded allocation, exact subject configuration and adapter, exact four-authorization admission, private result-envelope templates, exact-revision interoperability, receipts, mutation tests, and integrity CI](program/THEORY_MAP.md); 75 of 75 program-registry, theory-registry, theory-map-hash, generated-status, repository anti-forgetting, EXP-001 freeze/preregistration/coordinator, Gearbox evidence, and Visible Value tests passed locally; PR #13 and exact main CI validated the provider-free coordinator release, and the offline harness replayed all 48 arms | EXP-001 lacks an exact live budgeted authorization set and catalogue/pricing preflight; its one-block coordinator is provider-free qualified, zero model/provider subjects have run, and the program has no canonical measured concept experiment. | Create and provider-free validate one exact four-label LIVE_PROVIDER_RUN authorization set plus a model catalogue/pricing preflight artifact; do not launch a provider/model subject. | — | active | +| 18 | [research](https://github.com/opsle/research) | program infrastructure | `9ee43197880c` | `PROTOTYPED` | [authoritative 20-repository ledger, machine-readable 17-concept theory registry, canonical theory map, normative Visible Value controls, and provider-free EXP-001 benchmark, launch, one-block coordinator, external four-label LIVE_PROVIDER_RUN authorization, and current catalogue/pricing preflight artifacts with six content-addressed tasks, deterministic oracle, four arm contracts, sealed blinded allocation, exact subject configuration and adapter, exact authorization admission, private boundaries, receipts, mutation tests, and integrity CI](program/THEORY_MAP.md); 88 of 88 repository tests pass on the unreleased provider-free live-preflight branch, including 13 authorization validations and two byte-identical replays; PR #13 and exact main CI remain the released coordinator baseline | The exact live authorization set remains unconsumed and unreleased, account-specific API entitlement is unverified under the zero-provider-call policy, no immutable dated model snapshot is documented, and the program has no canonical measured concept experiment. | Independently review and release the provider-free live-authorization and catalogue/pricing preflight; do not consume authorization or launch a provider/model subject. | — | active | | 19 | [site](https://github.com/opsle/site) | program infrastructure | `28ad65be4750` | `PROTOTYPED` | [React/Vinext source implementation with content routes](https://github.com/opsle/site/blob/28ad65be4750dc849976fbf5c9eae9501c6bbb25/README.md); automated build/render tests present; not rerun because this reconciliation kept other repositories read-only | Wait for validated registry data and measured research; deployment requires separate authorization. | After registry merge, add a read-only registry ingestion design without deploying the site. | `research` | waiting | | 20 | [.github](https://github.com/opsle/.github) | program infrastructure | `01c38e726db7` | `THEORY` | [documentation-only organization profile](https://github.com/opsle/.github/blob/01c38e726db7c3e45059d25fccce55e071e35938/profile/README.md); not applicable to current single Markdown profile; consistency is unverified | No mechanical registry consistency check exists in this repository. | After registry merge, design a read-only consistency check for organization-profile repository links. | `research` | waiting | @@ -55,11 +55,11 @@ Canonical map: `program/THEORY_MAP.md`. Machine registry: `program/theory-regist ## Highest-priority workstream -Prepare EXP-001's final non-launch admission boundary: one exact four-label LIVE_PROVIDER_RUN authorization set plus a model catalogue/pricing preflight artifact, validated without launching any provider/model subject. +Independently review and release the provider-free EXP-001 live-authorization and current catalogue/pricing preflight; do not consume authorization or launch any provider/model subject. `EXP-001` — **PLANNED** — How much context can an AI coding agent safely not see? -Blockers: No machine-readable four-label budgeted authorization set exists; zero subject/provider runs are authorized or recorded. No distinct dated gpt-5.6-sol snapshot was publicly listed on 2026-08-29, so immutable model weights cannot be claimed and launch requires a catalogue-drift preflight. +Blockers: The validated live authorization set remains unconsumed; separate execution authority is required before any provider/model subject launch. Account-specific API entitlement remains unverified under the zero-provider-call policy and requires a fresh fail-closed launch-time catalogue check. No distinct dated gpt-5.6-sol snapshot was publicly listed on 2026-08-30, so immutable model weights cannot be claimed. ## Exact recommended next execution diff --git a/experiments/exp-001/README.md b/experiments/exp-001/README.md index 13e5284..5c03e5e 100644 --- a/experiments/exp-001/README.md +++ b/experiments/exp-001/README.md @@ -64,6 +64,13 @@ commitments. Its committed qualification evidence is in Qualification uses non-launchable fixture authorizations and therefore neither authorizes nor executes a provider request. +The unreleased provider-free live admission preparation is recorded in +[`live-preflight-v1/`](live-preflight-v1/). Its public evidence is in +[`program/evidence/exp-001-live-preflight/`](../../program/evidence/exp-001-live-preflight/). +The actual exact four-label `LIVE_PROVIDER_RUN` set remains external to Git and +outside every subject context. It is validated but unconsumed; this preparation +does not authorize lifecycle advancement or launch a subject. + ## Provider-free verification The harness fails closed unless the three pinned public Opsle dependencies are diff --git a/experiments/exp-001/live-preflight-v1/README.md b/experiments/exp-001/live-preflight-v1/README.md new file mode 100644 index 0000000..ad5f458 --- /dev/null +++ b/experiments/exp-001/live-preflight-v1/README.md @@ -0,0 +1,29 @@ +# EXP-001 live authorization and catalogue preflight v1 + +Status: `PROVIDER_FREE_VALIDATION_ONLY`. + +This directory implements the provider-free creation and validation of one +external four-label `LIVE_PROVIDER_RUN` authorization set and the current model +catalogue/pricing preflight for the exact preregistered `gpt-5.6-sol` +configuration. + +The live authorization set is intentionally outside Git and outside every +subject context. The public evidence contains its auditable set ID and content +identity but not the block ID, subject labels, per-label authorization IDs, +allocation seed, arm mapping, raw evidence, or result envelopes. + +`preflight.py create` is exclusive-create: it refuses an existing destination, +creates four unique label-bound records, marks every record `UNCONSUMED`, and +does not expose any provider transport or execution path. `preflight.py qualify` +validates without mutating the set, reproduces it twice from frozen inputs, and +executes the required fail-closed negative cases in temporary copies. + +The catalogue contains only the exact preregistered candidate. Other providers +or models are not eligible without a versioned preregistration amendment. Model +availability and prices come from official OpenAI API documentation; account +entitlement remains unresolved because this task permits zero provider calls. + +This preparation is not an EXP-001 run or result. It consumes no authorization, +launches no subject, creates no result envelope, advances no lifecycle state, +and makes no correctness, token, cost, latency, savings, or causal-benefit +claim. diff --git a/experiments/exp-001/live-preflight-v1/model-catalogue.json b/experiments/exp-001/live-preflight-v1/model-catalogue.json new file mode 100644 index 0000000..826f9e1 --- /dev/null +++ b/experiments/exp-001/live-preflight-v1/model-catalogue.json @@ -0,0 +1,79 @@ +{ + "protocol_version": "opsle.exp001.model-catalogue/v1", + "experiment_id": "EXP-001", + "purpose": "Current provider-free catalogue preflight for the exact preregistered EXP-001 live block configuration; this artifact does not authorize or launch any provider/model subject.", + "retrieved_at": "2026-08-30T18:49:10Z", + "catalogue_scope": "Only the exact preregistered candidate is eligible. Any provider or model substitution requires a versioned preregistration amendment and is outside this catalogue.", + "eligible_candidates": [ + { + "provider": "OpenAI API", + "provider_id": "openai", + "model_id": "gpt-5.6-sol", + "model_family": "GPT-5.6", + "wire_api": "Responses API", + "endpoint": "https://api.openai.com/v1/responses", + "availability": "DOCUMENTED_API_MODEL_ACCOUNT_ACCESS_UNVERIFIED", + "availability_detail": "The official model page documents gpt-5.6-sol on v1/responses and lists API rate tiers. This provider-free task did not query account-specific entitlement.", + "pricing_basis": "USD_PER_1M_TEXT_TOKENS_STANDARD_SHORT_CONTEXT", + "input_price_usd": 4.0, + "cached_input_price_usd": 0.4, + "cache_write_price_usd": 5.0, + "output_price_usd": 20.0, + "long_context_input_price_usd": 8.0, + "long_context_cached_input_price_usd": 0.8, + "long_context_cache_write_price_usd": 10.0, + "long_context_output_price_usd": 30.0, + "long_context_threshold_input_tokens": 272000, + "context_window_tokens": 1050000, + "max_output_tokens": 128000, + "subscription_api_distinction": "Direct API usage pricing; ChatGPT or Codex subscription access is not used and does not authorize this experiment.", + "snapshot_id": null, + "pricing_status": "AUTHORITATIVELY_DOCUMENTED_FOR_DIRECT_API_STANDARD_TIER", + "pricing_promotion": "Official documentation states promotional pricing is available at least through 2026-11-21; a fresh launch-time check remains mandatory.", + "sources": [ + { + "publisher": "OpenAI", + "title": "GPT-5.6 Sol Model", + "url": "https://developers.openai.com/api/docs/models/gpt-5.6-sol", + "retrieved_at": "2026-08-30T18:49:10Z", + "supports": [ + "exact model identifier", + "model family", + "Responses API availability", + "context and output limits", + "short-context prices", + "long-context threshold", + "snapshot listing" + ] + }, + { + "publisher": "OpenAI", + "title": "OpenAI API Pricing", + "url": "https://developers.openai.com/api/docs/pricing", + "retrieved_at": "2026-08-30T18:49:10Z", + "supports": [ + "standard short-context token prices", + "standard long-context token prices", + "cached-input and cache-write prices", + "promotional pricing horizon" + ] + } + ], + "uncertainty_or_unavailable_fields": [ + { + "field": "account_specific_api_entitlement", + "value": null, + "status": "UNRESOLVED_ZERO_PROVIDER_CALL_POLICY" + }, + { + "field": "distinct_dated_snapshot_id", + "value": null, + "status": "UNAVAILABLE_OFFICIAL_PAGE_LISTS_ONLY_PUBLIC_MODEL_ID" + } + ] + } + ], + "authorization_effect": "NONE", + "provider_model_launch_count": 0, + "identity": "sha256:334a48850dacf3d9ad5d5c9088287fecc78925e64e10f974cb63575b74592b3c" +} diff --git a/experiments/exp-001/live-preflight-v1/preflight.py b/experiments/exp-001/live-preflight-v1/preflight.py new file mode 100755 index 0000000..9adb3d3 --- /dev/null +++ b/experiments/exp-001/live-preflight-v1/preflight.py @@ -0,0 +1,948 @@ +#!/usr/bin/env python3 +"""Provider-free EXP-001 live authorization and catalogue preflight.""" + +from __future__ import annotations + +import argparse +import ast +import hashlib +import json +import secrets +import shutil +import tempfile +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +ROOT = Path(__file__).resolve().parents[3] +EXP_ROOT = ROOT / "experiments/exp-001" +PREREG_ROOT = EXP_ROOT / "preregistration-v1" +CATALOGUE_PATH = Path(__file__).with_name("model-catalogue.json") +VALUE_VALIDATOR_PATH = ROOT / "tools/validate_value_receipt.py" + +AUTHORIZATION_PROTOCOL = "opsle.exp001.live-provider-authorization/v1" +SET_PROTOCOL = "opsle.exp001.live-provider-authorization-set/v1" +CATALOGUE_PROTOCOL = "opsle.exp001.model-catalogue/v1" +PRICING_PROTOCOL = "opsle.exp001.pricing-preflight/v1" +REPORT_PROTOCOL = "opsle.exp001.live-preflight-qualification/v1" +LIVE_AUTHORIZATION = "LIVE_PROVIDER_RUN" +FIXTURE_AUTHORIZATION = "PROVIDER_FREE_FIXTURE" + + +class PreflightError(RuntimeError): + """Fail-closed provider-free preflight error.""" + + +def canonical_bytes(value: Any) -> bytes: + return ( + json.dumps( + value, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=False, + ).encode("utf-8") + + b"\n" + ) + + +def sha256_bytes(value: bytes) -> str: + return "sha256:" + hashlib.sha256(value).hexdigest() + + +def object_identity(value: dict[str, Any]) -> str: + payload = {key: item for key, item in value.items() if key != "identity"} + return sha256_bytes(canonical_bytes(payload)) + + +def load_json(path: Path) -> dict[str, Any]: + try: + value = json.loads(path.read_text(encoding="utf-8")) + except (OSError, UnicodeDecodeError, json.JSONDecodeError) as error: + raise PreflightError(f"invalid JSON artifact: {path.name}") from error + if not isinstance(value, dict): + raise PreflightError(f"JSON artifact must be an object: {path.name}") + return value + + +def parse_timestamp(value: Any, field: str) -> datetime: + if not isinstance(value, str) or not value.endswith("Z"): + raise PreflightError(f"{field} must be an RFC3339 UTC timestamp") + try: + parsed = datetime.fromisoformat(value[:-1] + "+00:00") + except ValueError as error: + raise PreflightError(f"{field} must be an RFC3339 UTC timestamp") from error + if parsed.tzinfo != timezone.utc: + raise PreflightError(f"{field} must be UTC") + return parsed + + +def selected_public_block(block_id: str) -> list[dict[str, Any]]: + index = load_json(PREREG_ROOT / "allocation-index.json") + entries = [ + entry + for entry in index.get("entries", []) + if isinstance(entry, dict) and entry.get("block_id") == block_id + ] + if len(entries) != 4: + raise PreflightError("selected public block must contain exactly four labels") + labels = [entry.get("subject_label") for entry in entries] + if any(not isinstance(label, str) or not label for label in labels): + raise PreflightError("selected public block contains a malformed label") + if len(set(labels)) != 4: + raise PreflightError("selected public block contains duplicate labels") + if {entry.get("position") for entry in entries} != {1, 2, 3, 4}: + raise PreflightError("selected public block positions are malformed") + configuration_ids = {entry.get("configuration_id") for entry in entries} + if configuration_ids != {index.get("configuration_id")}: + raise PreflightError("selected public block configuration binding drifted") + return sorted(entries, key=lambda entry: entry["position"]) + + +def first_public_block_id() -> str: + index = load_json(PREREG_ROOT / "allocation-index.json") + entries = index.get("entries") + if not isinstance(entries, list) or not entries: + raise PreflightError("allocation index contains no public block") + block_id = entries[0].get("block_id") + if not isinstance(block_id, str) or not block_id: + raise PreflightError("first public block ID is malformed") + return block_id + + +def authorization_payload( + *, + entry: dict[str, Any], + set_id: str, + authorization_id: str, + issued_at: str, + valid_from: str, + expires_at: str, + preregistration_identity: str, + maximum_spend_usd: float, +) -> dict[str, Any]: + return { + "protocol_version": AUTHORIZATION_PROTOCOL, + "authorization_id": authorization_id, + "authorization_set_id": set_id, + "authorization_class": LIVE_AUTHORIZATION, + "experiment_id": "EXP-001", + "preregistration_identity": preregistration_identity, + "configuration_id": entry["configuration_id"], + "block_id": entry["block_id"], + "subject_label": entry["subject_label"], + "provider_run_authorized": True, + "max_provider_runs": 1, + "maximum_spend_usd": maximum_spend_usd, + "fixture_only": False, + "provider_launch_permitted": True, + "issued_at": issued_at, + "valid_from": valid_from, + "expires_at": expires_at, + "authorization_state": "UNCONSUMED", + "consumed_at": None, + "result_envelope": None, + } + + +def build_set_manifest( + *, + set_id: str, + block_id: str, + issued_at: str, + valid_from: str, + expires_at: str, + authorization_records: list[dict[str, Any]], + authorization_files: dict[str, bytes], + preregistration_identity: str, + configuration_id: str, +) -> dict[str, Any]: + manifest = { + "protocol_version": SET_PROTOCOL, + "authorization_set_id": set_id, + "authorization_class": LIVE_AUTHORIZATION, + "experiment_id": "EXP-001", + "preregistration_identity": preregistration_identity, + "configuration_id": configuration_id, + "block_id": block_id, + "issued_at": issued_at, + "valid_from": valid_from, + "expires_at": expires_at, + "authorization_count": 4, + "authorization_state": "UNCONSUMED", + "authorization_consumption_count": 0, + "result_envelope_count": 0, + "provider_model_launch_count": 0, + "experiment_run_count": 0, + "experiment_result_count": 0, + "authorizations": [ + { + "subject_label": record["subject_label"], + "authorization_id": record["authorization_id"], + "path": f"authorizations/{record['subject_label']}.json", + "sha256": sha256_bytes(authorization_files[record["subject_label"]]), + } + for record in authorization_records + ], + } + manifest["identity"] = object_identity(manifest) + return manifest + + +def materialize_authorization_set( + *, + destination: Path, + block_id: str, + set_id: str, + authorization_ids: list[str], + issued_at: str, + valid_from: str, + expires_at: str, +) -> dict[str, Any]: + if destination.exists(): + raise PreflightError("authorization set destination must be absent") + if len(authorization_ids) != 4 or len(set(authorization_ids)) != 4: + raise PreflightError("exactly four unique authorization IDs are required") + issued = parse_timestamp(issued_at, "issued_at") + starts = parse_timestamp(valid_from, "valid_from") + ends = parse_timestamp(expires_at, "expires_at") + if not issued <= starts < ends: + raise PreflightError("authorization temporal bounds are inconsistent") + + entries = selected_public_block(block_id) + preregistration = load_json(PREREG_ROOT / "preregistration.json") + configuration = load_json(PREREG_ROOT / "subject-config.json") + maximum_spend = configuration["subject_limits"]["maximum_spend_usd_per_subject"] + records = [ + authorization_payload( + entry=entry, + set_id=set_id, + authorization_id=authorization_id, + issued_at=issued_at, + valid_from=valid_from, + expires_at=expires_at, + preregistration_identity=preregistration["identity"], + maximum_spend_usd=maximum_spend, + ) + for entry, authorization_id in zip(entries, authorization_ids, strict=True) + ] + files = {record["subject_label"]: canonical_bytes(record) for record in records} + manifest = build_set_manifest( + set_id=set_id, + block_id=block_id, + issued_at=issued_at, + valid_from=valid_from, + expires_at=expires_at, + authorization_records=records, + authorization_files=files, + preregistration_identity=preregistration["identity"], + configuration_id=configuration["identity"], + ) + + destination.mkdir(mode=0o700, parents=True) + authorization_dir = destination / "authorizations" + authorization_dir.mkdir(mode=0o700) + for label, content in files.items(): + path = authorization_dir / f"{label}.json" + path.write_bytes(content) + path.chmod(0o600) + manifest_path = destination / "set-manifest.json" + manifest_path.write_bytes(canonical_bytes(manifest)) + manifest_path.chmod(0o600) + return manifest + + +def validate_authorization_set( + set_directory: Path, + *, + validated_at: str, +) -> dict[str, Any]: + validation_time = parse_timestamp(validated_at, "validated_at") + if not set_directory.is_dir() or set_directory.is_symlink(): + raise PreflightError("authorization set directory is unavailable or unsafe") + manifest_path = set_directory / "set-manifest.json" + if manifest_path.is_symlink() or not manifest_path.is_file(): + raise PreflightError("authorization set manifest is unavailable or unsafe") + manifest = load_json(manifest_path) + if manifest.get("protocol_version") != SET_PROTOCOL: + raise PreflightError("authorization set protocol is malformed") + if manifest.get("identity") != object_identity(manifest): + raise PreflightError("authorization set identity mismatch") + if manifest.get("authorization_count") != 4: + raise PreflightError("authorization set must declare exactly four labels") + + block_id = manifest.get("block_id") + if not isinstance(block_id, str): + raise PreflightError("authorization set block binding is malformed") + entries = selected_public_block(block_id) + expected_by_label = {entry["subject_label"]: entry for entry in entries} + authorization_dir = set_directory / "authorizations" + if not authorization_dir.is_dir() or authorization_dir.is_symlink(): + raise PreflightError("authorization directory is unavailable or unsafe") + expected_names = {f"{label}.json" for label in expected_by_label} + observed_names = {path.name for path in authorization_dir.iterdir()} + if observed_names != expected_names: + raise PreflightError("authorization directory must contain exactly four labels") + + preregistration = load_json(PREREG_ROOT / "preregistration.json") + configuration = load_json(PREREG_ROOT / "subject-config.json") + required_set = { + "authorization_class": LIVE_AUTHORIZATION, + "experiment_id": "EXP-001", + "preregistration_identity": preregistration["identity"], + "configuration_id": configuration["identity"], + "authorization_state": "UNCONSUMED", + "authorization_consumption_count": 0, + "result_envelope_count": 0, + "provider_model_launch_count": 0, + "experiment_run_count": 0, + "experiment_result_count": 0, + } + if any(manifest.get(key) != value for key, value in required_set.items()): + raise PreflightError("authorization set binding or state drifted") + starts = parse_timestamp(manifest.get("valid_from"), "valid_from") + ends = parse_timestamp(manifest.get("expires_at"), "expires_at") + if validation_time < starts: + raise PreflightError("authorization set is not yet valid") + if validation_time >= ends: + raise PreflightError("authorization set is expired") + + records: list[dict[str, Any]] = [] + labels: set[str] = set() + authorization_ids: set[str] = set() + manifest_records = manifest.get("authorizations") + if not isinstance(manifest_records, list) or len(manifest_records) != 4: + raise PreflightError("authorization manifest must contain exactly four labels") + manifest_by_label = { + item.get("subject_label"): item + for item in manifest_records + if isinstance(item, dict) + } + if set(manifest_by_label) != set(expected_by_label): + raise PreflightError("authorization manifest labels are missing or duplicated") + + for label, entry in expected_by_label.items(): + path = authorization_dir / f"{label}.json" + if path.is_symlink() or not path.is_file(): + raise PreflightError("authorization files must be regular files") + content = path.read_bytes() + record = load_json(path) + required = { + "protocol_version": AUTHORIZATION_PROTOCOL, + "authorization_set_id": manifest["authorization_set_id"], + "authorization_class": LIVE_AUTHORIZATION, + "experiment_id": "EXP-001", + "preregistration_identity": preregistration["identity"], + "configuration_id": configuration["identity"], + "block_id": block_id, + "subject_label": label, + "provider_run_authorized": True, + "max_provider_runs": 1, + "fixture_only": False, + "provider_launch_permitted": True, + "authorization_state": "UNCONSUMED", + "consumed_at": None, + "result_envelope": None, + } + if any(record.get(key) != value for key, value in required.items()): + raise PreflightError("authorization schema, class, or binding drifted") + if record.get("block_id") != entry["block_id"]: + raise PreflightError("authorization block binding drifted") + authorization_id = record.get("authorization_id") + if not isinstance(authorization_id, str) or not authorization_id: + raise PreflightError("authorization identity is malformed") + if label in labels: + raise PreflightError("authorization labels must be unique") + if authorization_id in authorization_ids: + raise PreflightError("authorization identities must be unique") + labels.add(label) + authorization_ids.add(authorization_id) + budget = record.get("maximum_spend_usd") + required_budget = configuration["subject_limits"][ + "maximum_spend_usd_per_subject" + ] + if not isinstance(budget, (int, float)) or budget < required_budget: + raise PreflightError("authorization budget is below the frozen ceiling") + record_start = parse_timestamp(record.get("valid_from"), "valid_from") + record_end = parse_timestamp(record.get("expires_at"), "expires_at") + if validation_time < record_start: + raise PreflightError("authorization is not yet valid") + if validation_time >= record_end: + raise PreflightError("authorization is expired") + manifest_record = manifest_by_label[label] + if ( + manifest_record.get("authorization_id") != authorization_id + or manifest_record.get("path") != f"authorizations/{label}.json" + or manifest_record.get("sha256") != sha256_bytes(content) + ): + raise PreflightError("authorization manifest binding mismatch") + records.append(record) + + return { + "authorization_set_id": manifest["authorization_set_id"], + "authorization_set_identity": manifest["identity"], + "authorization_class": LIVE_AUTHORIZATION, + "authorization_count": len(records), + "label_count": len(labels), + "authorization_consumption_count": 0, + "result_envelope_count": 0, + "provider_model_launch_count": 0, + "experiment_run_count": 0, + "experiment_result_count": 0, + "valid": True, + } + + +def validate_catalogue(catalogue: dict[str, Any]) -> dict[str, Any]: + if catalogue.get("protocol_version") != CATALOGUE_PROTOCOL: + raise PreflightError("model catalogue protocol is malformed") + if catalogue.get("identity") != object_identity(catalogue): + raise PreflightError("model catalogue identity mismatch") + candidates = catalogue.get("eligible_candidates") + if not isinstance(candidates, list) or len(candidates) != 1: + raise PreflightError("catalogue must contain the one preregistered candidate") + candidate = candidates[0] + configuration = load_json(PREREG_ROOT / "subject-config.json") + model = configuration["model_provider_configuration"] + required = { + "provider_id": model["provider_id"], + "model_id": model["model_id"], + "wire_api": model["wire_api"], + "availability": "DOCUMENTED_API_MODEL_ACCOUNT_ACCESS_UNVERIFIED", + "pricing_basis": "USD_PER_1M_TEXT_TOKENS_STANDARD_SHORT_CONTEXT", + "input_price_usd": 4.0, + "cached_input_price_usd": 0.4, + "output_price_usd": 20.0, + "context_window_tokens": 1_050_000, + "max_output_tokens": 128_000, + } + if any(candidate.get(key) != value for key, value in required.items()): + raise PreflightError("catalogue candidate drifted from current documentation") + if candidate.get("subscription_api_distinction") != ( + "Direct API usage pricing; ChatGPT or Codex subscription access is not " + "used and does not authorize this experiment." + ): + raise PreflightError("catalogue API/subscription distinction is missing") + sources = candidate.get("sources") + if not isinstance(sources, list) or len(sources) < 2: + raise PreflightError("catalogue provenance is incomplete") + parse_timestamp(catalogue.get("retrieved_at"), "retrieved_at") + return candidate + + +def pricing_preflight(catalogue: dict[str, Any]) -> dict[str, Any]: + candidate = validate_catalogue(catalogue) + configuration = load_json(PREREG_ROOT / "subject-config.json") + limits = configuration["subject_limits"] + input_ceiling = limits["max_api_calls"] * limits["max_request_body_bytes"] + output_ceiling = limits["max_api_calls"] * limits["max_output_tokens_per_response"] + spend_ceiling = ( + input_ceiling * candidate["input_price_usd"] / 1_000_000 + + output_ceiling * candidate["output_price_usd"] / 1_000_000 + ) + result = { + "protocol_version": PRICING_PROTOCOL, + "experiment_id": "EXP-001", + "subject_configuration_id": configuration["identity"], + "model_catalogue_identity": catalogue["identity"], + "provider_id": candidate["provider_id"], + "model_id": candidate["model_id"], + "pricing_basis": candidate["pricing_basis"], + "input_price_usd_per_million_tokens": candidate["input_price_usd"], + "cached_input_price_usd_per_million_tokens": candidate[ + "cached_input_price_usd" + ], + "output_price_usd_per_million_tokens": candidate["output_price_usd"], + "conservative_input_token_ceiling": input_ceiling, + "conservative_output_token_ceiling": output_ceiling, + "derived_spend_ceiling_usd": spend_ceiling, + "registered_spend_ceiling_usd": limits["maximum_spend_usd_per_subject"], + "long_context_threshold_tokens": 272_000, + "long_context_multiplier_applies": False, + "price_drift_from_frozen_configuration": False, + "account_access_verified": False, + "provider_call_count": 0, + "admission": "PASS_PROVIDER_FREE_CURRENT_DOCUMENTATION", + "limitations": [ + "Provider documentation establishes catalogue availability, not this account's entitlement.", + "No API request was made and no provider/model subject was launched.", + "Promotional pricing may change; launch requires a fresh fail-closed catalogue check.", + ], + } + if spend_ceiling > limits["maximum_spend_usd_per_subject"]: + raise PreflightError("current pricing exceeds the registered spend ceiling") + result["identity"] = object_identity(result) + return result + + +def snapshot(directory: Path) -> dict[str, bytes]: + return { + path.relative_to(directory).as_posix(): path.read_bytes() + for path in sorted(item for item in directory.rglob("*") if item.is_file()) + } + + +def replay_inputs(manifest: dict[str, Any]) -> dict[str, Any]: + records = manifest.get("authorizations") + if not isinstance(records, list) or len(records) != 4: + raise PreflightError("authorization replay inputs are malformed") + return { + "block_id": manifest["block_id"], + "set_id": manifest["authorization_set_id"], + "authorization_ids": [record["authorization_id"] for record in records], + "issued_at": manifest["issued_at"], + "valid_from": manifest["valid_from"], + "expires_at": manifest["expires_at"], + } + + +def mutate_case(source: Path, root: Path, name: str) -> Path: + destination = root / name + shutil.copytree(source, destination) + return destination + + +def first_authorization_path(directory: Path) -> Path: + return min((directory / "authorizations").glob("*.json")) + + +def second_authorization_path(directory: Path) -> Path: + return sorted((directory / "authorizations").glob("*.json"))[1] + + +def rewrite_record(path: Path, mutator: Any) -> None: + record = load_json(path) + mutator(record) + path.write_bytes(canonical_bytes(record)) + + +def rewrite_manifest(directory: Path, mutator: Any) -> None: + path = directory / "set-manifest.json" + manifest = load_json(path) + mutator(manifest) + manifest["identity"] = object_identity(manifest) + path.write_bytes(canonical_bytes(manifest)) + + +def expect_rejection( + directory: Path, + *, + validated_at: str, + expected: str, +) -> None: + try: + validate_authorization_set(directory, validated_at=validated_at) + except PreflightError as error: + if expected not in str(error): + raise PreflightError( + f"negative validation returned unexpected reason: {error}" + ) from error + else: + raise PreflightError("negative authorization validation unexpectedly passed") + + +def provider_free_source_audit() -> None: + tree = ast.parse(Path(__file__).read_text(encoding="utf-8")) + forbidden_modules = {"http", "requests", "socket", "subprocess", "urllib"} + forbidden_calls = {"exec", "eval", "system", "popen", "spawn"} + for node in ast.walk(tree): + if isinstance(node, ast.Import) and any( + alias.name.split(".")[0] in forbidden_modules for alias in node.names + ): + raise PreflightError("validation path imports provider-capable transport") + if ( + isinstance(node, ast.ImportFrom) + and (node.module or "").split(".")[0] in forbidden_modules + ): + raise PreflightError("validation path imports provider-capable transport") + if isinstance(node, ast.Call): + name = None + if isinstance(node.func, ast.Name): + name = node.func.id.lower() + elif isinstance(node.func, ast.Attribute): + name = node.func.attr.lower() + if name in forbidden_calls: + raise PreflightError( + "validation path contains process-launch capability" + ) + + +def measurement(identity: str, result: int, evidence_ref: str) -> dict[str, Any]: + return { + "id": identity, + "unit": "count", + "class": "EXACT", + "direction": "PROTECTION_SIGNAL", + "baseline": None, + "result": result, + "delta": None, + "derivation": None, + "source_verification": "VERIFIED", + "evidence_refs": [evidence_ref], + "operator_display": True, + "aggregation": {"method": "SUM", "safe": True}, + "limitations": [], + } + + +def value_receipt( + *, + set_id: str, + set_identity: str, + catalogue_identity: str, + pricing_identity: str, + validation_count: int, +) -> dict[str, Any]: + return { + "schema": "opsle.value-receipt.v1", + "mechanism": { + "id": "opsle.exp001-live-provider-preflight", + "name": "EXP-001 Live Provider Authorization Preflight", + "version": "1.0.0", + "revision": sha256_bytes(Path(__file__).read_bytes()), + }, + "operation": { + "id": set_id, + "name": "prepare-live-authorization-set", + "configuration_id": catalogue_identity, + "policy_id": pricing_identity, + }, + "run": { + "id": set_id, + "repository": "opsle/research", + "task_classification": "EXP-001_PROVIDER_FREE_LIVE_AUTHORIZATION_PREFLIGHT", + "work_classification": "DETERMINISTIC_COORDINATION", + }, + "measurements": [ + measurement("live_authorization_label_count", 4, "authorization_set"), + measurement( + "authorization_validation_count", validation_count, "qualification" + ), + measurement("subject_rendering_count", 0, "qualification"), + measurement( + "subject_visible_canonical_arm_identifier_count", 0, "qualification" + ), + measurement("provider_model_launch_count", 0, "qualification"), + measurement("authorization_consumption_count", 0, "authorization_set"), + measurement("experiment_run_count", 0, "qualification"), + measurement("experiment_result_count", 0, "qualification"), + ], + "evidence": [ + { + "id": "authorization_set", + "kind": "CONTENT_HASH", + "locator": set_identity, + "trust": "VERIFIED", + }, + { + "id": "model_catalogue", + "kind": "CONTENT_HASH", + "locator": catalogue_identity, + "trust": "VERIFIED", + }, + { + "id": "pricing_preflight", + "kind": "CONTENT_HASH", + "locator": pricing_identity, + "trust": "VERIFIED", + }, + { + "id": "qualification", + "kind": "RUN_ARTIFACT", + "locator": set_id, + "trust": "VERIFIED", + }, + ], + "limitations": [ + "Preparation and validation do not consume authorization or execute a provider/model subject.", + "No subject rendering or result envelope was created by this path.", + "Account-specific model access is unverified because no provider call was made.", + "No token, cost, latency, correctness, savings, or causal-benefit claim is made.", + ], + } + + +def qualify( + *, + set_directory: Path, + catalogue_path: Path, + validated_at: str, +) -> dict[str, bytes]: + provider_free_source_audit() + catalogue = load_json(catalogue_path) + pricing = pricing_preflight(catalogue) + validation = validate_authorization_set( + set_directory, + validated_at=validated_at, + ) + manifest = load_json(set_directory / "set-manifest.json") + inputs = replay_inputs(manifest) + validation_count = 1 + + with tempfile.TemporaryDirectory(prefix="opsle-exp001-live-preflight-") as temp: + root = Path(temp) + first = root / "replay-first" + second = root / "replay-second" + materialize_authorization_set(destination=first, **inputs) + materialize_authorization_set(destination=second, **inputs) + first_validation = validate_authorization_set(first, validated_at=validated_at) + second_validation = validate_authorization_set( + second, validated_at=validated_at + ) + validation_count += 2 + if first_validation != second_validation or snapshot(first) != snapshot(second): + raise PreflightError("deterministic authorization replay drifted") + if snapshot(set_directory) != snapshot(first): + raise PreflightError( + "created authorization set differs from deterministic replay" + ) + + fixture = mutate_case(set_directory, root, "fixture") + fixture_path = first_authorization_path(fixture) + rewrite_record( + fixture_path, + lambda record: record.update( + { + "authorization_class": FIXTURE_AUTHORIZATION, + "fixture_only": True, + "provider_launch_permitted": False, + } + ), + ) + expect_rejection( + fixture, + validated_at=validated_at, + expected="class, or binding", + ) + validation_count += 1 + + duplicate_label = mutate_case(set_directory, root, "duplicate-label") + first_record = load_json(first_authorization_path(duplicate_label)) + rewrite_record( + second_authorization_path(duplicate_label), + lambda record: record.update( + {"subject_label": first_record["subject_label"]} + ), + ) + expect_rejection( + duplicate_label, + validated_at=validated_at, + expected="class, or binding", + ) + validation_count += 1 + + duplicate_identity = mutate_case(set_directory, root, "duplicate-identity") + first_record = load_json(first_authorization_path(duplicate_identity)) + rewrite_record( + second_authorization_path(duplicate_identity), + lambda record: record.update( + {"authorization_id": first_record["authorization_id"]} + ), + ) + expect_rejection( + duplicate_identity, + validated_at=validated_at, + expected="identities must be unique", + ) + validation_count += 1 + + missing = mutate_case(set_directory, root, "missing") + first_authorization_path(missing).unlink() + expect_rejection(missing, validated_at=validated_at, expected="exactly four") + validation_count += 1 + + extra = mutate_case(set_directory, root, "extra") + (extra / "authorizations" / "unexpected.json").write_text( + "{}\n", encoding="utf-8" + ) + expect_rejection(extra, validated_at=validated_at, expected="exactly four") + validation_count += 1 + + wrong_experiment = mutate_case(set_directory, root, "wrong-experiment") + rewrite_record( + first_authorization_path(wrong_experiment), + lambda record: record.update({"experiment_id": "EXP-OTHER"}), + ) + expect_rejection( + wrong_experiment, + validated_at=validated_at, + expected="class, or binding", + ) + validation_count += 1 + + wrong_block = mutate_case(set_directory, root, "wrong-block") + rewrite_record( + first_authorization_path(wrong_block), + lambda record: record.update({"block_id": "wrong-block"}), + ) + expect_rejection( + wrong_block, + validated_at=validated_at, + expected="class, or binding", + ) + validation_count += 1 + + malformed = mutate_case(set_directory, root, "malformed") + first_authorization_path(malformed).write_bytes(b"{\n") + expect_rejection(malformed, validated_at=validated_at, expected="invalid JSON") + validation_count += 1 + + expired = mutate_case(set_directory, root, "expired") + rewrite_manifest( + expired, + lambda value: value.update({"expires_at": "2026-08-30T00:00:00Z"}), + ) + expect_rejection(expired, validated_at=validated_at, expected="expired") + validation_count += 1 + + not_yet_valid = mutate_case(set_directory, root, "not-yet-valid") + rewrite_manifest( + not_yet_valid, + lambda value: value.update({"valid_from": "2026-12-01T00:00:00Z"}), + ) + expect_rejection( + not_yet_valid, + validated_at=validated_at, + expected="not yet valid", + ) + validation_count += 1 + + if validation_count != 13: + raise PreflightError("authorization validation accounting drifted") + + receipt = value_receipt( + set_id=validation["authorization_set_id"], + set_identity=validation["authorization_set_identity"], + catalogue_identity=catalogue["identity"], + pricing_identity=pricing["identity"], + validation_count=validation_count, + ) + report = { + "protocol_version": REPORT_PROTOCOL, + "experiment_id": "EXP-001", + "qualification": "PASS", + "classification": "PROVIDER_FREE_LIVE_AUTHORIZATION_PREFLIGHT", + "validated_at": validated_at, + "authorization_set_id": validation["authorization_set_id"], + "authorization_set_identity": validation["authorization_set_identity"], + "authorization_class": LIVE_AUTHORIZATION, + "live_authorization_label_count": 4, + "live_authorization_record_count": 4, + "authorization_validation_count": validation_count, + "fixture_rejected_from_live_gate": True, + "duplicate_label_rejected": True, + "duplicate_authorization_identity_rejected": True, + "missing_label_rejected": True, + "extra_label_rejected": True, + "wrong_experiment_binding_rejected": True, + "wrong_block_binding_rejected": True, + "malformed_authorization_rejected": True, + "expired_authorization_rejected": True, + "not_yet_valid_authorization_rejected": True, + "deterministic_replay_count": 2, + "deterministic_payloads_byte_identical": True, + "model_catalogue_identity": catalogue["identity"], + "pricing_preflight_identity": pricing["identity"], + "catalogue_candidate_count": 1, + "catalogue_unresolved_field_count": 2, + "subject_rendering_count": 0, + "subject_visible_canonical_arm_identifier_count": 0, + "private_mapping_access_count": 0, + "raw_evidence_access_count": 0, + "provider_model_launch_count": 0, + "authorization_consumption_count": 0, + "result_envelope_count": 0, + "experiment_run_count": 0, + "experiment_result_count": 0, + "lifecycle_status": "PLANNED", + "lifecycle_impact": "NONE_NO_ADVANCEMENT", + "value_receipt_sha256": sha256_bytes(canonical_bytes(receipt)), + "limitations": [ + "This provider-free preparation does not establish account-specific model access.", + "No subject was rendered or launched and no authorization was consumed.", + "No experiment run, result envelope, or experiment result was created.", + "No token, cost, latency, correctness, savings, or causal-benefit claim is made.", + ], + } + public_bytes = canonical_bytes(report) + forbidden = ( + b'"subject_label"', + b'"authorization_id"', + manifest["block_id"].encode("utf-8"), + ) + if any(value in public_bytes for value in forbidden): + raise PreflightError("public qualification evidence leaks private set detail") + return { + "model-catalogue.json": canonical_bytes(catalogue), + "pricing-preflight.json": canonical_bytes(pricing), + "qualification-report.json": public_bytes, + "value-receipt.json": canonical_bytes(receipt), + } + + +def write_evidence(destination: Path, evidence: dict[str, bytes]) -> None: + if destination.exists(): + raise PreflightError("evidence destination must be absent") + destination.mkdir(mode=0o755, parents=True) + for name, content in evidence.items(): + (destination / name).write_bytes(content) + + +def new_id(prefix: str) -> str: + return f"{prefix}-{secrets.token_hex(16)}" + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser() + subparsers = parser.add_subparsers(dest="command", required=True) + create = subparsers.add_parser("create") + create.add_argument("--destination", type=Path, required=True) + create.add_argument("--block-id", default=None) + create.add_argument("--set-id", default=None) + create.add_argument("--issued-at", required=True) + create.add_argument("--valid-from", required=True) + create.add_argument("--expires-at", required=True) + qualify_parser = subparsers.add_parser("qualify") + qualify_parser.add_argument("--authorization-set", type=Path, required=True) + qualify_parser.add_argument("--catalogue", type=Path, default=CATALOGUE_PATH) + qualify_parser.add_argument("--validated-at", required=True) + qualify_parser.add_argument("--evidence-dir", type=Path) + args = parser.parse_args(argv) + try: + if args.command == "create": + manifest = materialize_authorization_set( + destination=args.destination, + block_id=args.block_id or first_public_block_id(), + set_id=args.set_id or new_id("exp001-live-authset"), + authorization_ids=[new_id("exp001-live-authz") for _ in range(4)], + issued_at=args.issued_at, + valid_from=args.valid_from, + expires_at=args.expires_at, + ) + print( + canonical_bytes( + { + "authorization_set_id": manifest["authorization_set_id"], + "authorization_count": 4, + "authorization_consumption_count": 0, + "provider_model_launch_count": 0, + } + ).decode("utf-8"), + end="", + ) + else: + evidence = qualify( + set_directory=args.authorization_set, + catalogue_path=args.catalogue, + validated_at=args.validated_at, + ) + if args.evidence_dir is not None: + write_evidence(args.evidence_dir, evidence) + report = json.loads(evidence["qualification-report.json"]) + print(canonical_bytes(report).decode("utf-8"), end="") + return 0 + except PreflightError as error: + parser.error(str(error)) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/program/PRIORITY.md b/program/PRIORITY.md index 722cef9..7561545 100644 --- a/program/PRIORITY.md +++ b/program/PRIORITY.md @@ -82,7 +82,7 @@ conversation-local. ## Exact next execution -In `opsle/research`, create and provider-free validate one exact four-label -`LIVE_PROVIDER_RUN` authorization set plus a current model catalogue/pricing -preflight artifact. Do not consume authorization or launch a provider/model -subject. EXP-001 has no technical dependency on Gearbox. +In `opsle/research`, independently review and release the provider-free exact +four-label `LIVE_PROVIDER_RUN` authorization set and current model +catalogue/pricing preflight branch. Do not consume authorization or launch a +provider/model subject. EXP-001 has no technical dependency on Gearbox. diff --git a/program/evidence/exp-001-live-preflight/README.md b/program/evidence/exp-001-live-preflight/README.md new file mode 100644 index 0000000..d2561f6 --- /dev/null +++ b/program/evidence/exp-001-live-preflight/README.md @@ -0,0 +1,34 @@ +# EXP-001 live authorization/provider preflight evidence + +This directory records provider-free validation of one external, exact +four-label `LIVE_PROVIDER_RUN` authorization set and the current catalogue and +pricing preflight for the preregistered `gpt-5.6-sol` configuration. + +The external authorization material is not committed. These public artifacts +contain only the authorization set ID and content identity, aggregate counts, +the current model catalogue, pricing derivation, validation outcomes, and the +provider-free value receipt. They omit the selected block ID, subject labels, +per-label authorization IDs, allocation seed, arm mapping, raw evidence, and +result envelopes. + +Current provider facts were retrieved at `2026-08-30T18:49:10Z` from official +OpenAI documentation: + +- +- + +The current direct-API standard short-context prices match the frozen EXP-001 +configuration: USD 4.00 input, USD 0.40 cached input, and USD 20.00 output per +million text tokens. The official model page lists a 1,050,000-token context +window and 128,000 maximum output tokens. Account-specific entitlement and a +distinct dated snapshot remain unresolved; no provider call was made. + +Qualification validates 13 authorization-set cases, including two +byte-identical deterministic replays and the required fail-closed negative +cases. It records four live labels, zero subject renderings, zero subject-visible +canonical arm identifiers, zero provider/model launches, zero authorization +consumptions, zero result envelopes, and zero experiment runs/results. + +EXP-001 remains `PLANNED`. This evidence does not establish model correctness, +token or cost savings, latency savings, comparative performance, or causal +benefit. diff --git a/program/evidence/exp-001-live-preflight/model-catalogue.json b/program/evidence/exp-001-live-preflight/model-catalogue.json new file mode 100644 index 0000000..e14c601 --- /dev/null +++ b/program/evidence/exp-001-live-preflight/model-catalogue.json @@ -0,0 +1 @@ +{"authorization_effect":"NONE","catalogue_scope":"Only the exact preregistered candidate is eligible. Any provider or model substitution requires a versioned preregistration amendment and is outside this catalogue.","eligible_candidates":[{"availability":"DOCUMENTED_API_MODEL_ACCOUNT_ACCESS_UNVERIFIED","availability_detail":"The official model page documents gpt-5.6-sol on v1/responses and lists API rate tiers. This provider-free task did not query account-specific entitlement.","cache_write_price_usd":5.0,"cached_input_price_usd":0.4,"context_window_tokens":1050000,"endpoint":"https://api.openai.com/v1/responses","input_price_usd":4.0,"long_context_cache_write_price_usd":10.0,"long_context_cached_input_price_usd":0.8,"long_context_input_price_usd":8.0,"long_context_output_price_usd":30.0,"long_context_threshold_input_tokens":272000,"max_output_tokens":128000,"model_family":"GPT-5.6","model_id":"gpt-5.6-sol","output_price_usd":20.0,"pricing_basis":"USD_PER_1M_TEXT_TOKENS_STANDARD_SHORT_CONTEXT","pricing_promotion":"Official documentation states promotional pricing is available at least through 2026-11-21; a fresh launch-time check remains mandatory.","pricing_status":"AUTHORITATIVELY_DOCUMENTED_FOR_DIRECT_API_STANDARD_TIER","provider":"OpenAI API","provider_id":"openai","snapshot_id":null,"sources":[{"publisher":"OpenAI","retrieved_at":"2026-08-30T18:49:10Z","supports":["exact model identifier","model family","Responses API availability","context and output limits","short-context prices","long-context threshold","snapshot listing"],"title":"GPT-5.6 Sol Model","url":"https://developers.openai.com/api/docs/models/gpt-5.6-sol"},{"publisher":"OpenAI","retrieved_at":"2026-08-30T18:49:10Z","supports":["standard short-context token prices","standard long-context token prices","cached-input and cache-write prices","promotional pricing horizon"],"title":"OpenAI API Pricing","url":"https://developers.openai.com/api/docs/pricing"}],"subscription_api_distinction":"Direct API usage pricing; ChatGPT or Codex subscription access is not used and does not authorize this experiment.","uncertainty_or_unavailable_fields":[{"field":"account_specific_api_entitlement","status":"UNRESOLVED_ZERO_PROVIDER_CALL_POLICY","value":null},{"field":"distinct_dated_snapshot_id","status":"UNAVAILABLE_OFFICIAL_PAGE_LISTS_ONLY_PUBLIC_MODEL_ID","value":null}],"wire_api":"Responses API"}],"experiment_id":"EXP-001","identity":"sha256:334a48850dacf3d9ad5d5c9088287fecc78925e64e10f974cb63575b74592b3c","protocol_version":"opsle.exp001.model-catalogue/v1","provider_model_launch_count":0,"purpose":"Current provider-free catalogue preflight for the exact preregistered EXP-001 live block configuration; this artifact does not authorize or launch any provider/model subject.","retrieved_at":"2026-08-30T18:49:10Z"} diff --git a/program/evidence/exp-001-live-preflight/pricing-preflight.json b/program/evidence/exp-001-live-preflight/pricing-preflight.json new file mode 100644 index 0000000..fbc8305 --- /dev/null +++ b/program/evidence/exp-001-live-preflight/pricing-preflight.json @@ -0,0 +1 @@ +{"account_access_verified":false,"admission":"PASS_PROVIDER_FREE_CURRENT_DOCUMENTATION","cached_input_price_usd_per_million_tokens":0.4,"conservative_input_token_ceiling":1440000,"conservative_output_token_ceiling":24576,"derived_spend_ceiling_usd":6.25152,"experiment_id":"EXP-001","identity":"sha256:a19e793c2c2453d949687f614bee65be84b9abfa6218ea25c5302af7608a38db","input_price_usd_per_million_tokens":4.0,"limitations":["Provider documentation establishes catalogue availability, not this account's entitlement.","No API request was made and no provider/model subject was launched.","Promotional pricing may change; launch requires a fresh fail-closed catalogue check."],"long_context_multiplier_applies":false,"long_context_threshold_tokens":272000,"model_catalogue_identity":"sha256:334a48850dacf3d9ad5d5c9088287fecc78925e64e10f974cb63575b74592b3c","model_id":"gpt-5.6-sol","output_price_usd_per_million_tokens":20.0,"price_drift_from_frozen_configuration":false,"pricing_basis":"USD_PER_1M_TEXT_TOKENS_STANDARD_SHORT_CONTEXT","protocol_version":"opsle.exp001.pricing-preflight/v1","provider_call_count":0,"provider_id":"openai","registered_spend_ceiling_usd":6.3,"subject_configuration_id":"sha256:bc46a7d72ab776db966e84d9d473efbd0a7e2028ce993eecf4a09d22577ee6cf"} diff --git a/program/evidence/exp-001-live-preflight/qualification-report.json b/program/evidence/exp-001-live-preflight/qualification-report.json new file mode 100644 index 0000000..595f698 --- /dev/null +++ b/program/evidence/exp-001-live-preflight/qualification-report.json @@ -0,0 +1 @@ +{"authorization_class":"LIVE_PROVIDER_RUN","authorization_consumption_count":0,"authorization_set_id":"exp001-live-authset-03703f1af7db48c4eab9fe33b8f55073","authorization_set_identity":"sha256:8259f5b664235e5914ad4a53966a1155f8f90ad2014dc704f50ea394af32d0ec","authorization_validation_count":13,"catalogue_candidate_count":1,"catalogue_unresolved_field_count":2,"classification":"PROVIDER_FREE_LIVE_AUTHORIZATION_PREFLIGHT","deterministic_payloads_byte_identical":true,"deterministic_replay_count":2,"duplicate_authorization_identity_rejected":true,"duplicate_label_rejected":true,"experiment_id":"EXP-001","experiment_result_count":0,"experiment_run_count":0,"expired_authorization_rejected":true,"extra_label_rejected":true,"fixture_rejected_from_live_gate":true,"lifecycle_impact":"NONE_NO_ADVANCEMENT","lifecycle_status":"PLANNED","limitations":["This provider-free preparation does not establish account-specific model access.","No subject was rendered or launched and no authorization was consumed.","No experiment run, result envelope, or experiment result was created.","No token, cost, latency, correctness, savings, or causal-benefit claim is made."],"live_authorization_label_count":4,"live_authorization_record_count":4,"malformed_authorization_rejected":true,"missing_label_rejected":true,"model_catalogue_identity":"sha256:334a48850dacf3d9ad5d5c9088287fecc78925e64e10f974cb63575b74592b3c","not_yet_valid_authorization_rejected":true,"pricing_preflight_identity":"sha256:a19e793c2c2453d949687f614bee65be84b9abfa6218ea25c5302af7608a38db","private_mapping_access_count":0,"protocol_version":"opsle.exp001.live-preflight-qualification/v1","provider_model_launch_count":0,"qualification":"PASS","raw_evidence_access_count":0,"result_envelope_count":0,"subject_rendering_count":0,"subject_visible_canonical_arm_identifier_count":0,"validated_at":"2026-08-30T18:49:10Z","value_receipt_sha256":"sha256:0d5d00262a660fc09590dc9a427435aefe7bdf5eebe917abe91b2587a8e0167d","wrong_block_binding_rejected":true,"wrong_experiment_binding_rejected":true} diff --git a/program/evidence/exp-001-live-preflight/value-receipt.json b/program/evidence/exp-001-live-preflight/value-receipt.json new file mode 100644 index 0000000..2f65601 --- /dev/null +++ b/program/evidence/exp-001-live-preflight/value-receipt.json @@ -0,0 +1 @@ +{"evidence":[{"id":"authorization_set","kind":"CONTENT_HASH","locator":"sha256:8259f5b664235e5914ad4a53966a1155f8f90ad2014dc704f50ea394af32d0ec","trust":"VERIFIED"},{"id":"model_catalogue","kind":"CONTENT_HASH","locator":"sha256:334a48850dacf3d9ad5d5c9088287fecc78925e64e10f974cb63575b74592b3c","trust":"VERIFIED"},{"id":"pricing_preflight","kind":"CONTENT_HASH","locator":"sha256:a19e793c2c2453d949687f614bee65be84b9abfa6218ea25c5302af7608a38db","trust":"VERIFIED"},{"id":"qualification","kind":"RUN_ARTIFACT","locator":"exp001-live-authset-03703f1af7db48c4eab9fe33b8f55073","trust":"VERIFIED"}],"limitations":["Preparation and validation do not consume authorization or execute a provider/model subject.","No subject rendering or result envelope was created by this path.","Account-specific model access is unverified because no provider call was made.","No token, cost, latency, correctness, savings, or causal-benefit claim is made."],"measurements":[{"aggregation":{"method":"SUM","safe":true},"baseline":null,"class":"EXACT","delta":null,"derivation":null,"direction":"PROTECTION_SIGNAL","evidence_refs":["authorization_set"],"id":"live_authorization_label_count","limitations":[],"operator_display":true,"result":4,"source_verification":"VERIFIED","unit":"count"},{"aggregation":{"method":"SUM","safe":true},"baseline":null,"class":"EXACT","delta":null,"derivation":null,"direction":"PROTECTION_SIGNAL","evidence_refs":["qualification"],"id":"authorization_validation_count","limitations":[],"operator_display":true,"result":13,"source_verification":"VERIFIED","unit":"count"},{"aggregation":{"method":"SUM","safe":true},"baseline":null,"class":"EXACT","delta":null,"derivation":null,"direction":"PROTECTION_SIGNAL","evidence_refs":["qualification"],"id":"subject_rendering_count","limitations":[],"operator_display":true,"result":0,"source_verification":"VERIFIED","unit":"count"},{"aggregation":{"method":"SUM","safe":true},"baseline":null,"class":"EXACT","delta":null,"derivation":null,"direction":"PROTECTION_SIGNAL","evidence_refs":["qualification"],"id":"subject_visible_canonical_arm_identifier_count","limitations":[],"operator_display":true,"result":0,"source_verification":"VERIFIED","unit":"count"},{"aggregation":{"method":"SUM","safe":true},"baseline":null,"class":"EXACT","delta":null,"derivation":null,"direction":"PROTECTION_SIGNAL","evidence_refs":["qualification"],"id":"provider_model_launch_count","limitations":[],"operator_display":true,"result":0,"source_verification":"VERIFIED","unit":"count"},{"aggregation":{"method":"SUM","safe":true},"baseline":null,"class":"EXACT","delta":null,"derivation":null,"direction":"PROTECTION_SIGNAL","evidence_refs":["authorization_set"],"id":"authorization_consumption_count","limitations":[],"operator_display":true,"result":0,"source_verification":"VERIFIED","unit":"count"},{"aggregation":{"method":"SUM","safe":true},"baseline":null,"class":"EXACT","delta":null,"derivation":null,"direction":"PROTECTION_SIGNAL","evidence_refs":["qualification"],"id":"experiment_run_count","limitations":[],"operator_display":true,"result":0,"source_verification":"VERIFIED","unit":"count"},{"aggregation":{"method":"SUM","safe":true},"baseline":null,"class":"EXACT","delta":null,"derivation":null,"direction":"PROTECTION_SIGNAL","evidence_refs":["qualification"],"id":"experiment_result_count","limitations":[],"operator_display":true,"result":0,"source_verification":"VERIFIED","unit":"count"}],"mechanism":{"id":"opsle.exp001-live-provider-preflight","name":"EXP-001 Live Provider Authorization Preflight","revision":"sha256:6351437acab90419fb3215e2d6cec63dabcc385f8f419f31a6600d0c5e35c5bc","version":"1.0.0"},"operation":{"configuration_id":"sha256:334a48850dacf3d9ad5d5c9088287fecc78925e64e10f974cb63575b74592b3c","id":"exp001-live-authset-03703f1af7db48c4eab9fe33b8f55073","name":"prepare-live-authorization-set","policy_id":"sha256:a19e793c2c2453d949687f614bee65be84b9abfa6218ea25c5302af7608a38db"},"run":{"id":"exp001-live-authset-03703f1af7db48c4eab9fe33b8f55073","repository":"opsle/research","task_classification":"EXP-001_PROVIDER_FREE_LIVE_AUTHORIZATION_PREFLIGHT","work_classification":"DETERMINISTIC_COORDINATION"},"schema":"opsle.value-receipt.v1"} diff --git a/program/experiments.json b/program/experiments.json index 14dd63e..a86ac55 100644 --- a/program/experiments.json +++ b/program/experiments.json @@ -1,6 +1,6 @@ { "schema_version": 1, - "last_verified_at": "2026-08-29T14:49:44Z", + "last_verified_at": "2026-08-30T18:49:10Z", "experiments": [ { "id": "EXP-001", @@ -136,6 +136,44 @@ "No token, cost, latency, correctness, or causal savings claim is made." ] }, + "live_authorization_preflight": { + "status": "PROVIDER_FREE_VALIDATED_UNRELEASED", + "implementation_base_sha": "9056b35703d94d85055868ed6778d7d79804485d", + "authorization_set_id": "exp001-live-authset-03703f1af7db48c4eab9fe33b8f55073", + "authorization_set_identity": "sha256:8259f5b664235e5914ad4a53966a1155f8f90ad2014dc704f50ea394af32d0ec", + "authorization_class": "LIVE_PROVIDER_RUN", + "live_authorization_label_count": 4, + "authorization_validation_count": 13, + "model_catalogue_identity": "sha256:334a48850dacf3d9ad5d5c9088287fecc78925e64e10f974cb63575b74592b3c", + "pricing_preflight_identity": "sha256:a19e793c2c2453d949687f614bee65be84b9abfa6218ea25c5302af7608a38db", + "model_catalogue_artifact": "program/evidence/exp-001-live-preflight/model-catalogue.json", + "model_catalogue_artifact_sha256": "sha256:5f7d2f26c70fedd67d71f9146da87d0181700ecb0d73a11f6db9c969644de05f", + "pricing_preflight_artifact": "program/evidence/exp-001-live-preflight/pricing-preflight.json", + "pricing_preflight_artifact_sha256": "sha256:070cc44476e26b23a7c8bfa1d94a5c64ed9e62b6dbe81c0d1ce52a136b89cc6f", + "qualification_artifact": "program/evidence/exp-001-live-preflight/qualification-report.json", + "qualification_artifact_sha256": "sha256:bb2c6df8e6dcd3293af9034be58e159503ba2e083d30ef98eeb45922e4f9588f", + "value_receipt_artifact": "program/evidence/exp-001-live-preflight/value-receipt.json", + "value_receipt_artifact_sha256": "sha256:0d5d00262a660fc09590dc9a427435aefe7bdf5eebe917abe91b2587a8e0167d", + "qualification": "PASS", + "deterministic_replay_count": 2, + "deterministic_payloads_byte_identical": true, + "fixture_rejected_from_live_gate": true, + "subject_rendering_count": 0, + "subject_visible_canonical_arm_identifier_count": 0, + "provider_model_runs_added": 0, + "authorization_consumptions_added": 0, + "experiment_runs_added": 0, + "experiment_results_added": 0, + "result_envelopes_added": 0, + "lifecycle_status": "PLANNED", + "lifecycle_impact": "NONE_NO_ADVANCEMENT: the external live set and current pricing preflight are provider-free validated but unconsumed and unreleased; no subject, run, or result exists.", + "limitations": [ + "The external authorization set remains outside Git and every subject context.", + "Account-specific model access is unverified because no provider call was made.", + "No distinct dated gpt-5.6-sol snapshot is documented, so immutable weights cannot be claimed.", + "No token, cost, latency, correctness, savings, or causal-benefit claim is made." + ] + }, "prerequisite_progress": [ { "repository": "context-firewall", @@ -195,20 +233,21 @@ "replication_status": "NOT_STARTED", "verdict": "PENDING", "blockers": [ - "No machine-readable four-label budgeted authorization set exists; zero subject/provider runs are authorized or recorded.", - "No distinct dated gpt-5.6-sol snapshot was publicly listed on 2026-08-29, so immutable model weights cannot be claimed and launch requires a catalogue-drift preflight." + "The validated live authorization set remains unconsumed; separate execution authority is required before any provider/model subject launch.", + "Account-specific API entitlement remains unverified under the zero-provider-call policy and requires a fresh fail-closed launch-time catalogue check.", + "No distinct dated gpt-5.6-sol snapshot was publicly listed on 2026-08-30, so immutable model weights cannot be claimed." ], "theory_reconciliation": { "verified_at": "2026-08-29T11:24:05Z", "context_firewall_definition": "The deterministic multi-adapter evidence boundary is the concept; the current TAP-subset reducer is one adapter prototype.", "experimental_hypothesis": "The experiment continues to test correctness under reduced model-visible context, not the definition or existence of Context Firewall.", "gearbox_dependency": "NONE", - "sequence": "The offline benchmark, launch controls, and one-block coordinator are frozen and provider-free qualified; zero model/provider subjects have run, and one exact live authorization set plus a catalogue/pricing preflight are next.", + "sequence": "The offline benchmark, launch controls, one-block coordinator, exact live authorization set, and current catalogue/pricing preflight are provider-free validated; zero authorizations are consumed and zero model/provider subjects or experiment runs/results exist.", "gearbox_publication_status": "COMPLETED_INDEPENDENT_PREREQUISITE", "gearbox_repository_head_sha": "f3fab9f292cf4eabd7200615d444f98881f57d55", "model_provider_runs_added": 0 }, - "next_task": "Create and provider-free validate one exact four-label LIVE_PROVIDER_RUN authorization set plus a model catalogue/pricing preflight artifact; do not launch a provider/model subject." + "next_task": "Independently review and release the provider-free live-authorization preflight branch; do not consume authorization or launch a provider/model subject." }, { "id": "LEGACY-001", diff --git a/program/registry.json b/program/registry.json index b458ca9..d8d6f42 100644 --- a/program/registry.json +++ b/program/registry.json @@ -36,7 +36,7 @@ "provider_model_runs": 0, "repository_consolidations": 0 }, - "current_highest_priority_workstream": "Prepare EXP-001's final non-launch admission boundary: one exact four-label LIVE_PROVIDER_RUN authorization set plus a model catalogue/pricing preflight artifact, validated without launching any provider/model subject.", + "current_highest_priority_workstream": "Independently review and release the provider-free EXP-001 live-authorization and current catalogue/pricing preflight; do not consume authorization or launch any provider/model subject.", "recommended_next_execution": "In opsle/research, create and provider-free validate one exact four-label LIVE_PROVIDER_RUN authorization set plus a model catalogue/pricing preflight artifact; do not consume authorization or launch a provider/model subject.", "last_verified_at": "2026-08-29T14:49:44Z", "repositories": [ @@ -558,27 +558,27 @@ "project_type": "program infrastructure", "purpose": "Own public research methodology, portfolio reconciliation, experiment records, and the authoritative program ledger.", "lifecycle_stage": "PROTOTYPED", - "implementation_status": "authoritative 20-repository ledger, machine-readable 17-concept theory registry, canonical theory map, normative Visible Value controls, and provider-free EXP-001 benchmark, launch, and one-block coordinator freezes with six content-addressed tasks, deterministic oracle, four arm contracts, sealed blinded allocation, exact subject configuration and adapter, exact four-authorization admission, private result-envelope templates, exact-revision interoperability, receipts, mutation tests, and integrity CI", + "implementation_status": "authoritative 20-repository ledger, machine-readable 17-concept theory registry, canonical theory map, normative Visible Value controls, and provider-free EXP-001 benchmark, launch, one-block coordinator, external four-label LIVE_PROVIDER_RUN authorization, and current catalogue/pricing preflight artifacts with six content-addressed tasks, deterministic oracle, four arm contracts, sealed blinded allocation, exact subject configuration and adapter, exact authorization admission, private boundaries, receipts, mutation tests, and integrity CI", "implementation_requirement": "Program infrastructure requires a validated registry, generated dashboard, experiment ledger, operating rules, and CI.", "specification_status": "canonical lifecycle, theory classifications and dispositions, Gearbox and Context Firewall definitions, Gearbox-versus-Durable boundary, Visible Value receipt and measurement classes, operator/model channels, observational corpus, shadow/replay attachment, operating, priority, registry, experiment, and generated-status controls are specified", - "test_status": "75 of 75 program-registry, theory-registry, theory-map-hash, generated-status, repository anti-forgetting, EXP-001 freeze/preregistration/coordinator, Gearbox evidence, and Visible Value tests passed locally; PR #13 and exact main CI validated the provider-free coordinator release, and the offline harness replayed all 48 arms", - "benchmark_status": "the EXP-001 provider-free components are frozen at 04234a65bf36192d63f1dd173c440d45a6604d2b, launch controls are preregistered at 31848c3f25ff9371055932657e8e2f8ad54cc8c7, and the one-block coordinator is qualified at 9ee43197880c18d4e185cf7e29e02a151d22a12e: six tasks, 252 oracle cases, four arms, 10 repetitions, 240 blinded labels in 60 balanced blocks, an encrypted allocation, exact subject configuration/adapter, 48 offline renderings, eight secret-backed coordinator renderings, and zero provider/model runs; no causal experiment", + "test_status": "88 of 88 repository tests pass on the unreleased provider-free live-preflight branch, including 13 authorization validations and two byte-identical replays; PR #13 and exact main CI remain the released coordinator baseline", + "benchmark_status": "the EXP-001 provider-free components are frozen at 04234a65bf36192d63f1dd173c440d45a6604d2b, launch controls are preregistered at 31848c3f25ff9371055932657e8e2f8ad54cc8c7, and the one-block coordinator is qualified at 9ee43197880c18d4e185cf7e29e02a151d22a12e; the unreleased branch adds one external exact four-label LIVE_PROVIDER_RUN set and current gpt-5.6-sol catalogue/pricing evidence with zero consumptions, provider/model launches, experiment runs, or results; no causal experiment", "measured_experiment_status": "one legacy integration observation; no canonical measured concept experiment", - "reproducibility_status": "program and receipt validation, status generation, public dogfood artifacts, exact-revision EXP-001 offline qualification, public allocation checks, adapter self-test, structural preregistration verification, and coordinator unit checks are reproducible locally and in CI; full secret-backed coordinator replay requires the external seed, and no subject result exists", + "reproducibility_status": "program and receipt validation, status generation, public dogfood artifacts, exact-revision EXP-001 offline qualification, public allocation checks, adapter self-test, structural preregistration verification, coordinator unit checks, and live-authorization reconstruction from frozen private inputs are reproducible; the live set replay is byte-identical twice, full secret-backed coordinator replay still requires the external seed, and no subject result exists", "documentation_status": "public canonical theory map, project reconciliation, registered Gearbox boundary and prototype scope, Context Firewall adapter scope, consolidation provenance policy, program documentation, Visible Value semantics, machine controls, evidence, and generated portfolio status are present", "site_publication_status": "GitHub research hub only", - "known_limitations": ["The registry cannot self-reference the commit that contains its own SHA, default-branch HEAD verification remains operator-driven, the frozen EXP-001 corpus is Python-only, no exact live budgeted authorization or catalogue/pricing preflight exists, the public model ID is not an immutable dated snapshot, site content is not registry-derived, Gearbox comparative evidence is absent, and no canonical concept experiment has run."], + "known_limitations": ["The registry cannot self-reference the commit that contains its own SHA, default-branch HEAD verification remains operator-driven, the frozen EXP-001 corpus is Python-only, the exact live authorization set is external and unconsumed, account-specific API entitlement is unverified, the public model ID is not an immutable dated snapshot, site content is not registry-derived, Gearbox comparative evidence is absent, and no canonical concept experiment has run."], "dependencies": [], "dependents": [".github", "site"], "active_experiment_ids": ["EXP-001", "LEGACY-001"], - "blockers": ["EXP-001 lacks an exact live budgeted authorization set and catalogue/pricing preflight; its one-block coordinator is provider-free qualified, zero model/provider subjects have run, and the program has no canonical measured concept experiment."], - "next_task": "Create and provider-free validate one exact four-label LIVE_PROVIDER_RUN authorization set plus a model catalogue/pricing preflight artifact; do not launch a provider/model subject.", - "evidence": ["program/THEORY_MAP.md", "program/theory-registry.json", "program/evidence/gearbox-publication/README.md", "program/evidence/exp-001-offline-freeze/README.md", "program/evidence/exp-001-preregistration/README.md", "program/evidence/exp-001-block-coordinator/README.md", "https://github.com/opsle/research/blob/04234a65bf36192d63f1dd173c440d45a6604d2b/experiments/exp-001/benchmark.json", "https://github.com/opsle/research/blob/04234a65bf36192d63f1dd173c440d45a6604d2b/program/evidence/exp-001-offline-freeze/qualification-report.json", "https://github.com/opsle/research/blob/31848c3f25ff9371055932657e8e2f8ad54cc8c7/experiments/exp-001/preregistration-v1/preregistration.json", "https://github.com/opsle/research/blob/31848c3f25ff9371055932657e8e2f8ad54cc8c7/program/evidence/exp-001-preregistration/verification-report.json", "https://github.com/opsle/research/blob/9ee43197880c18d4e185cf7e29e02a151d22a12e/experiments/exp-001/coordinator-v1/coordinator.py", "https://github.com/opsle/research/blob/9ee43197880c18d4e185cf7e29e02a151d22a12e/program/evidence/exp-001-block-coordinator/qualification-report.json", "https://github.com/opsle/research/blob/72f9e4a0326d68d3870e2e79ce4e351acb1d8ffa/program/VISIBLE_VALUE_CONTRACT.md", "https://github.com/opsle/research/pull/9", "https://github.com/opsle/research/pull/11", "https://github.com/opsle/research/pull/13"], + "blockers": ["The exact live authorization set remains unconsumed and unreleased, account-specific API entitlement is unverified under the zero-provider-call policy, no immutable dated model snapshot is documented, and the program has no canonical measured concept experiment."], + "next_task": "Independently review and release the provider-free live-authorization and catalogue/pricing preflight; do not consume authorization or launch a provider/model subject.", + "evidence": ["program/THEORY_MAP.md", "program/theory-registry.json", "program/evidence/gearbox-publication/README.md", "program/evidence/exp-001-offline-freeze/README.md", "program/evidence/exp-001-preregistration/README.md", "program/evidence/exp-001-block-coordinator/README.md", "program/evidence/exp-001-live-preflight/README.md", "program/evidence/exp-001-live-preflight/model-catalogue.json", "program/evidence/exp-001-live-preflight/pricing-preflight.json", "program/evidence/exp-001-live-preflight/qualification-report.json", "program/evidence/exp-001-live-preflight/value-receipt.json", "https://github.com/opsle/research/blob/04234a65bf36192d63f1dd173c440d45a6604d2b/experiments/exp-001/benchmark.json", "https://github.com/opsle/research/blob/04234a65bf36192d63f1dd173c440d45a6604d2b/program/evidence/exp-001-offline-freeze/qualification-report.json", "https://github.com/opsle/research/blob/31848c3f25ff9371055932657e8e2f8ad54cc8c7/experiments/exp-001/preregistration-v1/preregistration.json", "https://github.com/opsle/research/blob/31848c3f25ff9371055932657e8e2f8ad54cc8c7/program/evidence/exp-001-preregistration/verification-report.json", "https://github.com/opsle/research/blob/9ee43197880c18d4e185cf7e29e02a151d22a12e/experiments/exp-001/coordinator-v1/coordinator.py", "https://github.com/opsle/research/blob/9ee43197880c18d4e185cf7e29e02a151d22a12e/program/evidence/exp-001-block-coordinator/qualification-report.json", "https://github.com/opsle/research/blob/72f9e4a0326d68d3870e2e79ce4e351acb1d8ffa/program/VISIBLE_VALUE_CONTRACT.md", "https://github.com/opsle/research/pull/9", "https://github.com/opsle/research/pull/11", "https://github.com/opsle/research/pull/13"], "completion_criteria": ["Keep the 20-repository ledger and dashboard mechanically consistent.", "Retain immutable experiment evidence and lifecycle promotion proof.", "Publish program documentation without stale or unsupported claims."], "completion_evidence": [], "completion_status": "INCOMPLETE", "program_state": "active", - "last_verified_at": "2026-08-29T15:25:30Z" + "last_verified_at": "2026-08-30T18:49:10Z" }, { "name": "site", diff --git a/tests/test_exp001_live_preflight.py b/tests/test_exp001_live_preflight.py new file mode 100644 index 0000000..567bc1c --- /dev/null +++ b/tests/test_exp001_live_preflight.py @@ -0,0 +1,205 @@ +from __future__ import annotations + +import importlib.util +import json +import sys +import tempfile +import unittest +from pathlib import Path +from typing import ClassVar + +ROOT = Path(__file__).resolve().parents[1] +PREFLIGHT_ROOT = ROOT / "experiments/exp-001/live-preflight-v1" + + +def load_module(name: str, path: Path): + spec = importlib.util.spec_from_file_location(name, path) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + sys.modules[name] = module + spec.loader.exec_module(module) + return module + + +PREFLIGHT = load_module( + "exp001_live_preflight", + PREFLIGHT_ROOT / "preflight.py", +) +VALUE_VALIDATOR = load_module( + "exp001_live_preflight_value_validator", + ROOT / "tools/validate_value_receipt.py", +) + + +class Exp001LivePreflightTests(unittest.TestCase): + issued_at = "2026-08-30T18:49:10Z" + valid_from = "2026-08-30T18:49:10Z" + expires_at = "2026-09-13T18:49:10Z" + validated_at = "2026-08-30T18:49:10Z" + set_id = "exp001-live-authset-11111111111111111111111111111111" + authorization_ids: ClassVar[list[str]] = [ + f"exp001-live-authz-{index:032x}" for index in range(1, 5) + ] + + def materialize(self, root: Path) -> Path: + destination = root / "authorization-set" + PREFLIGHT.materialize_authorization_set( + destination=destination, + block_id=PREFLIGHT.first_public_block_id(), + set_id=self.set_id, + authorization_ids=self.authorization_ids, + issued_at=self.issued_at, + valid_from=self.valid_from, + expires_at=self.expires_at, + ) + return destination + + def test_catalogue_matches_exact_preregistered_candidate(self): + catalogue = PREFLIGHT.load_json(PREFLIGHT.CATALOGUE_PATH) + candidate = PREFLIGHT.validate_catalogue(catalogue) + self.assertEqual(candidate["provider_id"], "openai") + self.assertEqual(candidate["model_id"], "gpt-5.6-sol") + self.assertEqual(candidate["model_family"], "GPT-5.6") + self.assertEqual(candidate["input_price_usd"], 4.0) + self.assertEqual(candidate["cached_input_price_usd"], 0.4) + self.assertEqual(candidate["output_price_usd"], 20.0) + self.assertEqual(candidate["context_window_tokens"], 1_050_000) + self.assertEqual(candidate["max_output_tokens"], 128_000) + self.assertEqual(len(candidate["uncertainty_or_unavailable_fields"]), 2) + + def test_pricing_preflight_preserves_registered_ceiling(self): + catalogue = PREFLIGHT.load_json(PREFLIGHT.CATALOGUE_PATH) + pricing = PREFLIGHT.pricing_preflight(catalogue) + self.assertEqual(pricing["derived_spend_ceiling_usd"], 6.25152) + self.assertEqual(pricing["registered_spend_ceiling_usd"], 6.3) + self.assertFalse(pricing["long_context_multiplier_applies"]) + self.assertFalse(pricing["price_drift_from_frozen_configuration"]) + self.assertFalse(pricing["account_access_verified"]) + self.assertEqual(pricing["provider_call_count"], 0) + + def test_exact_four_live_labels_validate_unconsumed(self): + with tempfile.TemporaryDirectory() as temporary: + directory = self.materialize(Path(temporary)) + result = PREFLIGHT.validate_authorization_set( + directory, + validated_at=self.validated_at, + ) + self.assertTrue(result["valid"]) + self.assertEqual(result["authorization_class"], "LIVE_PROVIDER_RUN") + self.assertEqual(result["label_count"], 4) + self.assertEqual(result["authorization_count"], 4) + self.assertEqual(result["authorization_consumption_count"], 0) + self.assertEqual(result["provider_model_launch_count"], 0) + self.assertEqual(result["experiment_run_count"], 0) + self.assertEqual(result["experiment_result_count"], 0) + + def test_authorizations_have_unique_auditable_identities_and_no_results(self): + with tempfile.TemporaryDirectory() as temporary: + directory = self.materialize(Path(temporary)) + records = [ + json.loads(path.read_text()) + for path in sorted((directory / "authorizations").glob("*.json")) + ] + self.assertEqual(len({record["authorization_id"] for record in records}), 4) + self.assertEqual(len({record["subject_label"] for record in records}), 4) + for record in records: + self.assertEqual(record["authorization_set_id"], self.set_id) + self.assertEqual(record["authorization_class"], "LIVE_PROVIDER_RUN") + self.assertEqual(record["authorization_state"], "UNCONSUMED") + self.assertIsNone(record["consumed_at"]) + self.assertIsNone(record["result_envelope"]) + + def test_fixture_duplicate_missing_extra_binding_malformed_and_time_reject(self): + with tempfile.TemporaryDirectory() as temporary: + directory = self.materialize(Path(temporary)) + evidence = PREFLIGHT.qualify( + set_directory=directory, + catalogue_path=PREFLIGHT.CATALOGUE_PATH, + validated_at=self.validated_at, + ) + report = json.loads(evidence["qualification-report.json"]) + self.assertEqual(report["authorization_validation_count"], 13) + self.assertTrue(report["fixture_rejected_from_live_gate"]) + self.assertTrue(report["duplicate_label_rejected"]) + self.assertTrue(report["duplicate_authorization_identity_rejected"]) + self.assertTrue(report["missing_label_rejected"]) + self.assertTrue(report["extra_label_rejected"]) + self.assertTrue(report["wrong_experiment_binding_rejected"]) + self.assertTrue(report["wrong_block_binding_rejected"]) + self.assertTrue(report["malformed_authorization_rejected"]) + self.assertTrue(report["expired_authorization_rejected"]) + self.assertTrue(report["not_yet_valid_authorization_rejected"]) + + def test_replay_is_byte_identical_and_creation_is_exclusive(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + directory = self.materialize(root) + manifest = PREFLIGHT.load_json(directory / "set-manifest.json") + replay = root / "replay" + PREFLIGHT.materialize_authorization_set( + destination=replay, + **PREFLIGHT.replay_inputs(manifest), + ) + self.assertEqual(PREFLIGHT.snapshot(directory), PREFLIGHT.snapshot(replay)) + with self.assertRaisesRegex( + PREFLIGHT.PreflightError, + "destination must be absent", + ): + PREFLIGHT.materialize_authorization_set( + destination=replay, + **PREFLIGHT.replay_inputs(manifest), + ) + + def test_public_evidence_discloses_no_labels_mapping_or_arm_ids(self): + arms = json.loads((ROOT / "experiments/exp-001/arms.json").read_text()) + arm_ids = [arm["id"].encode() for arm in arms["arms"]] + with tempfile.TemporaryDirectory() as temporary: + directory = self.materialize(Path(temporary)) + manifest = PREFLIGHT.load_json(directory / "set-manifest.json") + labels = [ + record["subject_label"].encode() + for record in manifest["authorizations"] + ] + evidence = PREFLIGHT.qualify( + set_directory=directory, + catalogue_path=PREFLIGHT.CATALOGUE_PATH, + validated_at=self.validated_at, + ) + public = b"".join(evidence.values()) + self.assertFalse(any(label in public for label in labels)) + self.assertFalse(any(arm_id in public for arm_id in arm_ids)) + report = json.loads(evidence["qualification-report.json"]) + self.assertEqual(report["subject_rendering_count"], 0) + self.assertEqual( + report["subject_visible_canonical_arm_identifier_count"], 0 + ) + self.assertEqual(report["private_mapping_access_count"], 0) + self.assertEqual(report["raw_evidence_access_count"], 0) + + def test_receipt_is_valid_and_claims_only_provider_free_counts(self): + with tempfile.TemporaryDirectory() as temporary: + directory = self.materialize(Path(temporary)) + evidence = PREFLIGHT.qualify( + set_directory=directory, + catalogue_path=PREFLIGHT.CATALOGUE_PATH, + validated_at=self.validated_at, + ) + receipt = json.loads(evidence["value-receipt.json"]) + self.assertEqual(VALUE_VALIDATOR.validate(receipt), []) + measurements = { + item["id"]: item["result"] for item in receipt["measurements"] + } + self.assertEqual(measurements["live_authorization_label_count"], 4) + self.assertEqual(measurements["authorization_validation_count"], 13) + self.assertEqual(measurements["subject_rendering_count"], 0) + self.assertEqual(measurements["provider_model_launch_count"], 0) + self.assertEqual(measurements["authorization_consumption_count"], 0) + self.assertEqual(measurements["experiment_run_count"], 0) + self.assertEqual(measurements["experiment_result_count"], 0) + + def test_validation_path_has_no_provider_or_process_launch_capability(self): + PREFLIGHT.provider_free_source_audit() + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_validate_program.py b/tests/test_validate_program.py index b4a97ab..71f692f 100644 --- a/tests/test_validate_program.py +++ b/tests/test_validate_program.py @@ -247,6 +247,67 @@ def test_exp001_block_coordinator_evidence_hash_cannot_drift(self): any("qualification_artifact_sha256 drifted" in error for error in errors) ) + def test_exp001_live_preflight_is_required(self): + experiments = copy.deepcopy(self.experiments) + del experiments["experiments"][0]["live_authorization_preflight"] + errors = self.errors_for(experiments=experiments) + self.assertTrue( + any( + "live_authorization_preflight must be an object" in error + for error in errors + ) + ) + + def test_exp001_live_preflight_cannot_claim_consumption_or_runs(self): + fields = ( + "provider_model_runs_added", + "authorization_consumptions_added", + "experiment_runs_added", + "experiment_results_added", + "result_envelopes_added", + ) + for field in fields: + with self.subTest(field=field): + experiments = copy.deepcopy(self.experiments) + experiments["experiments"][0]["live_authorization_preflight"][ + field + ] = 1 + errors = self.errors_for(experiments=experiments) + self.assertTrue( + any(f"{field} must be 0" in error for error in errors) + ) + + def test_exp001_live_preflight_evidence_hash_cannot_drift(self): + fields = ( + "model_catalogue_artifact_sha256", + "pricing_preflight_artifact_sha256", + "qualification_artifact_sha256", + "value_receipt_artifact_sha256", + ) + for field in fields: + with self.subTest(field=field): + experiments = copy.deepcopy(self.experiments) + experiments["experiments"][0]["live_authorization_preflight"][ + field + ] = "sha256:" + "0" * 64 + errors = self.errors_for(experiments=experiments) + self.assertTrue(any(f"{field} drifted" in error for error in errors)) + + def test_exp001_live_preflight_must_remain_planned_and_unrendered(self): + fields = ( + ("lifecycle_status", "EXPERIMENTED"), + ("subject_rendering_count", 1), + ("subject_visible_canonical_arm_identifier_count", 1), + ) + for field, value in fields: + with self.subTest(field=field): + experiments = copy.deepcopy(self.experiments) + experiments["experiments"][0]["live_authorization_preflight"][ + field + ] = value + errors = self.errors_for(experiments=experiments) + self.assertTrue(any(f"live preflight {field}" in error for error in errors)) + def test_dashboard_is_current(self): expected = (ROOT / "PROGRAM_STATUS.md").read_text(encoding="utf-8") self.assertEqual(render(self.registry, self.experiments), expected) diff --git a/tools/validate_program.py b/tools/validate_program.py index faa9e84..db2b521 100644 --- a/tools/validate_program.py +++ b/tools/validate_program.py @@ -134,9 +134,10 @@ ) EXP001_RECONCILED_SEQUENCE = ( - "The offline benchmark, launch controls, and one-block coordinator are frozen " - "and provider-free qualified; zero model/provider subjects have run, and one " - "exact live authorization set plus a catalogue/pricing preflight are next." + "The offline benchmark, launch controls, one-block coordinator, exact live " + "authorization set, and current catalogue/pricing preflight are provider-free " + "validated; zero authorizations are consumed and zero model/provider subjects " + "or experiment runs/results exist." ) EXP001_OFFLINE_RELEASE_SHA = "04234a65bf36192d63f1dd173c440d45a6604d2b" @@ -172,6 +173,19 @@ EXP001_COORDINATOR_REVISION = ( "sha256:d7a2e5709fdc3b36b0fedd464bd7ccb791b067da5f327d17f7a76314f3d0af90" ) +EXP001_LIVE_PREFLIGHT_BASE_SHA = "9056b35703d94d85055868ed6778d7d79804485d" +EXP001_LIVE_AUTHORIZATION_SET_ID = ( + "exp001-live-authset-03703f1af7db48c4eab9fe33b8f55073" +) +EXP001_LIVE_AUTHORIZATION_SET_IDENTITY = ( + "sha256:8259f5b664235e5914ad4a53966a1155f8f90ad2014dc704f50ea394af32d0ec" +) +EXP001_MODEL_CATALOGUE_IDENTITY = ( + "sha256:334a48850dacf3d9ad5d5c9088287fecc78925e64e10f974cb63575b74592b3c" +) +EXP001_PRICING_PREFLIGHT_IDENTITY = ( + "sha256:a19e793c2c2453d949687f614bee65be84b9abfa6218ea25c5302af7608a38db" +) REQUIRED_REPOSITORY_FIELDS = ( "name", @@ -757,6 +771,83 @@ def validate( errors.append( "EXP-001 block coordinator limitations must be a nonempty string array" ) + live_preflight = exp001.get("live_authorization_preflight") + if not isinstance(live_preflight, dict): + errors.append("EXP-001 live_authorization_preflight must be an object") + else: + expected_live_preflight_values = { + "status": "PROVIDER_FREE_VALIDATED_UNRELEASED", + "implementation_base_sha": EXP001_LIVE_PREFLIGHT_BASE_SHA, + "authorization_set_id": EXP001_LIVE_AUTHORIZATION_SET_ID, + "authorization_set_identity": ( + EXP001_LIVE_AUTHORIZATION_SET_IDENTITY + ), + "authorization_class": "LIVE_PROVIDER_RUN", + "live_authorization_label_count": 4, + "authorization_validation_count": 13, + "model_catalogue_identity": EXP001_MODEL_CATALOGUE_IDENTITY, + "pricing_preflight_identity": EXP001_PRICING_PREFLIGHT_IDENTITY, + "qualification": "PASS", + "deterministic_replay_count": 2, + "deterministic_payloads_byte_identical": True, + "fixture_rejected_from_live_gate": True, + "subject_rendering_count": 0, + "subject_visible_canonical_arm_identifier_count": 0, + "provider_model_runs_added": 0, + "authorization_consumptions_added": 0, + "experiment_runs_added": 0, + "experiment_results_added": 0, + "result_envelopes_added": 0, + "lifecycle_status": "PLANNED", + } + for field, expected_value in expected_live_preflight_values.items(): + if live_preflight.get(field) != expected_value: + errors.append( + f"EXP-001 live preflight {field} must be " + f"{expected_value!r}" + ) + artifacts = ( + ( + "model_catalogue_artifact", + "model_catalogue_artifact_sha256", + "program/evidence/exp-001-live-preflight/model-catalogue.json", + ), + ( + "pricing_preflight_artifact", + "pricing_preflight_artifact_sha256", + "program/evidence/exp-001-live-preflight/pricing-preflight.json", + ), + ( + "qualification_artifact", + "qualification_artifact_sha256", + "program/evidence/exp-001-live-preflight/qualification-report.json", + ), + ( + "value_receipt_artifact", + "value_receipt_artifact_sha256", + "program/evidence/exp-001-live-preflight/value-receipt.json", + ), + ) + for path_field, hash_field, expected_path in artifacts: + artifact = live_preflight.get(path_field) + if artifact != expected_path: + errors.append( + f"EXP-001 live preflight {path_field} path drifted" + ) + continue + artifact_path = ROOT / artifact + if not artifact_path.is_file(): + errors.append( + f"EXP-001 live preflight {path_field} is missing" + ) + continue + artifact_digest = hashlib.sha256(artifact_path.read_bytes()).hexdigest() + if live_preflight.get(hash_field) != f"sha256:{artifact_digest}": + errors.append(f"EXP-001 live preflight {hash_field} drifted") + if not _string_list(live_preflight.get("limitations"), allow_empty=False): + errors.append( + "EXP-001 live preflight limitations must be a nonempty string array" + ) return errors