diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e2ac92b..ce69719 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -66,6 +66,9 @@ jobs: - name: Verify embedded signature-verifier pins run: python scripts/sync-verifier-lock.py --check + - name: Test release qualification tooling + run: python -m pytest scripts/tests/ -q + - name: Run pytest run: python -m pytest tests/ -q diff --git a/CLAUDE.md b/CLAUDE.md index 0b5edf7..9108b2d 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -56,6 +56,11 @@ The source branch can contain fixes newer than the latest signed release. Check `CHANGELOG.md`, the release tag and `site/status.json` before claiming a fix is shipped; a source commit does not change an existing signed archive. +For the transition out of beta, follow [stable readiness](docs/stable-readiness.md). +The historical alpha RC ledger qualifies entry into beta, not a stable release. +Full `scripts/beta-gate.py` runs require an explicit, different `--previous` version +and scan all four locks. RC/stable labels require their own artifact evidence. + ## The rule everything rests on **The model proposes, it does not decide.** The reasoner emits `TOOL_CALL` text, the kernel diff --git a/README.md b/README.md index f456c6d..f3e1516 100644 --- a/README.md +++ b/README.md @@ -935,7 +935,10 @@ sitting — a gate you cannot read is not a gate. The [beta release evidence](docs/beta-readiness.md) separates source tests, signed artifact checks and the seven-day operating observation. The frozen [beta contract](docs/beta-contract.md), [recovery runbook](docs/recovery.md) and -local `python scripts/beta-gate.py` make these checks repeatable. The path to 1.0 +local `python scripts/beta-gate.py --previous ` make these checks repeatable. +The [RC and stable qualification plan](docs/stable-readiness.md) tracks the next +transition from beta through `0.20.0-rc.1` to `0.20.0`; these versions are not yet +published. The path to 1.0 requires evidence from the beta line and a verified upgrade from a previous beta; it is not a calendar promise. diff --git a/docs/beta-readiness.md b/docs/beta-readiness.md index 133242a..2dd7199 100644 --- a/docs/beta-readiness.md +++ b/docs/beta-readiness.md @@ -1,5 +1,8 @@ # Beta readiness +This page records the historical transition into beta. The next transition is +tracked separately in [RC and stable readiness](stable-readiness.md). + Talos' beta line freezes the operator-facing contract after separate source, signed-artifact and operating-observation gates. Beta is not a claim that the model, network or service never fails. This document records the criteria instead of @@ -88,9 +91,9 @@ Executed results for this baseline are in the [2026-09-25 evidence record](beta- Release-candidate accounting is in the [RC ledger](beta-rc-ledger.md). A source preflight is not a counted RC or a published beta. -Run `python scripts/beta-gate.py` from a prepared development environment for source +Run `python scripts/beta-gate.py --previous ` from a prepared development environment for source tests, red-team cases, public hygiene, locked-dependency OSV, a disposable clean -install and an isolated signed upgrade from the previous published alpha, including +install and an isolated signed upgrade from the explicitly selected published version, including schedule migration and rollback. The upgrade rehearsal creates and removes only a temporary installation; it does not restart or change the operating Pi. Use `--without-upgrade` only for a partial source diff --git a/docs/stable-readiness.md b/docs/stable-readiness.md new file mode 100644 index 0000000..e4c494b --- /dev/null +++ b/docs/stable-readiness.md @@ -0,0 +1,94 @@ +# From beta to a stable 0.20.0 + +The next proposed release is `0.20.0-rc.1`, followed by `0.20.0` only after the +evidence below passes. This is a qualification plan, not an announcement that +either version has shipped. The published baseline is `0.20.0-beta.9`. + +The historical `0.20.0-alpha.rc.*` artifacts qualified the earlier transition +**into beta**. They are not candidates for this transition out of beta. `1.0.0` +remains a separate compatibility commitment, not an automatic rename of 0.20.0. + +## Scope freeze + +Keep the [operator contract](beta-contract.md), command names, approval meanings, +configuration and durable-state boundaries. During candidate qualification, accept +release-blocking fixes and their regression tests; defer unrelated features. Any +compatibility change needs a before/after test and migration instructions. + +Terminal and Telegram on the documented Linux/macOS and Python 3.11–3.13 targets +remain the scope. A CLI release does not certify Windows, every optional computer +integration or a notarized desktop app. Unsupported configurations must be named, +not silently included in a broad claim that everything passed. + +## Candidate and stable gates + +| Gate | Required evidence | +|---|---| +| Source | Exact commit, clean checkout, full regression and adversarial suites; release-tooling and embedded verifier-pin checks; public hygiene and a reviewed secret scan. | +| Dependencies | Fresh OSV scans of all four locks: runtime, development, signature verifier and optional computer environment. Check installed distributions too, including pip. No unresolved critical/high finding. | +| Supported platforms | Native macOS and Linux hosted jobs on the exact commit; local Python 3.12/3.13 and ARM64 Linux checks with explicit skips and reasons. | +| Real model flow | Isolated real-model E2E: read, approval/refusal, exactly-once effect, restart/state restoration and the event chain. Record provider and results; do not send external test messages. | +| User-visible flow | Telegram reply/approval/background completion behavior, including conversation isolation and no false completion while waiting. Injected transport tests and an actual service probe are distinct evidence. | +| Exact artifact | Build from the qualified public commit; SHA-256, pinned Ed25519 verification, tamper rejection and archive hygiene. Source tests alone do not qualify an archive. | +| Installation/recovery | Clean install and beta.9→RC signed upgrade on macOS and ARM64 Linux; check operator identity/configuration, selected model, history, schedules, file modes and rollback. Rehearse isolated service restart and retain the prior runtime. | +| Delivery | Native Homebrew lifecycle, Pinokio clean/repeated install, and byte-for-byte HTTPS read-back of the release, website and installer definitions. Check main CI after each merge. | +| Observation before stable | Seven consecutive calendar days on a deployed RC, with daily health/event-chain records, investigated errors, no unexplained restart and no unresolved security or data-loss regression. Preserve failed samples. A release-blocking runtime fix starts a new candidate/window. | +| Stable promotion | Rebuild/sign the final version and requalify its exact bytes, including RC→stable upgrade and recovery. A version-only change is still a new artifact. | + +The [observation collector](beta-observation.md) can keep its existing name and +timer. A record must identify the running version. Beta observations establish +history; they must not be relabelled as days on a newer RC. Model/network errors +need diagnosis; neither an arbitrary error-count threshold nor `exit 0` alone +certifies stability. Human review is still required for the release conclusion. + +## Reproducible local preflight + +Start with the current published baseline in a prepared, hash-locked development +environment: + +```sh +python scripts/beta-gate.py --previous 0.20.0-beta.8 --current 0.20.0-beta.9 +python -m pytest macos/Tests -q +python e2e.py +``` + +The gate requires an explicit, different `--previous` version. It checks release +tooling, source tests, adversarial cases, hygiene, embedded verifier pins, all four +dependency locks, a disposable clean install, and signed upgrade/migration/rollback. +`--without-upgrade` is a partial source check and says so; it does not qualify a +candidate. Tests and artifact downloads require their normal dependencies/network. + +For an unpublished candidate, use `--current 0.20.0-rc.1`, `--previous +0.20.0-beta.9`, an explicit `--base` pointing to the candidate's `dist/` directory +parent, and `--assets-dir` containing both versions' archive/checksum/signature. +The checkout's installer must name the candidate. The existing scripts support +`file://` for an isolated pre-publication install. Run against the actual signed +bytes; do not change signature pins or skip verification to make a rehearsal pass. + +Retain full command logs and first failures privately. Public evidence contains +only redacted results, exact commit/artifact identities and links. Retrying an +unchanged failure is not a fix; disclose retries and the actual cause. + +## Publication order + +1. Qualify the exact source and candidate archive before changing public pointers. +2. Publish the immutable signed GitHub assets; download and verify all three bytes. +3. Publish website assets and pages with backups/read-back, switching `latest.txt` + last. Check its actual prior value rather than copying an old release constant. +4. Only then trigger Pinokio's live installer checks: its pinned installer downloads + the archive from the website. Verify the installer digest against the exact + published script before pushing a pin update. Qualify Homebrew's native lifecycle. +5. Merge package updates after their checks pass, verify public bytes and all main + CI jobs, then perform the separately backed-up operator rollout and read-back. + +Never rewrite an existing signed archive/tag or delete failed CI history to make +the release look clean. Publishing a release does not itself update a running +installation or restart its service. + +## Qualification ledger + +| Artifact | State | Evidence boundary | +|---|---|---| +| `0.20.0-beta.9` | Published baseline | [Core main CI](https://github.com/talos-kernel/Talos/actions/runs/40661310306), [Homebrew](https://github.com/talos-kernel/homebrew-tap/actions/runs/40661911664), [Pinokio](https://github.com/talos-kernel/talos-pinokio/actions/runs/40663657826). These qualify beta.9 only. | +| `0.20.0-rc.1` | Preparation started; not published | Needs its own commit, signed artifact, complete platform/install/recovery evidence and operator rollout. | +| `0.20.0` | Not qualified | Needs the RC observation window and final-artifact gates above. | diff --git a/scripts/beta-gate.py b/scripts/beta-gate.py index f2536e4..9d375c0 100644 --- a/scripts/beta-gate.py +++ b/scripts/beta-gate.py @@ -12,6 +12,13 @@ sys.path.insert(0, str(ROOT)) from talos import __version__ +LOCKFILES = ( + "requirements.lock", + "requirements-dev.lock", + "requirements-verifier.lock", + "deploy/computer-requirements.lock", +) + def run(label: str, argv: list[str]) -> bool: print(f"\n=== {label} ===", flush=True) @@ -28,23 +35,36 @@ def run(label: str, argv: list[str]) -> bool: def main() -> int: parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--previous", default="0.19.23-alpha") + parser.add_argument("--previous", help="Explicit published starting version for the upgrade rehearsal") parser.add_argument("--current", default=__version__) parser.add_argument("--base", default="https://talos-agent.ch") parser.add_argument("--assets-dir", type=Path) parser.add_argument("--without-upgrade", action="store_true", help="Run source gates only; does not qualify as a complete beta gate") args = parser.parse_args() + if not args.without_upgrade: + if not args.previous: + parser.error("--previous is required for a full gate; name the release being upgraded") + if args.previous == args.current: + parser.error("--previous and --current must differ; a no-op is not an upgrade rehearsal") + # The verifier and optional computer environment are independently installed. + # Scanning only the core runtime leaves both outside the release evidence. + missing = [name for name in LOCKFILES if not (ROOT / name).is_file()] + if missing: + print("FAILED: required dependency locks missing: " + ", ".join(missing), file=sys.stderr) + return 1 if shutil.which("osv-scanner") is None: print("FAILED: osv-scanner is required", file=sys.stderr) return 1 checks = ( + ("Release tooling", [sys.executable, "-m", "pytest", "scripts/tests/", "-q"]), ("Regression suite", [sys.executable, "-m", "pytest", "tests/", "-q"]), ("Adversarial suite", [sys.executable, "redteam.py"]), ("Public hygiene", [sys.executable, "scripts/check-public-hygiene.py"]), + ("Embedded verifier pins", [sys.executable, "scripts/sync-verifier-lock.py", "--check"]), ("Locked dependency scan", ["osv-scanner", "scan", "source", "--no-resolve", - "--lockfile", "requirements.txt:requirements.lock", - "--lockfile", "requirements.txt:requirements-dev.lock"]), + *[arg for name in LOCKFILES + for arg in ("--lockfile", "requirements.txt:" + name)]]), ) for label, argv in checks: if not run(label, argv): diff --git a/scripts/tests/test_beta_gate.py b/scripts/tests/test_beta_gate.py new file mode 100644 index 0000000..e692ee6 --- /dev/null +++ b/scripts/tests/test_beta_gate.py @@ -0,0 +1,84 @@ +"""Release tooling cannot count an incomplete or no-op rehearsal as qualified.""" +import importlib.util +from pathlib import Path +import sys + +import pytest + + +@pytest.fixture +def gate(tmp_path, monkeypatch): + script = Path(__file__).resolve().parents[1] / "beta-gate.py" + spec = importlib.util.spec_from_file_location("release_gate_under_test", script) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + monkeypatch.setattr(module, "ROOT", tmp_path) + for relative in module.LOCKFILES: + path = tmp_path / relative + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text("fixture==1.0\n") + monkeypatch.setattr(module.shutil, "which", lambda name: "/tools/" + name) + return module + + +@pytest.mark.parametrize("arguments", [[], ["--previous", "0.20.0-rc.1", "--current", "0.20.0-rc.1"]]) +def test_full_gate_refuses_missing_baseline_or_no_op(gate, monkeypatch, arguments): + monkeypatch.setattr(sys, "argv", ["beta-gate.py", *arguments]) + monkeypatch.setattr(gate, "run", lambda *args: pytest.fail("preflight must refuse before running checks")) + with pytest.raises(SystemExit) as error: + gate.main() + assert error.value.code == 2 + + +@pytest.mark.parametrize("missing", ["requirements.lock", "requirements-dev.lock", + "requirements-verifier.lock", "deploy/computer-requirements.lock"]) +def test_missing_environment_lock_refuses_the_gate(gate, monkeypatch, missing): + (gate.ROOT / missing).unlink() + monkeypatch.setattr(sys, "argv", ["beta-gate.py", "--without-upgrade"]) + monkeypatch.setattr(gate, "run", lambda *args: pytest.fail("missing lock must fail before checks")) + assert gate.main() == 1 + + +def test_source_gate_checks_every_environment_but_never_claims_upgrade(gate, monkeypatch, capsys): + calls = [] + monkeypatch.setattr(sys, "argv", ["beta-gate.py", "--without-upgrade"]) + monkeypatch.setattr(gate, "run", lambda label, argv: calls.append((label, argv)) or True) + assert gate.main() == 0 + scan = next(argv for label, argv in calls if label == "Locked dependency scan") + assert {arg for arg in scan if arg.startswith("requirements.txt:")} == { + "requirements.txt:requirements.lock", "requirements.txt:requirements-dev.lock", + "requirements.txt:requirements-verifier.lock", "requirements.txt:deploy/computer-requirements.lock", + } + assert "--no-resolve" in scan + assert any("scripts/sync-verifier-lock.py" in argv and "--check" in argv for _, argv in calls) + assert not any("scripts/beta-clean-install.py" in argv or "scripts/beta-upgrade-rehearsal.py" in argv + for _, argv in calls) + assert "signed upgrade/rollback was NOT run" in capsys.readouterr().out + + +def test_security_failure_stops_before_installation(gate, monkeypatch): + calls = [] + monkeypatch.setattr(sys, "argv", ["beta-gate.py", "--previous", "0.20.0-beta.9", + "--current", "0.20.0-rc.1"]) + def run(label, argv): + calls.append(label) + return label != "Locked dependency scan" + monkeypatch.setattr(gate, "run", run) + assert gate.main() == 1 + assert calls[-1] == "Locked dependency scan" + assert "Clean install from published release" not in calls + + +def test_full_gate_passes_explicit_versions_and_assets_to_real_rehearsal(gate, monkeypatch): + calls = [] + assets = gate.ROOT / "assets" + monkeypatch.setattr(sys, "argv", ["beta-gate.py", "--previous", "0.20.0-beta.9", + "--current", "0.20.0-rc.1", "--assets-dir", str(assets), + "--base", "https://example.invalid"]) + monkeypatch.setattr(gate, "run", lambda label, argv: calls.append((label, argv)) or True) + assert gate.main() == 0 + install = next(argv for label, argv in calls if label == "Clean install from published release") + upgrade = next(argv for label, argv in calls if label == "Signed upgrade, migration and rollback") + assert install[-4:] == ["--current", "0.20.0-rc.1", "--base", "https://example.invalid"] + assert upgrade[-6:] == ["--previous", "0.20.0-beta.9", "--current", "0.20.0-rc.1", + "--assets-dir", str(assets.resolve())]