From 7b05641f602cf8e8571499f124a339b014147f4d Mon Sep 17 00:00:00 2001 From: Ramesh Padmanabhaiah <22363102+codeforester@users.noreply.github.com> Date: Mon, 5 Oct 2026 22:55:48 +0530 Subject: [PATCH 1/4] ci: cover workspace update preflight safety --- docs/base-capability-matrix.md | 2 +- docs/workspace-scenarios.md | 10 ++- tests/scenarios/workspace.py | 121 +++++++++++++++++++++++++++++++++ 3 files changed, 130 insertions(+), 3 deletions(-) diff --git a/docs/base-capability-matrix.md b/docs/base-capability-matrix.md index 2c2fa58..0b12a52 100644 --- a/docs/base-capability-matrix.md +++ b/docs/base-capability-matrix.md @@ -25,7 +25,7 @@ either claim. | Representative build, test, service, environment, and non-interactive demo loop | `1.9.0` | The manifest targets in [`base_manifest.yaml`](../base_manifest.yaml), baseline gate in [`tests/validate.sh`](../tests/validate.sh), and focused suites in [`tests/services_test.bats`](../tests/services_test.bats), [`tests/environments_test.bats`](../tests/environments_test.bats), and [`tests/demo_test.bats`](../tests/demo_test.bats) | Full project loop is macOS; Ubuntu/Debian validates Base setup and project health only | Demonstrated | base-demo | Add executable evidence before calling a new service or command demonstrated. | | Linux and WSL2 read-only support boundary | `1.9.0` | The supported commands and explicit native-Windows boundary are in [`README.md`](../README.md) and [`docs/contracts.md`](contracts.md); CI's Ubuntu path is in [`.github/workflows/tests.yml`](../.github/workflows/tests.yml) | Ubuntu/Debian and WSL2 use setup, dev-profile, check, and doctor/read-only paths; native Windows is excluded | Demonstrated | Base + base-demo | Keep platform claims tied to a hosted or repository-local check. | | Base `1.9.0` released-compatibility pin and full Go/live-HTTP evidence | `1.9.0` | Structured pins in [supported inputs](../.release/supported-dependencies.json), verified by `bin/base-demo-dependencies`; full-language and live-HTTP gates run in [`.github/workflows/tests.yml`](../.github/workflows/tests.yml) | Exact macOS 14 full demo and Ubuntu 24.04 setup/read-only scope; no Linux full-demo claim | Demonstrated | Base + base-demo | Keep the stable pin unchanged until the final v1.10.0 release BOM is verified; candidate refresh is tracked by #324. | -| Workspace inventory, selected tests and targeted recovery | `1.9.0` reports; exact `1.10.0` candidate | `tests/scenarios/workspace.py` and [workspace scenario guide](workspace-scenarios.md), run by isolated scenario CI | macOS fixture lane; selection and expanded recovery require the exact candidate | Demonstrated | Base + base-demo | Refresh candidate evidence before release; do not claim stable 1.10 support. | +| Workspace inventory, selected tests, targeted recovery and update preflight safety | `1.9.0` reports; exact `1.10.0` candidate | `tests/scenarios/workspace.py` and [workspace scenario guide](workspace-scenarios.md), run by isolated scenario CI; candidate fixtures cover dirty, ancestor-checkout and mismatched-upstream protection | macOS fixture lane; selection, update preflight and expanded recovery require the exact candidate | Demonstrated | Base + base-demo | Refresh candidate evidence before release; do not claim stable 1.10 support. | | Trust lifecycle and separate runtime/IDE consent | `1.9.0` baseline; exact `1.10.0` candidate | `tests/scenarios/trust.py` and [trust scenario guide](trust-scenarios.md), executed by [isolated scenario CI](../.github/workflows/scenarios.yml) | macOS fixture lane; full historical revocation and runtime inspection require the pinned candidate | Demonstrated | Base + base-demo | Refresh exact candidate evidence before release; do not attribute candidate-only guarantees to v1.9.0. | | Optional `test.requirements` and uninstall guidance | `1.9.0` | The manifest test entry and contributor setup guidance are [`base_manifest.yaml`](../base_manifest.yaml) and [`README.md`](../README.md) | Optional metadata is not required for the baseline demo contract | Intentionally omitted | base-demo | Revisit when Base publishes a stable user-facing contract and a concrete scenario. | | Native Windows support | `1.11.0` (planned) | The current non-goal is recorded in [`README.md`](../README.md); the staged Base work is tracked by [base-demo#302](https://github.com/basefoundry/base-demo/issues/302) and Base [#2215](https://github.com/basefoundry/base/issues/2215) | Native Windows is not shipped; Git Bash and WSL2 do not count as native Windows evidence | Blocked upstream | Base + base-demo | Wait for the PowerShell-first Base contract and hosted Windows evidence. | diff --git a/docs/workspace-scenarios.md b/docs/workspace-scenarios.md index b92d971..31872bf 100644 --- a/docs/workspace-scenarios.md +++ b/docs/workspace-scenarios.md @@ -2,8 +2,10 @@ This compact scenario creates disposable healthy, failing, untrusted, no-test, missing-required and undeclared peers. It uses the same isolated HOME, cache, -provider checks and cleanup as the [trust scenario](trust-scenarios.md). No -clone, setup, repository initialization or learner-state mutation is performed. +provider checks and cleanup as the [trust scenario](trust-scenarios.md). It does +not clone or set up learner repositories or mutate learner state. Candidate +coverage also creates local Git fixtures to exercise update preflight safety; +those repositories are disposable and never leave the fixture directory. Generated recovery commands are inspected; only a reviewed fixture-local trust command is executed. Expected failures are assertions, not repair requests. @@ -43,6 +45,10 @@ The candidate asserts: 1; successful selection exits 0. `--fail-fast` skips later selected peers. - Two selected aliases of one manifest are rejected (exit 2); selecting one alias runs the intended checkout once. +- `workspace update --dry-run --format json` rejects a dirty checkout, a + manifest path nested inside an ancestor Git checkout, and a default branch + tracking the wrong upstream branch. The assertions verify the structured + preflight classification and that no checkout changes are made. Each completed assertion group prints `PASS`; fixtures are removed on success or exception. There is no second application stack and no host-readiness claim. diff --git a/tests/scenarios/workspace.py b/tests/scenarios/workspace.py index 0e28f9e..c32142b 100644 --- a/tests/scenarios/workspace.py +++ b/tests/scenarios/workspace.py @@ -3,10 +3,76 @@ import json import shlex +import subprocess from fixture import Fixture, arguments +def run_git(*args, cwd=None): + return subprocess.run( + ["git", *map(str, args)], + cwd=cwd, + check=True, + capture_output=True, + text=True, + ) + + +def create_bare_remote(root): + root.parent.mkdir(parents=True, exist_ok=True) + run_git("init", "--bare", "--initial-branch=main", root) + + +def create_git_checkout(root, remote=None): + root.parent.mkdir(parents=True, exist_ok=True) + run_git("init", "--initial-branch=main", root) + run_git("config", "user.name", "Base Demo Scenario", cwd=root) + run_git("config", "user.email", "scenario@example.com", cwd=root) + (root / "README.md").write_text("scenario\n") + run_git("add", "README.md", cwd=root) + run_git("commit", "-m", "initial", cwd=root) + if remote is not None: + run_git("remote", "add", "origin", remote, cwd=root) + run_git("push", "--set-upstream", "origin", "main", cwd=root) + + +def write_update_manifest(path, repository): + path.write_text( + "schema_version: 1\n" + "workspace:\n" + " name: update-safety\n" + "repos:\n" + f" - name: {repository}\n" + " default_branch: main\n" + ) + return path + + +def run_update(fixture, workspace, manifest): + result = fixture.run( + "workspace", + "update", + "--workspace", + workspace, + "--manifest", + manifest, + "--dry-run", + "--format", + "json", + expected=1, + ) + return json.loads(result.stdout) + + +def assert_preflight(payload, repository, issue): + report = next(item for item in payload["repositories"] if item["repository"] == repository) + assert report["action"] == "skip" + assert report["status"] == "skipped" + assert report["preflight"] == [issue] + assert report["fatal"] + assert payload["counts"] == {"planned": 0, "updated": 0, "unchanged": 0, "skipped": 1, "failed": 1} + + def main(): args = arguments(__doc__) with Fixture(args) as fixture: @@ -84,6 +150,61 @@ def main(): fixture.run("trust", "allow", name, "--workspace", fixture.workspace) print("PASS: ordered recovery actions bind the reviewed checkout and reject stale manifest evidence") + update_root = fixture.root / "workspace-update-fixtures" + update_root.mkdir() + + dirty_workspace = update_root / "dirty-workspace" + dirty_workspace.mkdir() + dirty_remote = update_root / "dirty-origin.git" + create_bare_remote(dirty_remote) + dirty_checkout = dirty_workspace / "dirty" + create_git_checkout(dirty_checkout, dirty_remote) + (dirty_checkout / "local-change").write_text("must survive\n") + dirty_head = run_git("rev-parse", "HEAD", cwd=dirty_checkout).stdout.strip() + dirty_payload = run_update( + fixture, + dirty_workspace, + write_update_manifest(update_root / "dirty-workspace.yaml", "dirty"), + ) + assert_preflight(dirty_payload, "dirty", "dirty") + assert run_git("rev-parse", "HEAD", cwd=dirty_checkout).stdout.strip() == dirty_head + assert (dirty_checkout / "local-change").read_text() == "must survive\n" + + ancestor_workspace = update_root / "ancestor-workspace" + ancestor_workspace.mkdir() + create_git_checkout(ancestor_workspace) + (ancestor_workspace / "nested").mkdir() + ancestor_payload = run_update( + fixture, + ancestor_workspace, + write_update_manifest(update_root / "ancestor-workspace.yaml", "nested"), + ) + assert_preflight(ancestor_payload, "nested", "checkout_root_mismatch") + + mismatch_workspace = update_root / "mismatch-workspace" + mismatch_workspace.mkdir() + mismatch_remote = update_root / "mismatch-origin.git" + mismatch_seed = update_root / "mismatch-seed" + create_bare_remote(mismatch_remote) + create_git_checkout(mismatch_seed, mismatch_remote) + run_git("switch", "-c", "feature", cwd=mismatch_seed) + (mismatch_seed / "feature.txt").write_text("feature\n") + run_git("add", "feature.txt", cwd=mismatch_seed) + run_git("commit", "-m", "feature", cwd=mismatch_seed) + run_git("push", "--set-upstream", "origin", "feature", cwd=mismatch_seed) + mismatch_checkout = mismatch_workspace / "mismatched" + run_git("clone", "--branch", "main", mismatch_remote, mismatch_checkout) + run_git("branch", "--set-upstream-to=origin/feature", "main", cwd=mismatch_checkout) + mismatch_head = run_git("rev-parse", "HEAD", cwd=mismatch_checkout).stdout.strip() + mismatch_payload = run_update( + fixture, + mismatch_workspace, + write_update_manifest(update_root / "mismatch-workspace.yaml", "mismatched"), + ) + assert_preflight(mismatch_payload, "mismatched", "upstream_mismatch") + assert run_git("rev-parse", "HEAD", cwd=mismatch_checkout).stdout.strip() == mismatch_head + print("PASS: workspace update dry-run rejects dirty, ancestor and mismatched-upstream checkouts without changing them") + # Selection is a filter, not an execution-order override. Put the # failing peer first in declaration order for the fail-fast example. manifest.write_text(manifest.read_text().replace( From 42c352fa25e2eaf245dcd86668500fdd1f5da8ca Mon Sep 17 00:00:00 2001 From: Ramesh Padmanabhaiah <22363102+codeforester@users.noreply.github.com> Date: Tue, 6 Oct 2026 18:24:42 +0530 Subject: [PATCH 2/4] docs: clarify candidate workspace scenario boundary --- docs/workspace-scenarios.md | 4 ++++ tests/scenarios/workspace.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/workspace-scenarios.md b/docs/workspace-scenarios.md index 31872bf..ce1255f 100644 --- a/docs/workspace-scenarios.md +++ b/docs/workspace-scenarios.md @@ -24,6 +24,10 @@ Base 1.9 checks the existing status, onboarding and agent-brief reports, their workspace identity, missing-peer reporting and read-only behavior. It prints an explicit supported-version boundary without invoking newer flags. +The candidate update fixtures use `git init --initial-branch` and `git switch`; +local runs therefore require Git 2.28 or newer, matching the hosted runner +toolchain. + For the implemented 1.10 candidate, select its checkout and exact commit `06715fed19aceff27844aba546ba2a0b4c594ac7`, and add `--candidate`. [Scenario CI](../.github/workflows/scenarios.yml) runs both lanes. This is diff --git a/tests/scenarios/workspace.py b/tests/scenarios/workspace.py index c32142b..17c1b46 100644 --- a/tests/scenarios/workspace.py +++ b/tests/scenarios/workspace.py @@ -101,7 +101,7 @@ def main(): assert not list(fixture.workspace.rglob("tested")), "read-only inventory executed a test" if not args.candidate: print("PASS: stable workspace status, onboarding and agent-brief are read-only") - print("BOUNDARY: selected workspace tests, expanded inventory and targeted next_actions require the v1.10 candidate") + print("BOUNDARY: selected workspace tests, expanded inventory, targeted next_actions and workspace-update preflight safety require the v1.10 candidate") return assert "test" in fixture.run("workspace", "--help").stdout From 87f20fcde0dca3cab38177d7ea193bae0135831a Mon Sep 17 00:00:00 2001 From: Ramesh Padmanabhaiah <22363102+codeforester@users.noreply.github.com> Date: Tue, 6 Oct 2026 07:59:03 +0530 Subject: [PATCH 3/4] fix(services): require lifecycle for process checks --- bin/base-demo-services | 2 ++ bin/base_demo_environment.py | 9 ++++++++ tests/services_test.bats | 45 ++++++++++++++++++++++++++++++++++++ 3 files changed, 56 insertions(+) diff --git a/bin/base-demo-services b/bin/base-demo-services index 601c2fd..d3881ff 100755 --- a/bin/base-demo-services +++ b/bin/base-demo-services @@ -696,6 +696,8 @@ def check_service( return CheckResult(False, label, f"process:{state_status.kind}") if check_type == "process": + if lifecycle.get("type") != "process": + return CheckResult(False, "process", "process:missing lifecycle") return CheckResult(True, "process", "process:running") if check_type == "none": diff --git a/bin/base_demo_environment.py b/bin/base_demo_environment.py index ee1eafe..51c8e42 100644 --- a/bin/base_demo_environment.py +++ b/bin/base_demo_environment.py @@ -115,6 +115,7 @@ def validate_service_catalog(payload: Any) -> list[dict[str, Any]]: ) check = service.get("check") + check_type: str | None = None if check is not None: if not isinstance(check, dict): raise ValueError(f"{location}.check must be an object") @@ -155,6 +156,14 @@ def validate_service_catalog(payload: Any) -> list[dict[str, Any]]: "must be a positive finite number" ) + if check_type == "process" and not ( + isinstance(lifecycle, dict) and lifecycle.get("type") == "process" + ): + raise ValueError( + f"{location}.check.type process requires " + f"{location}.lifecycle.type process" + ) + return services diff --git a/tests/services_test.bats b/tests/services_test.bats index c9b8b95..a0e2eaa 100644 --- a/tests/services_test.bats +++ b/tests/services_test.bats @@ -462,6 +462,51 @@ PY done } +@test "services rejects process checks without a process lifecycle" { + local catalog="$TEST_TMPDIR/catalog.json" + + cat > "$catalog" <<'EOF' +{ + "services": [ + { + "name": "missing-process-lifecycle", + "kind": "service", + "runtime": "test", + "required": true, + "check": {"type": "process"} + } + ] +} +EOF + + run "$TEST_ROOT/bin/base-demo-services" --catalog "$catalog" status + + [ "$status" -eq 2 ] + [[ "$output" == *"ERROR: service catalog.services[0].check.type process requires service catalog.services[0].lifecycle.type process"* ]] + [[ "$output" != *"Traceback"* ]] +} + +@test "the catalog validator rejects process checks through its direct API" { + run python3 - "$TEST_ROOT" <<'PY' +import sys + +sys.path.insert(0, sys.argv[1] + "/bin") +from base_demo_environment import validate_service_catalog + +try: + validate_service_catalog({"services": [{"name": "invalid", "check": {"type": "process"}}]}) +except ValueError as exc: + assert str(exc) == ( + "service catalog.services[0].check.type process requires " + "service catalog.services[0].lifecycle.type process" + ) +else: + raise AssertionError("process check without lifecycle was accepted") +PY + + [ "$status" -eq 0 ] +} + @test "services status shows catalog entries" { run "$TEST_ROOT/bin/base-demo-services" status From c614c0b2218fb3739e1f8f2d93a5cfb862810600 Mon Sep 17 00:00:00 2001 From: Ramesh Padmanabhaiah <22363102+codeforester@users.noreply.github.com> Date: Tue, 6 Oct 2026 07:59:57 +0530 Subject: [PATCH 4/4] fix(services): isolate command health failures --- bin/base-demo-services | 11 +++++-- tests/services_test.bats | 62 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 71 insertions(+), 2 deletions(-) diff --git a/bin/base-demo-services b/bin/base-demo-services index d3881ff..70e56cb 100755 --- a/bin/base-demo-services +++ b/bin/base-demo-services @@ -744,8 +744,15 @@ def check_service( return CheckResult(False, "command:", "command:") label = f"command:{shlex.join(command)}" try: - result = subprocess.run(command, cwd=root, check=False, capture_output=True, text=True, timeout=3.0) - except (FileNotFoundError, PermissionError, subprocess.TimeoutExpired) as exc: + result = subprocess.run( + command, + cwd=root, + check=False, + capture_output=True, + text=False, + timeout=3.0, + ) + except (OSError, subprocess.TimeoutExpired) as exc: return CheckResult(False, label, f"{label} ({exc})") return CheckResult(result.returncode == 0, label, label) diff --git a/tests/services_test.bats b/tests/services_test.bats index a0e2eaa..f59b24e 100644 --- a/tests/services_test.bats +++ b/tests/services_test.bats @@ -634,6 +634,68 @@ EOF [[ "$output" == *"missing-http skip optional http:"* ]] } +@test "services health isolates command execution and decoding failures" { + local catalog="$TEST_TMPDIR/catalog.json" + local bad_command="$TEST_TMPDIR/no-shebang" + local byte_command="$TEST_TMPDIR/invalid-bytes.py" + + printf 'exit 0\n' > "$bad_command" + chmod +x "$bad_command" + cat > "$byte_command" <<'PY' +#!/usr/bin/env python3 +import sys + +sys.stdout.buffer.write(b"\xff\n") +raise SystemExit(7) +PY + chmod +x "$byte_command" + + python3 - "$catalog" "$bad_command" "$byte_command" <<'PY' +import json +import sys + +catalog, bad_command, byte_command = sys.argv[1:] +with open(catalog, "w", encoding="utf-8") as handle: + json.dump( + { + "services": [ + { + "name": "bad-command", + "kind": "service", + "runtime": "test", + "required": True, + "check": {"type": "command", "command": [bad_command]}, + }, + { + "name": "invalid-bytes", + "kind": "service", + "runtime": "test", + "required": False, + "check": {"type": "command", "command": [byte_command]}, + }, + { + "name": "later-service", + "kind": "service", + "runtime": "test", + "required": False, + "check": {"type": "none"}, + }, + ] + }, + handle, + ) +PY + + run "$TEST_ROOT/bin/base-demo-services" --catalog "$catalog" check + + [ "$status" -eq 1 ] + [[ "$output" == *"bad-command fail command:"* ]] + [[ "$output" == *"invalid-bytes skip optional command:"* ]] + [[ "$output" == *"later-service ok"* ]] + [[ "$output" != *"Traceback"* ]] + [[ "$output" != *"UnicodeDecodeError"* ]] +} + @test "services validates the complete lifecycle plan before Compose mutation" { local catalog="$TEST_TMPDIR/catalog.json" local fake_bin="$TEST_TMPDIR/bin"