Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 11 additions & 2 deletions bin/base-demo-services
Original file line number Diff line number Diff line change
Expand Up @@ -696,6 +696,8 @@ def check_service(
return CheckResult(False, label, f"process:{state_status.kind}")

if check_type == "process":
if lifecycle.get("type") != "process":
return CheckResult(False, "process", "process:missing lifecycle")
return CheckResult(True, "process", "process:running")

if check_type == "none":
Expand Down Expand Up @@ -742,8 +744,15 @@ def check_service(
return CheckResult(False, "command:<invalid command>", "command:<invalid command>")
label = f"command:{shlex.join(command)}"
try:
result = subprocess.run(command, cwd=root, check=False, capture_output=True, text=True, timeout=3.0)
except (FileNotFoundError, PermissionError, subprocess.TimeoutExpired) as exc:
result = subprocess.run(
command,
cwd=root,
check=False,
capture_output=True,
text=False,
timeout=3.0,
)
except (OSError, subprocess.TimeoutExpired) as exc:
return CheckResult(False, label, f"{label} ({exc})")
return CheckResult(result.returncode == 0, label, label)

Expand Down
9 changes: 9 additions & 0 deletions bin/base_demo_environment.py
Original file line number Diff line number Diff line change
Expand Up @@ -115,6 +115,7 @@ def validate_service_catalog(payload: Any) -> list[dict[str, Any]]:
)

check = service.get("check")
check_type: str | None = None
if check is not None:
if not isinstance(check, dict):
raise ValueError(f"{location}.check must be an object")
Expand Down Expand Up @@ -155,6 +156,14 @@ def validate_service_catalog(payload: Any) -> list[dict[str, Any]]:
"must be a positive finite number"
)

if check_type == "process" and not (
isinstance(lifecycle, dict) and lifecycle.get("type") == "process"
):
raise ValueError(
f"{location}.check.type process requires "
f"{location}.lifecycle.type process"
)

return services


Expand Down
2 changes: 1 addition & 1 deletion docs/base-capability-matrix.md
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ either claim.
| Representative build, test, service, environment, and non-interactive demo loop | `1.9.0` | The manifest targets in [`base_manifest.yaml`](../base_manifest.yaml), baseline gate in [`tests/validate.sh`](../tests/validate.sh), and focused suites in [`tests/services_test.bats`](../tests/services_test.bats), [`tests/environments_test.bats`](../tests/environments_test.bats), and [`tests/demo_test.bats`](../tests/demo_test.bats) | Full project loop is macOS; Ubuntu/Debian validates Base setup and project health only | Demonstrated | base-demo | Add executable evidence before calling a new service or command demonstrated. |
| Linux and WSL2 read-only support boundary | `1.9.0` | The supported commands and explicit native-Windows boundary are in [`README.md`](../README.md) and [`docs/contracts.md`](contracts.md); CI's Ubuntu path is in [`.github/workflows/tests.yml`](../.github/workflows/tests.yml) | Ubuntu/Debian and WSL2 use setup, dev-profile, check, and doctor/read-only paths; native Windows is excluded | Demonstrated | Base + base-demo | Keep platform claims tied to a hosted or repository-local check. |
| Base `1.9.0` released-compatibility pin and full Go/live-HTTP evidence | `1.9.0` | Structured pins in [supported inputs](../.release/supported-dependencies.json), verified by `bin/base-demo-dependencies`; full-language and live-HTTP gates run in [`.github/workflows/tests.yml`](../.github/workflows/tests.yml) | Exact macOS 14 full demo and Ubuntu 24.04 setup/read-only scope; no Linux full-demo claim | Demonstrated | Base + base-demo | Keep the stable pin unchanged until the final v1.10.0 release BOM is verified; candidate refresh is tracked by #324. |
| Workspace inventory, selected tests and targeted recovery | `1.9.0` reports; exact `1.10.0` candidate | `tests/scenarios/workspace.py` and [workspace scenario guide](workspace-scenarios.md), run by isolated scenario CI | macOS fixture lane; selection and expanded recovery require the exact candidate | Demonstrated | Base + base-demo | Refresh candidate evidence before release; do not claim stable 1.10 support. |
| Workspace inventory, selected tests, targeted recovery and update preflight safety | `1.9.0` reports; exact `1.10.0` candidate | `tests/scenarios/workspace.py` and [workspace scenario guide](workspace-scenarios.md), run by isolated scenario CI; candidate fixtures cover dirty, ancestor-checkout and mismatched-upstream protection | macOS fixture lane; selection, update preflight and expanded recovery require the exact candidate | Demonstrated | Base + base-demo | Refresh candidate evidence before release; do not claim stable 1.10 support. |
| Trust lifecycle and separate runtime/IDE consent | `1.9.0` baseline; exact `1.10.0` candidate | `tests/scenarios/trust.py` and [trust scenario guide](trust-scenarios.md), executed by [isolated scenario CI](../.github/workflows/scenarios.yml) | macOS fixture lane; full historical revocation and runtime inspection require the pinned candidate | Demonstrated | Base + base-demo | Refresh exact candidate evidence before release; do not attribute candidate-only guarantees to v1.9.0. |
| Optional `test.requirements` and uninstall guidance | `1.9.0` | The manifest test entry and contributor setup guidance are [`base_manifest.yaml`](../base_manifest.yaml) and [`README.md`](../README.md) | Optional metadata is not required for the baseline demo contract | Intentionally omitted | base-demo | Revisit when Base publishes a stable user-facing contract and a concrete scenario. |
| Native Windows support | `1.11.0` (planned) | The current non-goal is recorded in [`README.md`](../README.md); the staged Base work is tracked by [base-demo#302](https://github.com/basefoundry/base-demo/issues/302) and Base [#2215](https://github.com/basefoundry/base/issues/2215) | Native Windows is not shipped; Git Bash and WSL2 do not count as native Windows evidence | Blocked upstream | Base + base-demo | Wait for the PowerShell-first Base contract and hosted Windows evidence. |
Expand Down
14 changes: 12 additions & 2 deletions docs/workspace-scenarios.md
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,10 @@

This compact scenario creates disposable healthy, failing, untrusted, no-test,
missing-required and undeclared peers. It uses the same isolated HOME, cache,
provider checks and cleanup as the [trust scenario](trust-scenarios.md). No
clone, setup, repository initialization or learner-state mutation is performed.
provider checks and cleanup as the [trust scenario](trust-scenarios.md). It does
not clone or set up learner repositories or mutate learner state. Candidate
coverage also creates local Git fixtures to exercise update preflight safety;
those repositories are disposable and never leave the fixture directory.
Generated recovery commands are inspected; only a reviewed fixture-local trust
command is executed. Expected failures are assertions, not repair requests.

Expand All @@ -22,6 +24,10 @@ Base 1.9 checks the existing status, onboarding and agent-brief reports, their
workspace identity, missing-peer reporting and read-only behavior. It prints
an explicit supported-version boundary without invoking newer flags.

The candidate update fixtures use `git init --initial-branch` and `git switch`;
local runs therefore require Git 2.28 or newer, matching the hosted runner
toolchain.

For the implemented 1.10 candidate, select its checkout and exact commit
`06715fed19aceff27844aba546ba2a0b4c594ac7`, and add `--candidate`.
[Scenario CI](../.github/workflows/scenarios.yml) runs both lanes. This is
Expand All @@ -43,6 +49,10 @@ The candidate asserts:
1; successful selection exits 0. `--fail-fast` skips later selected peers.
- Two selected aliases of one manifest are rejected (exit 2); selecting one
alias runs the intended checkout once.
- `workspace update --dry-run --format json` rejects a dirty checkout, a
manifest path nested inside an ancestor Git checkout, and a default branch
tracking the wrong upstream branch. The assertions verify the structured
preflight classification and that no checkout changes are made.

Each completed assertion group prints `PASS`; fixtures are removed on success
or exception. There is no second application stack and no host-readiness claim.
Expand Down
123 changes: 122 additions & 1 deletion tests/scenarios/workspace.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,10 +3,76 @@

import json
import shlex
import subprocess

from fixture import Fixture, arguments


def run_git(*args, cwd=None):
return subprocess.run(
["git", *map(str, args)],
cwd=cwd,
check=True,
capture_output=True,
text=True,
)


def create_bare_remote(root):
root.parent.mkdir(parents=True, exist_ok=True)
run_git("init", "--bare", "--initial-branch=main", root)


def create_git_checkout(root, remote=None):
root.parent.mkdir(parents=True, exist_ok=True)
run_git("init", "--initial-branch=main", root)
run_git("config", "user.name", "Base Demo Scenario", cwd=root)
run_git("config", "user.email", "scenario@example.com", cwd=root)
(root / "README.md").write_text("scenario\n")
run_git("add", "README.md", cwd=root)
run_git("commit", "-m", "initial", cwd=root)
if remote is not None:
run_git("remote", "add", "origin", remote, cwd=root)
run_git("push", "--set-upstream", "origin", "main", cwd=root)


def write_update_manifest(path, repository):
path.write_text(
"schema_version: 1\n"
"workspace:\n"
" name: update-safety\n"
"repos:\n"
f" - name: {repository}\n"
" default_branch: main\n"
)
return path


def run_update(fixture, workspace, manifest):
result = fixture.run(
"workspace",
"update",
"--workspace",
workspace,
"--manifest",
manifest,
"--dry-run",
"--format",
"json",
expected=1,
)
return json.loads(result.stdout)


def assert_preflight(payload, repository, issue):
report = next(item for item in payload["repositories"] if item["repository"] == repository)
assert report["action"] == "skip"
assert report["status"] == "skipped"
assert report["preflight"] == [issue]
assert report["fatal"]
assert payload["counts"] == {"planned": 0, "updated": 0, "unchanged": 0, "skipped": 1, "failed": 1}


def main():
args = arguments(__doc__)
with Fixture(args) as fixture:
Expand Down Expand Up @@ -35,7 +101,7 @@ def main():
assert not list(fixture.workspace.rglob("tested")), "read-only inventory executed a test"
if not args.candidate:
print("PASS: stable workspace status, onboarding and agent-brief are read-only")
print("BOUNDARY: selected workspace tests, expanded inventory and targeted next_actions require the v1.10 candidate")
print("BOUNDARY: selected workspace tests, expanded inventory, targeted next_actions and workspace-update preflight safety require the v1.10 candidate")
return

assert "test" in fixture.run("workspace", "--help").stdout
Expand Down Expand Up @@ -84,6 +150,61 @@ def main():
fixture.run("trust", "allow", name, "--workspace", fixture.workspace)
print("PASS: ordered recovery actions bind the reviewed checkout and reject stale manifest evidence")

update_root = fixture.root / "workspace-update-fixtures"
update_root.mkdir()

dirty_workspace = update_root / "dirty-workspace"
dirty_workspace.mkdir()
dirty_remote = update_root / "dirty-origin.git"
create_bare_remote(dirty_remote)
dirty_checkout = dirty_workspace / "dirty"
create_git_checkout(dirty_checkout, dirty_remote)
(dirty_checkout / "local-change").write_text("must survive\n")
dirty_head = run_git("rev-parse", "HEAD", cwd=dirty_checkout).stdout.strip()
dirty_payload = run_update(
fixture,
dirty_workspace,
write_update_manifest(update_root / "dirty-workspace.yaml", "dirty"),
)
assert_preflight(dirty_payload, "dirty", "dirty")
assert run_git("rev-parse", "HEAD", cwd=dirty_checkout).stdout.strip() == dirty_head
assert (dirty_checkout / "local-change").read_text() == "must survive\n"

ancestor_workspace = update_root / "ancestor-workspace"
ancestor_workspace.mkdir()
create_git_checkout(ancestor_workspace)
(ancestor_workspace / "nested").mkdir()
ancestor_payload = run_update(
fixture,
ancestor_workspace,
write_update_manifest(update_root / "ancestor-workspace.yaml", "nested"),
)
assert_preflight(ancestor_payload, "nested", "checkout_root_mismatch")

mismatch_workspace = update_root / "mismatch-workspace"
mismatch_workspace.mkdir()
mismatch_remote = update_root / "mismatch-origin.git"
mismatch_seed = update_root / "mismatch-seed"
create_bare_remote(mismatch_remote)
create_git_checkout(mismatch_seed, mismatch_remote)
run_git("switch", "-c", "feature", cwd=mismatch_seed)
(mismatch_seed / "feature.txt").write_text("feature\n")
run_git("add", "feature.txt", cwd=mismatch_seed)
run_git("commit", "-m", "feature", cwd=mismatch_seed)
run_git("push", "--set-upstream", "origin", "feature", cwd=mismatch_seed)
mismatch_checkout = mismatch_workspace / "mismatched"
run_git("clone", "--branch", "main", mismatch_remote, mismatch_checkout)
run_git("branch", "--set-upstream-to=origin/feature", "main", cwd=mismatch_checkout)
mismatch_head = run_git("rev-parse", "HEAD", cwd=mismatch_checkout).stdout.strip()
mismatch_payload = run_update(
fixture,
mismatch_workspace,
write_update_manifest(update_root / "mismatch-workspace.yaml", "mismatched"),
)
assert_preflight(mismatch_payload, "mismatched", "upstream_mismatch")
assert run_git("rev-parse", "HEAD", cwd=mismatch_checkout).stdout.strip() == mismatch_head
print("PASS: workspace update dry-run rejects dirty, ancestor and mismatched-upstream checkouts without changing them")

# Selection is a filter, not an execution-order override. Put the
# failing peer first in declaration order for the fail-fast example.
manifest.write_text(manifest.read_text().replace(
Expand Down
107 changes: 107 additions & 0 deletions tests/services_test.bats
Original file line number Diff line number Diff line change
Expand Up @@ -462,6 +462,51 @@ PY
done
}

@test "services rejects process checks without a process lifecycle" {
local catalog="$TEST_TMPDIR/catalog.json"

cat > "$catalog" <<'EOF'
{
"services": [
{
"name": "missing-process-lifecycle",
"kind": "service",
"runtime": "test",
"required": true,
"check": {"type": "process"}
}
]
}
EOF

run "$TEST_ROOT/bin/base-demo-services" --catalog "$catalog" status

[ "$status" -eq 2 ]
[[ "$output" == *"ERROR: service catalog.services[0].check.type process requires service catalog.services[0].lifecycle.type process"* ]]
[[ "$output" != *"Traceback"* ]]
}

@test "the catalog validator rejects process checks through its direct API" {
run python3 - "$TEST_ROOT" <<'PY'
import sys

sys.path.insert(0, sys.argv[1] + "/bin")
from base_demo_environment import validate_service_catalog

try:
validate_service_catalog({"services": [{"name": "invalid", "check": {"type": "process"}}]})
except ValueError as exc:
assert str(exc) == (
"service catalog.services[0].check.type process requires "
"service catalog.services[0].lifecycle.type process"
)
else:
raise AssertionError("process check without lifecycle was accepted")
PY

[ "$status" -eq 0 ]
}

@test "services status shows catalog entries" {
run "$TEST_ROOT/bin/base-demo-services" status

Expand Down Expand Up @@ -589,6 +634,68 @@ EOF
[[ "$output" == *"missing-http skip optional http:<missing health_url>"* ]]
}

@test "services health isolates command execution and decoding failures" {
local catalog="$TEST_TMPDIR/catalog.json"
local bad_command="$TEST_TMPDIR/no-shebang"
local byte_command="$TEST_TMPDIR/invalid-bytes.py"

printf 'exit 0\n' > "$bad_command"
chmod +x "$bad_command"
cat > "$byte_command" <<'PY'
#!/usr/bin/env python3
import sys

sys.stdout.buffer.write(b"\xff\n")
raise SystemExit(7)
PY
chmod +x "$byte_command"

python3 - "$catalog" "$bad_command" "$byte_command" <<'PY'
import json
import sys

catalog, bad_command, byte_command = sys.argv[1:]
with open(catalog, "w", encoding="utf-8") as handle:
json.dump(
{
"services": [
{
"name": "bad-command",
"kind": "service",
"runtime": "test",
"required": True,
"check": {"type": "command", "command": [bad_command]},
},
{
"name": "invalid-bytes",
"kind": "service",
"runtime": "test",
"required": False,
"check": {"type": "command", "command": [byte_command]},
},
{
"name": "later-service",
"kind": "service",
"runtime": "test",
"required": False,
"check": {"type": "none"},
},
]
},
handle,
)
PY

run "$TEST_ROOT/bin/base-demo-services" --catalog "$catalog" check

[ "$status" -eq 1 ]
[[ "$output" == *"bad-command fail command:"* ]]
[[ "$output" == *"invalid-bytes skip optional command:"* ]]
[[ "$output" == *"later-service ok"* ]]
[[ "$output" != *"Traceback"* ]]
[[ "$output" != *"UnicodeDecodeError"* ]]
}

@test "services validates the complete lifecycle plan before Compose mutation" {
local catalog="$TEST_TMPDIR/catalog.json"
local fake_bin="$TEST_TMPDIR/bin"
Expand Down
Loading