From 5f2b5256204a48b91c643ebe570aab57e2cc1699 Mon Sep 17 00:00:00 2001 From: fabbrik <22822543+fabbrik@users.noreply.github.com> Date: Sat, 14 Nov 2026 16:37:48 -0600 Subject: [PATCH 1/2] ci: prove the adoption path from the packed tool on Linux, macOS and Windows (story 29.2) eng/prove-adoption-path.sh installs AgentExperience.Cli as a global tool from the .nupkg CI packed for the same commit (never from nuget.org), then on a scratch project runs aexp init and aexp trust, finalizes a run the verifier fails and one it passes, and asserts what recall, lessons and benefit print. The adoption-path job runs it on ubuntu-24.04, macos-15 and windows-2025 from the pack job's artifact, so every OS proves the same bytes. A release test pins the job and the script. *.sh files are LF on every checkout so the Windows runner's bash can run them. Co-Authored-By: Claude Fable 5.1 --- .gitattributes | 2 + .github/workflows/ci.yml | 54 +++++ CHANGELOG.md | 4 + CONTRIBUTING.md | 4 +- docs/guide/cli-and-mcp.md | 4 +- eng/prove-adoption-path.sh | 197 ++++++++++++++++++ .../Workflows/WorkflowTests.cs | 77 +++++++ 7 files changed, 340 insertions(+), 2 deletions(-) create mode 100755 eng/prove-adoption-path.sh diff --git a/.gitattributes b/.gitattributes index ce56793e..709d2df1 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,3 +1,5 @@ *.sql text eol=lf +# Bash scripts run under `shell: bash` on the Windows runner too (ci.yml, adoption-path), where a CRLF checkout breaks them. +*.sh text eol=lf contract/** text eol=lf dogfood/** text eol=lf diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index abce0bba..0e3e3f6a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -164,6 +164,60 @@ jobs: path: artifacts/packages/ retention-days: 14 + # Story 29.2: the adoption path, proven on every OS from the same bytes. What the README promises -- install aexp, + # run init and trust, get a verified lesson back -- is done here as a stranger would do it: the tool is installed as + # a global tool from the pack job's artifact only (never from nuget.org, which may serve the published package of + # the same version), then eng/prove-adoption-path.sh runs init, trust, a failed and a verified run, and asserts what + # recall, lessons and benefit print. The three legs are required status checks on main. The job restores no project, + # so it does not use the NuGet cache; the SDK carries the runtime the tool needs. + adoption-path: + needs: pack + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-24.04, macos-15, windows-2025] + name: adoption-path (${{ matrix.os }}) + timeout-minutes: 20 + steps: + - name: Checkout + uses: actions/checkout@v5 + + # As the other jobs, but portable: GNU sed (Linux, Git Bash) and BSD sed (macOS) both take an attached backup + # suffix, where `sed -i -E` would need `-i ''` on macOS. The working copy is the runner's; nothing is committed. + - name: Pin the SDK exactly (CI only) + shell: bash + run: | + sed -i.bak -E 's/"rollForward": *"[^"]*"/"rollForward": "disable"/' global.json + rm -f global.json.bak + grep -q '"rollForward": "disable"' global.json + + # Exactly global.json's SDK (asserted below). It carries the net10.0 runtime the tool needs, so nothing else is + # installed. + - name: Setup .NET + uses: actions/setup-dotnet@v5 + with: + global-json-file: global.json + + # Belt and braces after the pin above: a different SDK fails here instead of silently winning. + - name: Assert the exact pinned SDK + shell: bash + run: | + pinned="$(sed -nE 's/.*"version": *"([^"]+)".*/\1/p' global.json)" + actual="$(dotnet --version)" + test "$actual" = "$pinned" || { echo "::error::SDK $actual is in use, but global.json pins $pinned"; exit 1; } + echo "SDK $actual, as pinned" + + - name: Download the packages the pack job built + uses: actions/download-artifact@v4 + with: + name: packages + path: artifacts/packages + + - name: Install aexp from the packed artifact and prove the adoption path + shell: bash + run: eng/prove-adoption-path.sh artifacts/packages + # The PostgreSQL matrix (story 6.3, KL-13): the storage suites, the pgvector proof, the sample's PostgreSQL # mode and the upgrades from the published previews' databases (story 12.1), on every supported major. One # variable picks both the pgvector image and the stock one, and each suite asserts the server it reached reports diff --git a/CHANGELOG.md b/CHANGELOG.md index 5adf434f..e881f8b2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,10 @@ AgentExperience.NET is a **preview**. It is not production ready, and public API trust like any edit), and prints the steps left to a person: `aexp trust`, then `aexp setup claude-code` or `claude mcp add`. It approves nothing. `aexp setup claude-code` writes its config and `.gitignore` entry through the same generator (`ProjectInit`), so there is one. See [CLI and MCP](docs/guide/cli-and-mcp.md#write-it-aexp-init). +- **The adoption path is proven in CI** on Linux, macOS and Windows: `eng/prove-adoption-path.sh` installs `aexp` from + the `.nupkg` packed for the same commit, runs `init` and `trust` on a scratch project, finalizes a failed and a + verified run, and asserts that `recall`, `lessons` and `benefit` report the lesson; the three `adoption-path` legs + gate every pull request. ### Changed diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index e9bb024d..340b6d42 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -124,7 +124,9 @@ _sdlc/ product brief, PRD, architecture, ep - Keep each PR to one concern, and explain the *why* in the description. - Use [Conventional Commits](https://www.conventionalcommits.org/) for commit messages (`feat:`, `fix:`, `docs:`, `chore:`). - Make sure `dotnet build` and `dotnet test` pass locally. CI (`.github/workflows/ci.yml`) runs on every pull request: - `build-and-test`, `pack` (with package verification), the `postgres` matrix (15 to 18, plaintext and + `build-and-test`, `pack` (with package verification), `adoption-path (ubuntu-24.04, macos-15, windows-2025)` (installs + `aexp` from the packed artifact on each OS and runs `eng/prove-adoption-path.sh`: init, trust, a failed and a + verified run, recall, lessons, benefit), the `postgres` matrix (15 to 18, plaintext and crypto-shredding), the MAF floor (`maf-compatibility (pinned)`), the Python port's `python` matrix (CPython 3.10 to 3.14, plus its PostgreSQL 16 and 18 legs; [python/README.md](python/README.md#running-the-tests) says how to run it locally) and the Go port's `go` matrix (the core on Go 1.24, the stores from 1.25, and the Agent Framework diff --git a/docs/guide/cli-and-mcp.md b/docs/guide/cli-and-mcp.md index dc61afe6..54ebfe38 100644 --- a/docs/guide/cli-and-mcp.md +++ b/docs/guide/cli-and-mcp.md @@ -17,7 +17,9 @@ aexp --version aexp init # in the project root: writes .aexp/config.json and the .gitignore entry ``` -It needs the .NET 10 runtime. The SQLite file is created (owner-only) on first use. +It needs the .NET 10 runtime. The SQLite file is created (owner-only) on first use. This path is proven end to end in CI +on Linux, macOS and Windows: every run installs the packed tool, runs `init` and `trust`, finalizes a failed and a +verified run, and checks that `recall`, `lessons` and `benefit` report the lesson (`eng/prove-adoption-path.sh`). ## Configure diff --git a/eng/prove-adoption-path.sh b/eng/prove-adoption-path.sh new file mode 100755 index 00000000..e890a563 --- /dev/null +++ b/eng/prove-adoption-path.sh @@ -0,0 +1,197 @@ +#!/usr/bin/env bash +# The adoption path, proven end to end (story 29.2). What the README promises a stranger -- install aexp, run init and +# trust, and get a verified lesson back -- is done here as that stranger would do it, from the packed tool package and +# nothing else: install the .nupkg CI packed for this commit as a global tool, then on a scratch project run +# `aexp init`, `aexp trust`, a run the configured verifier fails and a run it passes, finalize both, and assert that the +# next `aexp recall` prints the lesson, `aexp lessons` lists it and `aexp benefit` reports the holdout's estimate. Every +# assertion is on what the tool printed or wrote; nothing here reaches into the library. +# +# eng/prove-adoption-path.sh # the directory `dotnet pack --output ` filled +# +# CI runs it on Linux, macOS and Windows from the pack job's artifact, so every OS proves the same bytes (ci.yml, +# adoption-path). Locally: dotnet pack --configuration Release --output /tmp/aexp-pack -p:AgentExperienceReleaseBuild=true +# and point the script at that directory. The package is installed from that directory only, never from nuget.org, at +# the version scripts/release-version.sh derives, so the published package of the same version cannot stand in. The +# script refuses to run while AgentExperience.Cli is already installed as a global tool (it would uninstall yours); +# otherwise it uninstalls the tool and deletes the scratch directory when it exits, pass or fail. The trust approval is +# written to a config directory inside the scratch directory, never to yours. The verifier is the one the CLI tests use: +# it passes once the file `fixed` exists in the project (sh on Unix, cmd on Windows). +# +# Exit code 0 when every check passed, 1 at the first check that did not, 2 when the proof could not start. +set -euo pipefail + +packages="${1:-}" +if [ -z "$packages" ] || [ ! -d "$packages" ]; then + echo "usage: eng/prove-adoption-path.sh (a directory holding AgentExperience.Cli..nupkg)" >&2 + exit 2 +fi +packages="$(cd "$packages" && pwd)" +root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +version="$("$root/scripts/release-version.sh" dotnet)" + +# Git Bash sets HOME on Windows, and `dotnet tool install --global` puts the shim under ~/.dotnet/tools everywhere. +export PATH="$HOME/.dotnet/tools:$PATH" + +case "$(uname -s)" in + MINGW*|MSYS*|CYGWIN*) + windows=1 + verifier='["cmd","/c","if exist fixed (exit 0) else (exit 1)"]' + verifier_text='if exist fixed (exit 0) else (exit 1)' + ;; + *) + windows=0 + verifier='["sh","-c","test -f fixed"]' + verifier_text='test -f fixed' + ;; +esac + +nupkg="$packages/AgentExperience.Cli.$version.nupkg" +if [ ! -f "$nupkg" ]; then + echo "FAILED: no AgentExperience.Cli.$version.nupkg in $packages; it holds:" >&2 + ls -la "$packages" >&2 + exit 1 +fi + +if dotnet tool list --global | awk 'NR > 2 { print tolower($1) }' | grep -qx 'agentexperience.cli'; then + echo "AgentExperience.Cli is already installed as a global tool; this proof installs and uninstalls it, so it will not run over yours." >&2 + echo "Uninstall it first (dotnet tool uninstall --global AgentExperience.Cli), or run the proof on another machine." >&2 + exit 2 +fi + +pass() { echo "ok: $*"; } +fail() { echo "FAILED: $*" >&2; exit 1; } + +# The checks: each names what it looked at, and a miss prints the whole output before failing. +expect_contains() { + local what="$1" needle="$2" text="$3" + case "$text" in + *"$needle"*) pass "$what" ;; + *) printf '%s\n' "$text" >&2; fail "$what: the output does not contain '$needle'" ;; + esac +} +expect_not_contains() { + local what="$1" needle="$2" text="$3" + case "$text" in + *"$needle"*) printf '%s\n' "$text" >&2; fail "$what: the output contains '$needle'" ;; + *) pass "$what" ;; + esac +} +expect_starts_with() { + local what="$1" prefix="$2" text="$3" + case "$text" in + "$prefix"*) pass "$what" ;; + *) printf '%s\n' "$text" >&2; fail "$what: the output does not start with '$prefix'" ;; + esac +} +expect_empty() { + local what="$1" text="$2" + if [ -z "$text" ]; then pass "$what"; else printf '%s\n' "$text" >&2; fail "$what: expected no output"; fi +} +expect_equal() { + local what="$1" expected="$2" text="$3" + if [ "$text" = "$expected" ]; then pass "$what"; else fail "$what: expected '$expected', got '$text'"; fi +} +expect_file() { + local what="$1" path="$2" + if [ -f "$path" ]; then pass "$what"; else fail "$what: no file at $path"; fi +} + +scratch="$(mktemp -d)" +installed=0 +cleanup() { + local status=$? + # Out of the scratch directory first: Windows will not delete the directory a process is in. + cd "$root" 2>/dev/null || cd / + if [ "$installed" -eq 1 ]; then + dotnet tool uninstall --global AgentExperience.Cli >/dev/null \ + || echo "WARNING: could not uninstall AgentExperience.Cli; run: dotnet tool uninstall --global AgentExperience.Cli" >&2 + fi + rm -rf "$scratch" + if [ "$status" -eq 0 ]; then + echo "Adoption path: PASSED (AgentExperience.Cli $version from $packages)" + else + echo "Adoption path: FAILED (exit $status)" >&2 + fi +} +trap cleanup EXIT + +# The trust approval lands in the user's config directory: this run's own, inside the scratch directory, so the proof +# leaves nothing behind there either (XDG_CONFIG_HOME on Unix; APPDATA, as a Windows path, on Windows). +mkdir -p "$scratch/config" +if [ "$windows" -eq 1 ]; then + export APPDATA; APPDATA="$(cygpath -w "$scratch/config")" +else + export XDG_CONFIG_HOME="$scratch/config" +fi + +# Runs aexp, prints its stdout (CR stripped: on Windows the tool writes CRLF), and fails the proof when it exits +# non-zero, with what it wrote to stderr. +aexp_ok() { + local out status=0 + out="$(aexp "$@" 2>"$scratch/stderr" | tr -d '\r')" || status=$? + if [ "$status" -ne 0 ]; then + cat "$scratch/stderr" >&2 + fail "aexp $* exited $status" + fi + printf '%s' "$out" +} + +echo "Installing AgentExperience.Cli $version from $packages (and nowhere else)" +dotnet tool install --global AgentExperience.Cli --version "$version" --source "$packages" +installed=1 +command -v aexp >/dev/null || { echo "PATH=$PATH" >&2; ls -la "$HOME/.dotnet/tools" >&2 || true; fail "aexp is not on PATH after the install"; } +expect_equal "aexp --version prints $version" "$version" "$(aexp_ok --version)" + +project="$scratch/project" +mkdir -p "$project/.git" +cd "$project" + +init="$(aexp_ok init --verify "$verifier")" +expect_file "aexp init wrote .aexp/config.json" "$project/.aexp/config.json" +expect_contains "aexp init added .aexp/ to .gitignore" ".aexp/" "$(tr -d '\r' < "$project/.gitignore")" +expect_contains "aexp init names aexp trust as the next step" "aexp trust" "$init" + +trust="$(aexp_ok trust)" +expect_contains "aexp trust approved the config" "Trusted:" "$trust" +expect_contains "aexp trust showed the verifier" "$verifier_text" "$trust" + +task="Fix the failing build in the payments service" + +# Run A: the verifier fails (no `fixed` yet), so there is no lesson to give. +run_a="$(aexp_ok run start --task "$task")" +aexp_ok run call --run "$run_a" --tool run_tests --error "exit 2: build failed" >/dev/null +aexp_ok run end --run "$run_a" >/dev/null +expect_contains "run A: finalize reports the verifier's failure" "verifier exit 1 (fail)" "$(aexp_ok finalize --run "$run_a")" +expect_empty "run A: recall has nothing to give" "$(aexp_ok recall --task "$task")" + +# Run B: the same failure, then the fix; the verifier passes once `fixed` exists, and the run becomes a validated lesson. +run_b="$(aexp_ok run start --task "$task")" +aexp_ok run call --run "$run_b" --tool run_tests --args '{"filter":"payments"}' \ + --error "Process exited with code 2: error CS0246: The type 'Ledger' could not be found" >/dev/null +aexp_ok run call --run "$run_b" --tool add_reference --args '{"package":"Ledger"}' --result "added Ledger" >/dev/null +aexp_ok run call --run "$run_b" --tool run_tests --args '{"filter":"payments"}' --result "all 42 tests passed" >/dev/null +aexp_ok run end --run "$run_b" >/dev/null +echo fixed > "$project/fixed" +expect_contains "run B: finalize reports the verified run as Validated" "verifier exit 0 (pass); finalization Validated" "$(aexp_ok finalize --run "$run_b")" + +# The next recall for the task is handed the lesson, in the library's own block (no --run: no holdout arm is drawn). +block="$(aexp_ok recall --task "$task")" +expect_starts_with "recall opens the Historical Reference block" "=== BEGIN HISTORICAL REFERENCE" "$block" +expect_contains "recall names the call that worked" "add_reference [returned]" "$block" +expect_contains "recall closes the block" "=== END HISTORICAL REFERENCE ===" "$block" + +expect_contains "aexp lessons lists the validated lesson" "Validated" "$(aexp_ok lessons)" + +# Run C asks for its block as a run, so the holdout draws its arm (given or withheld, at the configured 20%), and is +# verified; benefit then has one run in one arm to report. Which arm is random, so no per-arm count is asserted. +run_c="$(aexp_ok run start --task "$task")" +aexp_ok recall --run "$run_c" >/dev/null +aexp_ok run call --run "$run_c" --tool run_tests --result "all passed" >/dev/null +aexp_ok run end --run "$run_c" >/dev/null +aexp_ok finalize --run "$run_c" >/dev/null + +benefit="$(aexp_ok benefit)" +expect_contains "aexp benefit reports a verdict" "Verdict:" "$benefit" +expect_not_contains "aexp benefit reads a store that tracks the holdout" "does not track the holdout" "$benefit" +aexp_ok benefit --markdown results.md >/dev/null +expect_file "aexp benefit --markdown wrote the results page" "$project/results.md" diff --git a/tests/AgentExperience.Release.Tests/Workflows/WorkflowTests.cs b/tests/AgentExperience.Release.Tests/Workflows/WorkflowTests.cs index 2dcd677a..d70dad09 100644 --- a/tests/AgentExperience.Release.Tests/Workflows/WorkflowTests.cs +++ b/tests/AgentExperience.Release.Tests/Workflows/WorkflowTests.cs @@ -248,6 +248,83 @@ public void The_CI_floating_dependency_leg_exists_and_gates_the_build_outside_pu Assert.True(File.Exists(Path.Combine(RepositoryRoot.Path, "eng", "probe-floating-dependencies.sh"))); } + /// + /// Story 29.2: the adoption path is proven from the packed artifact on Linux, macOS and Windows, and the job gates + /// (it is a required status check on main). The proof is a script anyone can run, so it has to be there and + /// executable; the job runs it under bash on every runner, from the pack job's artifact and nothing else, with a + /// timeout and without a softener. + /// + [Fact] + public void The_CI_adoption_path_job_proves_the_packed_tool_on_every_OS_and_gates() + { + var ci = File.ReadAllText(Path.Combine(RepositoryRoot.Path, ".github", "workflows", "ci.yml")).ReplaceLineEndings("\n"); + + var job = Job(ci, "adoption-path"); + + Assert.Contains("\n needs: pack\n", job, StringComparison.Ordinal); + Assert.Contains("os: [ubuntu-24.04, macos-15, windows-2025]", job, StringComparison.Ordinal); + Assert.Contains("runs-on: ${{ matrix.os }}", job, StringComparison.Ordinal); + Assert.Contains("name: adoption-path (${{ matrix.os }})", job, StringComparison.Ordinal); + Assert.Matches(new Regex(@"\n timeout-minutes: [0-9]+\n", RegexOptions.CultureInvariant), job); + Assert.DoesNotContain("continue-on-error", job, StringComparison.Ordinal); + Assert.DoesNotContain("permissions:", job, StringComparison.Ordinal); + + // The same bytes the pack job built and verified, on every OS: the artifact, never a fresh pack or nuget.org. + Assert.Matches( + new Regex(@"uses: actions/download-artifact@v4\n\s+with:\n\s+name: packages\n\s+path: artifacts/packages\n", RegexOptions.CultureInvariant), + job); + Assert.DoesNotContain("dotnet pack", job, StringComparison.Ordinal); + Assert.DoesNotContain("nuget.org", job, StringComparison.Ordinal); + Assert.Matches( + new Regex(@"\n shell: bash\n run: eng/prove-adoption-path.sh artifacts/packages\n", RegexOptions.CultureInvariant), + job); + + // The SDK global.json pins, and nothing beside it (the runtime the tool needs comes with it). + Assert.Contains("global-json-file: global.json", job, StringComparison.Ordinal); + Assert.DoesNotContain("dotnet-version:", job, StringComparison.Ordinal); + Assert.Contains("\"rollForward\": \"disable\"", job, StringComparison.Ordinal); + Assert.Contains("test \"$actual\" = \"$pinned\"", job, StringComparison.Ordinal); + + // The script is checked in, executable, and installs from the given directory only. + var script = Path.Combine(RepositoryRoot.Path, "eng", "prove-adoption-path.sh"); + Assert.True(File.Exists(script), "eng/prove-adoption-path.sh is missing."); + Assert.True(IsExecutable(script), "eng/prove-adoption-path.sh is not executable (mode 100755)."); + var text = File.ReadAllText(script).ReplaceLineEndings("\n"); + Assert.StartsWith("#!/usr/bin/env bash\n", text, StringComparison.Ordinal); + Assert.Contains("set -euo pipefail", text, StringComparison.Ordinal); + Assert.Contains("dotnet tool install --global AgentExperience.Cli --version \"$version\" --source \"$packages\"", text, StringComparison.Ordinal); + Assert.Contains("dotnet tool uninstall --global AgentExperience.Cli", text, StringComparison.Ordinal); + Assert.Contains("scripts/release-version.sh\" dotnet", text, StringComparison.Ordinal); + } + + /// + /// Whether the file is executable: its Unix mode on Unix; on Windows, where the file system has no such bit, the + /// mode git holds for it in the index (100755). + /// + private static bool IsExecutable(string path) + { + if (!OperatingSystem.IsWindows()) + { + return (File.GetUnixFileMode(path) & UnixFileMode.UserExecute) != 0; + } + + var start = new System.Diagnostics.ProcessStartInfo("git") + { + WorkingDirectory = RepositoryRoot.Path, + RedirectStandardOutput = true, + UseShellExecute = false, + }; + foreach (var arg in new[] { "ls-files", "-s", "--", Path.GetRelativePath(RepositoryRoot.Path, path).Replace('\\', '/') }) + { + start.ArgumentList.Add(arg); + } + + using var git = System.Diagnostics.Process.Start(start)!; + var line = git.StandardOutput.ReadToEnd(); + git.WaitForExit(); + return git.ExitCode == 0 && line.StartsWith("100755 ", StringComparison.Ordinal); + } + // ---- release.yml: the one workflow that publishes, and only like this ---- private static string ReadWorkflow(string name) => From 0e583d9e13a884eb30063007486c586131727627 Mon Sep 17 00:00:00 2001 From: fabbrik <22822543+fabbrik@users.noreply.github.com> Date: Sat, 14 Nov 2026 16:44:01 -0600 Subject: [PATCH 2/2] ci: the adoption proof keeps every aexp exit status, and tolerates CRLF and a locked scratch dir Review round 1: an aexp_ok output used inline as an argument lost its exit status (a crashing recall passed the emptiness check vacuously), so every output is assigned first; the Windows SDK assert strips the CR; the cleanup trap warns instead of aborting when the scratch dir cannot be removed; the already-installed check no longer depends on a pipeline grep -q may close early; the tools dir honours DOTNET_CLI_HOME. Co-Authored-By: Claude Fable 5.1 --- .github/workflows/ci.yml | 5 +++-- eng/prove-adoption-path.sh | 33 ++++++++++++++++++++++----------- 2 files changed, 25 insertions(+), 13 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0e3e3f6a..140f1cab 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -199,12 +199,13 @@ jobs: with: global-json-file: global.json - # Belt and braces after the pin above: a different SDK fails here instead of silently winning. + # Belt and braces after the pin above: a different SDK fails here instead of silently winning. The CR is + # stripped because on Windows `dotnet --version` prints CRLF, which `$(...)` keeps. - name: Assert the exact pinned SDK shell: bash run: | pinned="$(sed -nE 's/.*"version": *"([^"]+)".*/\1/p' global.json)" - actual="$(dotnet --version)" + actual="$(dotnet --version | tr -d '\r')" test "$actual" = "$pinned" || { echo "::error::SDK $actual is in use, but global.json pins $pinned"; exit 1; } echo "SDK $actual, as pinned" diff --git a/eng/prove-adoption-path.sh b/eng/prove-adoption-path.sh index e890a563..c55bf467 100755 --- a/eng/prove-adoption-path.sh +++ b/eng/prove-adoption-path.sh @@ -29,8 +29,9 @@ packages="$(cd "$packages" && pwd)" root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" version="$("$root/scripts/release-version.sh" dotnet)" -# Git Bash sets HOME on Windows, and `dotnet tool install --global` puts the shim under ~/.dotnet/tools everywhere. -export PATH="$HOME/.dotnet/tools:$PATH" +# Git Bash sets HOME on Windows, and `dotnet tool install --global` puts the shim under ~/.dotnet/tools everywhere +# (under DOTNET_CLI_HOME instead of HOME when that is set). +export PATH="${DOTNET_CLI_HOME:-$HOME}/.dotnet/tools:$PATH" case "$(uname -s)" in MINGW*|MSYS*|CYGWIN*) @@ -52,7 +53,9 @@ if [ ! -f "$nupkg" ]; then exit 1 fi -if dotnet tool list --global | awk 'NR > 2 { print tolower($1) }' | grep -qx 'agentexperience.cli'; then +# Captured first: under pipefail, `grep -q` exiting early would turn awk's SIGPIPE into a failed pipeline. +global_tools="$(dotnet tool list --global)" +if printf '%s\n' "$global_tools" | awk 'NR > 2 { print tolower($1) }' | grep -qx 'agentexperience.cli'; then echo "AgentExperience.Cli is already installed as a global tool; this proof installs and uninstalls it, so it will not run over yours." >&2 echo "Uninstall it first (dotnet tool uninstall --global AgentExperience.Cli), or run the proof on another machine." >&2 exit 2 @@ -106,7 +109,7 @@ cleanup() { dotnet tool uninstall --global AgentExperience.Cli >/dev/null \ || echo "WARNING: could not uninstall AgentExperience.Cli; run: dotnet tool uninstall --global AgentExperience.Cli" >&2 fi - rm -rf "$scratch" + rm -rf "$scratch" || echo "WARNING: could not remove $scratch" >&2 if [ "$status" -eq 0 ]; then echo "Adoption path: PASSED (AgentExperience.Cli $version from $packages)" else @@ -125,7 +128,9 @@ else fi # Runs aexp, prints its stdout (CR stripped: on Windows the tool writes CRLF), and fails the proof when it exits -# non-zero, with what it wrote to stderr. +# non-zero, with what it wrote to stderr. Its output is always assigned to a variable first (`out="$(aexp_ok ...)"`, +# whose failure `set -e` catches) and never written inline as an `expect_*` argument, where `fail` would only end the +# substitution's subshell and a crashed command would pass as empty output. aexp_ok() { local out status=0 out="$(aexp "$@" 2>"$scratch/stderr" | tr -d '\r')" || status=$? @@ -140,7 +145,8 @@ echo "Installing AgentExperience.Cli $version from $packages (and nowhere else)" dotnet tool install --global AgentExperience.Cli --version "$version" --source "$packages" installed=1 command -v aexp >/dev/null || { echo "PATH=$PATH" >&2; ls -la "$HOME/.dotnet/tools" >&2 || true; fail "aexp is not on PATH after the install"; } -expect_equal "aexp --version prints $version" "$version" "$(aexp_ok --version)" +printed="$(aexp_ok --version)" +expect_equal "aexp --version prints $version" "$version" "$printed" project="$scratch/project" mkdir -p "$project/.git" @@ -148,7 +154,8 @@ cd "$project" init="$(aexp_ok init --verify "$verifier")" expect_file "aexp init wrote .aexp/config.json" "$project/.aexp/config.json" -expect_contains "aexp init added .aexp/ to .gitignore" ".aexp/" "$(tr -d '\r' < "$project/.gitignore")" +gitignore="$(tr -d '\r' < "$project/.gitignore")" +expect_contains "aexp init added .aexp/ to .gitignore" ".aexp/" "$gitignore" expect_contains "aexp init names aexp trust as the next step" "aexp trust" "$init" trust="$(aexp_ok trust)" @@ -161,8 +168,10 @@ task="Fix the failing build in the payments service" run_a="$(aexp_ok run start --task "$task")" aexp_ok run call --run "$run_a" --tool run_tests --error "exit 2: build failed" >/dev/null aexp_ok run end --run "$run_a" >/dev/null -expect_contains "run A: finalize reports the verifier's failure" "verifier exit 1 (fail)" "$(aexp_ok finalize --run "$run_a")" -expect_empty "run A: recall has nothing to give" "$(aexp_ok recall --task "$task")" +finalized_a="$(aexp_ok finalize --run "$run_a")" +expect_contains "run A: finalize reports the verifier's failure" "verifier exit 1 (fail)" "$finalized_a" +recalled_a="$(aexp_ok recall --task "$task")" +expect_empty "run A: recall has nothing to give" "$recalled_a" # Run B: the same failure, then the fix; the verifier passes once `fixed` exists, and the run becomes a validated lesson. run_b="$(aexp_ok run start --task "$task")" @@ -172,7 +181,8 @@ aexp_ok run call --run "$run_b" --tool add_reference --args '{"package":"Ledger" aexp_ok run call --run "$run_b" --tool run_tests --args '{"filter":"payments"}' --result "all 42 tests passed" >/dev/null aexp_ok run end --run "$run_b" >/dev/null echo fixed > "$project/fixed" -expect_contains "run B: finalize reports the verified run as Validated" "verifier exit 0 (pass); finalization Validated" "$(aexp_ok finalize --run "$run_b")" +finalized_b="$(aexp_ok finalize --run "$run_b")" +expect_contains "run B: finalize reports the verified run as Validated" "verifier exit 0 (pass); finalization Validated" "$finalized_b" # The next recall for the task is handed the lesson, in the library's own block (no --run: no holdout arm is drawn). block="$(aexp_ok recall --task "$task")" @@ -180,7 +190,8 @@ expect_starts_with "recall opens the Historical Reference block" "=== BEGIN HIST expect_contains "recall names the call that worked" "add_reference [returned]" "$block" expect_contains "recall closes the block" "=== END HISTORICAL REFERENCE ===" "$block" -expect_contains "aexp lessons lists the validated lesson" "Validated" "$(aexp_ok lessons)" +lessons="$(aexp_ok lessons)" +expect_contains "aexp lessons lists the validated lesson" "Validated" "$lessons" # Run C asks for its block as a run, so the holdout draws its arm (given or withheld, at the configured 20%), and is # verified; benefit then has one run in one arm to report. Which arm is random, so no per-arm count is asserted.