diff --git a/docs/guide.md b/docs/guide.md index 399380d8..4114144f 100644 --- a/docs/guide.md +++ b/docs/guide.md @@ -4227,8 +4227,9 @@ qualify. The script uses a qualifying commit immediately even if the tip's CI is pending or failed. If none qualifies, it waits for a run to succeed, still considering older pending commits when `main` moves. The Experiment's Candidate is that release base. The tag names exactly that content -plus the version bump, so merges to `main` during a release do not change what -is measured or shipped. +plus the version bump. A repair or external head update can change that content; +the script validates it and refreshes the requested Experiment and release notes +before publishing. From a clone of this repo, signed in to `gh` and the User config's default Harness, run the release script with the new version: @@ -4236,7 +4237,26 @@ From a clone of this repo, signed in to `gh` and the User config's default Harne scripts/release.sh 0.4.0 ``` -It works from `origin/main` in a temporary worktree, so your checked-out branch and uncommitted changes don't matter and aren't touched. It opens a pull request from `release-` that bumps the version in `Cargo.toml` and `Cargo.lock` and nothing else. Its body is a summary of the release, then the version diff. The User config’s default Harness, with its configured Model and Effort (Claude Code when no default is set), writes the summary from the version diff and the title, number and body of each pull request merged between the last release tag and the release base, following the prompt in [`scripts/release-summary.md`](../scripts/release-summary.md). The script uses `cargo run` in this checkout to share thirdshift’s config parser and Harness adapters, and runs the summary in a temporary directory without Factory skills. Because pull request titles and bodies are contributor-editable, no Harness gets its session permission bypass, and each runs with as few tools as its CLI allows. Claude runs with no tools. Codex runs read-only and sandboxed, with its shell, MCP, web, app, plugin, browser and hook tools off. Grok gets only its to-do tool. OpenCode refuses every tool call. Muse has no shell, writes or web, and one model step. agy denies every tool that needs a permission, though it can still read files. Kimi Code runs an agent whose tool list is empty, with the release input as its instructions in a file beside the prompt. A summary that tries a refused tool usually comes back empty, so the script falls back as below. If the config is invalid, the chosen Harness fails, or it returns no summary, the script warns and uses GitHub's generated notes as the summary instead, with a note saying so. It writes the summary before it pushes anything. With `--review` (`scripts/release.sh --review 0.4.0`), it then prints the summary and asks `[y]es / [e]dit / [n]o`: `y` carries on, `e` opens the summary in `$EDITOR` and carries on with what you save, and `n` stops with no branch, pull request or tag pushed. Without `--review`, only a failed or inconclusive Experiment prompts. It waits for the pull request's checks, tags the bump commit at the head of `release-` as `v` and pushes the tag. Pushing the tag starts the release workflow (below). The script waits for that workflow on the bump commit, printing the run's URL so you can watch it. Only after the workflow succeeds does it merge the bump PR into `main` with a merge commit, preserving the tagged commit in main's history even if other content has arrived. If that merge fails, the published release stands: it names the open PR for the Day shift to merge and still exits `0`. CI's run on `main` for the merge isn't waited for. The script prints the GitHub Release's URL last. It prints a line on stderr for each step, stamped with the local time as `HH:MM:SS` as thirdshift's own progress lines are, so a long wait can be told from a stalled one. If the PR checks fail, it stops before tagging and leaves the pull request open and untagged. If the release workflow fails, it exits `1`, naming the failed jobs and run URL, leaving the tag in place and the PR open and unmerged. Recovery happens when you run the script again with the same version. Neither wait has an overall timeout. +It works from `origin/main` in a temporary worktree, so your checked-out branch and uncommitted changes don't matter and aren't touched. It opens a pull request from `release-` that bumps the version in `Cargo.toml` and `Cargo.lock` and nothing else. Its body is a summary of the release, then the version diff. The User config’s default Harness, with its configured Model and Effort (Claude Code when no default is set), writes the summary from the version diff and the title, number and body of each pull request merged between the last release tag and the release base, following the prompt in [`scripts/release-summary.md`](../scripts/release-summary.md). The script uses `cargo run` in this checkout to share thirdshift’s config parser and Harness adapters, and runs the summary in a temporary directory without Factory skills. Because pull request titles and bodies are contributor-editable, no Harness gets its session permission bypass, and each runs with as few tools as its CLI allows. Claude runs with no tools. Codex runs read-only and sandboxed, with its shell, MCP, web, app, plugin, browser and hook tools off. Grok gets only its to-do tool. OpenCode refuses every tool call. Muse has no shell, writes or web, and one model step. agy denies every tool that needs a permission, though it can still read files. Kimi Code runs an agent whose tool list is empty, with the release input as its instructions in a file beside the prompt. A summary that tries a refused tool usually comes back empty, so the script falls back as below. If the config is invalid, the chosen Harness fails, or it returns no summary, the script warns and uses GitHub's generated notes as the summary instead, with a note saying so. It writes the summary before it pushes anything. With `--review` (`scripts/release.sh --review 0.4.0`), it then prints the summary and asks `[y]es / [e]dit / [n]o`: `y` carries on, `e` opens the summary in `$EDITOR` and carries on with what you save, and `n` stops with no branch, pull request or tag pushed. Without `--review`, only a failed or inconclusive Experiment prompts. It waits for the pull request's checks, tags the bump commit at the head of `release-` as `v` and pushes the tag. Pushing the tag starts the release workflow (below). The script waits for that workflow on the bump commit, printing the run's URL so you can watch it. Only after the workflow succeeds does it merge the bump PR into `main` with a merge commit, preserving the tagged commit in main's history even if other content has arrived. If that merge fails, the published release stands: it names the open PR for the Day shift to merge and still exits `0`. CI's run on `main` for the merge isn't waited for. The script prints the GitHub Release's URL last. It prints a line on stderr for each step, stamped with the local time as `HH:MM:SS` as thirdshift's own progress lines are, so a long wait can be told from a stalled one. A Merge conflict or completed failing PR checks can start the bounded Fix run described below. If the release workflow fails, it exits `1`, naming the failed jobs and run URL, leaving the tag in place and the PR open and unmerged. Recovery happens when you run the script again with the same version. Running CI and publication have no overall timeout. + +While waiting for PR CI, the script observes that PR's current head and mergeability. +A Merge conflict, including one with no workflows, or completed failing CI starts +at most one targeted `thirdshift fix --no-merge --no-base-fix` per release +invocation, using this checkout's runner and User config. Those flags override +Self-merge and Base fix defaults: the release script still tags, waits for +publication, then merges. Healthy, queued and running CI start no Fix run. +Missing workflows without a conflict have a five-minute startup grace period +(`RELEASE_WORKFLOW_STARTUP_SECONDS=300`, adjustable for testing), then stop with +the PR, head and recovery command. This is not a deadline on CI runtime. + +After a Fix run or external push, the script fetches and validates the current +PR identity, open state, readiness, origin, branches, head and exact version-bump +tree against main. Extra source changes or a wrong version require human review +before tagging. A changed content base reruns the Experiment when requested and +regenerates the summary and version diff, honoring `--review`. More than five +head updates, a failed or interrupted Fix, or another conflict or CI failure +after the automatic Fix stops with recovery instructions and no new tag pushed. +Ctrl-C and rerunning the same version use the usual resume behavior. Before it pushes anything, it refuses a release that can't be cut and says why: a version that isn't plain `X.Y.Z` (no leading `v`), a version that isn't higher than the one in `Cargo.toml` on `origin/main`, or a `v` tag that already exists locally or on origin. A resumed PR must belong to origin and match its branch tip while open. Its tree (or a pushed branch without a PR) must be exactly the version bump in `Cargo.toml` and `Cargo.lock` from its merge base; extra content is refused even when already tagged. diff --git a/scripts/release.sh b/scripts/release.sh index 1e7782cb..38a0cf9c 100755 --- a/scripts/release.sh +++ b/scripts/release.sh @@ -68,6 +68,13 @@ # version is no longer higher than main. An invalid pin is selected again. # Main can gain content after selection: the tagged tree is still exactly the # Candidate plus the version bump, and merging later preserves it in history. +# While waiting for PR CI, a Merge conflict or completed failing CI starts at +# most one targeted Fix run, with --no-merge and --no-base-fix. After a Fix or +# external push, the current PR/head must still be an exact version bump. A +# changed content base refreshes the requested Experiment, summary and diff. +# More than five head changes, an unsuccessful Fix, or another blockage stops +# before tagging with recovery instructions. Workflows have 300 seconds to +# appear (RELEASE_WORKFLOW_STARTUP_SECONDS); queued/running CI has no deadline. # # Prints a progress line on stderr for each step, stamped with the local time # as HH:MM:SS, with the release workflow run's URL once it waits for that run, @@ -75,13 +82,13 @@ # Exits 0 once the release workflow run for the tag has succeeded: the GitHub # Release is published with its final body and the crate is on crates.io. It # exits 1 with nothing more pushed if it refuses or the review says no, exits -# 1 leaving the PR open if the PR's checks fail, and exits 1 leaving the tag in -# place and the PR open if the release workflow run fails, naming its failed +# 1 leaving the PR open if automatic CI recovery fails, and exits 1 leaving +# the tag in place and the PR open if the release workflow run fails, naming its failed # jobs and run URL. Recovery is only attempted on a subsequent invocation. # A failed merge after publication leaves the release standing, names # the PR for the Day shift and still exits 0. -# It waits with no overall timeout, so Ctrl-C and running it again -# with the same version carries on waiting. +# Running CI and publication wait with no overall timeout, so Ctrl-C and +# running it again with the same version carries on waiting. set -euo pipefail # The prompt the summary agent gets ahead of the release's input. @@ -90,10 +97,14 @@ summary_manifest=$(dirname "$summary_prompt")/../Cargo.toml # Seconds between reads of main's CI, the PR's runs and the release workflow. poll_seconds=${RELEASE_POLL_SECONDS:-15} +# Only workflow startup is timed out, never queued/running CI. +workflow_startup_seconds=${RELEASE_WORKFLOW_STARTUP_SECONDS:-300} main() { review= with_experiment= + fix_attempted= + head_updates=0 while :; do case ${1-} in --review) review=1 ;; @@ -117,6 +128,7 @@ main() { # first, including macOS's older Bash, which does not have this option. shopt -s inherit_errexit + release_origin=$(git remote get-url origin) fetch_main find_earlier_run case $pr_state in @@ -136,7 +148,7 @@ main() { validate_bump_pr recut_untagged_bump if [ -z "$recut" ]; then - if [ -z "$tagged" ]; then wait_for_experiment "$(git rev-parse "$head^1")"; fi + if [ -z "$tagged" ]; then wait_for_experiment "$(release_content_base)"; fi write_summary "no PR was opened" fi fi @@ -145,11 +157,11 @@ main() { OPEN) progress "resuming: $url is open" recut_untagged_bump - if [ -z "$tagged" ] && [ -z "$recut" ]; then wait_for_experiment "$(git rev-parse "$head^1")"; fi + if [ -z "$tagged" ] && [ -z "$recut" ]; then wait_for_experiment "$(release_content_base)"; fi ;; MERGED) progress "resuming: $url was merged as ${merge:0:7}" - if [ -z "$tagged" ]; then wait_for_experiment "$(git rev-parse "$head^1")"; fi + if [ -z "$tagged" ]; then wait_for_experiment "$(release_content_base)"; fi ;; *) refuse "$url was closed without merging, so there is nothing to resume; reopen it or delete $branch to start over" @@ -178,7 +190,7 @@ recut_untagged_bump() { recut= if [ -n "$tagged" ]; then return; fi previous=$(last_release_tag) - if [ -z "$previous" ] || git merge-base --is-ancestor "$previous" "$head^1"; then return; fi + if [ -z "$previous" ] || git merge-base --is-ancestor "$previous" "$(release_content_base)"; then return; fi base=$(check_can_release "$version" "$tag") progress "re-cutting untagged $tag from main at ${base:0:7}" commit_bump "$base" @@ -202,13 +214,13 @@ recover_failed_release() { run=$(release_run "$tagged") read -r id run_url status conclusion attempt <<<"$run" if [ "$status" != completed ] || [ "$conclusion" = success ]; then return; fi - base=$(newest_green_main "$(git rev-parse "$head^1")") + base=$(newest_green_main "$(release_content_base)") if [ -n "$base" ] || [ "$head" != "$tagged" ]; then if [ -n "$base" ]; then progress "re-cutting unpublished $tag from main at ${base:0:7}" commit_bump "$base" else - wait_for_experiment "$(git rev-parse "$head^1")" + wait_for_experiment "$(release_content_base)" fi write_summary "the release branch and tag were not moved" refuse_if_published @@ -304,21 +316,25 @@ find_earlier_run() { # bump_version result from its merge base. A merged PR uses main just before # its merge, since its head is already an ancestor of main now. validate_bump_pr() { - local base target expected candidate=${url:-$branch on origin} + local base expected candidate=${url:-$branch on origin} if ! git fetch --quiet origin "$head"; then refuse "$candidate head could not be fetched from origin" fi - target=origin/main - if [ "$pr_state" = MERGED ]; then - target=$merge^1 - fi - base=$(git merge-base "$head" "$target") + base=$(release_content_base) expected=$(bump_tree "$base") if [ "$expected" != "$(git rev-parse "$head^{tree}")" ]; then - refuse "$candidate changes more than the version bump to $version in Cargo.toml and Cargo.lock" + refuse "$candidate changes more than the version bump to $version in Cargo.toml and Cargo.lock; human review is required before tagging" fi } +# A repaired bump can be a merge commit whose first parent is the old bump. +# Its content is the merge base with main, not that first parent. +release_content_base() { + local target=origin/main + if [ "$pr_state" = MERGED ]; then target=$merge^1; fi + git merge-base "$head" "$target" +} + # bump_tree # The tree commit_bump would make, without committing or touching the caller's # checkout. The subshell owns the temporary worktree and its cleanup trap. @@ -338,7 +354,7 @@ bump_tree() ( # left behind by an interrupted recovery. With no earlier run, # check_can_release refuses an existing tag in its turn. find_existing_tag() { - local old_base base run id run_url status conclusion attempt + local old_base= parent base run id run_url status conclusion attempt tagged= if [ -z "$pr_state" ] && [ -z "$head" ]; then return @@ -355,13 +371,19 @@ find_existing_tag() { # tag. Accept only two exact bumps, a strictly newer content base, and an # unpublished failed workflow on the old tag; unrelated tags still refuse. git fetch --quiet origin "$tagged" - old_base=$(git rev-parse "$tagged^1") - base=$(git rev-parse "$head^1") + base=$(release_content_base) + # Fix can leave a merge with the old bump first and main second. Accept + # only a parent on the new content's history whose exact bump is the tag. + for parent in $(git show -s --format=%P "$tagged"); do + if [ "$parent" != "$base" ] && git merge-base --is-ancestor "$parent" "$base" && + [ "$(bump_tree "$parent")" = "$(git rev-parse "$tagged^{tree}")" ]; then + old_base=$parent + break + fi + done run=$(release_run "$tagged") read -r id run_url status conclusion attempt <<<"$run" - if [ "$pr_state" = OPEN ] && [ "$old_base" != "$base" ] && - git merge-base --is-ancestor "$old_base" "$base" && - [ "$(bump_tree "$old_base")" = "$(git rev-parse "$tagged^{tree}")" ] && + if [ "$pr_state" = OPEN ] && [ -n "$old_base" ] && [ "$status" = completed ] && [ "$conclusion" != success ] && release_is_absent; then progress "resuming re-cut of unpublished $tag on ${head:0:7}" @@ -484,6 +506,7 @@ experiment_status() { write_summary() { local base base=$(git merge-base "$head" origin/main) + summary_base=$base progress "writing the summary with the configured Harness" diff=$(version_diff "$base" "$head") summary=$(release_summary "$base" "$tag" "$diff") @@ -511,16 +534,122 @@ edit_pr() ( ) # wait_for_pr_checks -# Leaves the PR open and untagged and exits 1 if its checks failed. +# Observe this PR throughout CI. One targeted Fix run shares Delivery's Repair +# budget; a second blockage or continued head churn returns to the operator. wait_for_pr_checks() { - local failed tag_state="no tag was pushed" + local runs failed content_base next_base refresh_notes= startup=$SECONDS + local notes_ref=refs/thirdshift/release-notes/$version + [[ "$workflow_startup_seconds" =~ ^[0-9]+$ ]] || refuse "RELEASE_WORKFLOW_STARTUP_SECONDS must be a nonnegative integer" + observe_release_pr + content_base=$observed_notes_base + if [ "$content_base" = - ]; then + # Older PR bodies lack the marker. Remember their starting content + # before Fix can push and then fail, so rerunning still detects it. + content_base=$(git rev-parse --quiet --verify "$notes_ref^{commit}" || release_content_base) + # Without a marker the notes may predate this script/checkout. The + # current head alone cannot establish what their summary describes. + refresh_notes=1 + fi + git update-ref "$notes_ref" "$content_base" progress "waiting for CI on $url" - failed=$(wait_for_workflow_runs "$head") - if [ -n "$failed" ]; then - if [ -n "$tagged" ]; then tag_state="the existing tag is unchanged"; fi - progress "CI failed on $url ($failed), so the PR is left open and unmerged; $tag_state" - exit 1 + while :; do + observe_release_pr + if [ "$observed_head" != "$head" ] || [ "$(release_content_base)" != "$content_base" ]; then + if [ "$observed_head" != "$head" ]; then + head_updates=$((head_updates + 1)) + [ "$head_updates" -le 5 ] || release_blocked "the PR head changed more than five times" + fi + head=$observed_head + fetch_main + validate_bump_pr + next_base=$(git merge-base "$head" origin/main) + if [ "$next_base" != "$content_base" ]; then + wait_for_experiment "$next_base" + refresh_notes=1 + fi + content_base=$next_base + startup=$SECONDS + progress "adopted $url at ${head:0:7}" + fi + if [ "$observed_mergeable" = CONFLICTING ]; then + fix_release_pr "Merge conflict" + continue + fi + runs=$(workflow_runs "$head") + # A workflow read can race an external push or Fix completion. + observe_release_pr + if [ "$observed_head" != "$head" ] || [ "$observed_mergeable" = CONFLICTING ]; then continue; fi + if [ -n "$runs" ] && ! grep -qv '^completed ' <<<"$runs"; then + failed=$(failed_workflows <<<"$runs") + if [ -n "$failed" ]; then + fix_release_pr "CI failed ($failed)" + continue + fi + if [ "$observed_mergeable" = MERGEABLE ]; then + fetch_main + validate_bump_pr + # A newly merged release can invalidate ancestry during CI. + local previous + previous=$(last_release_tag) + if [ -n "$previous" ] && ! git merge-base --is-ancestor "$previous" "$content_base"; then + release_blocked "the content base predates the latest release $previous" + fi + if [ -n "$refresh_notes" ]; then + write_summary "no new tag was pushed" + # A Harness can take time; never edit a changed PR or + # publish these notes on a newer head. + observe_release_pr + if [ "$observed_head" != "$head" ]; then continue; fi + edit_pr + git update-ref "$notes_ref" "$content_base" + refresh_notes= + continue + fi + observe_release_pr + if [ "$observed_head" = "$head" ] && [ "$observed_mergeable" = MERGEABLE ]; then return; fi + continue + fi + elif [ -z "$runs" ] && [ "$((SECONDS - startup))" -ge "$workflow_startup_seconds" ]; then + release_blocked "no workflows appeared within ${workflow_startup_seconds}s; inspect GitHub Actions" + fi + sleep "$poll_seconds" + done +} + +# Re-read the captured PR, never search for a replacement. Refuse identity, +# state, origin or branch changes before adopting a head or publishing a tag. +observe_release_pr() { + local observation origin_head + [ "$(git remote get-url origin)" = "$release_origin" ] || release_blocked "origin changed" + if ! observation=$(gh pr view "$url" --json url,state,headRefName,baseRefName,headRefOid,isCrossRepository,isDraft,mergeable,body | + python3 -c ' +import json, re, sys +pr = json.load(sys.stdin) +url, branch = sys.argv[1:] +if (pr["url"] != url or pr["state"] != "OPEN" or pr["headRefName"] != branch + or pr["baseRefName"] != "main" or pr["isCrossRepository"] or pr["isDraft"]): + sys.exit("release PR identity/state/branch changed or PR is draft") +notes = re.search(r"^$", pr["body"], re.M) +print(pr["headRefOid"], pr["mergeable"], notes[1] if notes else "-") +' "$url" "$branch"); then + release_blocked "could not validate the release PR identity/state/branch" fi + read -r observed_head observed_mergeable observed_notes_base <<<"$observation" + origin_head=$(git ls-remote origin "refs/heads/$branch" | cut -f 1) + [ -n "$origin_head" ] && [ "$observed_head" = "$origin_head" ] || release_blocked "PR head does not match refs/heads/$branch on origin" +} + +fix_release_pr() { + [ -z "$fix_attempted" ] || release_blocked "$1 after the automatic Fix run" + fix_attempted=1 + progress "$1 on $url; starting one targeted Fix run" + if ! cargo run --quiet --manifest-path "$summary_manifest" -- fix "$url" --no-merge --no-base-fix; then + release_blocked "$1; the targeted Fix run failed or was interrupted" + fi +} + +release_blocked() { + refuse "$url at $head: $1; the PR is left open for human review and no new tag was pushed. Recover with thirdshift fix $url --no-merge --no-base-fix, then scripts/release.sh${with_experiment:+ --with-experiment} $version" } # merge_bump_pr @@ -721,6 +850,8 @@ pr_body() { $2 + + EOF } @@ -821,24 +952,19 @@ summary_input() { done } -# wait_for_workflow_runs -# Waits until has workflow runs and all of them have completed, then -# prints the names of those that did not succeed, comma-separated, or nothing. +# workflow_runs +# One observation; the PR wait owns startup grace, head refresh and recovery. # A workflow run completes only once all its jobs have, including jobs that # start after others finish, which a commit's check runs would not show yet. -wait_for_workflow_runs() { - local runs - while :; do - runs=$(gh api "repos/{owner}/{repo}/actions/runs?head_sha=$1&per_page=100" \ - --jq '.workflow_runs[] | "\(.status) \(.conclusion) \(.name)"') - if [ -n "$runs" ] && ! grep -qv '^completed ' <<<"$runs"; then - break - fi - sleep "$poll_seconds" - done +workflow_runs() { + gh api "repos/{owner}/{repo}/actions/runs?head_sha=$1&per_page=100" \ + --jq '.workflow_runs[] | "\(.status) \(.conclusion) \(.name)"' +} + +failed_workflows() { awk '$2 != "success" && $2 != "skipped" && $2 != "neutral" { sub(/^[^ ]+ [^ ]+ /, ""); printf "%s%s", sep, $0; sep = ", " - }' <<<"$runs" + }' } # Only when run, so the tests can source the script and call its functions. diff --git a/site/architecture/index.html b/site/architecture/index.html index bb5164f4..a24ae129 100644 --- a/site/architecture/index.html +++ b/site/architecture/index.html @@ -281,6 +281,7 @@

Architecture

The Experiment owns recheck history beside its immutable Grader Scores. thirdshift experiments <id> --json exposes pending borderline and decisive Scores, naming each decisive flip and its resulting provisional Verdict. thirdshift recheck <id> <trial> pass|fail --reason '…' takes the Experiment's exclusive lock and atomically saves the history, recalculated Verdict and pending list. thirdshift recheck <id> --pending lists the pending Scores and their fingerprint; thirdshift recheck <id> --batch <fingerprint> pass|fail --reason '…' checks that list under the same lock and records every listed Score in one atomic save. A stale fingerprint changes nothing. Rechecks stay in the CLI. They also update the latest invocation's Command outcome and Inbox group, preserving its completion time. The Verdict waits until the list is empty; flipping either pass/fail or a wrong-finding flag can make a Score decisive. Waiting rechecks place that invocation in Needs you; a confirmed pass moves it to Done.

scripts/release.sh runs on Linux. It refuses other hosts before Git or GitHub work. On macOS, use an Ubuntu 24.04 Linux VM such as Lima; on Windows, use WSL2.

thirdshift experiment --release <commit> compares the content commit with the last release tag on touched suites. The release script skips Experiments by default. Add --with-experiment to scripts/release.sh on each invocation to run one: it waits for rechecks and asks permission for a fail or inconclusive Verdict before pushing. Reruns with the flag reuse the Experiment, including before opening a pushed branch's PR. It selects the newest main commit descending from the last release tag whose latest push CI passed, tags that content plus the version bump, waits for publication, then merges the bump PR. A saved Candidate is selected again if a new release tag on main invalidates its ancestry; the requested version must still be higher than main. An untagged branch or open bump PR predating that tag is re-cut from a green main descendant, with a new summary and version diff, honoring --review before force-pushing the branch and updating the open PR body. Tagged or merged releases keep their resume behavior. After a failed unpublished release, rerunning re-cuts from newer green main content or automatically reruns failed jobs; published releases are never moved. If the publication check cannot be completed, the script refuses and moves nothing. Recovery retains the prior release boundary for the summary and Experiment. Main can move during the release; a failed merge leaves the release standing and the PR for the Day shift. No touched suite or a last release without the Trial flag carries on with a note. Releasing.

+

While waiting for release PR checks, a Merge conflict or completed failing CI starts at most one targeted thirdshift fix <PR URL> --no-merge --no-base-fix. Missing workflows have a five-minute startup grace period; queued and running CI keep waiting. A repaired or externally updated head must still be an exact version bump. A changed content base refreshes the requested Experiment, summary and version diff. A failed Fix, another blockage or more than five head changes stops before tagging with recovery instructions. The release script still tags, waits for publication, then merges.

diff --git a/site/docs/index.html b/site/docs/index.html index f5b2fd01..d179b9d1 100644 --- a/site/docs/index.html +++ b/site/docs/index.html @@ -3917,12 +3917,30 @@ if the tip's CI is pending or failed. If none qualifies, it waits for a run to succeed, still considering older pending commits when main moves. The Experiment's Candidate is that release base. The tag names exactly that content -plus the version bump, so merges to main during a release do not change what -is measured or shipped.

+plus the version bump. A repair or external head update can change that content; +the script validates it and refreshes the requested Experiment and release notes +before publishing.

From a clone of this repo, signed in to gh and the User config's default Harness, run the release script with the new version:

scripts/release.sh 0.4.0
-

It works from origin/main in a temporary worktree, so your checked-out branch and uncommitted changes don't matter and aren't touched. It opens a pull request from release-<version> that bumps the version in Cargo.toml and Cargo.lock and nothing else. Its body is a summary of the release, then the version diff. The User config’s default Harness, with its configured Model and Effort (Claude Code when no default is set), writes the summary from the version diff and the title, number and body of each pull request merged between the last release tag and the release base, following the prompt in scripts/release-summary.md. The script uses cargo run in this checkout to share thirdshift’s config parser and Harness adapters, and runs the summary in a temporary directory without Factory skills. Because pull request titles and bodies are contributor-editable, no Harness gets its session permission bypass, and each runs with as few tools as its CLI allows. Claude runs with no tools. Codex runs read-only and sandboxed, with its shell, MCP, web, app, plugin, browser and hook tools off. Grok gets only its to-do tool. OpenCode refuses every tool call. Muse has no shell, writes or web, and one model step. agy denies every tool that needs a permission, though it can still read files. Kimi Code runs an agent whose tool list is empty, with the release input as its instructions in a file beside the prompt. A summary that tries a refused tool usually comes back empty, so the script falls back as below. If the config is invalid, the chosen Harness fails, or it returns no summary, the script warns and uses GitHub's generated notes as the summary instead, with a note saying so. It writes the summary before it pushes anything. With --review (scripts/release.sh --review 0.4.0), it then prints the summary and asks [y]es / [e]dit / [n]o: y carries on, e opens the summary in $EDITOR and carries on with what you save, and n stops with no branch, pull request or tag pushed. Without --review, only a failed or inconclusive Experiment prompts. It waits for the pull request's checks, tags the bump commit at the head of release-<version> as v<version> and pushes the tag. Pushing the tag starts the release workflow (below). The script waits for that workflow on the bump commit, printing the run's URL so you can watch it. Only after the workflow succeeds does it merge the bump PR into main with a merge commit, preserving the tagged commit in main's history even if other content has arrived. If that merge fails, the published release stands: it names the open PR for the Day shift to merge and still exits 0. CI's run on main for the merge isn't waited for. The script prints the GitHub Release's URL last. It prints a line on stderr for each step, stamped with the local time as HH:MM:SS as thirdshift's own progress lines are, so a long wait can be told from a stalled one. If the PR checks fail, it stops before tagging and leaves the pull request open and untagged. If the release workflow fails, it exits 1, naming the failed jobs and run URL, leaving the tag in place and the PR open and unmerged. Recovery happens when you run the script again with the same version. Neither wait has an overall timeout.

+

It works from origin/main in a temporary worktree, so your checked-out branch and uncommitted changes don't matter and aren't touched. It opens a pull request from release-<version> that bumps the version in Cargo.toml and Cargo.lock and nothing else. Its body is a summary of the release, then the version diff. The User config’s default Harness, with its configured Model and Effort (Claude Code when no default is set), writes the summary from the version diff and the title, number and body of each pull request merged between the last release tag and the release base, following the prompt in scripts/release-summary.md. The script uses cargo run in this checkout to share thirdshift’s config parser and Harness adapters, and runs the summary in a temporary directory without Factory skills. Because pull request titles and bodies are contributor-editable, no Harness gets its session permission bypass, and each runs with as few tools as its CLI allows. Claude runs with no tools. Codex runs read-only and sandboxed, with its shell, MCP, web, app, plugin, browser and hook tools off. Grok gets only its to-do tool. OpenCode refuses every tool call. Muse has no shell, writes or web, and one model step. agy denies every tool that needs a permission, though it can still read files. Kimi Code runs an agent whose tool list is empty, with the release input as its instructions in a file beside the prompt. A summary that tries a refused tool usually comes back empty, so the script falls back as below. If the config is invalid, the chosen Harness fails, or it returns no summary, the script warns and uses GitHub's generated notes as the summary instead, with a note saying so. It writes the summary before it pushes anything. With --review (scripts/release.sh --review 0.4.0), it then prints the summary and asks [y]es / [e]dit / [n]o: y carries on, e opens the summary in $EDITOR and carries on with what you save, and n stops with no branch, pull request or tag pushed. Without --review, only a failed or inconclusive Experiment prompts. It waits for the pull request's checks, tags the bump commit at the head of release-<version> as v<version> and pushes the tag. Pushing the tag starts the release workflow (below). The script waits for that workflow on the bump commit, printing the run's URL so you can watch it. Only after the workflow succeeds does it merge the bump PR into main with a merge commit, preserving the tagged commit in main's history even if other content has arrived. If that merge fails, the published release stands: it names the open PR for the Day shift to merge and still exits 0. CI's run on main for the merge isn't waited for. The script prints the GitHub Release's URL last. It prints a line on stderr for each step, stamped with the local time as HH:MM:SS as thirdshift's own progress lines are, so a long wait can be told from a stalled one. A Merge conflict or completed failing PR checks can start the bounded Fix run described below. If the release workflow fails, it exits 1, naming the failed jobs and run URL, leaving the tag in place and the PR open and unmerged. Recovery happens when you run the script again with the same version. Running CI and publication have no overall timeout.

+

While waiting for PR CI, the script observes that PR's current head and mergeability. +A Merge conflict, including one with no workflows, or completed failing CI starts +at most one targeted thirdshift fix <PR URL> --no-merge --no-base-fix per release +invocation, using this checkout's runner and User config. Those flags override +Self-merge and Base fix defaults: the release script still tags, waits for +publication, then merges. Healthy, queued and running CI start no Fix run. +Missing workflows without a conflict have a five-minute startup grace period +(RELEASE_WORKFLOW_STARTUP_SECONDS=300, adjustable for testing), then stop with +the PR, head and recovery command. This is not a deadline on CI runtime.

+

After a Fix run or external push, the script fetches and validates the current +PR identity, open state, readiness, origin, branches, head and exact version-bump +tree against main. Extra source changes or a wrong version require human review +before tagging. A changed content base reruns the Experiment when requested and +regenerates the summary and version diff, honoring --review. More than five +head updates, a failed or interrupted Fix, or another conflict or CI failure +after the automatic Fix stops with recovery instructions and no new tag pushed. +Ctrl-C and rerunning the same version use the usual resume behavior.

Before it pushes anything, it refuses a release that can't be cut and says why: a version that isn't plain X.Y.Z (no leading v), a version that isn't higher than the one in Cargo.toml on origin/main, or a v<version> tag that already exists locally or on origin. A resumed PR must belong to origin and match its branch tip while open. Its tree (or a pushed branch without a PR) must be exactly the version bump in Cargo.toml and Cargo.lock from its merge base; extra content is refused even when already tagged.

If it's interrupted, run it again with the same version: it picks up the existing branch or pull request and carries on from the first step not yet done. With --review, it asks about a new summary before opening a pull request for a branch already pushed. It opens a PR for a pushed branch, waits for its checks, tags the bump head, waits for publication, then merges the PR. A PR already merged is still tagged at its bump head. Once the tag is on that head, it says the release is already tagged and resumes the release workflow: it waits for a run still going, or merges the open bump PR immediately when the workflow succeeded or the GitHub Release already exists, then prints the Release URL and exits 0.

An untagged pushed branch or open bump PR is re-cut if its content base does diff --git a/site/how-it-works/index.html b/site/how-it-works/index.html index fe0c8457..b29176e7 100644 --- a/site/how-it-works/index.html +++ b/site/how-it-works/index.html @@ -317,6 +317,7 @@

How thirdshift works

A Verdict stays waiting on rechecks until you have checked every borderline Score and every decisive Score, whose flip would change the Verdict. The pending list gives Trial IDs for thirdshift recheck <id> <trial> pass|fail --reason '…'. The JSON view names each decisive flip and the provisional Verdict it gives. thirdshift recheck <id> --pending lists all pending Scores and a fingerprint; thirdshift recheck <id> --batch <fingerprint> pass|fail --reason '…' records one decision and reason for that reviewed list in one locked save, refusing a changed list. Your latest decision feeds the calculation; the Grader's Score and your timestamped history remain. Each recheck updates the pending list and the latest invocation's Command outcome and Inbox group. A confirmed pass moves from Needs you to Done.

scripts/release.sh runs on Linux. It refuses other hosts before Git or GitHub work. On macOS, use an Ubuntu 24.04 Linux VM such as Lima; on Windows, use WSL2.

thirdshift experiment --release <commit> compares touched suites with the last release tag. The release script skips Experiments by default. Add --with-experiment to scripts/release.sh on each invocation to run one: it waits for rechecks before pushing and asks whether to release on a fail or inconclusive Verdict. Reruns with the flag reuse the Experiment, including before opening a pushed branch’s PR. It selects the newest main commit descending from the last release tag whose latest push CI passed, tags that content plus the version bump, waits for publication, then merges the bump PR. A saved Candidate is selected again if a new release tag on main invalidates its ancestry; the requested version must still be higher than main. An untagged branch or open bump PR predating that tag is re-cut from a green main descendant, with a new summary and version diff, honoring --review before force-pushing the branch and updating the open PR body. Tagged or merged releases keep their resume behavior. After a failed unpublished release, rerunning re-cuts from newer green main content or automatically reruns failed jobs; published releases are never moved. If the publication check cannot be completed, the script refuses and moves nothing. Recovery retains the prior release boundary for the summary and Experiment. Main can move during the release; a failed merge leaves the release standing and the PR for the Day shift. No touched suite or a last release with no Trial flag carries on with a note. Release checks.

+

While waiting for release PR checks, a Merge conflict or completed failing CI starts at most one targeted thirdshift fix <PR URL> --no-merge --no-base-fix. Missing workflows have a five-minute startup grace period; queued and running CI keep waiting. A repaired or externally updated head must still be an exact version bump. A changed content base refreshes the requested Experiment, summary and version diff. A failed Fix, another blockage or more than five head changes stops before tagging with recovery instructions. The release script still tags, waits for publication, then merges.

Architect runs every few minutes and Security runs on a schedule, with fixing allowed, started by your operating system.

Next: inspect every prompt and skill, see a real Ticket run's recorded output, see the architecture in six figures, use the manual for installation, commands and configuration, or watch the explainer videos.

diff --git a/site/index.html b/site/index.html index a1a9cec9..3b905ea7 100644 --- a/site/index.html +++ b/site/index.html @@ -523,6 +523,7 @@

Seven workflows

Each Trial removes its Harness caches when its session ends. After grading and all rechecks finish, the Experiment removes every attempt's repository copy. Paused Experiments keep repositories for resume; Session evidence, reports, packets and Scores remain readable.

scripts/release.sh runs on Linux. It refuses other hosts before Git or GitHub work. On macOS, use an Ubuntu 24.04 Linux VM such as Lima; on Windows, use WSL2.

thirdshift experiment --release <commit> compares touched suites with the last release tag. The release script skips Experiments by default. Add --with-experiment to scripts/release.sh on each invocation to run one: it waits for rechecks before pushing and asks whether to release on a fail or inconclusive Verdict. Reruns with the flag reuse the Experiment, including before opening a pushed branch’s PR. It selects the newest main commit descending from the last release tag whose latest push CI passed, tags that content plus the version bump, waits for publication, then merges the bump PR. A saved Candidate is selected again if a new release tag on main invalidates its ancestry; the requested version must still be higher than main. An untagged branch or open bump PR predating that tag is re-cut from a green main descendant, with a new summary and version diff, honoring --review before force-pushing the branch and updating the open PR body. Tagged or merged releases keep their resume behavior. After a failed unpublished release, rerunning re-cuts from newer green main content or automatically reruns failed jobs; published releases are never moved. If the publication check cannot be completed, the script refuses and moves nothing. Recovery retains the prior release boundary for the summary and Experiment. Main can move during the release; a failed merge leaves the release standing and the PR for the Day shift. No touched suite or a last release with no Trial flag carries on with a note. Release checks.

+

While waiting for release PR checks, a Merge conflict or completed failing CI starts at most one targeted thirdshift fix <PR URL> --no-merge --no-base-fix. Missing workflows have a five-minute startup grace period; queued and running CI keep waiting. A repaired or externally updated head must still be an exact version bump. A changed content base refreshes the requested Experiment, summary and version diff. A failed Fix, another blockage or more than five head changes stops before tagging with recovery instructions. The release script still tags, waits for publication, then merges.

diff --git a/tests/release.rs b/tests/release.rs index 475e796b..300dfc6d 100644 --- a/tests/release.rs +++ b/tests/release.rs @@ -21,6 +21,9 @@ mod recovery; #[path = "release/ancestry.rs"] mod ancestry; +#[path = "release/checks.rs"] +mod checks; + use std::fs; use std::os::unix::fs::PermissionsExt; use std::path::{Path, PathBuf}; @@ -230,7 +233,19 @@ impl Release { let cargo = release.bin().join("cargo"); write_executable( &cargo, - "#!/bin/sh\nwhile [ \"$1\" != -- ]; do shift; done\nshift\nexec \"$THIRDSHIFT_TEST_BINARY\" \"$@\"\n", + r#"#!/bin/sh +while [ "$1" != -- ]; do shift; done +shift +if [ "$1" = fix ]; then + printf '%s\n' "$*" >> "$FAKE_GH_STATE.fix-calls" + if [ -f "$FAKE_GH_STATE.fix" ]; then + exec bash "$FAKE_GH_STATE.fix" "$@" + fi + echo 'No simulated Fix repair configured' >&2 + exit 1 +fi +exec "$THIRDSHIFT_TEST_BINARY" "$@" +"#, ); fs::create_dir(root.join("home")).unwrap(); fs::write( @@ -547,7 +562,17 @@ fi /// Runs the script with `args`, with `stdin` on its standard input and /// `env` added to its environment. fn run_script_with(&self, args: &[&str], stdin: &str, env: &[(&str, &str)]) -> Output { - with_input( + self.run_script_with_bound(args, stdin, env, support::WAIT_BOUND) + } + + fn run_script_with_bound( + &self, + args: &[&str], + stdin: &str, + env: &[(&str, &str)], + bound: std::time::Duration, + ) -> Output { + support::child_lifetime::FixtureChild::spawn( self.command( &manifest_dir().join("scripts/release.sh"), &self.maintainer(), @@ -556,6 +581,7 @@ fi .envs(env.iter().copied()), stdin, ) + .wait(bound) } /// Sources the script in the maintainer's clone, with what `run_script` @@ -2309,7 +2335,14 @@ fn a_rerun_after_the_pr_was_opened_merges_and_tags_without_a_second_pr_or_bump() assert_eq!(release.prs_from("release-0.2.0").len(), 1); let pr = release.pr("release-0.2.0"); assert_eq!(pr["state"], "MERGED"); - assert_eq!(pr["body"], "Summary from the first run"); + // A legacy PR without a notes-base marker gets a verified summary once. + assert!(pr["body"].as_str().unwrap().contains(AGENT_SUMMARY)); + assert!( + pr["body"] + .as_str() + .unwrap() + .contains("release-content-base:") + ); assert_eq!(release.pr_head("release-0.2.0"), head); let merge = release.origin(&["rev-parse", "main"]); assert_eq!(release.origin(&["rev-parse", &format!("{merge}^2")]), head); diff --git a/tests/release/checks.rs b/tests/release/checks.rs new file mode 100644 index 00000000..7ab12c3b --- /dev/null +++ b/tests/release/checks.rs @@ -0,0 +1,512 @@ +use super::*; + +fn set_pr(release: &Release, field: &str, value: Value) { + release.gh(&["fake", "pr", "release-0.2.0", field, &value.to_string()]); +} + +fn fix_runs(release: &Release, script: &str) { + fs::write(release.root().join("gh-state.json.fix"), script).unwrap(); +} + +fn fix_calls(release: &Release) -> Vec { + fs::read_to_string(release.root().join("gh-state.json.fix-calls")) + .unwrap_or_default() + .lines() + .map(str::to_owned) + .collect() +} + +fn run(release: &Release, args: &[&str], env: &[(&str, &str)]) -> Output { + // Ten seconds covers script startup, Git/GitHub reads, fake Fix/summary/ + // Experiment work, publication/merge and output collection. Fixture setup + // happens before this bound; the guardian owns all fake Fix children too. + release.run_script_with_bound(args, "", env, std::time::Duration::from_secs(10)) +} + +/// Prepare the exact reported pattern: a bump based on older main, followed +/// by a competing version change. Keep a real conflict-resolution merge ready +/// for the fake Fix command to push, with the requested version retained. +fn conflicted_bump(release: &Release) -> (String, String) { + let old = release.push_bump("0.2.0"); + release.open_bump_pr("0.2.0"); + let contributor = release.contributor(); + for file in ["Cargo.toml", "Cargo.lock"] { + fs::write(contributor.join(file), with_version(file, "0.1.1")).unwrap(); + } + git( + &contributor, + &["commit", "-q", "-a", "-m", "Competing version"], + ); + git(&contributor, &["push", "-q", "origin", "HEAD:main"]); + git(&contributor, &["checkout", "-q", "release-0.2.0"]); + let conflict = isolated(&mut Command::new("git")) + .current_dir(&contributor) + .args(["merge", "--no-edit", "origin/main"]) + .output() + .unwrap(); + assert_eq!(conflict.status.code(), Some(1)); + assert!(stdout(&conflict).contains("CONFLICT")); + for file in ["Cargo.toml", "Cargo.lock"] { + fs::write(contributor.join(file), with_version(file, "0.2.0")).unwrap(); + } + git(&contributor, &["add", "Cargo.toml", "Cargo.lock"]); + git( + &contributor, + &["commit", "-q", "-m", "Resolve bump conflict"], + ); + let repaired = git(&contributor, &["rev-parse", "HEAD"]); + git( + &contributor, + &["push", "-q", "origin", "HEAD:prepared-repair"], + ); + set_pr(release, "mergeable", json!("CONFLICTING")); + (old, repaired) +} + +fn adopt_script(repaired: &str) -> String { + format!( + r#"set -euo pipefail +git --git-dir="${{FAKE_GH_STATE%/*}}/origin.git" update-ref refs/heads/release-0.2.0 {repaired} +gh fake pr release-0.2.0 mergeable '"MERGEABLE"' +gh fake checks {repaired} '{GREEN}' +"# + ) +} + +#[test] +fn a_conflict_without_workflows_runs_one_targeted_fix_and_releases_the_repaired_head() { + let release = Release::new(); + fs::create_dir_all(release.root().join("home/.thirdshift")).unwrap(); + fs::write( + release.root().join("home/.thirdshift/config.toml"), + "[merge]\nalways = true\n[base]\nfix = true\n", + ) + .unwrap(); + let (old, repaired) = conflicted_bump(&release); + fix_runs(&release, &adopt_script(&repaired)); + + let output = run(&release, &["0.2.0"], &[]); + + assert!(output.status.success(), "{}", stderr(&output)); + assert_eq!(release.origin_tag("v0.2.0"), Some(repaired.clone())); + assert_ne!(repaired, old); + assert_eq!(release.pr("release-0.2.0")["state"], "MERGED"); + assert_eq!( + fix_calls(&release), + [format!( + "fix {} --no-merge --no-base-fix", + release.pr("release-0.2.0")["url"].as_str().unwrap() + )] + ); + assert!( + release + .pr_body("release-0.2.0") + .contains("-version = \"0.1.1\"") + ); +} + +#[test] +fn missing_workflows_expire_with_the_pr_head_and_recovery_command() { + let release = Release::new(); + let head = release.push_bump("0.2.0"); + release.open_bump_pr("0.2.0"); + + let output = run( + &release, + &["0.2.0"], + &[("RELEASE_WORKFLOW_STARTUP_SECONDS", "0")], + ); + + assert_eq!(output.status.code(), Some(1), "{}", stderr(&output)); + for text in [ + "no workflows appeared", + &head, + "pull/4", + "scripts/release.sh 0.2.0", + ] { + assert!(stderr(&output).contains(text), "{}", stderr(&output)); + } + assert!(fix_calls(&release).is_empty()); + assert_eq!(release.origin_tag("v0.2.0"), None); +} + +#[test] +fn pending_and_healthy_ci_never_start_a_fix_even_after_startup_grace() { + for resume in [false, true] { + let release = Release::new(); + if resume { + release.push_bump("0.2.0"); + release.open_bump_pr("0.2.0"); + } + release.ci_reports(GREEN); + + let output = run( + &release, + &["0.2.0"], + &[("RELEASE_WORKFLOW_STARTUP_SECONDS", "0")], + ); + + assert!(output.status.success(), "{}", stderr(&output)); + assert!(fix_calls(&release).is_empty()); + } +} + +#[test] +fn completed_failing_ci_uses_the_same_fix_path() { + let release = Release::new(); + release.ci_reports(RED); + fix_runs( + &release, + &format!( + "gh fake checks \"$(git ls-remote origin refs/heads/release-0.2.0 | cut -f1)\" '{GREEN}'" + ), + ); + + let output = run(&release, &["0.2.0"], &[]); + + assert!(output.status.success(), "{}", stderr(&output)); + assert_eq!(fix_calls(&release).len(), 1); + assert!(stderr(&output).contains("CI failed (test)")); +} + +#[test] +fn failed_interrupted_or_ineffective_fixes_leave_no_tag_and_are_not_retried() { + for script in ["exit 1", "exit 130", "exit 0"] { + let release = Release::new(); + conflicted_bump(&release); + fix_runs(&release, script); + + let output = run(&release, &["0.2.0"], &[]); + + assert_eq!(output.status.code(), Some(1), "{}", stderr(&output)); + assert_eq!(fix_calls(&release).len(), 1); + assert_eq!(release.origin_tag("v0.2.0"), None); + assert!(stderr(&output).contains("thirdshift fix")); + assert!(stderr(&output).contains("scripts/release.sh 0.2.0")); + } +} + +#[test] +fn a_second_ci_failure_after_a_successful_fix_stops_without_another_fix() { + let release = Release::new(); + let (_, repaired) = conflicted_bump(&release); + fix_runs(&release, &adopt_script(&repaired).replace(GREEN, RED)); + + let output = run(&release, &["0.2.0"], &[]); + + assert_eq!(output.status.code(), Some(1), "{}", stderr(&output)); + assert!(stderr(&output).contains("CI failed (test) after the automatic Fix run")); + assert_eq!(fix_calls(&release).len(), 1); + assert_eq!(release.origin_tag("v0.2.0"), None); +} + +#[test] +fn an_external_head_change_during_a_green_read_is_adopted_without_a_fix() { + let release = Release::new(); + let (old, repaired) = conflicted_bump(&release); + set_pr(&release, "mergeable", json!("MERGEABLE")); + release.gh(&[ + "fake", + "on-ci-read", + "1", + &format!( + "{}\ngh fake checks {old} '{GREEN}'", + adopt_script(&repaired) + ), + ]); + + let output = run(&release, &["0.2.0"], &[]); + + assert!(output.status.success(), "{}", stderr(&output)); + assert_eq!(release.origin_tag("v0.2.0"), Some(repaired)); + assert!(fix_calls(&release).is_empty()); +} + +#[test] +fn identity_mutations_after_fix_or_external_updates_refuse_publication() { + for external in [false, true] { + for (field, value) in [ + ("base", json!("elsewhere")), + ("head", json!("elsewhere")), + ("state", json!("CLOSED")), + ("state", json!("MERGED")), + ("url", json!("https://github.com/other/repo/pull/4")), + ("isCrossRepository", json!(true)), + ("isDraft", json!(true)), + ] { + let release = Release::new(); + release.push_bump("0.2.0"); + release.open_bump_pr("0.2.0"); + let mutation = format!("gh fake pr release-0.2.0 {field} '{value}'"); + if external { + release.gh(&["fake", "on-ci-read", "1", &mutation]); + } else { + set_pr(&release, "mergeable", json!("CONFLICTING")); + fix_runs(&release, &mutation); + } + + let output = run(&release, &["0.2.0"], &[]); + + assert_eq!( + output.status.code(), + Some(1), + "{field}: {}", + stderr(&output) + ); + assert_eq!(release.origin_tag("v0.2.0"), None, "{field}"); + assert!( + stderr(&output).contains("identity/state/branch"), + "{field}: {}", + stderr(&output) + ); + assert_eq!(fix_calls(&release).len(), usize::from(!external)); + } + } +} + +#[test] +fn non_bump_changes_after_fix_or_external_updates_refuse_publication() { + for external in [false, true] { + for change in [ + BumpChange::WrongVersion, + BumpChange::AddedFile, + BumpChange::AppendComment("Cargo.lock"), + BumpChange::FileMode, + ] { + let release = Release::new(); + let old = release.push_bump("0.2.0"); + release.open_bump_pr("0.2.0"); + let changed = release.push_change_to_bump("0.2.0", change); + release.origin(&["update-ref", "refs/heads/release-0.2.0", &old]); + if external { + release.gh(&["fake", "on-ci-read", "1", &adopt_script(&changed)]); + } else { + set_pr(&release, "mergeable", json!("CONFLICTING")); + fix_runs(&release, &adopt_script(&changed)); + } + + let output = run(&release, &["0.2.0"], &[]); + + assert_eq!( + output.status.code(), + Some(1), + "{change:?}: {}", + stderr(&output) + ); + assert_eq!(release.origin_tag("v0.2.0"), None); + assert!( + stderr(&output).contains("changes more than the version bump"), + "{change:?}: {}", + stderr(&output) + ); + } + } +} + +#[test] +fn a_resumed_repaired_merge_checks_the_experiment_on_its_content_base() { + let release = Release::new(); + let (_, repaired) = conflicted_bump(&release); + let base = release.origin(&["rev-parse", "main"]); + release.origin(&["update-ref", "refs/heads/release-0.2.0", &repaired]); + set_pr(&release, "mergeable", json!("MERGEABLE")); + release.ci_reports(GREEN); + experiments::cargo_reports(&release, &[experiments::verdict("pass")]); + + let output = run(&release, &["--with-experiment", "0.2.0"], &[]); + + assert!(output.status.success(), "{}", stderr(&output)); + assert_eq!(experiments::calls(&release)[0]["args"][2], base); +} + +#[test] +fn a_repair_refreshes_the_experiment_summary_and_diff_for_its_new_content_base() { + let release = Release::new(); + let (old, repaired) = conflicted_bump(&release); + let base = release.origin(&["rev-parse", "main"]); + let old_base = release.origin(&["rev-parse", &format!("{old}^")]); + experiments::cargo_reports( + &release, + &[experiments::verdict("pass"), experiments::verdict("pass")], + ); + fix_runs(&release, &adopt_script(&repaired)); + + let output = run(&release, &["--with-experiment", "0.2.0"], &[]); + + assert!(output.status.success(), "{}", stderr(&output)); + let calls = experiments::calls(&release); + assert_eq!(calls.len(), 2); + assert_eq!(calls[0]["args"][2], old_base); + assert_eq!(calls[1]["args"][2], base); + assert!( + release + .pr_body("release-0.2.0") + .contains("-version = \"0.1.1\"") + ); + assert_eq!(release.origin_tag("v0.2.0"), Some(repaired)); +} + +#[test] +fn a_new_content_base_cannot_publish_under_an_obsolete_experiment() { + let release = Release::new(); + let (_, repaired) = conflicted_bump(&release); + experiments::cargo_reports( + &release, + &[experiments::verdict("pass"), experiments::verdict("fail")], + ); + fix_runs(&release, &adopt_script(&repaired)); + + let output = run(&release, &["--with-experiment", "0.2.0"], &[]); + + assert_eq!(output.status.code(), Some(1), "{}", stderr(&output)); + assert!(stderr(&output).contains("Experiment fail; no release override given")); + assert_eq!(experiments::calls(&release).len(), 2); + assert_eq!(release.origin_tag("v0.2.0"), None); +} + +#[test] +fn continued_external_head_churn_is_bounded_without_starting_fix_runs() { + let release = Release::new(); + let old = release.push_bump("0.2.0"); + release.open_bump_pr("0.2.0"); + let contributor = release.contributor(); + git(&contributor, &["checkout", "-q", "release-0.2.0"]); + let mut heads = vec![old.clone()]; + for _ in 0..7 { + git( + &contributor, + &["commit", "-q", "--allow-empty", "-m", "Another head"], + ); + heads.push(git(&contributor, &["rev-parse", "HEAD"])); + } + git( + &contributor, + &["push", "-q", "origin", "HEAD:prepared-churn"], + ); + let mut script = String::from("case $FAKE_CI_SHA in\n"); + for pair in heads.windows(2) { + script.push_str(&format!( + "{}) git --git-dir=\"${{FAKE_GH_STATE%/*}}/origin.git\" update-ref refs/heads/release-0.2.0 {} ;;\n", + pair[0], pair[1] + )); + } + script.push_str("esac\n"); + release.gh(&["fake", "on-ci-read", "10", &script]); + + let output = run(&release, &["0.2.0"], &[]); + + assert_eq!(output.status.code(), Some(1), "{}", stderr(&output)); + assert!(stderr(&output).contains("head changed more than five times")); + assert_eq!(release.origin_tag("v0.2.0"), None); + assert!(fix_calls(&release).is_empty()); +} + +#[test] +fn spec_resume_after_interrupted_fix_refreshes_release_notes() { + let release = Release::new(); + let (_, repaired) = conflicted_bump(&release); + set_pr( + &release, + "body", + json!("Old release notes: version diff starts at 0.1.0"), + ); + fix_runs(&release, &(adopt_script(&repaired) + "\nexit 130\n")); + let interrupted = run(&release, &["0.2.0"], &[]); + assert_eq!( + interrupted.status.code(), + Some(1), + "{}", + stderr(&interrupted) + ); + assert_eq!(release.origin_tag("v0.2.0"), None); + assert_eq!( + release.pr_body("release-0.2.0"), + "Old release notes: version diff starts at 0.1.0" + ); + + let resumed = run(&release, &["0.2.0"], &[]); + assert!(resumed.status.success(), "{}", stderr(&resumed)); + assert_eq!(release.origin_tag("v0.2.0"), Some(repaired)); + assert!( + release + .pr_body("release-0.2.0") + .contains("-version = \"0.1.1\""), + "Published repaired content with stale release notes: {}", + release.pr_body("release-0.2.0") + ); +} + +#[test] +fn a_notes_marker_detects_an_external_repair_from_another_checkout() { + let release = Release::new(); + let (old, repaired) = conflicted_bump(&release); + let old_base = release.origin(&["rev-parse", &format!("{old}^")]); + set_pr( + &release, + "body", + json!(format!( + "Old release notes\n" + )), + ); + release.origin(&["update-ref", "refs/heads/release-0.2.0", &repaired]); + set_pr(&release, "mergeable", json!("MERGEABLE")); + release.ci_reports(GREEN); + + let output = run(&release, &["0.2.0"], &[]); + + assert!(output.status.success(), "{}", stderr(&output)); + assert_eq!(release.origin_tag("v0.2.0"), Some(repaired)); + assert!(fix_calls(&release).is_empty()); + assert!( + release + .pr_body("release-0.2.0") + .contains("-version = \"0.1.1\"") + ); +} + +#[test] +fn recovery_after_a_repaired_tag_resumes_between_branch_and_tag_pushes() { + let release = Release::new(); + let (_, repaired) = conflicted_bump(&release); + fix_runs(&release, &adopt_script(&repaired)); + release.release_workflow(failed_release()); + let failed = run(&release, &["0.2.0"], &[]); + assert_release_failed(&release, &failed, &repaired); + release.merge_pr("newer-content", "Recovery content", ""); + release.main_ci("completed", "success"); + release.ci_reports(RED); + fix_runs(&release, "exit 1"); + let stopped = run(&release, &["0.2.0"], &[]); + assert_eq!(stopped.status.code(), Some(1), "{}", stderr(&stopped)); + assert_eq!(release.origin_tag("v0.2.0"), Some(repaired)); + let head = release.pr_head("release-0.2.0"); + release.gh(&["fake", "checks", &head, GREEN]); + release.release_workflow(json!({})); + let resumed = run(&release, &["0.2.0"], &[]); + assert!(resumed.status.success(), "{}", stderr(&resumed)); + assert_eq!(release.origin_tag("v0.2.0"), Some(head)); +} + +#[test] +fn an_unmarked_legacy_pr_repaired_before_resume_refreshes_its_notes() { + let release = Release::new(); + let (_, repaired) = conflicted_bump(&release); + set_pr( + &release, + "body", + json!("Legacy notes: version diff starts at 0.1.0"), + ); + release.origin(&["update-ref", "refs/heads/release-0.2.0", &repaired]); + set_pr(&release, "mergeable", json!("MERGEABLE")); + release.ci_reports(GREEN); + let output = run(&release, &["0.2.0"], &[]); + assert!(output.status.success(), "{}", stderr(&output)); + assert_eq!(release.origin_tag("v0.2.0"), Some(repaired)); + assert!( + release + .pr_body("release-0.2.0") + .contains("-version = \"0.1.1\""), + "Published repaired legacy content with stale release notes: {}", + release.pr_body("release-0.2.0") + ); +} diff --git a/tests/release/experiments.rs b/tests/release/experiments.rs index 116dbb4c..43418917 100644 --- a/tests/release/experiments.rs +++ b/tests/release/experiments.rs @@ -2,7 +2,7 @@ use super::*; const ID: &str = "eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee"; -fn verdict(outcome: &str) -> Value { +pub(super) fn verdict(outcome: &str) -> Value { json!({ "id": ID, "state": "completed", "pending_rechecks": [], "verdict": {"verdict": outcome, "suites": {"code-review": { @@ -12,7 +12,11 @@ fn verdict(outcome: &str) -> Value { }) } -fn cargo_reports(release: &Release, views: &[Value]) { +pub(super) fn cargo_reports(release: &Release, views: &[Value]) { + let driver = release.bin().join("cargo-driver"); + if !driver.exists() { + fs::copy(release.bin().join("cargo"), driver).unwrap(); + } fs::write( release.root().join("experiment-views.json"), json!(views).to_string(), @@ -21,9 +25,10 @@ fn cargo_reports(release: &Release, views: &[Value]) { fs::write(release.bin().join("cargo"), r#"#!/usr/bin/env python3 import json, os, pathlib, subprocess, sys args = sys.argv[sys.argv.index('--') + 1:] -if args == ['--release-summary']: - os.execv(os.environ['THIRDSHIFT_TEST_BINARY'], [os.environ['THIRDSHIFT_TEST_BINARY'], *args]) root = pathlib.Path(os.environ['FAKE_GH_STATE']).parent +if args == ['--release-summary'] or args[0] == 'fix': + driver = str(root / 'bin/cargo-driver') + os.execv(driver, [driver, *sys.argv[1:]]) branch = subprocess.check_output(['git', 'ls-remote', 'origin', 'refs/heads/release-0.2.0'], text=True).strip() with (root / 'experiment-calls.jsonl').open('a') as calls: calls.write(json.dumps(dict(args=args, branch=branch)) + '\n') @@ -38,7 +43,7 @@ print(json.dumps(view)) "#).unwrap(); } -fn calls(release: &Release) -> Vec { +pub(super) fn calls(release: &Release) -> Vec { fs::read_to_string(release.root().join("experiment-calls.jsonl")) .unwrap_or_default() .lines()