From 2c37ce9e17eaf7452078179a57098381d7a96f5d Mon Sep 17 00:00:00 2001 From: yunlishao-vibe <267182001+yunlishao-vibe@users.noreply.github.com> Date: Wed, 23 Sep 2026 18:54:39 -0400 Subject: [PATCH 1/2] ci(#273): install the packed zip before publishing, and the release after update_fixs.{ps1,sh} gain --zip (install a local build zip) and --list (the picker's releases). release.yml installs each platform's fresh zip with them before publish, and the published channel after it, checking the result with check_fixs_install.{ps1,sh}. A .sha256 the release lists is now required; only releases that list none install unverified. --- .github/workflows/release.yml | 102 ++++++++++++++- scripts/check_fixs_install.ps1 | 50 ++++++++ scripts/check_fixs_install.sh | 41 ++++++ scripts/update_fixs.ps1 | 93 ++++++++++---- scripts/update_fixs.sh | 224 ++++++++++++++++++++------------- 5 files changed, 397 insertions(+), 113 deletions(-) create mode 100644 scripts/check_fixs_install.ps1 create mode 100755 scripts/check_fixs_install.sh diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 7b7d2ff2..fbd71821 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -101,12 +101,13 @@ jobs: # here - building validates nothing about it. # pytest is the right gate for Python (#57). # environment.yml copied into the zip only (8_create_zip.ps1:110); - # a real env check is #248, not a C++ compile - # scripts/update_fixs.* consumer-side installer (#272) - not a build - # input, and not packaged (8_create_zip.ps1:119) + # checked by python-env.yml (#248), not a C++ compile # NOTE: ^scripts/ must NOT be ignored wholesale - scripts/dispatch/** IS - # the build system, and generate_version.ps1 stamps the version. - ignore='^doc/|^tests/|^\.github/|^\.vscode/|\.md$|\.py$|^LICENSE$|^\.gitignore$|^\.gitattributes$|^environment\.yml$|^scripts/update_fixs\.(ps1|sh)$' + # the build system, and generate_version.ps1 stamps the version. Nor + # scripts/update_fixs.* (ignored until #273): the build now installs its + # fresh zip with them, and every app on a channel runs them, so a change + # to the installer is exactly the change that must reach that check. + ignore='^doc/|^tests/|^\.github/|^\.vscode/|\.md$|\.py$|^LICENSE$|^\.gitignore$|^\.gitattributes$|^environment\.yml$' relevant=$(echo "$files" | grep -vE "$ignore" || true) if [ -z "$relevant" ]; then echo "No build-relevant changes -> skipping build (required check still passes)." @@ -373,6 +374,26 @@ jobs: Write-Host "VirtualEnvironment.lib absent (expected on the current/main train)." } + # Install the zip just packed exactly as a consumer would, BEFORE anything is + # published (#273). #238 changed the packaging, the installer was not updated + # to match, and it shipped: nothing ran the install path, so the first to + # notice was a user's --update-fixs. The checks after the publish below + # cover the download and the checksum; this one is what keeps a zip that + # cannot install off the channel. + - name: Install the packed zip as a consumer would (update_fixs.ps1 -Zip) + if: needs.changes.outputs.should_build == 'true' + shell: pwsh + env: + # PR builds have no channel; any tag will do, it is only stamped and read back. + TAG: ${{ steps.channel.outputs.tag || 'pr-build' }} + run: | + $zip = Get-ChildItem build -Filter 'fixs-build-*.zip' | Select-Object -First 1 + $root = Join-Path $env:RUNNER_TEMP 'consumer' + & powershell -NoProfile -ExecutionPolicy Bypass -File scripts/update_fixs.ps1 -Root $root -Zip $zip.FullName -Version $env:TAG + if ($LASTEXITCODE -ne 0) { exit 1 } + & powershell -NoProfile -ExecutionPolicy Bypass -File scripts/check_fixs_install.ps1 -Dir (Join-Path $root 'FIXS') -Tag $env:TAG + if ($LASTEXITCODE -ne 0) { exit 1 } + - name: Upload packaged zip as a workflow artifact (inspect on PRs) if: needs.changes.outputs.should_build == 'true' uses: actions/upload-artifact@v4 @@ -505,6 +526,17 @@ jobs: if: needs.changes.outputs.should_build == 'true' run: ./scripts/dispatch/pack_linux_bundle.sh --out-dir dist + # The Linux half of build-publish's pre-publish install (#273), on the same + # focal image the bundle targets. linux-publish needs this job, so a bundle + # that cannot install is never uploaded. + - name: Install the packed zip as a consumer would (update_fixs.sh --zip) + if: needs.changes.outputs.should_build == 'true' + run: | + zip="$(ls dist/fixs-build-*-linux-x86_64.zip)" + root="$RUNNER_TEMP/consumer"; mkdir -p "$root" + bash scripts/update_fixs.sh --root "$root" --zip "$zip" --version pr-build + bash scripts/check_fixs_install.sh "$root/FIXS" pr-build + - name: Upload bundle as a workflow artifact (inspect on PRs) if: needs.changes.outputs.should_build == 'true' uses: actions/upload-artifact@v4 @@ -553,6 +585,66 @@ jobs: # same rolling release is untouched. gh release upload "$CHANNEL_TAG" "$want" "$want.sha256" --repo "$GITHUB_REPOSITORY" --clobber + # =========================================================================== + # After the publish: install from the channel exactly as an app's --update-fixs + # does (#273). The pre-publish installs above use -Zip, so they cannot see the + # download or the checksum; these can. A failure here means the channel is + # already serving a bad release - it cannot un-publish, only make that loud. + # =========================================================================== + install-check-windows: + name: Install the published release (Windows) + needs: build-publish + if: needs.build-publish.outputs.publish == 'true' && (github.event_name == 'push' || (github.event_name == 'workflow_dispatch' && inputs.publish)) + runs-on: windows-2022 + steps: + - uses: actions/checkout@v4 + with: + sparse-checkout: scripts + - name: update_fixs.ps1 -Version + shell: pwsh + env: + TAG: ${{ needs.build-publish.outputs.tag }} + run: | + $root = Join-Path $env:RUNNER_TEMP 'consumer' + & powershell -NoProfile -ExecutionPolicy Bypass -File scripts/update_fixs.ps1 -Root $root -Version $env:TAG -Repo $env:GITHUB_REPOSITORY + if ($LASTEXITCODE -ne 0) { exit 1 } + & powershell -NoProfile -ExecutionPolicy Bypass -File scripts/check_fixs_install.ps1 -Dir (Join-Path $root 'FIXS') -Tag $env:TAG + if ($LASTEXITCODE -ne 0) { exit 1 } + + install-check-linux: + name: Install the published release (Linux) + needs: [build-publish, linux-publish] + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + sparse-checkout: scripts + - name: update_fixs.sh --version + env: + TAG: ${{ needs.build-publish.outputs.tag }} + run: | + root="$RUNNER_TEMP/consumer"; mkdir -p "$root" + bash scripts/update_fixs.sh --root "$root" --version "$TAG" --repo "$GITHUB_REPOSITORY" + bash scripts/check_fixs_install.sh "$root/FIXS" "$TAG" + + # The picker must offer exactly the releases that carry a build zip. The + # script decides that by walking the API's JSON as a key stream without + # jq, so check its answer against jq's reading of the same API. Neither + # list is allowed to be empty or to lack the channel just published. + - name: The picker offers exactly the installable releases + env: + GH_TOKEN: ${{ github.token }} + TAG: ${{ needs.build-publish.outputs.tag }} + run: | + offered="$(bash scripts/update_fixs.sh --list --repo "$GITHUB_REPOSITORY" | sort)" + expected="$(gh api "repos/$GITHUB_REPOSITORY/releases?per_page=30" \ + --jq '.[] | select(.draft | not) | select(any(.assets[]; .name | startswith("fixs-build-") and endswith(".zip"))) | .tag_name' | sort)" + echo "offered:"; echo "$offered" + echo "expected:"; echo "$expected" + [ -n "$offered" ] || { echo "::error::the picker offers nothing"; exit 1; } + echo "$offered" | grep -qx "$TAG" || { echo "::error::the picker does not offer $TAG, just published"; exit 1; } + [ "$offered" = "$expected" ] || { echo "::error::the picker's list differs from the releases that carry a fixs-build-*.zip"; exit 1; } + # Read the Docs serves one docs version per channel under the channel's own # name (stable, vX.Y.Z-beta, vX.Y.Z-alpha), built from the tag published above # (#137). Ask RTD to rebuild it here, so the docs move exactly when the channel diff --git a/scripts/check_fixs_install.ps1 b/scripts/check_fixs_install.ps1 new file mode 100644 index 00000000..958741e8 --- /dev/null +++ b/scripts/check_fixs_install.ps1 @@ -0,0 +1,50 @@ +# ============================================================================ +# Is a complete, runnable FIXS install? (Windows) #273 +# +# Run by release.yml against what update_fixs.ps1 produced - once from the zip it +# just packed (before publishing) and once from the published release (after). +# scripts/check_fixs_install.sh is the Linux counterpart; change them together. +# +# check_fixs_install.ps1 -Dir C:\tmp\t\FIXS -Tag v0.9.1-alpha +# +# Exit 0 = complete. Exit 1 = says what is missing. +# ============================================================================ +param( + [Parameter(Mandatory = $true)] [string]$Dir, + [Parameter(Mandatory = $true)] [string]$Tag +) +$ErrorActionPreference = 'Stop' +$bad = @() + +if (-not (Test-Path (Join-Path $Dir 'TrafficLayer.exe'))) { $bad += 'TrafficLayer.exe is missing' } + +# A load, not a presence check: the vendored bin/ once lacked geos_c.dll for +# months with every probed DLL present (#70). libsumo_verify.ps1 walks the +# import graph the way TrafficLayer does at run time. +. (Join-Path $PSScriptRoot 'dispatch\libsumo_verify.ps1') +try { + Test-LibsumoLoadable -BinDir (Join-Path $Dir 'CommonLib\libsumo\bin') -Context 'The installed native runtime is incomplete.' +} catch { $bad += "$_" } + +# Line 1 is '' for a pinned release and ' ()' for a rolling +# one. run_cosim.py reads it byte-for-byte, where a BOM becomes part of the tag. +$vf = Join-Path $Dir 'FIXS_VERSION.txt' +if (-not (Test-Path $vf)) { + $bad += 'FIXS_VERSION.txt is missing' +} else { + $bytes = [System.IO.File]::ReadAllBytes($vf) + if ($bytes.Length -ge 3 -and $bytes[0] -eq 0xEF -and $bytes[1] -eq 0xBB -and $bytes[2] -eq 0xBF) { + $bad += 'FIXS_VERSION.txt starts with a UTF-8 BOM' + } + $line1 = ([System.IO.File]::ReadAllLines($vf) | Select-Object -First 1) + if ($line1 -ne $Tag -and -not $line1.StartsWith("$Tag (")) { + $bad += "FIXS_VERSION.txt line 1 is '$line1', expected '$Tag'" + } +} + +if ($bad) { + $bad | ForEach-Object { Write-Host "[FAIL] $_" } + exit 1 +} +Write-Host "Install in $Dir is complete ($Tag)." +exit 0 diff --git a/scripts/check_fixs_install.sh b/scripts/check_fixs_install.sh new file mode 100755 index 00000000..d7f63f78 --- /dev/null +++ b/scripts/check_fixs_install.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash +# ============================================================================ +# Is a complete, runnable FIXS install? (Linux) #273 +# +# Run by release.yml against what update_fixs.sh produced - once from the zip it +# just packed (before publishing) and once from the published release (after). +# scripts/check_fixs_install.ps1 is the Windows counterpart; change them together. +# +# check_fixs_install.sh /tmp/t/FIXS v0.9.1-alpha +# +# Exit 0 = complete. Exit 1 = says what is missing. +# ============================================================================ +set -uo pipefail +dir="$1" tag="$2" +bad=0 +fail() { echo "[FAIL] $*"; bad=1; } + +if [[ ! -x "$dir/TrafficLayer" ]]; then + fail "TrafficLayer is missing or not executable" +else + # A load check, not a presence check. TrafficLayer finds libtracicpp.so + # through its $ORIGIN-relative RPATH (CMakeLists.txt), so with no + # LD_LIBRARY_PATH set this is exactly what a consumer's first run resolves. + missing="$(env -u LD_LIBRARY_PATH ldd "$dir/TrafficLayer" | grep 'not found' || true)" + [[ -z "$missing" ]] || fail "TrafficLayer cannot resolve: $(echo "$missing" | awk '{print $1}' | tr '\n' ' ')" +fi +[[ -f "$dir/CommonLib/libsumo/bin/libtracicpp.so" ]] || fail "CommonLib/libsumo/bin/libtracicpp.so is missing" + +# Line 1 is '' for a pinned release and ' ()' for a rolling +# one. run_cosim.py reads it byte-for-byte, where a BOM becomes part of the tag. +vf="$dir/FIXS_VERSION.txt" +if [[ ! -f "$vf" ]]; then + fail "FIXS_VERSION.txt is missing" +else + [[ "$(head -c3 "$vf" | od -An -tx1 | tr -d ' \n')" != "efbbbf" ]] || fail "FIXS_VERSION.txt starts with a UTF-8 BOM" + line1="$(head -n1 "$vf")" + [[ "$line1" == "$tag" || "$line1" == "$tag ("* ]] || fail "FIXS_VERSION.txt line 1 is '$line1', expected '$tag'" +fi + +[[ "$bad" == 0 ]] || exit 1 +echo "Install in $dir is complete ($tag)." diff --git a/scripts/update_fixs.ps1 b/scripts/update_fixs.ps1 index a33cde80..d603b8dc 100644 --- a/scripts/update_fixs.ps1 +++ b/scripts/update_fixs.ps1 @@ -13,6 +13,12 @@ # update_fixs.ps1 -Root C:\path\to\app # newest installable release # update_fixs.ps1 -Root C:\path\to\app -Version v0.9.0 # a specific release # update_fixs.ps1 -Root . -Repo my-fork/FIXS +# update_fixs.ps1 -Root C:\tmp\t -Zip .\fixs-build-v0.9.1-alpha.zip -Version v0.9.1-alpha +# update_fixs.ps1 -List # installable releases, then exit +# +# -Zip installs a build zip already on disk instead of downloading one; release.yml +# uses it to test the bundle it just packed BEFORE publishing it (#273). Everything +# after the download - unpack, native runtime, version marker - runs unchanged. # # Installs into \FIXS. Needs PowerShell 5.1+ and network access. No git, no # GitHub CLI, no auth - FIXS is public. @@ -20,7 +26,7 @@ # Exit codes: 0 = a complete bundle is in place; 1 = nothing usable was installed. # ============================================================================ param( - [Parameter(Mandatory = $true)] [string]$Root, + [string]$Root, [string]$Version, [string]$Repo = 'ORNL-Real-Sim/FIXS', # The ref this copy of the script was downloaded from. Supplied by the app's @@ -29,7 +35,12 @@ param( [string]$SelfRef, # The consuming app's preferred channel, offered as the Enter-default in the # picker. App config, not engine policy - see Select-Release. - [string]$DefaultVersion + [string]$DefaultVersion, + # A build zip on disk to install instead of a release asset. Needs -Version, + # which becomes the stamp in FIXS_VERSION.txt. + [string]$Zip, + # Print the releases the picker would offer, one tag per line, and exit. + [switch]$List ) # 'Stop' is safe here: this script shells out to nothing. The old fetch_fixs.ps1 @@ -44,9 +55,17 @@ $Api = "https://api.github.com/repos/$Repo" $Headers = @{ 'User-Agent' = 'fixs-fetch'; 'Accept' = 'application/vnd.github+json' } $DepsTag = 'fixs-native-deps' # rolling release carrying the packed native runtimes -$Root = [System.IO.Path]::GetFullPath($Root) -$OutputDir = Join-Path $Root 'FIXS' -$VersionFile = Join-Path $OutputDir 'FIXS_VERSION.txt' +if (-not $List -and -not $Root) { Write-Error '-Root is required.'; exit 1 } +if ($Zip) { + if (-not $Version) { Write-Error '-Zip needs -Version: it is what FIXS_VERSION.txt records.'; exit 1 } + if (-not (Test-Path $Zip -PathType Leaf)) { Write-Error "-Zip: no such file: $Zip"; exit 1 } + $Zip = (Resolve-Path $Zip).Path +} +if ($Root) { + $Root = [System.IO.Path]::GetFullPath($Root) + $OutputDir = Join-Path $Root 'FIXS' + $VersionFile = Join-Path $OutputDir 'FIXS_VERSION.txt' +} # --------------------------------------------------------------------------- # A release is INSTALLABLE iff it carries a fixs-build-*.zip. That one rule is @@ -99,27 +118,35 @@ function Get-InstallableReleases { @($rels | Where-Object { -not $_.draft -and (Get-BuildAsset $_) }) } +function Test-ListsSidecar($assets, [string]$name) { + [bool]($assets | Where-Object { $_.name -eq "$name.sha256" }) +} + function Get-Asset { # Download to and verify it against the published .sha256 sidecar. - # A missing/unreachable sidecar warns and proceeds (older assets may predate - # them); a MISMATCH is always fatal - that is a corrupt or tampered download. - param([string]$Url, [string]$Dest) + # -Required says the release LISTS that sidecar, so failing to fetch it is + # fatal (#273): the file is there, and something between us and it is not. + # Only a release that lists none - published before #204 added them - is + # installed unverified, with a warning. A MISMATCH is always fatal. + param([string]$Url, [string]$Dest, [bool]$Required) + $leaf = Split-Path $Url -Leaf Invoke-WebRequest -UseBasicParsing -Uri $Url -OutFile $Dest -Headers $Headers try { # The CDN serves the tiny .sha256 as octet-stream, so .Content comes back # as byte[] (Invoke-WebRequest only string-decodes text content types). $c = (Invoke-WebRequest -UseBasicParsing -Uri "$Url.sha256" -Headers $Headers).Content if ($c -is [byte[]]) { $c = [System.Text.Encoding]::UTF8.GetString($c) } - $expected = ($c -split '\s+')[0].Trim().ToLower() - $actual = (Get-FileHash $Dest -Algorithm SHA256).Hash.ToLower() - if ($expected -ne $actual) { - throw "checksum mismatch for $(Split-Path $Url -Leaf): expected $expected, got $actual" - } - Write-Host " checksum OK ($actual)" } catch { - if ("$_" -match 'checksum mismatch') { throw } - Write-Warning " could not verify .sha256 for $(Split-Path $Url -Leaf): $($_.Exception.Message) - proceeding WITHOUT verification." + if ($Required) { throw "the release lists $leaf.sha256 but it could not be fetched: $($_.Exception.Message)" } + Write-Warning " the release publishes no $leaf.sha256 (it predates them) - proceeding WITHOUT verification." + return + } + $expected = ($c -split '\s+')[0].Trim().ToLower() + $actual = (Get-FileHash $Dest -Algorithm SHA256).Hash.ToLower() + if ($expected -ne $actual) { + throw "checksum mismatch for ${leaf}: expected $expected, got $actual" } + Write-Host " checksum OK ($actual)" } # --------------------------------------------------------------------------- @@ -226,7 +253,18 @@ function Select-Release($releases) { # INVARIANT: -Version given => never prompt, never hand off. That is what bounds # the hand-off below to a single step and makes an infinite bounce impossible. # --------------------------------------------------------------------------- -if ($Version) { +if ($List) { + # What the picker offers, without the prompt, so CI can check the filter + # against the real release list (#273). + Get-InstallableReleases | ForEach-Object { $_.tag_name } + exit 0 +} +if ($Zip) { + # A local zip has no release behind it: nothing to resolve, nothing to verify + # against, no hand-off. Only the fields the rest of the script reads. + $release = [pscustomobject]@{ prerelease = $false; target_commitish = 'local'; published_at = '(local zip)'; assets = @() } + $asset = [pscustomobject]@{ name = (Split-Path $Zip -Leaf) } +} elseif ($Version) { try { $release = Invoke-RestMethod -UseBasicParsing -Uri "$Api/releases/tags/$Version" -Headers $Headers } catch { @@ -286,7 +324,7 @@ if ($Version) { # which commit is inside. Printing tag + commit + published-date together is what # pins down exactly what was installed. $sha7 = if ($release.target_commitish -match '^[0-9a-f]{40}$') { $release.target_commitish.Substring(0, 7) } else { $release.target_commitish } -Write-Host "=== Fetching the FIXS build from $Repo (public, no auth) ===" +Write-Host $(if ($Zip) { "=== Installing a local FIXS build zip ===" } else { "=== Fetching the FIXS build from $Repo (public, no auth) ===" }) Write-Host " release: $Version$(if ($release.prerelease) { ' (rolling prerelease - always re-fetched)' })" Write-Host " commit: $sha7" Write-Host " asset: $($asset.name)" @@ -299,7 +337,7 @@ Write-Host " into: $OutputDir" # 'prerelease' flag marks exactly the rolling channels release.yml publishes, # which is why this reads the flag instead of hardcoding a tag list: the .sh # used to hardcode one, and the two rules agreed only by coincidence. -if ((Test-Path $VersionFile) -and -not $release.prerelease) { +if (-not $Zip -and (Test-Path $VersionFile) -and -not $release.prerelease) { $current = (Get-Content $VersionFile | Select-Object -First 1).Trim() if ($current -eq $Version) { Write-Host "FIXS $Version is already installed (delete $VersionFile to force)." @@ -311,9 +349,15 @@ $TempDir = Join-Path ([System.IO.Path]::GetTempPath()) "fixs-fetch-$(Get-Random) New-Item -ItemType Directory -Path $TempDir -Force | Out-Null try { # --- The build zip ----------------------------------------------------- - $ZipPath = Join-Path $TempDir $asset.name - Write-Host "Downloading $($asset.name) ..." - Get-Asset -Url $asset.browser_download_url -Dest $ZipPath + if ($Zip) { + $ZipPath = $Zip + Write-Host "Installing the local $($asset.name) (no download, no checksum) ..." + } else { + $ZipPath = Join-Path $TempDir $asset.name + Write-Host "Downloading $($asset.name) ..." + Get-Asset -Url $asset.browser_download_url -Dest $ZipPath ` + -Required (Test-ListsSidecar $release.assets $asset.name) + } Write-Host "Extracting to $OutputDir ..." if (Test-Path $OutputDir) { Remove-Item -Path $OutputDir -Recurse -Force } @@ -337,7 +381,8 @@ try { $sumoAsset = Select-NativeAsset $depsRelease.assets 'libsumo' (Get-BundleDepVersion 'SUMO') Write-Host " downloading $($sumoAsset.name) ..." $SumoZip = Join-Path $TempDir $sumoAsset.name - Get-Asset -Url $sumoAsset.browser_download_url -Dest $SumoZip + Get-Asset -Url $sumoAsset.browser_download_url -Dest $SumoZip ` + -Required (Test-ListsSidecar $depsRelease.assets $sumoAsset.name) # Extracting into CommonLib/ reproduces the /bin layout the # executables already search; any future runtime asset lands the same way. Expand-Archive -Path $SumoZip -DestinationPath (Join-Path $OutputDir 'CommonLib') -Force @@ -357,7 +402,7 @@ try { $versionText = @" $stamp Fetched: $(Get-Date -Format 'yyyy-MM-dd HH:mm:ss') -Source: $Repo +Source: $(if ($Zip) { $Zip } else { $Repo }) Commit: $sha7 Zip: $($asset.name) "@ diff --git a/scripts/update_fixs.sh b/scripts/update_fixs.sh index b74e41d1..1a4562a6 100644 --- a/scripts/update_fixs.sh +++ b/scripts/update_fixs.sh @@ -15,6 +15,12 @@ # update_fixs.sh --root /path/to/app # pick interactively # update_fixs.sh --root /path/to/app --version v0.9.0 # a specific release # update_fixs.sh --root . --repo my-fork/FIXS +# update_fixs.sh --root /tmp/t --zip ./fixs-build-v0.9.1-alpha-linux-x86_64.zip --version v0.9.1-alpha +# update_fixs.sh --list # installable releases, then exit +# +# --zip installs a build zip already on disk instead of downloading one; +# release.yml uses it to test the bundle it just packed BEFORE publishing it +# (#273). Everything after the download runs unchanged. # # Installs into /FIXS. Requires bash, curl, unzip. No git, no GitHub CLI, # no auth - FIXS is public. @@ -28,6 +34,8 @@ VERSION="" REPO="ORNL-Real-Sim/FIXS" SELF_REF="" # the ref this copy came from; enables the hand-off below DEFAULT_VERSION="" # the app's preferred channel, offered as the Enter-default +ZIP_FILE="" # a build zip on disk to install instead of a release asset +LIST_ONLY=0 # print the releases the picker would offer, then exit while [[ $# -gt 0 ]]; do case "$1" in @@ -36,10 +44,17 @@ while [[ $# -gt 0 ]]; do --repo) REPO="$2"; shift 2 ;; --self-ref) SELF_REF="$2"; shift 2 ;; --default-version) DEFAULT_VERSION="$2"; shift 2 ;; + --zip) ZIP_FILE="$2"; shift 2 ;; + --list) LIST_ONLY=1; shift ;; *) echo "[ERROR] unknown argument: $1" >&2; exit 1 ;; esac done -[[ -n "$ROOT" ]] || { echo "[ERROR] --root is required." >&2; exit 1; } +[[ -n "$ROOT" || "$LIST_ONLY" == 1 ]] || { echo "[ERROR] --root is required." >&2; exit 1; } +if [[ -n "$ZIP_FILE" ]]; then + [[ -n "$VERSION" ]] || { echo "[ERROR] --zip needs --version: it is what FIXS_VERSION.txt records." >&2; exit 1; } + [[ -f "$ZIP_FILE" ]] || { echo "[ERROR] --zip: no such file: $ZIP_FILE" >&2; exit 1; } + ZIP_FILE="$(cd "$(dirname "$ZIP_FILE")" && pwd)/$(basename "$ZIP_FILE")" +fi # git is deliberately NOT required. It used to be, for a sparse clone of # CommonLib/libsumo/bin; #238 deleted that path from git and moved the runtime to @@ -48,9 +63,11 @@ for tool in curl unzip; do command -v "$tool" >/dev/null 2>&1 || { echo "[ERROR] '$tool' is required but not on PATH." >&2; exit 1; } done -ROOT="$(cd "$ROOT" && pwd)" -OUTPUT_DIR="$ROOT/FIXS" -VERSION_FILE="$OUTPUT_DIR/FIXS_VERSION.txt" +if [[ -n "$ROOT" ]]; then + ROOT="$(cd "$ROOT" && pwd)" + OUTPUT_DIR="$ROOT/FIXS" + VERSION_FILE="$OUTPUT_DIR/FIXS_VERSION.txt" +fi API="https://api.github.com/repos/$REPO" DEPS_TAG="fixs-native-deps" # rolling release carrying the packed native runtimes CURL=(curl -fsSL -H 'User-Agent: fixs-fetch' -H 'Accept: application/vnd.github+json') @@ -158,14 +175,20 @@ sha256_of() { else echo ""; fi } -get_asset() { # get_asset - download and verify against the .sha256 sidecar - local url="$1" dest="$2" expected actual +get_asset() { # get_asset - download and verify against the .sha256 sidecar + # =1 says the release LISTS that sidecar, so failing to fetch or + # check it is fatal (#273). Only a release that lists none - published before + # #204 added them - is installed unverified, with a warning. A MISMATCH is + # always fatal: that is a corrupt or tampered download. + local url="$1" dest="$2" required="$3" expected actual curl -fsSL -H 'User-Agent: fixs-fetch' -o "$dest" "$url" - # A missing/unreachable sidecar warns and proceeds (older assets may predate - # them); a MISMATCH is always fatal - that is a corrupt or tampered download. if expected="$(curl -fsSL "$url.sha256" 2>/dev/null | awk '{print tolower($1); exit}')" && [[ -n "$expected" ]]; then actual="$(sha256_of "$dest")" if [[ -z "$actual" ]]; then + if [[ "$required" == 1 ]]; then + echo "[ERROR] no sha256sum/shasum on PATH to verify $(basename "$url") against its published .sha256." >&2 + exit 1 + fi echo " [warn] no sha256sum/shasum on PATH - proceeding WITHOUT verification." >&2 elif [[ "$expected" != "$actual" ]]; then echo "[ERROR] checksum mismatch for $(basename "$url"): expected $expected, got $actual" >&2 @@ -173,17 +196,31 @@ get_asset() { # get_asset - download and verify against the .sha25 else echo " checksum OK ($actual)" fi + elif [[ "$required" == 1 ]]; then + echo "[ERROR] the release lists $(basename "$url").sha256 but it could not be fetched." >&2 + exit 1 else - echo " [warn] could not fetch $(basename "$url").sha256 - proceeding WITHOUT verification." >&2 + echo " [warn] the release publishes no $(basename "$url").sha256 (it predates them) - proceeding WITHOUT verification." >&2 fi } +lists_sidecar() { # lists_sidecar -> 1 if .sha256 is listed, else 0 + grep -qxF "$1.sha256" "$2" && echo 1 || echo 0 +} + # --------------------------------------------------------------------------- # Resolve the release. # # INVARIANT: --version given => never prompt, never hand off. That is what bounds # the hand-off below to a single step and makes an infinite bounce impossible. # --------------------------------------------------------------------------- +if [[ "$LIST_ONLY" == 1 ]]; then + # What the picker offers, without the prompt, so CI can check the filter + # against the real release list (#273). + installable_releases | cut -f1 + exit 0 +fi + if [[ -z "$VERSION" ]]; then LIST="$(installable_releases)" [[ -n "$LIST" ]] || { echo "[ERROR] No installable FIXS release at $REPO (none carries a 'fixs-build-*.zip')." >&2; exit 1; } @@ -211,85 +248,96 @@ if [[ -z "$VERSION" ]]; then fi fi -RELEASE_JSON="$TMP_DIR/release.json" -"${CURL[@]}" "$API/releases/tags/$VERSION" -o "$RELEASE_JSON" \ - || { echo "[ERROR] Could not find FIXS release '$VERSION' at $REPO." >&2; exit 1; } +if [[ -n "$ZIP_FILE" ]]; then + # A local zip has no release behind it: nothing to resolve, nothing to + # verify against. Only the fields the rest of the script reads. + ASSET_NAME="$(basename "$ZIP_FILE")" + SHA="local"; PUBLISHED_AT="(local zip)"; PRERELEASE="false" +else + RELEASE_JSON="$TMP_DIR/release.json" + "${CURL[@]}" "$API/releases/tags/$VERSION" -o "$RELEASE_JSON" \ + || { echo "[ERROR] Could not find FIXS release '$VERSION' at $REPO." >&2; exit 1; } -# --------------------------------------------------------------------------- -# Pick ONE build bundle, by the same two rules #306 applied to the native-deps -# assets -- because the same trap is one level up here. This was -# 'fixs-build-.*\.zip$ | head -n1', and the moment a release carries both -# -# fixs-build-v0.9.0-alpha-windows-x86_64.zip -# fixs-build-v0.9.0-alpha-linux-x86_64.zip -# -# that pattern matches both and head -n1 picks whichever GitHub lists first -- -# installing Linux binaries on Windows, or the reverse, with no complaint. The -# platform is known ($PLATFORM_TAG); use it. -# -# 1. Prefer the platform-qualified name, then fall back to the legacy -# unqualified one, so releases published before the split still install. -# 2. Never take the first of several. Two candidates means the release is -# ambiguous: say so and stop. -# --------------------------------------------------------------------------- -BUILD_URLS=() -while IFS= read -r _u; do - [[ -n "$_u" ]] && BUILD_URLS+=("$_u") -done < <(grep -o '"browser_download_url":[[:space:]]*"[^"]*"' "$RELEASE_JSON" \ - | sed 's/.*"\(https[^"]*\)".*/\1/' | grep -E 'fixs-build-.*\.zip$') -# Guarded before the loop: expanding an empty array under `set -u` is itself an -# error on bash < 4.4, and that trace would replace the real diagnosis. -[[ "${#BUILD_URLS[@]}" -gt 0 ]] \ - || { echo "[ERROR] Release '$VERSION' carries no 'fixs-build-*.zip' and cannot be installed." >&2; exit 1; } + # --------------------------------------------------------------------------- + # Pick ONE build bundle, by the same two rules #306 applied to the native-deps + # assets -- because the same trap is one level up here. This was + # 'fixs-build-.*\.zip$ | head -n1', and the moment a release carries both + # + # fixs-build-v0.9.0-alpha-windows-x86_64.zip + # fixs-build-v0.9.0-alpha-linux-x86_64.zip + # + # that pattern matches both and head -n1 picks whichever GitHub lists first -- + # installing Linux binaries on Windows, or the reverse, with no complaint. The + # platform is known ($PLATFORM_TAG); use it. + # + # 1. Prefer the platform-qualified name, then fall back to the legacy + # unqualified one, so releases published before the split still install. + # 2. Never take the first of several. Two candidates means the release is + # ambiguous: say so and stop. + # --------------------------------------------------------------------------- + BUILD_URLS=() + while IFS= read -r _u; do + [[ -n "$_u" ]] && BUILD_URLS+=("$_u") + done < <(grep -o '"browser_download_url":[[:space:]]*"[^"]*"' "$RELEASE_JSON" \ + | sed 's/.*"\(https[^"]*\)".*/\1/' | grep -E 'fixs-build-.*\.zip$') + # Guarded before the loop: expanding an empty array under `set -u` is itself an + # error on bash < 4.4, and that trace would replace the real diagnosis. + [[ "${#BUILD_URLS[@]}" -gt 0 ]] \ + || { echo "[ERROR] Release '$VERSION' carries no 'fixs-build-*.zip' and cannot be installed." >&2; exit 1; } -# Pass 1 takes the platform-qualified name. Pass 2 accepts a legacy unqualified -# one, but only after excluding every name that carries SOME platform tag -- -# otherwise 'fixs-build-.*\.zip' would match the other platform's bundle and we -# would be back to picking by luck. -select_build_asset() { # uses BUILD_URLS + PLATFORM_TAG, sets ASSET_URL - local rx u b - local -a hits - ASSET_URL="" - # The legacy fallback is offered to WINDOWS only. Every unqualified bundle - # ever published was built on Windows and holds .exe files, so accepting one - # on Linux would install a tree with no runnable binary in it and report - # success. Windows keeps installing today's releases untouched. - local -a patterns=("^fixs-build-.*-${PLATFORM_TAG}\.zip$") - [[ "$PLATFORM_TAG" == "windows-x86_64" ]] && patterns+=("^fixs-build-.*\.zip$") - for rx in "${patterns[@]}"; do - hits=() - for u in "${BUILD_URLS[@]}"; do - b="${u##*/}" - if [[ "$rx" == "^fixs-build-.*\.zip$" ]]; then - case "$b" in *-windows-x86_64.zip|*-linux-x86_64.zip|*-macos-x86_64.zip) continue ;; esac + # Pass 1 takes the platform-qualified name. Pass 2 accepts a legacy unqualified + # one, but only after excluding every name that carries SOME platform tag -- + # otherwise 'fixs-build-.*\.zip' would match the other platform's bundle and we + # would be back to picking by luck. + select_build_asset() { # uses BUILD_URLS + PLATFORM_TAG, sets ASSET_URL + local rx u b + local -a hits + ASSET_URL="" + # The legacy fallback is offered to WINDOWS only. Every unqualified bundle + # ever published was built on Windows and holds .exe files, so accepting one + # on Linux would install a tree with no runnable binary in it and report + # success. Windows keeps installing today's releases untouched. + local -a patterns=("^fixs-build-.*-${PLATFORM_TAG}\.zip$") + [[ "$PLATFORM_TAG" == "windows-x86_64" ]] && patterns+=("^fixs-build-.*\.zip$") + for rx in "${patterns[@]}"; do + hits=() + for u in "${BUILD_URLS[@]}"; do + b="${u##*/}" + if [[ "$rx" == "^fixs-build-.*\.zip$" ]]; then + case "$b" in *-windows-x86_64.zip|*-linux-x86_64.zip|*-macos-x86_64.zip) continue ;; esac + fi + [[ "$b" =~ $rx ]] && hits+=("$u") + done + if [[ "${#hits[@]}" -eq 1 ]]; then ASSET_URL="${hits[0]}"; return 0; fi + if [[ "${#hits[@]}" -gt 1 ]]; then + echo "[ERROR] Release carries ${#hits[@]} build bundles for $PLATFORM_TAG:" >&2 + printf ' %s\n' "${hits[@]##*/}" >&2 + echo " nothing says which to install. Retire the stale one." >&2 + return 1 fi - [[ "$b" =~ $rx ]] && hits+=("$u") done - if [[ "${#hits[@]}" -eq 1 ]]; then ASSET_URL="${hits[0]}"; return 0; fi - if [[ "${#hits[@]}" -gt 1 ]]; then - echo "[ERROR] Release carries ${#hits[@]} build bundles for $PLATFORM_TAG:" >&2 - printf ' %s\n' "${hits[@]##*/}" >&2 - echo " nothing says which to install. Retire the stale one." >&2 - return 1 - fi - done - echo "[ERROR] no build bundle for $PLATFORM_TAG (looked for" >&2 - echo " fixs-build--$PLATFORM_TAG.zip, then fixs-build-.zip):" >&2 - printf ' %s\n' "${BUILD_URLS[@]##*/}" >&2 - return 1 -} -select_build_asset || exit 1 -ASSET_NAME="$(basename "$ASSET_URL")" -SHA="$(json_field "$RELEASE_JSON" target_commitish)" -PUBLISHED_AT="$(json_field "$RELEASE_JSON" published_at)" -PRERELEASE="$(grep -o '"prerelease":[[:space:]]*[a-z]*' "$RELEASE_JSON" | head -n1 | grep -o 'true\|false')" + echo "[ERROR] no build bundle for $PLATFORM_TAG (looked for" >&2 + echo " fixs-build--$PLATFORM_TAG.zip, then fixs-build-.zip):" >&2 + printf ' %s\n' "${BUILD_URLS[@]##*/}" >&2 + return 1 + } + select_build_asset || exit 1 + ASSET_NAME="$(basename "$ASSET_URL")" + SHA="$(json_field "$RELEASE_JSON" target_commitish)" + PUBLISHED_AT="$(json_field "$RELEASE_JSON" published_at)" + PRERELEASE="$(grep -o '"prerelease":[[:space:]]*[a-z]*' "$RELEASE_JSON" | head -n1 | grep -o 'true\|false')" +fi # State the resolution in full. A rolling tag's asset is named after the nearest # annotated tag plus a git-describe suffix, so 'latest' unpacks a zip called # fixs-build-v0.9.0-alpha-1-g65f2970d.zip - which reads like the alpha was # installed instead of what was asked for. Printing tag + commit + asset together # is what makes the two distinguishable. -echo "=== Fetching the FIXS build from $REPO (public, no auth) ===" +if [[ -n "$ZIP_FILE" ]]; then + echo "=== Installing a local FIXS build zip ===" +else + echo "=== Fetching the FIXS build from $REPO (public, no auth) ===" +fi echo " release: $VERSION$([[ "$PRERELEASE" == "true" ]] && echo ' (rolling prerelease - always re-fetched)')" echo " commit: ${SHA:0:7}" echo " asset: $ASSET_NAME" @@ -301,7 +349,7 @@ echo " into: $OUTPUT_DIR" # current. This reads the 'prerelease' flag rather than hardcoding which tags are # rolling - the hardcoded list this replaces agreed with the flag only by # coincidence, and would have skipped a re-fetch for any newly named channel. -if [[ -f "$VERSION_FILE" && "$PRERELEASE" != "true" ]]; then +if [[ -z "$ZIP_FILE" && -f "$VERSION_FILE" && "$PRERELEASE" != "true" ]]; then current="$(head -n1 "$VERSION_FILE" | tr -d '[:space:]')" if [[ "$current" == "$VERSION" ]]; then echo "FIXS $VERSION is already installed (delete $VERSION_FILE to force)." @@ -310,9 +358,16 @@ if [[ -f "$VERSION_FILE" && "$PRERELEASE" != "true" ]]; then fi # --- The build zip --------------------------------------------------------- -ZIP_PATH="$TMP_DIR/$ASSET_NAME" -echo "Downloading $ASSET_NAME ..." -get_asset "$ASSET_URL" "$ZIP_PATH" +if [[ -n "$ZIP_FILE" ]]; then + ZIP_PATH="$ZIP_FILE" + echo "Installing the local $ASSET_NAME (no download, no checksum) ..." +else + ZIP_PATH="$TMP_DIR/$ASSET_NAME" + grep -o '"browser_download_url":[[:space:]]*"[^"]*"' "$RELEASE_JSON" \ + | sed 's/.*"\(https[^"]*\)"$/\1/' > "$TMP_DIR/release_urls.txt" + echo "Downloading $ASSET_NAME ..." + get_asset "$ASSET_URL" "$ZIP_PATH" "$(lists_sidecar "$ASSET_URL" "$TMP_DIR/release_urls.txt")" +fi echo "Extracting to $OUTPUT_DIR ..." # The Windows-packed zip ships some directories without their EXECUTE bit (e.g. @@ -423,7 +478,8 @@ select_native_asset() { # select_native_asset libsumo "$SUMO_VER" || exit 1 DEP_URL="$SELECTED_URL" echo " downloading ${DEP_URL##*/} ..." -get_asset "$DEP_URL" "$TMP_DIR/libsumo.zip" +printf '%s\n' "${DEP_URLS[@]}" > "$TMP_DIR/deps_urls.txt" +get_asset "$DEP_URL" "$TMP_DIR/libsumo.zip" "$(lists_sidecar "$DEP_URL" "$TMP_DIR/deps_urls.txt")" # Extracting into CommonLib/ reproduces the /bin layout the # executables already search; any future runtime asset lands the same way. set +e @@ -458,7 +514,7 @@ if [[ "$PRERELEASE" == "true" ]]; then STAMP="$VERSION ($PUBLISHED_AT)"; else ST cat > "$VERSION_FILE" < Date: Wed, 23 Sep 2026 18:54:39 -0400 Subject: [PATCH 2/2] ci(#248): build the env from environment.yml and run the unit tests in it Windows runner: the spec's vc14/vs2015 runtimes are win-64 only. --- .github/workflows/python-env.yml | 85 ++++++++++++++++++++++++++++++++ 1 file changed, 85 insertions(+) create mode 100644 .github/workflows/python-env.yml diff --git a/.github/workflows/python-env.yml b/.github/workflows/python-env.yml new file mode 100644 index 00000000..5b87bc7d --- /dev/null +++ b/.github/workflows/python-env.yml @@ -0,0 +1,85 @@ +name: Python env (environment.yml) + +# Builds the conda env from environment.yml ALONE and runs the Python side in it +# (#248). environment.yml is what every consumer builds from - carla_env_setup +# offers to create it, FIXS_Applications derives its env from it - and two broken +# specs shipped with CI green: shapely missing (#236, traffic-light sync silently +# off) and a wheel path the release zip strips (#221, `conda env create` failed on +# every published release). Nothing ever ran the command. +# +# Windows, not Linux: the spec pins vs2015_runtime / vc14_runtime, which +# conda-forge publishes for win-64 only, so it does not resolve on linux-64. +# +# No path filter, on purpose. release.yml skips its build when a PR touches only +# .py / environment.yml / tests/, and still reports green - so a Python-only PR +# ran nothing at all. This job exists for exactly those PRs. + +on: + workflow_dispatch: + pull_request: + branches: [dev, main, 'dev_v*', 'beta_v*'] + push: + branches: [main, 'dev_v*', 'beta_v*'] + +jobs: + env: + name: env from environment.yml + unit tests + runs-on: windows-2022 + defaults: + run: + shell: bash -el {0} + steps: + - uses: actions/checkout@v4 + with: + sparse-checkout: | + Carla + CommonLib + tests/Python + environment.yml + + # Miniforge: conda-forge only, and no Anaconda terms-of-service plugin to + # stop a non-interactive create on the spec's `defaults` channel. + - uses: conda-incubator/setup-miniconda@v3 + with: + miniforge-version: latest + environment-file: environment.yml + activate-environment: realsim + + # The engine's own list of what a co-sim interpreter must import, read from + # carla_env_setup rather than copied here, minus what the spec leaves out + # on purpose: carla (never in CHECKED_MODULES - ensure_carla installs it) + # and the SUMO clients, which environment.yml's trailing comment says are + # installed after the env exists. Where they should come from is #221. + - name: The env imports what the co-sim needs + run: | + python - <<'EOF' + import importlib, sys + sys.path.insert(0, "Carla") + import carla_env_setup as s + want = [m for m in s.CHECKED_MODULES if m not in s.SUMO_MODULES] + bad = [] + for m in want: + try: + importlib.import_module(m) + except Exception as e: + bad.append(f"{m}: {e!r}") + print("checked:", ", ".join(want)) + print("left to #221:", ", ".join(s.SUMO_MODULES)) + if bad: + sys.exit("cannot import:\n " + "\n ".join(bad)) + EOF + + # Excludes the vendored C++ trees: yaml-cpp carries googletest's Python 2 + # scripts, which FIXS never runs. No slashes in the pattern - MSYS bash + # rewrites a slash-bearing argument before python sees it. + - name: Byte-compile the shipped Python + run: python -m compileall -q -x 'yaml-cpp|libsumo|libcarla' Carla CommonLib + + # What a consumer's env has once setup finishes: carla_env_setup.ensure_carla + # adds the PyPI client right after creating the env from the spec, and the + # unit tests import it (CommonLib/fixs/_driver.py -> carla_agents). + - name: Add the carla client, as ensure_carla does + run: python -m pip install carla==0.9.15 + + - name: Unit tests + run: python -m pytest tests/Python/unit -q