#273 #248/test the consumer install path and the shipped env in CI

.github/workflows/python-env.yml

@@ -0,0 +1,85 @@
+name: Python env (environment.yml)
+
+# Builds the conda env from environment.yml ALONE and runs the Python side in it
+# (#248). environment.yml is what every consumer builds from - carla_env_setup
+# offers to create it, FIXS_Applications derives its env from it - and two broken
+# specs shipped with CI green: shapely missing (#236, traffic-light sync silently
+# off) and a wheel path the release zip strips (#221, `conda env create` failed on
+# every published release). Nothing ever ran the command.
+#
+# Windows, not Linux: the spec pins vs2015_runtime / vc14_runtime, which
+# conda-forge publishes for win-64 only, so it does not resolve on linux-64.
+#
+# No path filter, on purpose. release.yml skips its build when a PR touches only
+# .py / environment.yml / tests/, and still reports green - so a Python-only PR
+# ran nothing at all. This job exists for exactly those PRs.
+
+on:
+  workflow_dispatch:
+  pull_request:
+    branches: [dev, main, 'dev_v*', 'beta_v*']
+  push:
+    branches: [main, 'dev_v*', 'beta_v*']
+
+jobs:
+  env:
+    name: env from environment.yml + unit tests
+    runs-on: windows-2022
+    defaults:
+      run:
+        shell: bash -el {0}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          sparse-checkout: |
+            Carla
+            CommonLib
+            tests/Python
+            environment.yml
+
+      # Miniforge: conda-forge only, and no Anaconda terms-of-service plugin to
+      # stop a non-interactive create on the spec's `defaults` channel.
+      - uses: conda-incubator/setup-miniconda@v3
+        with:
+          miniforge-version: latest
+          environment-file: environment.yml
+          activate-environment: realsim
+
+      # The engine's own list of what a co-sim interpreter must import, read from
+      # carla_env_setup rather than copied here, minus what the spec leaves out
+      # on purpose: carla (never in CHECKED_MODULES - ensure_carla installs it)
+      # and the SUMO clients, which environment.yml's trailing comment says are
+      # installed after the env exists. Where they should come from is #221.
+      - name: The env imports what the co-sim needs
+        run: |
+          python - <<'EOF'
+          import importlib, sys
+          sys.path.insert(0, "Carla")
+          import carla_env_setup as s
+          want = [m for m in s.CHECKED_MODULES if m not in s.SUMO_MODULES]
+          bad = []
+          for m in want:
+              try:
+                  importlib.import_module(m)
+              except Exception as e:
+                  bad.append(f"{m}: {e!r}")
+          print("checked:", ", ".join(want))
+          print("left to #221:", ", ".join(s.SUMO_MODULES))
+          if bad:
+              sys.exit("cannot import:\n  " + "\n  ".join(bad))
+          EOF
+
+      # Excludes the vendored C++ trees: yaml-cpp carries googletest's Python 2
+      # scripts, which FIXS never runs. No slashes in the pattern - MSYS bash
+      # rewrites a slash-bearing argument before python sees it.
+      - name: Byte-compile the shipped Python
+        run: python -m compileall -q -x 'yaml-cpp|libsumo|libcarla' Carla CommonLib
+
+      # What a consumer's env has once setup finishes: carla_env_setup.ensure_carla
+      # adds the PyPI client right after creating the env from the spec, and the
+      # unit tests import it (CommonLib/fixs/_driver.py -> carla_agents).
+      - name: Add the carla client, as ensure_carla does
+        run: python -m pip install carla==0.9.15
+
+      - name: Unit tests
+        run: python -m pytest tests/Python/unit -q

.github/workflows/release.yml

@@ -101,12 +101,13 @@ jobs:
           #                          here - building validates nothing about it.
           #                          pytest is the right gate for Python (#57).
           #   environment.yml        copied into the zip only (8_create_zip.ps1:110);
-          #                          a real env check is #248, not a C++ compile
-          #   scripts/update_fixs.*  consumer-side installer (#272) - not a build
-          #                          input, and not packaged (8_create_zip.ps1:119)
+          #                          checked by python-env.yml (#248), not a C++ compile
           # NOTE: ^scripts/ must NOT be ignored wholesale - scripts/dispatch/** IS
-          # the build system, and generate_version.ps1 stamps the version.
-          ignore='^doc/|^tests/|^\.github/|^\.vscode/|\.md$|\.py$|^LICENSE$|^\.gitignore$|^\.gitattributes$|^environment\.yml$|^scripts/update_fixs\.(ps1|sh)$'
+          # the build system, and generate_version.ps1 stamps the version. Nor
+          # scripts/update_fixs.* (ignored until #273): the build now installs its
+          # fresh zip with them, and every app on a channel runs them, so a change
+          # to the installer is exactly the change that must reach that check.
+          ignore='^doc/|^tests/|^\.github/|^\.vscode/|\.md$|\.py$|^LICENSE$|^\.gitignore$|^\.gitattributes$|^environment\.yml$'
           relevant=$(echo "$files" | grep -vE "$ignore" || true)
           if [ -z "$relevant" ]; then
             echo "No build-relevant changes -> skipping build (required check still passes)."
@@ -373,6 +374,26 @@ jobs:
             Write-Host "VirtualEnvironment.lib absent (expected on the current/main train)."
           }
 
+      # Install the zip just packed exactly as a consumer would, BEFORE anything is
+      # published (#273). #238 changed the packaging, the installer was not updated
+      # to match, and it shipped: nothing ran the install path, so the first to
+      # notice was a user's --update-fixs. The checks after the publish below
+      # cover the download and the checksum; this one is what keeps a zip that
+      # cannot install off the channel.
+      - name: Install the packed zip as a consumer would (update_fixs.ps1 -Zip)
+        if: needs.changes.outputs.should_build == 'true'
+        shell: pwsh
+        env:
+          # PR builds have no channel; any tag will do, it is only stamped and read back.
+          TAG: ${{ steps.channel.outputs.tag || 'pr-build' }}
+        run: |
+          $zip = Get-ChildItem build -Filter 'fixs-build-*.zip' | Select-Object -First 1
+          $root = Join-Path $env:RUNNER_TEMP 'consumer'
+          & powershell -NoProfile -ExecutionPolicy Bypass -File scripts/update_fixs.ps1 -Root $root -Zip $zip.FullName -Version $env:TAG
+          if ($LASTEXITCODE -ne 0) { exit 1 }
+          & powershell -NoProfile -ExecutionPolicy Bypass -File scripts/check_fixs_install.ps1 -Dir (Join-Path $root 'FIXS') -Tag $env:TAG
+          if ($LASTEXITCODE -ne 0) { exit 1 }
+
       - name: Upload packaged zip as a workflow artifact (inspect on PRs)
         if: needs.changes.outputs.should_build == 'true'
         uses: actions/upload-artifact@v4
@@ -505,6 +526,17 @@ jobs:
         if: needs.changes.outputs.should_build == 'true'
         run: ./scripts/dispatch/pack_linux_bundle.sh --out-dir dist
 
+      # The Linux half of build-publish's pre-publish install (#273), on the same
+      # focal image the bundle targets. linux-publish needs this job, so a bundle
+      # that cannot install is never uploaded.
+      - name: Install the packed zip as a consumer would (update_fixs.sh --zip)
+        if: needs.changes.outputs.should_build == 'true'
+        run: |
+          zip="$(ls dist/fixs-build-*-linux-x86_64.zip)"
+          root="$RUNNER_TEMP/consumer"; mkdir -p "$root"
+          bash scripts/update_fixs.sh --root "$root" --zip "$zip" --version pr-build
+          bash scripts/check_fixs_install.sh "$root/FIXS" pr-build
+
       - name: Upload bundle as a workflow artifact (inspect on PRs)
         if: needs.changes.outputs.should_build == 'true'
         uses: actions/upload-artifact@v4
@@ -553,6 +585,66 @@ jobs:
           # same rolling release is untouched.
           gh release upload "$CHANNEL_TAG" "$want" "$want.sha256" --repo "$GITHUB_REPOSITORY" --clobber
 
+  # ===========================================================================
+  # After the publish: install from the channel exactly as an app's --update-fixs
+  # does (#273). The pre-publish installs above use -Zip, so they cannot see the
+  # download or the checksum; these can. A failure here means the channel is
+  # already serving a bad release - it cannot un-publish, only make that loud.
+  # ===========================================================================
+  install-check-windows:
+    name: Install the published release (Windows)
+    needs: build-publish
+    if: needs.build-publish.outputs.publish == 'true' && (github.event_name == 'push' || (github.event_name == 'workflow_dispatch' && inputs.publish))
+    runs-on: windows-2022
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          sparse-checkout: scripts
+      - name: update_fixs.ps1 -Version <channel>
+        shell: pwsh
+        env:
+          TAG: ${{ needs.build-publish.outputs.tag }}
+        run: |
+          $root = Join-Path $env:RUNNER_TEMP 'consumer'
+          & powershell -NoProfile -ExecutionPolicy Bypass -File scripts/update_fixs.ps1 -Root $root -Version $env:TAG -Repo $env:GITHUB_REPOSITORY
+          if ($LASTEXITCODE -ne 0) { exit 1 }
+          & powershell -NoProfile -ExecutionPolicy Bypass -File scripts/check_fixs_install.ps1 -Dir (Join-Path $root 'FIXS') -Tag $env:TAG
+          if ($LASTEXITCODE -ne 0) { exit 1 }
+
+  install-check-linux:
+    name: Install the published release (Linux)
+    needs: [build-publish, linux-publish]
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          sparse-checkout: scripts
+      - name: update_fixs.sh --version <channel>
+        env:
+          TAG: ${{ needs.build-publish.outputs.tag }}
+        run: |
+          root="$RUNNER_TEMP/consumer"; mkdir -p "$root"
+          bash scripts/update_fixs.sh --root "$root" --version "$TAG" --repo "$GITHUB_REPOSITORY"
+          bash scripts/check_fixs_install.sh "$root/FIXS" "$TAG"
+
+      # The picker must offer exactly the releases that carry a build zip. The
+      # script decides that by walking the API's JSON as a key stream without
+      # jq, so check its answer against jq's reading of the same API. Neither
+      # list is allowed to be empty or to lack the channel just published.
+      - name: The picker offers exactly the installable releases
+        env:
+          GH_TOKEN: ${{ github.token }}
+          TAG: ${{ needs.build-publish.outputs.tag }}
+        run: |
+          offered="$(bash scripts/update_fixs.sh --list --repo "$GITHUB_REPOSITORY" | sort)"
+          expected="$(gh api "repos/$GITHUB_REPOSITORY/releases?per_page=30" \
+                        --jq '.[] | select(.draft | not) | select(any(.assets[]; .name | startswith("fixs-build-") and endswith(".zip"))) | .tag_name' | sort)"
+          echo "offered:";  echo "$offered"
+          echo "expected:"; echo "$expected"
+          [ -n "$offered" ] || { echo "::error::the picker offers nothing"; exit 1; }
+          echo "$offered" | grep -qx "$TAG" || { echo "::error::the picker does not offer $TAG, just published"; exit 1; }
+          [ "$offered" = "$expected" ] || { echo "::error::the picker's list differs from the releases that carry a fixs-build-*.zip"; exit 1; }
+
   # Read the Docs serves one docs version per channel under the channel's own
   # name (stable, vX.Y.Z-beta, vX.Y.Z-alpha), built from the tag published above
   # (#137). Ask RTD to rebuild it here, so the docs move exactly when the channel

scripts/check_fixs_install.ps1

@@ -0,0 +1,50 @@
+# ============================================================================
+# Is <Dir> a complete, runnable FIXS install? (Windows)  #273
+#
+# Run by release.yml against what update_fixs.ps1 produced - once from the zip it
+# just packed (before publishing) and once from the published release (after).
+# scripts/check_fixs_install.sh is the Linux counterpart; change them together.
+#
+#   check_fixs_install.ps1 -Dir C:\tmp\t\FIXS -Tag v0.9.1-alpha
+#
+# Exit 0 = complete. Exit 1 = says what is missing.
+# ============================================================================
+param(
+    [Parameter(Mandatory = $true)] [string]$Dir,
+    [Parameter(Mandatory = $true)] [string]$Tag
+)
+$ErrorActionPreference = 'Stop'
+$bad = @()
+
+if (-not (Test-Path (Join-Path $Dir 'TrafficLayer.exe'))) { $bad += 'TrafficLayer.exe is missing' }
+
+# A load, not a presence check: the vendored bin/ once lacked geos_c.dll for
+# months with every probed DLL present (#70). libsumo_verify.ps1 walks the
+# import graph the way TrafficLayer does at run time.
+. (Join-Path $PSScriptRoot 'dispatch\libsumo_verify.ps1')
+try {
+    Test-LibsumoLoadable -BinDir (Join-Path $Dir 'CommonLib\libsumo\bin') -Context 'The installed native runtime is incomplete.'
+} catch { $bad += "$_" }
+
+# Line 1 is '<tag>' for a pinned release and '<tag> (<published>)' for a rolling
+# one. run_cosim.py reads it byte-for-byte, where a BOM becomes part of the tag.
+$vf = Join-Path $Dir 'FIXS_VERSION.txt'
+if (-not (Test-Path $vf)) {
+    $bad += 'FIXS_VERSION.txt is missing'
+} else {
+    $bytes = [System.IO.File]::ReadAllBytes($vf)
+    if ($bytes.Length -ge 3 -and $bytes[0] -eq 0xEF -and $bytes[1] -eq 0xBB -and $bytes[2] -eq 0xBF) {
+        $bad += 'FIXS_VERSION.txt starts with a UTF-8 BOM'
+    }
+    $line1 = ([System.IO.File]::ReadAllLines($vf) | Select-Object -First 1)
+    if ($line1 -ne $Tag -and -not $line1.StartsWith("$Tag (")) {
+        $bad += "FIXS_VERSION.txt line 1 is '$line1', expected '$Tag'"
+    }
+}
+
+if ($bad) {
+    $bad | ForEach-Object { Write-Host "[FAIL] $_" }
+    exit 1
+}
+Write-Host "Install in $Dir is complete ($Tag)."
+exit 0

scripts/check_fixs_install.sh

@@ -0,0 +1,41 @@
+#!/usr/bin/env bash
+# ============================================================================
+#  Is <dir> a complete, runnable FIXS install? (Linux)  #273
+#
+#  Run by release.yml against what update_fixs.sh produced - once from the zip it
+#  just packed (before publishing) and once from the published release (after).
+#  scripts/check_fixs_install.ps1 is the Windows counterpart; change them together.
+#
+#    check_fixs_install.sh /tmp/t/FIXS v0.9.1-alpha
+#
+#  Exit 0 = complete. Exit 1 = says what is missing.
+# ============================================================================
+set -uo pipefail
+dir="$1" tag="$2"
+bad=0
+fail() { echo "[FAIL] $*"; bad=1; }
+
+if [[ ! -x "$dir/TrafficLayer" ]]; then
+    fail "TrafficLayer is missing or not executable"
+else
+    # A load check, not a presence check. TrafficLayer finds libtracicpp.so
+    # through its $ORIGIN-relative RPATH (CMakeLists.txt), so with no
+    # LD_LIBRARY_PATH set this is exactly what a consumer's first run resolves.
+    missing="$(env -u LD_LIBRARY_PATH ldd "$dir/TrafficLayer" | grep 'not found' || true)"
+    [[ -z "$missing" ]] || fail "TrafficLayer cannot resolve: $(echo "$missing" | awk '{print $1}' | tr '\n' ' ')"
+fi
+[[ -f "$dir/CommonLib/libsumo/bin/libtracicpp.so" ]] || fail "CommonLib/libsumo/bin/libtracicpp.so is missing"
+
+# Line 1 is '<tag>' for a pinned release and '<tag> (<published>)' for a rolling
+# one. run_cosim.py reads it byte-for-byte, where a BOM becomes part of the tag.
+vf="$dir/FIXS_VERSION.txt"
+if [[ ! -f "$vf" ]]; then
+    fail "FIXS_VERSION.txt is missing"
+else
+    [[ "$(head -c3 "$vf" | od -An -tx1 | tr -d ' \n')" != "efbbbf" ]] || fail "FIXS_VERSION.txt starts with a UTF-8 BOM"
+    line1="$(head -n1 "$vf")"
+    [[ "$line1" == "$tag" || "$line1" == "$tag ("* ]] || fail "FIXS_VERSION.txt line 1 is '$line1', expected '$tag'"
+fi
+
+[[ "$bad" == 0 ]] || exit 1
+echo "Install in $dir is complete ($tag)."