Refactor CI/CD to widen the Unity versions range

.github/workflows/main.ci.cd.workflow.yml

@@ -9,91 +9,55 @@ on:
   schedule:
     - cron: "0 5 * * *"  # run daily at 5 AM (UTC)
 
+# Cancel superseded runs on the same ref (e.g. rapid PR pushes) so queued matrix
+# jobs don't stack up against the org's concurrent-runner limit. The event name is
+# part of the group because the nightly schedule runs on the default branch ref and
+# would otherwise cancel (or be cancelled by) a push run on that same branch.
+# Only PR runs are cancelled; push and schedule runs are always allowed to finish.
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.ref }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+
 permissions:
   actions: write
   contents: read
   pull-requests: write
 
 jobs:
   build:
+    name: build (${{ matrix.unity_version }}, ${{ matrix.target_platform }}, ${{ matrix.dotnet_version }}, ${{ matrix.compiler }})
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
+      # Testing every version x platform x dotnet_version combination would explode
+      # the job count. Instead we pick a representative subset: one Android IL2CPP
+      # config per Unity version, alternating API compatibility so both NET_4_x and
+      # STANDARD_2_x are still exercised.
+      #
+      # Video-specific constraints vs stream-chat-unity:
+      # - Floor is 2021.3 (package.json). Vendored Google.Protobuf needs C# 9.
+      # - compiler is always il2cpp (Unity WebRTC requires it; no Mono).
+      # - iOS stays off: ubuntu/ios fails Unity's build post-process cp (PBE-5316).
+      # - testMode is `all` because this repo has both Editor and Runtime tests.
+      # Patch releases must be public LTS/tech builds from the Unity archive, not
+      # Extended LTS (xLTS) patches that require Industry/Enterprise licenses in CI.
+      # dataset_index must be unique per row and within 0-15 (test data set count).
       matrix:
-        target_platform: [android] #TODO: investigate why ubuntu/ios fails on Unity's internal build post-process cp operation (jira PBE-5316)
-        unity_version: [2021, 2022]
-        dotnet_version: [NET_4_x, STANDARD_2_x]
-        compiler: [il2cpp] #no `mono` since Unity's webRTC requires il2cpp
+        include:
+          - { unity_version: "2021.3", target_platform: android, dotnet_version: STANDARD_2_x, compiler: il2cpp, dataset_index: 0, image: "unityci/editor:ubuntu-2021.3.36f1-android-3.1.0" }
+          - { unity_version: "2022.3", target_platform: android, dotnet_version: NET_4_x, compiler: il2cpp, dataset_index: 1, image: "unityci/editor:ubuntu-2022.3.62f2-android-3.2.2" }
+          - { unity_version: "2023.2", target_platform: android, dotnet_version: STANDARD_2_x, compiler: il2cpp, dataset_index: 2, image: "unityci/editor:ubuntu-2023.2.20f1-android-3.2.2" }
+          - { unity_version: "6000.0", target_platform: android, dotnet_version: NET_4_x, compiler: il2cpp, dataset_index: 3, image: "unityci/editor:ubuntu-6000.0.63f1-android-3.2.2" }
+          - { unity_version: "6000.1", target_platform: android, dotnet_version: STANDARD_2_x, compiler: il2cpp, dataset_index: 4, image: "unityci/editor:ubuntu-6000.1.17f1-android-3.2.2" }
+          - { unity_version: "6000.2", target_platform: android, dotnet_version: NET_4_x, compiler: il2cpp, dataset_index: 5, image: "unityci/editor:ubuntu-6000.2.12f1-android-3.2.2" }
 
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
 
-      - name: Check disk space before cleanup
-        run: df -h
-
-      - name: Free up disk space
-        run: |
-          echo "Freeing up disk space..."
-          echo "Unity version: ${{ matrix.unity_version }}"
-          echo "Target platform: ${{ matrix.target_platform }}"
-
-          # Always safe to remove - not used by Unity builds
-          sudo rm -rf /opt/ghc
-          sudo rm -rf /opt/hostedtoolcache/CodeQL
-          sudo rm -rf /usr/local/share/boost
-          sudo rm -rf "$AGENT_TOOLSDIRECTORY"
-
-          # Remove large packages that aren't needed
-          sudo apt-get remove -y '^llvm-.*' --fix-missing || echo "No llvm packages to remove"
-          sudo apt-get remove -y 'php.*' --fix-missing || echo "No php packages to remove"
-          sudo apt-get remove -y '^mongodb-.*' --fix-missing || echo "No mongodb packages to remove"
-          sudo apt-get remove -y '^mysql-.*' --fix-missing || echo "No mysql packages to remove"
-          sudo apt-get remove -y azure-cli google-chrome-stable firefox powershell mono-devel libgl1-mesa-dri --fix-missing || echo "Continuing..."
-          sudo apt-get autoremove -y || echo "Autoremove completed"
-          sudo apt-get clean || echo "Clean completed"
-
-          # Remove system .NET SDK for Unity 2021 and 2022 (they use bundled Mono)
-          # Skip for Unity 2023+ in case of CoreCLR dependencies
-          if [ "${{ matrix.unity_version }}" == "2021" ] || [ "${{ matrix.unity_version }}" == "2022" ] || [ "${{ matrix.unity_version }}" == "2020" ]; then
-            echo "Removing system .NET SDK (Unity ${{ matrix.unity_version }} uses bundled Mono runtime)"
-            sudo apt-get remove -y '^aspnetcore-.*' --fix-missing || echo "No aspnetcore packages to remove"
-            sudo apt-get remove -y '^dotnet-.*' --fix-missing || echo "No dotnet packages to remove"
-            sudo rm -rf /usr/share/dotnet
-          else
-            echo "Keeping system .NET SDK (Unity ${{ matrix.unity_version }} may use CoreCLR)"
-          fi
-
-          # Remove Node modules (not needed for Unity)
-          sudo rm -rf /usr/local/lib/node_modules
-
-          # Only remove Android SDK/NDK if NOT building for Android
-          if [ "${{ matrix.target_platform }}" != "android" ]; then
-            echo "Removing Android SDK/NDK (not needed for ${{ matrix.target_platform }} build)"
-            sudo rm -rf /usr/local/lib/android
-          else
-            echo "Keeping Android SDK/NDK (needed for Android build)"
-          fi
-
-          # Clean up Docker
-          sudo docker image prune --all --force
-
-          echo "Cleanup completed."
-
-      - name: Check disk space after cleanup
-        run: df -h
-
-      - name: Calculate Sequential Index
-        id: calculate-index
+      - name: Set Test Data Set Index
         run: |
-          target_index=$([[ "${{ matrix.target_platform }}" == 'android' ]] && echo '0' || echo '1')
-          unity_index=$([[ "${{ matrix.unity_version }}" == '2020' ]] && echo '0' || echo '1')
-          dotnet_index=$([[ "${{ matrix.dotnet_version }}" == 'NET_4_x' ]] && echo '0' || echo '1')
-          compiler_index=$([[ "${{ matrix.compiler }}" == 'mono' ]] && echo '0' || echo '1')
-
-          index=$((target_index * 1 + unity_index * 2 + dotnet_index * 4 + compiler_index * 8))
-
-          echo "SEQUENTIAL_INDEX=$index" >> $GITHUB_ENV
+          echo "SEQUENTIAL_INDEX=${{ matrix.dataset_index }}" >> $GITHUB_ENV
 
       - name: Print Sequential Index
         run: |
@@ -102,10 +66,15 @@ jobs:
       - name: Install Git
         run: git config --global --add safe.directory /github/workspace
 
+      - name: Free Disk space
+        uses: jlumbroso/free-disk-space@v1.2.0
+        with:
+          dotnet: false
+
       - name: Install dependencies (Linux)
         if: runner.os == 'Linux'
         run: sudo apt-get update
-      
+
       - name: Install dependencies (macOS)
         if: runner.os == 'macOS'
         run: brew update
@@ -123,38 +92,7 @@ jobs:
       - name: Determine Docker Image
         id: dockerImageSelector
         run: |
-          if [ "${{ matrix.unity_version }}" == '2020' ]; then
-            if [ "${{ matrix.target_platform }}" == 'android' ]; then
-              TAG='unityci/editor:ubuntu-2020.3.40f1-android-3.1.0'
-            elif [ "${{ matrix.target_platform }}" == 'ios' ]; then
-              TAG='unityci/editor:ubuntu-2020.3.40f1-ios-3.1.0'
-            else
-              echo "Unsupported platform"
-              exit 1
-            fi
-          elif [ "${{ matrix.unity_version }}" == '2021' ]; then
-            if [ "${{ matrix.target_platform }}" == 'android' ]; then
-              TAG='unityci/editor:ubuntu-2021.3.36f1-android-3.1.0'
-            elif [ "${{ matrix.target_platform }}" == 'ios' ]; then
-              TAG='unityci/editor:ubuntu-2021.3.36f1-ios-3.1.0'
-            else
-              echo "Unsupported platform"
-              exit 1
-            fi
-          elif [ "${{ matrix.unity_version }}" == '2022' ]; then
-            if [ "${{ matrix.target_platform }}" == 'android' ]; then
-              TAG='unityci/editor:ubuntu-2022.3.36f1-android-3.1.0'
-            elif [ "${{ matrix.target_platform }}" == 'ios' ]; then
-              TAG='unityci/editor:ubuntu-2022.3.36f1-ios-3.1.0'
-            else
-              echo "Unsupported platform"
-              exit 1
-            fi
-          else
-            echo "Unsupported Unity version"
-            exit 1
-          fi
-          echo "DOCKER_TAG=$TAG" >> $GITHUB_ENV
+          echo "DOCKER_TAG=${{ matrix.image }}" >> $GITHUB_ENV
 
       - name: Echo Docker Image
         run: |
@@ -163,7 +101,7 @@ jobs:
       - name: Determine Build Name
         run: |
           RUNNER_ID="${{ matrix.unity_version }}_${{ matrix.target_platform }}_${{ matrix.compiler }}_${{ matrix.dotnet_version }}"
-          
+
           if [ "${{ matrix.target_platform }}" == "android" ]; then
             BUILD_NAME="${RUNNER_ID}.apk"
           elif [ "${{ matrix.target_platform }}" == "ios" ]; then
@@ -172,10 +110,10 @@ jobs:
             echo "Unsupported platform"
             exit 1
           fi
-          
+
           echo "RUNNER_ID=$RUNNER_ID" >> $GITHUB_ENV
           echo "BUILD_NAME=$BUILD_NAME" >> $GITHUB_ENV
-      
+
       - name: Enable Tests
         uses: game-ci/unity-builder@v4
         env:
@@ -185,7 +123,9 @@ jobs:
         with:
           buildMethod: StreamVideo.EditorTools.StreamEditorTools.EnableStreamTestsEnabledCompilerFlag
           customImage: ${{ env.DOCKER_TAG }}
+          allowDirtyBuild: true
 
+      # Editor + Runtime assemblies. Chat is editmode-only; do not copy that.
       - name: Run Tests (Attempt 1)
         id: run_tests_1
         uses: game-ci/unity-test-runner@v4
@@ -194,11 +134,17 @@ jobs:
           UNITY_EMAIL: ${{ secrets.UNITY_EMAIL }}
           UNITY_PASSWORD: ${{ secrets.UNITY_PASSWORD }}
         with:
+          testMode: all
           customParameters: -streamBase64TestDataSet "${{ secrets.STREAM_AUTH_TEST_DATA_BASE64 }}" -testDataSetIndex ${{ env.SEQUENTIAL_INDEX }}
           customImage: ${{ env.DOCKER_TAG }}
-        timeout-minutes: 20
+          allowDirtyBuild: true
+        timeout-minutes: 40
         continue-on-error: true
 
+      - name: Reset Unity state before retry 2
+        if: steps.run_tests_1.outcome == 'failure'
+        run: bash .github/workflows/scripts/reset-unity-state.sh
+
       - name: Run Tests (Attempt 2)
         id: run_tests_2
         if: steps.run_tests_1.outcome == 'failure'
@@ -208,11 +154,17 @@ jobs:
           UNITY_EMAIL: ${{ secrets.UNITY_EMAIL }}
           UNITY_PASSWORD: ${{ secrets.UNITY_PASSWORD }}
         with:
+          testMode: all
           customParameters: -streamBase64TestDataSet "${{ secrets.STREAM_AUTH_TEST_DATA_BASE64 }}" -testDataSetIndex ${{ env.SEQUENTIAL_INDEX }}
           customImage: ${{ env.DOCKER_TAG }}
-        timeout-minutes: 20
+          allowDirtyBuild: true
+        timeout-minutes: 50
         continue-on-error: true
 
+      - name: Reset Unity state before retry 3
+        if: steps.run_tests_2.outcome == 'failure'
+        run: bash .github/workflows/scripts/reset-unity-state.sh
+
       - name: Run Tests (Attempt 3)
         id: run_tests_3
         if: steps.run_tests_2.outcome == 'failure'
@@ -222,50 +174,33 @@ jobs:
           UNITY_EMAIL: ${{ secrets.UNITY_EMAIL }}
           UNITY_PASSWORD: ${{ secrets.UNITY_PASSWORD }}
         with:
+          testMode: all
           customParameters: -streamBase64TestDataSet "${{ secrets.STREAM_AUTH_TEST_DATA_BASE64 }}" -testDataSetIndex ${{ env.SEQUENTIAL_INDEX }}
           customImage: ${{ env.DOCKER_TAG }}
-        timeout-minutes: 20
-        continue-on-error: true
-
-      - name: Run Tests (Attempt 4)
-        id: run_tests_4
-        if: steps.run_tests_3.outcome == 'failure'
-        uses: game-ci/unity-test-runner@v4
-        env:
-          UNITY_LICENSE: ${{ secrets.UNITY_LICENSE }}
-          UNITY_EMAIL: ${{ secrets.UNITY_EMAIL }}
-          UNITY_PASSWORD: ${{ secrets.UNITY_PASSWORD }}
-        with:
-          customParameters: -streamBase64TestDataSet "${{ secrets.STREAM_AUTH_TEST_DATA_BASE64 }}" -testDataSetIndex ${{ env.SEQUENTIAL_INDEX }}
-          customImage: ${{ env.DOCKER_TAG }}
-        timeout-minutes: 20
-        continue-on-error: true
-
-      - name: Run Tests (Attempt 5)
-        id: run_tests_5
-        if: steps.run_tests_4.outcome == 'failure'
-        uses: game-ci/unity-test-runner@v4
-        env:
-          UNITY_LICENSE: ${{ secrets.UNITY_LICENSE }}
-          UNITY_EMAIL: ${{ secrets.UNITY_EMAIL }}
-          UNITY_PASSWORD: ${{ secrets.UNITY_PASSWORD }}
-        with:
-          customParameters: -streamBase64TestDataSet "${{ secrets.STREAM_AUTH_TEST_DATA_BASE64 }}" -testDataSetIndex ${{ env.SEQUENTIAL_INDEX }}
-          customImage: ${{ env.DOCKER_TAG }}
-        timeout-minutes: 20
+          allowDirtyBuild: true
+        timeout-minutes: 60
 
       - name: Upload Test Results as Artifact
         uses: actions/upload-artifact@v4
+        if: always()
         with:
           name: Test_Results_${{ env.RUNNER_ID }}
           path: artifacts
 
+      - name: Verify tests actually ran
+        run: |
+          bash .github/workflows/scripts/assert-tests-ran.sh artifacts/editmode-results.xml
+          bash .github/workflows/scripts/assert-tests-ran.sh artifacts/playmode-results.xml
+
+      - name: Free Disk space
+        uses: jlumbroso/free-disk-space@v1.2.0
+        if: matrix.target_platform == 'android' || matrix.target_platform == 'ios'
+        with:
+          dotnet: false
+
       - name: List changes
         run: |
           git diff
-
-      - name: Check disk space before build
-        run: df -h
 
       - name: Build Sample Project
         uses: game-ci/unity-builder@v4
@@ -275,16 +210,16 @@ jobs:
           UNITY_LICENSE: ${{ secrets.UNITY_LICENSE }}
         with:
           buildMethod: StreamVideo.EditorTools.StreamEditorTools.BuildSampleApp
-          customParameters: -streamBase64TestDataSet ${{ secrets.STREAM_AUTH_TEST_DATA_BASE64 }} -testDataSetIndex ${{ env.SEQUENTIAL_INDEX }} -apiCompatibility ${{ matrix.dotnet_version }} -scriptingBackend ${{ matrix.compiler }} -buildTargetPlatform ${{ matrix.target_platform }} -buildTargetPath $(pwd)/SampleAppBuild/${{ env.BUILD_NAME }}
+          customParameters: -streamBase64TestDataSet ${{ secrets.STREAM_AUTH_TEST_DATA_BASE64 }} -testDataSetIndex ${{ env.SEQUENTIAL_INDEX }} -apiCompatibility ${{ matrix.dotnet_version }} -scriptingBackend ${{ matrix.compiler }} -buildTargetPlatform ${{ matrix.target_platform }} -buildTargetPath SampleAppBuild/${{ env.BUILD_NAME }}
           customImage: ${{ env.DOCKER_TAG }}
           allowDirtyBuild: true #Needed because the import process may update ProjectSettings
 
       - name: Upload Build as Artifact
         uses: actions/upload-artifact@v4
         with:
           name: Build_${{ env.BUILD_NAME }}
-          path: $(pwd)/SampleAppBuild/${{ env.BUILD_NAME }}
-          
+          path: ${{ github.workspace }}/SampleAppBuild/${{ env.BUILD_NAME }}
+
       - name: Notify Slack if failed
         uses: voxmedia/github-action-slack-notify-build@v1
         if: always() && failure()

.github/workflows/scripts/assert-tests-ran.sh

@@ -0,0 +1,28 @@
+#!/usr/bin/env bash
+# Fails the job when the test runner reported success without executing any test.
+#
+# Unity exits 0 when it cannot activate a license, and game-ci then prints
+# "Run succeeded, no failures occurred" with an empty result file. Without this guard a
+# job that ran zero tests is indistinguishable from a job where everything passed.
+set -euo pipefail
+
+results_file="${1:?path to the results xml is required}"
+
+if [ ! -f "${results_file}" ]; then
+  echo "::error::${results_file} is missing - the test runner never produced results."
+  exit 1
+fi
+
+test_case_count="$(grep -oE 'testcasecount="[0-9]+"' "${results_file}" | head -n 1 | grep -oE '[0-9]+' || true)"
+
+if [ -z "${test_case_count}" ]; then
+  echo "::error::Could not read testcasecount from ${results_file}."
+  exit 1
+fi
+
+if [ "${test_case_count}" -eq 0 ]; then
+  echo "::error::The test runner executed 0 tests in ${results_file}. Treating this as a failure - see the log for license or compilation errors."
+  exit 1
+fi
+
+echo "Test runner executed ${test_case_count} tests in ${results_file}."

.github/workflows/scripts/reset-unity-state.sh

@@ -0,0 +1,26 @@
+#!/usr/bin/env bash
+# Resets everything a killed Unity attempt leaves behind, so the next attempt can start.
+#
+# When a "Run Tests" step hits its step timeout, GitHub kills the action but not the
+# docker container it started. Without this cleanup the next attempt dies immediately
+# with either "Multiple Unity instances cannot open the same project" (the orphan still
+# holds the project) or "Machine identification is invalid for current license" - and in
+# the licensing case Unity exits 0 after running zero tests, which silently turns the
+# job green.
+set -uo pipefail
+
+running_containers="$(docker ps -q)"
+if [ -n "${running_containers}" ]; then
+  echo "Stopping leftover containers: ${running_containers}"
+  # shellcheck disable=SC2086
+  docker stop --time 10 ${running_containers} || true
+fi
+
+echo "Removing Unity project lock file"
+sudo rm -f Temp/UnityLockfile || true
+
+HOME_DIR="${RUNNER_TEMP}/_github_home"
+echo "Removing Unity license state under ${HOME_DIR}"
+sudo find "${HOME_DIR}" -iname '*.ulf' -delete -print || true
+sudo rm -rf "${HOME_DIR}/.config/unity3d/Unity/licenses" \
+  "${HOME_DIR}/.local/share/unity3d/Unity" || true