ci: Add WebGL PlayMode test job (MTT-15569)

Split the WebGL CI into a build phase and a run phase. webgl-build.yml
now exposes the built player as a dedicated 'players' artifact, and a new
webgl-test.yml run job consumes it and executes the PlayMode tests inside
Firefox on a GPU-backed Ubuntu agent (Unity::VM::GPU, rtx2080). The Firefox
browser flags are passed explicitly so UTR does not attempt to download the
browser from Stevedore.

The run job is wired into the Nightly and Weekly (QV) triggers only, not
into PR triggers, since WebGL failures are infrequent and the build is slow.
Runtime RuntimePlatform.WebGLPlayer exclusions are intentionally deferred to
be added reactively after the first real CI run.

Author: michalChrobot

.yamato/README.md

@@ -21,7 +21,7 @@ CI related files are present inside .yamato/ folder and we can distinguish speci
 - `.yamato/project-standards.yml` responsible for running standards check on a package.
 - `.yamato/code-coverage.yml` responsible for running code coverage analysis.
 - `.yamato/performance-tests.yml` responsible for running performance tests.
-- `.yamato/webgl-build.yml` responsible for making a build for webgl platform.
+- `.yamato/webgl-test.yml` responsible for building a webgl player and running its PlayMode tests in a browser.
 
 ### Standalone Tests
 - `.yamato/desktop-standalone-tests.yml` responsible for running tests on standalone desktops.

.yamato/_run-all.yml

@@ -129,7 +129,7 @@ run_all_webgl_builds:
 {% for project in projects.default -%}
 {% for platform in test_platforms.desktop -%}
 {% for editor in validation_editors.all -%}
-    - .yamato/webgl-build.yml#webgl_build_{{ project.name }}_{{ platform.name }}_{{ editor }}
+    - .yamato/webgl-test.yml#webgl_build_{{ project.name }}_{{ platform.name }}_{{ editor }}
 {% endfor -%}
 {% endfor -%}
 {% endfor -%}
@@ -141,7 +141,7 @@ run_all_webgl_builds_trunk:
   dependencies:
 {% for project in projects.default -%}
 {% for platform in test_platforms.desktop -%}
-    - .yamato/webgl-build.yml#webgl_build_{{ project.name }}_{{ platform.name }}_trunk
+    - .yamato/webgl-test.yml#webgl_build_{{ project.name }}_{{ platform.name }}_trunk
 {% endfor -%}
 {% endfor -%}
 
@@ -152,7 +152,7 @@ run_all_webgl_builds_pinnedTrunk:
   dependencies:
 {% for project in projects.default -%}
 {% for platform in test_platforms.desktop -%}
-    - .yamato/webgl-build.yml#webgl_build_{{ project.name }}_{{ platform.name }}_{{ pinnedTrunk }}
+    - .yamato/webgl-test.yml#webgl_build_{{ project.name }}_{{ platform.name }}_{{ pinnedTrunk }}
 {% endfor -%}
 {% endfor -%}
 
@@ -162,7 +162,41 @@ run_all_webgl_builds_6000:
   dependencies:
 {% for project in projects.default -%}
 {% for platform in test_platforms.desktop -%}
-    - .yamato/webgl-build.yml#webgl_build_{{ project.name }}_{{ platform.name }}_6000.0
+    - .yamato/webgl-test.yml#webgl_build_{{ project.name }}_{{ platform.name }}_6000.0
+{% endfor -%}
+{% endfor -%}
+
+
+# Runs all WebGL PlayMode tests (runs the produced player in a browser on a GPU agent)
+run_all_webgl_tests:
+  name: Run All WebGl Tests
+  dependencies:
+{% for project in projects.default -%}
+{% for platform in test_platforms.webgl_test -%}
+{% for editor in validation_editors.all -%}
+    - .yamato/webgl-test.yml#webgl_test_{{ project.name }}_{{ platform.name }}_{{ editor }}
+{% endfor -%}
+{% endfor -%}
+{% endfor -%}
+
+
+# Runs all WebGL PlayMode tests on trunk editor
+run_all_webgl_tests_trunk:
+  name: Run All WebGl Tests [Trunk only]
+  dependencies:
+{% for project in projects.default -%}
+{% for platform in test_platforms.webgl_test -%}
+    - .yamato/webgl-test.yml#webgl_test_{{ project.name }}_{{ platform.name }}_trunk
+{% endfor -%}
+{% endfor -%}
+
+# Runs all WebGL PlayMode tests on 6000.0 editor
+run_all_webgl_tests_6000:
+  name: Run All WebGl Tests [6000.0]
+  dependencies:
+{% for project in projects.default -%}
+{% for platform in test_platforms.webgl_test -%}
+    - .yamato/webgl-test.yml#webgl_test_{{ project.name }}_{{ platform.name }}_6000.0
 {% endfor -%}
 {% endfor -%}
 

.yamato/_triggers.yml

@@ -151,6 +151,9 @@ develop_nightly:
     # Build player for webgl platform on trunk and 6000.0 editors
     - .yamato/_run-all.yml#run_all_webgl_builds_trunk
     - .yamato/_run-all.yml#run_all_webgl_builds_6000
+    # Run WebGL PlayMode tests (browser on GPU agent) on trunk and 6000.0 editors
+    - .yamato/_run-all.yml#run_all_webgl_tests_trunk
+    - .yamato/_run-all.yml#run_all_webgl_tests_6000
     # Run Runtime tests against cmb service on trunk and 6000.0 editors
     - .yamato/_run-all.yml#run_all_project_tests_cmb_service_trunk
     - .yamato/_run-all.yml#run_all_project_tests_cmb_service_6000
@@ -159,7 +162,7 @@ develop_nightly:
     - .yamato/project-updated-dependencies-test.yml#updated-dependencies_testproject_NGO_win_6000.0
     # Run API validation to early-detect all new APIs that would force us to release new minor version of the package. Note that for this to work the package version in package.json must correspond to "actual package state" which means that it should be higher than last released version
     - .yamato/vetting-test.yml#vetting_test
-  # Run code coverage test (nightly uses actual trunk)
+    # Run code coverage test (nightly uses actual trunk)
     - .yamato/code-coverage.yml#code_coverage_project_test_testproject_ubuntu_trunk
 
 
@@ -189,5 +192,7 @@ develop_weekly_trunk:
     - .yamato/_run-all.yml#run_all_project_tests_console_standalone
     # Build player for webgl platform on trunk
     - .yamato/_run-all.yml#run_all_webgl_builds
+    # Run WebGL PlayMode tests (browser on GPU agent)
+    - .yamato/_run-all.yml#run_all_webgl_tests
     # Run Runtime tests against CMB service
     - .yamato/_run-all.yml#run_all_project_tests_cmb_service

.yamato/project.metafile

@@ -168,6 +168,15 @@ test_platforms:
           flavor: b1.large
           larger_flavor: b1.xlarge
           standalone: GameCoreScarlett
+    # WebGL runtime tests run the built player inside a browser, which requires a real GPU and display.
+    # We therefore run them on a GPU-backed Ubuntu VM (rtx2080). Firefox is preinstalled on the package-ci Ubuntu image.
+    webgl_test:
+        - name: ubuntu
+          type: Unity::VM::GPU
+          image: package-ci/ubuntu-22.04:v4.87.0
+          flavor: b1.large
+          model: rtx2080
+          standalone: WebGL
 
 # EDITOR CONFIGURATIONS-------------------------------------------------------------------------------
 # Editors to be used for testing. NGOv2.X official support started from 6000.0 editor

.yamato/webgl-build.yml .yamato/webgl-test.yml.yamato/webgl-build.yml renamed to .yamato/webgl-test.yml

@@ -2,28 +2,36 @@
 ---
 
 # DESCRIPTION--------------------------------------------------------------------------
-  # This job is responsible for validating a successful building of a player on WebGl standalone platform.
-  # This job WILL NOT execute any tests, we only validate the building process.
-  # This is because WebGL runs in browser and for tests to be executed we would need to consider having a web server, browser automation and overall complex test setup which currently we don't have.
+  # This job is responsible for WebGL platform test validation.
+  # It is split into a build phase (produces the WebGL player) and a run phase (executes the produced player inside a browser to run PlayMode tests).
 
 # CONFIGURATION STRUCTURE--------------------------------------------------------------
-  # Jobs configurations are generated using nested loops through:
-    # 1. For the default project (project is used only as a context for the build).
-    # 2. For all desktop platforms (Windows, Ubuntu, macOS)
-    # 3. For all supported Unity Editor versions (For NGOv2.X it means 6000+)
+  # Jobs are generated using nested loops (separate build phase and run phase). Worth noting that run phase uses the build as dependency:
+    # 1. Build phase runs for all desktop platforms (Windows, Ubuntu, macOS); run phase runs on a single GPU-backed Ubuntu agent.
+    # 2. For all supported Unity Editor versions (For NGOv2.X it means 6000+).
+    # 3. For the default project.
 
 # TECHNICAL CONSIDERATIONS----------------------------------------------------------------
-  # WebGL requires IL2CPP scripting backend (Mono is not supported)
-  # We are not using ARM64 architectures since we only perform a build action. x64 architectures are preferred for build phase (in order to optimize available resource usage)
-  # We only perform build validation (no runtime testing)
+  # WebGL requires IL2CPP scripting backend (Mono is not supported).
+  # Build phase:
+    # We are not using ARM64 architectures since we only perform a build action. x64 architectures are preferred for build phase (in order to optimize available resource usage).
+  # Run phase:
+    # Running the player means launching a real browser, which needs a GPU and a display, therefore we run on a GPU-backed Ubuntu agent (Unity::VM::GPU, rtx2080). See webgl_test in project.metafile.
+    # We do NOT pass -nographics here (unlike the build job) because the browser needs graphics to run.
+    # Firefox is preinstalled on the package-ci Ubuntu image (/usr/bin/firefox).
+    # Passing --target-browser and --target-browser-executable explicitly is mandatory: without them UTR tries to download Firefox from Stevedore via perl, which fails on the CI agent.
+    # --player-connection-ip=${BOKKEN_HOST_IP} is required so the browser-hosted player can connect back to the test runner.
 
 # QUALITY CONSIDERATIONS--------------------------------------------------------------------
-  # In the future we could try to implement an infrastructure to run test in webgl context but this could be quite complicated and would need to be evaluated if it's worth it
-  # To see where this job is included (in trigger job definitions) look into _triggers.yml file
-  # WebGL jobs were timing up more often in develop-2.0.0 branch (especially on trunk) so we increased the timeout from 1800 to 3600. We should investigate why it times more often in develop-2.0.0/ubuntu/trunk
+  # WebGL failures are infrequent and the build is slow, so this job is placed on Nightly/Weekly (QV) only, not on PR triggers.
+  # Runtime platform exclusions (RuntimePlatform.WebGLPlayer) are intentionally not pre-added; they are introduced reactively after the first real CI run.
+  # To see where this job is included (in trigger job definitions) look into _triggers.yml file.
+  # WebGL jobs were timing up more often in develop-2.0.0 branch (especially on trunk) so we increased the timeout from 1800 to 3600. We should investigate why it times more often in develop-2.0.0/ubuntu/trunk.
 
 #--------------------------------------------------------------------------------------
 
+
+# BUILD PHASE CONFIGURATION------------------------------------------------------------------------------------
 {% for project in projects.default -%}
 {% for platform in test_platforms.desktop -%}
 {% for editor in validation_editors.all -%}
@@ -47,10 +55,37 @@ webgl_build_{{ project.name }}_{{ platform.name }}_{{ editor }}:
     # This will perform only building phase (--build-only) with a timeout set by the test_timeout variable in the project metafile.
     - UnifiedTestRunner --suite=playmode --platform=WebGL --scripting-backend=il2cpp --testproject={{ project.path }} --editor-location=.Editor --artifacts_path=artifacts --player-save-path=build/players --extra-editor-arg=-batchmode --extra-editor-arg=-nographics --reruncount=1 --clean-library-on-rerun --build-only --timeout={{ test_timeout }}
   artifacts:
+    players:
+      paths:
+        - "build/players/**/*"
     logs:
       paths:
         - "artifacts/**/*"
-        - "build/players/**/*"
+{% endfor -%}
+{% endfor -%}
+{% endfor -%}
+
+
+# RUN PHASE CONFIGURATION------------------------------------------------------------------------------------
+{% for project in projects.default -%}
+{% for platform in test_platforms.webgl_test -%}
+{% for editor in validation_editors.all -%}
+webgl_test_{{ project.name }}_{{ platform.name }}_{{ editor }}:
+  name: WebGl Test - {{ project.name }} [{{ platform.name }}, {{ editor }}, il2cpp]
+  agent:
+    type: {{ platform.type }}
+    image: {{ platform.image }}
+    flavor: {{ platform.flavor }}
+    model: {{ platform.model }} # GPU model required to run a browser (more information in project.metafile)
+  commands:
+    - unity-downloader-cli --fast --wait -u {{ editor }} -c Editor -c webgl -c il2cpp # Downloading the editor with additional webgl and il2cpp components
+    - UnifiedTestRunner --suite=playmode --platform=WebGL --scripting-backend=il2cpp --testproject={{ project.path }} --editor-location=.Editor --artifacts_path=artifacts --player-load-path=build/players --target-browser=Firefox --target-browser-executable=/usr/bin/firefox --player-connection-ip=${BOKKEN_HOST_IP} --reruncount=1 --clean-library-on-rerun --timeout={{ test_timeout }}
+  artifacts:
+    logs:
+      paths:
+        - "artifacts/**/*"
+  dependencies:
+    - .yamato/webgl-test.yml#webgl_build_{{ project.name }}_ubuntu_{{ editor }}
 {% endfor -%}
 {% endfor -%}
 {% endfor -%}