smoke: isolate and loop flaky Chat Sessions / Agents Window tests [do not merge]

.github/workflows/pr-darwin-test.yml

@@ -189,8 +189,14 @@ jobs:
 
       - name: 🧪 Run smoke tests (Electron)
         if: ${{ inputs.electron_tests && inputs.smoke_tests }}
-        timeout-minutes: 20
-        run: npm run smoketest-no-compile -- --tracing
+        timeout-minutes: 120
+        run: |
+          # NOTE: Temporarily looping smoke tests to flush out flaky Chat Sessions / Agents Window tests. Do not merge.
+          for i in $(seq 1 10); do
+            echo "::group::Smoke test run $i of 10"
+            npm run smoketest-no-compile -- --tracing
+            echo "::endgroup::"
+          done
 
       - name: 🧪 Run smoke tests (Browser, Chromium)
         if: ${{ inputs.browser_tests && inputs.smoke_tests }}

.github/workflows/pr-linux-test.yml

@@ -367,8 +367,14 @@ jobs:
 
       - name: 🧪 Run smoke tests (Electron)
         if: ${{ inputs.electron_tests && inputs.smoke_tests }}
-        timeout-minutes: 20
-        run: npm run smoketest-no-compile -- --tracing
+        timeout-minutes: 120
+        run: |
+          # NOTE: Temporarily looping smoke tests to flush out flaky Chat Sessions / Agents Window tests. Do not merge.
+          for i in $(seq 1 10); do
+            echo "::group::Smoke test run $i of 10"
+            npm run smoketest-no-compile -- --tracing
+            echo "::endgroup::"
+          done
         env:
           DISPLAY: ":10"
 

.github/workflows/pr-win32-test.yml

@@ -220,9 +220,16 @@ jobs:
 
       - name: 🧪 Run smoke tests (Electron)
         if: ${{ inputs.electron_tests && inputs.smoke_tests }}
-        timeout-minutes: 20
+        timeout-minutes: 120
         shell: pwsh
-        run: npm run smoketest-no-compile -- --tracing
+        run: |
+          # NOTE: Temporarily looping smoke tests to flush out flaky Chat Sessions / Agents Window tests. Do not merge.
+          for ($i = 1; $i -le 10; $i++) {
+            Write-Host "::group::Smoke test run $i of 10"
+            npm run smoketest-no-compile -- --tracing
+            if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
+            Write-Host "::endgroup::"
+          }
 
       - name: 🧪 Run smoke tests (Browser, Chromium)
         if: ${{ inputs.browser_tests && inputs.smoke_tests }}

build/azure-pipelines/darwin/steps/product-build-darwin-test.yml

@@ -107,8 +107,13 @@ steps:
         set -e
         APP_ROOT=$(agent.builddirectory)/VSCode-darwin-$(VSCODE_ARCH)
         APP_NAME="`ls $APP_ROOT | head -n 1`"
-        npm run smoketest-no-compile -- --tracing --build "$APP_ROOT/$APP_NAME"
-      timeoutInMinutes: 20
+        # NOTE: Temporarily looping smoke tests to flush out flaky Chat Sessions / Agents Window tests. Do not merge.
+        for i in $(seq 1 10); do
+          echo "##[group]Smoke test run $i of 10"
+          npm run smoketest-no-compile -- --tracing --build "$APP_ROOT/$APP_NAME"
+          echo "##[endgroup]"
+        done
+      timeoutInMinutes: 120
       displayName: 🧪 Run smoke tests (Electron)
 
   - ${{ if eq(parameters.VSCODE_RUN_BROWSER_TESTS, true) }}:

build/azure-pipelines/linux/steps/product-build-linux-test.yml

@@ -119,8 +119,13 @@ steps:
   - ${{ if eq(parameters.VSCODE_RUN_ELECTRON_TESTS, true) }}:
     - script: |
         set -e
-        npm run smoketest-no-compile -- --tracing --build "$(agent.builddirectory)/VSCode-linux-$(VSCODE_ARCH)"
-      timeoutInMinutes: 20
+        # NOTE: Temporarily looping smoke tests to flush out flaky Chat Sessions / Agents Window tests. Do not merge.
+        for i in $(seq 1 10); do
+          echo "##[group]Smoke test run $i of 10"
+          npm run smoketest-no-compile -- --tracing --build "$(agent.builddirectory)/VSCode-linux-$(VSCODE_ARCH)"
+          echo "##[endgroup]"
+        done
+      timeoutInMinutes: 120
       displayName: 🧪 Run smoke tests (Electron)
 
   - ${{ if eq(parameters.VSCODE_RUN_BROWSER_TESTS, true) }}:

build/azure-pipelines/win32/steps/product-build-win32-test.yml

@@ -130,9 +130,15 @@ steps:
 
   - ${{ if eq(parameters.VSCODE_RUN_ELECTRON_TESTS, true) }}:
     - powershell: |
-        npm run smoketest-no-compile -- --tracing --build "$(agent.builddirectory)\test\VSCode-win32-$(VSCODE_ARCH)"
+        # NOTE: Temporarily looping smoke tests to flush out flaky Chat Sessions / Agents Window tests. Do not merge.
+        for ($i = 1; $i -le 10; $i++) {
+          Write-Host "##[group]Smoke test run $i of 10"
+          npm run smoketest-no-compile -- --tracing --build "$(agent.builddirectory)\test\VSCode-win32-$(VSCODE_ARCH)"
+          if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
+          Write-Host "##[endgroup]"
+        }
       displayName: 🧪 Run smoke tests (Electron)
-      timeoutInMinutes: 20
+      timeoutInMinutes: 120
 
   - ${{ if eq(parameters.VSCODE_RUN_BROWSER_TESTS, true) }}:
     - powershell: npm run smoketest-no-compile -- --web --tracing --headless

test/smoke/src/main.ts

@@ -14,23 +14,24 @@ import fetch from 'node-fetch';
 import { Quality, MultiLogger, Logger, ConsoleLogger, FileLogger, measureAndLog, getDevElectronPath, getBuildElectronPath, getBuildVersion, ApplicationOptions } from '../../automation';
 import { retry } from './utils';
 
-import { setup as setupDataLossTests } from './areas/workbench/data-loss.test';
-import { setup as setupPreferencesTests } from './areas/preferences/preferences.test';
-import { setup as setupSearchTests } from './areas/search/search.test';
-import { setup as setupNotebookTests } from './areas/notebook/notebook.test';
-import { setup as setupLanguagesTests } from './areas/languages/languages.test';
-import { setup as setupStatusbarTests } from './areas/statusbar/statusbar.test';
-import { setup as setupExtensionTests } from './areas/extensions/extensions.test';
-import { setup as setupExtensionHostRestartTests } from './areas/extensions/extension-host-restart.test';
-import { setup as setupMultirootTests } from './areas/multiroot/multiroot.test';
-import { setup as setupLocalizationTests } from './areas/workbench/localization.test';
-import { setup as setupLaunchTests } from './areas/workbench/launch.test';
-import { setup as setupTerminalTests } from './areas/terminal/terminal.test';
-import { setup as setupTaskTests } from './areas/task/task.test';
-import { setup as setupChatTests } from './areas/chat/chatDisabled.test';
-import { setup as setupCopilotCliTests } from './areas/chat/copilotCli.test';
+// NOTE: Temporarily disabled to flush out flaky Chat Sessions / Agents Window smoke tests. Do not merge.
+// import { setup as setupDataLossTests } from './areas/workbench/data-loss.test';
+// import { setup as setupPreferencesTests } from './areas/preferences/preferences.test';
+// import { setup as setupSearchTests } from './areas/search/search.test';
+// import { setup as setupNotebookTests } from './areas/notebook/notebook.test';
+// import { setup as setupLanguagesTests } from './areas/languages/languages.test';
+// import { setup as setupStatusbarTests } from './areas/statusbar/statusbar.test';
+// import { setup as setupExtensionTests } from './areas/extensions/extensions.test';
+// import { setup as setupExtensionHostRestartTests } from './areas/extensions/extension-host-restart.test';
+// import { setup as setupMultirootTests } from './areas/multiroot/multiroot.test';
+// import { setup as setupLocalizationTests } from './areas/workbench/localization.test';
+// import { setup as setupLaunchTests } from './areas/workbench/launch.test';
+// import { setup as setupTerminalTests } from './areas/terminal/terminal.test';
+// import { setup as setupTaskTests } from './areas/task/task.test';
+// import { setup as setupChatTests } from './areas/chat/chatDisabled.test';
+// import { setup as setupCopilotCliTests } from './areas/chat/copilotCli.test';
 import { setup as setupChatSessionsTests } from './areas/chat/chatSessions.test';
-import { setup as setupAccessibilityTests } from './areas/accessibility/accessibility.test';
+// import { setup as setupAccessibilityTests } from './areas/accessibility/accessibility.test';
 import { setup as setupAgentsWindowTests } from './areas/agentsWindow/agentsWindow.test';
 
 const rootPath = path.join(__dirname, '..', '..', '..');
@@ -407,22 +408,23 @@ after(async function () {
 });
 
 describe(`VSCode Smoke Tests (${opts.web ? 'Web' : 'Electron'})`, () => {
-	if (!opts.web) { setupDataLossTests(() => { return { stableCodePath: opts['stable-build'], stableCodeVersion: opts['stable-version'] } /* Do not change, deferred for a reason! */; }, logger); }
-	setupPreferencesTests(logger);
-	setupSearchTests(logger);
-	if (!opts.web) { setupNotebookTests(logger); }
-	setupLanguagesTests(logger);
-	setupTerminalTests(logger);
-	setupTaskTests(logger);
-	setupStatusbarTests(logger);
-	if (quality !== Quality.Dev && quality !== Quality.OSS) { setupExtensionTests(logger); }
-	if (!opts.web && !opts.remote) { setupExtensionHostRestartTests(logger); }
-	if (!(opts.web && process.platform === 'win32' /* TODO@bpasero flaky */)) { setupMultirootTests(logger); }
-	if (!opts.web && !opts.remote && quality !== Quality.Dev && quality !== Quality.OSS) { setupLocalizationTests(logger); }
-	if (!opts.web && !opts.remote) { setupLaunchTests(logger); }
-	if (!opts.web) { setupChatTests(logger); }
-	if (!opts.web && !opts.remote && quality !== Quality.Dev && quality !== Quality.OSS) { setupCopilotCliTests(logger); }
+	// NOTE: Temporarily disabled to flush out flaky Chat Sessions / Agents Window smoke tests. Do not merge.
+	// if (!opts.web) { setupDataLossTests(() => { return { stableCodePath: opts['stable-build'], stableCodeVersion: opts['stable-version'] } /* Do not change, deferred for a reason! */; }, logger); }
+	// setupPreferencesTests(logger);
+	// setupSearchTests(logger);
+	// if (!opts.web) { setupNotebookTests(logger); }
+	// setupLanguagesTests(logger);
+	// setupTerminalTests(logger);
+	// setupTaskTests(logger);
+	// setupStatusbarTests(logger);
+	// if (quality !== Quality.Dev && quality !== Quality.OSS) { setupExtensionTests(logger); }
+	// if (!opts.web && !opts.remote) { setupExtensionHostRestartTests(logger); }
+	// if (!(opts.web && process.platform === 'win32' /* TODO@bpasero flaky */)) { setupMultirootTests(logger); }
+	// if (!opts.web && !opts.remote && quality !== Quality.Dev && quality !== Quality.OSS) { setupLocalizationTests(logger); }
+	// if (!opts.web && !opts.remote) { setupLaunchTests(logger); }
+	// if (!opts.web) { setupChatTests(logger); }
+	// if (!opts.web && !opts.remote && quality !== Quality.Dev && quality !== Quality.OSS) { setupCopilotCliTests(logger); }
 	if (!opts.web && !opts.remote) { setupChatSessionsTests(logger); }
 	if (!opts.web && !opts.remote) { setupAgentsWindowTests(logger); }
-	setupAccessibilityTests(logger, opts, quality);
+	// setupAccessibilityTests(logger, opts, quality);
 });