From a94af95589b925d549914b9c030688c2bae21bc3 Mon Sep 17 00:00:00 2001 From: Alex Dima Date: Sat, 20 Jun 2026 23:49:44 +0200 Subject: [PATCH 1/2] smoke: isolate and loop flaky Chat Sessions / Agents Window tests Temporary, do-not-merge investigation change to flush out smoke test flakiness. Disables all smoke suites except Chat Sessions and Agents Window in main.ts, and loops the Electron smoke test step 10 times on Linux, macOS and Windows so a single flaky failure fails the build while preserving uploaded logs/traces. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../steps/product-build-darwin-test.yml | 9 ++- .../linux/steps/product-build-linux-test.yml | 9 ++- .../win32/steps/product-build-win32-test.yml | 10 ++- test/smoke/src/main.ts | 66 ++++++++++--------- 4 files changed, 56 insertions(+), 38 deletions(-) diff --git a/build/azure-pipelines/darwin/steps/product-build-darwin-test.yml b/build/azure-pipelines/darwin/steps/product-build-darwin-test.yml index 2cb75de8dddd6..e54ee1e3e5eb9 100644 --- a/build/azure-pipelines/darwin/steps/product-build-darwin-test.yml +++ b/build/azure-pipelines/darwin/steps/product-build-darwin-test.yml @@ -107,8 +107,13 @@ steps: set -e APP_ROOT=$(agent.builddirectory)/VSCode-darwin-$(VSCODE_ARCH) APP_NAME="`ls $APP_ROOT | head -n 1`" - npm run smoketest-no-compile -- --tracing --build "$APP_ROOT/$APP_NAME" - timeoutInMinutes: 20 + # NOTE: Temporarily looping smoke tests to flush out flaky Chat Sessions / Agents Window tests. Do not merge. + for i in $(seq 1 10); do + echo "##[group]Smoke test run $i of 10" + npm run smoketest-no-compile -- --tracing --build "$APP_ROOT/$APP_NAME" + echo "##[endgroup]" + done + timeoutInMinutes: 120 displayName: 🧪 Run smoke tests (Electron) - ${{ if eq(parameters.VSCODE_RUN_BROWSER_TESTS, true) }}: diff --git a/build/azure-pipelines/linux/steps/product-build-linux-test.yml b/build/azure-pipelines/linux/steps/product-build-linux-test.yml index 5c6223cf371d8..c28a377e78317 100644 --- a/build/azure-pipelines/linux/steps/product-build-linux-test.yml +++ b/build/azure-pipelines/linux/steps/product-build-linux-test.yml @@ -119,8 +119,13 @@ steps: - ${{ if eq(parameters.VSCODE_RUN_ELECTRON_TESTS, true) }}: - script: | set -e - npm run smoketest-no-compile -- --tracing --build "$(agent.builddirectory)/VSCode-linux-$(VSCODE_ARCH)" - timeoutInMinutes: 20 + # NOTE: Temporarily looping smoke tests to flush out flaky Chat Sessions / Agents Window tests. Do not merge. + for i in $(seq 1 10); do + echo "##[group]Smoke test run $i of 10" + npm run smoketest-no-compile -- --tracing --build "$(agent.builddirectory)/VSCode-linux-$(VSCODE_ARCH)" + echo "##[endgroup]" + done + timeoutInMinutes: 120 displayName: 🧪 Run smoke tests (Electron) - ${{ if eq(parameters.VSCODE_RUN_BROWSER_TESTS, true) }}: diff --git a/build/azure-pipelines/win32/steps/product-build-win32-test.yml b/build/azure-pipelines/win32/steps/product-build-win32-test.yml index 187856b227229..ec4624039d491 100644 --- a/build/azure-pipelines/win32/steps/product-build-win32-test.yml +++ b/build/azure-pipelines/win32/steps/product-build-win32-test.yml @@ -130,9 +130,15 @@ steps: - ${{ if eq(parameters.VSCODE_RUN_ELECTRON_TESTS, true) }}: - powershell: | - npm run smoketest-no-compile -- --tracing --build "$(agent.builddirectory)\test\VSCode-win32-$(VSCODE_ARCH)" + # NOTE: Temporarily looping smoke tests to flush out flaky Chat Sessions / Agents Window tests. Do not merge. + for ($i = 1; $i -le 10; $i++) { + Write-Host "##[group]Smoke test run $i of 10" + npm run smoketest-no-compile -- --tracing --build "$(agent.builddirectory)\test\VSCode-win32-$(VSCODE_ARCH)" + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + Write-Host "##[endgroup]" + } displayName: 🧪 Run smoke tests (Electron) - timeoutInMinutes: 20 + timeoutInMinutes: 120 - ${{ if eq(parameters.VSCODE_RUN_BROWSER_TESTS, true) }}: - powershell: npm run smoketest-no-compile -- --web --tracing --headless diff --git a/test/smoke/src/main.ts b/test/smoke/src/main.ts index d51710a0a87b8..bcd4c14b4832a 100644 --- a/test/smoke/src/main.ts +++ b/test/smoke/src/main.ts @@ -14,23 +14,24 @@ import fetch from 'node-fetch'; import { Quality, MultiLogger, Logger, ConsoleLogger, FileLogger, measureAndLog, getDevElectronPath, getBuildElectronPath, getBuildVersion, ApplicationOptions } from '../../automation'; import { retry } from './utils'; -import { setup as setupDataLossTests } from './areas/workbench/data-loss.test'; -import { setup as setupPreferencesTests } from './areas/preferences/preferences.test'; -import { setup as setupSearchTests } from './areas/search/search.test'; -import { setup as setupNotebookTests } from './areas/notebook/notebook.test'; -import { setup as setupLanguagesTests } from './areas/languages/languages.test'; -import { setup as setupStatusbarTests } from './areas/statusbar/statusbar.test'; -import { setup as setupExtensionTests } from './areas/extensions/extensions.test'; -import { setup as setupExtensionHostRestartTests } from './areas/extensions/extension-host-restart.test'; -import { setup as setupMultirootTests } from './areas/multiroot/multiroot.test'; -import { setup as setupLocalizationTests } from './areas/workbench/localization.test'; -import { setup as setupLaunchTests } from './areas/workbench/launch.test'; -import { setup as setupTerminalTests } from './areas/terminal/terminal.test'; -import { setup as setupTaskTests } from './areas/task/task.test'; -import { setup as setupChatTests } from './areas/chat/chatDisabled.test'; -import { setup as setupCopilotCliTests } from './areas/chat/copilotCli.test'; +// NOTE: Temporarily disabled to flush out flaky Chat Sessions / Agents Window smoke tests. Do not merge. +// import { setup as setupDataLossTests } from './areas/workbench/data-loss.test'; +// import { setup as setupPreferencesTests } from './areas/preferences/preferences.test'; +// import { setup as setupSearchTests } from './areas/search/search.test'; +// import { setup as setupNotebookTests } from './areas/notebook/notebook.test'; +// import { setup as setupLanguagesTests } from './areas/languages/languages.test'; +// import { setup as setupStatusbarTests } from './areas/statusbar/statusbar.test'; +// import { setup as setupExtensionTests } from './areas/extensions/extensions.test'; +// import { setup as setupExtensionHostRestartTests } from './areas/extensions/extension-host-restart.test'; +// import { setup as setupMultirootTests } from './areas/multiroot/multiroot.test'; +// import { setup as setupLocalizationTests } from './areas/workbench/localization.test'; +// import { setup as setupLaunchTests } from './areas/workbench/launch.test'; +// import { setup as setupTerminalTests } from './areas/terminal/terminal.test'; +// import { setup as setupTaskTests } from './areas/task/task.test'; +// import { setup as setupChatTests } from './areas/chat/chatDisabled.test'; +// import { setup as setupCopilotCliTests } from './areas/chat/copilotCli.test'; import { setup as setupChatSessionsTests } from './areas/chat/chatSessions.test'; -import { setup as setupAccessibilityTests } from './areas/accessibility/accessibility.test'; +// import { setup as setupAccessibilityTests } from './areas/accessibility/accessibility.test'; import { setup as setupAgentsWindowTests } from './areas/agentsWindow/agentsWindow.test'; const rootPath = path.join(__dirname, '..', '..', '..'); @@ -407,22 +408,23 @@ after(async function () { }); describe(`VSCode Smoke Tests (${opts.web ? 'Web' : 'Electron'})`, () => { - if (!opts.web) { setupDataLossTests(() => { return { stableCodePath: opts['stable-build'], stableCodeVersion: opts['stable-version'] } /* Do not change, deferred for a reason! */; }, logger); } - setupPreferencesTests(logger); - setupSearchTests(logger); - if (!opts.web) { setupNotebookTests(logger); } - setupLanguagesTests(logger); - setupTerminalTests(logger); - setupTaskTests(logger); - setupStatusbarTests(logger); - if (quality !== Quality.Dev && quality !== Quality.OSS) { setupExtensionTests(logger); } - if (!opts.web && !opts.remote) { setupExtensionHostRestartTests(logger); } - if (!(opts.web && process.platform === 'win32' /* TODO@bpasero flaky */)) { setupMultirootTests(logger); } - if (!opts.web && !opts.remote && quality !== Quality.Dev && quality !== Quality.OSS) { setupLocalizationTests(logger); } - if (!opts.web && !opts.remote) { setupLaunchTests(logger); } - if (!opts.web) { setupChatTests(logger); } - if (!opts.web && !opts.remote && quality !== Quality.Dev && quality !== Quality.OSS) { setupCopilotCliTests(logger); } + // NOTE: Temporarily disabled to flush out flaky Chat Sessions / Agents Window smoke tests. Do not merge. + // if (!opts.web) { setupDataLossTests(() => { return { stableCodePath: opts['stable-build'], stableCodeVersion: opts['stable-version'] } /* Do not change, deferred for a reason! */; }, logger); } + // setupPreferencesTests(logger); + // setupSearchTests(logger); + // if (!opts.web) { setupNotebookTests(logger); } + // setupLanguagesTests(logger); + // setupTerminalTests(logger); + // setupTaskTests(logger); + // setupStatusbarTests(logger); + // if (quality !== Quality.Dev && quality !== Quality.OSS) { setupExtensionTests(logger); } + // if (!opts.web && !opts.remote) { setupExtensionHostRestartTests(logger); } + // if (!(opts.web && process.platform === 'win32' /* TODO@bpasero flaky */)) { setupMultirootTests(logger); } + // if (!opts.web && !opts.remote && quality !== Quality.Dev && quality !== Quality.OSS) { setupLocalizationTests(logger); } + // if (!opts.web && !opts.remote) { setupLaunchTests(logger); } + // if (!opts.web) { setupChatTests(logger); } + // if (!opts.web && !opts.remote && quality !== Quality.Dev && quality !== Quality.OSS) { setupCopilotCliTests(logger); } if (!opts.web && !opts.remote) { setupChatSessionsTests(logger); } if (!opts.web && !opts.remote) { setupAgentsWindowTests(logger); } - setupAccessibilityTests(logger, opts, quality); + // setupAccessibilityTests(logger, opts, quality); }); From 865a295bd3d9d46ef76be66f002f900e2bf06085 Mon Sep 17 00:00:00 2001 From: Alex Dima Date: Sun, 21 Jun 2026 10:48:56 +0200 Subject: [PATCH 2/2] smoke: loop GitHub Actions Electron smoke step 10x The PR validation runs via GitHub Actions (.github/workflows/pr-*-test.yml), not the Azure Pipelines steps edited earlier. Apply the 10x loop and 120m timeout to the GitHub Actions Electron smoke step on Linux, macOS and Windows so the flaky-test investigation actually loops in PR CI. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/workflows/pr-darwin-test.yml | 10 ++++++++-- .github/workflows/pr-linux-test.yml | 10 ++++++++-- .github/workflows/pr-win32-test.yml | 11 +++++++++-- 3 files changed, 25 insertions(+), 6 deletions(-) diff --git a/.github/workflows/pr-darwin-test.yml b/.github/workflows/pr-darwin-test.yml index 971fda8191f1e..97e184cc9f4b1 100644 --- a/.github/workflows/pr-darwin-test.yml +++ b/.github/workflows/pr-darwin-test.yml @@ -189,8 +189,14 @@ jobs: - name: 🧪 Run smoke tests (Electron) if: ${{ inputs.electron_tests && inputs.smoke_tests }} - timeout-minutes: 20 - run: npm run smoketest-no-compile -- --tracing + timeout-minutes: 120 + run: | + # NOTE: Temporarily looping smoke tests to flush out flaky Chat Sessions / Agents Window tests. Do not merge. + for i in $(seq 1 10); do + echo "::group::Smoke test run $i of 10" + npm run smoketest-no-compile -- --tracing + echo "::endgroup::" + done - name: 🧪 Run smoke tests (Browser, Chromium) if: ${{ inputs.browser_tests && inputs.smoke_tests }} diff --git a/.github/workflows/pr-linux-test.yml b/.github/workflows/pr-linux-test.yml index 113d41cdda3bb..ddb8b4b1d4574 100644 --- a/.github/workflows/pr-linux-test.yml +++ b/.github/workflows/pr-linux-test.yml @@ -367,8 +367,14 @@ jobs: - name: 🧪 Run smoke tests (Electron) if: ${{ inputs.electron_tests && inputs.smoke_tests }} - timeout-minutes: 20 - run: npm run smoketest-no-compile -- --tracing + timeout-minutes: 120 + run: | + # NOTE: Temporarily looping smoke tests to flush out flaky Chat Sessions / Agents Window tests. Do not merge. + for i in $(seq 1 10); do + echo "::group::Smoke test run $i of 10" + npm run smoketest-no-compile -- --tracing + echo "::endgroup::" + done env: DISPLAY: ":10" diff --git a/.github/workflows/pr-win32-test.yml b/.github/workflows/pr-win32-test.yml index 45681aff7077b..ebbfe21819167 100644 --- a/.github/workflows/pr-win32-test.yml +++ b/.github/workflows/pr-win32-test.yml @@ -220,9 +220,16 @@ jobs: - name: 🧪 Run smoke tests (Electron) if: ${{ inputs.electron_tests && inputs.smoke_tests }} - timeout-minutes: 20 + timeout-minutes: 120 shell: pwsh - run: npm run smoketest-no-compile -- --tracing + run: | + # NOTE: Temporarily looping smoke tests to flush out flaky Chat Sessions / Agents Window tests. Do not merge. + for ($i = 1; $i -le 10; $i++) { + Write-Host "::group::Smoke test run $i of 10" + npm run smoketest-no-compile -- --tracing + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + Write-Host "::endgroup::" + } - name: 🧪 Run smoke tests (Browser, Chromium) if: ${{ inputs.browser_tests && inputs.smoke_tests }}