diff --git a/.github/workflows/pr-darwin-test.yml b/.github/workflows/pr-darwin-test.yml index 971fda8191f1e..97e184cc9f4b1 100644 --- a/.github/workflows/pr-darwin-test.yml +++ b/.github/workflows/pr-darwin-test.yml @@ -189,8 +189,14 @@ jobs: - name: 🧪 Run smoke tests (Electron) if: ${{ inputs.electron_tests && inputs.smoke_tests }} - timeout-minutes: 20 - run: npm run smoketest-no-compile -- --tracing + timeout-minutes: 120 + run: | + # NOTE: Temporarily looping smoke tests to flush out flaky Chat Sessions / Agents Window tests. Do not merge. + for i in $(seq 1 10); do + echo "::group::Smoke test run $i of 10" + npm run smoketest-no-compile -- --tracing + echo "::endgroup::" + done - name: 🧪 Run smoke tests (Browser, Chromium) if: ${{ inputs.browser_tests && inputs.smoke_tests }} diff --git a/.github/workflows/pr-linux-test.yml b/.github/workflows/pr-linux-test.yml index 113d41cdda3bb..ddb8b4b1d4574 100644 --- a/.github/workflows/pr-linux-test.yml +++ b/.github/workflows/pr-linux-test.yml @@ -367,8 +367,14 @@ jobs: - name: 🧪 Run smoke tests (Electron) if: ${{ inputs.electron_tests && inputs.smoke_tests }} - timeout-minutes: 20 - run: npm run smoketest-no-compile -- --tracing + timeout-minutes: 120 + run: | + # NOTE: Temporarily looping smoke tests to flush out flaky Chat Sessions / Agents Window tests. Do not merge. + for i in $(seq 1 10); do + echo "::group::Smoke test run $i of 10" + npm run smoketest-no-compile -- --tracing + echo "::endgroup::" + done env: DISPLAY: ":10" diff --git a/.github/workflows/pr-win32-test.yml b/.github/workflows/pr-win32-test.yml index 45681aff7077b..ebbfe21819167 100644 --- a/.github/workflows/pr-win32-test.yml +++ b/.github/workflows/pr-win32-test.yml @@ -220,9 +220,16 @@ jobs: - name: 🧪 Run smoke tests (Electron) if: ${{ inputs.electron_tests && inputs.smoke_tests }} - timeout-minutes: 20 + timeout-minutes: 120 shell: pwsh - run: npm run smoketest-no-compile -- --tracing + run: | + # NOTE: Temporarily looping smoke tests to flush out flaky Chat Sessions / Agents Window tests. Do not merge. + for ($i = 1; $i -le 10; $i++) { + Write-Host "::group::Smoke test run $i of 10" + npm run smoketest-no-compile -- --tracing + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + Write-Host "::endgroup::" + } - name: 🧪 Run smoke tests (Browser, Chromium) if: ${{ inputs.browser_tests && inputs.smoke_tests }} diff --git a/build/azure-pipelines/darwin/steps/product-build-darwin-test.yml b/build/azure-pipelines/darwin/steps/product-build-darwin-test.yml index 2cb75de8dddd6..e54ee1e3e5eb9 100644 --- a/build/azure-pipelines/darwin/steps/product-build-darwin-test.yml +++ b/build/azure-pipelines/darwin/steps/product-build-darwin-test.yml @@ -107,8 +107,13 @@ steps: set -e APP_ROOT=$(agent.builddirectory)/VSCode-darwin-$(VSCODE_ARCH) APP_NAME="`ls $APP_ROOT | head -n 1`" - npm run smoketest-no-compile -- --tracing --build "$APP_ROOT/$APP_NAME" - timeoutInMinutes: 20 + # NOTE: Temporarily looping smoke tests to flush out flaky Chat Sessions / Agents Window tests. Do not merge. + for i in $(seq 1 10); do + echo "##[group]Smoke test run $i of 10" + npm run smoketest-no-compile -- --tracing --build "$APP_ROOT/$APP_NAME" + echo "##[endgroup]" + done + timeoutInMinutes: 120 displayName: 🧪 Run smoke tests (Electron) - ${{ if eq(parameters.VSCODE_RUN_BROWSER_TESTS, true) }}: diff --git a/build/azure-pipelines/linux/steps/product-build-linux-test.yml b/build/azure-pipelines/linux/steps/product-build-linux-test.yml index 5c6223cf371d8..c28a377e78317 100644 --- a/build/azure-pipelines/linux/steps/product-build-linux-test.yml +++ b/build/azure-pipelines/linux/steps/product-build-linux-test.yml @@ -119,8 +119,13 @@ steps: - ${{ if eq(parameters.VSCODE_RUN_ELECTRON_TESTS, true) }}: - script: | set -e - npm run smoketest-no-compile -- --tracing --build "$(agent.builddirectory)/VSCode-linux-$(VSCODE_ARCH)" - timeoutInMinutes: 20 + # NOTE: Temporarily looping smoke tests to flush out flaky Chat Sessions / Agents Window tests. Do not merge. + for i in $(seq 1 10); do + echo "##[group]Smoke test run $i of 10" + npm run smoketest-no-compile -- --tracing --build "$(agent.builddirectory)/VSCode-linux-$(VSCODE_ARCH)" + echo "##[endgroup]" + done + timeoutInMinutes: 120 displayName: 🧪 Run smoke tests (Electron) - ${{ if eq(parameters.VSCODE_RUN_BROWSER_TESTS, true) }}: diff --git a/build/azure-pipelines/win32/steps/product-build-win32-test.yml b/build/azure-pipelines/win32/steps/product-build-win32-test.yml index 187856b227229..ec4624039d491 100644 --- a/build/azure-pipelines/win32/steps/product-build-win32-test.yml +++ b/build/azure-pipelines/win32/steps/product-build-win32-test.yml @@ -130,9 +130,15 @@ steps: - ${{ if eq(parameters.VSCODE_RUN_ELECTRON_TESTS, true) }}: - powershell: | - npm run smoketest-no-compile -- --tracing --build "$(agent.builddirectory)\test\VSCode-win32-$(VSCODE_ARCH)" + # NOTE: Temporarily looping smoke tests to flush out flaky Chat Sessions / Agents Window tests. Do not merge. + for ($i = 1; $i -le 10; $i++) { + Write-Host "##[group]Smoke test run $i of 10" + npm run smoketest-no-compile -- --tracing --build "$(agent.builddirectory)\test\VSCode-win32-$(VSCODE_ARCH)" + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + Write-Host "##[endgroup]" + } displayName: 🧪 Run smoke tests (Electron) - timeoutInMinutes: 20 + timeoutInMinutes: 120 - ${{ if eq(parameters.VSCODE_RUN_BROWSER_TESTS, true) }}: - powershell: npm run smoketest-no-compile -- --web --tracing --headless diff --git a/test/smoke/src/main.ts b/test/smoke/src/main.ts index d51710a0a87b8..bcd4c14b4832a 100644 --- a/test/smoke/src/main.ts +++ b/test/smoke/src/main.ts @@ -14,23 +14,24 @@ import fetch from 'node-fetch'; import { Quality, MultiLogger, Logger, ConsoleLogger, FileLogger, measureAndLog, getDevElectronPath, getBuildElectronPath, getBuildVersion, ApplicationOptions } from '../../automation'; import { retry } from './utils'; -import { setup as setupDataLossTests } from './areas/workbench/data-loss.test'; -import { setup as setupPreferencesTests } from './areas/preferences/preferences.test'; -import { setup as setupSearchTests } from './areas/search/search.test'; -import { setup as setupNotebookTests } from './areas/notebook/notebook.test'; -import { setup as setupLanguagesTests } from './areas/languages/languages.test'; -import { setup as setupStatusbarTests } from './areas/statusbar/statusbar.test'; -import { setup as setupExtensionTests } from './areas/extensions/extensions.test'; -import { setup as setupExtensionHostRestartTests } from './areas/extensions/extension-host-restart.test'; -import { setup as setupMultirootTests } from './areas/multiroot/multiroot.test'; -import { setup as setupLocalizationTests } from './areas/workbench/localization.test'; -import { setup as setupLaunchTests } from './areas/workbench/launch.test'; -import { setup as setupTerminalTests } from './areas/terminal/terminal.test'; -import { setup as setupTaskTests } from './areas/task/task.test'; -import { setup as setupChatTests } from './areas/chat/chatDisabled.test'; -import { setup as setupCopilotCliTests } from './areas/chat/copilotCli.test'; +// NOTE: Temporarily disabled to flush out flaky Chat Sessions / Agents Window smoke tests. Do not merge. +// import { setup as setupDataLossTests } from './areas/workbench/data-loss.test'; +// import { setup as setupPreferencesTests } from './areas/preferences/preferences.test'; +// import { setup as setupSearchTests } from './areas/search/search.test'; +// import { setup as setupNotebookTests } from './areas/notebook/notebook.test'; +// import { setup as setupLanguagesTests } from './areas/languages/languages.test'; +// import { setup as setupStatusbarTests } from './areas/statusbar/statusbar.test'; +// import { setup as setupExtensionTests } from './areas/extensions/extensions.test'; +// import { setup as setupExtensionHostRestartTests } from './areas/extensions/extension-host-restart.test'; +// import { setup as setupMultirootTests } from './areas/multiroot/multiroot.test'; +// import { setup as setupLocalizationTests } from './areas/workbench/localization.test'; +// import { setup as setupLaunchTests } from './areas/workbench/launch.test'; +// import { setup as setupTerminalTests } from './areas/terminal/terminal.test'; +// import { setup as setupTaskTests } from './areas/task/task.test'; +// import { setup as setupChatTests } from './areas/chat/chatDisabled.test'; +// import { setup as setupCopilotCliTests } from './areas/chat/copilotCli.test'; import { setup as setupChatSessionsTests } from './areas/chat/chatSessions.test'; -import { setup as setupAccessibilityTests } from './areas/accessibility/accessibility.test'; +// import { setup as setupAccessibilityTests } from './areas/accessibility/accessibility.test'; import { setup as setupAgentsWindowTests } from './areas/agentsWindow/agentsWindow.test'; const rootPath = path.join(__dirname, '..', '..', '..'); @@ -407,22 +408,23 @@ after(async function () { }); describe(`VSCode Smoke Tests (${opts.web ? 'Web' : 'Electron'})`, () => { - if (!opts.web) { setupDataLossTests(() => { return { stableCodePath: opts['stable-build'], stableCodeVersion: opts['stable-version'] } /* Do not change, deferred for a reason! */; }, logger); } - setupPreferencesTests(logger); - setupSearchTests(logger); - if (!opts.web) { setupNotebookTests(logger); } - setupLanguagesTests(logger); - setupTerminalTests(logger); - setupTaskTests(logger); - setupStatusbarTests(logger); - if (quality !== Quality.Dev && quality !== Quality.OSS) { setupExtensionTests(logger); } - if (!opts.web && !opts.remote) { setupExtensionHostRestartTests(logger); } - if (!(opts.web && process.platform === 'win32' /* TODO@bpasero flaky */)) { setupMultirootTests(logger); } - if (!opts.web && !opts.remote && quality !== Quality.Dev && quality !== Quality.OSS) { setupLocalizationTests(logger); } - if (!opts.web && !opts.remote) { setupLaunchTests(logger); } - if (!opts.web) { setupChatTests(logger); } - if (!opts.web && !opts.remote && quality !== Quality.Dev && quality !== Quality.OSS) { setupCopilotCliTests(logger); } + // NOTE: Temporarily disabled to flush out flaky Chat Sessions / Agents Window smoke tests. Do not merge. + // if (!opts.web) { setupDataLossTests(() => { return { stableCodePath: opts['stable-build'], stableCodeVersion: opts['stable-version'] } /* Do not change, deferred for a reason! */; }, logger); } + // setupPreferencesTests(logger); + // setupSearchTests(logger); + // if (!opts.web) { setupNotebookTests(logger); } + // setupLanguagesTests(logger); + // setupTerminalTests(logger); + // setupTaskTests(logger); + // setupStatusbarTests(logger); + // if (quality !== Quality.Dev && quality !== Quality.OSS) { setupExtensionTests(logger); } + // if (!opts.web && !opts.remote) { setupExtensionHostRestartTests(logger); } + // if (!(opts.web && process.platform === 'win32' /* TODO@bpasero flaky */)) { setupMultirootTests(logger); } + // if (!opts.web && !opts.remote && quality !== Quality.Dev && quality !== Quality.OSS) { setupLocalizationTests(logger); } + // if (!opts.web && !opts.remote) { setupLaunchTests(logger); } + // if (!opts.web) { setupChatTests(logger); } + // if (!opts.web && !opts.remote && quality !== Quality.Dev && quality !== Quality.OSS) { setupCopilotCliTests(logger); } if (!opts.web && !opts.remote) { setupChatSessionsTests(logger); } if (!opts.web && !opts.remote) { setupAgentsWindowTests(logger); } - setupAccessibilityTests(logger, opts, quality); + // setupAccessibilityTests(logger, opts, quality); });