From f152835f16c1864f7e2cf657576ca3ff94cf8cad Mon Sep 17 00:00:00 2001 From: mehmet turac Date: Fri, 19 Jun 2026 15:19:37 +0300 Subject: [PATCH] fix(test runner): rerun last-failed dependency projects --- packages/playwright/src/runner/loadUtils.ts | 20 ++ tests/playwright-test/deps.spec.ts | 233 ++++++++++++++++++++ 2 files changed, 253 insertions(+) diff --git a/packages/playwright/src/runner/loadUtils.ts b/packages/playwright/src/runner/loadUtils.ts index a2cce3b386459..ecde7ab563f6e 100644 --- a/packages/playwright/src/runner/loadUtils.ts +++ b/packages/playwright/src/runner/loadUtils.ts @@ -147,6 +147,26 @@ export async function createRootSuite(testRun: TestRun, errors: TestError[], sho } } + if (testRun.options.lastFailedTestIds?.length) { + const failedTestIds = new Set(testRun.options.lastFailedTestIds); + const projectsWithLastFailedTests: commonConfig.FullProjectInternal[] = []; + for (const [project, filteredProjectSuite] of filteredProjectSuites) { + if (filteredProjectSuite.allTests().some(test => failedTestIds.has(test.id))) + projectsWithLastFailedTests.push(project); + } + + if (projectsWithLastFailedTests.length) { + const lastFailedProjectClosure = buildProjectsClosure(projectsWithLastFailedTests, project => filteredProjectSuites.get(project)!._hasTests()); + const topLevelProjectsWithLastFailedTests = new Set([...lastFailedProjectClosure] + .filter(([, type]) => type === 'top-level') + .map(([project]) => project)); + for (const project of filteredProjectSuites.keys()) { + if (!topLevelProjectsWithLastFailedTests.has(project)) + filteredProjectSuites.delete(project); + } + } + } + // Add post-filtered top-level projects to the root suite for sharding and 'only' processing. const projectClosure = buildProjectsClosure([...filteredProjectSuites.keys()], project => filteredProjectSuites.get(project)!._hasTests()); for (const [project, type] of projectClosure) { diff --git a/tests/playwright-test/deps.spec.ts b/tests/playwright-test/deps.spec.ts index 26417bd7bc9b0..05faa3580b8d8 100644 --- a/tests/playwright-test/deps.spec.ts +++ b/tests/playwright-test/deps.spec.ts @@ -38,6 +38,239 @@ test('should run projects with dependencies', async ({ runInlineTest }) => { expect(result.outputLines).toEqual(['A', 'B', 'C']); }); +test('should run last failed tests from a dependency project', async ({ runInlineTest }) => { + const files = { + 'playwright.config.ts': ` + module.exports = { + projects: [ + { name: 'setup', testMatch: /setup.spec.ts/ }, + { name: 'app', testMatch: /app.spec.ts/, dependencies: ['setup'] }, + ], + };`, + 'setup.spec.ts': ` + import { test, expect } from '@playwright/test'; + test('setup passes', async () => { + console.log('\\n%%setup passes'); + }); + test('setup fails 1', async () => { + console.log('\\n%%setup fails 1'); + expect(1).toBe(2); + }); + test('setup fails 2', async () => { + console.log('\\n%%setup fails 2'); + expect(1).toBe(2); + }); + `, + 'app.spec.ts': ` + import { test, expect } from '@playwright/test'; + test('app passes', async () => { + console.log('\\n%%app passes'); + }); + `, + }; + const result1 = await runInlineTest(files, { workers: 1 }); + expect(result1.exitCode).toBe(1); + expect(result1.passed).toBe(1); + expect(result1.failed).toBe(2); + expect(result1.didNotRun).toBe(1); + expect(result1.outputLines).toEqual(['setup passes', 'setup fails 1', 'setup fails 2']); + + const result2 = await runInlineTest(files, { workers: 1 }, {}, { additionalArgs: ['--last-failed'] }); + expect(result2.exitCode).toBe(1); + expect(result2.passed).toBe(0); + expect(result2.failed).toBe(2); + expect(result2.didNotRun).toBe(0); + expect(result2.outputLines).toEqual(['setup fails 1', 'setup fails 2']); +}); + +test('should run dependencies unfiltered when rerunning a top-level last failed test', async ({ runInlineTest }) => { + const files = { + 'playwright.config.ts': ` + module.exports = { + projects: [ + { name: 'setup', testMatch: /setup.spec.ts/ }, + { name: 'app', testMatch: /app.spec.ts/, dependencies: ['setup'] }, + ], + };`, + 'setup.spec.ts': ` + import { test, expect } from '@playwright/test'; + test('setup 1', async () => { + console.log('\\n%%setup 1'); + }); + test('setup 2', async () => { + console.log('\\n%%setup 2'); + }); + `, + 'app.spec.ts': ` + import { test, expect } from '@playwright/test'; + test('app passes', async () => { + console.log('\\n%%app passes'); + }); + test('app fails', async () => { + console.log('\\n%%app fails'); + expect(1).toBe(2); + }); + `, + }; + const result1 = await runInlineTest(files, { workers: 1 }); + expect(result1.exitCode).toBe(1); + expect(result1.passed).toBe(3); + expect(result1.failed).toBe(1); + expect(result1.outputLines).toEqual(['setup 1', 'setup 2', 'app passes', 'app fails']); + + const result2 = await runInlineTest(files, { workers: 1 }, {}, { additionalArgs: ['--last-failed'] }); + expect(result2.exitCode).toBe(1); + expect(result2.passed).toBe(2); + expect(result2.failed).toBe(1); + expect(result2.outputLines).toEqual(['setup 1', 'setup 2', 'app fails']); +}); + +test('should prefer top-level last failed tests over their dependency failures', async ({ runInlineTest }) => { + const files = { + 'playwright.config.ts': ` + module.exports = { + projects: [ + { name: 'setup', testMatch: /setup.spec.ts/ }, + { name: 'app', testMatch: /app.spec.ts/, dependencies: ['setup'] }, + ], + };`, + 'setup.spec.ts': ` + import { test, expect } from '@playwright/test'; + test('setup passes', async () => { + console.log('\\n%%setup passes'); + }); + test('setup fails', async () => { + console.log('\\n%%setup fails'); + expect(1).toBe(2); + }); + `, + 'app.spec.ts': ` + import { test, expect } from '@playwright/test'; + test('app passes', async () => { + console.log('\\n%%app passes'); + }); + test('app fails', async () => { + console.log('\\n%%app fails'); + expect(1).toBe(2); + }); + `, + }; + + const result1 = await runInlineTest(files, { workers: 1 }, {}, { additionalArgs: ['--no-deps'] }); + expect(result1.exitCode).toBe(1); + expect(result1.passed).toBe(2); + expect(result1.failed).toBe(2); + expect(result1.outputLines).toEqual(['setup passes', 'setup fails', 'app passes', 'app fails']); + + const result2 = await runInlineTest(files, { workers: 1 }, {}, { additionalArgs: ['--last-failed'] }); + expect(result2.exitCode).toBe(1); + expect(result2.passed).toBe(1); + expect(result2.failed).toBe(1); + expect(result2.didNotRun).toBe(1); + expect(result2.outputLines).toEqual(['setup passes', 'setup fails']); +}); + +test('should run transitive dependencies for a last failed dependency project', async ({ runInlineTest }) => { + const files = { + 'playwright.config.ts': ` + module.exports = { + projects: [ + { name: 'setup', testMatch: /setup.spec.ts/ }, + { name: 'seed', testMatch: /seed.spec.ts/, dependencies: ['setup'] }, + { name: 'app', testMatch: /app.spec.ts/, dependencies: ['seed'] }, + ], + };`, + 'setup.spec.ts': ` + import { test, expect } from '@playwright/test'; + test('setup passes', async () => { + console.log('\\n%%setup passes'); + }); + `, + 'seed.spec.ts': ` + import { test, expect } from '@playwright/test'; + test('seed passes', async () => { + console.log('\\n%%seed passes'); + }); + test('seed fails', async () => { + console.log('\\n%%seed fails'); + expect(1).toBe(2); + }); + `, + 'app.spec.ts': ` + import { test, expect } from '@playwright/test'; + test('app passes', async () => { + console.log('\\n%%app passes'); + }); + `, + }; + const result1 = await runInlineTest(files, { workers: 1 }); + expect(result1.exitCode).toBe(1); + expect(result1.passed).toBe(2); + expect(result1.failed).toBe(1); + expect(result1.didNotRun).toBe(1); + expect(result1.outputLines).toEqual(['setup passes', 'seed passes', 'seed fails']); + + const result2 = await runInlineTest(files, { workers: 1 }, {}, { additionalArgs: ['--last-failed'] }); + expect(result2.exitCode).toBe(1); + expect(result2.passed).toBe(1); + expect(result2.failed).toBe(1); + expect(result2.didNotRun).toBe(0); + expect(result2.outputLines).toEqual(['setup passes', 'seed fails']); +}); + +test('should run shared dependencies once for multiple top-level last failed tests', async ({ runInlineTest }) => { + const files = { + 'playwright.config.ts': ` + module.exports = { + projects: [ + { name: 'setup', testMatch: /setup.spec.ts/ }, + { name: 'chromium', testMatch: /chromium.spec.ts/, dependencies: ['setup'] }, + { name: 'firefox', testMatch: /firefox.spec.ts/, dependencies: ['setup'] }, + ], + };`, + 'setup.spec.ts': ` + import { test, expect } from '@playwright/test'; + test('setup 1', async () => { + console.log('\\n%%setup 1'); + }); + test('setup 2', async () => { + console.log('\\n%%setup 2'); + }); + `, + 'chromium.spec.ts': ` + import { test, expect } from '@playwright/test'; + test('chromium passes', async () => { + console.log('\\n%%chromium passes'); + }); + test('chromium fails', async () => { + console.log('\\n%%chromium fails'); + expect(1).toBe(2); + }); + `, + 'firefox.spec.ts': ` + import { test, expect } from '@playwright/test'; + test('firefox passes', async () => { + console.log('\\n%%firefox passes'); + }); + test('firefox fails', async () => { + console.log('\\n%%firefox fails'); + expect(1).toBe(2); + }); + `, + }; + const result1 = await runInlineTest(files, { workers: 1 }); + expect(result1.exitCode).toBe(1); + expect(result1.passed).toBe(4); + expect(result1.failed).toBe(2); + expect(result1.outputLines).toEqual(['setup 1', 'setup 2', 'chromium passes', 'chromium fails', 'firefox passes', 'firefox fails']); + + const result2 = await runInlineTest(files, { workers: 1 }, {}, { additionalArgs: ['--last-failed'] }); + expect(result2.exitCode).toBe(1); + expect(result2.passed).toBe(2); + expect(result2.failed).toBe(2); + expect(result2.outputLines).toEqual(['setup 1', 'setup 2', 'chromium fails', 'firefox fails']); +}); + test('should inherit env changes from dependencies', async ({ runInlineTest }) => { const result = await runInlineTest({ 'playwright.config.ts': `