From c7e7b1d5a5ba0051698d1c3d355ac49a0aa5e15b Mon Sep 17 00:00:00 2001 From: "M.P. Korstanje" Date: Sat, 30 May 2026 13:19:29 +0200 Subject: [PATCH 1/5] Setup tests --- .../suite/engine/SuiteEngineTests.java | 27 +++++++++++++++++++ .../testcases/SingleFailingTestTestCase.java | 25 +++++++++++++++++ .../suite/engine/testsuites/FailingSuite.java | 23 ++++++++++++++++ 3 files changed, 75 insertions(+) create mode 100644 platform-tests/src/test/java/org/junit/platform/suite/engine/testcases/SingleFailingTestTestCase.java create mode 100644 platform-tests/src/test/java/org/junit/platform/suite/engine/testsuites/FailingSuite.java diff --git a/platform-tests/src/test/java/org/junit/platform/suite/engine/SuiteEngineTests.java b/platform-tests/src/test/java/org/junit/platform/suite/engine/SuiteEngineTests.java index e5199135941e..eb9981081aa8 100644 --- a/platform-tests/src/test/java/org/junit/platform/suite/engine/SuiteEngineTests.java +++ b/platform-tests/src/test/java/org/junit/platform/suite/engine/SuiteEngineTests.java @@ -35,15 +35,18 @@ import static org.mockito.Mockito.when; import java.nio.file.Path; +import java.util.logging.Level; import org.jspecify.annotations.Nullable; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.fixtures.TrackLogRecords; import org.junit.jupiter.api.io.TempDir; import org.junit.jupiter.engine.descriptor.ClassTestDescriptor; import org.junit.jupiter.engine.descriptor.JupiterEngineDescriptor; import org.junit.jupiter.engine.descriptor.TestMethodTestDescriptor; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ValueSource; +import org.junit.platform.commons.logging.LogRecordListener; import org.junit.platform.engine.CancellationToken; import org.junit.platform.engine.DiscoveryIssue; import org.junit.platform.engine.DiscoveryIssue.Severity; @@ -67,6 +70,7 @@ import org.junit.platform.suite.engine.testcases.ErroneousTestCase; import org.junit.platform.suite.engine.testcases.JUnit4TestsTestCase; import org.junit.platform.suite.engine.testcases.MultipleTestsTestCase; +import org.junit.platform.suite.engine.testcases.SingleFailingTestTestCase; import org.junit.platform.suite.engine.testcases.SingleTestTestCase; import org.junit.platform.suite.engine.testcases.TaggedTestTestCase; import org.junit.platform.suite.engine.testsuites.AbstractSuite; @@ -80,6 +84,7 @@ import org.junit.platform.suite.engine.testsuites.EmptyTestCaseSuite; import org.junit.platform.suite.engine.testsuites.EmptyTestCaseWithFailIfNoTestFalseSuite; import org.junit.platform.suite.engine.testsuites.ErroneousTestSuite; +import org.junit.platform.suite.engine.testsuites.FailingSuite; import org.junit.platform.suite.engine.testsuites.InheritedSuite; import org.junit.platform.suite.engine.testsuites.MultiEngineSuite; import org.junit.platform.suite.engine.testsuites.MultipleSuite; @@ -630,6 +635,28 @@ void threePartCyclicSuite() { // @formatter:on } + @Test + void failingSuite(@TrackLogRecords LogRecordListener listener) { + // @formatter:off + EngineTestKit.Builder testKit = EngineTestKit.engine(ENGINE_ID) + .selectors(selectClass(FailingSuite.class)); + + assertThat(testKit.discover().getDiscoveryIssues()) + .isEmpty(); + + testKit + .execute() + .testEvents() + .debug() + .assertThatEvents() + .haveExactly(1, event(test(FailingSuite.class.getName()), finishedWithFailure())) + .haveExactly(1, event(test(SingleFailingTestTestCase.class.getName()),finishedWithFailure())); + // @formatter:on + + // Warnings from failing listeners. + assertThat(listener.stream(Level.WARNING)).isEmpty(); + } + @Test void selectByIdentifier() { // @formatter:off diff --git a/platform-tests/src/test/java/org/junit/platform/suite/engine/testcases/SingleFailingTestTestCase.java b/platform-tests/src/test/java/org/junit/platform/suite/engine/testcases/SingleFailingTestTestCase.java new file mode 100644 index 000000000000..4eebe57e64fe --- /dev/null +++ b/platform-tests/src/test/java/org/junit/platform/suite/engine/testcases/SingleFailingTestTestCase.java @@ -0,0 +1,25 @@ +/* + * Copyright 2015-2026 the original author or authors. + * + * All rights reserved. This program and the accompanying materials are + * made available under the terms of the Eclipse Public License v2.0 which + * accompanies this distribution and is available at + * + * https://www.eclipse.org/legal/epl-v20.html + */ + +package org.junit.platform.suite.engine.testcases; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +/** + * @since 1.8 + */ +public class SingleFailingTestTestCase { + + @Test + void test() { + Assertions.fail(); + } +} diff --git a/platform-tests/src/test/java/org/junit/platform/suite/engine/testsuites/FailingSuite.java b/platform-tests/src/test/java/org/junit/platform/suite/engine/testsuites/FailingSuite.java new file mode 100644 index 000000000000..12a35d469dd8 --- /dev/null +++ b/platform-tests/src/test/java/org/junit/platform/suite/engine/testsuites/FailingSuite.java @@ -0,0 +1,23 @@ +/* + * Copyright 2015-2026 the original author or authors. + * + * All rights reserved. This program and the accompanying materials are + * made available under the terms of the Eclipse Public License v2.0 which + * accompanies this distribution and is available at + * + * https://www.eclipse.org/legal/epl-v20.html + */ + +package org.junit.platform.suite.engine.testsuites; + +import org.junit.platform.suite.api.SelectClasses; +import org.junit.platform.suite.api.Suite; +import org.junit.platform.suite.engine.testcases.SingleFailingTestTestCase; + +/** + * @since 1.8 + */ +@Suite +@SelectClasses(SingleFailingTestTestCase.class) +public class FailingSuite { +} From 5eb4a47e11496e5bd6052598a777bda8a54ea1cb Mon Sep 17 00:00:00 2001 From: "M.P. Korstanje" Date: Sat, 30 May 2026 13:28:57 +0200 Subject: [PATCH 2/5] Use minimal summary listener for Suite Engine The Suite engine uses the `SummaryGeneratingListener` to track if any tests were discovered. After #5588 this fails because it now tries to construct a complete description of the failing test. This assumes that all ancestors of a test descriptor are in the plan. Which isn't true for plans from the SuiteEngine, as these are a subtree of the overall plan. Using a minimal implementation of the `SummaryGeneratingListener` we can avoid this problem. --- .../platform/suite/engine/SuiteLauncher.java | 13 ++++--- .../SuiteSummaryGeneratingListener.java | 39 +++++++++++++++++++ .../suite/engine/SuiteTestDescriptor.java | 7 ++-- 3 files changed, 49 insertions(+), 10 deletions(-) create mode 100644 junit-platform-suite-engine/src/main/java/org/junit/platform/suite/engine/SuiteSummaryGeneratingListener.java diff --git a/junit-platform-suite-engine/src/main/java/org/junit/platform/suite/engine/SuiteLauncher.java b/junit-platform-suite-engine/src/main/java/org/junit/platform/suite/engine/SuiteLauncher.java index 34a5f3beff34..6cc3dc1568fe 100644 --- a/junit-platform-suite-engine/src/main/java/org/junit/platform/suite/engine/SuiteLauncher.java +++ b/junit-platform-suite-engine/src/main/java/org/junit/platform/suite/engine/SuiteLauncher.java @@ -27,8 +27,6 @@ import org.junit.platform.launcher.core.EngineExecutionOrchestrator; import org.junit.platform.launcher.core.LauncherDiscoveryResult; import org.junit.platform.launcher.core.ServiceLoaderTestEngineRegistry; -import org.junit.platform.launcher.listeners.SummaryGeneratingListener; -import org.junit.platform.launcher.listeners.TestExecutionSummary; /** * @since 1.8 @@ -59,12 +57,15 @@ LauncherDiscoveryResult discover(LauncherDiscoveryRequest discoveryRequest, Uniq return discoveryOrchestrator.discover(discoveryRequest, parentId); } - TestExecutionSummary execute(LauncherDiscoveryResult discoveryResult, EngineExecutionListener executionListener, - NamespacedHierarchicalStore requestLevelStore, CancellationToken cancellationToken) { - SummaryGeneratingListener listener = new SummaryGeneratingListener(); + SuiteSummaryGeneratingListener execute(LauncherDiscoveryResult discoveryResult, + EngineExecutionListener executionListener, NamespacedHierarchicalStore requestLevelStore, + CancellationToken cancellationToken) { + + SuiteSummaryGeneratingListener listener = new SuiteSummaryGeneratingListener(); + executionOrchestrator.execute(discoveryResult, executionListener, listener, requestLevelStore, cancellationToken); - return listener.getSummary(); + return listener; } } diff --git a/junit-platform-suite-engine/src/main/java/org/junit/platform/suite/engine/SuiteSummaryGeneratingListener.java b/junit-platform-suite-engine/src/main/java/org/junit/platform/suite/engine/SuiteSummaryGeneratingListener.java new file mode 100644 index 000000000000..25cb82013156 --- /dev/null +++ b/junit-platform-suite-engine/src/main/java/org/junit/platform/suite/engine/SuiteSummaryGeneratingListener.java @@ -0,0 +1,39 @@ +/* + * Copyright 2026 the original author or authors. + * + * All rights reserved. This program and the accompanying materials are + * made available under the terms of the Eclipse Public License v2.0 which + * accompanies this distribution and is available at + * + * https://www.eclipse.org/legal/epl-v20.html + */ + +package org.junit.platform.suite.engine; + +import java.util.concurrent.atomic.AtomicLong; + +import org.junit.platform.launcher.TestExecutionListener; +import org.junit.platform.launcher.TestIdentifier; +import org.junit.platform.launcher.TestPlan; +import org.junit.platform.launcher.listeners.SummaryGeneratingListener; + +/** + * A minimal implementation of the {@link SummaryGeneratingListener}. + *

+ * The {@code SummaryGeneratingListener} assumes that all the ancestors all + * test descriptors are in the test plan. This isn't true for the suite engine + * which only executes a subtree of the test plan. This implementation tracks + * only the essentials. + */ +final class SuiteSummaryGeneratingListener implements TestExecutionListener { + private final AtomicLong testsFound = new AtomicLong(); + + @Override + public void testPlanExecutionStarted(TestPlan testPlan) { + this.testsFound.set(testPlan.countTestIdentifiers(TestIdentifier::isTest)); + } + + public long getTestsFoundCount() { + return testsFound.get(); + } +} diff --git a/junit-platform-suite-engine/src/main/java/org/junit/platform/suite/engine/SuiteTestDescriptor.java b/junit-platform-suite-engine/src/main/java/org/junit/platform/suite/engine/SuiteTestDescriptor.java index bba954d5ba9a..0d0d405f1551 100644 --- a/junit-platform-suite-engine/src/main/java/org/junit/platform/suite/engine/SuiteTestDescriptor.java +++ b/junit-platform-suite-engine/src/main/java/org/junit/platform/suite/engine/SuiteTestDescriptor.java @@ -48,7 +48,6 @@ import org.junit.platform.launcher.LauncherDiscoveryListener; import org.junit.platform.launcher.LauncherDiscoveryRequest; import org.junit.platform.launcher.core.LauncherDiscoveryResult; -import org.junit.platform.launcher.listeners.TestExecutionSummary; import org.junit.platform.suite.api.Suite; import org.junit.platform.suite.api.SuiteDisplayName; @@ -170,7 +169,7 @@ void execute(EngineExecutionListener executionListener, NamespacedHierarchicalSt executeBeforeSuiteMethods(throwableCollector); - TestExecutionSummary summary = executeTests(executionListener, requestLevelStore, cancellationToken, + SuiteSummaryGeneratingListener summary = executeTests(executionListener, requestLevelStore, cancellationToken, throwableCollector); executeAfterSuiteMethods(throwableCollector); @@ -191,7 +190,7 @@ private void executeBeforeSuiteMethods(ThrowableCollector throwableCollector) { } } - private @Nullable TestExecutionSummary executeTests(EngineExecutionListener executionListener, + private @Nullable SuiteSummaryGeneratingListener executeTests(EngineExecutionListener executionListener, NamespacedHierarchicalStore requestLevelStore, CancellationToken cancellationToken, ThrowableCollector throwableCollector) { @@ -215,7 +214,7 @@ private void executeAfterSuiteMethods(ThrowableCollector throwableCollector) { } } - private TestExecutionResult computeTestExecutionResult(@Nullable TestExecutionSummary summary, + private TestExecutionResult computeTestExecutionResult(@Nullable SuiteSummaryGeneratingListener summary, ThrowableCollector throwableCollector) { var throwable = throwableCollector.getThrowable(); if (throwable != null) { From f921db2ec332d53d531396c208f81d946a13c6fe Mon Sep 17 00:00:00 2001 From: "M.P. Korstanje" Date: Tue, 2 Jun 2026 00:58:41 +0200 Subject: [PATCH 3/5] Clean up --- .../platform/suite/engine/SuiteLauncher.java | 10 ++--- .../SuiteSummaryGeneratingListener.java | 39 ------------------- .../suite/engine/SuiteTestDescriptor.java | 36 ++++++++++++++--- 3 files changed, 35 insertions(+), 50 deletions(-) delete mode 100644 junit-platform-suite-engine/src/main/java/org/junit/platform/suite/engine/SuiteSummaryGeneratingListener.java diff --git a/junit-platform-suite-engine/src/main/java/org/junit/platform/suite/engine/SuiteLauncher.java b/junit-platform-suite-engine/src/main/java/org/junit/platform/suite/engine/SuiteLauncher.java index 6cc3dc1568fe..393a694527f6 100644 --- a/junit-platform-suite-engine/src/main/java/org/junit/platform/suite/engine/SuiteLauncher.java +++ b/junit-platform-suite-engine/src/main/java/org/junit/platform/suite/engine/SuiteLauncher.java @@ -23,6 +23,7 @@ import org.junit.platform.engine.support.store.Namespace; import org.junit.platform.engine.support.store.NamespacedHierarchicalStore; import org.junit.platform.launcher.LauncherDiscoveryRequest; +import org.junit.platform.launcher.TestExecutionListener; import org.junit.platform.launcher.core.EngineDiscoveryOrchestrator; import org.junit.platform.launcher.core.EngineExecutionOrchestrator; import org.junit.platform.launcher.core.LauncherDiscoveryResult; @@ -57,15 +58,12 @@ LauncherDiscoveryResult discover(LauncherDiscoveryRequest discoveryRequest, Uniq return discoveryOrchestrator.discover(discoveryRequest, parentId); } - SuiteSummaryGeneratingListener execute(LauncherDiscoveryResult discoveryResult, - EngineExecutionListener executionListener, NamespacedHierarchicalStore requestLevelStore, + void execute(LauncherDiscoveryResult discoveryResult, EngineExecutionListener executionListener, + TestExecutionListener testExecutionListener, NamespacedHierarchicalStore requestLevelStore, CancellationToken cancellationToken) { - SuiteSummaryGeneratingListener listener = new SuiteSummaryGeneratingListener(); - - executionOrchestrator.execute(discoveryResult, executionListener, listener, requestLevelStore, + executionOrchestrator.execute(discoveryResult, executionListener, testExecutionListener, requestLevelStore, cancellationToken); - return listener; } } diff --git a/junit-platform-suite-engine/src/main/java/org/junit/platform/suite/engine/SuiteSummaryGeneratingListener.java b/junit-platform-suite-engine/src/main/java/org/junit/platform/suite/engine/SuiteSummaryGeneratingListener.java deleted file mode 100644 index 25cb82013156..000000000000 --- a/junit-platform-suite-engine/src/main/java/org/junit/platform/suite/engine/SuiteSummaryGeneratingListener.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Copyright 2026 the original author or authors. - * - * All rights reserved. This program and the accompanying materials are - * made available under the terms of the Eclipse Public License v2.0 which - * accompanies this distribution and is available at - * - * https://www.eclipse.org/legal/epl-v20.html - */ - -package org.junit.platform.suite.engine; - -import java.util.concurrent.atomic.AtomicLong; - -import org.junit.platform.launcher.TestExecutionListener; -import org.junit.platform.launcher.TestIdentifier; -import org.junit.platform.launcher.TestPlan; -import org.junit.platform.launcher.listeners.SummaryGeneratingListener; - -/** - * A minimal implementation of the {@link SummaryGeneratingListener}. - *

- * The {@code SummaryGeneratingListener} assumes that all the ancestors all - * test descriptors are in the test plan. This isn't true for the suite engine - * which only executes a subtree of the test plan. This implementation tracks - * only the essentials. - */ -final class SuiteSummaryGeneratingListener implements TestExecutionListener { - private final AtomicLong testsFound = new AtomicLong(); - - @Override - public void testPlanExecutionStarted(TestPlan testPlan) { - this.testsFound.set(testPlan.countTestIdentifiers(TestIdentifier::isTest)); - } - - public long getTestsFoundCount() { - return testsFound.get(); - } -} diff --git a/junit-platform-suite-engine/src/main/java/org/junit/platform/suite/engine/SuiteTestDescriptor.java b/junit-platform-suite-engine/src/main/java/org/junit/platform/suite/engine/SuiteTestDescriptor.java index 0d0d405f1551..295e9e892515 100644 --- a/junit-platform-suite-engine/src/main/java/org/junit/platform/suite/engine/SuiteTestDescriptor.java +++ b/junit-platform-suite-engine/src/main/java/org/junit/platform/suite/engine/SuiteTestDescriptor.java @@ -19,6 +19,7 @@ import java.lang.reflect.Method; import java.util.List; +import java.util.concurrent.atomic.AtomicLong; import java.util.function.BiFunction; import java.util.function.Predicate; @@ -47,7 +48,11 @@ import org.junit.platform.engine.support.store.NamespacedHierarchicalStore; import org.junit.platform.launcher.LauncherDiscoveryListener; import org.junit.platform.launcher.LauncherDiscoveryRequest; +import org.junit.platform.launcher.TestExecutionListener; +import org.junit.platform.launcher.TestIdentifier; +import org.junit.platform.launcher.TestPlan; import org.junit.platform.launcher.core.LauncherDiscoveryResult; +import org.junit.platform.launcher.listeners.SummaryGeneratingListener; import org.junit.platform.suite.api.Suite; import org.junit.platform.suite.api.SuiteDisplayName; @@ -169,8 +174,8 @@ void execute(EngineExecutionListener executionListener, NamespacedHierarchicalSt executeBeforeSuiteMethods(throwableCollector); - SuiteSummaryGeneratingListener summary = executeTests(executionListener, requestLevelStore, cancellationToken, - throwableCollector); + var summary = new SuiteSummaryGeneratingListener(); + executeTests(executionListener, summary, requestLevelStore, cancellationToken, throwableCollector); executeAfterSuiteMethods(throwableCollector); @@ -190,12 +195,12 @@ private void executeBeforeSuiteMethods(ThrowableCollector throwableCollector) { } } - private @Nullable SuiteSummaryGeneratingListener executeTests(EngineExecutionListener executionListener, + private void executeTests(EngineExecutionListener executionListener, TestExecutionListener testExecutionListener, NamespacedHierarchicalStore requestLevelStore, CancellationToken cancellationToken, ThrowableCollector throwableCollector) { if (throwableCollector.isNotEmpty()) { - return null; + return; } // #2838: The discovery result from a suite may have been filtered by @@ -204,7 +209,7 @@ private void executeBeforeSuiteMethods(ThrowableCollector throwableCollector) { LauncherDiscoveryResult discoveryResult = requireNonNull(this.launcherDiscoveryResult).withRetainedEngines( getChildren()::contains); - return requireNonNull(launcher).execute(discoveryResult, executionListener, requestLevelStore, + requireNonNull(launcher).execute(discoveryResult, executionListener, testExecutionListener, requestLevelStore, cancellationToken); } @@ -279,4 +284,25 @@ public void issueEncountered(UniqueId engineUniqueId, DiscoveryIssue issue) { this.discoveryListener.issueEncountered(engineUniqueId, transformedIssue); } } + + /** + * A minimal implementation of the {@link SummaryGeneratingListener}. + *

+ * The {@code SummaryGeneratingListener} assumes that all the ancestors all + * test descriptors are in the test plan. This isn't true for the suite engine + * which only executes a subtree of the test plan. This implementation tracks + * only the essentials. + */ + private static final class SuiteSummaryGeneratingListener implements TestExecutionListener { + private final AtomicLong testsFound = new AtomicLong(); + + @Override + public void testPlanExecutionStarted(TestPlan testPlan) { + this.testsFound.set(testPlan.countTestIdentifiers(TestIdentifier::isTest)); + } + + long getTestsFoundCount() { + return testsFound.get(); + } + } } From 48b25f21176dafc07fbdcb0e399f3aea85140c17 Mon Sep 17 00:00:00 2001 From: "M.P. Korstanje" Date: Tue, 2 Jun 2026 01:06:58 +0200 Subject: [PATCH 4/5] Clean up --- .../org/junit/platform/suite/engine/SuiteTestDescriptor.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/junit-platform-suite-engine/src/main/java/org/junit/platform/suite/engine/SuiteTestDescriptor.java b/junit-platform-suite-engine/src/main/java/org/junit/platform/suite/engine/SuiteTestDescriptor.java index 295e9e892515..166840495a09 100644 --- a/junit-platform-suite-engine/src/main/java/org/junit/platform/suite/engine/SuiteTestDescriptor.java +++ b/junit-platform-suite-engine/src/main/java/org/junit/platform/suite/engine/SuiteTestDescriptor.java @@ -219,13 +219,13 @@ private void executeAfterSuiteMethods(ThrowableCollector throwableCollector) { } } - private TestExecutionResult computeTestExecutionResult(@Nullable SuiteSummaryGeneratingListener summary, + private TestExecutionResult computeTestExecutionResult(SuiteSummaryGeneratingListener summary, ThrowableCollector throwableCollector) { var throwable = throwableCollector.getThrowable(); if (throwable != null) { return TestExecutionResult.failed(throwable); } - if (failIfNoTests && requireNonNull(summary).getTestsFoundCount() == 0) { + if (failIfNoTests && summary.getTestsFoundCount() == 0) { return TestExecutionResult.failed(new NoTestsDiscoveredException(suiteClass)); } return TestExecutionResult.successful(); From d972dcae9ec4774e8a393d9dc5e9c47640ad5904 Mon Sep 17 00:00:00 2001 From: "M.P. Korstanje" Date: Tue, 2 Jun 2026 01:08:34 +0200 Subject: [PATCH 5/5] Update release notes --- .../ROOT/partials/release-notes/release-notes-6.2.0-M1.adoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/documentation/modules/ROOT/partials/release-notes/release-notes-6.2.0-M1.adoc b/documentation/modules/ROOT/partials/release-notes/release-notes-6.2.0-M1.adoc index 9f3d6b92d9f3..19ca4135c2bf 100644 --- a/documentation/modules/ROOT/partials/release-notes/release-notes-6.2.0-M1.adoc +++ b/documentation/modules/ROOT/partials/release-notes/release-notes-6.2.0-M1.adoc @@ -16,7 +16,7 @@ repository on GitHub. [[v6.2.0-M1-junit-platform-bug-fixes]] ==== Bug Fixes -* ❓ +* The Suite Engine no longer logs "No TestIdentifier with unique ID" for failing tests. [[v6.2.0-M1-junit-platform-deprecations-and-breaking-changes]] ==== Deprecations and Breaking Changes