Change to use worst Test Step result as the Test Case result

CHANGELOG.md

@@ -22,6 +22,10 @@ Please visit [cucumber/CONTRIBUTING.md](https://github.com/cucumber/cucumber/blo
 - Changed the base event inheritance to a new base class structure
 - Refactored some internals of the library to be slightly more performant (Including removing redundant nil checks)
 
+### Changed
+- Change to use worst Test Step result as the Test Case result
+([#317](https://github.com/cucumber/cucumber-ruby-core/pull/317))
+
 ## [16.2.0] - 2026-02-06
 ### Changed
 - Added the test result type 'ambiguous'

lib/cucumber/core/test/result/boolean_methods.rb

@@ -10,7 +10,7 @@ module Result
         # Simple module that when included generates all the boolean methods for each category of result
         # The single exception to this is the class method `self.ok?` which is defined for each result individually
         module BooleanMethods
-          TYPES = %i[failed ambiguous flaky skipped undefined pending passed unknown].freeze
+          TYPES = %i[failed ambiguous flaky undefined pending skipped passed unknown].freeze
 
           TYPES.each do |result|
             define_method("#{result}?") do

lib/cucumber/core/test/result/summary.rb

@@ -16,7 +16,7 @@ module Result
         #     => 1
         #
         class Summary
-          TYPES = %i[failed ambiguous flaky skipped undefined pending passed unknown].freeze
+          TYPES = %i[failed ambiguous flaky undefined pending skipped passed unknown].freeze
 
           attr_reader :exceptions, :durations
 

lib/cucumber/core/test/runner.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require 'cucumber/messages/helpers/test_step_result_comparator'
+
 require_relative 'timer'
 
 module Cucumber
@@ -45,6 +47,8 @@ def done
         end
 
         class RunningTestCase
+          include Cucumber::Messages::Helpers::TestStepResultComparator
+
           def initialize
             @timer = Timer.new.start
             @status = Status::Unknown.new(Result::Unknown.new)
@@ -59,27 +63,27 @@ def result
           end
 
           def failed(step_result)
-            @status = Status::Failing.new(step_result)
+            not_passing(step_result)
             self
           end
 
           def ambiguous(step_result)
-            @status = Status::Ambiguous.new(step_result)
+            failed(step_result)
             self
           end
 
           def passed(step_result)
-            @status = Status::Passing.new(step_result)
+            @status = Status::Passing.new(step_result) if test_step_result_rankings[step_result.to_message.status] > test_step_result_rankings[status.step_result_message.status]
             self
           end
 
           def pending(_message, step_result)
-            @status = Status::Pending.new(step_result)
+            failed(step_result)
             self
           end
 
           def skipped(step_result)
-            @status = Status::Skipping.new(step_result)
+            failed(step_result)
             self
           end
 
@@ -96,6 +100,13 @@ def duration(_step_duration, _step_result)
             self
           end
 
+          private
+
+          def not_passing(step_result)
+            @status = Status::NotPassing.new(step_result) if test_step_result_rankings[step_result.to_message.status] > test_step_result_rankings[status.step_result_message.status]
+            self
+          end
+
           attr_reader :status
           private :status
 
@@ -118,6 +129,10 @@ def execute(test_step, monitor, &)
               def result
                 raise NoMethodError, 'Override me'
               end
+
+              def step_result_message
+                step_result.to_message
+              end
             end
 
             class Unknown < Base
@@ -132,26 +147,14 @@ def result(duration)
               end
             end
 
-            class Failing < Base
+            class NotPassing < Base
               def execute(test_step, monitor)
                 result = test_step.skip(monitor.result)
-                if result.undefined?
-                  result = result.with_message(%(Undefined step: "#{test_step.text}"))
-                  result = result.with_appended_backtrace(test_step)
-                end
-                result
-              end
-
-              def result(duration)
-                step_result.with_duration(duration)
+                result = result.with_message(%(Undefined step: "#{test_step.text}")) if result.undefined?
+                result = result.with_appended_backtrace(test_step) unless test_step.hook?
+                result.describe_to(monitor, result)
               end
-            end
-
-            Pending = Class.new(Failing)
-
-            Ambiguous = Class.new(Failing)
 
-            class Skipping < Failing
               def result(duration)
                 step_result.with_duration(duration)
               end

spec/cucumber/core/test/runner_spec.rb

@@ -222,14 +222,14 @@
       let(:test_steps) { [failing_step, passing_step] }
 
       it 'emits the test_step_finished event with a failed result' do
-        expect(event_bus).to receive(:test_step_finished).with(failing_step, anything) do |_reported_test_case, result|
+        expect(event_bus).to receive(:test_step_finished).with(failing_step, anything) do |_reported_test_step, result|
           expect(result).to be_failed
         end
         test_case.describe_to(runner)
       end
 
       it 'emits a test_step_finished event with a skipped result' do
-        expect(event_bus).to receive(:test_step_finished).with(passing_step, anything) do |_reported_test_case, result|
+        expect(event_bus).to receive(:test_step_finished).with(passing_step, anything) do |_reported_test_step, result|
           expect(result).to be_skipped
         end
         test_case.describe_to(runner)
@@ -250,6 +250,158 @@
         test_case.describe_to(runner)
       end
     end
+
+    context 'with an initial undefined step' do
+      let(:test_steps) { [undefined_step, passing_step] }
+
+      it 'emits a test_step_finished event when executing an undefined step' do
+        expect(event_bus).to receive(:test_step_finished).with(undefined_step, anything) do |reported_test_step, _result|
+          expect(reported_test_step).to be_a(Cucumber::Core::Test::Step)
+        end
+        test_case.describe_to(runner)
+      end
+
+      it 'emits a test_step_finished event with an undefined result' do
+        expect(event_bus).to receive(:test_step_finished).with(undefined_step, anything) do |_reported_test_step, result|
+          expect(result).to be_undefined
+        end
+        test_case.describe_to(runner)
+      end
+
+      it 'skips, rather than executing the second step' do
+        expect(passing_step).not_to receive(:execute)
+
+        allow(passing_step).to receive(:skip).and_return(Cucumber::Core::Test::Result::Skipped.new)
+        test_case.describe_to(runner)
+      end
+
+      it 'emits a test_step_finished event when executing a skipped step' do
+        expect(event_bus).to receive(:test_step_finished).with(passing_step, anything) do |reported_test_step, _result|
+          expect(reported_test_step).to be_a(Cucumber::Core::Test::Step)
+        end
+        test_case.describe_to(runner)
+      end
+
+      it 'emits a test_step_finished event with a skipped result' do
+        expect(event_bus).to receive(:test_step_finished).with(passing_step, anything) do |_reported_test_step, result|
+          expect(result).to be_skipped
+        end
+        test_case.describe_to(runner)
+      end
+
+      it 'emits a test_case_finished event with an undefined result' do
+        expect(event_bus).to receive(:test_case_finished) do |_reported_test_case, result|
+          expect(result).to be_undefined
+        end
+        test_case.describe_to(runner)
+      end
+
+      it 'emits a test_case_finished event with an exception object' do
+        expect(event_bus).to receive(:test_case_finished) do |_reported_test_case, result|
+          expect(result.exception).to be_a StandardError
+        end
+        test_case.describe_to(runner)
+      end
+
+      context 'with an undefined step and a subsequent ambiguous step' do
+        let(:test_steps) { [undefined_step, ambiguous_step] }
+
+        it 'emits a test_step_finished event when executing an undefined step' do
+          expect(event_bus).to receive(:test_step_finished).with(undefined_step, anything) do |reported_test_step, _result|
+            expect(reported_test_step).to be_a(Cucumber::Core::Test::Step)
+          end
+          test_case.describe_to(runner)
+        end
+
+        it 'emits a test_step_finished event with an undefined result' do
+          expect(event_bus).to receive(:test_step_finished).with(undefined_step, anything) do |_reported_test_step, result|
+            expect(result).to be_undefined
+          end
+          test_case.describe_to(runner)
+        end
+
+        it 'skips, rather than executing the second step' do
+          expect(passing_step).not_to receive(:execute)
+
+          allow(passing_step).to receive(:skip).and_return(Cucumber::Core::Test::Result::Skipped.new)
+          test_case.describe_to(runner)
+        end
+
+        it 'emits a test_step_finished event when executing a skipped step' do
+          expect(event_bus).to receive(:test_step_finished).with(ambiguous_step, anything) do |reported_test_step, _result|
+            expect(reported_test_step).to be_a(Cucumber::Core::Test::Step)
+          end
+          test_case.describe_to(runner)
+        end
+
+        it 'emits a test_step_finished event with an ambiguous result' do
+          expect(event_bus).to receive(:test_step_finished).with(ambiguous_step, anything) do |_reported_test_step, result|
+            expect(result).to be_ambiguous
+          end
+          test_case.describe_to(runner)
+        end
+
+        it 'emits a test_case_finished event with an ambiguous result' do
+          expect(event_bus).to receive(:test_case_finished) do |_reported_test_case, result|
+            expect(result).to be_ambiguous
+          end
+          test_case.describe_to(runner)
+        end
+
+        it 'emits a test_case_finished event with an exception object' do
+          expect(event_bus).to receive(:test_case_finished) do |_reported_test_case, result|
+            expect(result.exception).to be_a StandardError
+          end
+          test_case.describe_to(runner)
+        end
+      end
+
+      context 'with a failing after hook' do
+        let(:test_steps) { [undefined_step, failing_hook] }
+
+        it 'emits a test_step_finished event when executing an undefined step' do
+          expect(event_bus).to receive(:test_step_finished).with(undefined_step, anything) do |reported_test_step, _result|
+            expect(reported_test_step).to be_a(Cucumber::Core::Test::Step)
+          end
+          test_case.describe_to(runner)
+        end
+
+        it 'emits a test_step_finished event with an undefined result' do
+          expect(event_bus).to receive(:test_step_finished).with(undefined_step, anything) do |_reported_test_step, result|
+            expect(result).to be_undefined
+          end
+          test_case.describe_to(runner)
+        end
+
+        it 'emits a test_step_finished event with a failing result' do
+          expect(event_bus).to receive(:test_step_finished).with(failing_hook, anything) do |_reported_test_step, result|
+            expect(result).to be_failed
+          end
+          test_case.describe_to(runner)
+        end
+
+        it 'emits a test_case_finished event with a failing result' do
+          expect(event_bus).to receive(:test_case_finished) do |_reported_test_case, result|
+            expect(result).to be_failed
+          end
+          test_case.describe_to(runner)
+        end
+
+        it 'emits a test_case_finished event with an exception object' do
+          expect(event_bus).to receive(:test_case_finished) do |_reported_test_case, result|
+            expect(result.exception).to be_a StandardError
+          end
+          test_case.describe_to(runner)
+        end
+
+        it 'calls skip, rather than execute on test step of the hook' do
+          expect(failing_hook).not_to receive(:execute)
+
+          allow(failing_hook).to receive(:skip).and_return(Cucumber::Core::Test::Result::Failed.new(Cucumber::Core::Test::Result::UnknownDuration.new, instance_double(StandardError, backtrace: [])))
+          test_case.describe_to(runner)
+        end
+      end
+    end
   end
 
   context 'with multiple test cases' do

spec/support/shared_context/test_step_types.rb

@@ -8,4 +8,6 @@
   let(:pending_step)   { Cucumber::Core::Test::Step.new(3, 'Pending Step', double).with_action { raise Cucumber::Core::Test::Result::Pending, 'TODO' } }
   let(:skipping_step)  { Cucumber::Core::Test::Step.new(4, 'Skipped Step', double).with_action { raise Cucumber::Core::Test::Result::Skipped } }
   let(:undefined_step) { Cucumber::Core::Test::Step.new(5, 'Undefined Step', double) }
+  let(:ambiguous_step) { Cucumber::Core::Test::Step.new(6, 'Ambiguous Step', double).with_unskippable_action { raise Cucumber::Core::Test::Result::Ambiguous } }
+  let(:failing_hook)   { Cucumber::Core::Test::Step.new(7, 'Failing Step', double).with_unskippable_action { raise StandardError, 'Error' } }
 end