Apply module timeout per retry attempt

docs/docs/how-to/retry-policy.md

@@ -97,6 +97,12 @@ public class ResilientModule : Module<CommandResult>
 }
 ```
 
+`WithTimeout` applies to each execution attempt, not to the whole retry chain. Backoff delays run
+outside that timeout. A timed-out attempt is passed to the resilience shield as a
+`ModuleTimeoutException`, so exception filters still decide whether it should be retried. If the
+attempt remains active after the cancellation grace period, the engine bypasses retries to avoid
+running the same module instance concurrently with its abandoned attempt.
+
 ## Default Retry Configuration
 
 Retries are off by default. You can set a default retry count on the `PipelineOptions`:

docs/docs/how-to/timeouts.md

@@ -75,9 +75,16 @@ public class ResilientModule : Module<CommandResult>
 
 ## Timeout Behavior
 
+Timeouts apply to each execution attempt. With retries enabled, every attempt receives the full
+timeout and retry backoff delays do not consume it. A module configured with a five-minute timeout
+and three retries can therefore spend up to five minutes in each of its four attempts, plus retry
+delays.
+
 When a timeout occurs:
 
 - The `CancellationToken` passed to `ExecuteAsync` will be cancelled
 - The module will fail with a `ModuleTimeoutException`
-- If retry policies are configured, the module may be retried
+- If retry policies are configured and the attempt stops within the cancellation grace period,
+  the module may be retried. An attempt that remains active after the grace period is never retried,
+  preventing concurrent executions of the same module instance.
 - If `WithIgnoreFailures()` is configured, the pipeline will continue despite the timeout

src/ModularPipelines/Configuration/ModuleConfiguration.cs

@@ -58,10 +58,10 @@ public sealed class ModuleConfiguration
     internal Func<IModuleContext, CancellationToken, ValueTask<SkipDecision?>>? PlanningSkipCondition { get; init; }
 
     /// <summary>
-    /// Gets the timeout duration for module execution.
+    /// Gets the timeout duration for each module execution attempt.
     /// </summary>
     /// <value>
-    /// A <see cref="TimeSpan"/> representing the maximum execution time,
+    /// A <see cref="TimeSpan"/> representing the maximum time for each attempt,
     /// or null if no timeout is configured.
     /// </value>
     public TimeSpan? Timeout { get; init; }

src/ModularPipelines/Configuration/ModuleConfigurationBuilder.cs

@@ -325,10 +325,13 @@ public ModuleConfigurationBuilder DependsOnIf(Type moduleType, bool condition)
     #region WithTimeout
 
     /// <summary>
-    /// Sets the timeout duration for module execution.
+    /// Sets the timeout duration for each module execution attempt.
     /// </summary>
-    /// <param name="timeout">The maximum duration allowed for module execution.</param>
+    /// <param name="timeout">The maximum duration allowed for each execution attempt.</param>
     /// <returns>This builder instance for method chaining.</returns>
+    /// <remarks>
+    /// When retries are configured, the timeout restarts for every attempt and does not include retry delays.
+    /// </remarks>
     public ModuleConfigurationBuilder WithTimeout(TimeSpan timeout)
     {
         _timeout = timeout;
@@ -349,6 +352,7 @@ public ModuleConfigurationBuilder WithTimeout(TimeSpan timeout)
     /// <remarks>
     /// Each delay uses equal jitter between half and all of its exponential-backoff ceiling.
     /// A null <paramref name="shouldRetry"/> retries every exception handled by the retry engine.
+    /// A configured module timeout applies separately to each attempt and does not include these delays.
     /// </remarks>
     public ModuleConfigurationBuilder WithRetry(
         int count,

src/ModularPipelines/Context/Command.cs

@@ -553,10 +553,7 @@ await WaitForForcefulCancellationAsync(
                         command.WorkingDirPath));
                 var failure = loggingFailures.CombineWith(e);
 
-                if (ShouldPreserveCallerCancellation(e, failure, callerCancellationToken))
-                {
-                    throw;
-                }
+                ThrowCallerCancellationIfRequired(e, failure, callerCancellationToken);
 
                 throw CreateExecutionFailure(
                     e,
@@ -627,6 +624,38 @@ private static bool ShouldPreserveCallerCancellation(
                && ReferenceEquals(combinedFailure, executionFailure);
     }
 
+    private static void ThrowCallerCancellationIfRequired(
+        Exception executionFailure,
+        Exception combinedFailure,
+        CancellationToken callerCancellationToken)
+    {
+        if (!ShouldPreserveCallerCancellation(
+                executionFailure,
+                combinedFailure,
+                callerCancellationToken))
+        {
+            return;
+        }
+
+        if (executionFailure is OperationCanceledException cancellationException
+            && cancellationException.CancellationToken != callerCancellationToken)
+        {
+            throw cancellationException is TaskCanceledException
+                ? new TaskCanceledException(
+                    cancellationException.Message,
+                    cancellationException,
+                    callerCancellationToken)
+                : new OperationCanceledException(
+                    cancellationException.Message,
+                    cancellationException,
+                    callerCancellationToken);
+        }
+
+        System.Runtime.ExceptionServices.ExceptionDispatchInfo
+            .Capture(executionFailure)
+            .Throw();
+    }
+
     private Exception CreateExecutionFailure(
         Exception executionFailure,
         Exception combinedFailure,

src/ModularPipelines/Engine/ModuleExecutionPipeline.cs

@@ -472,80 +472,126 @@ private async Task<T> ExecuteWithPolicies<T>(
         IModuleContext moduleContext)
     {
         var timeout = GetTimeout(config);
-        if (config.Timeout is null)
-        {
-            if (timeout == TimeSpan.Zero)
-            {
-                moduleContext.Logger.LogTrace("No module timeout configured. The pipeline default timeout is disabled");
-            }
-            else
-            {
-                moduleContext.Logger.LogTrace("No module timeout configured. Using pipeline default timeout {Timeout}", timeout);
-            }
-        }
+        LogTimeoutConfiguration(config, timeout, moduleContext.Logger);
 
         var cancellationToken = executionContext.ModuleCancellationTokenSource.Token;
 
         // Get resilience shield if applicable
         var resilienceShield = GetResilienceShield(config, moduleContext);
-        var moduleAttemptCount = 0;
-        var moduleAttemptRespondedToCancellation = 0;
+        using var retryCancellationTokenSource = resilienceShield is null
+            ? null
+            : CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
+        var policyExecutionState = new PolicyExecutionState();
+
+        // Keep timeout enforcement inside the resilience shield so each attempt gets a fresh budget
+        // and shield-owned backoff delays are not mistaken for unresponsive module execution.
+        Task<T> ExecuteModuleAttempt(CancellationToken ct) => ExecuteModuleAttemptAsync(
+            module,
+            executionContext,
+            moduleContext,
+            timeout,
+            retryCancellationTokenSource,
+            policyExecutionState,
+            ct);
 
-        async Task<T> ExecuteModuleAttempt(CancellationToken ct)
+        T result;
+        try
         {
-            Interlocked.Increment(ref moduleAttemptCount);
-            try
-            {
-                return await module.ExecuteAsync(moduleContext, ct).ConfigureAwait(false);
-            }
-            finally
-            {
-                if (ct.IsCancellationRequested)
-                {
-                    Volatile.Write(ref moduleAttemptRespondedToCancellation, 1);
-                }
-            }
+            result = resilienceShield != null
+                ? await resilienceShield.ExecuteAsync(
+                    async shieldToken => await ExecuteModuleAttempt(shieldToken).ConfigureAwait(false),
+                    retryCancellationTokenSource!.Token).ConfigureAwait(false)
+                : await ExecuteModuleAttempt(cancellationToken).ConfigureAwait(false);
         }
-
-        // Create the execution function that optionally includes resilience strategies
-        Func<CancellationToken, Task<T>> executeFunc = resilienceShield != null
-            ? async ct => await resilienceShield.ExecuteAsync(
-                async shieldToken => await ExecuteModuleAttempt(shieldToken).ConfigureAwait(false),
-                ct).ConfigureAwait(false)
-            : ExecuteModuleAttempt;
-
-        // Use TimeoutHelper with detailed results to get information about token cooperation
-        TimeoutExecutionResult<T> timeoutResult;
-        try
+        catch (OperationCanceledException) when (policyExecutionState.AbandonedAttemptTimeout is not null
+                                                 && !cancellationToken.IsCancellationRequested)
         {
-            timeoutResult = await TimeoutHelper.ExecuteWithTimeoutAndDetailsAsync(
-                executeFunc,
-                timeout == TimeSpan.Zero ? null : timeout,
-                cancellationToken,
-                $"Module {executionContext.ModuleType.Name} timed out after {timeout}").ConfigureAwait(false);
+            return ThrowPreservingStack<T>(policyExecutionState.AbandonedAttemptTimeout);
         }
         finally
         {
             ModuleActivityTracing.RecordModuleRetries(
                 executionContext.ModuleType,
-                Math.Max(0, Volatile.Read(ref moduleAttemptCount) - 1));
+                policyExecutionState.RetryCount);
         }
 
-        if (timeoutResult.TimedOut)
+        if (policyExecutionState.AbandonedAttemptTimeout is { } abandonedAttemptTimeout)
         {
-            var wasCancellationTokenRespected = timeoutResult.WasCancellationTokenRespected
-                                                && (resilienceShield is null
-                                                    || Volatile.Read(ref moduleAttemptRespondedToCancellation) == 1);
+            return ThrowPreservingStack<T>(abandonedAttemptTimeout);
+        }
 
-            // Create a detailed timeout exception with information about token cooperation
-            throw new ModuleTimeoutException(
-                executionContext.ModuleType,
-                timeout,
-                timeoutResult.ElapsedTime,
-                wasCancellationTokenRespected);
+        return result;
+    }
+
+    private static void LogTimeoutConfiguration(
+        ModuleConfiguration config,
+        TimeSpan timeout,
+        IModuleLogger logger)
+    {
+        if (config.Timeout is not null)
+        {
+            return;
+        }
+
+        if (timeout == TimeSpan.Zero)
+        {
+            logger.LogTrace("No module timeout configured. The pipeline default timeout is disabled");
+            return;
         }
 
-        return timeoutResult.Value!;
+        logger.LogTrace("No module timeout configured. Using pipeline default timeout {Timeout}", timeout);
+    }
+
+    private static async Task<T> ExecuteModuleAttemptAsync<T>(
+        Module<T> module,
+        ModuleExecutionContext<T> executionContext,
+        IModuleContext moduleContext,
+        TimeSpan timeout,
+        CancellationTokenSource? retryCancellationTokenSource,
+        PolicyExecutionState policyExecutionState,
+        CancellationToken cancellationToken)
+    {
+        if (policyExecutionState.AbandonedAttemptTimeout is { } abandonedAttemptTimeout)
+        {
+            return ThrowPreservingStack<T>(abandonedAttemptTimeout);
+        }
+
+        policyExecutionState.RecordAttempt();
+
+        var timeoutResult = await TimeoutHelper.ExecuteWithTimeoutAndDetailsAsync(
+            attemptToken => module.ExecuteAsync(moduleContext, attemptToken),
+            timeout == TimeSpan.Zero ? null : timeout,
+            cancellationToken,
+            $"Module {executionContext.ModuleType.Name} timed out after {timeout}").ConfigureAwait(false);
+
+        if (!timeoutResult.TimedOut)
+        {
+            return timeoutResult.Value!;
+        }
+
+        var timeoutException = new ModuleTimeoutException(
+            executionContext.ModuleType,
+            timeout,
+            timeoutResult.ElapsedTime,
+            timeoutResult.WasCancellationTokenRespected);
+
+        if (!timeoutResult.WasCancellationTokenRespected)
+        {
+            policyExecutionState.AbandonedAttemptTimeout = timeoutException;
+            // Cancel the shield before its retry delay because re-entering this module while
+            // the abandoned attempt is active is unsafe.
+            retryCancellationTokenSource?.Cancel();
+        }
+
+        throw timeoutException;
+    }
+
+    private static T ThrowPreservingStack<T>(Exception exception)
+    {
+        System.Runtime.ExceptionServices.ExceptionDispatchInfo
+            .Capture(exception)
+            .Throw();
+        throw new System.Diagnostics.UnreachableException();
     }
 
     private TimeSpan GetTimeout(ModuleConfiguration config)
@@ -680,7 +726,7 @@ private async Task<ModuleResult<T>> HandleException<T>(
 
         executionContext.Exception = exception;
 
-        executionContext.Status = ClassifyException(config, executionContext, exception);
+        executionContext.Status = ClassifyException(config, exception);
 
         // Use the enhanced exception type for detailed timeout logging.
         if (exception is ModuleTimeoutException timeoutException)
@@ -693,12 +739,10 @@ private async Task<ModuleResult<T>> HandleException<T>(
                     timeoutException.ElapsedTime.ToDisplayString());
             }
         }
-
         // Check if we should ignore failures
         if (config.IgnoreFailuresCondition != null
             && (executionContext.Status != Status.PipelineTerminated
-                || exception is ModuleTimeoutException
-                || IsTimeout(config, executionContext, exception)))
+                || exception is ModuleTimeoutException))
         {
             if (await config.IgnoreFailuresCondition(moduleContext, exception).ConfigureAwait(false))
             {
@@ -747,31 +791,23 @@ await SaveResults(
 
     private Status ClassifyException(
         ModuleConfiguration config,
-        ModuleExecutionContext executionContext,
         Exception exception)
     {
         if (!config.AlwaysRun
-            && _engineCancellationToken.IsCancelled
-            && exception is OperationCanceledException or ModuleTimeoutException)
+            && IsPipelineCancelled(exception))
         {
             return Status.PipelineTerminated;
         }
 
-        return exception is ModuleTimeoutException || IsTimeout(config, executionContext, exception)
+        return exception is ModuleTimeoutException
             ? Status.TimedOut
             : Status.Failed;
     }
 
-    private bool IsTimeout(ModuleConfiguration config, ModuleExecutionContext executionContext, Exception exception)
+    private bool IsPipelineCancelled(Exception exception)
     {
-        var timeout = GetTimeout(config);
-        if (timeout == TimeSpan.Zero)
-        {
-            return false;
-        }
-
-        var isTimeoutExceeded = executionContext.Stopwatch.Elapsed >= timeout;
-        return isTimeoutExceeded && exception is OperationCanceledException;
+        return exception is OperationCanceledException or ModuleTimeoutException
+               && _engineCancellationToken.IsCancelled;
     }
 
     private void CancelPipelineAndThrow(
@@ -841,6 +877,17 @@ public void Complete(ModuleResult<T> result)
         }
     }
 
+    private sealed class PolicyExecutionState
+    {
+        private int _moduleAttemptCount;
+
+        public ModuleTimeoutException? AbandonedAttemptTimeout { get; set; }
+
+        public int RetryCount => Math.Max(0, Volatile.Read(ref _moduleAttemptCount) - 1);
+
+        public void RecordAttempt() => Interlocked.Increment(ref _moduleAttemptCount);
+    }
+
     private static void LogModuleStatus(ModuleExecutionContext executionContext, IModuleLogger logger)
     {
         var moduleName = executionContext.ModuleType.Name;

src/ModularPipelines/Exceptions/ModuleTimeoutException.cs

@@ -7,12 +7,13 @@ namespace ModularPipelines.Exceptions;
 /// </summary>
 /// <remarks>
 /// <para>
-/// This exception is thrown when a module's execution time exceeds the configured timeout.
+/// This exception is thrown when a module execution attempt exceeds the configured timeout.
 /// The timeout can be configured per-module using the <c>Timeout</c> property or module options.
+/// When retries are configured, each attempt receives a fresh timeout and retry delays are excluded.
 /// </para>
 /// <para><b>When this is thrown:</b></para>
 /// <list type="bullet">
-/// <item>When a module's <c>ExecuteAsync</c> takes longer than the configured timeout</item>
+/// <item>When a module's <c>ExecuteAsync</c> attempt takes longer than the configured timeout</item>
 /// <item>When the module does not respond to cancellation token within the grace period</item>
 /// </list>
 /// <para><b>Properties available:</b></para>