fix(runtime): carry typed pre-stream errors on QUIC response plane (#14750)

.github/actions/compliance-extract/action.yml

@@ -130,7 +130,6 @@ runs:
         GIT_SHA: ${{ inputs.git_sha }}
         EPP_IMAGE: ${{ inputs.epp_image }}
       run: |
-        mkdir -p /tmp/compliance
         # Reuse the warm cache the build just populated by matching ALL the
         # cache-key-affecting build-args the build passed. Otherwise the shared
         # stages rebuild here instead of cache-hitting:
@@ -174,7 +173,8 @@ runs:
         # --build-arg (it would clear the ARG and reintroduce the failure).
         EPP_IMAGE_ARG=""
         [ -n "${EPP_IMAGE:-}" ] && EPP_IMAGE_ARG="--build-arg EPP_IMAGE=${EPP_IMAGE}"
-        docker buildx build \
+        bash "${{ github.action_path }}/extract-with-retry.sh" /tmp/compliance \
+            docker buildx build \
             --builder ${{ inputs.builder_name }} \
             --platform ${{ inputs.platform }} \
             --target compliance_artifact \
@@ -183,7 +183,6 @@ runs:
             --build-arg SCCACHE_BUCKET="${SCCACHE_BUCKET}" \
             --build-arg SCCACHE_REGION="${SCCACHE_REGION}" \
             $EPP_IMAGE_ARG \
-            --output type=local,dest=/tmp/compliance \
             --progress=plain \
             -f "${DOCKERFILE_PATH}" \
             $SECRET_ARGS \
@@ -357,7 +356,6 @@ runs:
         GIT_SHA: ${{ inputs.git_sha }}
         EPP_IMAGE: ${{ inputs.epp_image }}
       run: |
-        mkdir -p /tmp/sources
         # ENABLE_SOURCE_ARCHIVAL=true to actually run sources_collect; the other
         # build-args match the build so this shares the warm cache (same reason
         # as the compliance_artifact extract above). Same IRSA secrets too, so a
@@ -385,7 +383,8 @@ runs:
         # resolves the GC'd default staging tag. Empty for non-frontend targets.
         EPP_IMAGE_ARG=""
         [ -n "${EPP_IMAGE:-}" ] && EPP_IMAGE_ARG="--build-arg EPP_IMAGE=${EPP_IMAGE}"
-        docker buildx build \
+        bash "${{ github.action_path }}/extract-with-retry.sh" /tmp/sources \
+            docker buildx build \
             --builder ${{ inputs.builder_name }} \
             --platform ${{ inputs.platform }} \
             --target sources_archive \
@@ -395,7 +394,6 @@ runs:
             --build-arg SCCACHE_REGION="${SCCACHE_REGION}" \
             --build-arg ENABLE_SOURCE_ARCHIVAL=true \
             $EPP_IMAGE_ARG \
-            --output type=local,dest=/tmp/sources \
             --progress=plain \
             -f "${DOCKERFILE_PATH}" \
             $SECRET_ARGS \

.github/actions/compliance-extract/extract-with-retry.sh

@@ -0,0 +1,41 @@
+#!/usr/bin/env bash
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+# Retry only transient BuildKit transport failures, preserving each attempt's log.
+set -euo pipefail
+
+destination=$1
+shift
+attempt_root=$(mktemp -d "${RUNNER_TEMP:-${TMPDIR:-/tmp}}/compliance-extract.XXXXXX")
+echo "Compliance extraction logs: ${attempt_root}"
+
+for attempt in 1 2 3; do
+    output="${attempt_root}/output-${attempt}"
+    log="${attempt_root}/attempt-${attempt}.log"
+    mkdir -p "$output"
+    status=0
+    "$@" --output "type=local,dest=${output}" 2>&1 | tee "$log" || status=$?
+    if [ "$status" -eq 0 ]; then
+        # Publish only a complete export; failed attempts never reach consumers.
+        rm -rf -- "$destination"
+        mv -- "$output" "$destination"
+        exit 0
+    fi
+    rm -rf -- "$output"
+
+    # Inspect the terminal build error, not earlier warnings or command output.
+    error=$(grep '^ERROR:' "$log" | tail -n 1 || true)
+    if printf '%s\n' "$error" | grep -Eiq \
+        '(process .*did not complete successfully|401 Unauthorized|403 Forbidden|denied:|manifest unknown)'; then
+        exit "$status"
+    fi
+    if [ "$attempt" -eq 3 ] || ! printf '%s\n' "$error" | grep -Eiq \
+        '(Unavailable:.*error reading from server: (unexpected )?EOF|connection reset by peer|transport is closing|unexpected status.*(502 Bad Gateway|503 Service Unavailable|504 Gateway Timeout))'; then
+        exit "$status"
+    fi
+    delay=10
+    [ "$attempt" -eq 2 ] && delay=30
+    echo "::warning::Compliance extraction attempt ${attempt}/3 failed with a transient transport error; retrying in ${delay}s"
+    sleep "$delay"
+done

.github/actions/compliance-extract/test_extract_with_retry.py

@@ -0,0 +1,118 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""Exercise the real retry wrapper with a fake BuildKit command and sleep."""
+
+import os
+import subprocess
+import tempfile
+import unittest
+from pathlib import Path
+
+SCRIPT = Path(__file__).with_name("extract-with-retry.sh")
+TRANSIENT = "ERROR: failed to build: failed to solve: Unavailable: error reading from server: EOF"
+
+
+class ExtractRetryTest(unittest.TestCase):
+    def run_extract(self, errors):
+        with tempfile.TemporaryDirectory() as temp:
+            root = Path(temp)
+            for index, error in enumerate(errors, 1):
+                (root / f"error-{index}").write_text(error)
+            fake = root / "build"
+            fake.write_text(
+                """#!/usr/bin/env bash
+set -eu
+count=0
+[ ! -f "$STATE/count" ] || count=$(cat "$STATE/count")
+count=$((count + 1))
+echo "$count" > "$STATE/count"
+for arg in "$@"; do
+    case "$arg" in type=local,dest=*) output=${arg#type=local,dest=} ;; esac
+done
+mkdir -p "$output"
+echo "$count" > "$output/attempt-$count"
+error="$STATE/error-$count"
+if [ -s "$error" ]; then
+    cat "$error"
+    exit 7
+fi
+"""
+            )
+            fake.chmod(0o755)
+            sleep = root / "sleep"
+            sleep.write_text('#!/usr/bin/env bash\necho "$1" >> "$STATE/delays"\n')
+            sleep.chmod(0o755)
+            destination = root / "result"
+            env = dict(
+                os.environ,
+                STATE=temp,
+                RUNNER_TEMP=temp,
+                PATH=temp + os.pathsep + os.environ["PATH"],
+            )
+            result = subprocess.run(
+                ["bash", str(SCRIPT), str(destination), str(fake)],
+                env=env,
+                capture_output=True,
+                text=True,
+            )
+            count = int((root / "count").read_text())
+            delays = (
+                (root / "delays").read_text().splitlines()
+                if (root / "delays").exists()
+                else []
+            )
+            files = sorted(p.name for p in destination.glob("*"))
+            logs = list(root.glob("compliance-extract.*/attempt-*.log"))
+            self.assertEqual(len(logs), count)
+            return result.returncode, count, delays, files
+
+    def test_success(self):
+        self.assertEqual(self.run_extract([""]), (0, 1, [], ["attempt-1"]))
+
+    def test_transient_recovers_without_partial_files(self):
+        self.assertEqual(
+            self.run_extract([TRANSIENT, ""]), (0, 2, ["10"], ["attempt-2"])
+        )
+
+    def test_exhaustion_preserves_failure(self):
+        self.assertEqual(self.run_extract([TRANSIENT] * 3), (7, 3, ["10", "30"], []))
+
+    def test_deterministic_errors_do_not_retry(self):
+        for error in [
+            "ERROR: failed to solve: process did not complete: exit code: 1",
+            "ERROR: failed to authorize: 401 Unauthorized",
+            "ERROR: manifest unknown",
+            "No solution found when resolving dependencies",
+        ]:
+            with self.subTest(error=error):
+                self.assertEqual(self.run_extract([error]), (7, 1, [], []))
+
+    def test_earlier_transient_does_not_mask_terminal_failure(self):
+        self.assertEqual(
+            self.run_extract([TRANSIENT + "\nERROR: manifest unknown"]), (7, 1, [], [])
+        )
+
+    def test_other_transport_errors_recover(self):
+        for error in [
+            "ERROR: failed to solve: connection reset by peer",
+            "ERROR: failed to solve: rpc error: transport is closing",
+            "ERROR: failed to solve: unexpected status from HEAD request: 503 Service Unavailable",
+        ]:
+            with self.subTest(error=error):
+                self.assertEqual(
+                    self.run_extract([error, ""]), (0, 2, ["10"], ["attempt-2"])
+                )
+
+    def test_build_command_containing_transport_text_does_not_retry(self):
+        error = 'ERROR: process "echo connection reset by peer" did not complete successfully: exit code: 1'
+        self.assertEqual(self.run_extract([error]), (7, 1, [], []))
+
+    def test_permanent_error_after_retry_stops(self):
+        self.assertEqual(
+            self.run_extract([TRANSIENT, "ERROR: manifest unknown"]), (7, 2, ["10"], [])
+        )
+
+
+if __name__ == "__main__":
+    unittest.main()

.github/actions/connect-vcluster/action.yml

@@ -167,5 +167,8 @@ runs:
         fi
 
         KUBECONFIG_B64=$(base64 < "${KUBECONFIG_PATH}" | tr -d '\n')
+        # Masked before export: the value carries a vCluster client certificate,
+        # and every consumer interpolates it into a shell script.
+        echo "::add-mask::${KUBECONFIG_B64}"
         echo "kubeconfig_base64=${KUBECONFIG_B64}" >> $GITHUB_OUTPUT
         echo "::endgroup::"

.github/actions/pytest-local/action.yml

@@ -176,10 +176,19 @@ runs:
         # Artifacts go to the GitHub Actions checkout so the per-job upload in
         # shared-test.yml can find them.
         TEST_RESULTS_DIR="${GITHUB_WORKSPACE}/test-results"
-        mkdir -p "${TEST_RESULTS_DIR}"
+        mkdir -p "${TEST_RESULTS_DIR}/status"
         chmod 777 "${TEST_RESULTS_DIR}"
         echo "📁 Test results will be saved to: ${TEST_RESULTS_DIR}"
 
+        # Emptied up front so the file read at the end of the job can only have
+        # come from this run of this stage. Truncated rather than removed: the
+        # distroless planner test image ships no rm (see the tool list in
+        # container/Dockerfile.test), and an empty file is the better signal
+        # anyway - absent means the stage never started, empty means it started
+        # and was cut off before recording an exit code.
+        STATUS_FILE="${TEST_RESULTS_DIR}/status/${STR_TEST_TYPE}.exitcode"
+        : > "${STATUS_FILE}"
+
         # NVMe guard: GPU runners mount ${NVME_DIR} (nvme-scratch emptyDir, NVMe-backed via
         # instanceStorePolicy: RAID0). CPU/ARM runners have no ${NVME_DIR} mount.
         if [[ -d "${NVME_DIR}" ]]; then
@@ -303,8 +312,40 @@ runs:
           echo "⚠️  JUnit XML file not found - test results may not be available for upload"
         fi
 
+        # Checked by "Verify test stages completed" in shared-test.yml, which
+        # cannot use the exit below or $GITHUB_ENV: a truncated script is still
+        # reported as a successful step and its $GITHUB_ENV writes are dropped.
+        # Keep this last - its absence is what proves the script was truncated.
+        echo "${TEST_EXIT_CODE}" > "${STATUS_FILE}"
+
         exit ${TEST_EXIT_CODE}
 
+    # conftest.py's autouse `logger` fixture writes every test's full root-logger
+    # output (ManagedProcess/EngineProcess health-check status, retry reasons,
+    # subprocess log tails, ...) to /tmp/dynamo_tests/<test-name>/test.log.txt --
+    # a much richer trace than the JUnit XML's truncated captured-output, but
+    # nothing ever uploaded it. failure()-gated (not always()) to keep this
+    # opt-in-by-outcome: pytest-local is shared by every framework's CI, and
+    # most runs pass.
+    - name: Collect per-test log files
+      if: failure()
+      shell: bash
+      run: |
+        set -euo pipefail
+        mkdir -p test-results/dynamo-tests-logs
+        if [ -d /tmp/dynamo_tests ]; then
+          cp -r /tmp/dynamo_tests/. test-results/dynamo-tests-logs/
+        fi
+
+    - name: Upload per-test log files
+      uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a  # v7.0.1
+      if: failure()
+      with:
+        name: dynamo-tests-logs-${{ inputs.test_suite_name }}-${{ inputs.test_type }}-${{ inputs.platform_arch }}-${{ github.run_id }}-${{ job.check_run_id }}
+        path: test-results/dynamo-tests-logs/
+        retention-days: 7
+        if-no-files-found: ignore
+
     # Allure results and coverage data are uploaded per stage, under the same
     # artifact names as before the step collapse, because an external CI-metrics
     # pipeline consumes them by name. `always()` because the step above exits

.github/codeowners/areas.yaml

@@ -234,7 +234,6 @@ areas:
   - components/src/dynamo/common/utils/
   - components/src/dynamo/experimental/
   - examples/backends/sample/
-  - examples/backends/tritonserver/
   - examples/common/
   - examples/custom_backend/
   - lib/backend-common/
@@ -459,6 +458,7 @@ areas:
   github_team: '@ai-dynamo/dynamo-rl-codeowners'
   path_globs:
   - components/src/dynamo/common/rl/
+  - examples/rl/
   - lib/rl/
   - tests/rl/
 - label: performance
@@ -1063,7 +1063,7 @@ shared:
   owners:
   - router
   - kv-memory
-- glob: lib/llm/src/preprocessor/lightseek_mm.rs
+- glob: lib/llm/src/preprocessor/mm_routing/
   owners:
   - frontend
   - multimodal
@@ -1275,10 +1275,6 @@ shared:
   owners:
   - ops
   - backend-vllm
-- glob: examples/backends/tritonserver/Dockerfile
-  owners:
-  - runtime
-  - ops
 - glob: examples/diffusers/Dockerfile
   owners:
   - diffusion

.github/filters.yaml

@@ -352,6 +352,7 @@ sidecar:
   # acceptable CI fan-out is agreed. Keep the remaining scope narrow for now.
   - 'lib/sidecar/**'
   - '.github/workflows/shared-build-sidecar.yml'
+  - 'tests/serve/test_sidecar.py'
   - 'Cargo.toml'
   - 'Cargo.lock'
   # Compliance policy and baseline changes can fail the sidecar image build.