From 935b4b7871849d83b8bbb52d874447c62c82086d Mon Sep 17 00:00:00 2001
From: Veera Rajasekhar <VeeraRajasekharReddy.Gopu@amd.com>
Date: Fri, 31 Jul 2026 07:13:17 +0000
Subject: [PATCH 1/9] Ci test run optimization

---
 .github/scripts/ci_sgpu_queue.conf |  19 ++
 .github/scripts/run_queue_sgpu.sh  | 434 +++++++++++++++++++++++++++++
 .github/workflows/rocm-ci.yml      |  66 ++++-
 ci/_utils.sh                       |  11 +
 ci/build_weights.py                | 194 +++++++++++++
 ci/core_scaling.sh                 | 135 +++++++++
 ci/jax.sh                          |  18 +-
 ci/pytorch.sh                      |  40 ++-
 ci/test_weights.gfx950.txt         |  93 +++++++
 9 files changed, 990 insertions(+), 20 deletions(-)
 create mode 100644 .github/scripts/ci_sgpu_queue.conf
 create mode 100755 .github/scripts/run_queue_sgpu.sh
 create mode 100755 ci/build_weights.py
 create mode 100755 ci/core_scaling.sh
 create mode 100644 ci/test_weights.gfx950.txt

diff --git a/.github/scripts/ci_sgpu_queue.conf b/.github/scripts/ci_sgpu_queue.conf
new file mode 100644
index 0000000000..598d530c7c
--- /dev/null
+++ b/.github/scripts/ci_sgpu_queue.conf
@@ -0,0 +1,19 @@
+# Copyright (c) 2026, Advanced Micro Devices, Inc. All rights reserved.
+#
+# See LICENSE for license information.
+#
+# sGPU suite definitions for the work-queue scheduler (run_queue_sgpu.sh).
+# Format: <label>  <logfile>  <mode>  <command> [args...]
+#
+#   mode=list    suite honours TE_CI_LIST_ONLY, so it is expanded into one work
+#                item per test invocation and those items are packed across all
+#                GPUs. Items are re-dispatched with TEST_FILTER=<tag>.
+#   mode=opaque  suite is scheduled as a single work item (whole script).
+#
+# Unlike ci_sgpu_jobs.conf, line order carries no GPU assignment: every item
+# from every suite goes into one queue and workers pull on completion.
+
+examples  examples.log  opaque  .github/scripts/run_examples.sh
+torch     torch.log     list    ci/pytorch.sh
+jax       jax.log       list    ci/jax.sh
+core      core.log      opaque  ci/core.sh
diff --git a/.github/scripts/run_queue_sgpu.sh b/.github/scripts/run_queue_sgpu.sh
new file mode 100755
index 0000000000..d554ebac1b
--- /dev/null
+++ b/.github/scripts/run_queue_sgpu.sh
@@ -0,0 +1,434 @@
+#!/bin/bash
+# Copyright (c) 2026, Advanced Micro Devices, Inc. All rights reserved.
+#
+# See LICENSE for license information.
+#
+# Run all sGPU test suites as one global work queue across N GPUs.
+#
+# Usage: run_queue_sgpu.sh [options] <config>...
+#   -n, --gpus <n>        number of GPUs / concurrent workers (default: 4)
+#   -g, --first-gpu <n>   first GPU id to use (default: 0)
+#   -l, --log-dir <dir>   where suite and per-item logs are written
+#   -w, --weights <file>  "<label>/<tag> <weight>" pairs used to order the queue
+#       --only <regex>    keep only items whose "<label>/<tag>" matches (smoke
+#                         tests, or re-running just the items that failed)
+#       --skip-setup      container prerequisites and the CK JIT cache are
+#                         already in place; go straight to the queue
+#       --dry-run         build and print the queue, run nothing
+#
+# Why a queue instead of one suite per GPU (run_parallel_sgpu.sh): the suites
+# are very unevenly sized, so a static 1:1 map leaves most GPUs idle waiting for
+# the largest suite. Here every suite is expanded into individual work items,
+# the items are ordered longest-first, and each worker pulls the next one when
+# it goes idle -- so a GPU only stops working when the queue is empty.
+#
+# Compatibility: after the queue drains, each suite's item logs are concatenated
+# into <log-dir>/<logfile> and the worst item exit code is written to
+# <logfile>.rc, so existing consumers of those paths keep working unchanged.
+
+set -u
+
+SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
+REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)"
+
+NUM_GPUS=${TEST_NUM_GPUS:-4}
+FIRST_GPU=${TEST_FIRST_GPU:-0}
+LOG_DIR=${LOG_DIR:-/tmp/te_ci_logs}
+WEIGHTS_FILE=""
+DRY_RUN=""
+ONLY_RE=""
+SKIP_SETUP=""
+# Items with no recorded weight sort first: an unknown item is more likely to be
+# a new (or newly slow) one, and a long item started late is what stretches the
+# tail. Losing the gamble costs far less than mis-scheduling a genuinely big item.
+DEFAULT_WEIGHT=${TE_CI_DEFAULT_WEIGHT:-999999}
+# Ceiling for the per-item deadline derived from the weight below. Also the
+# deadline for unknown items, whose DEFAULT_WEIGHT makes the derived budget
+# meaningless.
+MAX_ITEM_SECS=${TE_CI_MAX_ITEM_SECS:-7200}
+
+while [[ $# -gt 0 ]]; do
+    case "$1" in
+        -n|--gpus)       NUM_GPUS="$2"; shift 2 ;;
+        --gpus=*)        NUM_GPUS="${1#*=}"; shift ;;
+        -g|--first-gpu)  FIRST_GPU="$2"; shift 2 ;;
+        --first-gpu=*)   FIRST_GPU="${1#*=}"; shift ;;
+        -l|--log-dir)    LOG_DIR="$2"; shift 2 ;;
+        --log-dir=*)     LOG_DIR="${1#*=}"; shift ;;
+        -w|--weights)    WEIGHTS_FILE="$2"; shift 2 ;;
+        --weights=*)     WEIGHTS_FILE="${1#*=}"; shift ;;
+        --only)          ONLY_RE="$2"; shift 2 ;;
+        --only=*)        ONLY_RE="${1#*=}"; shift ;;
+        --skip-setup)    SKIP_SETUP=1; shift ;;
+        --dry-run)       DRY_RUN=1; shift ;;
+        -*)              echo "Unknown option: $1" >&2; exit 1 ;;
+        *)               break ;;
+    esac
+done
+
+if [[ $# -eq 0 ]]; then
+    echo "Error: at least one config file is required." >&2
+    exit 1
+fi
+
+resolved_configs=()
+for c in "$@"; do resolved_configs+=( "$(realpath -m "$c")" ); done
+[[ -n "$WEIGHTS_FILE" ]] && WEIGHTS_FILE="$(realpath -m "$WEIGHTS_FILE")"
+[[ "$LOG_DIR" != /* ]] && LOG_DIR="$(realpath -m "$LOG_DIR")"
+
+ITEM_LOG_DIR="$LOG_DIR/items"
+mkdir -p "$ITEM_LOG_DIR"
+cd "$REPO_ROOT" || { echo "Error: cannot cd to '${REPO_ROOT}'" >&2; exit 1; }
+
+QUEUE_FILE="$LOG_DIR/queue.tsv"
+IDX_FILE="$LOG_DIR/queue.idx"
+LOCK_FILE="$LOG_DIR/queue.lock"
+: > "$QUEUE_FILE"
+
+# ---------------------------------------------------------------------------
+# Phase 1: expand every suite into work items
+#
+# List mode runs the suite script with TE_CI_LIST_ONLY=1, which makes pytest_run
+# echo "TE_CI_ITEM <tag>" for each invocation it would have made instead of
+# running it. Level gating, backend matrix and capability probes all still apply,
+# so the emitted list is exactly what the suite would have executed.
+echo "== Expanding suites into work items =="
+declare -a SUITE_LABELS SUITE_LOGFILES
+while IFS= read -r line; do
+    [[ "$line" =~ ^[[:space:]]*# ]] && continue
+    [[ -z "${line//[[:space:]]/}" ]] && continue
+    read -r label logfile mode cmd rest <<< "$line"
+    SUITE_LABELS+=( "$label" )
+    SUITE_LOGFILES+=( "$logfile" )
+    if [[ "$mode" == "list" ]]; then
+        mapfile -t tags < <(TE_CI_LIST_ONLY=1 "$cmd" ${rest:-} 2>/dev/null \
+                            | sed -n 's/^TE_CI_ITEM //p')
+        if [[ ${#tags[@]} -eq 0 ]]; then
+            echo "::error::suite '${label}' (${cmd}) produced no work items" >&2
+            exit 1
+        fi
+        for tag in "${tags[@]}"; do
+            printf '%s\t%s\t%s\t%s\n' "$label" "$cmd" "$tag" "${rest:-}" >> "$QUEUE_FILE.raw"
+        done
+        echo "  ${label}: ${#tags[@]} items"
+    else
+        printf '%s\t%s\t%s\t%s\n' "$label" "$cmd" "" "${rest:-}" >> "$QUEUE_FILE.raw"
+        echo "  ${label}: 1 item (opaque)"
+    fi
+done < <(cat "${resolved_configs[@]}")
+
+# Duplicate tags within a suite would collide on both the JUnit XML filename and
+# the TEST_FILTER dispatch (one invocation would run both). Fail loudly instead.
+dupes=$(cut -f1,3 "$QUEUE_FILE.raw" | grep -v '^\S*\t$' | sort | uniq -d)
+if [[ -n "$dupes" ]]; then
+    echo "::error::duplicate work item tags detected:" >&2
+    echo "$dupes" >&2
+    exit 1
+fi
+
+# ---------------------------------------------------------------------------
+# Phase 2: weight and order the queue (longest processing time first)
+awk -v wf="$WEIGHTS_FILE" -v dw="$DEFAULT_WEIGHT" -v only="$ONLY_RE" '
+    BEGIN {
+        FS = OFS = "\t"
+        if (wf != "" && (getline line < wf) >= 0) {
+            do {
+                if (line !~ /^[[:space:]]*#/ && line !~ /^[[:space:]]*$/) {
+                    split(line, a, " "); if (a[1] != "") w[a[1]] = a[2]
+                }
+            } while ((getline line < wf) > 0)
+        }
+    }
+    {
+        key = $3 == "" ? $1 : $1 "/" $3
+        if (only != "" && key !~ only) next
+        # Truncate to an integer: the weight is also used in shell arithmetic to
+        # derive the per-item deadline, and bash cannot parse "675.0".
+        print (key in w ? int(w[key]) : dw), $0
+    }
+' "$QUEUE_FILE.raw" | sort -t$'\t' -k1,1nr > "$QUEUE_FILE"
+rm -f "$QUEUE_FILE.raw"
+
+if [[ ! -s "$QUEUE_FILE" ]]; then
+    echo "::error::queue is empty${ONLY_RE:+ (no item matched --only '${ONLY_RE}')}" >&2
+    exit 1
+fi
+
+TOTAL_ITEMS=$(wc -l < "$QUEUE_FILE")
+echo "== Queue: ${TOTAL_ITEMS} items over ${NUM_GPUS} GPUs (first=${FIRST_GPU}) =="
+if [[ -n "$WEIGHTS_FILE" && -f "$WEIGHTS_FILE" ]]; then
+    echo "   weights: $WEIGHTS_FILE"
+else
+    echo "   weights: none (unordered -- run ci/build_weights.py after this run)"
+fi
+
+# The plan is printed on every run, not just --dry-run: it is the only record of
+# what the scheduler intended, and comparing it against the schedule reported at
+# the end is how a mis-weighted item is spotted.
+echo "== Queue plan (dispatch order; est = weight used to sort) =="
+awk -F'\t' -v dw="$DEFAULT_WEIGHT" -v n="$NUM_GPUS" '
+    { total += $1
+      printf "  %3d. est=%-8s %-9s %s\n", NR, ($1 == dw ? "unknown" : $1 "s"), $2,
+             ($4 == "" ? "(whole suite)" : $4) }
+    END {
+        printf "\n  estimated work %ss over %s GPUs -> lower bound %.0fs", total, n, total / n
+        printf " (largest item %ss)\n", first
+    }
+    NR == 1 { first = ($1 == dw ? "unknown" : $1) }
+' "$QUEUE_FILE"
+
+if [[ -n "$DRY_RUN" ]]; then
+    echo "== Dry run: nothing executed =="
+    exit 0
+fi
+
+# ---------------------------------------------------------------------------
+# Phase 3: one-time, container-wide setup
+#
+# Prerequisites and the CK JIT blob cache are shared filesystem state, so they
+# are built once here rather than by each of the N*items invocations. Running
+# the suites sequentially also removes the race they previously had over the
+# shared cache directory when launched in parallel.
+if [[ -n "$SKIP_SETUP" ]]; then
+    echo "== One-time suite setup: skipped (--skip-setup) =="
+fi
+for i in "${!SUITE_LABELS[@]}"; do
+    [[ -n "$SKIP_SETUP" ]] && break
+    [[ $i -eq 0 ]] && echo "== One-time suite setup =="
+    label="${SUITE_LABELS[$i]}"
+    # Columns are: weight, label, cmd, tag, rest. Only list-mode suites (those
+    # with a non-empty tag) have a setup/dispatch split; opaque suites are a
+    # single invocation that still does its own setup.
+    cmd=$(awk -F'\t' -v l="$label" '$2==l && $4!="" {print $3; exit}' "$QUEUE_FILE")
+    [[ -z "$cmd" ]] && continue
+    echo "  setup: $label ($cmd)"
+    if ! HIP_VISIBLE_DEVICES=$FIRST_GPU TE_CI_SETUP_ONLY=1 "$cmd" \
+            > "$LOG_DIR/setup.${label}.log" 2>&1; then
+        echo "::error::setup failed for ${label}; see $LOG_DIR/setup.${label}.log" >&2
+        tail -30 "$LOG_DIR/setup.${label}.log" >&2
+        exit 1
+    fi
+done
+
+# ---------------------------------------------------------------------------
+# Phase 4: run the queue
+echo 1 > "$IDX_FILE"
+: > "$LOCK_FILE"
+START_TS=$(date +%s)
+
+take_next() {
+    local i
+    {
+        flock 9
+        i=$(cat "$IDX_FILE")
+        echo $((i + 1)) > "$IDX_FILE"
+    } 9<>"$LOCK_FILE"
+    echo "$i"
+}
+
+worker() {
+    local gpu=$1
+    local i line weight label cmd tag rest itemlog rc safetag junit_dir start end
+    while :; do
+        i=$(take_next)
+        [[ "$i" -gt "$TOTAL_ITEMS" ]] && break
+        line=$(sed -n "${i}p" "$QUEUE_FILE")
+        [[ -z "$line" ]] && break
+        IFS=$'\t' read -r weight label cmd tag rest <<< "$line"
+
+        safetag="${tag:-whole}"
+        itemlog="$ITEM_LOG_DIR/${label}.${safetag}.log"
+        if [[ -n "${JUNITXML_PREFIX:-}${JUNITXML_SUFFIX:-}" ]]; then
+            junit_dir="${JUNITXML_PREFIX:-}${label}/"
+            mkdir -p "$junit_dir"
+        else
+            junit_dir=""
+        fi
+
+        # Bound the item at 3x its learned weight, plus slack for the ~5% noise
+        # floor and a genuinely slower run; past that it is hung, not slow.
+        # Nothing else bounds an *item*: pytest's --timeout is per test, so an
+        # item of many just-under-PYTEST_TIMEOUT tests -- or a hang during
+        # collection or conftest import, which pytest-timeout does not cover --
+        # otherwise runs until the job's timeout-minutes kills all N workers at
+        # once, discarding every in-flight item on every GPU. rc 124 here means
+        # the deadline fired; build_weights.py drops those rows, since the
+        # measurement is the ceiling itself and would compound run over run.
+        budget=$(( weight * 3 + 300 ))
+        [[ $budget -gt $MAX_ITEM_SECS ]] && budget=$MAX_ITEM_SECS
+
+        start=$(date +%s)
+        if [[ -n "$tag" ]]; then
+            HIP_VISIBLE_DEVICES=$gpu TE_CI_SKIP_SETUP=1 TEST_FILTER="$tag" \
+                JUNITXML_PREFIX="$junit_dir" \
+                timeout --kill-after=60 "$budget" "$cmd" ${rest:-} > "$itemlog" 2>&1
+        else
+            HIP_VISIBLE_DEVICES=$gpu JUNITXML_PREFIX="$junit_dir" \
+                timeout --kill-after=60 "$budget" "$cmd" ${rest:-} > "$itemlog" 2>&1
+        fi
+        rc=$?
+        if [[ $rc -eq 124 || $rc -eq 137 ]]; then
+            echo "::error::[${label}] ${safetag} killed at its ${budget}s deadline" >&2
+            echo "TE_CI_ITEM_DEADLINE_EXCEEDED after ${budget}s" >> "$itemlog"
+        fi
+        end=$(date +%s)
+
+        echo "$rc" > "${itemlog}.rc"
+        # Machine-readable timing record. The first five columns are the stable
+        # contract ci/build_weights.py --from-timing reads; the scheduling
+        # columns are appended after them so that reader stays unchanged. Written
+        # incrementally so a run killed by the job timeout still leaves a usable
+        # record of everything that had completed.
+        printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n' \
+            "$label" "$safetag" "$gpu" "$((end - start))" "$rc" \
+            "$((start - START_TS))" "$((end - START_TS))" "$weight" \
+            >> "$LOG_DIR/timings.tsv"
+        printf '[%s] gpu%-2s t+%-7s %5ss rc=%-4s est=%-8s %-9s %s\n' \
+            "$(date '+%H:%M:%S')" "$gpu" "$((start - START_TS))s" "$((end - start))" \
+            "$rc" "$([[ $weight -eq $DEFAULT_WEIGHT ]] && echo unknown || echo "${weight}s")" \
+            "$label" "$safetag"
+    done
+}
+
+: > "$LOG_DIR/timings.tsv"
+for k in $(seq 0 $((NUM_GPUS - 1))); do
+    worker $((FIRST_GPU + k)) &
+done
+wait
+
+WALL=$(( $(date +%s) - START_TS ))
+# The report divides by WALL in several places; a sub-second run (a --only of one
+# trivial item) would otherwise abort the whole report on a division by zero.
+[[ $WALL -lt 1 ]] && WALL=1
+
+# ---------------------------------------------------------------------------
+# Phase 5: fold per-item results back into per-suite logs and exit codes
+OVERALL_RC=0
+for i in "${!SUITE_LABELS[@]}"; do
+    label="${SUITE_LABELS[$i]}"
+    # A suite with no items in the queue (filtered out by --only) must not get a
+    # suite log and rc=0 -- that would read as "passed" to downstream consumers.
+    awk -F'\t' -v l="$label" '$2==l {found=1} END {exit !found}' "$QUEUE_FILE" || continue
+    suite_log="$LOG_DIR/${SUITE_LOGFILES[$i]}"
+    : > "$suite_log"
+    worst=0
+    for itemlog in "$ITEM_LOG_DIR/${label}."*.log; do
+        [[ -e "$itemlog" ]] || continue
+        rc=$(cat "${itemlog}.rc" 2>/dev/null || echo 1)
+        echo "##### $(basename "$itemlog" .log) (rc=${rc}) #####" >> "$suite_log"
+        cat "$itemlog" >> "$suite_log"
+        [[ "$rc" != "0" ]] && worst=$rc
+    done
+    echo "$worst" > "${suite_log}.rc"
+    [[ "$worst" != "0" ]] && OVERALL_RC=$worst
+done
+
+# ---------------------------------------------------------------------------
+# Phase 6: scheduling report
+#
+# timings.tsv columns: label, tag, gpu, secs, rc, start_off, end_off, est.
+# Everything below is derived from that one file, so the same report can be
+# regenerated after the fact from an uploaded artifact.
+report() {
+    local tf="$LOG_DIR/timings.tsv"
+
+    echo "=== sGPU queue: ${TOTAL_ITEMS} items, ${NUM_GPUS} GPUs, drained in ${WALL}s ==="
+    echo
+    echo "-- schedule: what ran where, in execution order --"
+    sort -t$'\t' -k3,3n -k6,6n "$tf" | awk -F'\t' -v dw="$DEFAULT_WEIGHT" '
+        $3 != last { if (NR > 1) print ""; last = $3 }
+        {
+            est = ($8 == dw ? "unknown" : $8 "s")
+            # Estimate error is the scheduler feedback loop made visible: a large
+            # positive miss is an item that should have been dispatched earlier.
+            err = ($8 > 0 && $8 != dw) ? sprintf("%+.0f%%", ($4 - $8) * 100 / $8) : "n/a"
+            printf "  gpu%-2s  t+%-7s %6ss  rc=%-4s est=%-9s %-7s %s/%s\n",
+                   $3, $6 "s", $4, $5, est, err, $1, $2
+        }'
+
+    echo
+    echo "-- per-GPU utilisation --"
+    awk -F'\t' -v w="$WALL" '
+        { busy[$3] += $4; n[$3]++; if ($5 != 0) bad[$3]++; if ($7 > last[$3]) last[$3] = $7 }
+        END {
+            for (g in busy)
+                printf "  gpu%-3s %3d items  %6ss busy  %5ss idle  %5.1f%% util  %d failed\n",
+                       g, n[g], busy[g], w - busy[g], busy[g] * 100 / w, bad[g] + 0
+        }' "$tf" | sort
+
+    echo
+    echo "-- timeline (one column ~$(( WALL / 60 + 1 ))s; = ran, # failed, . idle) --"
+    # Idle gaps are the whole point: a run of dots on one GPU while another is
+    # still busy is exactly the tail a better ordering would have filled.
+    awk -F'\t' -v w="$WALL" -v cols=60 -v first="$FIRST_GPU" -v n="$NUM_GPUS" '
+        { from = int($6 * cols / w); to = int($7 * cols / w)
+          if (to >= cols) to = cols - 1
+          for (c = from; c <= to; c++) cell[$3, c] = ($5 == 0 ? "=" : "#") }
+        END {
+            for (g = first; g < first + n; g++) {
+                line = ""
+                for (c = 0; c < cols; c++) line = line ((g, c) in cell ? cell[g, c] : ".")
+                printf "  gpu%-3s |%s|\n", g, line
+            }
+            printf "  %7s 0s%*ss\n", "", cols - 2, w
+        }' "$tf"
+
+    echo
+    echo "-- efficiency --"
+    awk -F'\t' -v w="$WALL" -v n="$NUM_GPUS" '
+        { work += $4; if ($4 > big) { big = $4; bigname = $1 "/" $2 } }
+        END {
+            bound = work / n; if (big > bound) bound = big
+            printf "  total work      %8.0fs across %d items\n", work, NR
+            printf "  capacity        %8.0fs (%ss x %d GPUs)\n", w * n, w, n
+            printf "  lower bound     %8.0fs  max(work/%d, largest item)\n", bound, n
+            printf "  actual makespan %8.0fs  %+.1f%% over the bound\n", w, (w - bound) * 100 / bound
+            printf "  utilisation     %8.1f%%\n", work * 100 / (w * n)
+            printf "  largest item    %8.0fs  %s%s\n", big, bigname,
+                   (big > work / n ? "   <-- item-bound: splitting would now pay" : "")
+        }' "$tf"
+
+    echo
+    echo "-- weight accuracy (misses over 30s; these are what the next run corrects) --"
+    # 30s is roughly one process startup, and below the ~5% run-to-run noise for
+    # anything long enough to matter. Listing smaller misses would bury the real
+    # ones -- a healthy table has almost every item within a few seconds.
+    awk -F'\t' -v dw="$DEFAULT_WEIGHT" '$8 != dw && $8 > 0 {
+            d = $4 - $8; a = (d < 0 ? -d : d)
+            if (a >= 30) printf "%.0f\t%+.0f\t%s\t%s\t%s\n", a, d, $8, $4, $1 "/" $2 }' "$tf" \
+        | sort -k1,1nr | head -15 \
+        | awk -F'\t' '{printf "  %+7ss  est %6ss -> actual %6ss  %s\n", $2, $3, $4, $5}'
+    awk -F'\t' -v dw="$DEFAULT_WEIGHT" '$8 == dw {printf "  unknown     -> actual %6ss  %s/%s\n", $4, $1, $2}' "$tf"
+
+    echo
+    echo "-- failures --"
+    if awk -F'\t' '$5 != 0 {found = 1} END {exit !found}' "$tf"; then
+        awk -F'\t' '$5 != 0 {
+            printf "  rc=%-4s %6ss  %s/%s%s\n", $5, $4, $1, $2,
+                   ($5 == 124 || $5 == 137 ? "   (killed at per-item deadline)" : "") }' "$tf"
+    else
+        echo "  none"
+    fi
+
+    # An item that never got a timings.tsv row was never dispatched -- only
+    # possible if a worker died outright. Silence here would read as success.
+    local ran; ran=$(wc -l < "$tf")
+    if [[ "$ran" -ne "$TOTAL_ITEMS" ]]; then
+        echo
+        echo "  !! ${ran}/${TOTAL_ITEMS} items produced a timing record; the rest never ran:"
+        awk -F'\t' 'NR == FNR {seen[$1 "/" $2] = 1; next}
+                    { k = ($4 == "" ? $2 "/whole" : $2 "/" $4); if (!(k in seen)) print "     " k }' \
+            "$tf" "$QUEUE_FILE"
+    fi
+}
+
+report | tee "$LOG_DIR/schedule.txt"
+{
+    echo "## sGPU queue schedule (${WALL}s over ${NUM_GPUS} GPUs)"
+    echo
+    echo '```'
+    cat "$LOG_DIR/schedule.txt"
+    echo '```'
+} > "$LOG_DIR/schedule.md"
+
+exit $OVERALL_RC
diff --git a/.github/workflows/rocm-ci.yml b/.github/workflows/rocm-ci.yml
index 5591f1f399..ceeb9a134f 100644
--- a/.github/workflows/rocm-ci.yml
+++ b/.github/workflows/rocm-ci.yml
@@ -182,29 +182,81 @@ jobs:
           EOF
           )"
 
-      - name: Run sGPU tests in parallel (pytorch, jax, examples, core)
+      # The scheduler orders its queue longest-first, so it needs a per-item cost
+      # estimate. That table is produced by the previous run of this same job and
+      # carried forward in the cache -- there is no separate measurement pass.
+      # `restore` and `save` are separate actions on purpose: `actions/cache`
+      # skips its implicit save when the job fails, and a failing test run is
+      # exactly when the timings are most worth keeping.
+      - name: Restore learned test weights
+        uses: actions/cache/restore@v4
+        with:
+          path: ci-weights
+          key: te-sgpu-weights-${{ matrix.arch_label }}-${{ github.run_id }}
+          restore-keys: te-sgpu-weights-${{ matrix.arch_label }}-
+
+      - name: Run sGPU tests as one queue (pytorch, jax, examples, core)
         id: run-tests
         # Below the job's timeout-minutes so an overrun kills only this step;
         # the `if: always()` report + upload steps still run (artifacts survive).
         timeout-minutes: 240
         env:
           HF_TOKEN: ${{ secrets.HF_TOKEN }}
+          ARCH: ${{ matrix.arch_label == 'mi30x' && 'gfx942' || 'gfx950' }}
         run: |
           # One subdir per suite so PyTorch/JAX/Core get independent reports and
           # never collide on identically-named XML (e.g. test_sanity_import.auto.xml
           # exists in both the torch and jax suites and runs in parallel).
           rm -rf test-results && mkdir -p test-results/torch test-results/jax test-results/core
 
+          # Seed from the in-tree table on a cache miss; after that the cached
+          # copy is authoritative, since it has been refined by every run since.
+          mkdir -p ci-weights
+          W="ci-weights/test_weights.${ARCH}.txt"
+          [ -s "$W" ] || cp "ci/test_weights.${ARCH}.txt" "$W" 2>/dev/null || true
+          echo "weights: $W ($(wc -l < "$W" 2>/dev/null || echo 0) items)"
+
           docker exec \
             -e TEST_SGPU=1 \
             -e TEST_LEVEL=${{ env.TEST_LEVEL }} \
             -e JUNITXML_PREFIX=/workspace/test-results/ \
             -e JUNITXML_SUFFIX=.xml \
             -e HF_TOKEN="$HF_TOKEN" \
-            te-runner bash /workspace/.github/scripts/run_parallel_sgpu.sh \
-              --first-gpu 0 \
+            te-runner bash /workspace/.github/scripts/run_queue_sgpu.sh \
+              --gpus 4 --first-gpu 0 \
               --log-dir /workspace/ \
-              /workspace/.github/scripts/ci_sgpu_jobs.conf
+              --weights "/workspace/$W" \
+              /workspace/.github/scripts/ci_sgpu_queue.conf
+
+      # Runs even when tests failed: a red run still measures how long each item
+      # took, and build_weights.py's asymmetric blend is built to absorb the bias
+      # a short-because-it-crashed measurement introduces. Only items killed at
+      # their deadline (rc 124/137) are dropped, since that measurement is a
+      # ceiling that would otherwise compound run over run.
+      - name: Update learned test weights
+        if: always() && hashFiles('timings.tsv') != ''
+        env:
+          ARCH: ${{ matrix.arch_label == 'mi30x' && 'gfx942' || 'gfx950' }}
+        run: |
+          docker exec te-runner python3 /workspace/ci/build_weights.py \
+            --from-timing /workspace/timings.tsv \
+            --merge -o "/workspace/ci-weights/test_weights.${ARCH}.txt"
+
+      - name: Save learned test weights
+        if: always() && hashFiles('ci-weights/*') != ''
+        uses: actions/cache/save@v4
+        with:
+          path: ci-weights
+          key: te-sgpu-weights-${{ matrix.arch_label }}-${{ github.run_id }}
+
+      - name: Publish scheduling report
+        if: always()
+        run: |
+          if [ -f schedule.md ]; then
+            cat schedule.md >> "$GITHUB_STEP_SUMMARY"
+          else
+            echo "::warning::no schedule.md -- the queue did not reach its report phase"
+          fi
 
       - name: Generate test report
         if: always()
@@ -224,7 +276,7 @@ jobs:
         if: always()
         run: |
           EXIT_STATUS=0
-          for config in .github/scripts/ci_sgpu_jobs.conf; do
+          for config in .github/scripts/ci_sgpu_queue.conf; do
             while IFS= read -r line || [[ -n "$line" ]]; do
               [[ "$line" =~ ^[[:space:]]*# ]] && continue
               [[ -z "${line//[[:space:]]/}" ]] && continue
@@ -249,6 +301,10 @@ jobs:
           path: |
             *.log
             test-results/**/*.xml
+            timings.tsv
+            schedule.txt
+            queue.tsv
+            ci-weights/*.txt
           if-no-files-found: ignore
           retention-days: 5
 
diff --git a/ci/_utils.sh b/ci/_utils.sh
index f2478fc762..f54f7e74a9 100644
--- a/ci/_utils.sh
+++ b/ci/_utils.sh
@@ -350,6 +350,17 @@ pytest_run() {
     shift 3
     _test_name_tag=`get_test_name_tag $1 $_test_variant_tag`
     check_test_filter $_test_name_tag || return
+    # List mode: emit the work item instead of running it, so an external
+    # scheduler can pack items across GPUs. The tag alone is enough to
+    # re-dispatch the item: setting TEST_FILTER to it and re-entering this
+    # script replays the very same call line, so the inline NVTE_* prefixes and
+    # -k expressions are reapplied by the script itself and never have to be
+    # serialized here. The suite scripts stay the single source of truth for
+    # what runs at each TEST_LEVEL.
+    if [ -n "$TE_CI_LIST_ONLY" ]; then
+        echo "TE_CI_ITEM $_test_name_tag"
+        return
+    fi
     _start_ts=`date +%s`
     echo "Run [$_test_variant_tag] $@ at `time_elapsed $TEST_START_TS`"
     # A per-test timeout is applied to every item. Callers may still append their
diff --git a/ci/build_weights.py b/ci/build_weights.py
new file mode 100755
index 0000000000..239fa7ee81
--- /dev/null
+++ b/ci/build_weights.py
@@ -0,0 +1,194 @@
+#!/usr/bin/env python3
+# Copyright (c) 2026, Advanced Micro Devices, Inc. All rights reserved.
+#
+# See LICENSE for license information.
+
+"""Build a work-item weight table for the sGPU queue scheduler.
+
+``run_queue_sgpu.sh`` orders its queue longest-first, which needs a per-item
+cost estimate. Both sources below are produced by a normal CI run, so no extra
+measurement pass is required:
+
+  --from-timing <file>  read ``timings.tsv`` written by the scheduler itself.
+                        Preferred for scheduling: it is per-item wall clock, so
+                        it includes process startup and CUDA/HIP init, which is
+                        exactly what the makespan is made of.
+  --from-junit <dir>    sum the per-testcase ``time`` attributes of every
+                        ``<label>/<tag>.xml`` the run wrote. Useful when only
+                        the ``logs-sgpu-*`` artifacts are available, but it
+                        counts test time only -- a short item can look ~4x
+                        cheaper than it really is once startup is included.
+
+Output is ``<label>/<tag> <seconds>`` per line, plus ``<label> <seconds>`` for
+opaque whole-suite items. Weights are per GPU architecture -- skip patterns
+differ between gfx942 and gfx950 -- so keep one table per arch.
+
+With ``--merge`` the output file is read first and the new measurement is
+blended into it, so every CI run refines the table instead of replacing it from
+a single noisy sample. See ``merge_weights`` for why the blend is asymmetric.
+"""
+
+import argparse
+import os
+import sys
+import xml.etree.ElementTree as ET
+from collections import defaultdict
+
+
+def from_junit(root_dir):
+    """Sum testcase durations for every <label>/<tag>.xml under root_dir."""
+    weights = defaultdict(float)
+    for label in sorted(os.listdir(root_dir)):
+        subdir = os.path.join(root_dir, label)
+        if not os.path.isdir(subdir):
+            continue
+        for name in sorted(os.listdir(subdir)):
+            if not name.endswith(".xml"):
+                continue
+            tag = name[: -len(".xml")]
+            path = os.path.join(subdir, name)
+            try:
+                tree = ET.parse(path)
+            except ET.ParseError as err:
+                # A truncated report means the run was cut off mid-item. Skipping
+                # it keeps the weight unknown, which sorts the item first next
+                # time -- the safe direction for something that may be slow.
+                print(f"warning: skipping unparsable {path}: {err}", file=sys.stderr)
+                continue
+            total = 0.0
+            for case in tree.getroot().iter("testcase"):
+                try:
+                    total += float(case.get("time") or 0.0)
+                except ValueError:
+                    pass
+            weights[f"{label}/{tag}"] += total
+    return weights
+
+
+# rc values that mean the item was killed rather than measured. 124 is the
+# scheduler's per-item deadline (GNU timeout), 137 its SIGKILL follow-up or an
+# OOM-kill. Either way the recorded duration is a ceiling imposed from outside,
+# not the item's cost -- and merging a ceiling is the one case the asymmetric
+# blend cannot absorb, because it raises the weight, which raises next run's
+# deadline, which raises the next ceiling. Drop them; the previous weight stands.
+KILLED_RCS = frozenset(("124", "137"))
+
+
+def from_timing(path):
+    """Read the scheduler's timings.tsv: label, tag, gpu, seconds, rc."""
+    weights = defaultdict(float)
+    with open(path, encoding="utf-8") as handle:
+        for line in handle:
+            fields = line.rstrip("\n").split("\t")
+            if len(fields) < 4:
+                continue
+            label, tag, _gpu, secs = fields[0], fields[1], fields[2], fields[3]
+            if len(fields) > 4 and fields[4] in KILLED_RCS:
+                continue
+            try:
+                value = float(secs)
+            except ValueError:
+                continue
+            # "whole" is the scheduler's placeholder for an opaque suite item,
+            # which the queue keys by bare label.
+            key = label if tag == "whole" else f"{label}/{tag}"
+            weights[key] += value
+    return weights
+
+
+def read_weights(path):
+    """Read an existing ``<key> <seconds>`` table. Missing file -> empty table."""
+    weights = {}
+    try:
+        handle = open(path, encoding="utf-8")
+    except OSError:
+        return weights
+    with handle:
+        for line in handle:
+            line = line.strip()
+            if not line or line.startswith("#"):
+                continue
+            key, _, value = line.rpartition(" ")
+            try:
+                weights[key] = float(value)
+            except ValueError:
+                continue
+    return weights
+
+
+def merge_weights(old, new, alpha_up=0.5, alpha_down=0.1):
+    """Blend a fresh measurement into the running table.
+
+    The blend is deliberately asymmetric, because the scheduler's two error
+    directions are not equally expensive. Over-estimating an item only starts it
+    earlier than it needed to be started; under-estimating one leaves it running
+    after the other GPUs have gone idle, and that is what sets the makespan. So
+    a rise is trusted quickly (half the gap in one run) while a fall is trusted
+    slowly (a tenth), which also makes the table shrug off a single short run --
+    a crash partway through, or a machine having a fast day -- without needing
+    to decide whether that run was "valid".
+    """
+    merged = dict(old)
+    for key, value in new.items():
+        previous = merged.get(key)
+        if previous is None:
+            merged[key] = value
+            continue
+        alpha = alpha_up if value > previous else alpha_down
+        merged[key] = (1.0 - alpha) * previous + alpha * value
+    return merged
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter
+    )
+    source = parser.add_mutually_exclusive_group(required=True)
+    source.add_argument(
+        "--from-junit", metavar="DIR", help="directory of per-suite JUnit XML subdirectories"
+    )
+    source.add_argument(
+        "--from-timing", metavar="FILE", help="timings.tsv written by run_queue_sgpu.sh"
+    )
+    parser.add_argument("-o", "--output", default="-", help="output file (default: stdout)")
+    parser.add_argument(
+        "--merge",
+        action="store_true",
+        help="blend into the existing --output table instead of replacing it",
+    )
+    args = parser.parse_args()
+
+    if args.from_junit:
+        weights = from_junit(args.from_junit)
+    else:
+        weights = from_timing(args.from_timing)
+
+    if not weights:
+        print("error: no weights produced from the given source", file=sys.stderr)
+        return 1
+
+    if args.merge:
+        if args.output == "-":
+            parser.error("--merge needs -o/--output (the table to merge into)")
+        previous = read_weights(args.output)
+        weights = merge_weights(previous, weights)
+        print(
+            f"merged {len(weights)} weights "
+            f"({len(previous)} known, {len(set(weights) - set(previous))} new)",
+            file=sys.stderr,
+        )
+
+    lines = [
+        f"{key} {value:.1f}\n" for key, value in sorted(weights.items(), key=lambda kv: -kv[1])
+    ]
+    if args.output == "-":
+        sys.stdout.writelines(lines)
+    else:
+        with open(args.output, "w", encoding="utf-8") as handle:
+            handle.writelines(lines)
+        print(f"wrote {len(lines)} weights to {args.output}", file=sys.stderr)
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())
diff --git a/ci/core_scaling.sh b/ci/core_scaling.sh
new file mode 100755
index 0000000000..d21ca42713
--- /dev/null
+++ b/ci/core_scaling.sh
@@ -0,0 +1,135 @@
+#!/bin/bash
+# Copyright (c) 2026, Advanced Micro Devices, Inc. All rights reserved.
+#
+# See LICENSE for license information.
+#
+# Measure how ci/core.sh scales with ctest job count and OMP thread count.
+#
+# ci/core.sh hardcodes n_parallel_jobs=4 and derives OMP_NUM_THREADS from it as
+# physical_cores/n_parallel_jobs. That 4 is a CPU-partitioning number -- it makes
+# jobs*threads equal the core count -- not a measured optimum; it was raised to 8
+# once (PR #426) and came back to 4 as part of an unrelated revert (PR #440),
+# never having been evaluated with core pinned to a single GPU. This script
+# answers the question that revert skipped:
+#
+#   A  current policy, OMP = cores/j: does raising j help under the coupling?
+#   B  decoupled, OMP pinned to 8:    is the coupling what limits j?
+#   C  j fixed at 4, OMP swept:       do these tests use OMP threads at all?
+#
+# The gtest suites are registered one-ctest-test-per-case (gtest_discover_tests),
+# so the workload is dominated by process startup rather than GPU work. If C
+# shows a flat curve, core's CPU budget is being spent on threads it does not use
+# instead of on concurrency it does.
+#
+# Usage:  [GPU=0] [FILTER='-E GEMMTestSuite'] ci/core_scaling.sh
+#
+#   GPU     device to pin every run to. Keep this set -- leaving all devices
+#           visible is what made PR #426 pile every job onto device 0.
+#   FILTER  ctest selection, passed through verbatim. Defaults to the non-GEMM
+#           half (the larger one, ~6.4k tests). Use '-R GEMMTestSuite' for the
+#           other half, or '' for everything.
+#
+# Run on an idle machine: a concurrent CI job invalidates every number here.
+# Results are appended to /tmp/core_scaling.tsv; per-config ctest output stays in
+# /tmp/core_run.<name>.log.
+
+set -u
+
+GPU=${GPU:-0}
+FILTER=${FILTER:--E GEMMTestSuite}
+OUT=${OUT:-/tmp/core_scaling.tsv}
+
+DIR=$(cd "$(dirname "$0")" && pwd)
+TEST_DIR=${TE_PATH:-$(cd "$DIR/.." && pwd)/}tests/cpp
+
+cd "$TEST_DIR" || { echo "cannot cd to $TEST_DIR" >&2; exit 1; }
+
+CORES=$(( $(lscpu | awk '/^CPU\(s\):/{print $2}') \
+        / $(lscpu | awk '/Thread\(s\) per core:/{print $NF}') ))
+
+# ---------------------------------------------------------------------------
+# Preflight. The previous version of this script reported 0.0s for every config
+# because ctest exited immediately and its output was discarded. Refuse to
+# produce a number unless tests were actually discovered.
+if [ ! -d build ]; then
+    echo "== configuring + building (one time, 10-20 min) =="
+    if ! cmake -GNinja -Bbuild . > /tmp/core_cfg.log 2>&1; then
+        tail -30 /tmp/core_cfg.log >&2; exit 1
+    fi
+    if ! cmake --build build >> /tmp/core_cfg.log 2>&1; then
+        tail -30 /tmp/core_cfg.log >&2; exit 1
+    fi
+fi
+
+NTESTS=$(HIP_VISIBLE_DEVICES=$GPU ctest --test-dir build -N $FILTER 2>/dev/null \
+         | awk '/Total Tests:/{print $3}')
+if [ -z "$NTESTS" ] || [ "$NTESTS" -eq 0 ]; then
+    echo "ABORT: ctest discovered no tests in $TEST_DIR/build" >&2
+    echo "A build configured for a different container path will do this;" >&2
+    echo "remove build/ and re-run to reconfigure. Raw ctest output:" >&2
+    HIP_VISIBLE_DEVICES=$GPU ctest --test-dir build -N $FILTER 2>&1 | tail -20 >&2
+    exit 1
+fi
+
+echo "cores=$CORES  gpu=$GPU  filter=${FILTER:-<all>}  discovered=$NTESTS tests"
+
+# ---------------------------------------------------------------------------
+run() {
+    _name=$1; _jobs=$2; _omp=$3
+    _log=/tmp/core_run.${_name}.log
+    _smi=/tmp/core_smi.${_name}
+
+    # Sample utilisation and VRAM alongside the run. Peak VRAM is the ceiling on
+    # how far j can be raised regardless of what the wall clock says.
+    ( while :; do
+          rocm-smi --showuse --showmeminfo vram --csv 2>/dev/null \
+              | awk -F, -v g="card$GPU" '$1 ~ g {print $2"\t"$4}'
+          sleep 2
+      done ) > "$_smi" 2>/dev/null &
+    _smi_pid=$!
+
+    _t0=$(date +%s)
+    OMP_NUM_THREADS=$_omp HIP_VISIBLE_DEVICES=$GPU \
+        ctest --test-dir build -j"$_jobs" --timeout 600 $FILTER > "$_log" 2>&1
+    _rc=$?
+    _t1=$(date +%s)
+    kill $_smi_pid 2>/dev/null
+    wait $_smi_pid 2>/dev/null
+
+    _secs=$((_t1 - _t0))
+    _passed=$(awk '/tests passed/{print $1}' "$_log" | tail -1)
+    _gpu=$(awk '{s+=$1; n++} END {if (n) printf "%.0f", s/n}' "$_smi")
+    _vram=$(awk '{if ($2 > m) m = $2} END {printf "%.0f", m/1048576}' "$_smi")
+
+    printf '%-9s j=%-3s omp=%-3s %6ss  rc=%-3s %-16s gpu=%3s%%  vram=%sMB\n' \
+        "$_name" "$_jobs" "$_omp" "$_secs" "$_rc" "${_passed:-NO-SUMMARY}" \
+        "$_gpu" "$_vram" | tee -a "$OUT"
+
+    # A config that finishes implausibly fast did not run the suite. Say so
+    # rather than folding a zero into the comparison.
+    if [ "$_secs" -lt 30 ]; then
+        echo "  !! suspiciously fast -- tail of $_log:" | tee -a "$OUT"
+        tail -5 "$_log" | sed 's/^/     /' | tee -a "$OUT"
+    fi
+}
+
+: > "$OUT"
+
+echo "=== A: current policy (OMP = cores/j) ===" | tee -a "$OUT"
+for j in 4 8 16 32; do
+    run "A_j$j" $j $((CORES / j))
+done
+
+echo "=== B: decoupled (OMP pinned to 8) ===" | tee -a "$OUT"
+for j in 8 16 32; do
+    run "B_j$j" $j 8
+done
+
+echo "=== C: is OMP used at all? (j=4) ===" | tee -a "$OUT"
+for o in 32 8 1; do
+    run "C_omp$o" 4 $o
+done
+
+echo
+echo "================ results: $OUT ================"
+cat "$OUT"
diff --git a/ci/jax.sh b/ci/jax.sh
index dbca249c9b..a5bcc094be 100755
--- a/ci/jax.sh
+++ b/ci/jax.sh
@@ -116,11 +116,17 @@ fi
 
 #Master script mode: prepares testing prerequisites
 start_message
-install_prerequisites
-pip list | egrep "flax|fidle|jax|ml_dtypes|numpy|transformer_e|typing_ext"
-#check_test_jobs_requested
-#test $? -eq 0 && init_test_jobs `python -c "import jax; print(len([d for d in jax.devices() if 'rocm' in d.client.platform_version]))"`
-ck_jit_prebuild build || exit $?
+#See ci/pytorch.sh for the TE_CI_SETUP_ONLY / TE_CI_SKIP_SETUP contract. Running
+#the CK JIT prebuild here once, sequentially, also removes the race this script
+#used to have with ci/pytorch.sh over the shared blob cache.
+if [ -z "$TE_CI_SKIP_SETUP$TE_CI_LIST_ONLY" ]; then
+    install_prerequisites
+    pip list | egrep "flax|fidle|jax|ml_dtypes|numpy|transformer_e|typing_ext"
+    #check_test_jobs_requested
+    #test $? -eq 0 && init_test_jobs `python -c "import jax; print(len([d for d in jax.devices() if 'rocm' in d.client.platform_version]))"`
+    ck_jit_prebuild build || exit $?
+    test -n "$TE_CI_SETUP_ONLY" && exit 0
+fi
 
 for _fus_attn in auto ck aotriton; do
     configure_fused_attn_env $_fus_attn || continue
@@ -155,5 +161,5 @@ if [ -n "$TEST_JOBS_MODE" -a -n "$TEST_MGPU" ]; then
     done
 fi
 
-ck_jit_prebuild list
+test -z "$TE_CI_SKIP_SETUP$TE_CI_LIST_ONLY" && ck_jit_prebuild list
 return_run_results
diff --git a/ci/pytorch.sh b/ci/pytorch.sh
index 38cef08ce6..4bd0627c27 100755
--- a/ci/pytorch.sh
+++ b/ci/pytorch.sh
@@ -56,10 +56,22 @@ run_test_config(){
     export NVTE_GROUPED_LINEAR_SINGLE_PARAM=1
     run_default_fa 1 test_backward_override.py
     if [ $_fus_attn = "$_DEFAULT_FUSED_ATTN" ]; then
-        mkdir -p ${TEST_DIR}/checkpoint
-        python ${TEST_DIR}/test_checkpoint.py --save-checkpoint all --checkpoint-dir ${TEST_DIR}/checkpoint
-        NVTE_TEST_CHECKPOINT_ARTIFACT_PATH=${TEST_DIR}/checkpoint run 1 test_checkpoint.py
-        rm -rf ${TEST_DIR}/checkpoint
+        #The checkpoint artifact is generated outside pytest_run, so it is not
+        #covered by TEST_FILTER. Gate it on the same tag the test itself uses,
+        #otherwise every filtered invocation regenerates the artifact -- and
+        #concurrent ones race on the shared checkpoint directory.
+        _ckpt_variant=`get_test_variant_tag $_fus_attn ""`
+        _ckpt_tag=`get_test_name_tag test_checkpoint.py $_ckpt_variant`
+        if [ -n "$TE_CI_LIST_ONLY" ]; then
+            #Only emit the work item; do not generate the artifact. Save, test
+            #and cleanup stay one unit when the item is dispatched.
+            run 1 test_checkpoint.py
+        elif check_test_filter "$_ckpt_tag"; then
+            mkdir -p ${TEST_DIR}/checkpoint
+            python ${TEST_DIR}/test_checkpoint.py --save-checkpoint all --checkpoint-dir ${TEST_DIR}/checkpoint
+            NVTE_TEST_CHECKPOINT_ARTIFACT_PATH=${TEST_DIR}/checkpoint run 1 test_checkpoint.py
+            rm -rf ${TEST_DIR}/checkpoint
+        fi
     fi
     run 1 test_cuda_graphs.py
     run_default_fa 1 test_deferred_init.py
@@ -133,6 +145,10 @@ run_test_config_mgpu(){
 
 run_benchmark() {
     check_test_filter benchmark || return
+    if [ -n "$TE_CI_LIST_ONLY" ]; then
+        echo "TE_CI_ITEM benchmark"
+        return
+    fi
     echo "\n============= Running benchmarks attention script ============="
     BENCH_SCRIPT="$DIR/../benchmarks/attention/benchmark_attention_rocm.py"
     
@@ -168,10 +184,16 @@ check_flash_attn_installed() {
 
 #Master script mode: prepare testing prerequisites first
 start_message
-install_prerequisites
-pip list | egrep "flash|ml_dtypes|numpy|torch|transformer_e|typing_ext"
-#check_test_jobs_requested && init_test_jobs `python -c "import torch; print(torch.cuda.device_count())"`
-ck_jit_prebuild build || exit $?
+#Prerequisites and the CK JIT cache are container-wide state, so a scheduler
+#that dispatches many TEST_FILTER-ed invocations does them once up front with
+#TE_CI_SETUP_ONLY and then passes TE_CI_SKIP_SETUP on every item.
+if [ -z "$TE_CI_SKIP_SETUP$TE_CI_LIST_ONLY" ]; then
+    install_prerequisites
+    pip list | egrep "flash|ml_dtypes|numpy|torch|transformer_e|typing_ext"
+    #check_test_jobs_requested && init_test_jobs `python -c "import torch; print(torch.cuda.device_count())"`
+    ck_jit_prebuild build || exit $?
+    test -n "$TE_CI_SETUP_ONLY" && exit 0
+fi
 
 for _fus_attn in auto flash ck aotriton unfused; do
     configure_fused_attn_env $_fus_attn || continue
@@ -220,5 +242,5 @@ if [ $TEST_LEVEL -ge 3 ]; then
     fi
 fi
 
-ck_jit_prebuild list
+test -z "$TE_CI_SKIP_SETUP$TE_CI_LIST_ONLY" && ck_jit_prebuild list
 return_run_results
diff --git a/ci/test_weights.gfx950.txt b/ci/test_weights.gfx950.txt
new file mode 100644
index 0000000000..bc62b84daa
--- /dev/null
+++ b/ci/test_weights.gfx950.txt
@@ -0,0 +1,93 @@
+# Bootstrap work-item weights for .github/scripts/run_queue_sgpu.sh (gfx950).
+# Units: MEASURED SECONDS of per-item wall clock, from a full TEST_LEVEL=3 run.
+# Seconds matter, not just relative order: run_queue_sgpu.sh also derives each
+# item's kill deadline from its weight (3x + 300s), so a value in any other
+# unit will kill healthy items or fail to bound hung ones.
+#
+# Items absent here take the unknown-weight default: scheduled first, and given
+# the MAX_ITEM_SECS deadline rather than a derived one. Both are the safe
+# direction for something whose cost is not yet known.
+#
+# CI refreshes this table into the actions/cache after every run; regenerate the
+# in-tree copy with:
+#   ci/build_weights.py --from-timing <log-dir>/timings.tsv -o ci/test_weights.gfx950.txt
+core 1251.0
+torch/triton_kernels.test_norms.ck.autotune 675.0
+torch/triton_kernels.test_grouped_gemm.ck 578.0
+jax/test_custom_call_compute.auto 539.0
+jax/test_fused_attn.auto 528.0
+jax/test_fused_attn.auto.v2 524.0
+torch/test_numerics.ck.triton 372.0
+examples 324.0
+torch/test_numerics.ck.amax+triton 260.0
+torch/mxfp4..ck 241.0
+jax/test_layer.auto 239.0
+torch/test_numerics.flash 223.0
+torch/test_numerics.unfused 221.0
+torch/test_numerics.aotriton 210.0
+torch/test_numerics.ck 209.0
+torch/test_numerics.ck.amax 209.0
+torch/test_permutation.ck 201.0
+torch/test_sanity.aotriton 154.0
+torch/test_sanity.ck 152.0
+torch/test_fusible_ops.ck.triton 143.0
+torch/triton_kernels.test_cast.ck.amax 139.0
+torch/triton_kernels.test_cast.ck 138.0
+torch/test_sanity.unfused 137.0
+torch/test_sanity.flash 136.0
+torch/test_fusible_ops.ck.amax+triton 124.0
+torch/test_cpu_offloading.ck 103.0
+torch/test_fused_rope.ck 102.0
+torch/benchmark 95.0
+torch/test_cuda_graphs.unfused 86.0
+torch/test_cuda_graphs.aotriton 84.0
+torch/test_cuda_graphs.ck 83.0
+torch/test_cuda_graphs.flash 83.0
+torch/nvfp4..ck 82.0
+torch/attention.test_attention.ck 81.0
+torch/attention.test_kv_cache.ck 79.0
+torch/triton_kernels.test_norms.ck 73.0
+torch/test_backward_override.ck 65.0
+jax/test_fused_attn.aotriton 57.0
+torch/test_cpu_offloading_v1.aotriton 56.0
+torch/test_fusible_ops.ck 56.0
+torch/test_grouped_linear.unfused 56.0
+torch/test_grouped_linear.aotriton 55.0
+torch/test_grouped_linear.ck 55.0
+torch/test_grouped_linear.flash 55.0
+torch/test_fusible_ops.ck.amax 54.0
+torch/test_cpu_offloading_v1.ck 51.0
+torch/test_numerics.ck.mxfp8 45.0
+torch/triton_kernels.test_cast_mxfp4.ck 44.0
+torch/test_multi_tensor.ck 43.0
+torch/test_float8_blockwise_scaling_exact.ck 43.0
+torch/triton_kernels.test_cast_mxfp8.ck 42.0
+jax/test_fused_attn.auto.deterministic 41.0
+torch/test_jit.unfused 37.0
+torch/test_float8_current_scaling_exact.ck.triton 37.0
+torch/test_gemm_autotune.ck 37.0
+torch/test_jit.ck 36.0
+torch/test_jit.flash 36.0
+torch/test_jit.aotriton 35.0
+jax/test_functions.auto 33.0
+torch/test_float8_current_scaling_exact.ck 32.0
+torch/test_recipe.ck 31.0
+torch/test_torch_compile.ck 31.0
+torch/test_gqa.ck 30.0
+torch/test_gqa.flash 30.0
+torch/test_fused_router.ck 29.0
+torch/test_gqa.aotriton 29.0
+torch/attention.test_attention.ck.deterministic 28.0
+torch/test_gqa.unfused 28.0
+torch/test_checkpoint.ck 28.0
+torch/test_sanity_import.ck 27.0
+torch/test_quantized_tensor.ck 26.0
+torch/test_float8blockwisetensor.ck 26.0
+torch/test_parallel_cross_entropy.ck 26.0
+torch/test_sanity_hipified_cast_transpose.ck 25.0
+torch/test_nvfp4_fsdp2_hooks.ck 25.0
+torch/attention.test_cp_utils.ck 25.0
+torch/test_deferred_init.ck 24.0
+torch/triton_kernels.test_utils.ck 24.0
+jax/test_softmax.auto 22.0
+jax/test_sanity_import.auto 4.0

From 6f4c3e339e626e48b98d02872b0bb42205064db9 Mon Sep 17 00:00:00 2001
From: Veera Rajasekhar <VeeraRajasekharReddy.Gopu@amd.com>
Date: Fri, 31 Jul 2026 16:08:51 +0000
Subject: [PATCH 2/9] Removed mgpu runs and use previous built wheels

---
 .github/workflows/rocm-ci.yml | 40 ++++++++++++++++++++++++++++++++---
 1 file changed, 37 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/rocm-ci.yml b/.github/workflows/rocm-ci.yml
index ceeb9a134f..faeebc96ed 100644
--- a/.github/workflows/rocm-ci.yml
+++ b/.github/workflows/rocm-ci.yml
@@ -48,6 +48,24 @@ concurrency:
 env:
   TEST_LEVEL: ${{ (github.event_name == 'push' && '3') || inputs.test_level || '1' }}
 
+  # ---------------------------------------------------------------------------
+  # DEBUG, BRANCH-LOCAL -- REVERT BEFORE MERGING TO dev
+  #
+  # Scheduler experiment: run only the sGPU mi30x job and reuse an earlier run's
+  # wheels. These are hardcoded rather than workflow_dispatch inputs because the
+  # dispatch form is rendered from the default branch's copy of this file, so
+  # inputs added on a topic branch never show up in the UI.
+  #
+  # Reverting means: drop this block, drop `if: false` from `build` and
+  # `mgpu_tests`, restore `arch_label: [mi30x, mi35x]` in `sgpu_tests`, and drop
+  # `run-id`/`github-token` from the download-artifact steps.
+  #
+  # A reusable workflow shares its caller's run id, so this is the id of the whole
+  # CI run that built the wheels -- the same run whose logs-sgpu-* artifacts you
+  # downloaded. te-rocm-wheels is uploaded with retention-days: 1, so this only
+  # works within 24h of that build; after that, set `build` back to running.
+  REUSE_WHEELS_RUN_ID: '30612285497'
+
 jobs:
   select_image:
     name: Select Docker Image
@@ -100,20 +118,28 @@ jobs:
 
   build:
     # Delegate wheel building to the reusable workflow on dev. It produces a core .whl plus framework .tar.gz sdists under artifact name `te-rocm-wheels`.
+    # DEBUG: disabled, reusing REUSE_WHEELS_RUN_ID's wheels. The test jobs below
+    # tolerate a skipped (but not a failed) build.
+    if: false
     uses: ./.github/workflows/rocm-wheels-build.yml
     secrets: inherit
 
   sgpu_tests:
     name: sGPU Tests (${{ matrix.arch_label }})
     needs: [select_image, build]
+    if: ${{ !cancelled() && needs.select_image.result == 'success' && needs.build.result != 'failure' }}
     timeout-minutes: 270
     runs-on: ${{ matrix.arch_label == 'mi30x' && 'linux-te-mi30x-4' || matrix.arch_label == 'mi35x' && 'linux-te-mi35x-4' }}
+    # actions:read lets download-artifact reach another workflow run's artifacts.
+    permissions:
+      contents: read
+      actions: read
     env:
       DOCKER_IMAGE: ${{ inputs.docker_image_override || format('{0}{1}', needs.select_image.outputs.image-tag, matrix.arch_label == 'mi30x' && '_gfx942' || matrix.arch_label == 'mi35x' && '_gfx950') }}
     strategy:
       fail-fast: false
       matrix:
-        arch_label: [mi30x, mi35x]
+        arch_label: [mi30x]  # DEBUG: was [mi30x, mi35x]
     steps:
       - name: Checkout repository
         uses: actions/checkout@v6
@@ -129,6 +155,8 @@ jobs:
         with:
           name: te-rocm-wheels
           path: dist/
+          run-id: ${{ env.REUSE_WHEELS_RUN_ID || github.run_id }}  # DEBUG
+          github-token: ${{ secrets.GITHUB_TOKEN }}                # DEBUG
 
       - name: Host Diagnostics
         run: |
@@ -192,7 +220,7 @@ jobs:
         uses: actions/cache/restore@v4
         with:
           path: ci-weights
-          key: te-sgpu-weights-${{ matrix.arch_label }}-${{ github.run_id }}
+          key: te-sgpu-weights-${{ matrix.arch_label }}-${{ github.run_id }}-${{ github.run_attempt }}
           restore-keys: te-sgpu-weights-${{ matrix.arch_label }}-
 
       - name: Run sGPU tests as one queue (pytorch, jax, examples, core)
@@ -247,7 +275,7 @@ jobs:
         uses: actions/cache/save@v4
         with:
           path: ci-weights
-          key: te-sgpu-weights-${{ matrix.arch_label }}-${{ github.run_id }}
+          key: te-sgpu-weights-${{ matrix.arch_label }}-${{ github.run_id }}-${{ github.run_attempt }}
 
       - name: Publish scheduling report
         if: always()
@@ -315,8 +343,12 @@ jobs:
   mgpu_tests:
     name: mGPU ${{ matrix.framework == 'pytorch' && 'Torch' || 'JAX' }} (${{ matrix.arch_label }})
     needs: [select_image, build]
+    if: false  # DEBUG: was ${{ !cancelled() && needs.select_image.result == 'success' && needs.build.result != 'failure' }}
     timeout-minutes: 210
     runs-on: ${{ matrix.arch_label == 'mi30x' && 'linux-te-mi30x-8' || matrix.arch_label == 'mi35x' && 'linux-te-mi35x-8' }}
+    permissions:
+      contents: read
+      actions: read
     env:
       DOCKER_IMAGE: ${{ inputs.docker_image_override || format('{0}{1}', needs.select_image.outputs.image-tag, matrix.arch_label == 'mi30x' && '_gfx942' || matrix.arch_label == 'mi35x' && '_gfx950') }}
     strategy:
@@ -333,6 +365,8 @@ jobs:
         with:
           name: te-rocm-wheels
           path: dist/
+          run-id: ${{ env.REUSE_WHEELS_RUN_ID || github.run_id }}  # DEBUG
+          github-token: ${{ secrets.GITHUB_TOKEN }}                # DEBUG
 
       - name: Host Diagnostics
         run: |

From 8594f496a24d54158a7a1a851ecb09f22af292e3 Mon Sep 17 00:00:00 2001
From: Veera Rajasekhar <VeeraRajasekharReddy.Gopu@amd.com>
Date: Sun, 2 Aug 2026 03:37:48 +0000
Subject: [PATCH 3/9] Revert "Removed mgpu runs and use previous built wheels"

This reverts commit 4d0e28b973f0d6472caf0c70547d5cbae477470b.
---
 .github/workflows/rocm-ci.yml | 40 +++--------------------------------
 1 file changed, 3 insertions(+), 37 deletions(-)

diff --git a/.github/workflows/rocm-ci.yml b/.github/workflows/rocm-ci.yml
index faeebc96ed..ceeb9a134f 100644
--- a/.github/workflows/rocm-ci.yml
+++ b/.github/workflows/rocm-ci.yml
@@ -48,24 +48,6 @@ concurrency:
 env:
   TEST_LEVEL: ${{ (github.event_name == 'push' && '3') || inputs.test_level || '1' }}
 
-  # ---------------------------------------------------------------------------
-  # DEBUG, BRANCH-LOCAL -- REVERT BEFORE MERGING TO dev
-  #
-  # Scheduler experiment: run only the sGPU mi30x job and reuse an earlier run's
-  # wheels. These are hardcoded rather than workflow_dispatch inputs because the
-  # dispatch form is rendered from the default branch's copy of this file, so
-  # inputs added on a topic branch never show up in the UI.
-  #
-  # Reverting means: drop this block, drop `if: false` from `build` and
-  # `mgpu_tests`, restore `arch_label: [mi30x, mi35x]` in `sgpu_tests`, and drop
-  # `run-id`/`github-token` from the download-artifact steps.
-  #
-  # A reusable workflow shares its caller's run id, so this is the id of the whole
-  # CI run that built the wheels -- the same run whose logs-sgpu-* artifacts you
-  # downloaded. te-rocm-wheels is uploaded with retention-days: 1, so this only
-  # works within 24h of that build; after that, set `build` back to running.
-  REUSE_WHEELS_RUN_ID: '30612285497'
-
 jobs:
   select_image:
     name: Select Docker Image
@@ -118,28 +100,20 @@ jobs:
 
   build:
     # Delegate wheel building to the reusable workflow on dev. It produces a core .whl plus framework .tar.gz sdists under artifact name `te-rocm-wheels`.
-    # DEBUG: disabled, reusing REUSE_WHEELS_RUN_ID's wheels. The test jobs below
-    # tolerate a skipped (but not a failed) build.
-    if: false
     uses: ./.github/workflows/rocm-wheels-build.yml
     secrets: inherit
 
   sgpu_tests:
     name: sGPU Tests (${{ matrix.arch_label }})
     needs: [select_image, build]
-    if: ${{ !cancelled() && needs.select_image.result == 'success' && needs.build.result != 'failure' }}
     timeout-minutes: 270
     runs-on: ${{ matrix.arch_label == 'mi30x' && 'linux-te-mi30x-4' || matrix.arch_label == 'mi35x' && 'linux-te-mi35x-4' }}
-    # actions:read lets download-artifact reach another workflow run's artifacts.
-    permissions:
-      contents: read
-      actions: read
     env:
       DOCKER_IMAGE: ${{ inputs.docker_image_override || format('{0}{1}', needs.select_image.outputs.image-tag, matrix.arch_label == 'mi30x' && '_gfx942' || matrix.arch_label == 'mi35x' && '_gfx950') }}
     strategy:
       fail-fast: false
       matrix:
-        arch_label: [mi30x]  # DEBUG: was [mi30x, mi35x]
+        arch_label: [mi30x, mi35x]
     steps:
       - name: Checkout repository
         uses: actions/checkout@v6
@@ -155,8 +129,6 @@ jobs:
         with:
           name: te-rocm-wheels
           path: dist/
-          run-id: ${{ env.REUSE_WHEELS_RUN_ID || github.run_id }}  # DEBUG
-          github-token: ${{ secrets.GITHUB_TOKEN }}                # DEBUG
 
       - name: Host Diagnostics
         run: |
@@ -220,7 +192,7 @@ jobs:
         uses: actions/cache/restore@v4
         with:
           path: ci-weights
-          key: te-sgpu-weights-${{ matrix.arch_label }}-${{ github.run_id }}-${{ github.run_attempt }}
+          key: te-sgpu-weights-${{ matrix.arch_label }}-${{ github.run_id }}
           restore-keys: te-sgpu-weights-${{ matrix.arch_label }}-
 
       - name: Run sGPU tests as one queue (pytorch, jax, examples, core)
@@ -275,7 +247,7 @@ jobs:
         uses: actions/cache/save@v4
         with:
           path: ci-weights
-          key: te-sgpu-weights-${{ matrix.arch_label }}-${{ github.run_id }}-${{ github.run_attempt }}
+          key: te-sgpu-weights-${{ matrix.arch_label }}-${{ github.run_id }}
 
       - name: Publish scheduling report
         if: always()
@@ -343,12 +315,8 @@ jobs:
   mgpu_tests:
     name: mGPU ${{ matrix.framework == 'pytorch' && 'Torch' || 'JAX' }} (${{ matrix.arch_label }})
     needs: [select_image, build]
-    if: false  # DEBUG: was ${{ !cancelled() && needs.select_image.result == 'success' && needs.build.result != 'failure' }}
     timeout-minutes: 210
     runs-on: ${{ matrix.arch_label == 'mi30x' && 'linux-te-mi30x-8' || matrix.arch_label == 'mi35x' && 'linux-te-mi35x-8' }}
-    permissions:
-      contents: read
-      actions: read
     env:
       DOCKER_IMAGE: ${{ inputs.docker_image_override || format('{0}{1}', needs.select_image.outputs.image-tag, matrix.arch_label == 'mi30x' && '_gfx942' || matrix.arch_label == 'mi35x' && '_gfx950') }}
     strategy:
@@ -365,8 +333,6 @@ jobs:
         with:
           name: te-rocm-wheels
           path: dist/
-          run-id: ${{ env.REUSE_WHEELS_RUN_ID || github.run_id }}  # DEBUG
-          github-token: ${{ secrets.GITHUB_TOKEN }}                # DEBUG
 
       - name: Host Diagnostics
         run: |

From b32560c52010559e883005da61b5e4cd46bd1afd Mon Sep 17 00:00:00 2001
From: Veera Rajasekhar <VeeraRajasekharReddy.Gopu@amd.com>
Date: Sun, 2 Aug 2026 05:10:56 +0000
Subject: [PATCH 4/9] Refactor logging

---
 .github/scripts/run_queue_sgpu.sh | 130 ++++++++++++++++--------------
 .github/workflows/rocm-ci.yml     |  11 +--
 ci/build_weights.py               |   9 +--
 ci/test_weights.gfx950.txt        |  93 ---------------------
 4 files changed, 79 insertions(+), 164 deletions(-)
 delete mode 100644 ci/test_weights.gfx950.txt

diff --git a/.github/scripts/run_queue_sgpu.sh b/.github/scripts/run_queue_sgpu.sh
index d554ebac1b..89c92be121 100755
--- a/.github/scripts/run_queue_sgpu.sh
+++ b/.github/scripts/run_queue_sgpu.sh
@@ -41,11 +41,10 @@ SKIP_SETUP=""
 # Items with no recorded weight sort first: an unknown item is more likely to be
 # a new (or newly slow) one, and a long item started late is what stretches the
 # tail. Losing the gamble costs far less than mis-scheduling a genuinely big item.
+# With no weights file at all -- the first run on a new arch, or a cache miss --
+# every item takes this, the queue keeps its natural order, and the run is simply
+# unordered. That costs makespan once; the table it writes fixes the next run.
 DEFAULT_WEIGHT=${TE_CI_DEFAULT_WEIGHT:-999999}
-# Ceiling for the per-item deadline derived from the weight below. Also the
-# deadline for unknown items, whose DEFAULT_WEIGHT makes the derived budget
-# meaningless.
-MAX_ITEM_SECS=${TE_CI_MAX_ITEM_SECS:-7200}
 
 while [[ $# -gt 0 ]]; do
     case "$1" in
@@ -245,32 +244,23 @@ worker() {
             junit_dir=""
         fi
 
-        # Bound the item at 3x its learned weight, plus slack for the ~5% noise
-        # floor and a genuinely slower run; past that it is hung, not slow.
-        # Nothing else bounds an *item*: pytest's --timeout is per test, so an
-        # item of many just-under-PYTEST_TIMEOUT tests -- or a hang during
-        # collection or conftest import, which pytest-timeout does not cover --
-        # otherwise runs until the job's timeout-minutes kills all N workers at
-        # once, discarding every in-flight item on every GPU. rc 124 here means
-        # the deadline fired; build_weights.py drops those rows, since the
-        # measurement is the ceiling itself and would compound run over run.
-        budget=$(( weight * 3 + 300 ))
-        [[ $budget -gt $MAX_ITEM_SECS ]] && budget=$MAX_ITEM_SECS
-
+        # No scheduler-imposed deadline: the suite scripts' own PYTEST_TIMEOUT and
+        # the workflow's timeout-minutes are the only limits, exactly as they are
+        # outside the queue. Deriving one from the weight would make the timeout
+        # depend on measurement history, so the same test could pass one run and
+        # be killed the next -- the queue must not change what "a test failed"
+        # means. build_weights.py still drops rc 124/137 rows, because a duration
+        # that ends in a kill is a ceiling wherever the kill came from.
         start=$(date +%s)
         if [[ -n "$tag" ]]; then
             HIP_VISIBLE_DEVICES=$gpu TE_CI_SKIP_SETUP=1 TEST_FILTER="$tag" \
                 JUNITXML_PREFIX="$junit_dir" \
-                timeout --kill-after=60 "$budget" "$cmd" ${rest:-} > "$itemlog" 2>&1
+                "$cmd" ${rest:-} > "$itemlog" 2>&1
         else
             HIP_VISIBLE_DEVICES=$gpu JUNITXML_PREFIX="$junit_dir" \
-                timeout --kill-after=60 "$budget" "$cmd" ${rest:-} > "$itemlog" 2>&1
+                "$cmd" ${rest:-} > "$itemlog" 2>&1
         fi
         rc=$?
-        if [[ $rc -eq 124 || $rc -eq 137 ]]; then
-            echo "::error::[${label}] ${safetag} killed at its ${budget}s deadline" >&2
-            echo "TE_CI_ITEM_DEADLINE_EXCEEDED after ${budget}s" >> "$itemlog"
-        fi
         end=$(date +%s)
 
         echo "$rc" > "${itemlog}.rc"
@@ -332,83 +322,101 @@ done
 report() {
     local tf="$LOG_DIR/timings.tsv"
 
+    # Every section is a fixed-width table with a header row, so the report can be
+    # read straight out of the GHA step summary or grepped/cut by column.
+    local rule="  ------ -------- -------- ------- ---------- -------- ----------------------------------------"
+
     echo "=== sGPU queue: ${TOTAL_ITEMS} items, ${NUM_GPUS} GPUs, drained in ${WALL}s ==="
     echo
     echo "-- schedule: what ran where, in execution order --"
-    sort -t$'\t' -k3,3n -k6,6n "$tf" | awk -F'\t' -v dw="$DEFAULT_WEIGHT" '
-        $3 != last { if (NR > 1) print ""; last = $3 }
+    echo "$rule"
+    printf "  %-6s %-8s %-8s %-7s %-10s %-8s %s\n" \
+           GPU START DURATION RESULT ESTIMATE "% CHANGE" TEST_NAME
+    echo "$rule"
+    # rc is reported as a word, not a number: the report is read by people, and
+    # "killed" vs "fail" is the distinction that actually changes what you do next.
+    sort -t$'\t' -k3,3n -k6,6n "$tf" | awk -F'\t' -v dw="$DEFAULT_WEIGHT" -v rule="$rule" '
+        $3 != last { if (NR > 1) print rule; last = $3 }
         {
             est = ($8 == dw ? "unknown" : $8 "s")
             # Estimate error is the scheduler feedback loop made visible: a large
             # positive miss is an item that should have been dispatched earlier.
             err = ($8 > 0 && $8 != dw) ? sprintf("%+.0f%%", ($4 - $8) * 100 / $8) : "n/a"
-            printf "  gpu%-2s  t+%-7s %6ss  rc=%-4s est=%-9s %-7s %s/%s\n",
-                   $3, $6 "s", $4, $5, est, err, $1, $2
+            res = ($5 == 0 ? "pass" : ($5 == 1 ? "fail" : \
+                  ($5 == 124 || $5 == 137 ? "killed" : "error")))
+            printf "  gpu%-3s %-8s %-8s %-7s %-10s %-8s %s/%s\n",
+                   $3, "t+" $6 "s", $4 "s", res, est, err, $1, $2
         }'
+    echo "$rule"
+    echo "  START = seconds after the queue opened. DURATION = wall clock for the whole"
+    echo "  item, process startup included. ESTIMATE = the weight it was scheduled on."
 
     echo
     echo "-- per-GPU utilisation --"
+    echo "  ------ ------- --------- --------- --------- --------"
+    printf "  %-6s %-7s %-9s %-9s %-9s %s\n" GPU ITEMS BUSY IDLE UTIL FAILED
+    echo "  ------ ------- --------- --------- --------- --------"
     awk -F'\t' -v w="$WALL" '
-        { busy[$3] += $4; n[$3]++; if ($5 != 0) bad[$3]++; if ($7 > last[$3]) last[$3] = $7 }
+        { busy[$3] += $4; n[$3]++; if ($5 != 0) bad[$3]++ }
         END {
             for (g in busy)
-                printf "  gpu%-3s %3d items  %6ss busy  %5ss idle  %5.1f%% util  %d failed\n",
-                       g, n[g], busy[g], w - busy[g], busy[g] * 100 / w, bad[g] + 0
+                printf "  gpu%-3s %-7d %-9s %-9s %-9s %d\n",
+                       g, n[g], busy[g] "s", (w - busy[g]) "s",
+                       sprintf("%.1f%%", busy[g] * 100 / w), bad[g] + 0
         }' "$tf" | sort
-
-    echo
-    echo "-- timeline (one column ~$(( WALL / 60 + 1 ))s; = ran, # failed, . idle) --"
-    # Idle gaps are the whole point: a run of dots on one GPU while another is
-    # still busy is exactly the tail a better ordering would have filled.
-    awk -F'\t' -v w="$WALL" -v cols=60 -v first="$FIRST_GPU" -v n="$NUM_GPUS" '
-        { from = int($6 * cols / w); to = int($7 * cols / w)
-          if (to >= cols) to = cols - 1
-          for (c = from; c <= to; c++) cell[$3, c] = ($5 == 0 ? "=" : "#") }
-        END {
-            for (g = first; g < first + n; g++) {
-                line = ""
-                for (c = 0; c < cols; c++) line = line ((g, c) in cell ? cell[g, c] : ".")
-                printf "  gpu%-3s |%s|\n", g, line
-            }
-            printf "  %7s 0s%*ss\n", "", cols - 2, w
-        }' "$tf"
+    echo "  ------ ------- --------- --------- --------- --------"
 
     echo
     echo "-- efficiency --"
+    echo "  ----------------- ---------- --------------------------------------------------"
+    printf "  %-17s %-10s %s\n" METRIC VALUE MEANING
+    echo "  ----------------- ---------- --------------------------------------------------"
     awk -F'\t' -v w="$WALL" -v n="$NUM_GPUS" '
         { work += $4; if ($4 > big) { big = $4; bigname = $1 "/" $2 } }
         END {
-            bound = work / n; if (big > bound) bound = big
-            printf "  total work      %8.0fs across %d items\n", work, NR
-            printf "  capacity        %8.0fs (%ss x %d GPUs)\n", w * n, w, n
-            printf "  lower bound     %8.0fs  max(work/%d, largest item)\n", bound, n
-            printf "  actual makespan %8.0fs  %+.1f%% over the bound\n", w, (w - bound) * 100 / bound
-            printf "  utilisation     %8.1f%%\n", work * 100 / (w * n)
-            printf "  largest item    %8.0fs  %s%s\n", big, bigname,
-                   (big > work / n ? "   <-- item-bound: splitting would now pay" : "")
+            printf "  %-17s %-10s %s\n", "total work", sprintf("%.0fs", work),
+                   sprintf("sum of all %d item durations", NR)
+            printf "  %-17s %-10s %s\n", "actual run time", sprintf("%.0fs", w),
+                   sprintf("wall clock, first item start to last item finish")
+            printf "  %-17s %-10s %s\n", "utilisation", sprintf("%.1f%%", work * 100 / (w * n)),
+                   sprintf("share of %ss x %d GPUs actually spent running tests", w, n)
+            printf "  %-17s %-10s %s\n", "largest item", sprintf("%.0fs", big),
+                   bigname (big > work / n ? "  <-- floor: splitting it would now pay" : "")
         }' "$tf"
+    echo "  ----------------- ---------- --------------------------------------------------"
 
     echo
-    echo "-- weight accuracy (misses over 30s; these are what the next run corrects) --"
+    echo "-- weight accuracy: misses over 30s, which the next run corrects --"
     # 30s is roughly one process startup, and below the ~5% run-to-run noise for
     # anything long enough to matter. Listing smaller misses would bury the real
     # ones -- a healthy table has almost every item within a few seconds.
+    echo "  --------- --------- --------- -------- ---------------------------------------"
+    printf "  %-9s %-9s %-9s %-8s %s\n" MISS ESTIMATE ACTUAL "% CHANGE" TEST_NAME
+    echo "  --------- --------- --------- -------- ---------------------------------------"
     awk -F'\t' -v dw="$DEFAULT_WEIGHT" '$8 != dw && $8 > 0 {
             d = $4 - $8; a = (d < 0 ? -d : d)
-            if (a >= 30) printf "%.0f\t%+.0f\t%s\t%s\t%s\n", a, d, $8, $4, $1 "/" $2 }' "$tf" \
+            if (a >= 30) printf "%.0f\t%+.0f\t%s\t%s\t%.0f\t%s\n", a, d, $8, $4,
+                                (d * 100 / $8), $1 "/" $2 }' "$tf" \
         | sort -k1,1nr | head -15 \
-        | awk -F'\t' '{printf "  %+7ss  est %6ss -> actual %6ss  %s\n", $2, $3, $4, $5}'
-    awk -F'\t' -v dw="$DEFAULT_WEIGHT" '$8 == dw {printf "  unknown     -> actual %6ss  %s/%s\n", $4, $1, $2}' "$tf"
+        | awk -F'\t' '{printf "  %-9s %-9s %-9s %-8s %s\n",
+                              $2 "s", $3 "s", $4 "s", sprintf("%+d%%", $5), $6}'
+    awk -F'\t' -v dw="$DEFAULT_WEIGHT" '$8 == dw {
+            printf "  %-9s %-9s %-9s %-8s %s\n", "n/a", "unknown", $4 "s", "n/a", $1 "/" $2 }' "$tf"
+    echo "  --------- --------- --------- -------- ---------------------------------------"
 
     echo
     echo "-- failures --"
+    echo "  -------- --------- --------------------------------------------------------------"
+    printf "  %-8s %-9s %s\n" RESULT DURATION TEST_NAME
+    echo "  -------- --------- --------------------------------------------------------------"
     if awk -F'\t' '$5 != 0 {found = 1} END {exit !found}' "$tf"; then
         awk -F'\t' '$5 != 0 {
-            printf "  rc=%-4s %6ss  %s/%s%s\n", $5, $4, $1, $2,
-                   ($5 == 124 || $5 == 137 ? "   (killed at per-item deadline)" : "") }' "$tf"
+            res = ($5 == 1 ? "fail" : ($5 == 124 || $5 == 137 ? "killed" : "error"))
+            printf "  %-8s %-9s %s/%s\n", res, $4 "s", $1, $2 }' "$tf"
     else
-        echo "  none"
+        printf "  %-8s %-9s %s\n" "-" "-" "none"
     fi
+    echo "  -------- --------- --------------------------------------------------------------"
 
     # An item that never got a timings.tsv row was never dispatched -- only
     # possible if a worker died outright. Silence here would read as success.
diff --git a/.github/workflows/rocm-ci.yml b/.github/workflows/rocm-ci.yml
index ceeb9a134f..16f12dd3b6 100644
--- a/.github/workflows/rocm-ci.yml
+++ b/.github/workflows/rocm-ci.yml
@@ -209,11 +209,12 @@ jobs:
           # exists in both the torch and jax suites and runs in parallel).
           rm -rf test-results && mkdir -p test-results/torch test-results/jax test-results/core
 
-          # Seed from the in-tree table on a cache miss; after that the cached
-          # copy is authoritative, since it has been refined by every run since.
+          # No in-tree seed table: on a cache miss the file is simply absent, the
+          # queue runs in its natural order for one run, and this run's timings
+          # create the table. A checked-in copy would go stale silently and is
+          # worth less than the one unordered run it saves.
           mkdir -p ci-weights
           W="ci-weights/test_weights.${ARCH}.txt"
-          [ -s "$W" ] || cp "ci/test_weights.${ARCH}.txt" "$W" 2>/dev/null || true
           echo "weights: $W ($(wc -l < "$W" 2>/dev/null || echo 0) items)"
 
           docker exec \
@@ -230,8 +231,8 @@ jobs:
 
       # Runs even when tests failed: a red run still measures how long each item
       # took, and build_weights.py's asymmetric blend is built to absorb the bias
-      # a short-because-it-crashed measurement introduces. Only items killed at
-      # their deadline (rc 124/137) are dropped, since that measurement is a
+      # a short-because-it-crashed measurement introduces. Only items whose
+      # process was killed (rc 124/137) are dropped, since that measurement is a
       # ceiling that would otherwise compound run over run.
       - name: Update learned test weights
         if: always() && hashFiles('timings.tsv') != ''
diff --git a/ci/build_weights.py b/ci/build_weights.py
index 239fa7ee81..584797048e 100755
--- a/ci/build_weights.py
+++ b/ci/build_weights.py
@@ -65,12 +65,11 @@ def from_junit(root_dir):
     return weights
 
 
-# rc values that mean the item was killed rather than measured. 124 is the
-# scheduler's per-item deadline (GNU timeout), 137 its SIGKILL follow-up or an
+# rc values that mean the item was killed rather than measured. 124 is a GNU
+# timeout expiry (the job's own step timeout, for instance), 137 a SIGKILL or an
 # OOM-kill. Either way the recorded duration is a ceiling imposed from outside,
-# not the item's cost -- and merging a ceiling is the one case the asymmetric
-# blend cannot absorb, because it raises the weight, which raises next run's
-# deadline, which raises the next ceiling. Drop them; the previous weight stands.
+# not the item's cost, so blending it in would teach the table a number the item
+# never actually took. Drop them; the previous weight stands.
 KILLED_RCS = frozenset(("124", "137"))
 
 
diff --git a/ci/test_weights.gfx950.txt b/ci/test_weights.gfx950.txt
deleted file mode 100644
index bc62b84daa..0000000000
--- a/ci/test_weights.gfx950.txt
+++ /dev/null
@@ -1,93 +0,0 @@
-# Bootstrap work-item weights for .github/scripts/run_queue_sgpu.sh (gfx950).
-# Units: MEASURED SECONDS of per-item wall clock, from a full TEST_LEVEL=3 run.
-# Seconds matter, not just relative order: run_queue_sgpu.sh also derives each
-# item's kill deadline from its weight (3x + 300s), so a value in any other
-# unit will kill healthy items or fail to bound hung ones.
-#
-# Items absent here take the unknown-weight default: scheduled first, and given
-# the MAX_ITEM_SECS deadline rather than a derived one. Both are the safe
-# direction for something whose cost is not yet known.
-#
-# CI refreshes this table into the actions/cache after every run; regenerate the
-# in-tree copy with:
-#   ci/build_weights.py --from-timing <log-dir>/timings.tsv -o ci/test_weights.gfx950.txt
-core 1251.0
-torch/triton_kernels.test_norms.ck.autotune 675.0
-torch/triton_kernels.test_grouped_gemm.ck 578.0
-jax/test_custom_call_compute.auto 539.0
-jax/test_fused_attn.auto 528.0
-jax/test_fused_attn.auto.v2 524.0
-torch/test_numerics.ck.triton 372.0
-examples 324.0
-torch/test_numerics.ck.amax+triton 260.0
-torch/mxfp4..ck 241.0
-jax/test_layer.auto 239.0
-torch/test_numerics.flash 223.0
-torch/test_numerics.unfused 221.0
-torch/test_numerics.aotriton 210.0
-torch/test_numerics.ck 209.0
-torch/test_numerics.ck.amax 209.0
-torch/test_permutation.ck 201.0
-torch/test_sanity.aotriton 154.0
-torch/test_sanity.ck 152.0
-torch/test_fusible_ops.ck.triton 143.0
-torch/triton_kernels.test_cast.ck.amax 139.0
-torch/triton_kernels.test_cast.ck 138.0
-torch/test_sanity.unfused 137.0
-torch/test_sanity.flash 136.0
-torch/test_fusible_ops.ck.amax+triton 124.0
-torch/test_cpu_offloading.ck 103.0
-torch/test_fused_rope.ck 102.0
-torch/benchmark 95.0
-torch/test_cuda_graphs.unfused 86.0
-torch/test_cuda_graphs.aotriton 84.0
-torch/test_cuda_graphs.ck 83.0
-torch/test_cuda_graphs.flash 83.0
-torch/nvfp4..ck 82.0
-torch/attention.test_attention.ck 81.0
-torch/attention.test_kv_cache.ck 79.0
-torch/triton_kernels.test_norms.ck 73.0
-torch/test_backward_override.ck 65.0
-jax/test_fused_attn.aotriton 57.0
-torch/test_cpu_offloading_v1.aotriton 56.0
-torch/test_fusible_ops.ck 56.0
-torch/test_grouped_linear.unfused 56.0
-torch/test_grouped_linear.aotriton 55.0
-torch/test_grouped_linear.ck 55.0
-torch/test_grouped_linear.flash 55.0
-torch/test_fusible_ops.ck.amax 54.0
-torch/test_cpu_offloading_v1.ck 51.0
-torch/test_numerics.ck.mxfp8 45.0
-torch/triton_kernels.test_cast_mxfp4.ck 44.0
-torch/test_multi_tensor.ck 43.0
-torch/test_float8_blockwise_scaling_exact.ck 43.0
-torch/triton_kernels.test_cast_mxfp8.ck 42.0
-jax/test_fused_attn.auto.deterministic 41.0
-torch/test_jit.unfused 37.0
-torch/test_float8_current_scaling_exact.ck.triton 37.0
-torch/test_gemm_autotune.ck 37.0
-torch/test_jit.ck 36.0
-torch/test_jit.flash 36.0
-torch/test_jit.aotriton 35.0
-jax/test_functions.auto 33.0
-torch/test_float8_current_scaling_exact.ck 32.0
-torch/test_recipe.ck 31.0
-torch/test_torch_compile.ck 31.0
-torch/test_gqa.ck 30.0
-torch/test_gqa.flash 30.0
-torch/test_fused_router.ck 29.0
-torch/test_gqa.aotriton 29.0
-torch/attention.test_attention.ck.deterministic 28.0
-torch/test_gqa.unfused 28.0
-torch/test_checkpoint.ck 28.0
-torch/test_sanity_import.ck 27.0
-torch/test_quantized_tensor.ck 26.0
-torch/test_float8blockwisetensor.ck 26.0
-torch/test_parallel_cross_entropy.ck 26.0
-torch/test_sanity_hipified_cast_transpose.ck 25.0
-torch/test_nvfp4_fsdp2_hooks.ck 25.0
-torch/attention.test_cp_utils.ck 25.0
-torch/test_deferred_init.ck 24.0
-torch/triton_kernels.test_utils.ck 24.0
-jax/test_softmax.auto 22.0
-jax/test_sanity_import.auto 4.0

From cc8f0fb035069e95dda6270a548b94b1f96399ef Mon Sep 17 00:00:00 2001
From: Veera Rajasekhar <VeeraRajasekharReddy.Gopu@amd.com>
Date: Wed, 5 Aug 2026 06:02:53 +0000
Subject: [PATCH 5/9] Refactor implementation

---
 .github/scripts/run_queue_sgpu.sh | 614 +++++++++++++++++++++---------
 .github/workflows/rocm-ci.yml     | 105 ++---
 .gitignore                        |   2 +
 ci/_utils.sh                      |  11 +
 ci/build_weights.py               | 251 +++++++-----
 ci/pytorch.sh                     |   2 +
 6 files changed, 657 insertions(+), 328 deletions(-)

diff --git a/.github/scripts/run_queue_sgpu.sh b/.github/scripts/run_queue_sgpu.sh
index 89c92be121..4b6c08218e 100755
--- a/.github/scripts/run_queue_sgpu.sh
+++ b/.github/scripts/run_queue_sgpu.sh
@@ -5,39 +5,55 @@
 #
 # Run all sGPU test suites as one global work queue across N GPUs.
 #
-# Usage: run_queue_sgpu.sh [options] <config>...
-#   -n, --gpus <n>        number of GPUs / concurrent workers (default: 4)
-#   -g, --first-gpu <n>   first GPU id to use (default: 0)
-#   -l, --log-dir <dir>   where suite and per-item logs are written
-#   -w, --weights <file>  "<label>/<tag> <weight>" pairs used to order the queue
+# Usage: run_queue_sgpu.sh --arch <gfx> [options] [config]...
+#   -a, --arch <gfx>      GPU arch these timings belong to, e.g. gfx942. Required
+#                         (or TE_CI_ARCH); it keys the learned weight table
+#   -l, --log-dir <dir>   where the per-phase log tree is written
+#                         (default test-results/logs)
 #       --only <regex>    keep only items whose "<label>/<tag>" matches (smoke
 #                         tests, or re-running just the items that failed)
-#       --skip-setup      container prerequisites and the CK JIT cache are
-#                         already in place; go straight to the queue
-#       --dry-run         build and print the queue, run nothing
+#   -h, --help            this text
 #
-# Why a queue instead of one suite per GPU (run_parallel_sgpu.sh): the suites
-# are very unevenly sized, so a static 1:1 map leaves most GPUs idle waiting for
-# the largest suite. Here every suite is expanded into individual work items,
-# the items are ordered longest-first, and each worker pulls the next one when
-# it goes idle -- so a GPU only stops working when the queue is empty.
+# With no config the full sGPU set (ci_sgpu_queue.conf) is run. TEST_LEVEL and
+# the other ci/_utils.sh variables are read from the environment as usual, so a
+# local run inside the dev container is just:
+#
+#   TEST_LEVEL=1 .github/scripts/run_queue_sgpu.sh --arch gfx942
+#
+# The queue uses every GPU it can see. To restrict what it can see the same way you would for any other
+# ROCm program
+#
+#   HIP_VISIBLE_DEVICES=0,1 .github/scripts/run_queue_sgpu.sh --arch gfx942
 #
-# Compatibility: after the queue drains, each suite's item logs are concatenated
-# into <log-dir>/<logfile> and the worst item exit code is written to
-# <logfile>.rc, so existing consumers of those paths keep working unchanged.
-
 set -u
 
 SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
 REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)"
 
-NUM_GPUS=${TEST_NUM_GPUS:-4}
-FIRST_GPU=${TEST_FIRST_GPU:-0}
-LOG_DIR=${LOG_DIR:-/tmp/te_ci_logs}
-WEIGHTS_FILE=""
-DRY_RUN=""
+# Two directories in the repo root:
+#
+#   test-results/  everything one run produced. This script adds logs/ under it:
+#                  the per-phase log tree
+#   ci-weights/    the learned weight table, which is the one thing that must
+#                  outlive the run that measured it. Same name the workflow uses
+#                  for the cached copy.
+LOG_DIR=${LOG_DIR:-${REPO_ROOT}/test-results/logs}
+ARCH=${TE_CI_ARCH:-}
 ONLY_RE=""
-SKIP_SETUP=""
+REPORT_TITLE="sGPU queue schedule"
+
+# ::error:: and ::warning:: are GitHub Actions annotations, and this script also
+# runs by hand in the dev container -- emit them only where they mean something.
+if [[ -n "${GITHUB_ACTIONS:-}" ]]; then
+    log_error() { echo "::error::$*" >&2; }
+    log_warn()  { echo "::warning::$*" >&2; }
+else
+    log_error() { echo "Error: $*" >&2; }
+    log_warn()  { echo "Warning: $*" >&2; }
+fi
+
+# The header comment is the usage text; reprinting it keeps the two in sync.
+usage() { awk 'NR > 5 && /^#/ {sub(/^# ?/, ""); print; next} NR > 5 {exit}' "$0"; }
 # Items with no recorded weight sort first: an unknown item is more likely to be
 # a new (or newly slow) one, and a long item started late is what stretches the
 # tail. Losing the gamble costs far less than mis-scheduling a genuinely big item.
@@ -48,49 +64,126 @@ DEFAULT_WEIGHT=${TE_CI_DEFAULT_WEIGHT:-999999}
 
 while [[ $# -gt 0 ]]; do
     case "$1" in
-        -n|--gpus)       NUM_GPUS="$2"; shift 2 ;;
-        --gpus=*)        NUM_GPUS="${1#*=}"; shift ;;
-        -g|--first-gpu)  FIRST_GPU="$2"; shift 2 ;;
-        --first-gpu=*)   FIRST_GPU="${1#*=}"; shift ;;
+        -a|--arch)       ARCH="$2"; shift 2 ;;
+        --arch=*)        ARCH="${1#*=}"; shift ;;
         -l|--log-dir)    LOG_DIR="$2"; shift 2 ;;
         --log-dir=*)     LOG_DIR="${1#*=}"; shift ;;
-        -w|--weights)    WEIGHTS_FILE="$2"; shift 2 ;;
-        --weights=*)     WEIGHTS_FILE="${1#*=}"; shift ;;
         --only)          ONLY_RE="$2"; shift 2 ;;
         --only=*)        ONLY_RE="${1#*=}"; shift ;;
-        --skip-setup)    SKIP_SETUP=1; shift ;;
-        --dry-run)       DRY_RUN=1; shift ;;
-        -*)              echo "Unknown option: $1" >&2; exit 1 ;;
+        -h|--help)       usage; exit 0 ;;
+        -*)              echo "Unknown option: $1" >&2; usage >&2; exit 1 ;;
         *)               break ;;
     esac
 done
 
+if [[ -z "$ARCH" ]]; then
+    log_error "--arch is required (e.g. --arch gfx942), or set TE_CI_ARCH"
+    exit 1
+fi
+
 if [[ $# -eq 0 ]]; then
-    echo "Error: at least one config file is required." >&2
+    set -- "${SCRIPT_DIR}/ci_sgpu_queue.conf"
+    echo "No config given; using $1"
+fi
+
+# The pool is every GPU this run can see: HIP_VISIBLE_DEVICES if set
+# else what rocminfo counts.
+declare -a GPU_IDS=()
+GPU_SOURCE=""
+detect_gpu_pool() {
+    local n k
+    if [[ -n "${HIP_VISIBLE_DEVICES:-}" ]]; then
+        IFS=',' read -r -a GPU_IDS <<< "$HIP_VISIBLE_DEVICES"
+        GPU_SOURCE="HIP_VISIBLE_DEVICES=$HIP_VISIBLE_DEVICES"
+        return 0
+    fi
+    n=$(rocminfo 2>/dev/null | grep -c 'Device Type:.*GPU')
+    [[ -n "$n" && "$n" -gt 0 ]] || return 1
+    for ((k = 0; k < n; k++)); do GPU_IDS+=( "$k" ); done
+    GPU_SOURCE="rocminfo${ROCR_VISIBLE_DEVICES:+, ROCR_VISIBLE_DEVICES=$ROCR_VISIBLE_DEVICES}"
+}
+
+if ! detect_gpu_pool; then
+    log_error "no GPU found: rocminfo reports none and HIP_VISIBLE_DEVICES is unset." \
+              "Set HIP_VISIBLE_DEVICES to the devices this run may use."
     exit 1
 fi
 
+NUM_GPUS=${#GPU_IDS[@]}
+echo "== GPUs: ${NUM_GPUS} visible -- ids ${GPU_IDS[*]} (via ${GPU_SOURCE}) =="
+
+# The weight table is keyed by arch and TEST_LEVEL -- the coarsest keying under
+# which a weight still means one thing. Read below to order the queue and
+# rewritten in Phase 7, so a first local run is unordered and every run after it
+# is not.
+WEIGHTS_FILE="${REPO_ROOT}/ci-weights/test_weights.${ARCH}.l${TEST_LEVEL:-99}.txt"
+mkdir -p "$(dirname "$WEIGHTS_FILE")" 2>/dev/null
+
 resolved_configs=()
 for c in "$@"; do resolved_configs+=( "$(realpath -m "$c")" ); done
-[[ -n "$WEIGHTS_FILE" ]] && WEIGHTS_FILE="$(realpath -m "$WEIGHTS_FILE")"
 [[ "$LOG_DIR" != /* ]] && LOG_DIR="$(realpath -m "$LOG_DIR")"
 
+
+# One directory per phase, so "where did that come from" is answered by the path
+# and a phase can be cleared without touching the others:
+#
+#   prerequisite_ck_jit_status/ Phase 3  one-time prerequisites per suite
+#   items/                      Phase 4  the test output itself -- one file per item
+#   suites/                     Phase 5  per-suite verdict: rc + index into items/
+#   report/                     Phase 6  the human-readable schedule
+#   queue/                               the machine-readable state: what to run,
+#                                        what it cost
+#
+# Phase 1 has no directory: what it produced is queue.tsv and items.tsv, and all
+# it has of its own is whatever the suites printed on stderr while listing, which
+# is one file.
+EXPAND_LOG="$LOG_DIR/expand.log"
+SETUP_DIR="$LOG_DIR/prerequisite_ck_jit_status"
 ITEM_LOG_DIR="$LOG_DIR/items"
-mkdir -p "$ITEM_LOG_DIR"
+SUITE_LOG_DIR="$LOG_DIR/suites"
+REPORT_DIR="$LOG_DIR/report"
+QUEUE_DIR="$LOG_DIR/queue"
+
+rm -rf "${REPO_ROOT}/test-results"
+rm -rf "$ITEM_LOG_DIR" "$SUITE_LOG_DIR"
+
+mkdir -p "$SETUP_DIR" "$ITEM_LOG_DIR" "$SUITE_LOG_DIR" "$REPORT_DIR" "$QUEUE_DIR"
+: > "$EXPAND_LOG"
 cd "$REPO_ROOT" || { echo "Error: cannot cd to '${REPO_ROOT}'" >&2; exit 1; }
 
-QUEUE_FILE="$LOG_DIR/queue.tsv"
-IDX_FILE="$LOG_DIR/queue.idx"
-LOCK_FILE="$LOG_DIR/queue.lock"
+QUEUE_FILE="$QUEUE_DIR/queue.tsv"
+ITEMS_FILE="$QUEUE_DIR/items.tsv"
+TIMINGS_FILE="$QUEUE_DIR/timings.tsv"
+IDX_FILE="$QUEUE_DIR/queue.idx"
+LOCK_FILE="$QUEUE_DIR/queue.lock"
 : > "$QUEUE_FILE"
+: > "$ITEMS_FILE"
+# Phase 2 removes this, but only if it gets that far: an expansion that bails out
+# leaves it behind, and re-running into the same log dir would then append to the
+# previous attempt and trip the duplicate-tag check. Local runs do that a lot.
+: > "$QUEUE_FILE.raw"
 
 # ---------------------------------------------------------------------------
 # Phase 1: expand every suite into work items
 #
 # List mode runs the suite script with TE_CI_LIST_ONLY=1, which makes pytest_run
 # echo "TE_CI_ITEM <tag>" for each invocation it would have made instead of
-# running it. Level gating, backend matrix and capability probes all still apply,
-# so the emitted list is exactly what the suite would have executed.
+# running it. The suite's own control flow produces the list, so nothing here has
+# to know how a suite decomposes.
+#
+# Every list-mode suite is listed twice, because "will run" and "exists" are
+# different questions:
+#
+#   run 1  LIST_ONLY             what this host will run     -> the queue
+#   run 2  LIST_ONLY + LIST_ALL  what exists at this level   -> items.tsv
+#
+# Run 1 keeps every gate: level, backend matrix, and the host capability probes.
+# Run 2 answers the capability probes yes without probing (check_list_all in
+# ci/_utils.sh); the level and matrix gates still apply.
+#
+# So run2 - run1 is what this host skipped -- no flash-attn, say -- and those
+# tests do still exist. Only a tag in neither list is gone for good, and that is
+# what ci/build_weights.py prunes on.
 echo "== Expanding suites into work items =="
 declare -a SUITE_LABELS SUITE_LOGFILES
 while IFS= read -r line; do
@@ -100,18 +193,44 @@ while IFS= read -r line; do
     SUITE_LABELS+=( "$label" )
     SUITE_LOGFILES+=( "$logfile" )
     if [[ "$mode" == "list" ]]; then
-        mapfile -t tags < <(TE_CI_LIST_ONLY=1 "$cmd" ${rest:-} 2>/dev/null \
+        # The tags themselves are not logged here -- they go straight into
+        # queue.tsv and items.tsv, which is where anything reading them looks.
+        # What is left is stderr, banner-separated by suite and pass: the suite
+        # scripts report a declined capability probe on it, which is noise on a
+        # good run but the only explanation of a list that came back short.
+        echo "=== ${label}: list -- what this runner will run ===" >> "$EXPAND_LOG"
+        mapfile -t tags < <(TE_CI_LIST_ONLY=1 "$cmd" ${rest:-} \
+                            2>> "$EXPAND_LOG" \
                             | sed -n 's/^TE_CI_ITEM //p')
         if [[ ${#tags[@]} -eq 0 ]]; then
-            echo "::error::suite '${label}' (${cmd}) produced no work items" >&2
+            log_error "suite '${label}' (${cmd}) produced no work items"
+            tail -20 "$EXPAND_LOG" >&2
             exit 1
         fi
         for tag in "${tags[@]}"; do
             printf '%s\t%s\t%s\t%s\n' "$label" "$cmd" "$tag" "${rest:-}" >> "$QUEUE_FILE.raw"
         done
-        echo "  ${label}: ${#tags[@]} items"
+        echo "=== ${label}: list-all -- what exists at this level ===" >> "$EXPAND_LOG"
+        mapfile -t all_tags < <(TE_CI_LIST_ONLY=1 TE_CI_LIST_ALL=1 "$cmd" ${rest:-} \
+                                2>> "$EXPAND_LOG" \
+                                | sed -n 's/^TE_CI_ITEM //p')
+        # The wider list must be a superset. If it is not, it cannot be trusted
+        # as "everything that exists", so the label is left out of items.tsv --
+        # which is exactly what tells build_weights.py to prune nothing for it.
+        # Warn rather than fail: this only degrades bookkeeping, and refusing to
+        # run the tests over it would be a much worse trade.
+        if [[ ${#all_tags[@]} -lt ${#tags[@]} ]]; then
+            log_warn "suite '${label}': list-all returned ${#all_tags[@]} items," \
+                     "fewer than the ${#tags[@]} queued; its weights will not be pruned"
+        else
+            for tag in "${all_tags[@]}"; do
+                printf '%s\t%s\n' "$label" "$tag" >> "$ITEMS_FILE"
+            done
+        fi
+        echo "  ${label}: ${#tags[@]} items (${#all_tags[@]} exist at this level)"
     else
         printf '%s\t%s\t%s\t%s\n' "$label" "$cmd" "" "${rest:-}" >> "$QUEUE_FILE.raw"
+        printf '%s\t%s\n' "$label" "" >> "$ITEMS_FILE"
         echo "  ${label}: 1 item (opaque)"
     fi
 done < <(cat "${resolved_configs[@]}")
@@ -120,7 +239,7 @@ done < <(cat "${resolved_configs[@]}")
 # the TEST_FILTER dispatch (one invocation would run both). Fail loudly instead.
 dupes=$(cut -f1,3 "$QUEUE_FILE.raw" | grep -v '^\S*\t$' | sort | uniq -d)
 if [[ -n "$dupes" ]]; then
-    echo "::error::duplicate work item tags detected:" >&2
+    log_error "duplicate work item tags detected:"
     echo "$dupes" >&2
     exit 1
 fi
@@ -149,21 +268,21 @@ awk -v wf="$WEIGHTS_FILE" -v dw="$DEFAULT_WEIGHT" -v only="$ONLY_RE" '
 rm -f "$QUEUE_FILE.raw"
 
 if [[ ! -s "$QUEUE_FILE" ]]; then
-    echo "::error::queue is empty${ONLY_RE:+ (no item matched --only '${ONLY_RE}')}" >&2
+    log_error "queue is empty${ONLY_RE:+ (no item matched --only '${ONLY_RE}')}"
     exit 1
 fi
 
 TOTAL_ITEMS=$(wc -l < "$QUEUE_FILE")
-echo "== Queue: ${TOTAL_ITEMS} items over ${NUM_GPUS} GPUs (first=${FIRST_GPU}) =="
-if [[ -n "$WEIGHTS_FILE" && -f "$WEIGHTS_FILE" ]]; then
-    echo "   weights: $WEIGHTS_FILE"
+echo "== Queue: ${TOTAL_ITEMS} items over ${NUM_GPUS} GPUs (${GPU_IDS[*]}) =="
+if [[ -f "$WEIGHTS_FILE" ]]; then
+    echo "   weights: $WEIGHTS_FILE ($(wc -l < "$WEIGHTS_FILE") items)"
 else
-    echo "   weights: none (unordered -- run ci/build_weights.py after this run)"
+    echo "   weights: none yet -- this run is unordered and will write $WEIGHTS_FILE"
 fi
 
-# The plan is printed on every run, not just --dry-run: it is the only record of
-# what the scheduler intended, and comparing it against the schedule reported at
-# the end is how a mis-weighted item is spotted.
+# The plan is printed before anything runs: it is the only record of what the
+# scheduler intended, and comparing it against the schedule reported at the end
+# is how a mis-weighted item is spotted.
 echo "== Queue plan (dispatch order; est = weight used to sort) =="
 awk -F'\t' -v dw="$DEFAULT_WEIGHT" -v n="$NUM_GPUS" '
     { total += $1
@@ -176,37 +295,33 @@ awk -F'\t' -v dw="$DEFAULT_WEIGHT" -v n="$NUM_GPUS" '
     NR == 1 { first = ($1 == dw ? "unknown" : $1) }
 ' "$QUEUE_FILE"
 
-if [[ -n "$DRY_RUN" ]]; then
-    echo "== Dry run: nothing executed =="
-    exit 0
-fi
-
 # ---------------------------------------------------------------------------
 # Phase 3: one-time, container-wide setup
 #
-# Prerequisites and the CK JIT blob cache are shared filesystem state, so they
-# are built once here rather than by each of the N*items invocations. Running
-# the suites sequentially also removes the race they previously had over the
-# shared cache directory when launched in parallel.
-if [[ -n "$SKIP_SETUP" ]]; then
-    echo "== One-time suite setup: skipped (--skip-setup) =="
-fi
+# pip prerequisites and the CK JIT blob cache are container-wide filesystem
+# state, so each suite installs and prebuilds them once here.
+setup_banner=""
 for i in "${!SUITE_LABELS[@]}"; do
-    [[ -n "$SKIP_SETUP" ]] && break
-    [[ $i -eq 0 ]] && echo "== One-time suite setup =="
     label="${SUITE_LABELS[$i]}"
     # Columns are: weight, label, cmd, tag, rest. Only list-mode suites (those
     # with a non-empty tag) have a setup/dispatch split; opaque suites are a
     # single invocation that still does its own setup.
     cmd=$(awk -F'\t' -v l="$label" '$2==l && $4!="" {print $3; exit}' "$QUEUE_FILE")
     [[ -z "$cmd" ]] && continue
-    echo "  setup: $label ($cmd)"
-    if ! HIP_VISIBLE_DEVICES=$FIRST_GPU TE_CI_SETUP_ONLY=1 "$cmd" \
-            > "$LOG_DIR/setup.${label}.log" 2>&1; then
-        echo "::error::setup failed for ${label}; see $LOG_DIR/setup.${label}.log" >&2
-        tail -30 "$LOG_DIR/setup.${label}.log" >&2
+    if [[ -z "$setup_banner" ]]; then
+        echo "== One-time setup: pip prerequisites + CK JIT prebuild (once per suite) =="
+        setup_banner=1
+    fi
+    printf '  %s: installing prerequisites, prebuilding CK JIT blobs (%s) ... ' "$label" "$cmd"
+    setup_start=$(date +%s)
+    if ! HIP_VISIBLE_DEVICES=${GPU_IDS[0]} TE_CI_SETUP_ONLY=1 "$cmd" \
+            > "$SETUP_DIR/${label}.log" 2>&1; then
+        echo "FAILED"
+        log_error "setup failed for ${label}; see $SETUP_DIR/${label}.log"
+        tail -30 "$SETUP_DIR/${label}.log" >&2
         exit 1
     fi
+    echo "done in $(( $(date +%s) - setup_start ))s"
 done
 
 # ---------------------------------------------------------------------------
@@ -227,7 +342,7 @@ take_next() {
 
 worker() {
     local gpu=$1
-    local i line weight label cmd tag rest itemlog rc safetag junit_dir start end
+    local i line weight label cmd tag rest itemlog rc safetag junit_dir start end incomplete
     while :; do
         i=$(take_next)
         [[ "$i" -gt "$TOTAL_ITEMS" ]] && break
@@ -246,11 +361,7 @@ worker() {
 
         # No scheduler-imposed deadline: the suite scripts' own PYTEST_TIMEOUT and
         # the workflow's timeout-minutes are the only limits, exactly as they are
-        # outside the queue. Deriving one from the weight would make the timeout
-        # depend on measurement history, so the same test could pass one run and
-        # be killed the next -- the queue must not change what "a test failed"
-        # means. build_weights.py still drops rc 124/137 rows, because a duration
-        # that ends in a kill is a ceiling wherever the kill came from.
+        # outside the queue.
         start=$(date +%s)
         if [[ -n "$tag" ]]; then
             HIP_VISIBLE_DEVICES=$gpu TE_CI_SKIP_SETUP=1 TEST_FILTER="$tag" \
@@ -264,15 +375,35 @@ worker() {
         end=$(date +%s)
 
         echo "$rc" > "${itemlog}.rc"
+
+        # A te_ci_result_sink sidecar that outlived the process means pytest
+        # never reached its end-of-session write -- a --timeout-method=thread
+        # expiry, a segfault, or an OOM-kill -- so this duration is where the
+        # item was cut off, not what it costs. rc cannot be used to tell: a
+        # thread-method timeout exits 1, indistinguishable from an ordinary test
+        # failure, which is a perfectly good measurement. build_weights.py drops
+        # the flagged rows rather than teaching the table a truncated number.
+        incomplete=0
+        if [[ -n "$junit_dir" ]]; then
+            if [[ -n "$tag" ]]; then
+                # Items sharing a label share junit_dir, so only this item's own
+                # sidecar may be consulted -- a glob would see the in-flight
+                # sidecar of an item still running on another GPU.
+                [[ -e "${junit_dir}${tag}${JUNITXML_SUFFIX:-}.partial" ]] && incomplete=1
+            elif compgen -G "${junit_dir}*.partial" > /dev/null; then
+                incomplete=1   # opaque suite: it is the only item in its dir
+            fi
+        fi
+
         # Machine-readable timing record. The first five columns are the stable
-        # contract ci/build_weights.py --from-timing reads; the scheduling
-        # columns are appended after them so that reader stays unchanged. Written
-        # incrementally so a run killed by the job timeout still leaves a usable
-        # record of everything that had completed.
-        printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n' \
+        # contract ci/build_weights.py reads; the scheduling columns are appended
+        # after them so that reader stays unchanged. Written incrementally so a
+        # run killed by the job timeout still leaves a usable record of
+        # everything that had completed.
+        printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n' \
             "$label" "$safetag" "$gpu" "$((end - start))" "$rc" \
-            "$((start - START_TS))" "$((end - START_TS))" "$weight" \
-            >> "$LOG_DIR/timings.tsv"
+            "$((start - START_TS))" "$((end - START_TS))" "$weight" "$incomplete" \
+            >> "$TIMINGS_FILE"
         printf '[%s] gpu%-2s t+%-7s %5ss rc=%-4s est=%-8s %-9s %s\n' \
             "$(date '+%H:%M:%S')" "$gpu" "$((start - START_TS))s" "$((end - start))" \
             "$rc" "$([[ $weight -eq $DEFAULT_WEIGHT ]] && echo unknown || echo "${weight}s")" \
@@ -280,9 +411,9 @@ worker() {
     done
 }
 
-: > "$LOG_DIR/timings.tsv"
-for k in $(seq 0 $((NUM_GPUS - 1))); do
-    worker $((FIRST_GPU + k)) &
+: > "$TIMINGS_FILE"
+for gpu in "${GPU_IDS[@]}"; do
+    worker "$gpu" &
 done
 wait
 
@@ -292,21 +423,31 @@ WALL=$(( $(date +%s) - START_TS ))
 [[ $WALL -lt 1 ]] && WALL=1
 
 # ---------------------------------------------------------------------------
-# Phase 5: fold per-item results back into per-suite logs and exit codes
+# Phase 5: roll per-item results up into a per-suite verdict
+#
+# Two outputs per suite, both keyed by the logfile name from the config so the
+# workflow's failure gate keeps reading the same name: the exit code it gates on,
+# and an index of which items ran and how each one ended.
+#
+# The item logs are not concatenated into that index. They are the largest files
+# the run produces, and copying them would double both the artifact and the
+# reader's work -- the index says which item to open, and the item log is right
+# there under items/ with only that item's output in it.
 OVERALL_RC=0
 for i in "${!SUITE_LABELS[@]}"; do
     label="${SUITE_LABELS[$i]}"
     # A suite with no items in the queue (filtered out by --only) must not get a
     # suite log and rc=0 -- that would read as "passed" to downstream consumers.
     awk -F'\t' -v l="$label" '$2==l {found=1} END {exit !found}' "$QUEUE_FILE" || continue
-    suite_log="$LOG_DIR/${SUITE_LOGFILES[$i]}"
+    suite_log="$SUITE_LOG_DIR/${SUITE_LOGFILES[$i]}"
     : > "$suite_log"
     worst=0
     for itemlog in "$ITEM_LOG_DIR/${label}."*.log; do
         [[ -e "$itemlog" ]] || continue
         rc=$(cat "${itemlog}.rc" 2>/dev/null || echo 1)
-        echo "##### $(basename "$itemlog" .log) (rc=${rc}) #####" >> "$suite_log"
-        cat "$itemlog" >> "$suite_log"
+        printf '%-4s rc=%-4s items/%s\n' \
+            "$([[ "$rc" == "0" ]] && echo ok || echo FAIL)" "$rc" \
+            "$(basename "$itemlog")" >> "$suite_log"
         [[ "$rc" != "0" ]] && worst=$rc
     done
     echo "$worst" > "${suite_log}.rc"
@@ -316,127 +457,228 @@ done
 # ---------------------------------------------------------------------------
 # Phase 6: scheduling report
 #
-# timings.tsv columns: label, tag, gpu, secs, rc, start_off, end_off, est.
+# timings.tsv columns: label, tag, gpu, secs, rc, start_off, end_off, est,
+# incomplete.
 # Everything below is derived from that one file, so the same report can be
 # regenerated after the fact from an uploaded artifact.
-report() {
-    local tf="$LOG_DIR/timings.tsv"
+#
+# Each section is a rows_* function emitting TSV whose first line is the header,
+# and one of two renderers turns that into either an aligned plain-text table
+# (job log + schedule.txt) or a GitHub-flavoured Markdown one (schedule.md, which
+# is appended to the job summary alongside the junit_report.py sections). Keeping
+# the data and the formatting separate is what lets both exist without the awk
+# being written twice.
+
+txt_table() {
+    awk -F'\t' '
+        { rows = NR; if (NF > cols) cols = NF
+          for (i = 1; i <= NF; i++) {
+              cell[NR, i] = $i
+              if (length($i) > w[i]) w[i] = length($i) } }
+        END {
+            for (r = 1; r <= rows; r++) {
+                line = "  "
+                for (i = 1; i <= cols; i++) line = line sprintf("%-" w[i] "s  ", cell[r, i])
+                sub(/ +$/, "", line); print line
+                if (r == 1) {                       # rule under the header row
+                    line = "  "
+                    for (i = 1; i <= cols; i++) {
+                        d = ""; while (length(d) < w[i]) d = d "-"
+                        line = line d "  " }
+                    sub(/ +$/, "", line); print line
+                }
+            }
+        }'
+}
 
-    # Every section is a fixed-width table with a header row, so the report can be
-    # read straight out of the GHA step summary or grepped/cut by column.
-    local rule="  ------ -------- -------- ------- ---------- -------- ----------------------------------------"
+md_table() {
+    awk -F'\t' '
+        { out = ""
+          for (i = 1; i <= NF; i++) { c = $i; gsub(/\|/, "\\|", c); out = out "| " c " " }
+          print out "|"
+          if (NR == 1) { s = "|"; for (i = 1; i <= NF; i++) s = s "---|"; print s } }'
+}
 
-    echo "=== sGPU queue: ${TOTAL_ITEMS} items, ${NUM_GPUS} GPUs, drained in ${WALL}s ==="
-    echo
-    echo "-- schedule: what ran where, in execution order --"
-    echo "$rule"
-    printf "  %-6s %-8s %-8s %-7s %-10s %-8s %s\n" \
-           GPU START DURATION RESULT ESTIMATE "% CHANGE" TEST_NAME
-    echo "$rule"
-    # rc is reported as a word, not a number: the report is read by people, and
-    # "killed" vs "fail" is the distinction that actually changes what you do next.
-    sort -t$'\t' -k3,3n -k6,6n "$tf" | awk -F'\t' -v dw="$DEFAULT_WEIGHT" -v rule="$rule" '
-        $3 != last { if (NR > 1) print rule; last = $3 }
-        {
+# rc is reported as a word, not a number: the report is read by people, and
+# "killed" vs "fail" is the distinction that changes what you do next. "cut"
+# marks a duration that is where the item was stopped rather than what it costs,
+# which is also why the next run's weight table ignores that row.
+rows_schedule() {
+    printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\n' \
+           GPU Start Duration Result Estimate "% change" "Test name"
+    sort -t$'\t' -k3,3n -k6,6n "$TIMINGS_FILE" \
+        | awk -F'\t' -v dw="$DEFAULT_WEIGHT" -v OFS='\t' '{
             est = ($8 == dw ? "unknown" : $8 "s")
-            # Estimate error is the scheduler feedback loop made visible: a large
+            # The % change is the scheduler feedback loop made visible: a large
             # positive miss is an item that should have been dispatched earlier.
-            err = ($8 > 0 && $8 != dw) ? sprintf("%+.0f%%", ($4 - $8) * 100 / $8) : "n/a"
+            chg = ($8 > 0 && $8 != dw) ? sprintf("%+.0f%%", ($4 - $8) * 100 / $8) : "n/a"
             res = ($5 == 0 ? "pass" : ($5 == 1 ? "fail" : \
                   ($5 == 124 || $5 == 137 ? "killed" : "error")))
-            printf "  gpu%-3s %-8s %-8s %-7s %-10s %-8s %s/%s\n",
-                   $3, "t+" $6 "s", $4 "s", res, est, err, $1, $2
-        }'
-    echo "$rule"
-    echo "  START = seconds after the queue opened. DURATION = wall clock for the whole"
-    echo "  item, process startup included. ESTIMATE = the weight it was scheduled on."
+            if ($9 == 1) res = res " (cut)"
+            print "gpu" $3, "t+" $6 "s", $4 "s", res, est, chg, $1 "/" $2 }'
+}
 
-    echo
-    echo "-- per-GPU utilisation --"
-    echo "  ------ ------- --------- --------- --------- --------"
-    printf "  %-6s %-7s %-9s %-9s %-9s %s\n" GPU ITEMS BUSY IDLE UTIL FAILED
-    echo "  ------ ------- --------- --------- --------- --------"
-    awk -F'\t' -v w="$WALL" '
+# Iterating the assigned ids rather than the keys present keeps the rows ordered
+# and surfaces a GPU that took no work at all -- which would otherwise just
+# vanish. The ids need not be contiguous or start at zero, since they may have
+# come from HIP_VISIBLE_DEVICES.
+rows_gpu() {
+    printf '%s\t%s\t%s\t%s\t%s\t%s\n' GPU Items Busy Idle Util Failed
+    awk -F'\t' -v w="$WALL" -v ids="${GPU_IDS[*]}" -v OFS='\t' '
         { busy[$3] += $4; n[$3]++; if ($5 != 0) bad[$3]++ }
         END {
-            for (g in busy)
-                printf "  gpu%-3s %-7d %-9s %-9s %-9s %d\n",
-                       g, n[g], busy[g] "s", (w - busy[g]) "s",
-                       sprintf("%.1f%%", busy[g] * 100 / w), bad[g] + 0
-        }' "$tf" | sort
-    echo "  ------ ------- --------- --------- --------- --------"
+            ng = split(ids, g, " ")
+            for (i = 1; i <= ng; i++)
+                print "gpu" g[i], n[g[i]] + 0, (busy[g[i]] + 0) "s", (w - busy[g[i]]) "s",
+                      sprintf("%.1f%%", (busy[g[i]] + 0) * 100 / w), bad[g[i]] + 0
+        }' "$TIMINGS_FILE"
+}
 
-    echo
-    echo "-- efficiency --"
-    echo "  ----------------- ---------- --------------------------------------------------"
-    printf "  %-17s %-10s %s\n" METRIC VALUE MEANING
-    echo "  ----------------- ---------- --------------------------------------------------"
-    awk -F'\t' -v w="$WALL" -v n="$NUM_GPUS" '
+rows_efficiency() {
+    printf '%s\t%s\t%s\n' Metric Value Meaning
+    awk -F'\t' -v w="$WALL" -v n="$NUM_GPUS" -v OFS='\t' '
         { work += $4; if ($4 > big) { big = $4; bigname = $1 "/" $2 } }
         END {
-            printf "  %-17s %-10s %s\n", "total work", sprintf("%.0fs", work),
-                   sprintf("sum of all %d item durations", NR)
-            printf "  %-17s %-10s %s\n", "actual run time", sprintf("%.0fs", w),
-                   sprintf("wall clock, first item start to last item finish")
-            printf "  %-17s %-10s %s\n", "utilisation", sprintf("%.1f%%", work * 100 / (w * n)),
-                   sprintf("share of %ss x %d GPUs actually spent running tests", w, n)
-            printf "  %-17s %-10s %s\n", "largest item", sprintf("%.0fs", big),
-                   bigname (big > work / n ? "  <-- floor: splitting it would now pay" : "")
-        }' "$tf"
-    echo "  ----------------- ---------- --------------------------------------------------"
+            print "total work", sprintf("%.0fs", work),
+                  sprintf("sum of all %d item durations", NR)
+            print "actual run time", sprintf("%.0fs", w),
+                  "wall clock, first item start to last item finish"
+            print "utilisation", sprintf("%.1f%%", work * 100 / (w * n)),
+                  sprintf("share of %ss x %d GPUs actually spent running tests", w, n)
+            # Once one item exceeds the per-GPU average there is no ordering that
+            # finishes sooner than that item, so it becomes the thing to split.
+            print "largest item", sprintf("%.0fs", big),
+                  bigname (big > work / n ? " -- floor: splitting it would now pay" : "")
+        }' "$TIMINGS_FILE"
+}
 
-    echo
-    echo "-- weight accuracy: misses over 30s, which the next run corrects --"
-    # 30s is roughly one process startup, and below the ~5% run-to-run noise for
-    # anything long enough to matter. Listing smaller misses would bury the real
-    # ones -- a healthy table has almost every item within a few seconds.
-    echo "  --------- --------- --------- -------- ---------------------------------------"
-    printf "  %-9s %-9s %-9s %-8s %s\n" MISS ESTIMATE ACTUAL "% CHANGE" TEST_NAME
-    echo "  --------- --------- --------- -------- ---------------------------------------"
+# 30s is roughly one process startup, and below the ~5% run-to-run noise for
+# anything long enough to matter. Listing smaller misses would bury the real
+# ones -- a healthy table has almost every item within a few seconds.
+rows_weights() {
+    printf '%s\t%s\t%s\t%s\t%s\n' Miss Estimate Actual "% change" "Test name"
     awk -F'\t' -v dw="$DEFAULT_WEIGHT" '$8 != dw && $8 > 0 {
             d = $4 - $8; a = (d < 0 ? -d : d)
-            if (a >= 30) printf "%.0f\t%+.0f\t%s\t%s\t%.0f\t%s\n", a, d, $8, $4,
-                                (d * 100 / $8), $1 "/" $2 }' "$tf" \
-        | sort -k1,1nr | head -15 \
-        | awk -F'\t' '{printf "  %-9s %-9s %-9s %-8s %s\n",
-                              $2 "s", $3 "s", $4 "s", sprintf("%+d%%", $5), $6}'
-    awk -F'\t' -v dw="$DEFAULT_WEIGHT" '$8 == dw {
-            printf "  %-9s %-9s %-9s %-8s %s\n", "n/a", "unknown", $4 "s", "n/a", $1 "/" $2 }' "$tf"
-    echo "  --------- --------- --------- -------- ---------------------------------------"
+            if (a >= 30) printf "%.0f\t%+.0fs\t%ss\t%ss\t%+.0f%%\t%s\n",
+                                a, d, $8, $4, d * 100 / $8, $1 "/" $2 }' "$TIMINGS_FILE" \
+        | sort -k1,1nr | head -15 | cut -f2-
+    awk -F'\t' -v dw="$DEFAULT_WEIGHT" -v OFS='\t' '$8 == dw {
+            print "n/a", "unknown", $4 "s", "n/a", $1 "/" $2 }' "$TIMINGS_FILE"
+}
 
-    echo
-    echo "-- failures --"
-    echo "  -------- --------- --------------------------------------------------------------"
-    printf "  %-8s %-9s %s\n" RESULT DURATION TEST_NAME
-    echo "  -------- --------- --------------------------------------------------------------"
-    if awk -F'\t' '$5 != 0 {found = 1} END {exit !found}' "$tf"; then
-        awk -F'\t' '$5 != 0 {
+rows_failures() {
+    printf '%s\t%s\t%s\n' Result Duration "Test name"
+    if awk -F'\t' '$5 != 0 {found = 1} END {exit !found}' "$TIMINGS_FILE"; then
+        awk -F'\t' -v OFS='\t' '$5 != 0 {
             res = ($5 == 1 ? "fail" : ($5 == 124 || $5 == 137 ? "killed" : "error"))
-            printf "  %-8s %-9s %s/%s\n", res, $4 "s", $1, $2 }' "$tf"
+            print res, $4 "s", $1 "/" $2 }' "$TIMINGS_FILE"
     else
-        printf "  %-8s %-9s %s\n" "-" "-" "none"
+        printf '%s\t%s\t%s\n' - - none
     fi
-    echo "  -------- --------- --------------------------------------------------------------"
+}
+
+# An item that never got a timings.tsv row was never dispatched -- only possible
+# if a worker died outright. Silence here would read as success.
+missing_items() {
+    awk -F'\t' 'NR == FNR {seen[$1 "/" $2] = 1; next}
+                { k = ($4 == "" ? $2 "/whole" : $2 "/" $4); if (!(k in seen)) print k }' \
+        "$TIMINGS_FILE" "$QUEUE_FILE"
+}
+
+RAN_ITEMS=$(wc -l < "$TIMINGS_FILE")
+FAILED_ITEMS=$(awk -F'\t' '$5 != 0' "$TIMINGS_FILE" | wc -l)
 
-    # An item that never got a timings.tsv row was never dispatched -- only
-    # possible if a worker died outright. Silence here would read as success.
-    local ran; ran=$(wc -l < "$tf")
-    if [[ "$ran" -ne "$TOTAL_ITEMS" ]]; then
+report() {
+    echo "=== sGPU queue: ${TOTAL_ITEMS} items, ${NUM_GPUS} GPUs, drained in ${WALL}s ==="
+    echo
+    echo "-- efficiency --"
+    rows_efficiency | txt_table
+    echo
+    echo "-- per-GPU utilisation --"
+    rows_gpu | txt_table
+    echo
+    echo "-- failures --"
+    rows_failures | txt_table
+    echo
+    echo "-- schedule: what ran where, in execution order --"
+    rows_schedule | txt_table
+    echo
+    echo "-- weight accuracy: misses over 30s, which the next run corrects --"
+    rows_weights | txt_table
+    if [[ "$RAN_ITEMS" -ne "$TOTAL_ITEMS" ]]; then
         echo
-        echo "  !! ${ran}/${TOTAL_ITEMS} items produced a timing record; the rest never ran:"
-        awk -F'\t' 'NR == FNR {seen[$1 "/" $2] = 1; next}
-                    { k = ($4 == "" ? $2 "/whole" : $2 "/" $4); if (!(k in seen)) print "     " k }' \
-            "$tf" "$QUEUE_FILE"
+        echo "  !! ${RAN_ITEMS}/${TOTAL_ITEMS} items produced a timing record; the rest never ran:"
+        missing_items | sed 's/^/     /'
     fi
 }
 
-report | tee "$LOG_DIR/schedule.txt"
-{
-    echo "## sGPU queue schedule (${WALL}s over ${NUM_GPUS} GPUs)"
+report_md() {
+    local mark=":white_check_mark:"
+    [[ "$FAILED_ITEMS" -gt 0 || "$RAN_ITEMS" -ne "$TOTAL_ITEMS" ]] && mark=":x:"
+    local util
+    util=$(awk -F'\t' -v w="$WALL" -v n="$NUM_GPUS" \
+               '{work += $4} END {printf "%.1f", work * 100 / (w * n)}' "$TIMINGS_FILE")
+
+    echo "## ${REPORT_TITLE}"
+    echo
+    echo "${mark} **${RAN_ITEMS} items** on ${NUM_GPUS} GPUs -- ${FAILED_ITEMS} failed" \
+         "-- ${WALL}s wall clock at ${util}% GPU utilisation"
+    echo
+    if [[ "$RAN_ITEMS" -ne "$TOTAL_ITEMS" ]]; then
+        echo "> :warning: **Only ${RAN_ITEMS} of ${TOTAL_ITEMS} items produced a timing"
+        echo "> record.** The rest were never dispatched, which means a worker died:"
+        echo
+        missing_items | sed 's/^/> - `/; s/$/`/'
+        echo
+    fi
+    echo "### Efficiency"
+    echo
+    rows_efficiency | md_table
+    echo
+    echo "### Per-GPU utilisation"
+    echo
+    rows_gpu | md_table
+    echo
+    echo "### Failures"
+    echo
+    rows_failures | md_table
+    echo
+    echo "<details><summary>Schedule -- what ran where, in execution order</summary>"
     echo
-    echo '```'
-    cat "$LOG_DIR/schedule.txt"
-    echo '```'
-} > "$LOG_DIR/schedule.md"
+    rows_schedule | md_table
+    echo
+    echo "</details>"
+    echo
+    echo "<details><summary>Weight accuracy -- misses over 30s, which the next run corrects</summary>"
+    echo
+    rows_weights | md_table
+    echo
+    echo "</details>"
+    echo
+}
+
+report | tee "$REPORT_DIR/schedule.txt"
+report_md > "$REPORT_DIR/schedule.md"
+
+# ---------------------------------------------------------------------------
+# Phase 7: fold this run's timings back into the weight table
+#
+# Every run that gets here updates the table, and only runs that get here do --
+# one owner, so the asymmetric blend sees each measurement exactly once. A queue
+# killed outright (the CI step's timeout, say) therefore teaches the table
+# nothing, which is the right way round: its durations are mostly ceilings
+# imposed by the kill, not costs.
+#
+# Unconditional on the test results, though. A red run still measures how long
+# each item took, and build_weights.py drops the individual rows that were killed
+# or cut short.
+echo
+if ! command -v python3 > /dev/null 2>&1; then
+    log_warn "python3 not found; $WEIGHTS_FILE was not updated" \
+             "and the next run will be unordered again"
+elif ! python3 "$REPO_ROOT/ci/build_weights.py" "$TIMINGS_FILE" \
+        --items "$ITEMS_FILE" -o "$WEIGHTS_FILE"; then
+    log_warn "could not update $WEIGHTS_FILE; the next run will use the table as it stands"
+fi
 
 exit $OVERALL_RC
diff --git a/.github/workflows/rocm-ci.yml b/.github/workflows/rocm-ci.yml
index 16f12dd3b6..9d837946a9 100644
--- a/.github/workflows/rocm-ci.yml
+++ b/.github/workflows/rocm-ci.yml
@@ -188,12 +188,18 @@ jobs:
       # `restore` and `save` are separate actions on purpose: `actions/cache`
       # skips its implicit save when the job fails, and a failing test run is
       # exactly when the timings are most worth keeping.
+      #
+      # The key carries the test level as well as the arch. A level decides both
+      # which items run and how long some of them take (tests/jax switches on
+      # NVTE_JAX_UNITTEST_LEVEL, test_grouped_gemm.py adds its real shapes at
+      # level 2), so a shared table would have L1 and L3 runs continually
+      # overwriting each other's weights for the same key with different numbers.
       - name: Restore learned test weights
         uses: actions/cache/restore@v4
         with:
           path: ci-weights
-          key: te-sgpu-weights-${{ matrix.arch_label }}-${{ github.run_id }}
-          restore-keys: te-sgpu-weights-${{ matrix.arch_label }}-
+          key: te-sgpu-weights-${{ matrix.arch_label }}-l${{ env.TEST_LEVEL }}-${{ github.run_id }}
+          restore-keys: te-sgpu-weights-${{ matrix.arch_label }}-l${{ env.TEST_LEVEL }}-
 
       - name: Run sGPU tests as one queue (pytorch, jax, examples, core)
         id: run-tests
@@ -204,17 +210,19 @@ jobs:
           HF_TOKEN: ${{ secrets.HF_TOKEN }}
           ARCH: ${{ matrix.arch_label == 'mi30x' && 'gfx942' || 'gfx950' }}
         run: |
-          # One subdir per suite so PyTorch/JAX/Core get independent reports and
-          # never collide on identically-named XML (e.g. test_sanity_import.auto.xml
-          # exists in both the torch and jax suites and runs in parallel).
-          rm -rf test-results && mkdir -p test-results/torch test-results/jax test-results/core
+          # The queue clears test-results/ itself and creates one XML subdir per
+          # suite, so PyTorch/JAX/Core get independent reports and never collide
+          # on identically-named XML (test_sanity_import.auto.xml exists in both
+          # the torch and jax suites and runs in parallel). Clearing it here too
+          # covers the case where the container never starts.
+          rm -rf test-results
 
           # No in-tree seed table: on a cache miss the file is simply absent, the
           # queue runs in its natural order for one run, and this run's timings
           # create the table. A checked-in copy would go stale silently and is
           # worth less than the one unordered run it saves.
           mkdir -p ci-weights
-          W="ci-weights/test_weights.${ARCH}.txt"
+          W="ci-weights/test_weights.${ARCH}.l${{ env.TEST_LEVEL }}.txt"
           echo "weights: $W ($(wc -l < "$W" 2>/dev/null || echo 0) items)"
 
           docker exec \
@@ -223,41 +231,24 @@ jobs:
             -e JUNITXML_PREFIX=/workspace/test-results/ \
             -e JUNITXML_SUFFIX=.xml \
             -e HF_TOKEN="$HF_TOKEN" \
+            -e GITHUB_ACTIONS \
             te-runner bash /workspace/.github/scripts/run_queue_sgpu.sh \
-              --gpus 4 --first-gpu 0 \
-              --log-dir /workspace/ \
-              --weights "/workspace/$W" \
+              --arch "$ARCH" \
+              --log-dir /workspace/test-results/logs \
               /workspace/.github/scripts/ci_sgpu_queue.conf
 
-      # Runs even when tests failed: a red run still measures how long each item
-      # took, and build_weights.py's asymmetric blend is built to absorb the bias
-      # a short-because-it-crashed measurement introduces. Only items whose
-      # process was killed (rc 124/137) are dropped, since that measurement is a
-      # ceiling that would otherwise compound run over run.
-      - name: Update learned test weights
-        if: always() && hashFiles('timings.tsv') != ''
-        env:
-          ARCH: ${{ matrix.arch_label == 'mi30x' && 'gfx942' || 'gfx950' }}
-        run: |
-          docker exec te-runner python3 /workspace/ci/build_weights.py \
-            --from-timing /workspace/timings.tsv \
-            --merge -o "/workspace/ci-weights/test_weights.${ARCH}.txt"
-
+      # The queue's own last phase updated ci-weights/ before it exited, including
+      # when tests failed -- a red run still measures how long each item took. Only
+      # a run killed outright (this step's timeout) skips it, and that run's
+      # durations are ceilings imposed by the kill rather than costs, so losing
+      # them is no loss. Saving is separate because it must happen either way:
+      # a failed run's table is still better than the one restored above.
       - name: Save learned test weights
         if: always() && hashFiles('ci-weights/*') != ''
         uses: actions/cache/save@v4
         with:
           path: ci-weights
-          key: te-sgpu-weights-${{ matrix.arch_label }}-${{ github.run_id }}
-
-      - name: Publish scheduling report
-        if: always()
-        run: |
-          if [ -f schedule.md ]; then
-            cat schedule.md >> "$GITHUB_STEP_SUMMARY"
-          else
-            echo "::warning::no schedule.md -- the queue did not reach its report phase"
-          fi
+          key: te-sgpu-weights-${{ matrix.arch_label }}-l${{ env.TEST_LEVEL }}-${{ github.run_id }}
 
       - name: Generate test report
         if: always()
@@ -273,6 +264,22 @@ jobs:
           python3 ci/junit_report.py test-results/core \
             --title "sGPU Core (${{ matrix.arch_label }})"
 
+      # Last, so the summary leads with test results: how the work was scheduled
+      # only matters once you know what the tests did.
+      - name: Publish scheduling report
+        if: always()
+        run: |
+          if [ -f test-results/logs/report/schedule.md ]; then
+            cat test-results/logs/report/schedule.md >> "$GITHUB_STEP_SUMMARY"
+          else
+            echo "::warning::no schedule.md -- the queue did not reach its report phase"
+          fi
+
+      # The gate is per suite, as before -- the queue interleaves suites across
+      # GPUs, but a suite is still what "PyTorch tests failed" means. What is
+      # dumped inline is per item: only the item logs that actually ended non-zero,
+      # rather than the whole suite's output with the failure buried in it. The
+      # passing items are in the artifact for anyone who wants them.
       - name: Check suite failure status
         if: always()
         run: |
@@ -282,29 +289,35 @@ jobs:
               [[ "$line" =~ ^[[:space:]]*# ]] && continue
               [[ -z "${line//[[:space:]]/}" ]] && continue
               read -r label logfile rest <<< "$line"
-              rc=$(cat "${logfile}.rc" 2>/dev/null || echo "missing")
-              if [[ "$rc" != "0" ]]; then
-                echo "::group::[FAILED] ${label} Log (rc=${rc})"
-                cat "$logfile" 2>/dev/null || echo "(log not found)"
+              rc=$(cat "test-results/logs/suites/${logfile}.rc" 2>/dev/null || echo "missing")
+              [[ "$rc" == "0" ]] && continue
+              echo "::error::${label} tests FAILED (rc=${rc})."
+              EXIT_STATUS=1
+              for f in test-results/logs/items/"${label}".*.log; do
+                [[ -e "$f" ]] || continue
+                irc=$(cat "${f}.rc" 2>/dev/null || echo 1)
+                [[ "$irc" == "0" ]] && continue
+                echo "::group::[FAILED] $(basename "$f" .log) (rc=${irc})"
+                cat "$f"
                 echo "::endgroup::"
-                echo "::error::${label} tests FAILED."
-                EXIT_STATUS=1
-              fi
+              done
             done < "$config"
           done
           exit $EXIT_STATUS
 
+      # test-results/ is the whole run: XML per suite plus one directory per
+      # scheduler phase under logs/. A download answers "what did it plan, what
+      # did it set up, what did each item print, what did each item cost" from a
+      # single tree. logs/queue/ is what makes it reproducible after the fact --
+      # schedule.txt and the weight table can both be regenerated from
+      # timings.tsv + items.tsv alone.
       - name: Upload logs
         if: always()
         uses: actions/upload-artifact@v4
         with:
           name: logs-sgpu-${{ matrix.arch_label }}
           path: |
-            *.log
-            test-results/**/*.xml
-            timings.tsv
-            schedule.txt
-            queue.tsv
+            test-results/**
             ci-weights/*.txt
           if-no-files-found: ignore
           retention-days: 5
diff --git a/.gitignore b/.gitignore
index 940c43b720..563145f080 100644
--- a/.gitignore
+++ b/.gitignore
@@ -64,3 +64,5 @@ transformer_engine/common/util/hip_nvml.*
 .codex/
 .cline_storage/
 .claude/
+test-results/
+ci-weights/
diff --git a/ci/_utils.sh b/ci/_utils.sh
index f54f7e74a9..34e506e87d 100644
--- a/ci/_utils.sh
+++ b/ci/_utils.sh
@@ -139,6 +139,17 @@ check_level() {
     test $TEST_LEVEL -ge $1
 }
 
+# TE_CI_LIST_ALL widens list mode from "what this host would run" to "what this
+# TEST_LEVEL could run anywhere". Level gating, the backend matrix and TEST_FILTER
+# still apply -- only host capability probes are answered yes without probing.
+# The scheduler uses the wider list as the authoritative set of work items that
+# exist, so ci/build_weights.py can tell a test that was deleted or renamed (drop
+# its weight) from one this host merely skipped (keep it). A probe result varies
+# by machine; whether a test file exists does not.
+check_list_all() {
+    test -n "$TE_CI_LIST_ONLY" -a -n "$TE_CI_LIST_ALL"
+}
+
 check_test_jobs_requested() {
     return 1 #Disable parallel jobs because some tests do not support parallel execution
     test -z "$SINGLE_CONFIG" -a -n "$TEST_JOBS" || return 1
diff --git a/ci/build_weights.py b/ci/build_weights.py
index 584797048e..a0e7f349df 100755
--- a/ci/build_weights.py
+++ b/ci/build_weights.py
@@ -3,68 +3,43 @@
 #
 # See LICENSE for license information.
 
-"""Build a work-item weight table for the sGPU queue scheduler.
-
-``run_queue_sgpu.sh`` orders its queue longest-first, which needs a per-item
-cost estimate. Both sources below are produced by a normal CI run, so no extra
-measurement pass is required:
-
-  --from-timing <file>  read ``timings.tsv`` written by the scheduler itself.
-                        Preferred for scheduling: it is per-item wall clock, so
-                        it includes process startup and CUDA/HIP init, which is
-                        exactly what the makespan is made of.
-  --from-junit <dir>    sum the per-testcase ``time`` attributes of every
-                        ``<label>/<tag>.xml`` the run wrote. Useful when only
-                        the ``logs-sgpu-*`` artifacts are available, but it
-                        counts test time only -- a short item can look ~4x
-                        cheaper than it really is once startup is included.
-
-Output is ``<label>/<tag> <seconds>`` per line, plus ``<label> <seconds>`` for
-opaque whole-suite items. Weights are per GPU architecture -- skip patterns
-differ between gfx942 and gfx950 -- so keep one table per arch.
-
-With ``--merge`` the output file is read first and the new measurement is
-blended into it, so every CI run refines the table instead of replacing it from
-a single noisy sample. See ``merge_weights`` for why the blend is asymmetric.
+"""Fold one CI run's measured timings into the sGPU scheduler's weight table.
+
+``run_queue_sgpu.sh`` orders its queue longest-first, which needs a per-item cost
+estimate. That estimate comes from the run that just finished, so no separate
+measurement pass is required: the scheduler writes ``timings.tsv`` as it
+dispatches, and every run refines the table that the next one reads.
+
+Inputs and outputs:
+
+  TIMINGS     ``timings.tsv`` written by the scheduler. Per-item wall clock, so
+              it includes process startup and HIP init -- which is what the
+              makespan is actually made of, and what a test-time-only source
+              such as JUnit XML would understate several-fold.
+  --items     ``items.tsv``, also from the scheduler: every work item that
+              exists at this TEST_LEVEL, whether or not this host ran it.
+  -o TABLE    the weight table, read and rewritten in place. Lines are
+              ``<label>/<tag> <seconds>``, plus ``<label> <seconds>`` for opaque
+              whole-suite items.
+
+Keep one table per (GPU arch, TEST_LEVEL). Skip patterns differ between gfx942
+and gfx950, and the level changes both which items run and how long several of
+them take (tests/jax reads NVTE_JAX_UNITTEST_LEVEL, test_grouped_gemm.py reads
+TEST_LEVEL), so that is the coarsest keying under which a weight still means one
+thing. Holding them apart is the caller's job -- the workflow puts arch and level
+in the file name.
+
+Each run is blended into the table rather than replacing it, so no single noisy
+sample can move a weight far; see ``merge_weights`` for why the blend is
+asymmetric. Measurements that were cut short are dropped instead of blended
+(``read_timings``), and items that no longer exist are dropped from the table
+(``prune_weights``).
 """
 
 import argparse
-import os
 import sys
-import xml.etree.ElementTree as ET
 from collections import defaultdict
 
-
-def from_junit(root_dir):
-    """Sum testcase durations for every <label>/<tag>.xml under root_dir."""
-    weights = defaultdict(float)
-    for label in sorted(os.listdir(root_dir)):
-        subdir = os.path.join(root_dir, label)
-        if not os.path.isdir(subdir):
-            continue
-        for name in sorted(os.listdir(subdir)):
-            if not name.endswith(".xml"):
-                continue
-            tag = name[: -len(".xml")]
-            path = os.path.join(subdir, name)
-            try:
-                tree = ET.parse(path)
-            except ET.ParseError as err:
-                # A truncated report means the run was cut off mid-item. Skipping
-                # it keeps the weight unknown, which sorts the item first next
-                # time -- the safe direction for something that may be slow.
-                print(f"warning: skipping unparsable {path}: {err}", file=sys.stderr)
-                continue
-            total = 0.0
-            for case in tree.getroot().iter("testcase"):
-                try:
-                    total += float(case.get("time") or 0.0)
-                except ValueError:
-                    pass
-            weights[f"{label}/{tag}"] += total
-    return weights
-
-
 # rc values that mean the item was killed rather than measured. 124 is a GNU
 # timeout expiry (the job's own step timeout, for instance), 137 a SIGKILL or an
 # OOM-kill. Either way the recorded duration is a ceiling imposed from outside,
@@ -73,26 +48,81 @@ def from_junit(root_dir):
 KILLED_RCS = frozenset(("124", "137"))
 
 
-def from_timing(path):
-    """Read the scheduler's timings.tsv: label, tag, gpu, seconds, rc."""
-    weights = defaultdict(float)
+def read_timings(path):
+    """Read the scheduler's timings.tsv.
+
+    Columns are label, tag, gpu, secs, rc, start_off, end_off, est, incomplete;
+    only the first five and the last are read here.
+
+    Returns ``(measured, dispatched)``. ``measured`` maps key -> seconds and
+    holds only the rows worth learning from; ``dispatched`` is every key the run
+    started, which is a weaker but separate fact -- an item can have run and
+    still have produced no usable number, and only the second set answers "does
+    this item still exist" (see ``prune_weights``).
+
+    Two kinds of row are excluded from ``measured``, for the same reason: the
+    duration records where the item was stopped, not what it costs. An rc in
+    ``KILLED_RCS`` says so outright. The ``incomplete`` flag covers what rc
+    cannot -- a ``--timeout-method=thread`` expiry exits 1, indistinguishable
+    from an ordinary test failure (which is a perfectly good measurement), and a
+    segfault exits 139. The scheduler sets that flag from the result sidecar the
+    dying process left behind, which is the only reliable witness.
+    """
+    measured = defaultdict(float)
+    dispatched = set()
     with open(path, encoding="utf-8") as handle:
         for line in handle:
             fields = line.rstrip("\n").split("\t")
             if len(fields) < 4:
                 continue
             label, tag, _gpu, secs = fields[0], fields[1], fields[2], fields[3]
+            # "whole" is the scheduler's placeholder for an opaque suite item,
+            # which the queue keys by bare label.
+            key = label if tag == "whole" else f"{label}/{tag}"
+            dispatched.add(key)
             if len(fields) > 4 and fields[4] in KILLED_RCS:
                 continue
+            if len(fields) > 8 and fields[8] == "1":
+                continue
             try:
                 value = float(secs)
             except ValueError:
                 continue
-            # "whole" is the scheduler's placeholder for an opaque suite item,
-            # which the queue keys by bare label.
-            key = label if tag == "whole" else f"{label}/{tag}"
-            weights[key] += value
-    return weights
+            measured[key] += value
+    return measured, dispatched
+
+
+def read_items(path):
+    """Read items.tsv -- every work item that exists at this TEST_LEVEL.
+
+    The file is ``<label>\\t<tag>`` per line, with an empty tag for an opaque
+    whole-suite item. Returns ``(keys, labels)``: the item keys, and the set of
+    suite labels the file speaks for.
+
+    Both are needed because pruning has to be per label. A suite can be absent
+    from the file entirely -- it was filtered out of the run, or its expansion
+    came back untrustworthy -- and that absence must not be read as "none of its
+    items exist any more".
+
+    Returns ``(None, None)`` when the file cannot be read, which disables
+    pruning. That is the conservative direction: a weight kept for a test that
+    is gone wastes one line, while a weight dropped for a live test makes the
+    scheduler treat it as unknown and mis-order a whole run.
+    """
+    try:
+        handle = open(path, encoding="utf-8")
+    except OSError as err:
+        print(f"warning: cannot read {path} ({err}); nothing will be pruned", file=sys.stderr)
+        return None, None
+    keys, labels = set(), set()
+    with handle:
+        for line in handle:
+            label, _, tag = line.rstrip("\n").partition("\t")
+            if not label:
+                continue
+            labels.add(label)
+            keys.add(f"{label}/{tag}" if tag else label)
+    return keys, labels
 
 
 def read_weights(path):
@@ -138,54 +168,83 @@ def merge_weights(old, new, alpha_up=0.5, alpha_down=0.1):
     return merged
 
 
+def prune_weights(weights, dispatched, live_keys, live_labels):
+    """Drop the weights of work items that no longer exist.
+
+    A key survives if any of these holds:
+
+      * the run dispatched it -- then it plainly exists, whatever else says, and
+        that holds even when it was killed and so taught the table nothing;
+      * its suite is not covered by the item list -- nothing is known about it;
+      * the item list contains it.
+
+    What is left is a key whose suite was listed in full and which that listing
+    did not contain: a test that was deleted, or renamed. Renames need no
+    handling at this end -- the new name simply arrives as an unknown key, which
+    the scheduler already sorts first.
+
+    Returns ``(kept, pruned)``.
+    """
+    kept, pruned = {}, []
+    for key, value in weights.items():
+        label = key.split("/", 1)[0]
+        if key in dispatched or label not in live_labels or key in live_keys:
+            kept[key] = value
+        else:
+            pruned.append(key)
+    return kept, pruned
+
+
 def main():
     parser = argparse.ArgumentParser(
         description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter
     )
-    source = parser.add_mutually_exclusive_group(required=True)
-    source.add_argument(
-        "--from-junit", metavar="DIR", help="directory of per-suite JUnit XML subdirectories"
+    parser.add_argument(
+        "timings", metavar="TIMINGS", help="timings.tsv written by run_queue_sgpu.sh"
     )
-    source.add_argument(
-        "--from-timing", metavar="FILE", help="timings.tsv written by run_queue_sgpu.sh"
+    parser.add_argument(
+        "-o", "--output", required=True, metavar="TABLE", help="weight table to update in place"
     )
-    parser.add_argument("-o", "--output", default="-", help="output file (default: stdout)")
     parser.add_argument(
-        "--merge",
-        action="store_true",
-        help="blend into the existing --output table instead of replacing it",
+        "--items",
+        metavar="FILE",
+        help="items.tsv listing every work item that exists at this level; "
+        "without it, no weight is ever pruned",
     )
     args = parser.parse_args()
 
-    if args.from_junit:
-        weights = from_junit(args.from_junit)
-    else:
-        weights = from_timing(args.from_timing)
-
-    if not weights:
-        print("error: no weights produced from the given source", file=sys.stderr)
+    measured, dispatched = read_timings(args.timings)
+    if not measured:
+        # Every row was killed, cut short or malformed. There is nothing to
+        # learn and the existing table is still the best estimate available, so
+        # leave it exactly as it is rather than rewriting it from nothing.
+        print(f"error: no usable measurements in {args.timings}", file=sys.stderr)
         return 1
 
-    if args.merge:
-        if args.output == "-":
-            parser.error("--merge needs -o/--output (the table to merge into)")
-        previous = read_weights(args.output)
-        weights = merge_weights(previous, weights)
-        print(
-            f"merged {len(weights)} weights "
-            f"({len(previous)} known, {len(set(weights) - set(previous))} new)",
-            file=sys.stderr,
-        )
+    previous = read_weights(args.output)
+    weights = merge_weights(previous, measured)
+    print(
+        f"merged {len(measured)} measurements into {len(previous)} known weights "
+        f"({len(set(weights) - set(previous))} new)",
+        file=sys.stderr,
+    )
+
+    if args.items:
+        live_keys, live_labels = read_items(args.items)
+        if live_keys is not None:
+            weights, pruned = prune_weights(weights, dispatched, live_keys, live_labels)
+            # Named individually rather than counted: a prune is how a deleted
+            # test leaves the table, and it is also exactly what a broken suite
+            # expansion would look like, so the list is worth reading.
+            for key in sorted(pruned):
+                print(f"pruned {key}: its suite no longer lists it", file=sys.stderr)
 
     lines = [
         f"{key} {value:.1f}\n" for key, value in sorted(weights.items(), key=lambda kv: -kv[1])
     ]
-    if args.output == "-":
-        sys.stdout.writelines(lines)
-    else:
-        with open(args.output, "w", encoding="utf-8") as handle:
-            handle.writelines(lines)
-        print(f"wrote {len(lines)} weights to {args.output}", file=sys.stderr)
+    with open(args.output, "w", encoding="utf-8") as handle:
+        handle.writelines(lines)
+    print(f"wrote {len(lines)} weights to {args.output}", file=sys.stderr)
     return 0
 
 
diff --git a/ci/pytorch.sh b/ci/pytorch.sh
index 4bd0627c27..6c5e000088 100755
--- a/ci/pytorch.sh
+++ b/ci/pytorch.sh
@@ -40,6 +40,7 @@ run_default_fa_lbl() {
 
 check_mxfp8_supported() {
     #Guard MXFP8-only test filters, which collect no tests on unsupported archs
+    check_list_all && return 0
     _result=$(NVTE_ROCM_ENABLE_MXFP8=1 python -c "${PYTHON_TE_IMPORT}; from transformer_engine.pytorch.quantization import is_mxfp8_available; print(is_mxfp8_available())" 2>/dev/null)
     if [ "$_result" = "True" ]; then
         return 0
@@ -173,6 +174,7 @@ if [ -n "$SINGLE_CONFIG" ]; then
 fi
 
 check_flash_attn_installed() {
+    check_list_all && return 0
     _result=$(python -c "${PYTHON_TE_IMPORT}; from transformer_engine.pytorch.attention.dot_product_attention.utils import FlashAttentionUtils; print(FlashAttentionUtils.is_installed)" 2>/dev/null)
     if [ "$_result" = "True" ]; then
         return 0

From 0f812af5b8d3313d4515d059f0c45514803effff Mon Sep 17 00:00:00 2001
From: Veera Rajasekhar <VeeraRajasekharReddy.Gopu@amd.com>
Date: Thu, 6 Aug 2026 06:01:04 +0000
Subject: [PATCH 6/9] Refactored the code

---
 .github/scripts/run_queue_sgpu.sh | 441 +++++++++---------------------
 .github/workflows/rocm-ci.yml     |  51 ++--
 2 files changed, 151 insertions(+), 341 deletions(-)

diff --git a/.github/scripts/run_queue_sgpu.sh b/.github/scripts/run_queue_sgpu.sh
index 4b6c08218e..b9beca5dde 100755
--- a/.github/scripts/run_queue_sgpu.sh
+++ b/.github/scripts/run_queue_sgpu.sh
@@ -5,25 +5,22 @@
 #
 # Run all sGPU test suites as one global work queue across N GPUs.
 #
-# Usage: run_queue_sgpu.sh --arch <gfx> [options] [config]...
-#   -a, --arch <gfx>      GPU arch these timings belong to, e.g. gfx942. Required
-#                         (or TE_CI_ARCH); it keys the learned weight table
-#   -l, --log-dir <dir>   where the per-phase log tree is written
-#                         (default test-results/logs)
-#       --only <regex>    keep only items whose "<label>/<tag>" matches (smoke
-#                         tests, or re-running just the items that failed)
-#   -h, --help            this text
+# Usage: run_queue_sgpu.sh [-l|--log-dir <dir>] [<config>...]
 #
-# With no config the full sGPU set (ci_sgpu_queue.conf) is run. TEST_LEVEL and
-# the other ci/_utils.sh variables are read from the environment as usual, so a
-# local run inside the dev container is just:
+# Config format (one suite per line; # comments and blank lines are ignored):
+#   <label>  <logfile>  <mode>  <command> [args...]
+#     mode=list    expanded into one work item per test invocation
+#     mode=opaque  scheduled as a single work item
 #
-#   TEST_LEVEL=1 .github/scripts/run_queue_sgpu.sh --arch gfx942
+# With no config given, .github/scripts/ci_sgpu_queue.conf is used. Unlike
+# run_parallel_sgpu.sh, line order carries no GPU assignment: every item from
+# every config goes into one queue and workers pull on completion.
 #
-# The queue uses every GPU it can see. To restrict what it can see the same way you would for any other
-# ROCm program
+# The queue uses every GPU it can see; restrict it with HIP_VISIBLE_DEVICES.
 #
-#   HIP_VISIBLE_DEVICES=0,1 .github/scripts/run_queue_sgpu.sh --arch gfx942
+# Example usage:
+#   TEST_LEVEL=1 .github/scripts/run_queue_sgpu.sh
+#   HIP_VISIBLE_DEVICES=0,1 TEST_LEVEL=1 .github/scripts/run_queue_sgpu.sh
 #
 set -u
 
@@ -32,18 +29,11 @@ REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)"
 
 # Two directories in the repo root:
 #
-#   test-results/  everything one run produced. This script adds logs/ under it:
-#                  the per-phase log tree
-#   ci-weights/    the learned weight table, which is the one thing that must
-#                  outlive the run that measured it. Same name the workflow uses
-#                  for the cached copy.
+#   test-results/  everything one run produced; this script adds logs/ under it
+#   ci-weights/    the learned weight table, the one thing that must outlive the
+#                  run that measured it. Same name the workflow caches.
 LOG_DIR=${LOG_DIR:-${REPO_ROOT}/test-results/logs}
-ARCH=${TE_CI_ARCH:-}
-ONLY_RE=""
-REPORT_TITLE="sGPU queue schedule"
 
-# ::error:: and ::warning:: are GitHub Actions annotations, and this script also
-# runs by hand in the dev container -- emit them only where they mean something.
 if [[ -n "${GITHUB_ACTIONS:-}" ]]; then
     log_error() { echo "::error::$*" >&2; }
     log_warn()  { echo "::warning::$*" >&2; }
@@ -52,8 +42,6 @@ else
     log_warn()  { echo "Warning: $*" >&2; }
 fi
 
-# The header comment is the usage text; reprinting it keeps the two in sync.
-usage() { awk 'NR > 5 && /^#/ {sub(/^# ?/, ""); print; next} NR > 5 {exit}' "$0"; }
 # Items with no recorded weight sort first: an unknown item is more likely to be
 # a new (or newly slow) one, and a long item started late is what stretches the
 # tail. Losing the gamble costs far less than mis-scheduling a genuinely big item.
@@ -62,29 +50,38 @@ usage() { awk 'NR > 5 && /^#/ {sub(/^# ?/, ""); print; next} NR > 5 {exit}' "$0"
 # unordered. That costs makespan once; the table it writes fixes the next run.
 DEFAULT_WEIGHT=${TE_CI_DEFAULT_WEIGHT:-999999}
 
+# ---------------------------------------------------------------------------
+# Parse arguments
 while [[ $# -gt 0 ]]; do
     case "$1" in
-        -a|--arch)       ARCH="$2"; shift 2 ;;
-        --arch=*)        ARCH="${1#*=}"; shift ;;
-        -l|--log-dir)    LOG_DIR="$2"; shift 2 ;;
-        --log-dir=*)     LOG_DIR="${1#*=}"; shift ;;
-        --only)          ONLY_RE="$2"; shift 2 ;;
-        --only=*)        ONLY_RE="${1#*=}"; shift ;;
-        -h|--help)       usage; exit 0 ;;
-        -*)              echo "Unknown option: $1" >&2; usage >&2; exit 1 ;;
-        *)               break ;;
+        -l|--log-dir)
+            LOG_DIR="$2"; shift 2 ;;
+        --log-dir=*)
+            LOG_DIR="${1#*=}"; shift ;;
+        -*)
+            echo "Unknown option: $1" >&2
+            echo "Usage: $0 [-l|--log-dir <dir>] [<config>...]" >&2
+            exit 1 ;;
+        *)
+            break ;;
     esac
 done
 
-if [[ -z "$ARCH" ]]; then
-    log_error "--arch is required (e.g. --arch gfx942), or set TE_CI_ARCH"
-    exit 1
-fi
-
-if [[ $# -eq 0 ]]; then
-    set -- "${SCRIPT_DIR}/ci_sgpu_queue.conf"
-    echo "No config given; using $1"
+# Resolve config paths to absolute against the caller's CWD *before* changing
+# directory, so relative paths passed by the caller remain valid: the configs
+# are not read until Phase 1, long after the cd to the repo root below.
+declare -a CONFIGS=()
+if [[ $# -gt 0 ]]; then
+    for c in "$@"; do CONFIGS+=( "$(realpath -m "$c")" ); done
+else
+    CONFIGS=( "${SCRIPT_DIR}/ci_sgpu_queue.conf" )
 fi
+for c in "${CONFIGS[@]}"; do
+    if [[ ! -f "$c" ]]; then
+        log_error "suite list not found: $c"
+        exit 1
+    fi
+done
 
 # The pool is every GPU this run can see: HIP_VISIBLE_DEVICES if set
 # else what rocminfo counts.
@@ -103,6 +100,10 @@ detect_gpu_pool() {
     GPU_SOURCE="rocminfo${ROCR_VISIBLE_DEVICES:+, ROCR_VISIBLE_DEVICES=$ROCR_VISIBLE_DEVICES}"
 }
 
+detect_arch() {
+    rocminfo 2>/dev/null | grep -E "^ *Name: *gfx" | head -1 | sed "s/.*gfx/gfx/;s/[: ].*//"
+}
+
 if ! detect_gpu_pool; then
     log_error "no GPU found: rocminfo reports none and HIP_VISIBLE_DEVICES is unset." \
               "Set HIP_VISIBLE_DEVICES to the devices this run may use."
@@ -112,6 +113,13 @@ fi
 NUM_GPUS=${#GPU_IDS[@]}
 echo "== GPUs: ${NUM_GPUS} visible -- ids ${GPU_IDS[*]} (via ${GPU_SOURCE}) =="
 
+ARCH=$(detect_arch)
+if [[ -z "$ARCH" ]]; then
+    log_error "could not read the GPU arch from rocminfo; it keys the weight table"
+    exit 1
+fi
+echo "== Arch: ${ARCH} =="
+
 # The weight table is keyed by arch and TEST_LEVEL -- the coarsest keying under
 # which a weight still means one thing. Read below to order the queue and
 # rewritten in Phase 7, so a first local run is unordered and every run after it
@@ -119,8 +127,6 @@ echo "== GPUs: ${NUM_GPUS} visible -- ids ${GPU_IDS[*]} (via ${GPU_SOURCE}) =="
 WEIGHTS_FILE="${REPO_ROOT}/ci-weights/test_weights.${ARCH}.l${TEST_LEVEL:-99}.txt"
 mkdir -p "$(dirname "$WEIGHTS_FILE")" 2>/dev/null
 
-resolved_configs=()
-for c in "$@"; do resolved_configs+=( "$(realpath -m "$c")" ); done
 [[ "$LOG_DIR" != /* ]] && LOG_DIR="$(realpath -m "$LOG_DIR")"
 
 
@@ -186,54 +192,67 @@ LOCK_FILE="$QUEUE_DIR/queue.lock"
 # what ci/build_weights.py prunes on.
 echo "== Expanding suites into work items =="
 declare -a SUITE_LABELS SUITE_LOGFILES
-while IFS= read -r line; do
-    [[ "$line" =~ ^[[:space:]]*# ]] && continue
-    [[ -z "${line//[[:space:]]/}" ]] && continue
-    read -r label logfile mode cmd rest <<< "$line"
-    SUITE_LABELS+=( "$label" )
-    SUITE_LOGFILES+=( "$logfile" )
-    if [[ "$mode" == "list" ]]; then
-        # The tags themselves are not logged here -- they go straight into
-        # queue.tsv and items.tsv, which is where anything reading them looks.
-        # What is left is stderr, banner-separated by suite and pass: the suite
-        # scripts report a declined capability probe on it, which is noise on a
-        # good run but the only explanation of a list that came back short.
-        echo "=== ${label}: list -- what this runner will run ===" >> "$EXPAND_LOG"
-        mapfile -t tags < <(TE_CI_LIST_ONLY=1 "$cmd" ${rest:-} \
-                            2>> "$EXPAND_LOG" \
-                            | sed -n 's/^TE_CI_ITEM //p')
-        if [[ ${#tags[@]} -eq 0 ]]; then
-            log_error "suite '${label}' (${cmd}) produced no work items"
-            tail -20 "$EXPAND_LOG" >&2
-            exit 1
-        fi
-        for tag in "${tags[@]}"; do
-            printf '%s\t%s\t%s\t%s\n' "$label" "$cmd" "$tag" "${rest:-}" >> "$QUEUE_FILE.raw"
-        done
-        echo "=== ${label}: list-all -- what exists at this level ===" >> "$EXPAND_LOG"
-        mapfile -t all_tags < <(TE_CI_LIST_ONLY=1 TE_CI_LIST_ALL=1 "$cmd" ${rest:-} \
+for config in "${CONFIGS[@]}"; do
+    while IFS= read -r line || [[ -n "$line" ]]; do
+        [[ "$line" =~ ^[[:space:]]*# ]] && continue
+        [[ -z "${line//[[:space:]]/}" ]] && continue
+        read -r label logfile mode cmd rest <<< "$line"
+        SUITE_LABELS+=( "$label" )
+        SUITE_LOGFILES+=( "$logfile" )
+        if [[ "$mode" == "list" ]]; then
+            # The tags themselves are not logged here -- they go straight into
+            # queue.tsv and items.tsv, which is where anything reading them
+            # looks. What is left is stderr, banner-separated by suite and pass:
+            # the suite scripts report a declined capability probe on it, which
+            # is noise on a good run but the only explanation of a list that
+            # came back short.
+            echo "=== ${label}: list -- what this runner will run ===" >> "$EXPAND_LOG"
+            mapfile -t tags < <(TE_CI_LIST_ONLY=1 "$cmd" ${rest:-} \
                                 2>> "$EXPAND_LOG" \
                                 | sed -n 's/^TE_CI_ITEM //p')
-        # The wider list must be a superset. If it is not, it cannot be trusted
-        # as "everything that exists", so the label is left out of items.tsv --
-        # which is exactly what tells build_weights.py to prune nothing for it.
-        # Warn rather than fail: this only degrades bookkeeping, and refusing to
-        # run the tests over it would be a much worse trade.
-        if [[ ${#all_tags[@]} -lt ${#tags[@]} ]]; then
-            log_warn "suite '${label}': list-all returned ${#all_tags[@]} items," \
-                     "fewer than the ${#tags[@]} queued; its weights will not be pruned"
-        else
-            for tag in "${all_tags[@]}"; do
-                printf '%s\t%s\n' "$label" "$tag" >> "$ITEMS_FILE"
+            if [[ ${#tags[@]} -eq 0 ]]; then
+                log_error "suite '${label}' (${cmd}) produced no work items"
+                tail -20 "$EXPAND_LOG" >&2
+                exit 1
+            fi
+            for tag in "${tags[@]}"; do
+                printf '%s\t%s\t%s\t%s\n' "$label" "$cmd" "$tag" "${rest:-}" >> "$QUEUE_FILE.raw"
             done
+            echo "=== ${label}: list-all -- what exists at this level ===" >> "$EXPAND_LOG"
+            mapfile -t all_tags < <(TE_CI_LIST_ONLY=1 TE_CI_LIST_ALL=1 "$cmd" ${rest:-} \
+                                    2>> "$EXPAND_LOG" \
+                                    | sed -n 's/^TE_CI_ITEM //p')
+            # The wider list must be a superset. If it is not, it cannot be
+            # trusted as "everything that exists", so the label is left out of
+            # items.tsv -- which is exactly what tells build_weights.py to prune
+            # nothing for it. Warn rather than fail: this only degrades
+            # bookkeeping, and refusing to run the tests would be a worse trade.
+            if [[ ${#all_tags[@]} -lt ${#tags[@]} ]]; then
+                log_warn "suite '${label}': list-all returned ${#all_tags[@]} items," \
+                         "fewer than the ${#tags[@]} queued; its weights will not be pruned"
+            else
+                for tag in "${all_tags[@]}"; do
+                    printf '%s\t%s\n' "$label" "$tag" >> "$ITEMS_FILE"
+                done
+            fi
+            echo "  ${label}: ${#tags[@]} items (${#all_tags[@]} exist at this level)"
+        else
+            printf '%s\t%s\t%s\t%s\n' "$label" "$cmd" "" "${rest:-}" >> "$QUEUE_FILE.raw"
+            printf '%s\t%s\n' "$label" "" >> "$ITEMS_FILE"
+            echo "  ${label}: 1 item (opaque)"
         fi
-        echo "  ${label}: ${#tags[@]} items (${#all_tags[@]} exist at this level)"
-    else
-        printf '%s\t%s\t%s\t%s\n' "$label" "$cmd" "" "${rest:-}" >> "$QUEUE_FILE.raw"
-        printf '%s\t%s\n' "$label" "" >> "$ITEMS_FILE"
-        echo "  ${label}: 1 item (opaque)"
-    fi
-done < <(cat "${resolved_configs[@]}")
+    done < "$config"
+done
+
+# A label repeated across configs would install its prerequisites twice in
+# Phase 3 and have Phase 5 roll the same item logs into the same suite log
+# twice. The tag check below cannot catch it: an opaque suite has no tag.
+dupe_labels=$(printf '%s\n' "${SUITE_LABELS[@]}" | sort | uniq -d)
+if [[ -n "$dupe_labels" ]]; then
+    log_error "duplicate suite labels across configs:"
+    echo "$dupe_labels" >&2
+    exit 1
+fi
 
 # Duplicate tags within a suite would collide on both the JUnit XML filename and
 # the TEST_FILTER dispatch (one invocation would run both). Fail loudly instead.
@@ -246,7 +265,7 @@ fi
 
 # ---------------------------------------------------------------------------
 # Phase 2: weight and order the queue (longest processing time first)
-awk -v wf="$WEIGHTS_FILE" -v dw="$DEFAULT_WEIGHT" -v only="$ONLY_RE" '
+awk -v wf="$WEIGHTS_FILE" -v dw="$DEFAULT_WEIGHT" '
     BEGIN {
         FS = OFS = "\t"
         if (wf != "" && (getline line < wf) >= 0) {
@@ -259,7 +278,6 @@ awk -v wf="$WEIGHTS_FILE" -v dw="$DEFAULT_WEIGHT" -v only="$ONLY_RE" '
     }
     {
         key = $3 == "" ? $1 : $1 "/" $3
-        if (only != "" && key !~ only) next
         # Truncate to an integer: the weight is also used in shell arithmetic to
         # derive the per-item deadline, and bash cannot parse "675.0".
         print (key in w ? int(w[key]) : dw), $0
@@ -268,7 +286,7 @@ awk -v wf="$WEIGHTS_FILE" -v dw="$DEFAULT_WEIGHT" -v only="$ONLY_RE" '
 rm -f "$QUEUE_FILE.raw"
 
 if [[ ! -s "$QUEUE_FILE" ]]; then
-    log_error "queue is empty${ONLY_RE:+ (no item matched --only '${ONLY_RE}')}"
+    log_error "queue is empty: every suite expanded to nothing"
     exit 1
 fi
 
@@ -418,8 +436,8 @@ done
 wait
 
 WALL=$(( $(date +%s) - START_TS ))
-# The report divides by WALL in several places; a sub-second run (a --only of one
-# trivial item) would otherwise abort the whole report on a division by zero.
+# The report divides by WALL in several places; a run short enough to round to
+# zero seconds would otherwise abort the whole report on a division by zero.
 [[ $WALL -lt 1 ]] && WALL=1
 
 # ---------------------------------------------------------------------------
@@ -436,8 +454,9 @@ WALL=$(( $(date +%s) - START_TS ))
 OVERALL_RC=0
 for i in "${!SUITE_LABELS[@]}"; do
     label="${SUITE_LABELS[$i]}"
-    # A suite with no items in the queue (filtered out by --only) must not get a
-    # suite log and rc=0 -- that would read as "passed" to downstream consumers.
+    # Phase 1 fails a suite that expands to nothing, so this should never skip.
+    # It stays because the alternative to skipping is an empty suite log and
+    # rc=0, which reads as "passed" to the workflow's gate.
     awk -F'\t' -v l="$label" '$2==l {found=1} END {exit !found}' "$QUEUE_FILE" || continue
     suite_log="$SUITE_LOG_DIR/${SUITE_LOGFILES[$i]}"
     : > "$suite_log"
@@ -456,227 +475,19 @@ done
 
 # ---------------------------------------------------------------------------
 # Phase 6: scheduling report
-#
-# timings.tsv columns: label, tag, gpu, secs, rc, start_off, end_off, est,
-# incomplete.
-# Everything below is derived from that one file, so the same report can be
-# regenerated after the fact from an uploaded artifact.
-#
-# Each section is a rows_* function emitting TSV whose first line is the header,
-# and one of two renderers turns that into either an aligned plain-text table
-# (job log + schedule.txt) or a GitHub-flavoured Markdown one (schedule.md, which
-# is appended to the job summary alongside the junit_report.py sections). Keeping
-# the data and the formatting separate is what lets both exist without the awk
-# being written twice.
-
-txt_table() {
-    awk -F'\t' '
-        { rows = NR; if (NF > cols) cols = NF
-          for (i = 1; i <= NF; i++) {
-              cell[NR, i] = $i
-              if (length($i) > w[i]) w[i] = length($i) } }
-        END {
-            for (r = 1; r <= rows; r++) {
-                line = "  "
-                for (i = 1; i <= cols; i++) line = line sprintf("%-" w[i] "s  ", cell[r, i])
-                sub(/ +$/, "", line); print line
-                if (r == 1) {                       # rule under the header row
-                    line = "  "
-                    for (i = 1; i <= cols; i++) {
-                        d = ""; while (length(d) < w[i]) d = d "-"
-                        line = line d "  " }
-                    sub(/ +$/, "", line); print line
-                }
-            }
-        }'
-}
-
-md_table() {
-    awk -F'\t' '
-        { out = ""
-          for (i = 1; i <= NF; i++) { c = $i; gsub(/\|/, "\\|", c); out = out "| " c " " }
-          print out "|"
-          if (NR == 1) { s = "|"; for (i = 1; i <= NF; i++) s = s "---|"; print s } }'
-}
-
-# rc is reported as a word, not a number: the report is read by people, and
-# "killed" vs "fail" is the distinction that changes what you do next. "cut"
-# marks a duration that is where the item was stopped rather than what it costs,
-# which is also why the next run's weight table ignores that row.
-rows_schedule() {
-    printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\n' \
-           GPU Start Duration Result Estimate "% change" "Test name"
-    sort -t$'\t' -k3,3n -k6,6n "$TIMINGS_FILE" \
-        | awk -F'\t' -v dw="$DEFAULT_WEIGHT" -v OFS='\t' '{
-            est = ($8 == dw ? "unknown" : $8 "s")
-            # The % change is the scheduler feedback loop made visible: a large
-            # positive miss is an item that should have been dispatched earlier.
-            chg = ($8 > 0 && $8 != dw) ? sprintf("%+.0f%%", ($4 - $8) * 100 / $8) : "n/a"
-            res = ($5 == 0 ? "pass" : ($5 == 1 ? "fail" : \
-                  ($5 == 124 || $5 == 137 ? "killed" : "error")))
-            if ($9 == 1) res = res " (cut)"
-            print "gpu" $3, "t+" $6 "s", $4 "s", res, est, chg, $1 "/" $2 }'
-}
-
-# Iterating the assigned ids rather than the keys present keeps the rows ordered
-# and surfaces a GPU that took no work at all -- which would otherwise just
-# vanish. The ids need not be contiguous or start at zero, since they may have
-# come from HIP_VISIBLE_DEVICES.
-rows_gpu() {
-    printf '%s\t%s\t%s\t%s\t%s\t%s\n' GPU Items Busy Idle Util Failed
-    awk -F'\t' -v w="$WALL" -v ids="${GPU_IDS[*]}" -v OFS='\t' '
-        { busy[$3] += $4; n[$3]++; if ($5 != 0) bad[$3]++ }
-        END {
-            ng = split(ids, g, " ")
-            for (i = 1; i <= ng; i++)
-                print "gpu" g[i], n[g[i]] + 0, (busy[g[i]] + 0) "s", (w - busy[g[i]]) "s",
-                      sprintf("%.1f%%", (busy[g[i]] + 0) * 100 / w), bad[g[i]] + 0
-        }' "$TIMINGS_FILE"
-}
-
-rows_efficiency() {
-    printf '%s\t%s\t%s\n' Metric Value Meaning
-    awk -F'\t' -v w="$WALL" -v n="$NUM_GPUS" -v OFS='\t' '
-        { work += $4; if ($4 > big) { big = $4; bigname = $1 "/" $2 } }
-        END {
-            print "total work", sprintf("%.0fs", work),
-                  sprintf("sum of all %d item durations", NR)
-            print "actual run time", sprintf("%.0fs", w),
-                  "wall clock, first item start to last item finish"
-            print "utilisation", sprintf("%.1f%%", work * 100 / (w * n)),
-                  sprintf("share of %ss x %d GPUs actually spent running tests", w, n)
-            # Once one item exceeds the per-GPU average there is no ordering that
-            # finishes sooner than that item, so it becomes the thing to split.
-            print "largest item", sprintf("%.0fs", big),
-                  bigname (big > work / n ? " -- floor: splitting it would now pay" : "")
-        }' "$TIMINGS_FILE"
-}
-
-# 30s is roughly one process startup, and below the ~5% run-to-run noise for
-# anything long enough to matter. Listing smaller misses would bury the real
-# ones -- a healthy table has almost every item within a few seconds.
-rows_weights() {
-    printf '%s\t%s\t%s\t%s\t%s\n' Miss Estimate Actual "% change" "Test name"
-    awk -F'\t' -v dw="$DEFAULT_WEIGHT" '$8 != dw && $8 > 0 {
-            d = $4 - $8; a = (d < 0 ? -d : d)
-            if (a >= 30) printf "%.0f\t%+.0fs\t%ss\t%ss\t%+.0f%%\t%s\n",
-                                a, d, $8, $4, d * 100 / $8, $1 "/" $2 }' "$TIMINGS_FILE" \
-        | sort -k1,1nr | head -15 | cut -f2-
-    awk -F'\t' -v dw="$DEFAULT_WEIGHT" -v OFS='\t' '$8 == dw {
-            print "n/a", "unknown", $4 "s", "n/a", $1 "/" $2 }' "$TIMINGS_FILE"
-}
-
-rows_failures() {
-    printf '%s\t%s\t%s\n' Result Duration "Test name"
-    if awk -F'\t' '$5 != 0 {found = 1} END {exit !found}' "$TIMINGS_FILE"; then
-        awk -F'\t' -v OFS='\t' '$5 != 0 {
-            res = ($5 == 1 ? "fail" : ($5 == 124 || $5 == 137 ? "killed" : "error"))
-            print res, $4 "s", $1 "/" $2 }' "$TIMINGS_FILE"
-    else
-        printf '%s\t%s\t%s\n' - - none
-    fi
-}
-
-# An item that never got a timings.tsv row was never dispatched -- only possible
-# if a worker died outright. Silence here would read as success.
-missing_items() {
-    awk -F'\t' 'NR == FNR {seen[$1 "/" $2] = 1; next}
-                { k = ($4 == "" ? $2 "/whole" : $2 "/" $4); if (!(k in seen)) print k }' \
-        "$TIMINGS_FILE" "$QUEUE_FILE"
-}
-
-RAN_ITEMS=$(wc -l < "$TIMINGS_FILE")
-FAILED_ITEMS=$(awk -F'\t' '$5 != 0' "$TIMINGS_FILE" | wc -l)
-
-report() {
-    echo "=== sGPU queue: ${TOTAL_ITEMS} items, ${NUM_GPUS} GPUs, drained in ${WALL}s ==="
-    echo
-    echo "-- efficiency --"
-    rows_efficiency | txt_table
-    echo
-    echo "-- per-GPU utilisation --"
-    rows_gpu | txt_table
-    echo
-    echo "-- failures --"
-    rows_failures | txt_table
-    echo
-    echo "-- schedule: what ran where, in execution order --"
-    rows_schedule | txt_table
-    echo
-    echo "-- weight accuracy: misses over 30s, which the next run corrects --"
-    rows_weights | txt_table
-    if [[ "$RAN_ITEMS" -ne "$TOTAL_ITEMS" ]]; then
-        echo
-        echo "  !! ${RAN_ITEMS}/${TOTAL_ITEMS} items produced a timing record; the rest never ran:"
-        missing_items | sed 's/^/     /'
-    fi
-}
-
-report_md() {
-    local mark=":white_check_mark:"
-    [[ "$FAILED_ITEMS" -gt 0 || "$RAN_ITEMS" -ne "$TOTAL_ITEMS" ]] && mark=":x:"
-    local util
-    util=$(awk -F'\t' -v w="$WALL" -v n="$NUM_GPUS" \
-               '{work += $4} END {printf "%.1f", work * 100 / (w * n)}' "$TIMINGS_FILE")
-
-    echo "## ${REPORT_TITLE}"
-    echo
-    echo "${mark} **${RAN_ITEMS} items** on ${NUM_GPUS} GPUs -- ${FAILED_ITEMS} failed" \
-         "-- ${WALL}s wall clock at ${util}% GPU utilisation"
-    echo
-    if [[ "$RAN_ITEMS" -ne "$TOTAL_ITEMS" ]]; then
-        echo "> :warning: **Only ${RAN_ITEMS} of ${TOTAL_ITEMS} items produced a timing"
-        echo "> record.** The rest were never dispatched, which means a worker died:"
-        echo
-        missing_items | sed 's/^/> - `/; s/$/`/'
-        echo
-    fi
-    echo "### Efficiency"
-    echo
-    rows_efficiency | md_table
-    echo
-    echo "### Per-GPU utilisation"
-    echo
-    rows_gpu | md_table
-    echo
-    echo "### Failures"
-    echo
-    rows_failures | md_table
-    echo
-    echo "<details><summary>Schedule -- what ran where, in execution order</summary>"
-    echo
-    rows_schedule | md_table
-    echo
-    echo "</details>"
-    echo
-    echo "<details><summary>Weight accuracy -- misses over 30s, which the next run corrects</summary>"
-    echo
-    rows_weights | md_table
-    echo
-    echo "</details>"
-    echo
-}
-
-report | tee "$REPORT_DIR/schedule.txt"
-report_md > "$REPORT_DIR/schedule.md"
+# ---------------------------------------------------------------------------
+if ! python3 "$REPO_ROOT/ci/schedule_report.py" "$LOG_DIR" \
+        --gpus "${GPU_IDS[*]}" \
+        --wall "$WALL" \
+        --default-weight "$DEFAULT_WEIGHT"; then
+    log_warn "could not write the scheduling report"
+fi
 
 # ---------------------------------------------------------------------------
-# Phase 7: fold this run's timings back into the weight table
-#
-# Every run that gets here updates the table, and only runs that get here do --
-# one owner, so the asymmetric blend sees each measurement exactly once. A queue
-# killed outright (the CI step's timeout, say) therefore teaches the table
-# nothing, which is the right way round: its durations are mostly ceilings
-# imposed by the kill, not costs.
-#
-# Unconditional on the test results, though. A red run still measures how long
-# each item took, and build_weights.py drops the individual rows that were killed
-# or cut short.
+# Phase 7: Update the learned weight table for the next run. 
+# ---------------------------------------------------------------------------
 echo
-if ! command -v python3 > /dev/null 2>&1; then
-    log_warn "python3 not found; $WEIGHTS_FILE was not updated" \
-             "and the next run will be unordered again"
-elif ! python3 "$REPO_ROOT/ci/build_weights.py" "$TIMINGS_FILE" \
+if ! python3 "$REPO_ROOT/ci/build_weights.py" "$TIMINGS_FILE" \
         --items "$ITEMS_FILE" -o "$WEIGHTS_FILE"; then
     log_warn "could not update $WEIGHTS_FILE; the next run will use the table as it stands"
 fi
diff --git a/.github/workflows/rocm-ci.yml b/.github/workflows/rocm-ci.yml
index 9d837946a9..b778a68359 100644
--- a/.github/workflows/rocm-ci.yml
+++ b/.github/workflows/rocm-ci.yml
@@ -208,7 +208,6 @@ jobs:
         timeout-minutes: 240
         env:
           HF_TOKEN: ${{ secrets.HF_TOKEN }}
-          ARCH: ${{ matrix.arch_label == 'mi30x' && 'gfx942' || 'gfx950' }}
         run: |
           # The queue clears test-results/ itself and creates one XML subdir per
           # suite, so PyTorch/JAX/Core get independent reports and never collide
@@ -221,9 +220,13 @@ jobs:
           # queue runs in its natural order for one run, and this run's timings
           # create the table. A checked-in copy would go stale silently and is
           # worth less than the one unordered run it saves.
+          #
+          # Listed rather than named: the queue derives the file from the arch it
+          # reads off rocminfo, and deriving it a second time from the matrix
+          # label would be a second thing to keep in step.
           mkdir -p ci-weights
-          W="ci-weights/test_weights.${ARCH}.l${{ env.TEST_LEVEL }}.txt"
-          echo "weights: $W ($(wc -l < "$W" 2>/dev/null || echo 0) items)"
+          wc -l ci-weights/*.txt 2>/dev/null \
+            || echo "no weight table restored; this run will be unordered"
 
           docker exec \
             -e TEST_SGPU=1 \
@@ -233,9 +236,7 @@ jobs:
             -e HF_TOKEN="$HF_TOKEN" \
             -e GITHUB_ACTIONS \
             te-runner bash /workspace/.github/scripts/run_queue_sgpu.sh \
-              --arch "$ARCH" \
-              --log-dir /workspace/test-results/logs \
-              /workspace/.github/scripts/ci_sgpu_queue.conf
+              --log-dir /workspace/test-results/logs
 
       # The queue's own last phase updated ci-weights/ before it exited, including
       # when tests failed -- a red run still measures how long each item took. Only
@@ -284,32 +285,30 @@ jobs:
         if: always()
         run: |
           EXIT_STATUS=0
-          for config in .github/scripts/ci_sgpu_queue.conf; do
-            while IFS= read -r line || [[ -n "$line" ]]; do
-              [[ "$line" =~ ^[[:space:]]*# ]] && continue
-              [[ -z "${line//[[:space:]]/}" ]] && continue
-              read -r label logfile rest <<< "$line"
-              rc=$(cat "test-results/logs/suites/${logfile}.rc" 2>/dev/null || echo "missing")
-              [[ "$rc" == "0" ]] && continue
-              echo "::error::${label} tests FAILED (rc=${rc})."
-              EXIT_STATUS=1
-              for f in test-results/logs/items/"${label}".*.log; do
-                [[ -e "$f" ]] || continue
-                irc=$(cat "${f}.rc" 2>/dev/null || echo 1)
-                [[ "$irc" == "0" ]] && continue
-                echo "::group::[FAILED] $(basename "$f" .log) (rc=${irc})"
-                cat "$f"
-                echo "::endgroup::"
-              done
-            done < "$config"
-          done
+          while IFS= read -r line || [[ -n "$line" ]]; do
+            [[ "$line" =~ ^[[:space:]]*# ]] && continue
+            [[ -z "${line//[[:space:]]/}" ]] && continue
+            read -r label logfile rest <<< "$line"
+            rc=$(cat "test-results/logs/suites/${logfile}.rc" 2>/dev/null || echo "missing")
+            [[ "$rc" == "0" ]] && continue
+            echo "::error::${label} tests FAILED (rc=${rc})."
+            EXIT_STATUS=1
+            for f in test-results/logs/items/"${label}".*.log; do
+              [[ -e "$f" ]] || continue
+              irc=$(cat "${f}.rc" 2>/dev/null || echo 1)
+              [[ "$irc" == "0" ]] && continue
+              echo "::group::[FAILED] $(basename "$f" .log) (rc=${irc})"
+              cat "$f"
+              echo "::endgroup::"
+            done
+          done < .github/scripts/ci_sgpu_queue.conf
           exit $EXIT_STATUS
 
       # test-results/ is the whole run: XML per suite plus one directory per
       # scheduler phase under logs/. A download answers "what did it plan, what
       # did it set up, what did each item print, what did each item cost" from a
       # single tree. logs/queue/ is what makes it reproducible after the fact --
-      # schedule.txt and the weight table can both be regenerated from
+      # schedule.md and the weight table can both be regenerated from
       # timings.tsv + items.tsv alone.
       - name: Upload logs
         if: always()

From 2f91be8a8d3bb2605af62f3dadcde3572dfedf47 Mon Sep 17 00:00:00 2001
From: Veera Rajasekhar <VeeraRajasekharReddy.Gopu@amd.com>
Date: Fri, 7 Aug 2026 05:37:32 +0000
Subject: [PATCH 7/9] Refactored the code

---
 .github/scripts/run_queue_sgpu.sh            | 501 +++++++++----------
 .github/scripts/scheduler/build_weights.py   | 318 ++++++++++++
 .github/scripts/scheduler/queue_files.py     |  61 +++
 .github/scripts/scheduler/schedule_report.py | 285 +++++++++++
 ci/_utils.sh                                 |  56 ++-
 ci/build_weights.py                          | 252 ----------
 ci/jax.sh                                    |   4 +-
 ci/pytorch.sh                                |  29 +-
 8 files changed, 965 insertions(+), 541 deletions(-)
 create mode 100755 .github/scripts/scheduler/build_weights.py
 create mode 100644 .github/scripts/scheduler/queue_files.py
 create mode 100644 .github/scripts/scheduler/schedule_report.py
 delete mode 100755 ci/build_weights.py

diff --git a/.github/scripts/run_queue_sgpu.sh b/.github/scripts/run_queue_sgpu.sh
index b9beca5dde..abacad733c 100755
--- a/.github/scripts/run_queue_sgpu.sh
+++ b/.github/scripts/run_queue_sgpu.sh
@@ -32,7 +32,28 @@ REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)"
 #   test-results/  everything one run produced; this script adds logs/ under it
 #   ci-weights/    the learned weight table, the one thing that must outlive the
 #                  run that measured it. Same name the workflow caches.
-LOG_DIR=${LOG_DIR:-${REPO_ROOT}/test-results/logs}
+: "${LOG_DIR:=${REPO_ROOT}/test-results/logs}"
+
+# Items with no recorded weight sort first: an unknown item is more likely to be
+# a new (or newly slow) one, and a long item started late is what stretches the
+# tail. Losing the gamble costs far less than mis-scheduling a genuinely big item.
+: "${DEFAULT_WEIGHT:=${TE_CI_DEFAULT_WEIGHT:-999999}}"
+
+# Run state:  CONFIGS[]   suite lists to read, in the order given
+#             GPU_IDS[]   the worker pool, and where it was read from
+#             SUITE_*[]   the configs as parsed by Phase 0: one entry per config
+#                         line, in file order, shared index across all five.
+#                         Phases 1, 3 and 5 all walk them rather than re-reading
+#                         the files, so the parse happens once.
+declare -a CONFIGS=()
+declare -a GPU_IDS=()
+declare -a SUITE_LABELS=()
+declare -a SUITE_LOGFILES=()
+declare -a SUITE_MODES=()
+declare -a SUITE_CMDS=()
+declare -a SUITE_ARGS=()
+GPU_SOURCE=""
+OVERALL_RC=0
 
 if [[ -n "${GITHUB_ACTIONS:-}" ]]; then
     log_error() { echo "::error::$*" >&2; }
@@ -42,16 +63,114 @@ else
     log_warn()  { echo "Warning: $*" >&2; }
 fi
 
-# Items with no recorded weight sort first: an unknown item is more likely to be
-# a new (or newly slow) one, and a long item started late is what stretches the
-# tail. Losing the gamble costs far less than mis-scheduling a genuinely big item.
-# With no weights file at all -- the first run on a new arch, or a cache miss --
-# every item takes this, the queue keeps its natural order, and the run is simply
-# unordered. That costs makespan once; the table it writes fixes the next run.
-DEFAULT_WEIGHT=${TE_CI_DEFAULT_WEIGHT:-999999}
+# Fill GPU_IDS with every GPU this run can see: HIP_VISIBLE_DEVICES if set
+# else what rocminfo counts. Returns 1 if there is none.
+detect_gpu_pool() {
+    local n k
+    if [[ -n "${HIP_VISIBLE_DEVICES:-}" ]]; then
+        IFS=',' read -r -a GPU_IDS <<< "$HIP_VISIBLE_DEVICES"
+        GPU_SOURCE="HIP_VISIBLE_DEVICES=$HIP_VISIBLE_DEVICES"
+        return 0
+    fi
+    n=$(rocminfo 2>/dev/null | grep -c 'Device Type:.*GPU')
+    [[ -n "$n" && "$n" -gt 0 ]] || return 1
+    for ((k = 0; k < n; k++)); do GPU_IDS+=( "$k" ); done
+    GPU_SOURCE="rocminfo${ROCR_VISIBLE_DEVICES:+, ROCR_VISIBLE_DEVICES=$ROCR_VISIBLE_DEVICES}"
+}
+
+detect_arch() {
+    rocminfo 2>/dev/null | grep -E "^ *Name: *gfx" | head -1 | sed "s/.*gfx/gfx/;s/[: ].*//"
+}
+
+# Echo the next queue index and advance it. Every worker calls this, so the
+# read-modify-write is done under flock.
+take_next() {
+    local i
+    {
+        flock 9
+        i=$(cat "$IDX_FILE")
+        echo $((i + 1)) > "$IDX_FILE"
+    } 9<>"$LOCK_FILE"
+    echo "$i"
+}
+
+# Usage: worker <gpu>
+# Pull items off the queue until it is empty, running each one on <gpu> and
+# appending its timing record to TIMINGS_FILE.
+worker() {
+    local gpu=$1
+    local i line weight label cmd tag rest itemlog rc safetag junit_dir start end incomplete
+    while :; do
+        i=$(take_next)
+        [[ "$i" -gt "$TOTAL_ITEMS" ]] && break
+        line=$(sed -n "${i}p" "$QUEUE_FILE")
+        [[ -z "$line" ]] && break
+        IFS=$'\t' read -r weight label cmd tag rest <<< "$line"
+
+        safetag="${tag:-whole}"
+        itemlog="$ITEM_LOG_DIR/${label}.${safetag}.log"
+        if [[ -n "${JUNITXML_PREFIX:-}${JUNITXML_SUFFIX:-}" ]]; then
+            junit_dir="${JUNITXML_PREFIX:-}${label}/"
+            mkdir -p "$junit_dir"
+        else
+            junit_dir=""
+        fi
+
+        # No scheduler-imposed deadline: the suite scripts' own PYTEST_TIMEOUT and
+        # the workflow's timeout-minutes are the only limits, exactly as they are
+        # outside the queue.
+        start=$(date +%s)
+        if [[ -n "$tag" ]]; then
+            HIP_VISIBLE_DEVICES=$gpu TE_CI_SKIP_SETUP=1 TEST_FILTER="$tag" \
+                JUNITXML_PREFIX="$junit_dir" \
+                "$cmd" ${rest:-} > "$itemlog" 2>&1
+        else
+            HIP_VISIBLE_DEVICES=$gpu JUNITXML_PREFIX="$junit_dir" \
+                "$cmd" ${rest:-} > "$itemlog" 2>&1
+        fi
+        rc=$?
+        end=$(date +%s)
+
+        echo "$rc" > "${itemlog}.rc"
+
+        # A te_ci_result_sink sidecar that outlived the process means pytest
+        # never reached its end-of-session write -- a --timeout-method=thread
+        # expiry, a segfault, or an OOM-kill -- so this duration is where the
+        # item was cut off, not what it costs. rc cannot be used to tell: a
+        # thread-method timeout exits 1, indistinguishable from an ordinary test
+        # failure, which is a perfectly good measurement. build_weights.py drops
+        # the flagged rows rather than teaching the table a truncated number.
+        incomplete=0
+        if [[ -n "$junit_dir" ]]; then
+            if [[ -n "$tag" ]]; then
+                # Items sharing a label share junit_dir, so only this item's own
+                # sidecar may be consulted -- a glob would see the in-flight
+                # sidecar of an item still running on another GPU.
+                [[ -e "${junit_dir}${tag}${JUNITXML_SUFFIX:-}.partial" ]] && incomplete=1
+            elif compgen -G "${junit_dir}*.partial" > /dev/null; then
+                incomplete=1   # opaque suite: it is the only item in its dir
+            fi
+        fi
+
+        # Machine-readable timing record. The first five columns are the stable
+        # contract build_weights.py reads; the scheduling columns are appended
+        # after them so that reader stays unchanged. Written incrementally so a
+        # run killed by the job timeout still leaves a usable record of
+        # everything that had completed.
+        printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n' \
+            "$label" "$safetag" "$gpu" "$((end - start))" "$rc" \
+            "$((start - START_TS))" "$((end - START_TS))" "$weight" "$incomplete" \
+            >> "$TIMINGS_FILE"
+        printf '[%s] gpu%-2s t+%-7s %5ss rc=%-4s est=%-8s %-9s %s\n' \
+            "$(date '+%H:%M:%S')" "$gpu" "$((start - START_TS))s" "$((end - start))" \
+            "$rc" "$([[ $weight -eq $DEFAULT_WEIGHT ]] && echo unknown || echo "${weight}s")" \
+            "$label" "$safetag"
+    done
+}
 
 # ---------------------------------------------------------------------------
 # Parse arguments
+# ---------------------------------------------------------------------------
 while [[ $# -gt 0 ]]; do
     case "$1" in
         -l|--log-dir)
@@ -68,9 +187,9 @@ while [[ $# -gt 0 ]]; do
 done
 
 # Resolve config paths to absolute against the caller's CWD *before* changing
-# directory, so relative paths passed by the caller remain valid: the configs
-# are not read until Phase 1, long after the cd to the repo root below.
-declare -a CONFIGS=()
+# directory, so relative paths passed by the caller remain valid: Phase 0 reads
+# them below, but the commands they name are not run until after the cd to the
+# repo root.
 if [[ $# -gt 0 ]]; then
     for c in "$@"; do CONFIGS+=( "$(realpath -m "$c")" ); done
 else
@@ -83,27 +202,38 @@ for c in "${CONFIGS[@]}"; do
     fi
 done
 
-# The pool is every GPU this run can see: HIP_VISIBLE_DEVICES if set
-# else what rocminfo counts.
-declare -a GPU_IDS=()
-GPU_SOURCE=""
-detect_gpu_pool() {
-    local n k
-    if [[ -n "${HIP_VISIBLE_DEVICES:-}" ]]; then
-        IFS=',' read -r -a GPU_IDS <<< "$HIP_VISIBLE_DEVICES"
-        GPU_SOURCE="HIP_VISIBLE_DEVICES=$HIP_VISIBLE_DEVICES"
-        return 0
-    fi
-    n=$(rocminfo 2>/dev/null | grep -c 'Device Type:.*GPU')
-    [[ -n "$n" && "$n" -gt 0 ]] || return 1
-    for ((k = 0; k < n; k++)); do GPU_IDS+=( "$k" ); done
-    GPU_SOURCE="rocminfo${ROCR_VISIBLE_DEVICES:+, ROCR_VISIBLE_DEVICES=$ROCR_VISIBLE_DEVICES}"
-}
+# ---------------------------------------------------------------------------
+# Phase 0: read the configs and reject a bad one
+# ---------------------------------------------------------------------------
+for config in "${CONFIGS[@]}"; do
+    while IFS= read -r line || [[ -n "$line" ]]; do
+        [[ "$line" =~ ^[[:space:]]*# ]] && continue
+        [[ -z "${line//[[:space:]]/}" ]] && continue
+        read -r label logfile mode cmd rest <<< "$line"
+        SUITE_LABELS+=( "$label" )
+        SUITE_LOGFILES+=( "$logfile" )
+        SUITE_MODES+=( "$mode" )
+        SUITE_CMDS+=( "$cmd" )
+        SUITE_ARGS+=( "${rest:-}" )
+    done < "$config"
+done
 
-detect_arch() {
-    rocminfo 2>/dev/null | grep -E "^ *Name: *gfx" | head -1 | sed "s/.*gfx/gfx/;s/[: ].*//"
-}
+if [[ ${#SUITE_LABELS[@]} -eq 0 ]]; then
+    log_error "no suites to run: ${CONFIGS[*]} contain only comments and blank lines"
+    exit 1
+fi
+
+# Reject duplicate labels across configs: the queue is keyed by label
+dupe_labels=$(printf '%s\n' "${SUITE_LABELS[@]}" | sort | uniq -d)
+if [[ -n "$dupe_labels" ]]; then
+    log_error "duplicate suite labels across configs:"
+    echo "$dupe_labels" >&2
+    exit 1
+fi
 
+# ---------------------------------------------------------------------------
+# Set up the run: GPU pool, arch, weight table, log tree
+# ---------------------------------------------------------------------------
 if ! detect_gpu_pool; then
     log_error "no GPU found: rocminfo reports none and HIP_VISIBLE_DEVICES is unset." \
               "Set HIP_VISIBLE_DEVICES to the devices this run may use."
@@ -120,10 +250,7 @@ if [[ -z "$ARCH" ]]; then
 fi
 echo "== Arch: ${ARCH} =="
 
-# The weight table is keyed by arch and TEST_LEVEL -- the coarsest keying under
-# which a weight still means one thing. Read below to order the queue and
-# rewritten in Phase 7, so a first local run is unordered and every run after it
-# is not.
+# The weight table is keyed by arch and TEST_LEVEL 
 WEIGHTS_FILE="${REPO_ROOT}/ci-weights/test_weights.${ARCH}.l${TEST_LEVEL:-99}.txt"
 mkdir -p "$(dirname "$WEIGHTS_FILE")" 2>/dev/null
 
@@ -136,7 +263,7 @@ mkdir -p "$(dirname "$WEIGHTS_FILE")" 2>/dev/null
 #   prerequisite_ck_jit_status/ Phase 3  one-time prerequisites per suite
 #   items/                      Phase 4  the test output itself -- one file per item
 #   suites/                     Phase 5  per-suite verdict: rc + index into items/
-#   report/                     Phase 6  the human-readable schedule
+#   report/                     Phase 7  the human-readable schedule
 #   queue/                               the machine-readable state: what to run,
 #                                        what it cost
 #
@@ -155,10 +282,13 @@ rm -rf "$ITEM_LOG_DIR" "$SUITE_LOG_DIR"
 
 mkdir -p "$SETUP_DIR" "$ITEM_LOG_DIR" "$SUITE_LOG_DIR" "$REPORT_DIR" "$QUEUE_DIR"
 : > "$EXPAND_LOG"
+
+# cd to repo root so that commands in configs (e.g. ci/pytorch.sh) resolve correctly
 cd "$REPO_ROOT" || { echo "Error: cannot cd to '${REPO_ROOT}'" >&2; exit 1; }
 
 QUEUE_FILE="$QUEUE_DIR/queue.tsv"
 ITEMS_FILE="$QUEUE_DIR/items.tsv"
+LIST_TMP="$QUEUE_DIR/.expand.tmp"
 TIMINGS_FILE="$QUEUE_DIR/timings.tsv"
 IDX_FILE="$QUEUE_DIR/queue.idx"
 LOCK_FILE="$QUEUE_DIR/queue.lock"
@@ -171,11 +301,11 @@ LOCK_FILE="$QUEUE_DIR/queue.lock"
 
 # ---------------------------------------------------------------------------
 # Phase 1: expand every suite into work items
+# ---------------------------------------------------------------------------
 #
 # List mode runs the suite script with TE_CI_LIST_ONLY=1, which makes pytest_run
 # echo "TE_CI_ITEM <tag>" for each invocation it would have made instead of
-# running it. The suite's own control flow produces the list, so nothing here has
-# to know how a suite decomposes.
+# running it.
 #
 # Every list-mode suite is listed twice, because "will run" and "exists" are
 # different questions:
@@ -183,149 +313,88 @@ LOCK_FILE="$QUEUE_DIR/queue.lock"
 #   run 1  LIST_ONLY             what this host will run     -> the queue
 #   run 2  LIST_ONLY + LIST_ALL  what exists at this level   -> items.tsv
 #
-# Run 1 keeps every gate: level, backend matrix, and the host capability probes.
-# Run 2 answers the capability probes yes without probing (check_list_all in
-# ci/_utils.sh); the level and matrix gates still apply.
-#
 # So run2 - run1 is what this host skipped -- no flash-attn, say -- and those
 # tests do still exist. Only a tag in neither list is gone for good, and that is
-# what ci/build_weights.py prunes on.
-echo "== Expanding suites into work items =="
-declare -a SUITE_LABELS SUITE_LOGFILES
-for config in "${CONFIGS[@]}"; do
-    while IFS= read -r line || [[ -n "$line" ]]; do
-        [[ "$line" =~ ^[[:space:]]*# ]] && continue
-        [[ -z "${line//[[:space:]]/}" ]] && continue
-        read -r label logfile mode cmd rest <<< "$line"
-        SUITE_LABELS+=( "$label" )
-        SUITE_LOGFILES+=( "$logfile" )
-        if [[ "$mode" == "list" ]]; then
-            # The tags themselves are not logged here -- they go straight into
-            # queue.tsv and items.tsv, which is where anything reading them
-            # looks. What is left is stderr, banner-separated by suite and pass:
-            # the suite scripts report a declined capability probe on it, which
-            # is noise on a good run but the only explanation of a list that
-            # came back short.
-            echo "=== ${label}: list -- what this runner will run ===" >> "$EXPAND_LOG"
-            mapfile -t tags < <(TE_CI_LIST_ONLY=1 "$cmd" ${rest:-} \
-                                2>> "$EXPAND_LOG" \
-                                | sed -n 's/^TE_CI_ITEM //p')
-            if [[ ${#tags[@]} -eq 0 ]]; then
-                log_error "suite '${label}' (${cmd}) produced no work items"
-                tail -20 "$EXPAND_LOG" >&2
-                exit 1
-            fi
-            for tag in "${tags[@]}"; do
-                printf '%s\t%s\t%s\t%s\n' "$label" "$cmd" "$tag" "${rest:-}" >> "$QUEUE_FILE.raw"
-            done
-            echo "=== ${label}: list-all -- what exists at this level ===" >> "$EXPAND_LOG"
-            mapfile -t all_tags < <(TE_CI_LIST_ONLY=1 TE_CI_LIST_ALL=1 "$cmd" ${rest:-} \
-                                    2>> "$EXPAND_LOG" \
-                                    | sed -n 's/^TE_CI_ITEM //p')
-            # The wider list must be a superset. If it is not, it cannot be
-            # trusted as "everything that exists", so the label is left out of
-            # items.tsv -- which is exactly what tells build_weights.py to prune
-            # nothing for it. Warn rather than fail: this only degrades
-            # bookkeeping, and refusing to run the tests would be a worse trade.
-            if [[ ${#all_tags[@]} -lt ${#tags[@]} ]]; then
-                log_warn "suite '${label}': list-all returned ${#all_tags[@]} items," \
-                         "fewer than the ${#tags[@]} queued; its weights will not be pruned"
-            else
-                for tag in "${all_tags[@]}"; do
-                    printf '%s\t%s\n' "$label" "$tag" >> "$ITEMS_FILE"
-                done
-            fi
-            echo "  ${label}: ${#tags[@]} items (${#all_tags[@]} exist at this level)"
+# what build_weights.py prunes on.
+echo "== Expanding test suites into work items =="
+for i in "${!SUITE_LABELS[@]}"; do
+    label="${SUITE_LABELS[$i]}"
+    mode="${SUITE_MODES[$i]}"
+    cmd="${SUITE_CMDS[$i]}"
+    rest="${SUITE_ARGS[$i]}"
+    if [[ "$mode" == "list" ]]; then
+        echo "=== ${label}: list -- what this runner will run ===" >> "$EXPAND_LOG"
+        TE_CI_LIST_ONLY=1 "$cmd" ${rest:-} > "$LIST_TMP" 2>> "$EXPAND_LOG"
+        list_rc=$?
+        mapfile -t tags < <(sed -n 's/^TE_CI_ITEM //p' "$LIST_TMP")
+
+        if [[ $list_rc -ne 0 ]]; then
+            log_error "suite '${label}' (${cmd}) reported an error while listing (rc=${list_rc})"
+            tail -20 "$EXPAND_LOG" >&2
+            exit 1
+        fi
+        if [[ ${#tags[@]} -eq 0 ]]; then
+            log_error "suite '${label}' (${cmd}) produced no work items"
+            tail -20 "$EXPAND_LOG" >&2
+            exit 1
+        fi
+        for tag in "${tags[@]}"; do
+            printf '%s\t%s\t%s\t%s\n' "$label" "$cmd" "$tag" "${rest:-}" >> "$QUEUE_FILE.raw"
+        done
+
+        echo "=== ${label}: list-all -- what exists at this level ===" >> "$EXPAND_LOG"
+        TE_CI_LIST_ONLY=1 TE_CI_LIST_ALL=1 "$cmd" ${rest:-} > "$LIST_TMP" 2>> "$EXPAND_LOG"
+        all_rc=$?
+        mapfile -t all_tags < <(sed -n 's/^TE_CI_ITEM //p' "$LIST_TMP")
+
+        if [[ $all_rc -ne 0 ]]; then
+            log_warn "suite '${label}': list-all reported an error (rc=${all_rc}) after" \
+                     "returning ${#all_tags[@]} items; its weights will not be pruned"
         else
-            printf '%s\t%s\t%s\t%s\n' "$label" "$cmd" "" "${rest:-}" >> "$QUEUE_FILE.raw"
-            printf '%s\t%s\n' "$label" "" >> "$ITEMS_FILE"
-            echo "  ${label}: 1 item (opaque)"
+            for tag in "${all_tags[@]}"; do
+                printf '%s\t%s\n' "$label" "$tag" >> "$ITEMS_FILE"
+            done
         fi
-    done < "$config"
+        echo "  ${label}: ${#tags[@]} items (${#all_tags[@]} exist at this level)"
+    else
+        printf '%s\t%s\t%s\t%s\n' "$label" "$cmd" "" "${rest:-}" >> "$QUEUE_FILE.raw"
+        printf '%s\t%s\n' "$label" "" >> "$ITEMS_FILE"
+        echo "  ${label}: 1 item (opaque)"
+    fi
 done
-
-# A label repeated across configs would install its prerequisites twice in
-# Phase 3 and have Phase 5 roll the same item logs into the same suite log
-# twice. The tag check below cannot catch it: an opaque suite has no tag.
-dupe_labels=$(printf '%s\n' "${SUITE_LABELS[@]}" | sort | uniq -d)
-if [[ -n "$dupe_labels" ]]; then
-    log_error "duplicate suite labels across configs:"
-    echo "$dupe_labels" >&2
-    exit 1
-fi
-
-# Duplicate tags within a suite would collide on both the JUnit XML filename and
-# the TEST_FILTER dispatch (one invocation would run both). Fail loudly instead.
-dupes=$(cut -f1,3 "$QUEUE_FILE.raw" | grep -v '^\S*\t$' | sort | uniq -d)
-if [[ -n "$dupes" ]]; then
-    log_error "duplicate work item tags detected:"
-    echo "$dupes" >&2
-    exit 1
-fi
+rm -f "$LIST_TMP"
 
 # ---------------------------------------------------------------------------
 # Phase 2: weight and order the queue (longest processing time first)
-awk -v wf="$WEIGHTS_FILE" -v dw="$DEFAULT_WEIGHT" '
-    BEGIN {
-        FS = OFS = "\t"
-        if (wf != "" && (getline line < wf) >= 0) {
-            do {
-                if (line !~ /^[[:space:]]*#/ && line !~ /^[[:space:]]*$/) {
-                    split(line, a, " "); if (a[1] != "") w[a[1]] = a[2]
-                }
-            } while ((getline line < wf) > 0)
-        }
-    }
-    {
-        key = $3 == "" ? $1 : $1 "/" $3
-        # Truncate to an integer: the weight is also used in shell arithmetic to
-        # derive the per-item deadline, and bash cannot parse "675.0".
-        print (key in w ? int(w[key]) : dw), $0
-    }
-' "$QUEUE_FILE.raw" | sort -t$'\t' -k1,1nr > "$QUEUE_FILE"
-rm -f "$QUEUE_FILE.raw"
-
-if [[ ! -s "$QUEUE_FILE" ]]; then
-    log_error "queue is empty: every suite expanded to nothing"
+# ---------------------------------------------------------------------------
+#
+# build_weights.py owns the weight table at both ends -- it applies the table
+# here and rewrites it in Phase 6.
+if ! python3 "$REPO_ROOT/.github/scripts/scheduler/build_weights.py" order "$QUEUE_FILE.raw" \
+        --weights "$WEIGHTS_FILE" \
+        --output "$QUEUE_FILE" \
+        --default-weight "$DEFAULT_WEIGHT" \
+        --gpus "${GPU_IDS[*]}"; then
+    log_error "could not order the queue"
     exit 1
 fi
+rm -f "$QUEUE_FILE.raw"
 
 TOTAL_ITEMS=$(wc -l < "$QUEUE_FILE")
-echo "== Queue: ${TOTAL_ITEMS} items over ${NUM_GPUS} GPUs (${GPU_IDS[*]}) =="
-if [[ -f "$WEIGHTS_FILE" ]]; then
-    echo "   weights: $WEIGHTS_FILE ($(wc -l < "$WEIGHTS_FILE") items)"
-else
-    echo "   weights: none yet -- this run is unordered and will write $WEIGHTS_FILE"
-fi
-
-# The plan is printed before anything runs: it is the only record of what the
-# scheduler intended, and comparing it against the schedule reported at the end
-# is how a mis-weighted item is spotted.
-echo "== Queue plan (dispatch order; est = weight used to sort) =="
-awk -F'\t' -v dw="$DEFAULT_WEIGHT" -v n="$NUM_GPUS" '
-    { total += $1
-      printf "  %3d. est=%-8s %-9s %s\n", NR, ($1 == dw ? "unknown" : $1 "s"), $2,
-             ($4 == "" ? "(whole suite)" : $4) }
-    END {
-        printf "\n  estimated work %ss over %s GPUs -> lower bound %.0fs", total, n, total / n
-        printf " (largest item %ss)\n", first
-    }
-    NR == 1 { first = ($1 == dw ? "unknown" : $1) }
-' "$QUEUE_FILE"
 
 # ---------------------------------------------------------------------------
 # Phase 3: one-time, container-wide setup
+# ---------------------------------------------------------------------------
 #
 # pip prerequisites and the CK JIT blob cache are container-wide filesystem
 # state, so each suite installs and prebuilds them once here.
 setup_banner=""
 for i in "${!SUITE_LABELS[@]}"; do
     label="${SUITE_LABELS[$i]}"
-    # Columns are: weight, label, cmd, tag, rest. Only list-mode suites (those
-    # with a non-empty tag) have a setup/dispatch split; opaque suites are a
-    # single invocation that still does its own setup.
-    cmd=$(awk -F'\t' -v l="$label" '$2==l && $4!="" {print $3; exit}' "$QUEUE_FILE")
-    [[ -z "$cmd" ]] && continue
+    cmd="${SUITE_CMDS[$i]}"
+    # Only list-mode suites have a setup/dispatch split; an opaque suite is a
+    # single invocation that still does its own setup inline.
+    [[ "${SUITE_MODES[$i]}" == "list" ]] || continue
     if [[ -z "$setup_banner" ]]; then
         echo "== One-time setup: pip prerequisites + CK JIT prebuild (once per suite) =="
         setup_banner=1
@@ -344,92 +413,12 @@ done
 
 # ---------------------------------------------------------------------------
 # Phase 4: run the queue
+# ---------------------------------------------------------------------------
 echo 1 > "$IDX_FILE"
 : > "$LOCK_FILE"
+: > "$TIMINGS_FILE"
 START_TS=$(date +%s)
 
-take_next() {
-    local i
-    {
-        flock 9
-        i=$(cat "$IDX_FILE")
-        echo $((i + 1)) > "$IDX_FILE"
-    } 9<>"$LOCK_FILE"
-    echo "$i"
-}
-
-worker() {
-    local gpu=$1
-    local i line weight label cmd tag rest itemlog rc safetag junit_dir start end incomplete
-    while :; do
-        i=$(take_next)
-        [[ "$i" -gt "$TOTAL_ITEMS" ]] && break
-        line=$(sed -n "${i}p" "$QUEUE_FILE")
-        [[ -z "$line" ]] && break
-        IFS=$'\t' read -r weight label cmd tag rest <<< "$line"
-
-        safetag="${tag:-whole}"
-        itemlog="$ITEM_LOG_DIR/${label}.${safetag}.log"
-        if [[ -n "${JUNITXML_PREFIX:-}${JUNITXML_SUFFIX:-}" ]]; then
-            junit_dir="${JUNITXML_PREFIX:-}${label}/"
-            mkdir -p "$junit_dir"
-        else
-            junit_dir=""
-        fi
-
-        # No scheduler-imposed deadline: the suite scripts' own PYTEST_TIMEOUT and
-        # the workflow's timeout-minutes are the only limits, exactly as they are
-        # outside the queue.
-        start=$(date +%s)
-        if [[ -n "$tag" ]]; then
-            HIP_VISIBLE_DEVICES=$gpu TE_CI_SKIP_SETUP=1 TEST_FILTER="$tag" \
-                JUNITXML_PREFIX="$junit_dir" \
-                "$cmd" ${rest:-} > "$itemlog" 2>&1
-        else
-            HIP_VISIBLE_DEVICES=$gpu JUNITXML_PREFIX="$junit_dir" \
-                "$cmd" ${rest:-} > "$itemlog" 2>&1
-        fi
-        rc=$?
-        end=$(date +%s)
-
-        echo "$rc" > "${itemlog}.rc"
-
-        # A te_ci_result_sink sidecar that outlived the process means pytest
-        # never reached its end-of-session write -- a --timeout-method=thread
-        # expiry, a segfault, or an OOM-kill -- so this duration is where the
-        # item was cut off, not what it costs. rc cannot be used to tell: a
-        # thread-method timeout exits 1, indistinguishable from an ordinary test
-        # failure, which is a perfectly good measurement. build_weights.py drops
-        # the flagged rows rather than teaching the table a truncated number.
-        incomplete=0
-        if [[ -n "$junit_dir" ]]; then
-            if [[ -n "$tag" ]]; then
-                # Items sharing a label share junit_dir, so only this item's own
-                # sidecar may be consulted -- a glob would see the in-flight
-                # sidecar of an item still running on another GPU.
-                [[ -e "${junit_dir}${tag}${JUNITXML_SUFFIX:-}.partial" ]] && incomplete=1
-            elif compgen -G "${junit_dir}*.partial" > /dev/null; then
-                incomplete=1   # opaque suite: it is the only item in its dir
-            fi
-        fi
-
-        # Machine-readable timing record. The first five columns are the stable
-        # contract ci/build_weights.py reads; the scheduling columns are appended
-        # after them so that reader stays unchanged. Written incrementally so a
-        # run killed by the job timeout still leaves a usable record of
-        # everything that had completed.
-        printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n' \
-            "$label" "$safetag" "$gpu" "$((end - start))" "$rc" \
-            "$((start - START_TS))" "$((end - START_TS))" "$weight" "$incomplete" \
-            >> "$TIMINGS_FILE"
-        printf '[%s] gpu%-2s t+%-7s %5ss rc=%-4s est=%-8s %-9s %s\n' \
-            "$(date '+%H:%M:%S')" "$gpu" "$((start - START_TS))s" "$((end - start))" \
-            "$rc" "$([[ $weight -eq $DEFAULT_WEIGHT ]] && echo unknown || echo "${weight}s")" \
-            "$label" "$safetag"
-    done
-}
-
-: > "$TIMINGS_FILE"
 for gpu in "${GPU_IDS[@]}"; do
     worker "$gpu" &
 done
@@ -442,16 +431,11 @@ WALL=$(( $(date +%s) - START_TS ))
 
 # ---------------------------------------------------------------------------
 # Phase 5: roll per-item results up into a per-suite verdict
+# ---------------------------------------------------------------------------
 #
 # Two outputs per suite, both keyed by the logfile name from the config so the
 # workflow's failure gate keeps reading the same name: the exit code it gates on,
 # and an index of which items ran and how each one ended.
-#
-# The item logs are not concatenated into that index. They are the largest files
-# the run produces, and copying them would double both the artifact and the
-# reader's work -- the index says which item to open, and the item log is right
-# there under items/ with only that item's output in it.
-OVERALL_RC=0
 for i in "${!SUITE_LABELS[@]}"; do
     label="${SUITE_LABELS[$i]}"
     # Phase 1 fails a suite that expands to nothing, so this should never skip.
@@ -474,22 +458,23 @@ for i in "${!SUITE_LABELS[@]}"; do
 done
 
 # ---------------------------------------------------------------------------
-# Phase 6: scheduling report
+# Phase 6: update the learned weight table for the next run
 # ---------------------------------------------------------------------------
-if ! python3 "$REPO_ROOT/ci/schedule_report.py" "$LOG_DIR" \
-        --gpus "${GPU_IDS[*]}" \
-        --wall "$WALL" \
-        --default-weight "$DEFAULT_WEIGHT"; then
-    log_warn "could not write the scheduling report"
+echo
+if ! python3 "$REPO_ROOT/.github/scripts/scheduler/build_weights.py" update "$TIMINGS_FILE" \
+        --items "$ITEMS_FILE" -o "$WEIGHTS_FILE"; then
+    log_warn "could not update $WEIGHTS_FILE; the next run will use the table as it stands"
 fi
 
 # ---------------------------------------------------------------------------
-# Phase 7: Update the learned weight table for the next run. 
+# Phase 7: scheduling report
 # ---------------------------------------------------------------------------
-echo
-if ! python3 "$REPO_ROOT/ci/build_weights.py" "$TIMINGS_FILE" \
-        --items "$ITEMS_FILE" -o "$WEIGHTS_FILE"; then
-    log_warn "could not update $WEIGHTS_FILE; the next run will use the table as it stands"
+if ! python3 "$REPO_ROOT/.github/scripts/scheduler/schedule_report.py" "$LOG_DIR" \
+        --gpus "${GPU_IDS[*]}" \
+        --wall "$WALL" \
+        --default-weight "$DEFAULT_WEIGHT" \
+        --weights "$WEIGHTS_FILE"; then
+    log_warn "could not write the scheduling report"
 fi
 
 exit $OVERALL_RC
diff --git a/.github/scripts/scheduler/build_weights.py b/.github/scripts/scheduler/build_weights.py
new file mode 100755
index 0000000000..344b500f46
--- /dev/null
+++ b/.github/scripts/scheduler/build_weights.py
@@ -0,0 +1,318 @@
+#!/usr/bin/env python3
+# Copyright (c) 2026, Advanced Micro Devices, Inc. All rights reserved.
+#
+# See LICENSE for license information.
+
+"""Order/Update the sGPU scheduler's weight table.
+
+  order  : Read the cached weight table and apply it to sort the scheduler's queue by longest-processing-time first.
+  update : Read a finished run's timings and blend them into the cached weight table.
+"""
+
+import argparse
+import sys
+
+from queue_files import KILLED_RCS, read_timings
+
+
+def measurements(frame):
+    """Split one run's timings into what it measured and what it merely ran.
+
+    Returns ``(measured, dispatched)``, which answer different questions and go
+    to different places:
+
+      measured    key -> seconds, for the rows worth learning from.
+                  Blended into the table by ``merge_weights``.
+      dispatched  every key the run started, learned from or not. Used by
+                  ``prune_weights``, which only needs to know the item exists.
+
+    A row is kept out of ``measured`` when its duration records where the item
+    was stopped rather than what it costs:
+
+      rc in KILLED_RCS   124 from a ``timeout`` expiry, 137 from a SIGKILL or an
+                         OOM-kill: killed from outside, so the duration is a
+                         ceiling imposed on it.
+      incomplete == 1    pytest never reached its end-of-session write, which rc
+                         cannot say on its own -- a ``--timeout-method=thread``
+                         expiry exits 1, indistinguishable from an ordinary test
+                         failure. The scheduler raises the flag from the result
+                         sidecar the dying process left behind.
+
+    Both still count as dispatched: the item plainly exists, it just taught the
+    table nothing this run.
+    """
+    # "whole" is the scheduler's placeholder for an opaque suite item, which the
+    # weight table keys by bare label rather than by "<label>/<tag>".
+    key = frame["label"].where(frame["tag"] == "whole", frame["name"])
+    usable = ~frame["rc"].isin(KILLED_RCS) & (frame["incomplete"] != 1)
+    measured = frame["secs"][usable].groupby(key[usable]).sum()
+    return measured.to_dict(), set(key)
+
+
+def item_key(label, tag):
+    """The table key for one work item.
+
+    An opaque whole-suite item has no tag and is keyed by bare label; everything
+    else by ``<label>/<tag>``. Every reader and writer goes through here
+    """
+    return f"{label}/{tag}" if tag else label
+
+
+def read_items(path):
+    """Read items.tsv -- every work item that exists at this TEST_LEVEL"""
+    try:
+        handle = open(path, encoding="utf-8")
+    except OSError as err:
+        print(f"warning: cannot read {path} ({err}); nothing will be pruned", file=sys.stderr)
+        return None, None
+    keys, labels = set(), set()
+    with handle:
+        for line in handle:
+            label, _, tag = line.rstrip("\n").partition("\t")
+            if not label:
+                continue
+            labels.add(label)
+            keys.add(item_key(label, tag))
+    return keys, labels
+
+
+def read_weights(path):
+    """Read the cached weight table the scheduler uses to sort the queue."""
+    weights = {}
+    try:
+        handle = open(path, encoding="utf-8")
+    except OSError:
+        return weights
+    with handle:
+        for line in handle:
+            line = line.strip()
+            if not line or line.startswith("#"):
+                continue
+            key, _, value = line.rpartition(" ")
+            try:
+                weights[key] = float(value)
+            except ValueError:
+                continue
+    return weights
+
+
+QUEUE_COLUMNS = 4  # label, cmd, tag, rest
+
+
+def read_queue(path):
+    """Read the unweighted queue the expansion phase wrote."""
+    rows = []
+    with open(path, encoding="utf-8") as handle:
+        for line in handle:
+            line = line.rstrip("\n")
+            if not line.strip():
+                continue
+            fields = line.split("\t")
+            fields += [""] * (QUEUE_COLUMNS - len(fields))
+            rows.append(tuple(fields[:QUEUE_COLUMNS]))
+    return rows
+
+
+def order_queue(rows, weights, default_weight):
+    """Weight every work item and sort the queue longest-processing-time first.
+
+    LPT is the standard makespan heuristic: dispatch the long items first and
+    what is left to fill the tail is short, so no GPU is still starting a big
+    item once the others have gone idle. An item the table has never seen takes
+    ``default_weight``, which sorts it first -- see the scheduler for why that
+    gamble is the cheap direction.
+    """
+    weighted = [
+        (int(weights.get(item_key(label, tag), default_weight)), label, cmd, tag, rest)
+        for label, cmd, tag, rest in rows
+    ]
+    weighted.sort(key=lambda row: -row[0])
+    return weighted
+
+
+def plan_lines(queue, weights_path, n_weights, gpu_ids, default_weight):
+    """Render what the scheduler intends to do, before it does any of it."""
+
+    def est(weight):
+        return "unknown" if weight == default_weight else f"{weight}s"
+
+    n_gpus = len(gpu_ids) or 1
+    lines = [f"== Queue: {len(queue)} items over {n_gpus} GPUs ({' '.join(gpu_ids)}) =="]
+    if n_weights:
+        lines.append(f"   weights: {weights_path} ({n_weights} items)")
+    else:
+        lines.append(f"   weights: none yet -- this run is unordered and will write {weights_path}")
+
+    lines.append("== Queue plan (dispatch order; est = weight used to sort) ==")
+    total = 0
+    for position, (weight, label, _cmd, tag, _rest) in enumerate(queue, 1):
+        total += weight
+        lines.append(f"  {position:3d}. est={est(weight):<8} {label:<9} {tag or '(whole suite)'}")
+    lines.append("")
+    lines.append(
+        f"  estimated work {total}s over {n_gpus} GPUs -> lower bound {total / n_gpus:.0f}s"
+        f" (largest item {est(queue[0][0])})"
+    )
+    return lines
+
+
+def merge_weights(old, new, alpha_up=0.5, alpha_down=0.1):
+    """Update cached table with new weights from current run.
+
+    Every key already in the table moves by an exponential average,
+    ``(1 - alpha) * previous + alpha * measured``; a key the table has never
+    seen takes its measurement whole. The rate depends on which way it moved:
+
+      alpha_up    0.5, when the item got slower -- half the gap closed in one
+                  run.
+      alpha_down  0.1, when it got faster -- a tenth of it.
+
+    The two differ because the scheduler's two error directions do not cost the
+    same. Over-estimating an item only starts it earlier than it needed to be
+    started; under-estimating one leaves it running after the other GPUs have
+    gone idle, and that is what sets the makespan. So a rise is trusted quickly
+    and a fall slowly.
+
+    The slow fall also makes the table shrug off a single short run -- a crash
+    partway through, or a machine having a fast day -- without anyone needing to
+    decide whether that run was "valid".
+    """
+    merged = dict(old)
+    for key, value in new.items():
+        previous = merged.get(key)
+        if previous is None:
+            merged[key] = value
+            continue
+        alpha = alpha_up if value > previous else alpha_down
+        merged[key] = (1.0 - alpha) * previous + alpha * value
+    return merged
+
+
+def prune_weights(weights, dispatched, live_keys, live_labels):
+    """Drop the weights of work items that no longer exist.
+
+    A key survives if item.tsv lists it.
+    """
+    kept, pruned = {}, []
+    for key, value in weights.items():
+        label = key.split("/", 1)[0]
+        if key in dispatched or label not in live_labels or key in live_keys:
+            kept[key] = value
+        else:
+            pruned.append(key)
+    return kept, pruned
+
+
+def run_order(args):
+    """Apply the cached weight table to the queue and send it to the scheduler."""
+    try:
+        rows = read_queue(args.queue)
+    except OSError as err:
+        print(f"error: cannot read {args.queue} ({err})", file=sys.stderr)
+        return 1
+    if not rows:
+        print("error: queue is empty: every suite expanded to nothing", file=sys.stderr)
+        return 1
+
+    weights = read_weights(args.weights)
+    queue = order_queue(rows, weights, args.default_weight)
+    try:
+        with open(args.output, "w", encoding="utf-8") as handle:
+            for row in queue:
+                handle.write("\t".join(str(field) for field in row) + "\n")
+    except OSError as err:
+        print(f"error: cannot write {args.output} ({err})", file=sys.stderr)
+        return 1
+
+    for line in plan_lines(
+        queue, args.weights, len(weights), args.gpus.split(), args.default_weight
+    ):
+        print(line)
+    return 0
+
+
+def run_update(args):
+    """Blend a finished run's measurements into the table and update the cache."""
+    measured, dispatched = measurements(read_timings(args.timings))
+    if not measured:
+        # Every row was killed, cut short or malformed. There is nothing to
+        # learn and the existing table is still the best estimate available, so
+        # leave it exactly as it is rather than rewriting it from nothing.
+        print(f"error: no usable measurements in {args.timings}", file=sys.stderr)
+        return 1
+
+    previous = read_weights(args.output)
+    weights = merge_weights(previous, measured)
+    print(
+        f"merged {len(measured)} measurements into {len(previous)} known weights "
+        f"({len(set(weights) - set(previous))} new)",
+        file=sys.stderr,
+    )
+
+    if args.items:
+        live_keys, live_labels = read_items(args.items)
+        if live_keys is not None:
+            weights, pruned = prune_weights(weights, dispatched, live_keys, live_labels)
+            # Named individually rather than counted: a prune is how a deleted
+            # test leaves the table, and it is also exactly what a broken suite
+            # expansion would look like, so the list is worth reading.
+            for key in sorted(pruned):
+                print(f"pruned {key}: its suite no longer lists it", file=sys.stderr)
+
+    lines = [
+        f"{key} {value:.1f}\n" for key, value in sorted(weights.items(), key=lambda kv: -kv[1])
+    ]
+    with open(args.output, "w", encoding="utf-8") as handle:
+        handle.writelines(lines)
+    print(f"wrote {len(lines)} weights to {args.output}", file=sys.stderr)
+    return 0
+
+
+def main():
+    """Dispatch to ``order`` (apply the table) or ``update`` (rewrite it)."""
+    parser = argparse.ArgumentParser(
+        description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter
+    )
+    sub = parser.add_subparsers(dest="command", required=True)
+
+    order = sub.add_parser(
+        "order", help="apply the table: weight a queue and sort it longest-first"
+    )
+    order.add_argument("queue", metavar="QUEUE", help="the unweighted queue from expansion")
+    order.add_argument(
+        "-w", "--weights", required=True, metavar="TABLE", help="weight table to read"
+    )
+    order.add_argument(
+        "-o", "--output", required=True, metavar="FILE", help="where to write the sorted queue"
+    )
+    order.add_argument(
+        "--default-weight",
+        type=int,
+        required=True,
+        metavar="SECS",
+        help="weight for an item the table has never seen; it sorts first",
+    )
+    order.add_argument(
+        "--gpus", default="", metavar="IDS", help='space-separated device ids, e.g. "0 1 2 4"'
+    )
+
+    update = sub.add_parser("update", help="fold a finished run's timings into the table")
+    update.add_argument(
+        "timings", metavar="TIMINGS", help="timings.tsv written by run_queue_sgpu.sh"
+    )
+    update.add_argument(
+        "-o", "--output", required=True, metavar="TABLE", help="weight table to update in place"
+    )
+    update.add_argument(
+        "--items",
+        metavar="FILE",
+        help="items.tsv listing every work item that exists at this level; "
+        "without it, no weight is ever pruned",
+    )
+
+    args = parser.parse_args()
+    return run_order(args) if args.command == "order" else run_update(args)
+
+
+if __name__ == "__main__":
+    sys.exit(main())
diff --git a/.github/scripts/scheduler/queue_files.py b/.github/scripts/scheduler/queue_files.py
new file mode 100644
index 0000000000..1beb0ed00d
--- /dev/null
+++ b/.github/scripts/scheduler/queue_files.py
@@ -0,0 +1,61 @@
+#!/usr/bin/env python3
+# Copyright (c) 2026, Advanced Micro Devices, Inc. All rights reserved.
+#
+# See LICENSE for license information.
+
+"""The ``timings.tsv`` contract, shared by the two tools that read it.
+
+``run_queue_sgpu.sh`` appends one row per work item as it dispatches. Two tools
+consume that file afterwards -- ``build_weights.py`` to learn the next run's
+schedule and ``schedule_report.py`` to report on this one
+
+Columns, in order:
+
+  label       suite label, e.g. "torch"
+  tag         work item within the suite, or "whole" for an opaque suite
+  gpu         HIP device id it ran on, as written (ids need not start at zero)
+  secs        wall-clock duration
+  rc          exit code of the suite invocation
+  start_off   seconds from the start of the queue to this item's start
+  end_off     seconds from the start of the queue to this item's finish
+  est         the weight used to schedule it, or the caller's default weight
+  incomplete  1 if the item was cut off mid-way
+"""
+
+import pandas as pd
+
+COLUMNS = ["label", "tag", "gpu", "secs", "rc", "start_off", "end_off", "est", "incomplete"]
+TEXT_COLUMNS = ["label", "tag", "gpu"]
+NUMERIC_COLUMNS = ["secs", "rc", "start_off", "end_off", "est", "incomplete"]
+
+# rc values that mean the item was killed rather than measured. 124 is a GNU
+# timeout expiry (the job's own step timeout, for instance), 137 a SIGKILL or an
+# OOM-kill.
+KILLED_RCS = frozenset((124, 137))
+
+
+def read_timings(path):
+    """``timings.tsv`` as a DataFrame, one row per dispatched work item."""
+    try:
+        frame = pd.read_csv(
+            path,
+            sep="\t",
+            header=None,
+            names=COLUMNS,
+            dtype={c: str for c in TEXT_COLUMNS},
+            keep_default_na=False,
+            na_values=[],
+            on_bad_lines="skip",
+        )
+    except (OSError, pd.errors.EmptyDataError):
+        frame = pd.DataFrame({c: pd.Series(dtype=str) for c in COLUMNS})
+
+    for column in NUMERIC_COLUMNS:
+        frame[column] = pd.to_numeric(frame[column], errors="coerce")
+    # A short row leaves NaN in the numeric columns; a torn one leaves garbage
+    # that to_numeric just turned into NaN. Either way there is no measurement.
+    frame = frame.dropna(subset=NUMERIC_COLUMNS)
+    frame[NUMERIC_COLUMNS] = frame[NUMERIC_COLUMNS].astype(int)
+
+    frame["name"] = frame["label"] + "/" + frame["tag"]
+    return frame.reset_index(drop=True)
diff --git a/.github/scripts/scheduler/schedule_report.py b/.github/scripts/scheduler/schedule_report.py
new file mode 100644
index 0000000000..009e86112c
--- /dev/null
+++ b/.github/scripts/scheduler/schedule_report.py
@@ -0,0 +1,285 @@
+#!/usr/bin/env python3
+# Copyright (c) 2026, Advanced Micro Devices, Inc. All rights reserved.
+#
+# See LICENSE for license information.
+
+"""Render one sGPU queue run's scheduling report from its measured timings.
+
+"""
+
+import argparse
+import os
+import sys
+
+from queue_files import KILLED_RCS, read_timings
+from build_weights import item_key, read_weights
+
+# Where the scheduler puts things under its log directory
+TIMINGS_PATH = ("queue", "timings.tsv")
+QUEUE_PATH = ("queue", "queue.tsv")
+REPORT_PATH = ("report", "schedule.md")
+
+REPORT_TITLE = "sGPU queue schedule"
+
+
+def outcome(rc):
+    """An exit code as the word that says what to do about it.
+    """
+    if rc == 0:
+        return "pass"
+    if rc == 1:
+        return "fail"
+    return "killed" if rc in KILLED_RCS else "error"
+
+
+def read_queue_keys(path):
+    """Every item the queue intended to run, as "<label>/<tag>" keys.
+
+    queue.tsv columns are weight, label, cmd, tag, rest; an empty tag is an
+    opaque whole-suite item, which the timings file records as "whole".
+    """
+    keys = []
+    with open(path, encoding="utf-8") as handle:
+        for line in handle:
+            fields = line.rstrip("\n").split("\t")
+            if len(fields) >= 4:
+                keys.append(f"{fields[1]}/{fields[3] or 'whole'}")
+    return keys
+
+def render_md(rows):
+    """Rows as a GitHub-flavoured Markdown table."""
+    out = []
+    for n, row in enumerate(rows):
+        out.append("| " + " | ".join(c.replace("|", "\\|") for c in row) + " |")
+        if n == 0:
+            out.append("|" + "---|" * len(row))
+    return out
+
+def calculate_efficiency_table(frame, wall, n_gpus):
+    """What the run cost, and how much of the GPU-time it bought was used."""
+    work = int(frame["secs"].sum())
+    if len(frame):
+        biggest = frame.loc[frame["secs"].idxmax()]
+        big, bigname = int(biggest["secs"]), biggest["name"]
+    else:
+        big, bigname = 0, ""
+    # Once one item exceeds the per-GPU average there is no ordering that
+    # finishes sooner than that item, so it becomes the thing to split.
+    floor = " -- floor: splitting it would now pay" if big > work / n_gpus else ""
+    return [
+        ["Metric", "Value", "Meaning"],
+        ["total work", f"{work}s", f"sum of all {len(frame)} item durations"],
+        ["actual run time", f"{wall}s", "wall clock, first item start to last item finish"],
+        [
+            "utilisation",
+            f"{work * 100 / (wall * n_gpus):.1f}%",
+            f"share of {wall}s x {n_gpus} GPUs actually spent running tests",
+        ],
+        ["largest item", f"{big}s", bigname + floor],
+    ]
+
+
+def calculate_gpu_utilisation_table(frame, wall, gpu_ids):
+    """Per-GPU utilisation.
+    """
+    per_gpu = frame.groupby("gpu")
+    busy = per_gpu["secs"].sum().reindex(gpu_ids, fill_value=0).astype(int)
+    count = per_gpu.size().reindex(gpu_ids, fill_value=0).astype(int)
+    failed = (
+        frame[frame["rc"] != 0].groupby("gpu").size().reindex(gpu_ids, fill_value=0).astype(int)
+    )
+
+    return [["GPU", "Items", "Busy", "Idle", "Util", "Failed"]] + [
+        [
+            f"gpu{gpu}",
+            f"{count[gpu]}",
+            f"{busy[gpu]}s",
+            f"{wall - busy[gpu]}s",
+            f"{busy[gpu] * 100 / wall:.1f}%",
+            f"{failed[gpu]}",
+        ]
+        for gpu in gpu_ids
+    ]
+
+
+def calculate_schedule_table(frame, default_weight):
+    """What ran where, in execution order, against what it was estimated to cost."""
+    order = frame.assign(_gpu=frame["gpu"].astype(int)).sort_values(
+        ["_gpu", "start_off", "name"], kind="stable"
+    )
+    rows = [["GPU", "Start", "Duration", "Result", "Estimate", "% change", "Test name"]]
+    for row in order.itertuples(index=False):
+        known = row.est > 0 and row.est != default_weight
+        # The % change is the scheduler feedback loop made visible: a large
+        # positive miss is an item that should have been dispatched earlier.
+        change = f"{(row.secs - row.est) * 100 / row.est:+.0f}%" if known else "n/a"
+        # "cut" marks a duration that is where the item was stopped rather than
+        # what it costs, which is also why the next run's weight table ignores
+        # that row.
+        result = outcome(row.rc) + (" (cut)" if row.incomplete == 1 else "")
+        rows.append(
+            [
+                f"gpu{row.gpu}",
+                f"t+{row.start_off}s",
+                f"{row.secs}s",
+                result,
+                "unknown" if row.est == default_weight else f"{row.est}s",
+                change,
+                row.name,
+            ]
+        )
+    return rows
+
+
+def calculate_updated_weights_table(frame, weights, default_weight):
+    """
+    What the next run will schedule each item with.
+    """
+    rows = []
+    for row in frame.itertuples(index=False):
+        key = item_key(row.label, "" if row.tag == "whole" else row.tag)
+        new = weights.get(key)
+        known = row.est > 0 and row.est != default_weight
+        rows.append(
+            (
+                # An item the table does not name is one the next run has no
+                # weight for, and order_queue sorts those first -- so it sorts
+                # first here too, for the ordering claim above to hold.
+                new if new is not None else float("inf"),
+                [
+                    f"{new:.0f}s" if new is not None else "unknown",
+                    "unknown" if not known else f"{row.est}s",
+                    f"{row.secs}s" + (" (cut)" if row.incomplete == 1 else ""),
+                    f"{(row.secs - row.est) * 100 / row.est:+.0f}%" if known else "n/a",
+                    row.name,
+                ],
+            )
+        )
+    # Name breaks ties so the order is stable run to run rather than dependent on
+    # dispatch order.
+    rows.sort(key=lambda r: (-r[0], r[1][4]))
+    return [["Updated weight", "This run", "Current weight", "% change", "Test name"]] + [
+        row for _, row in rows
+    ]
+
+
+def missing_items(frame, queue_keys):
+    """Items the queue held that produced no timing record.
+
+    Only possible if a worker died outright, and silence here would read as
+    success.
+    """
+    seen = set(frame["name"])
+    return [key for key in queue_keys if key not in seen]
+
+
+# ---------------------------------------------------------------------------
+# Report.
+
+
+def render_report_md(frame, wall, gpu_ids, total_items, default_weight, missing, weights):
+    """The Markdown report the workflow appends to the job summary."""
+    n_gpus = len(gpu_ids)
+    ran = len(frame)
+    failed = int((frame["rc"] != 0).sum())
+    util = int(frame["secs"].sum()) * 100 / (wall * n_gpus)
+    mark = ":x:" if failed or ran != total_items else ":white_check_mark:"
+
+    out = [
+        f"## {REPORT_TITLE}",
+        "",
+        f"{mark} **{ran} items** on {n_gpus} GPUs -- {failed} failed "
+        f"-- {wall}s wall clock at {util:.1f}% GPU utilisation",
+        "",
+    ]
+
+    if ran != total_items:
+        out.append(f"> :warning: **Only {ran} of {total_items} items produced a timing")
+        out.append("> record.** The rest were never dispatched, which means a worker died:")
+        out.append("")
+        out.extend(f"> - `{key}`" for key in missing)
+        out.append("")
+
+    for heading, rows in (
+        ("Efficiency", calculate_efficiency_table(frame, wall, n_gpus)),
+        ("Per-GPU utilisation", calculate_gpu_utilisation_table(frame, wall, gpu_ids)),
+    ):
+        out += [f"### {heading}", ""] + render_md(rows) + [""]
+
+    # The two big tables are collapsed: they are the detail you open once you
+    # know from the sections above that something is worth looking at.
+    for summary, rows in (
+        (
+            "Schedule -- what ran where, in execution order",
+            calculate_schedule_table(frame, default_weight),
+        ),
+        (
+            "Updated weights for next run",
+            calculate_updated_weights_table(frame, weights, default_weight),
+        ),
+    ):
+        out += [f"<details><summary>{summary}</summary>", ""] + render_md(rows)
+        out += ["", "</details>", ""]
+    return out
+
+
+def main():
+    """Parse arguments, render the report, and never fail the run over it."""
+    parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
+    parser.add_argument("log_dir", help="the queue's log directory, e.g. test-results/logs")
+    # Required, both of them: they are facts about the run that no file in the
+    # log directory records, and guessing at either yields a plausible-looking
+    # report with the wrong utilisation in it.
+    parser.add_argument(
+        "--gpus", required=True, metavar="IDS", help='space-separated device ids, e.g. "0 1 2 4"'
+    )
+    parser.add_argument(
+        "--wall", type=int, required=True, metavar="SECS", help="run wall clock in seconds"
+    )
+    parser.add_argument(
+        "--default-weight",
+        type=int,
+        default=999999,
+        help="the est value that means 'no weight was known'",
+    )
+    parser.add_argument(
+        "--weights",
+        required=True,
+        metavar="TABLE",
+        help="the weight table build_weights.py has just rewritten; the report "
+        "reads it to show what the next run will schedule with",
+    )
+    args = parser.parse_args()
+
+    timings = os.path.join(args.log_dir, *TIMINGS_PATH)
+    queue = os.path.join(args.log_dir, *QUEUE_PATH)
+    out = os.path.join(args.log_dir, *REPORT_PATH)
+    if not os.path.exists(timings):
+        print(f"no {timings}: nothing to report on", file=sys.stderr)
+        return 0
+
+    frame = read_timings(timings)
+
+    gpu_ids = args.gpus.split() or ["0"]
+    wall = max(args.wall, 1)
+
+    # Without queue.tsv there is nothing to say an item is missing, so every item
+    # that ran is taken to be every item there was.
+    queue_keys = read_queue_keys(queue) if os.path.exists(queue) else list(frame["name"])
+    missing = missing_items(frame, queue_keys)
+
+    weights = read_weights(args.weights)
+
+    report = render_report_md(
+        frame, wall, gpu_ids, len(queue_keys), args.default_weight, missing, weights
+    )
+    os.makedirs(os.path.dirname(out), exist_ok=True)
+    with open(out, "w", encoding="utf-8") as handle:
+        handle.write("\n".join(report) + "\n")
+
+    print(f"Scheduling report: {out}")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())
diff --git a/ci/_utils.sh b/ci/_utils.sh
index 34e506e87d..b23ed48156 100644
--- a/ci/_utils.sh
+++ b/ci/_utils.sh
@@ -79,6 +79,7 @@ export PYTHONSAFEPATH=${PYTHONSAFEPATH:-1}
 _script_error_count=0
 _run_error_count=0
 _ignored_error_count=0
+_seen_test_tags=""
 TEST_ERROR_IGNORE=""
 
 script_error() {
@@ -143,13 +144,40 @@ check_level() {
 # TEST_LEVEL could run anywhere". Level gating, the backend matrix and TEST_FILTER
 # still apply -- only host capability probes are answered yes without probing.
 # The scheduler uses the wider list as the authoritative set of work items that
-# exist, so ci/build_weights.py can tell a test that was deleted or renamed (drop
-# its weight) from one this host merely skipped (keep it). A probe result varies
-# by machine; whether a test file exists does not.
+# exist, so .github/scripts/scheduler/build_weights.py can tell a test that was deleted or
+# renamed (drop its weight) from one this host merely skipped (keep it). A probe
+# result varies by machine; whether a test file exists does not.
 check_list_all() {
     test -n "$TE_CI_LIST_ONLY" -a -n "$TE_CI_LIST_ALL"
 }
 
+# Every host capability probe goes through here, selected by label. Returns 0 to
+# run the guarded tests, 1 to skip them.
+check_supported() {
+    check_list_all && return 0
+    case "$1" in
+        "mxfp8")
+            #MXFP8-only test filters collect no tests on unsupported archs
+            _probe_result=$(NVTE_ROCM_ENABLE_MXFP8=1 python -c "${PYTHON_TE_IMPORT}; from transformer_engine.pytorch.quantization import is_mxfp8_available; print(is_mxfp8_available())" 2>/dev/null)
+            _probe_message="MXFP8 is not supported on this device, skipping MXFP8-only tests"
+        ;;
+        "flash_attn")
+            _probe_result=$(python -c "${PYTHON_TE_IMPORT}; from transformer_engine.pytorch.attention.dot_product_attention.utils import FlashAttentionUtils; print(FlashAttentionUtils.is_installed)" 2>/dev/null)
+            _probe_message="Flash attention is not installed"
+        ;;
+        *)
+            #A mistyped label must not read as "unsupported": that would skip the
+            #guarded tests on every host and look exactly like a machine that
+            #cannot run them. Count it so the run's exit code reports it.
+            script_error "check_supported: unknown capability $1"
+            return 1
+        ;;
+    esac
+    test "$_probe_result" = "True" && return 0
+    echo "$_probe_message" >&2
+    return 1
+}
+
 check_test_jobs_requested() {
     return 1 #Disable parallel jobs because some tests do not support parallel execution
     test -z "$SINGLE_CONFIG" -a -n "$TEST_JOBS" || return 1
@@ -322,6 +350,27 @@ check_test_filter() {
     return 1
 }
 
+# The test name tag names the JUnit XML file and is the TEST_FILTER key a
+# scheduler re-dispatches on, so two call lines may not share one: the second
+# would overwrite the first's results, and one dispatch would run both. Anything
+# that changes what a line runs -- an env prefix, a -k expression, extra pytest
+# args -- needs its own label to keep the tags apart.
+#
+# Fatal rather than counted-and-continued. Skipping the offending call line
+# leaves a run that looks complete but is quietly short by an item, and in list
+# mode that short list is what the scheduler queues -- so the tests the
+# duplicate displaced never run at all, on this run or any after it. The only
+# signal would be a non-zero exit code at the very end, long after the message
+# scrolled past.
+check_test_tag_unique() {
+    case " $_seen_test_tags " in
+    *" $1 "*)
+        script_error "Duplicate test tag $1: give the call site a distinct label"
+        exit 1
+    esac
+    _seen_test_tags="$_seen_test_tags $1"
+}
+
 start_message() {
     echo "Started with TEST_LEVEL=$TEST_LEVEL sGPU='$TEST_SGPU' mGPU='$TEST_MGPU' at `date`"
     _rocm_path=$(resolve_rocm_path)
@@ -361,6 +410,7 @@ pytest_run() {
     shift 3
     _test_name_tag=`get_test_name_tag $1 $_test_variant_tag`
     check_test_filter $_test_name_tag || return
+    check_test_tag_unique $_test_name_tag || return
     # List mode: emit the work item instead of running it, so an external
     # scheduler can pack items across GPUs. The tag alone is enough to
     # re-dispatch the item: setting TEST_FILTER to it and re-entering this
diff --git a/ci/build_weights.py b/ci/build_weights.py
deleted file mode 100755
index a0e7f349df..0000000000
--- a/ci/build_weights.py
+++ /dev/null
@@ -1,252 +0,0 @@
-#!/usr/bin/env python3
-# Copyright (c) 2026, Advanced Micro Devices, Inc. All rights reserved.
-#
-# See LICENSE for license information.
-
-"""Fold one CI run's measured timings into the sGPU scheduler's weight table.
-
-``run_queue_sgpu.sh`` orders its queue longest-first, which needs a per-item cost
-estimate. That estimate comes from the run that just finished, so no separate
-measurement pass is required: the scheduler writes ``timings.tsv`` as it
-dispatches, and every run refines the table that the next one reads.
-
-Inputs and outputs:
-
-  TIMINGS     ``timings.tsv`` written by the scheduler. Per-item wall clock, so
-              it includes process startup and HIP init -- which is what the
-              makespan is actually made of, and what a test-time-only source
-              such as JUnit XML would understate several-fold.
-  --items     ``items.tsv``, also from the scheduler: every work item that
-              exists at this TEST_LEVEL, whether or not this host ran it.
-  -o TABLE    the weight table, read and rewritten in place. Lines are
-              ``<label>/<tag> <seconds>``, plus ``<label> <seconds>`` for opaque
-              whole-suite items.
-
-Keep one table per (GPU arch, TEST_LEVEL). Skip patterns differ between gfx942
-and gfx950, and the level changes both which items run and how long several of
-them take (tests/jax reads NVTE_JAX_UNITTEST_LEVEL, test_grouped_gemm.py reads
-TEST_LEVEL), so that is the coarsest keying under which a weight still means one
-thing. Holding them apart is the caller's job -- the workflow puts arch and level
-in the file name.
-
-Each run is blended into the table rather than replacing it, so no single noisy
-sample can move a weight far; see ``merge_weights`` for why the blend is
-asymmetric. Measurements that were cut short are dropped instead of blended
-(``read_timings``), and items that no longer exist are dropped from the table
-(``prune_weights``).
-"""
-
-import argparse
-import sys
-from collections import defaultdict
-
-# rc values that mean the item was killed rather than measured. 124 is a GNU
-# timeout expiry (the job's own step timeout, for instance), 137 a SIGKILL or an
-# OOM-kill. Either way the recorded duration is a ceiling imposed from outside,
-# not the item's cost, so blending it in would teach the table a number the item
-# never actually took. Drop them; the previous weight stands.
-KILLED_RCS = frozenset(("124", "137"))
-
-
-def read_timings(path):
-    """Read the scheduler's timings.tsv.
-
-    Columns are label, tag, gpu, secs, rc, start_off, end_off, est, incomplete;
-    only the first five and the last are read here.
-
-    Returns ``(measured, dispatched)``. ``measured`` maps key -> seconds and
-    holds only the rows worth learning from; ``dispatched`` is every key the run
-    started, which is a weaker but separate fact -- an item can have run and
-    still have produced no usable number, and only the second set answers "does
-    this item still exist" (see ``prune_weights``).
-
-    Two kinds of row are excluded from ``measured``, for the same reason: the
-    duration records where the item was stopped, not what it costs. An rc in
-    ``KILLED_RCS`` says so outright. The ``incomplete`` flag covers what rc
-    cannot -- a ``--timeout-method=thread`` expiry exits 1, indistinguishable
-    from an ordinary test failure (which is a perfectly good measurement), and a
-    segfault exits 139. The scheduler sets that flag from the result sidecar the
-    dying process left behind, which is the only reliable witness.
-    """
-    measured = defaultdict(float)
-    dispatched = set()
-    with open(path, encoding="utf-8") as handle:
-        for line in handle:
-            fields = line.rstrip("\n").split("\t")
-            if len(fields) < 4:
-                continue
-            label, tag, _gpu, secs = fields[0], fields[1], fields[2], fields[3]
-            # "whole" is the scheduler's placeholder for an opaque suite item,
-            # which the queue keys by bare label.
-            key = label if tag == "whole" else f"{label}/{tag}"
-            dispatched.add(key)
-            if len(fields) > 4 and fields[4] in KILLED_RCS:
-                continue
-            if len(fields) > 8 and fields[8] == "1":
-                continue
-            try:
-                value = float(secs)
-            except ValueError:
-                continue
-            measured[key] += value
-    return measured, dispatched
-
-
-def read_items(path):
-    """Read items.tsv -- every work item that exists at this TEST_LEVEL.
-
-    The file is ``<label>\\t<tag>`` per line, with an empty tag for an opaque
-    whole-suite item. Returns ``(keys, labels)``: the item keys, and the set of
-    suite labels the file speaks for.
-
-    Both are needed because pruning has to be per label. A suite can be absent
-    from the file entirely -- it was filtered out of the run, or its expansion
-    came back untrustworthy -- and that absence must not be read as "none of its
-    items exist any more".
-
-    Returns ``(None, None)`` when the file cannot be read, which disables
-    pruning. That is the conservative direction: a weight kept for a test that
-    is gone wastes one line, while a weight dropped for a live test makes the
-    scheduler treat it as unknown and mis-order a whole run.
-    """
-    try:
-        handle = open(path, encoding="utf-8")
-    except OSError as err:
-        print(f"warning: cannot read {path} ({err}); nothing will be pruned", file=sys.stderr)
-        return None, None
-    keys, labels = set(), set()
-    with handle:
-        for line in handle:
-            label, _, tag = line.rstrip("\n").partition("\t")
-            if not label:
-                continue
-            labels.add(label)
-            keys.add(f"{label}/{tag}" if tag else label)
-    return keys, labels
-
-
-def read_weights(path):
-    """Read an existing ``<key> <seconds>`` table. Missing file -> empty table."""
-    weights = {}
-    try:
-        handle = open(path, encoding="utf-8")
-    except OSError:
-        return weights
-    with handle:
-        for line in handle:
-            line = line.strip()
-            if not line or line.startswith("#"):
-                continue
-            key, _, value = line.rpartition(" ")
-            try:
-                weights[key] = float(value)
-            except ValueError:
-                continue
-    return weights
-
-
-def merge_weights(old, new, alpha_up=0.5, alpha_down=0.1):
-    """Blend a fresh measurement into the running table.
-
-    The blend is deliberately asymmetric, because the scheduler's two error
-    directions are not equally expensive. Over-estimating an item only starts it
-    earlier than it needed to be started; under-estimating one leaves it running
-    after the other GPUs have gone idle, and that is what sets the makespan. So
-    a rise is trusted quickly (half the gap in one run) while a fall is trusted
-    slowly (a tenth), which also makes the table shrug off a single short run --
-    a crash partway through, or a machine having a fast day -- without needing
-    to decide whether that run was "valid".
-    """
-    merged = dict(old)
-    for key, value in new.items():
-        previous = merged.get(key)
-        if previous is None:
-            merged[key] = value
-            continue
-        alpha = alpha_up if value > previous else alpha_down
-        merged[key] = (1.0 - alpha) * previous + alpha * value
-    return merged
-
-
-def prune_weights(weights, dispatched, live_keys, live_labels):
-    """Drop the weights of work items that no longer exist.
-
-    A key survives if any of these holds:
-
-      * the run dispatched it -- then it plainly exists, whatever else says, and
-        that holds even when it was killed and so taught the table nothing;
-      * its suite is not covered by the item list -- nothing is known about it;
-      * the item list contains it.
-
-    What is left is a key whose suite was listed in full and which that listing
-    did not contain: a test that was deleted, or renamed. Renames need no
-    handling at this end -- the new name simply arrives as an unknown key, which
-    the scheduler already sorts first.
-
-    Returns ``(kept, pruned)``.
-    """
-    kept, pruned = {}, []
-    for key, value in weights.items():
-        label = key.split("/", 1)[0]
-        if key in dispatched or label not in live_labels or key in live_keys:
-            kept[key] = value
-        else:
-            pruned.append(key)
-    return kept, pruned
-
-
-def main():
-    parser = argparse.ArgumentParser(
-        description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter
-    )
-    parser.add_argument(
-        "timings", metavar="TIMINGS", help="timings.tsv written by run_queue_sgpu.sh"
-    )
-    parser.add_argument(
-        "-o", "--output", required=True, metavar="TABLE", help="weight table to update in place"
-    )
-    parser.add_argument(
-        "--items",
-        metavar="FILE",
-        help="items.tsv listing every work item that exists at this level; "
-        "without it, no weight is ever pruned",
-    )
-    args = parser.parse_args()
-
-    measured, dispatched = read_timings(args.timings)
-    if not measured:
-        # Every row was killed, cut short or malformed. There is nothing to
-        # learn and the existing table is still the best estimate available, so
-        # leave it exactly as it is rather than rewriting it from nothing.
-        print(f"error: no usable measurements in {args.timings}", file=sys.stderr)
-        return 1
-
-    previous = read_weights(args.output)
-    weights = merge_weights(previous, measured)
-    print(
-        f"merged {len(measured)} measurements into {len(previous)} known weights "
-        f"({len(set(weights) - set(previous))} new)",
-        file=sys.stderr,
-    )
-
-    if args.items:
-        live_keys, live_labels = read_items(args.items)
-        if live_keys is not None:
-            weights, pruned = prune_weights(weights, dispatched, live_keys, live_labels)
-            # Named individually rather than counted: a prune is how a deleted
-            # test leaves the table, and it is also exactly what a broken suite
-            # expansion would look like, so the list is worth reading.
-            for key in sorted(pruned):
-                print(f"pruned {key}: its suite no longer lists it", file=sys.stderr)
-
-    lines = [
-        f"{key} {value:.1f}\n" for key, value in sorted(weights.items(), key=lambda kv: -kv[1])
-    ]
-    with open(args.output, "w", encoding="utf-8") as handle:
-        handle.writelines(lines)
-    print(f"wrote {len(lines)} weights to {args.output}", file=sys.stderr)
-    return 0
-
-
-if __name__ == "__main__":
-    sys.exit(main())
diff --git a/ci/jax.sh b/ci/jax.sh
index a5bcc094be..c0305ba4d8 100755
--- a/ci/jax.sh
+++ b/ci/jax.sh
@@ -99,11 +99,11 @@ run_test_config_mgpu() {
     run_default_fa 2 test_distributed_layernorm_mlp.py -k "not (${_layernorm_mlp_jax_gemm_k})"
     _saved_xla_flags="$XLA_FLAGS"
     export XLA_FLAGS="${XLA_FLAGS} --xla_gpu_enable_triton_gemm=false"
-    run_default_fa 2 test_distributed_layernorm_mlp.py -k "${_layernorm_mlp_jax_gemm_k}"
+    run_default_fa_lbl "no_triton_gemm" 2 test_distributed_layernorm_mlp.py -k "${_layernorm_mlp_jax_gemm_k}"
     export XLA_FLAGS="$_saved_xla_flags"
     run_default_fa 3 test_distributed_softmax.py
 
-    run_default_fa 3 test_sanity_import.py
+    run_default_fa_lbl "mgpu" 3 test_sanity_import.py
 }
 
 # Single config mode, run it synchronously and return result
diff --git a/ci/pytorch.sh b/ci/pytorch.sh
index 6c5e000088..794297f5ef 100755
--- a/ci/pytorch.sh
+++ b/ci/pytorch.sh
@@ -38,18 +38,6 @@ run_default_fa_lbl() {
     fi
 }
 
-check_mxfp8_supported() {
-    #Guard MXFP8-only test filters, which collect no tests on unsupported archs
-    check_list_all && return 0
-    _result=$(NVTE_ROCM_ENABLE_MXFP8=1 python -c "${PYTHON_TE_IMPORT}; from transformer_engine.pytorch.quantization import is_mxfp8_available; print(is_mxfp8_available())" 2>/dev/null)
-    if [ "$_result" = "True" ]; then
-        return 0
-    else
-        echo "MXFP8 is not supported on this device, skipping MXFP8-only tests" >&2
-        return 1
-    fi
-}
-
 run_test_config(){
     echo ==== Run with Fused attention backend: $_fus_attn ====
     #_WORKERS_COUNT=$TEST_WORKERS
@@ -91,7 +79,7 @@ run_test_config(){
     run 1 test_jit.py
     NVTE_ROCM_ENABLE_MXFP8=1 run_default_fa 1 test_multi_tensor.py
     run 1 test_numerics.py
-    check_mxfp8_supported && NVTE_ROCM_ENABLE_MXFP8=1 run_default_fa_lbl "mxfp8" 1 test_numerics.py -k "MXFP8BlockScaling and 126m and not grouped"
+    check_supported mxfp8 && NVTE_ROCM_ENABLE_MXFP8=1 run_default_fa_lbl "mxfp8" 1 test_numerics.py -k "MXFP8BlockScaling and 126m and not grouped"
     run_default_fa 1 test_nvfp4_fsdp2_hooks.py
     run_default_fa 1 test_permutation.py
     run_default_fa 1 test_recipe.py
@@ -130,7 +118,7 @@ run_test_config_mgpu(){
     #this test is not really mGPU but time sensitive so run it here because sGPU tests
     #run in parallel on CI and it affects timing
     run_default_fa 1 test_gemm_sm_count.py
-    run_default_fa 3 test_sanity_import.py
+    run_default_fa_lbl "mgpu" 3 test_sanity_import.py
     run_default_fa 3 distributed/test_cast_master_weights_to_fp8.py
     run_default_fa 3 distributed/test_comm_gemm_overlap.py
     run_default_fa 2 distributed/test_fusible_ops.py
@@ -173,17 +161,6 @@ if [ -n "$SINGLE_CONFIG" ]; then
     exit $?
 fi
 
-check_flash_attn_installed() {
-    check_list_all && return 0
-    _result=$(python -c "${PYTHON_TE_IMPORT}; from transformer_engine.pytorch.attention.dot_product_attention.utils import FlashAttentionUtils; print(FlashAttentionUtils.is_installed)" 2>/dev/null)
-    if [ "$_result" = "True" ]; then
-        return 0
-    else
-        echo "Flash attention is not installed" >&2
-        return 1
-    fi
-}
-
 #Master script mode: prepare testing prerequisites first
 start_message
 #Prerequisites and the CK JIT cache are container-wide state, so a scheduler
@@ -219,7 +196,7 @@ for _fus_attn in auto flash ck aotriton unfused; do
     fi
 
     if [ $_fus_attn = flash ]; then
-        check_flash_attn_installed || continue
+        check_supported flash_attn || continue
     fi
 
     if [ -n "$TEST_JOBS_MODE" ]; then

From d5a5ed86d999a027d7e69bb94c3d74599dca7a70 Mon Sep 17 00:00:00 2001
From: Veera Rajasekhar <VeeraRajasekharReddy.Gopu@amd.com>
Date: Fri, 7 Aug 2026 14:28:58 +0000
Subject: [PATCH 8/9] Deleted unwanted files

---
 ci/core_scaling.sh | 135 ---------------------------------------------
 1 file changed, 135 deletions(-)
 delete mode 100755 ci/core_scaling.sh

diff --git a/ci/core_scaling.sh b/ci/core_scaling.sh
deleted file mode 100755
index d21ca42713..0000000000
--- a/ci/core_scaling.sh
+++ /dev/null
@@ -1,135 +0,0 @@
-#!/bin/bash
-# Copyright (c) 2026, Advanced Micro Devices, Inc. All rights reserved.
-#
-# See LICENSE for license information.
-#
-# Measure how ci/core.sh scales with ctest job count and OMP thread count.
-#
-# ci/core.sh hardcodes n_parallel_jobs=4 and derives OMP_NUM_THREADS from it as
-# physical_cores/n_parallel_jobs. That 4 is a CPU-partitioning number -- it makes
-# jobs*threads equal the core count -- not a measured optimum; it was raised to 8
-# once (PR #426) and came back to 4 as part of an unrelated revert (PR #440),
-# never having been evaluated with core pinned to a single GPU. This script
-# answers the question that revert skipped:
-#
-#   A  current policy, OMP = cores/j: does raising j help under the coupling?
-#   B  decoupled, OMP pinned to 8:    is the coupling what limits j?
-#   C  j fixed at 4, OMP swept:       do these tests use OMP threads at all?
-#
-# The gtest suites are registered one-ctest-test-per-case (gtest_discover_tests),
-# so the workload is dominated by process startup rather than GPU work. If C
-# shows a flat curve, core's CPU budget is being spent on threads it does not use
-# instead of on concurrency it does.
-#
-# Usage:  [GPU=0] [FILTER='-E GEMMTestSuite'] ci/core_scaling.sh
-#
-#   GPU     device to pin every run to. Keep this set -- leaving all devices
-#           visible is what made PR #426 pile every job onto device 0.
-#   FILTER  ctest selection, passed through verbatim. Defaults to the non-GEMM
-#           half (the larger one, ~6.4k tests). Use '-R GEMMTestSuite' for the
-#           other half, or '' for everything.
-#
-# Run on an idle machine: a concurrent CI job invalidates every number here.
-# Results are appended to /tmp/core_scaling.tsv; per-config ctest output stays in
-# /tmp/core_run.<name>.log.
-
-set -u
-
-GPU=${GPU:-0}
-FILTER=${FILTER:--E GEMMTestSuite}
-OUT=${OUT:-/tmp/core_scaling.tsv}
-
-DIR=$(cd "$(dirname "$0")" && pwd)
-TEST_DIR=${TE_PATH:-$(cd "$DIR/.." && pwd)/}tests/cpp
-
-cd "$TEST_DIR" || { echo "cannot cd to $TEST_DIR" >&2; exit 1; }
-
-CORES=$(( $(lscpu | awk '/^CPU\(s\):/{print $2}') \
-        / $(lscpu | awk '/Thread\(s\) per core:/{print $NF}') ))
-
-# ---------------------------------------------------------------------------
-# Preflight. The previous version of this script reported 0.0s for every config
-# because ctest exited immediately and its output was discarded. Refuse to
-# produce a number unless tests were actually discovered.
-if [ ! -d build ]; then
-    echo "== configuring + building (one time, 10-20 min) =="
-    if ! cmake -GNinja -Bbuild . > /tmp/core_cfg.log 2>&1; then
-        tail -30 /tmp/core_cfg.log >&2; exit 1
-    fi
-    if ! cmake --build build >> /tmp/core_cfg.log 2>&1; then
-        tail -30 /tmp/core_cfg.log >&2; exit 1
-    fi
-fi
-
-NTESTS=$(HIP_VISIBLE_DEVICES=$GPU ctest --test-dir build -N $FILTER 2>/dev/null \
-         | awk '/Total Tests:/{print $3}')
-if [ -z "$NTESTS" ] || [ "$NTESTS" -eq 0 ]; then
-    echo "ABORT: ctest discovered no tests in $TEST_DIR/build" >&2
-    echo "A build configured for a different container path will do this;" >&2
-    echo "remove build/ and re-run to reconfigure. Raw ctest output:" >&2
-    HIP_VISIBLE_DEVICES=$GPU ctest --test-dir build -N $FILTER 2>&1 | tail -20 >&2
-    exit 1
-fi
-
-echo "cores=$CORES  gpu=$GPU  filter=${FILTER:-<all>}  discovered=$NTESTS tests"
-
-# ---------------------------------------------------------------------------
-run() {
-    _name=$1; _jobs=$2; _omp=$3
-    _log=/tmp/core_run.${_name}.log
-    _smi=/tmp/core_smi.${_name}
-
-    # Sample utilisation and VRAM alongside the run. Peak VRAM is the ceiling on
-    # how far j can be raised regardless of what the wall clock says.
-    ( while :; do
-          rocm-smi --showuse --showmeminfo vram --csv 2>/dev/null \
-              | awk -F, -v g="card$GPU" '$1 ~ g {print $2"\t"$4}'
-          sleep 2
-      done ) > "$_smi" 2>/dev/null &
-    _smi_pid=$!
-
-    _t0=$(date +%s)
-    OMP_NUM_THREADS=$_omp HIP_VISIBLE_DEVICES=$GPU \
-        ctest --test-dir build -j"$_jobs" --timeout 600 $FILTER > "$_log" 2>&1
-    _rc=$?
-    _t1=$(date +%s)
-    kill $_smi_pid 2>/dev/null
-    wait $_smi_pid 2>/dev/null
-
-    _secs=$((_t1 - _t0))
-    _passed=$(awk '/tests passed/{print $1}' "$_log" | tail -1)
-    _gpu=$(awk '{s+=$1; n++} END {if (n) printf "%.0f", s/n}' "$_smi")
-    _vram=$(awk '{if ($2 > m) m = $2} END {printf "%.0f", m/1048576}' "$_smi")
-
-    printf '%-9s j=%-3s omp=%-3s %6ss  rc=%-3s %-16s gpu=%3s%%  vram=%sMB\n' \
-        "$_name" "$_jobs" "$_omp" "$_secs" "$_rc" "${_passed:-NO-SUMMARY}" \
-        "$_gpu" "$_vram" | tee -a "$OUT"
-
-    # A config that finishes implausibly fast did not run the suite. Say so
-    # rather than folding a zero into the comparison.
-    if [ "$_secs" -lt 30 ]; then
-        echo "  !! suspiciously fast -- tail of $_log:" | tee -a "$OUT"
-        tail -5 "$_log" | sed 's/^/     /' | tee -a "$OUT"
-    fi
-}
-
-: > "$OUT"
-
-echo "=== A: current policy (OMP = cores/j) ===" | tee -a "$OUT"
-for j in 4 8 16 32; do
-    run "A_j$j" $j $((CORES / j))
-done
-
-echo "=== B: decoupled (OMP pinned to 8) ===" | tee -a "$OUT"
-for j in 8 16 32; do
-    run "B_j$j" $j 8
-done
-
-echo "=== C: is OMP used at all? (j=4) ===" | tee -a "$OUT"
-for o in 32 8 1; do
-    run "C_omp$o" 4 $o
-done
-
-echo
-echo "================ results: $OUT ================"
-cat "$OUT"

From e7c1d095e47291ba760a4b21ab286e41c8ef72e2 Mon Sep 17 00:00:00 2001
From: Veera Rajasekhar <VeeraRajasekharReddy.Gopu@amd.com>
Date: Fri, 7 Aug 2026 14:58:17 +0000
Subject: [PATCH 9/9] Refactor ci scripts

---
 ci/_utils.sh  | 14 ++++++++++++++
 ci/jax.sh     |  4 ++--
 ci/pytorch.sh |  4 ++--
 3 files changed, 18 insertions(+), 4 deletions(-)

diff --git a/ci/_utils.sh b/ci/_utils.sh
index b23ed48156..d01736f8ee 100644
--- a/ci/_utils.sh
+++ b/ci/_utils.sh
@@ -151,6 +151,15 @@ check_list_all() {
     test -n "$TE_CI_LIST_ONLY" -a -n "$TE_CI_LIST_ALL"
 }
 
+# Whether this invocation is the one that should do container-wide setup. The
+# scheduler hoists that out of the per-item runs -- once per suite with
+# TE_CI_SETUP_ONLY, then TE_CI_SKIP_SETUP on every item it dispatches -- and list
+# mode sets nothing up at all, since it only prints what it would have run. Every
+# setup step goes through here rather than spelling the condition out again.
+check_setup_needed() {
+    test -z "$TE_CI_SKIP_SETUP$TE_CI_LIST_ONLY"
+}
+
 # Every host capability probe goes through here, selected by label. Returns 0 to
 # run the guarded tests, 1 to skip them.
 check_supported() {
@@ -456,7 +465,12 @@ pytest_run() {
 }
 
 PYTHON_TE_IMPORT="import sys; sys.path[:] = [p for p in sys.path if p not in ['', '.']]; import transformer_engine"
+# Usage: ck_jit_prebuild build|list
+# Build the CK JIT blob cache, or report what is in it. Like check_supported, the
+# guard lives with the thing it guards: a dispatched item or a list run returns
+# here without touching the cache, so no caller has to remember to ask.
 ck_jit_prebuild() {
+    check_setup_needed || return 0
     _prebuild_list="${TE_PATH}ci/ck_jit_prebuild.txt"
     if [ ! -f "$_prebuild_list" ]; then
         script_error "ck_jit_prebuild: blob list not found: $_prebuild_list"
diff --git a/ci/jax.sh b/ci/jax.sh
index c0305ba4d8..403ef44e96 100755
--- a/ci/jax.sh
+++ b/ci/jax.sh
@@ -119,7 +119,7 @@ start_message
 #See ci/pytorch.sh for the TE_CI_SETUP_ONLY / TE_CI_SKIP_SETUP contract. Running
 #the CK JIT prebuild here once, sequentially, also removes the race this script
 #used to have with ci/pytorch.sh over the shared blob cache.
-if [ -z "$TE_CI_SKIP_SETUP$TE_CI_LIST_ONLY" ]; then
+if check_setup_needed; then
     install_prerequisites
     pip list | egrep "flax|fidle|jax|ml_dtypes|numpy|transformer_e|typing_ext"
     #check_test_jobs_requested
@@ -161,5 +161,5 @@ if [ -n "$TEST_JOBS_MODE" -a -n "$TEST_MGPU" ]; then
     done
 fi
 
-test -z "$TE_CI_SKIP_SETUP$TE_CI_LIST_ONLY" && ck_jit_prebuild list
+ck_jit_prebuild list
 return_run_results
diff --git a/ci/pytorch.sh b/ci/pytorch.sh
index 794297f5ef..2d40f5e5e9 100755
--- a/ci/pytorch.sh
+++ b/ci/pytorch.sh
@@ -166,7 +166,7 @@ start_message
 #Prerequisites and the CK JIT cache are container-wide state, so a scheduler
 #that dispatches many TEST_FILTER-ed invocations does them once up front with
 #TE_CI_SETUP_ONLY and then passes TE_CI_SKIP_SETUP on every item.
-if [ -z "$TE_CI_SKIP_SETUP$TE_CI_LIST_ONLY" ]; then
+if check_setup_needed; then
     install_prerequisites
     pip list | egrep "flash|ml_dtypes|numpy|torch|transformer_e|typing_ext"
     #check_test_jobs_requested && init_test_jobs `python -c "import torch; print(torch.cuda.device_count())"`
@@ -221,5 +221,5 @@ if [ $TEST_LEVEL -ge 3 ]; then
     fi
 fi
 
-test -z "$TE_CI_SKIP_SETUP$TE_CI_LIST_ONLY" && ck_jit_prebuild list
+ck_jit_prebuild list
 return_run_results