From 834fe82a4302e7ec27c9230bb4caf9a352db56d4 Mon Sep 17 00:00:00 2001
From: root <root@slurm-sa-mi325-controller-02.ord.vultr.cpe.ice.amd.com>
Date: Tue, 31 Mar 2026 17:05:53 +0000
Subject: [PATCH 1/9] Add MI325X DeepSeek-R1 FP8 disaggregated inference (1P1D,
 Broadcom Thor 2 IBGDA)

Port the MI355X disagg recipe to MI325X (gfx942/CDNA3) on a Vultr Slurm cluster
with Broadcom BCM5760X Thor 2 NICs using IBGDA for GPU-Direct RDMA via MoRI.

Container image: ghcr.io/jordannanos/sgl-mi325x-mori:v0.5.9-bnxt
Built from akao-amd/sglang rocm.Dockerfile with:
  - GPU_ARCH=gfx942, ENABLE_MORI=1, NIC_BACKEND=ibgda
  - Broadcom bnxt_rocelib (bcm5760x_231.2.63.0a) for RDMA userspace
  - MoRI pinned to HEAD (c0eccaf2) for bundled bnxt headers + dlopen
  - smg-wasm pinned to =1.0.0 (v1.0.1 breaks sgl-model-gateway v0.5.9 API)

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
---
 .github/configs/amd-master.yaml               | 149 +++++++++++++
 .github/configs/runners.yaml                  |   5 +
 benchmarks/multi_node/amd_utils/env.sh        |   3 +
 benchmarks/multi_node/amd_utils/job.slurm     |  20 +-
 benchmarks/multi_node/amd_utils/server.sh     |   3 +-
 .../dsr1_fp8_mi325x_sglang-disagg.sh          |  82 +++++++
 runners/launch_mi325x-amd.sh                  | 206 +++++++++++++++---
 scripts/manual-test-mi325x.sh                 |  37 ++++
 8 files changed, 467 insertions(+), 38 deletions(-)
 create mode 100755 benchmarks/multi_node/dsr1_fp8_mi325x_sglang-disagg.sh
 create mode 100755 scripts/manual-test-mi325x.sh

diff --git a/.github/configs/amd-master.yaml b/.github/configs/amd-master.yaml
index e84fc0da5..6da2a4e22 100644
--- a/.github/configs/amd-master.yaml
+++ b/.github/configs/amd-master.yaml
@@ -1231,3 +1231,152 @@ dsr1-fp4-mi355x-sglang-disagg-mtp:
         - "DECODE_NODES=1"
         - "DECODE_MTP_SIZE=1"
 
+
+dsr1-fp8-mi325x-sglang-disagg:
+  image: ghcr.io/jordannanos/sgl-mi325x-mori:v0.5.9-bnxt
+  model: deepseek-ai/DeepSeek-R1-0528
+  model-prefix: dsr1
+  runner: mi325x-disagg
+  precision: fp8
+  framework: sglang-disagg
+  multinode: true
+  disagg: true
+  seq-len-configs:
+  - isl: 1024
+    osl: 1024
+    search-space:
+    # "Top of curve" (1 prefill worker at TP8, 1 decode worker at DEP8)
+    - spec-decoding: "none"
+      conc-list: [ 512, 1024 ]
+      prefill:
+        num-worker: 1
+        tp: 8
+        ep: 1
+        dp-attn: false
+        additional-settings:
+        - "PREFILL_NODES=1"
+      decode:
+        num-worker: 1
+        tp: 8
+        ep: 8
+        dp-attn: true
+        additional-settings:
+        - "DECODE_NODES=2"
+        - "DECODE_MTP_SIZE=0"
+
+    # "Middle of curve" (1 prefill worker at TP8, 2 decode workers at DEP8)
+    - spec-decoding: "none"
+      conc-list: [ 768, 512, 256 ]
+      prefill:
+        num-worker: 1
+        tp: 8
+        ep: 1
+        dp-attn: false
+        additional-settings:
+        - "PREFILL_NODES=1"
+      decode:
+        num-worker: 2
+        tp: 8
+        ep: 8
+        dp-attn: true
+        additional-settings:
+        - "DECODE_NODES=2"
+        - "DECODE_MTP_SIZE=0"
+
+    # "Bottom of curve" (1 prefill worker at TP8, 2 decode workers at TP8)
+    - spec-decoding: "none"
+      conc-list: [ 256, 128, 64, 32, 16, 8, 4 ]
+      prefill:
+        num-worker: 1
+        tp: 8
+        ep: 1
+        dp-attn: false
+        additional-settings:
+        - "PREFILL_NODES=1"
+      decode:
+        num-worker: 2
+        tp: 8
+        ep: 1
+        dp-attn: false
+        additional-settings:
+        - "DECODE_NODES=2"
+        - "DECODE_MTP_SIZE=0"
+
+    # "Low concurrency" (1 prefill worker at TP4, 1 decode worker at TP8)
+    - spec-decoding: "none"
+      conc-list: [ 64, 32, 16, 8, 4, 2, 1 ]
+      prefill:
+        num-worker: 1
+        tp: 4
+        ep: 1
+        dp-attn: false
+        additional-settings:
+        - "PREFILL_NODES=1"
+      decode:
+        num-worker: 1
+        tp: 8
+        ep: 1
+        dp-attn: false
+        additional-settings:
+        - "DECODE_NODES=1"
+        - "DECODE_MTP_SIZE=0"
+
+  - isl: 8192
+    osl: 1024
+    search-space:
+    # "Top of curve" (2 prefill workers at DEP8, 1 decode worker at DEP8)
+    - spec-decoding: "none"
+      conc-list: [ 512, 1024 ]
+      prefill:
+        num-worker: 2
+        tp: 8
+        ep: 8
+        dp-attn: true
+        additional-settings:
+        - "PREFILL_NODES=2"
+      decode:
+        num-worker: 1
+        tp: 8
+        ep: 8
+        dp-attn: true
+        additional-settings:
+        - "DECODE_NODES=1"
+        - "DECODE_MTP_SIZE=0"
+
+    # "Bottom of curve" (1 prefill worker at TP8, 2 decode workers at TP8)
+    - spec-decoding: "none"
+      conc-list: [ 256, 128, 64, 32, 16, 8, 4 ]
+      prefill:
+        num-worker: 1
+        tp: 8
+        ep: 1
+        dp-attn: false
+        additional-settings:
+        - "PREFILL_NODES=1"
+      decode:
+        num-worker: 2
+        tp: 8
+        ep: 1
+        dp-attn: false
+        additional-settings:
+        - "DECODE_NODES=2"
+        - "DECODE_MTP_SIZE=0"
+
+    # "Low concurrency" (1 prefill worker at TP4, 1 decode worker at TP8)
+    - spec-decoding: "none"
+      conc-list: [ 64, 32, 16, 8, 4, 2, 1 ]
+      prefill:
+        num-worker: 1
+        tp: 4
+        ep: 1
+        dp-attn: false
+        additional-settings:
+        - "PREFILL_NODES=1"
+      decode:
+        num-worker: 1
+        tp: 8
+        ep: 1
+        dp-attn: false
+        additional-settings:
+        - "DECODE_NODES=1"
+        - "DECODE_MTP_SIZE=0"
diff --git a/.github/configs/runners.yaml b/.github/configs/runners.yaml
index 1251e459d..f61e81e36 100644
--- a/.github/configs/runners.yaml
+++ b/.github/configs/runners.yaml
@@ -75,6 +75,11 @@ mi325x:
 - 'mi325x-amd_1'
 - 'mi325x-amd_2'
 - 'mi325x-amd_3'
+mi325x-disagg:
+- 'mi325x-amd_0'
+- 'mi325x-amd_1'
+- 'mi325x-amd_2'
+- 'mi325x-amd_3'
 mi355x:
 - 'mi355x-amds_0'
 - 'mi355x-amds_1'
diff --git a/benchmarks/multi_node/amd_utils/env.sh b/benchmarks/multi_node/amd_utils/env.sh
index 5565c5b3b..56572dfcf 100755
--- a/benchmarks/multi_node/amd_utils/env.sh
+++ b/benchmarks/multi_node/amd_utils/env.sh
@@ -20,6 +20,9 @@ if [[ -z "$IBDEVICES" ]]; then
         export IBDEVICES=ionic_0,ionic_1,ionic_2,ionic_3,ionic_4,ionic_5,ionic_6,ionic_7
     elif [[ $NODENAME == mia1* ]]; then
         export IBDEVICES=rdma0,rdma1,rdma2,rdma3,rdma4,rdma5,rdma6,rdma7
+    elif [[ $NODENAME == chi-mi325x* ]]; then
+        # Vultr/CPE MI325X cluster: Broadcom RoCE (bnxt_re); bnxt_re6 is DOWN, skip it
+        export IBDEVICES=bnxt_re0,bnxt_re1,bnxt_re2,bnxt_re3,bnxt_re4,bnxt_re5,bnxt_re7,bnxt_re8
     else
         echo "ERROR: Unable to detect cluster from hostname $NODENAME and IBDEVICES not set" >&2
         exit 1
diff --git a/benchmarks/multi_node/amd_utils/job.slurm b/benchmarks/multi_node/amd_utils/job.slurm
index 6b0352f24..0e8f465f5 100755
--- a/benchmarks/multi_node/amd_utils/job.slurm
+++ b/benchmarks/multi_node/amd_utils/job.slurm
@@ -30,14 +30,18 @@ if [[ ! -f "$MODELS_YAML" ]]; then
     exit 1
 fi
 
-# Validate MODEL_NAME exists as a top-level key in models.yaml
-if ! grep -q "^${MODEL_NAME}:" "$MODELS_YAML"; then
-    echo "Error: Model '$MODEL_NAME' not found in models.yaml"
+# MODEL_YAML_KEY is the models.yaml lookup key (bare model name, e.g. DeepSeek-R1-0528).
+# MODEL_NAME may be a longer HF cache path (e.g. models--org--repo/snapshots/<hash>).
+_MODEL_YAML_KEY="${MODEL_YAML_KEY:-$MODEL_NAME}"
+
+# Validate the yaml key exists as a top-level key in models.yaml
+if ! grep -q "^${_MODEL_YAML_KEY}:" "$MODELS_YAML"; then
+    echo "Error: Model '$_MODEL_YAML_KEY' not found in models.yaml"
     echo "Available models:"
     grep -E '^[A-Za-z]' "$MODELS_YAML" | sed 's/:.*$//' | sed 's/^/  - /'
     exit 1
 fi
-echo "Model found: $MODEL_NAME"
+echo "Model found: $_MODEL_YAML_KEY"
 
 # All models use server.sh as the entrypoint
 RUN_FILE="server.sh"
@@ -249,10 +253,9 @@ echo "NNODES is ${NNODES}"
 echo "REPO Directory is ${DI_REPO_DIR}"
 echo "USER_NAME is ${USER_NAME}"
 
-# Get the RDMA priority and DSCP value from the NIC
+# Get the RDMA priority and DSCP value from the NIC (optional - env.sh handles absence gracefully)
 if ! command -v nicctl >/dev/null 2>&1; then
-    echo "Error: nicctl command not found. Please ensure nicctl is installed and available." >&2
-    exit 1
+    echo "[INFO] nicctl not found. RDMA QoS configuration will be skipped inside the container." >&2
 fi
 
 # Reduce log spam
@@ -357,7 +360,7 @@ exec sudo docker run --rm \
     --privileged \
     -v ${MODEL_DIR}:/models \
     -v \$HOME/.ssh:/root/.ssh \
-    -v $(which nicctl):/usr/sbin/nicctl \
+    $(command -v nicctl &>/dev/null && echo "-v $(which nicctl):/usr/sbin/nicctl") \
     --shm-size 128G \
     -v /tmp:/run_logs \
     -v ${BENCHMARK_LOGS_DIR}:/benchmark_logs \
@@ -373,6 +376,7 @@ exec sudo docker run --rm \
     -e xP=\$xP \
     -e yD=\$yD \
     -e MODEL_NAME=\$MODEL_NAME \
+    -e MODEL_YAML_KEY=${_MODEL_YAML_KEY} \
     -e IPADDRS=\$IPADDRS \
     -e PREFILL_TP_SIZE=\$PREFILL_TP_SIZE \
     -e PREFILL_ENABLE_EP=\$PREFILL_ENABLE_EP \
diff --git a/benchmarks/multi_node/amd_utils/server.sh b/benchmarks/multi_node/amd_utils/server.sh
index 7f174b760..b477790b3 100755
--- a/benchmarks/multi_node/amd_utils/server.sh
+++ b/benchmarks/multi_node/amd_utils/server.sh
@@ -72,11 +72,12 @@ fi
 # Load model config via inline Python (PyYAML is available in SGLang containers)
 # Formula evaluation (e.g. "SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK * TP * xP")
 # is done here in Python to avoid bash glob-expanding the * characters.
+_MODEL_YAML_KEY="${MODEL_YAML_KEY:-$MODEL_NAME}"
 eval "$(python3 -c "
 import yaml, sys, os
 
 config_path = '${MODELS_YAML}'
-model_name = '${MODEL_NAME}'
+model_name = '${_MODEL_YAML_KEY}'
 
 with open(config_path) as f:
     models = yaml.safe_load(f)
diff --git a/benchmarks/multi_node/dsr1_fp8_mi325x_sglang-disagg.sh b/benchmarks/multi_node/dsr1_fp8_mi325x_sglang-disagg.sh
new file mode 100755
index 000000000..6a7314ab4
--- /dev/null
+++ b/benchmarks/multi_node/dsr1_fp8_mi325x_sglang-disagg.sh
@@ -0,0 +1,82 @@
+#!/usr/bin/env bash
+
+source "$(dirname "$0")/../benchmark_lib.sh"
+
+check_env_vars \
+    CONC_LIST \
+    ISL \
+    OSL \
+    IMAGE \
+    SPEC_DECODING \
+    MODEL_PATH \
+    PREFILL_NUM_WORKERS \
+    PREFILL_TP \
+    PREFILL_EP \
+    PREFILL_DP_ATTN \
+    DECODE_NUM_WORKERS \
+    DECODE_TP \
+    DECODE_EP \
+    DECODE_DP_ATTN \
+    PREFILL_NODES \
+    DECODE_NODES \
+    RANDOM_RANGE_RATIO
+
+if [[ -n "$SLURM_JOB_ID" ]]; then
+  echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME"
+fi
+
+set -x
+
+# Use upstreamed multi_node scripts (no external clone needed)
+cd "$GITHUB_WORKSPACE/benchmarks/multi_node/amd_utils" || exit 1
+
+# Set up SGL launch script-specific environment variables
+export TIME_LIMIT="08:00:00"
+export MODEL_PATH=$MODEL_PATH
+export MODEL_NAME=$MODEL_NAME
+export CONTAINER_IMAGE=$IMAGE
+
+if [[ "${PREFILL_EP:-1}" -eq 1 ]]; then
+export PREFILL_ENABLE_EP=false
+else
+export PREFILL_ENABLE_EP=true
+fi
+
+if [[ "$PREFILL_DP_ATTN" == "true" ]]; then
+export PREFILL_ENABLE_DP=true
+else
+export PREFILL_ENABLE_DP=false
+fi
+
+if [[ "${DECODE_EP:-1}" -eq 1 ]]; then
+export DECODE_ENABLE_EP=false
+else
+export DECODE_ENABLE_EP=true
+fi
+
+if [[ "$DECODE_DP_ATTN" == "true" ]]; then
+export DECODE_ENABLE_DP=true
+else
+export DECODE_ENABLE_DP=false
+fi
+
+# Launch jobs based on ISL/OSL
+# Replace ' ' in CONC_LIST with 'x' such that the concurrency list is represented
+# by a list of numbers delimited by 'x'. This is because of how the underlying launch script
+# expects the concurrencies.
+JOB_ID=$(bash ./submit.sh $PREFILL_NODES \
+    $PREFILL_NUM_WORKERS \
+    $DECODE_NODES \
+    $DECODE_NUM_WORKERS \
+    $ISL $OSL "${CONC_LIST// /x}" inf \
+    ${PREFILL_ENABLE_EP} ${PREFILL_ENABLE_DP} \
+    ${DECODE_ENABLE_EP} ${DECODE_ENABLE_DP} \
+    ${PREFILL_TP} ${DECODE_TP} \
+    ${RANDOM_RANGE_RATIO})
+
+if [[ $? -ne 0 ]]; then
+    echo "Failed to submit job" >&2
+    exit 1
+fi
+
+echo "$JOB_ID"
diff --git a/runners/launch_mi325x-amd.sh b/runners/launch_mi325x-amd.sh
index 67f93a309..4e76c205a 100644
--- a/runners/launch_mi325x-amd.sh
+++ b/runners/launch_mi325x-amd.sh
@@ -4,37 +4,185 @@ export HF_HUB_CACHE_MOUNT="/nfsdata/sa/gharunner/gharunners/hf-hub-cache/"
 export PORT=8888
 
 PARTITION="compute"
-SQUASH_FILE="/nfsdata/sa/gharunner/gharunners/squash/$(echo "$IMAGE" | sed 's/[\/:@#]/_/g').sqsh"
-LOCK_FILE="${SQUASH_FILE}.lock"
 
-set -x
-
-JOB_ID=$(salloc --partition=$PARTITION --gres=gpu:$TP --cpus-per-task=256 --time=480 --no-shell --job-name="$RUNNER_NAME" 2>&1 | tee /dev/stderr | grep -oP 'Granted job allocation \K[0-9]+')
-
-if [ -z "$JOB_ID" ]; then
-    echo "ERROR: salloc failed to allocate a job"
+# Detect benchmark subdir from where the script lives
+SCRIPT_NAME="${EXP_NAME%%_*}_${PRECISION}_mi325x_${FRAMEWORK}.sh"
+if [[ -f "benchmarks/multi_node/${SCRIPT_NAME}" ]]; then
+    BENCHMARK_SUBDIR="multi_node"
+elif [[ -f "benchmarks/single_node/${SCRIPT_NAME}" ]]; then
+    BENCHMARK_SUBDIR="single_node"
+else
+    echo "ERROR: ${SCRIPT_NAME} not found in benchmarks/multi_node or benchmarks/single_node"
     exit 1
 fi
 
-# Use flock to serialize concurrent imports to the same squash file
-srun --jobid=$JOB_ID --job-name="$RUNNER_NAME" bash -c "
-    exec 9>\"$LOCK_FILE\"
-    flock -w 600 9 || { echo 'Failed to acquire lock for $SQUASH_FILE'; exit 1; }
-    if unsquashfs -l \"$SQUASH_FILE\" > /dev/null 2>&1; then
-        echo 'Squash file already exists and is valid, skipping import'
-    else
-        rm -f \"$SQUASH_FILE\"
-        enroot import -o \"$SQUASH_FILE\" docker://$IMAGE
+# =============================================================================
+# Multi-node disaggregated path: sbatch + Docker via submit.sh
+# =============================================================================
+if [[ "$BENCHMARK_SUBDIR" == "multi_node" ]]; then
+
+    scancel_sync() {
+        local jobid=$1
+        local timeout=${2:-600}
+        local interval=10
+        local start
+        start=$(date +%s)
+
+        echo "[scancel_sync] Requesting cancel of job $jobid"
+        scancel "$jobid" || true
+
+        while [[ -n "$(squeue -j "$jobid" --noheader 2>/dev/null)" ]]; do
+            local now
+            now=$(date +%s)
+            if (( now - start >= timeout )); then
+                echo "[scancel_sync][WARN] job $jobid still present after ${timeout}s"
+                return 1
+            fi
+            echo "[scancel_sync] waiting for job $jobid to exit. $((timeout-(now-start))) secs remaining..."
+            sleep "$interval"
+        done
+        echo "[scancel_sync] job $jobid exited"
+        return 0
+    }
+
+    set -x
+
+    export SLURM_ACCOUNT="$USER"
+    export SLURM_PARTITION="$PARTITION"
+    export SLURM_JOB_NAME="benchmark-sglang-disagg.job"
+
+    export MODEL_PATH="${HF_HUB_CACHE_MOUNT%/}"
+
+    # MODEL_YAML_KEY: top-level key in models.yaml for server config lookup.
+    if [[ -z "${MODEL_YAML_KEY:-}" ]]; then
+        export MODEL_YAML_KEY="${MODEL##*/}"
+    fi
+
+    # MODEL_NAME: relative path under MODEL_PATH for --model-path inside the container.
+    # Auto-resolved from HF hub cache layout so no symlink is needed.
+    if [[ -z "${MODEL_NAME:-}" ]]; then
+        _HF_DIR="models--$(echo "${MODEL}" | tr '/' '--')"
+        _SNAPSHOT=$(ls "${MODEL_PATH}/${_HF_DIR}/snapshots/" 2>/dev/null | sort | tail -1)
+        if [[ -n "${_SNAPSHOT}" ]]; then
+            export MODEL_NAME="${_HF_DIR}/snapshots/${_SNAPSHOT}"
+        else
+            export MODEL_NAME="${MODEL_YAML_KEY}"
+        fi
     fi
-"
-srun --jobid=$JOB_ID \
---container-image=$SQUASH_FILE \
---container-mounts=$GITHUB_WORKSPACE:/workspace/,$HF_HUB_CACHE_MOUNT:$HF_HUB_CACHE \
---container-mount-home \
---container-writable \
---container-remap-root \
---container-workdir=/workspace/ \
---no-container-entrypoint --export=ALL \
-bash benchmarks/single_node/${EXP_NAME%%_*}_${PRECISION}_mi325x.sh
-
-scancel $JOB_ID
+
+    export GPUS_PER_NODE=8
+
+    export BENCHMARK_LOGS_DIR="${BENCHMARK_LOGS_DIR:-$GITHUB_WORKSPACE/benchmark_logs}"
+    mkdir -p "$BENCHMARK_LOGS_DIR"
+    sudo rm -rf "$BENCHMARK_LOGS_DIR/logs" 2>/dev/null || true
+
+    JOB_ID=$(bash "benchmarks/${BENCHMARK_SUBDIR}/${SCRIPT_NAME}")
+
+    LOG_FILE="$BENCHMARK_LOGS_DIR/slurm_job-${JOB_ID}.out"
+
+    sleep 10
+
+    while ! ls "$LOG_FILE" &>/dev/null; do
+        if ! squeue -u "$USER" --noheader --format='%i' | grep -q "$JOB_ID"; then
+            echo "ERROR: Job $JOB_ID failed before creating log file"
+            scontrol show job "$JOB_ID"
+            exit 1
+        fi
+        sleep 5
+    done
+
+    set +x
+
+    (
+        while squeue -u $USER --noheader --format='%i' | grep -q "$JOB_ID"; do
+            sleep 10
+        done
+    ) &
+    POLL_PID=$!
+
+    tail -F -s 2 -n+1 "$LOG_FILE" --pid=$POLL_PID 2>/dev/null
+
+    wait $POLL_PID
+
+    set -x
+
+    cat > collect_latest_results.py <<'PY'
+import os, sys
+sgl_job_dir, isl, osl, nexp = sys.argv[1], int(sys.argv[2]), int(sys.argv[3]), int(sys.argv[4])
+for path in sorted([f"{sgl_job_dir}/logs/{name}/sglang_isl_{isl}_osl_{osl}" for name in os.listdir(f"{sgl_job_dir}/logs/") if os.path.isdir(f"{sgl_job_dir}/logs/{name}/sglang_isl_{isl}_osl_{osl}")], key=os.path.getmtime, reverse=True)[:nexp]:
+    print(path)
+PY
+
+    LOGS_DIR=$(python3 collect_latest_results.py "$BENCHMARK_LOGS_DIR" "$ISL" "$OSL" 1)
+    if [ -z "$LOGS_DIR" ]; then
+        echo "No logs directory found for ISL=${ISL}, OSL=${OSL}"
+        exit 1
+    fi
+
+    echo "Found logs directory: $LOGS_DIR"
+    ls -la "$LOGS_DIR"
+
+    for result_file in $(find $LOGS_DIR -type f); do
+        file_name=$(basename $result_file)
+        if [ -f $result_file ]; then
+            WORKSPACE_RESULT_FILE="$GITHUB_WORKSPACE/${RESULT_FILENAME}_${file_name}"
+            echo "Found result file ${result_file}. Copying it to ${WORKSPACE_RESULT_FILE}"
+            cp $result_file $WORKSPACE_RESULT_FILE
+        fi
+    done
+
+    echo "All result files processed"
+    set +x
+    scancel_sync $JOB_ID
+    set -x
+    echo "Canceled the slurm job $JOB_ID"
+
+    sudo rm -rf "$BENCHMARK_LOGS_DIR/logs" 2>/dev/null || true
+
+    if [[ -n "${GITHUB_ACTIONS:-}" ]]; then
+        ARTIFACT_DIR="$GITHUB_WORKSPACE/benchmark_artifacts"
+        mkdir -p "$ARTIFACT_DIR"
+        cp -r "$BENCHMARK_LOGS_DIR"/slurm_job-${JOB_ID}.{out,err} "$ARTIFACT_DIR/" 2>/dev/null || true
+        echo "Logs copied to $ARTIFACT_DIR for artifact upload"
+    fi
+
+# =============================================================================
+# Single-node path: enroot via salloc + srun
+# =============================================================================
+else
+
+    SQUASH_FILE="/nfsdata/sa/gharunner/gharunners/squash/$(echo "$IMAGE" | sed 's/[\/:@#]/_/g').sqsh"
+    LOCK_FILE="${SQUASH_FILE}.lock"
+
+    set -x
+
+    JOB_ID=$(salloc --partition=$PARTITION --gres=gpu:$TP --cpus-per-task=256 --time=480 --no-shell --job-name="$RUNNER_NAME" 2>&1 | tee /dev/stderr | grep -oP 'Granted job allocation \K[0-9]+')
+
+    if [ -z "$JOB_ID" ]; then
+        echo "ERROR: salloc failed to allocate a job"
+        exit 1
+    fi
+
+    srun --jobid=$JOB_ID --job-name="$RUNNER_NAME" bash -c "
+        exec 9>\"$LOCK_FILE\"
+        flock -w 600 9 || { echo 'Failed to acquire lock for $SQUASH_FILE'; exit 1; }
+        if unsquashfs -l \"$SQUASH_FILE\" > /dev/null 2>&1; then
+            echo 'Squash file already exists and is valid, skipping import'
+        else
+            rm -f \"$SQUASH_FILE\"
+            enroot import -o \"$SQUASH_FILE\" docker://$IMAGE
+        fi
+    "
+    srun --jobid=$JOB_ID \
+    --container-image=$SQUASH_FILE \
+    --container-mounts=$GITHUB_WORKSPACE:/workspace/,$HF_HUB_CACHE_MOUNT:$HF_HUB_CACHE \
+    --container-mount-home \
+    --container-writable \
+    --container-remap-root \
+    --container-workdir=/workspace/ \
+    --no-container-entrypoint --export=ALL \
+    bash benchmarks/single_node/${SCRIPT_NAME}
+
+    scancel $JOB_ID
+
+fi
diff --git a/scripts/manual-test-mi325x.sh b/scripts/manual-test-mi325x.sh
new file mode 100755
index 000000000..c232ded2a
--- /dev/null
+++ b/scripts/manual-test-mi325x.sh
@@ -0,0 +1,37 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+cd "$(dirname "${BASH_SOURCE[0]}")/.."
+
+export GITHUB_WORKSPACE=$(pwd)
+export RUNNER_NAME=mi325x-amd-manual
+
+export MODEL=deepseek-ai/DeepSeek-R1-0528
+export EXP_NAME=dsr1_1k1k
+export PRECISION=fp8
+export FRAMEWORK=sglang-disagg
+
+export IMAGE=ghcr.io/jordannanos/sgl-mi325x-mori:v0.5.9-bnxt-good
+
+export ISL=1024
+export OSL=1024
+export CONC_LIST="4 2 1"
+export SPEC_DECODING=none
+export RANDOM_RANGE_RATIO=1
+
+export PREFILL_NODES=1
+export PREFILL_NUM_WORKERS=1
+export PREFILL_TP=4
+export PREFILL_EP=1
+export PREFILL_DP_ATTN=false
+
+export DECODE_NODES=1
+export DECODE_NUM_WORKERS=1
+export DECODE_TP=8
+export DECODE_EP=1
+export DECODE_DP_ATTN=false
+
+bash runners/launch_mi325x-amd.sh
+
+#model files are here:
+#/nfsdata/sa/gharunner/gharunners/hf-hub-cache/models--deepseek-ai--DeepSeek-R1-0528
\ No newline at end of file

From 7b5047673f6e33a310754b85175f665fd9d5f08f Mon Sep 17 00:00:00 2001
From: Jordan Nanos <jordan.nanos@gmail.com>
Date: Tue, 31 Mar 2026 10:36:30 -0700
Subject: [PATCH 2/9] Update amd-master.yaml

---
 .github/configs/amd-master.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/configs/amd-master.yaml b/.github/configs/amd-master.yaml
index 6da2a4e22..815023c55 100644
--- a/.github/configs/amd-master.yaml
+++ b/.github/configs/amd-master.yaml
@@ -1233,7 +1233,7 @@ dsr1-fp4-mi355x-sglang-disagg-mtp:
 
 
 dsr1-fp8-mi325x-sglang-disagg:
-  image: ghcr.io/jordannanos/sgl-mi325x-mori:v0.5.9-bnxt
+  image: ghcr.io/jordannanos/sgl-mi325x-mori:v0.5.9-bnxt-good
   model: deepseek-ai/DeepSeek-R1-0528
   model-prefix: dsr1
   runner: mi325x-disagg

From b40908ca814db79aea0248c641b0ee359e09c762 Mon Sep 17 00:00:00 2001
From: root <root@slurm-sa-mi325-controller-02.ord.vultr.cpe.ice.amd.com>
Date: Tue, 31 Mar 2026 17:50:20 +0000
Subject: [PATCH 3/9] Add MTP config, expand sweep to full pareto frontier, use
 -good image

- Add dsr1-fp8-mi325x-sglang-disagg-mtp config with MTP=1/2 across
  all curve points (top/middle/bottom/low-conc) for both 1k/1k and 8k/1k
- Expand concurrency lists to cover full pareto frontier including
  non-optimal points
- Update image tag to v0.5.9-bnxt-good (the pushed image)

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
---
 .github/configs/amd-master.yaml | 152 ++++++++++++++++++++++++++++++++
 scripts/manual-test-mi325x.sh   |   2 +-
 2 files changed, 153 insertions(+), 1 deletion(-)

diff --git a/.github/configs/amd-master.yaml b/.github/configs/amd-master.yaml
index 815023c55..00b6a26de 100644
--- a/.github/configs/amd-master.yaml
+++ b/.github/configs/amd-master.yaml
@@ -1380,3 +1380,155 @@ dsr1-fp8-mi325x-sglang-disagg:
         additional-settings:
         - "DECODE_NODES=1"
         - "DECODE_MTP_SIZE=0"
+
+
+dsr1-fp8-mi325x-sglang-disagg-mtp:
+  image: ghcr.io/jordannanos/sgl-mi325x-mori:v0.5.9-bnxt-good
+  model: deepseek-ai/DeepSeek-R1-0528
+  model-prefix: dsr1
+  runner: mi325x-disagg
+  precision: fp8
+  framework: sglang-disagg
+  multinode: true
+  disagg: true
+  seq-len-configs:
+  - isl: 1024
+    osl: 1024
+    search-space:
+    # MTP configurations
+    # "Top of curve" (1 prefill worker at TP8, 1 decode worker at DEP8)
+    - spec-decoding: "mtp"
+      conc-list: [ 512, 1024 ]
+      prefill:
+        num-worker: 1
+        tp: 8
+        ep: 1
+        dp-attn: false
+        additional-settings:
+        - "PREFILL_NODES=1"
+      decode:
+        num-worker: 1
+        tp: 8
+        ep: 8
+        dp-attn: true
+        additional-settings:
+        - "DECODE_NODES=2"
+        - "DECODE_MTP_SIZE=1"
+
+    # "Middle of curve" (1 prefill worker at TP8, 2 decode workers at DEP8)
+    - spec-decoding: "mtp"
+      conc-list: [ 768, 512, 256 ]
+      prefill:
+        num-worker: 1
+        tp: 8
+        ep: 1
+        dp-attn: false
+        additional-settings:
+        - "PREFILL_NODES=1"
+      decode:
+        num-worker: 2
+        tp: 8
+        ep: 8
+        dp-attn: true
+        additional-settings:
+        - "DECODE_NODES=2"
+        - "DECODE_MTP_SIZE=1"
+
+    # "Bottom of curve" (1 prefill worker at TP8, 2 decode workers at TP8)
+    - spec-decoding: "mtp"
+      conc-list: [ 256, 128, 64, 32, 16, 8, 4 ]
+      prefill:
+        num-worker: 1
+        tp: 8
+        ep: 1
+        dp-attn: false
+        additional-settings:
+        - "PREFILL_NODES=1"
+      decode:
+        num-worker: 2
+        tp: 8
+        ep: 1
+        dp-attn: false
+        additional-settings:
+        - "DECODE_NODES=2"
+        - "DECODE_MTP_SIZE=2"
+
+    # "Low concurrency" (1 prefill worker at TP4, 1 decode worker at TP8)
+    - spec-decoding: "mtp"
+      conc-list: [ 64, 32, 16, 8, 4, 2, 1 ]
+      prefill:
+        num-worker: 1
+        tp: 4
+        ep: 1
+        dp-attn: false
+        additional-settings:
+        - "PREFILL_NODES=1"
+      decode:
+        num-worker: 1
+        tp: 8
+        ep: 1
+        dp-attn: false
+        additional-settings:
+        - "DECODE_NODES=1"
+        - "DECODE_MTP_SIZE=2"
+
+  - isl: 8192
+    osl: 1024
+    search-space:
+    # MTP configurations
+    # "Top of curve" (2 prefill workers at DEP8, 1 decode worker at DEP8)
+    - spec-decoding: "mtp"
+      conc-list: [ 512, 1024 ]
+      prefill:
+        num-worker: 2
+        tp: 8
+        ep: 8
+        dp-attn: true
+        additional-settings:
+        - "PREFILL_NODES=2"
+      decode:
+        num-worker: 1
+        tp: 8
+        ep: 8
+        dp-attn: true
+        additional-settings:
+        - "DECODE_NODES=1"
+        - "DECODE_MTP_SIZE=1"
+
+    # "Bottom of curve" (1 prefill worker at TP8, 2 decode workers at TP8)
+    - spec-decoding: "mtp"
+      conc-list: [ 256, 128, 64, 32, 16, 8, 4, 2 ]
+      prefill:
+        num-worker: 1
+        tp: 8
+        ep: 1
+        dp-attn: false
+        additional-settings:
+        - "PREFILL_NODES=1"
+      decode:
+        num-worker: 2
+        tp: 8
+        ep: 1
+        dp-attn: false
+        additional-settings:
+        - "DECODE_NODES=2"
+        - "DECODE_MTP_SIZE=2"
+
+    # "Low concurrency" (1 prefill worker at TP4, 1 decode worker at TP8)
+    - spec-decoding: "mtp"
+      conc-list: [ 64, 32, 16, 8, 4, 2, 1 ]
+      prefill:
+        num-worker: 1
+        tp: 4
+        ep: 1
+        dp-attn: false
+        additional-settings:
+        - "PREFILL_NODES=1"
+      decode:
+        num-worker: 1
+        tp: 8
+        ep: 1
+        dp-attn: false
+        additional-settings:
+        - "DECODE_NODES=1"
+        - "DECODE_MTP_SIZE=2"
diff --git a/scripts/manual-test-mi325x.sh b/scripts/manual-test-mi325x.sh
index c232ded2a..30ec87d6a 100755
--- a/scripts/manual-test-mi325x.sh
+++ b/scripts/manual-test-mi325x.sh
@@ -15,7 +15,7 @@ export IMAGE=ghcr.io/jordannanos/sgl-mi325x-mori:v0.5.9-bnxt-good
 
 export ISL=1024
 export OSL=1024
-export CONC_LIST="4 2 1"
+export CONC_LIST="1024 512 256 128 64 32 16 8 4 2 1"
 export SPEC_DECODING=none
 export RANDOM_RANGE_RATIO=1
 

From 2421ca580cbb54491cd0bd12666ca1f660300908 Mon Sep 17 00:00:00 2001
From: root <root@slurm-sa-mi325-controller-02.ord.vultr.cpe.ice.amd.com>
Date: Tue, 31 Mar 2026 17:56:58 +0000
Subject: [PATCH 4/9] Add perf-changelog entry for MI325X disagg configs

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
---
 perf-changelog.yaml | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/perf-changelog.yaml b/perf-changelog.yaml
index 967edc19c..8e8ebc989 100644
--- a/perf-changelog.yaml
+++ b/perf-changelog.yaml
@@ -1,3 +1,14 @@
+- config-keys:
+    - dsr1-fp8-mi325x-sglang-disagg
+    - dsr1-fp8-mi325x-sglang-disagg-mtp
+  description:
+    - "Add MI325X DeepSeek-R1 FP8 disaggregated inference with Broadcom Thor 2 IBGDA"
+    - "Custom container image built from akao-amd/sglang with MORI + bnxt_rocelib patches"
+    - "Image: ghcr.io/jordannanos/sgl-mi325x-mori:v0.5.9-bnxt-good"
+    - "Full pareto sweep: non-MTP and MTP configs across 4 curve points, ISL 1k/1k and 8k/1k"
+    - "Dockerfile patches: https://github.com/JordanNanos/sglang/tree/main/docker"
+  pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/985
+
 - config-keys:
     - kimik2.5-int4-mi300x-vllm
   description:

From 6abdf85570d876220480a317ba9635451ccb055f Mon Sep 17 00:00:00 2001
From: JordanNanos <jordan@semianalysis.com>
Date: Wed, 1 Apr 2026 00:19:24 +0000
Subject: [PATCH 5/9] Fix MI325X QoS detection and NFS-safe cleanup for disagg
 benchmarks

- Add chi-mi325x* hostname detection in env.sh for RDMA QoS config
  (MORI_RDMA_TC=104, MORI_RDMA_SL=3, derived from DCB DSCP AF31->prio 3)
  since nicctl is not available on Vultr/CPE MI325X hosts
- Wrap sudo rm -rf calls with timeout 30s in launch_mi325x-amd.sh and
  job.slurm to prevent indefinite hangs on stale NFS locks

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
---
 benchmarks/multi_node/amd_utils/env.sh    | 10 ++++++++++
 benchmarks/multi_node/amd_utils/job.slurm |  4 ++--
 runners/launch_mi325x-amd.sh              |  6 ++++--
 3 files changed, 16 insertions(+), 4 deletions(-)

diff --git a/benchmarks/multi_node/amd_utils/env.sh b/benchmarks/multi_node/amd_utils/env.sh
index 56572dfcf..99f2d0238 100755
--- a/benchmarks/multi_node/amd_utils/env.sh
+++ b/benchmarks/multi_node/amd_utils/env.sh
@@ -104,6 +104,11 @@ $1 == "DSCP" && $2 == ":" && $NF == p {
         elif [[ $NODENAME == mia1* ]]; then
             export MORI_RDMA_TC=104
             echo "[INFO] Auto-detected MORI_RDMA_TC=$MORI_RDMA_TC from hostname $NODENAME"
+        elif [[ $NODENAME == chi-mi325x* ]]; then
+            # Vultr/CPE MI325X: Broadcom Thor 2, DSCP AF31(26)->prio 3, TC=4*26=104
+            export MORI_RDMA_TC=104
+            export MORI_RDMA_SL=3
+            echo "[INFO] Auto-detected MORI_RDMA_TC=$MORI_RDMA_TC, MORI_RDMA_SL=$MORI_RDMA_SL from hostname $NODENAME"
         else
             echo "[INFO] Unable to detect MORI_RDMA_TC from hostname. Skipping RDMA QoS configuration."
         fi
@@ -117,6 +122,11 @@ else
     elif [[ $NODENAME == mia1* ]]; then
         export MORI_RDMA_TC=104
         echo "[INFO] Auto-detected MORI_RDMA_TC=$MORI_RDMA_TC from hostname $NODENAME"
+    elif [[ $NODENAME == chi-mi325x* ]]; then
+        # Vultr/CPE MI325X: Broadcom Thor 2, DSCP AF31(26)->prio 3, TC=4*26=104
+        export MORI_RDMA_TC=104
+        export MORI_RDMA_SL=3
+        echo "[INFO] Auto-detected MORI_RDMA_TC=$MORI_RDMA_TC, MORI_RDMA_SL=$MORI_RDMA_SL from hostname $NODENAME"
     else
         echo "[INFO] nicctl not found and unable to detect from hostname. Skipping RDMA QoS configuration."
         echo "       This is normal for clusters without QoS or outside Docker containers."
diff --git a/benchmarks/multi_node/amd_utils/job.slurm b/benchmarks/multi_node/amd_utils/job.slurm
index 0e8f465f5..784161d06 100755
--- a/benchmarks/multi_node/amd_utils/job.slurm
+++ b/benchmarks/multi_node/amd_utils/job.slurm
@@ -299,8 +299,8 @@ SELECTED_NODELIST_SRUN=$(echo "$SELECTED_NODES" | paste -sd,)
 
 cleanup() {
   echo "[${SLURM_JOB_ID}] termination received on $(hostname); cleaning stale logs folder..."
-  # clean up the logs folder
-  sudo rm -rf ${SLURM_SUBMIT_DIR}/logs 2>/dev/null || true
+  # NFS-safe cleanup: use timeout to avoid hanging on stale NFS locks
+  timeout --kill-after=5 30 sudo rm -rf ${SLURM_SUBMIT_DIR}/logs 2>/dev/null || true
 
   echo "[${SLURM_JOB_ID}] cleanup done."
 }
diff --git a/runners/launch_mi325x-amd.sh b/runners/launch_mi325x-amd.sh
index 4e76c205a..a21d2fd58 100644
--- a/runners/launch_mi325x-amd.sh
+++ b/runners/launch_mi325x-amd.sh
@@ -74,7 +74,8 @@ if [[ "$BENCHMARK_SUBDIR" == "multi_node" ]]; then
 
     export BENCHMARK_LOGS_DIR="${BENCHMARK_LOGS_DIR:-$GITHUB_WORKSPACE/benchmark_logs}"
     mkdir -p "$BENCHMARK_LOGS_DIR"
-    sudo rm -rf "$BENCHMARK_LOGS_DIR/logs" 2>/dev/null || true
+    # NFS-safe cleanup: use timeout to avoid hanging on stale NFS locks
+    timeout --kill-after=5 30 sudo rm -rf "$BENCHMARK_LOGS_DIR/logs" 2>/dev/null || true
 
     JOB_ID=$(bash "benchmarks/${BENCHMARK_SUBDIR}/${SCRIPT_NAME}")
 
@@ -137,7 +138,8 @@ PY
     set -x
     echo "Canceled the slurm job $JOB_ID"
 
-    sudo rm -rf "$BENCHMARK_LOGS_DIR/logs" 2>/dev/null || true
+    # NFS-safe cleanup: use timeout to avoid hanging on stale NFS locks
+    timeout --kill-after=5 30 sudo rm -rf "$BENCHMARK_LOGS_DIR/logs" 2>/dev/null || true
 
     if [[ -n "${GITHUB_ACTIONS:-}" ]]; then
         ARTIFACT_DIR="$GITHUB_WORKSPACE/benchmark_artifacts"

From 37162588f5776b1e4282577f801da8fd93d7448c Mon Sep 17 00:00:00 2001
From: JordanNanos <jordan@semianalysis.com>
Date: Wed, 1 Apr 2026 06:44:31 +0000
Subject: [PATCH 6/9] Add local NVMe model caching for faster model loading

Pre-stage model weights from NFS/shared storage to local NVMe before
the inference server starts. Reduces model load time for large models
(e.g., DeepSeek-R1 ~340GB FP8) from NFS read speeds to NVMe speeds.

- utils/setup_local_nvme.sh: One-time NVMe setup script for compute nodes
  (format, mount, fstab entry). Supports single drive or RAID-0.
- utils/cache_model_locally.sh: Standalone/sourceable model caching utility
  using rsync with parallel blob sync for HF hub cache layout.
- job.slurm: When LOCAL_MODEL_CACHE_DIR is set, runs srun-based parallel
  rsync on all nodes before Docker starts. Idempotent (skips if cached).
  Falls back to shared storage if caching fails.
- launch_mi325x-amd.sh: Enable local caching at /local-nvme/models for
  MI325X cluster (8x 3.5TB NVMe per node).

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
---
 benchmarks/multi_node/amd_utils/job.slurm |  61 +++++++++++
 runners/launch_mi325x-amd.sh              |   5 +
 utils/cache_model_locally.sh              | 109 ++++++++++++++++++++
 utils/setup_local_nvme.sh                 | 118 ++++++++++++++++++++++
 4 files changed, 293 insertions(+)
 create mode 100755 utils/cache_model_locally.sh
 create mode 100755 utils/setup_local_nvme.sh

diff --git a/benchmarks/multi_node/amd_utils/job.slurm b/benchmarks/multi_node/amd_utils/job.slurm
index 784161d06..7c746b41a 100755
--- a/benchmarks/multi_node/amd_utils/job.slurm
+++ b/benchmarks/multi_node/amd_utils/job.slurm
@@ -321,6 +321,67 @@ srun --nodelist="$SELECTED_NODELIST_SRUN" bash -c '
     echo "NFS cache refreshed on $(hostname)"
 '
 
+# =============================================================================
+# Optional: Pre-stage model to local NVMe for faster loading
+# =============================================================================
+# LOCAL_MODEL_CACHE_DIR: mount point for fast local storage (NVMe/SSD) on compute nodes.
+# Set per-cluster via the runner/launch script. When set, model weights are rsync'd
+# from shared storage to local NVMe before Docker starts. This is idempotent —
+# subsequent runs skip files already cached locally.
+#
+# If unset or the local path doesn't exist, the model is served directly from
+# shared storage (NFS/Lustre) as before.
+if [[ -n "${LOCAL_MODEL_CACHE_DIR:-}" ]]; then
+    LOCAL_MODEL_FULL="${LOCAL_MODEL_CACHE_DIR}/${MODEL_NAME}"
+    echo "[cache] Pre-staging model to local NVMe on all nodes..."
+    echo "[cache]   Source: $MODEL_PATH"
+    echo "[cache]   Dest:   $LOCAL_MODEL_FULL"
+
+    srun --nodelist="$SELECTED_NODELIST_SRUN" bash -c '
+        set -euo pipefail
+        SRC="'"$MODEL_PATH"'"
+        DST="'"$LOCAL_MODEL_FULL"'"
+        CACHE_DIR="'"${LOCAL_MODEL_CACHE_DIR}"'"
+
+        # Create destination directory
+        sudo mkdir -p "$CACHE_DIR" 2>/dev/null || mkdir -p "$CACHE_DIR"
+        sudo chown -R "$(whoami)" "$CACHE_DIR" 2>/dev/null || true
+
+        SRC_COUNT=$(find "$SRC" -type f 2>/dev/null | wc -l)
+        DST_COUNT=$(find "$DST" -type f 2>/dev/null | wc -l)
+
+        if [[ "$SRC_COUNT" -eq "$DST_COUNT" ]] && [[ "$DST_COUNT" -gt 0 ]]; then
+            echo "[cache] $(hostname): Already cached ($DST_COUNT files)"
+        else
+            echo "[cache] $(hostname): Syncing $SRC_COUNT files..."
+            START=$(date +%s)
+
+            if [[ -d "$SRC/blobs" ]]; then
+                # HuggingFace hub cache layout: parallel-sync large blobs
+                mkdir -p "$DST/blobs"
+                find "$SRC/blobs" -type f -printf "%f\n" | \
+                    xargs -P '"${CACHE_PARALLEL_JOBS:-4}"' -I{} \
+                    rsync -a --whole-file --ignore-existing "$SRC/blobs/{}" "$DST/blobs/{}"
+                rsync -a --whole-file --ignore-existing --exclude="blobs/" "$SRC/" "$DST/"
+            else
+                # Flat model directory
+                rsync -a --whole-file --ignore-existing "$SRC/" "$DST/"
+            fi
+
+            ELAPSED=$(( $(date +%s) - START ))
+            SIZE=$(du -sh "$DST" 2>/dev/null | cut -f1)
+            echo "[cache] $(hostname): Done in ${ELAPSED}s ($SIZE)"
+        fi
+    ' 2>&1
+
+    if [[ $? -eq 0 ]]; then
+        echo "[cache] Model pre-staged successfully. Updating MODEL_DIR."
+        MODEL_DIR="${LOCAL_MODEL_CACHE_DIR}"
+    else
+        echo "[cache] WARNING: Local caching failed on some nodes. Falling back to shared storage."
+    fi
+fi
+
 srun \
   --nodelist="$SELECTED_NODELIST_SRUN" \
   --kill-on-bad-exit=1 \
diff --git a/runners/launch_mi325x-amd.sh b/runners/launch_mi325x-amd.sh
index a21d2fd58..107c68d7d 100644
--- a/runners/launch_mi325x-amd.sh
+++ b/runners/launch_mi325x-amd.sh
@@ -3,6 +3,11 @@
 export HF_HUB_CACHE_MOUNT="/nfsdata/sa/gharunner/gharunners/hf-hub-cache/"
 export PORT=8888
 
+# Local NVMe cache for model weights (set to empty to disable)
+# MI325X nodes have 8x 3.5TB NVMe drives; /local-nvme must be set up
+# via: sudo bash utils/setup_local_nvme.sh /local-nvme
+export LOCAL_MODEL_CACHE_DIR="${LOCAL_MODEL_CACHE_DIR:-/local-nvme/models}"
+
 PARTITION="compute"
 
 # Detect benchmark subdir from where the script lives
diff --git a/utils/cache_model_locally.sh b/utils/cache_model_locally.sh
new file mode 100755
index 000000000..37369d29e
--- /dev/null
+++ b/utils/cache_model_locally.sh
@@ -0,0 +1,109 @@
+#!/usr/bin/env bash
+# cache_model_locally.sh — Pre-stage model weights from shared storage to local NVMe.
+#
+# Syncs a model directory from NFS/shared storage to fast local NVMe before
+# the inference server starts, dramatically reducing model load time.
+#
+# Usage:
+#   source utils/cache_model_locally.sh
+#   cache_model_locally "/nfs/models/deepseek-r1" "/local-nvme/models/deepseek-r1"
+#
+# Or as a standalone script:
+#   bash utils/cache_model_locally.sh /nfs/models/deepseek-r1 /local-nvme/models/deepseek-r1
+#
+# Features:
+#   - Idempotent: skips files already present on the target
+#   - Preserves HuggingFace cache symlink structure
+#   - Concurrent execution safe (multiple nodes can cache simultaneously)
+#   - Configurable timeout to prevent NFS hangs
+#   - Works with both HF hub cache layout and flat model directories
+#
+# Environment variables:
+#   CACHE_PARALLEL_JOBS  — number of parallel rsync jobs for large blobs (default: 4)
+#   CACHE_TIMEOUT        — per-file timeout in seconds (default: 600)
+#   CACHE_DRY_RUN        — set to 1 to print what would be synced without copying
+
+set -euo pipefail
+
+CACHE_PARALLEL_JOBS="${CACHE_PARALLEL_JOBS:-4}"
+CACHE_TIMEOUT="${CACHE_TIMEOUT:-600}"
+CACHE_DRY_RUN="${CACHE_DRY_RUN:-0}"
+
+cache_model_locally() {
+    local src="${1:?Usage: cache_model_locally <source_path> <dest_path>}"
+    local dst="${2:?Usage: cache_model_locally <source_path> <dest_path>}"
+
+    if [[ ! -d "$src" ]]; then
+        echo "[cache] ERROR: Source path does not exist: $src" >&2
+        return 1
+    fi
+
+    # Quick check: if dest has the same number of regular files, skip entirely
+    local src_count dst_count
+    src_count=$(find "$src" -type f 2>/dev/null | wc -l)
+    dst_count=$(find "$dst" -type f 2>/dev/null | wc -l)
+
+    if [[ "$src_count" -eq "$dst_count" ]] && [[ "$dst_count" -gt 0 ]]; then
+        echo "[cache] Already cached: $dst ($dst_count files)"
+        echo "$dst"
+        return 0
+    fi
+
+    echo "[cache] Syncing model to local storage..."
+    echo "[cache]   Source: $src"
+    echo "[cache]   Dest:   $dst"
+    echo "[cache]   Parallel jobs: $CACHE_PARALLEL_JOBS"
+
+    mkdir -p "$dst"
+
+    local rsync_opts=(-a --whole-file --ignore-existing --info=name)
+    if [[ "$CACHE_DRY_RUN" -eq 1 ]]; then
+        rsync_opts+=(--dry-run)
+    fi
+
+    local start_time
+    start_time=$(date +%s)
+
+    # Check if this is a HuggingFace hub cache directory (has blobs/ subdir)
+    if [[ -d "$src/blobs" ]]; then
+        echo "[cache] Detected HuggingFace hub cache layout"
+
+        # Step 1: Parallel-sync the large blob files (the actual model weights)
+        mkdir -p "$dst/blobs"
+        find "$src/blobs" -type f -printf '%f\n' | \
+            xargs -P "$CACHE_PARALLEL_JOBS" -I{} \
+            timeout "$CACHE_TIMEOUT" rsync "${rsync_opts[@]}" "$src/blobs/{}" "$dst/blobs/{}"
+
+        # Step 2: Sync everything else (symlinks in snapshots/, refs/, etc.) — fast
+        rsync "${rsync_opts[@]}" --exclude='blobs/' "$src/" "$dst/"
+    else
+        # Flat model directory: parallel-sync large files, then the rest
+        echo "[cache] Detected flat model directory"
+
+        # Sync large files (>100MB) in parallel
+        find "$src" -type f -size +100M -printf '%P\n' | \
+            xargs -P "$CACHE_PARALLEL_JOBS" -I{} bash -c \
+            'mkdir -p "$(dirname "'"$dst"'/{}")"; timeout '"$CACHE_TIMEOUT"' rsync '"$(printf '%q ' "${rsync_opts[@]}")"' "'"$src"'/{}" "'"$dst"'/{}"'
+
+        # Sync remaining small files and symlinks
+        rsync "${rsync_opts[@]}" "$src/" "$dst/"
+    fi
+
+    local elapsed=$(( $(date +%s) - start_time ))
+    local size
+    size=$(du -sh "$dst" 2>/dev/null | cut -f1)
+
+    echo "[cache] Done in ${elapsed}s — $size cached at $dst"
+    echo "$dst"
+    return 0
+}
+
+# If run as a standalone script (not sourced), execute with args
+if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then
+    if [[ $# -lt 2 ]]; then
+        echo "Usage: $0 <source_model_path> <dest_model_path>" >&2
+        echo "  Env: CACHE_PARALLEL_JOBS=$CACHE_PARALLEL_JOBS CACHE_TIMEOUT=$CACHE_TIMEOUT" >&2
+        exit 1
+    fi
+    cache_model_locally "$1" "$2"
+fi
diff --git a/utils/setup_local_nvme.sh b/utils/setup_local_nvme.sh
new file mode 100755
index 000000000..03b81e8a4
--- /dev/null
+++ b/utils/setup_local_nvme.sh
@@ -0,0 +1,118 @@
+#!/usr/bin/env bash
+# setup_local_nvme.sh — Format and mount local NVMe drives for model caching.
+#
+# Detects unformatted/unmounted NVMe drives and sets up a mount point for
+# caching model weights locally. Designed to be run once per node (idempotent).
+#
+# Usage (run on each compute node, requires root):
+#   sudo bash utils/setup_local_nvme.sh [mount_point]
+#
+# Default mount point: /local-nvme
+#
+# This script:
+#   1. Finds the first available NVMe drive that is not the boot device
+#   2. Formats it with ext4 if not already formatted
+#   3. Mounts it at the specified mount point
+#   4. Adds an fstab entry for persistence across reboots
+#
+# For RAID-0 across multiple NVMe drives (maximum throughput), use:
+#   sudo bash utils/setup_local_nvme.sh --raid [mount_point]
+
+set -euo pipefail
+
+USE_RAID=false
+MOUNT_POINT="/local-nvme"
+
+while [[ $# -gt 0 ]]; do
+    case "$1" in
+        --raid) USE_RAID=true; shift ;;
+        *) MOUNT_POINT="$1"; shift ;;
+    esac
+done
+
+if [[ $EUID -ne 0 ]]; then
+    echo "ERROR: This script must be run as root (sudo)" >&2
+    exit 1
+fi
+
+echo "[nvme-setup] Mount point: $MOUNT_POINT"
+
+# Already mounted?
+if mountpoint -q "$MOUNT_POINT" 2>/dev/null; then
+    echo "[nvme-setup] $MOUNT_POINT is already mounted:"
+    df -h "$MOUNT_POINT"
+    exit 0
+fi
+
+# Find NVMe drives that are not part of the root filesystem
+ROOT_DEV=$(findmnt -n -o SOURCE / | sed 's/[0-9]*$//' | sed 's/p$//')
+NVME_DRIVES=()
+for dev in /dev/nvme*n1; do
+    [[ -b "$dev" ]] || continue
+    # Skip if this drive is part of root
+    if [[ "$dev" == "$ROOT_DEV"* ]]; then
+        echo "[nvme-setup] Skipping $dev (root device)"
+        continue
+    fi
+    # Skip if already mounted
+    if mount | grep -q "^$dev "; then
+        echo "[nvme-setup] Skipping $dev (already mounted)"
+        continue
+    fi
+    # Skip if part of an md array
+    if grep -q "$(basename "$dev")" /proc/mdstat 2>/dev/null; then
+        echo "[nvme-setup] Skipping $dev (part of md array)"
+        continue
+    fi
+    NVME_DRIVES+=("$dev")
+done
+
+if [[ ${#NVME_DRIVES[@]} -eq 0 ]]; then
+    echo "[nvme-setup] No available NVMe drives found."
+    exit 1
+fi
+
+echo "[nvme-setup] Found ${#NVME_DRIVES[@]} available NVMe drives: ${NVME_DRIVES[*]}"
+
+if [[ "$USE_RAID" == true ]] && [[ ${#NVME_DRIVES[@]} -gt 1 ]]; then
+    # RAID-0 for maximum throughput
+    MD_DEV="/dev/md10"
+    echo "[nvme-setup] Creating RAID-0 array across ${#NVME_DRIVES[@]} drives..."
+
+    if [[ -b "$MD_DEV" ]]; then
+        echo "[nvme-setup] $MD_DEV already exists, using it"
+    else
+        mdadm --create "$MD_DEV" --level=0 --raid-devices=${#NVME_DRIVES[@]} "${NVME_DRIVES[@]}" --run
+    fi
+
+    TARGET_DEV="$MD_DEV"
+else
+    # Single drive (use the first available)
+    TARGET_DEV="${NVME_DRIVES[0]}"
+    echo "[nvme-setup] Using single drive: $TARGET_DEV"
+fi
+
+# Format if needed
+if ! blkid "$TARGET_DEV" | grep -q 'TYPE="ext4"'; then
+    echo "[nvme-setup] Formatting $TARGET_DEV with ext4..."
+    mkfs.ext4 -F -L local-nvme "$TARGET_DEV"
+else
+    echo "[nvme-setup] $TARGET_DEV already has ext4 filesystem"
+fi
+
+# Mount
+mkdir -p "$MOUNT_POINT"
+mount -o noatime,discard "$TARGET_DEV" "$MOUNT_POINT"
+
+# Set permissions so non-root users can write
+chmod 1777 "$MOUNT_POINT"
+
+# Add fstab entry if not present
+if ! grep -q "$MOUNT_POINT" /etc/fstab; then
+    UUID=$(blkid -s UUID -o value "$TARGET_DEV")
+    echo "UUID=$UUID $MOUNT_POINT ext4 noatime,discard,nofail 0 2" >> /etc/fstab
+    echo "[nvme-setup] Added fstab entry"
+fi
+
+echo "[nvme-setup] Done:"
+df -h "$MOUNT_POINT"

From db677bd8fc2fc0a71ac57d67c35b88f6ccc06910 Mon Sep 17 00:00:00 2001
From: JordanNanos <jordan@semianalysis.com>
Date: Wed, 1 Apr 2026 06:54:51 +0000
Subject: [PATCH 7/9] Switch model caching from rsync to rclone sync

Use rclone sync with --transfers 32 --checkers 32 --links for
high-parallelism model pre-staging from NFS to local NVMe.
rclone is now installed on all MI325X compute nodes (v1.73.3).

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
---
 benchmarks/multi_node/amd_utils/job.slurm | 37 ++++--------
 utils/cache_model_locally.sh              | 71 ++++++-----------------
 2 files changed, 30 insertions(+), 78 deletions(-)

diff --git a/benchmarks/multi_node/amd_utils/job.slurm b/benchmarks/multi_node/amd_utils/job.slurm
index 7c746b41a..523bfd7c5 100755
--- a/benchmarks/multi_node/amd_utils/job.slurm
+++ b/benchmarks/multi_node/amd_utils/job.slurm
@@ -347,31 +347,18 @@ if [[ -n "${LOCAL_MODEL_CACHE_DIR:-}" ]]; then
         sudo mkdir -p "$CACHE_DIR" 2>/dev/null || mkdir -p "$CACHE_DIR"
         sudo chown -R "$(whoami)" "$CACHE_DIR" 2>/dev/null || true
 
-        SRC_COUNT=$(find "$SRC" -type f 2>/dev/null | wc -l)
-        DST_COUNT=$(find "$DST" -type f 2>/dev/null | wc -l)
-
-        if [[ "$SRC_COUNT" -eq "$DST_COUNT" ]] && [[ "$DST_COUNT" -gt 0 ]]; then
-            echo "[cache] $(hostname): Already cached ($DST_COUNT files)"
-        else
-            echo "[cache] $(hostname): Syncing $SRC_COUNT files..."
-            START=$(date +%s)
-
-            if [[ -d "$SRC/blobs" ]]; then
-                # HuggingFace hub cache layout: parallel-sync large blobs
-                mkdir -p "$DST/blobs"
-                find "$SRC/blobs" -type f -printf "%f\n" | \
-                    xargs -P '"${CACHE_PARALLEL_JOBS:-4}"' -I{} \
-                    rsync -a --whole-file --ignore-existing "$SRC/blobs/{}" "$DST/blobs/{}"
-                rsync -a --whole-file --ignore-existing --exclude="blobs/" "$SRC/" "$DST/"
-            else
-                # Flat model directory
-                rsync -a --whole-file --ignore-existing "$SRC/" "$DST/"
-            fi
-
-            ELAPSED=$(( $(date +%s) - START ))
-            SIZE=$(du -sh "$DST" 2>/dev/null | cut -f1)
-            echo "[cache] $(hostname): Done in ${ELAPSED}s ($SIZE)"
-        fi
+        echo "[cache] $(hostname): Syncing model to local NVMe..."
+        START=$(date +%s)
+
+        rclone sync "$SRC/" "$DST/" \
+            --transfers 32 \
+            --checkers 32 \
+            --links \
+            --progress
+
+        ELAPSED=$(( $(date +%s) - START ))
+        SIZE=$(du -sh "$DST" 2>/dev/null | cut -f1)
+        echo "[cache] $(hostname): Done in ${ELAPSED}s ($SIZE)"
     ' 2>&1
 
     if [[ $? -eq 0 ]]; then
diff --git a/utils/cache_model_locally.sh b/utils/cache_model_locally.sh
index 37369d29e..0b1480231 100755
--- a/utils/cache_model_locally.sh
+++ b/utils/cache_model_locally.sh
@@ -2,31 +2,30 @@
 # cache_model_locally.sh — Pre-stage model weights from shared storage to local NVMe.
 #
 # Syncs a model directory from NFS/shared storage to fast local NVMe before
-# the inference server starts, dramatically reducing model load time.
+# the inference server starts, using rclone for high-parallelism transfers.
 #
 # Usage:
 #   source utils/cache_model_locally.sh
-#   cache_model_locally "/nfs/models/deepseek-r1" "/local-nvme/models/deepseek-r1"
+#   cache_model_locally "/nfs/hub/models--org--repo" "/local-nvme/hub/models--org--repo"
 #
 # Or as a standalone script:
-#   bash utils/cache_model_locally.sh /nfs/models/deepseek-r1 /local-nvme/models/deepseek-r1
+#   bash utils/cache_model_locally.sh /nfs/hub/models--org--repo /local-nvme/hub/models--org--repo
 #
 # Features:
-#   - Idempotent: skips files already present on the target
-#   - Preserves HuggingFace cache symlink structure
-#   - Concurrent execution safe (multiple nodes can cache simultaneously)
-#   - Configurable timeout to prevent NFS hangs
+#   - Uses rclone sync with 32 parallel transfers for maximum throughput
+#   - Preserves HuggingFace cache symlink structure (--links)
+#   - Idempotent: rclone skips files already present and identical
 #   - Works with both HF hub cache layout and flat model directories
 #
 # Environment variables:
-#   CACHE_PARALLEL_JOBS  — number of parallel rsync jobs for large blobs (default: 4)
-#   CACHE_TIMEOUT        — per-file timeout in seconds (default: 600)
-#   CACHE_DRY_RUN        — set to 1 to print what would be synced without copying
+#   CACHE_TRANSFERS  — number of parallel rclone transfers (default: 32)
+#   CACHE_CHECKERS   — number of parallel rclone checkers (default: 32)
+#   CACHE_DRY_RUN    — set to 1 to print what would be synced without copying
 
 set -euo pipefail
 
-CACHE_PARALLEL_JOBS="${CACHE_PARALLEL_JOBS:-4}"
-CACHE_TIMEOUT="${CACHE_TIMEOUT:-600}"
+CACHE_TRANSFERS="${CACHE_TRANSFERS:-32}"
+CACHE_CHECKERS="${CACHE_CHECKERS:-32}"
 CACHE_DRY_RUN="${CACHE_DRY_RUN:-0}"
 
 cache_model_locally() {
@@ -38,57 +37,23 @@ cache_model_locally() {
         return 1
     fi
 
-    # Quick check: if dest has the same number of regular files, skip entirely
-    local src_count dst_count
-    src_count=$(find "$src" -type f 2>/dev/null | wc -l)
-    dst_count=$(find "$dst" -type f 2>/dev/null | wc -l)
-
-    if [[ "$src_count" -eq "$dst_count" ]] && [[ "$dst_count" -gt 0 ]]; then
-        echo "[cache] Already cached: $dst ($dst_count files)"
-        echo "$dst"
-        return 0
-    fi
-
     echo "[cache] Syncing model to local storage..."
     echo "[cache]   Source: $src"
     echo "[cache]   Dest:   $dst"
-    echo "[cache]   Parallel jobs: $CACHE_PARALLEL_JOBS"
+    echo "[cache]   Transfers: $CACHE_TRANSFERS, Checkers: $CACHE_CHECKERS"
 
     mkdir -p "$dst"
 
-    local rsync_opts=(-a --whole-file --ignore-existing --info=name)
-    if [[ "$CACHE_DRY_RUN" -eq 1 ]]; then
-        rsync_opts+=(--dry-run)
-    fi
-
     local start_time
     start_time=$(date +%s)
 
-    # Check if this is a HuggingFace hub cache directory (has blobs/ subdir)
-    if [[ -d "$src/blobs" ]]; then
-        echo "[cache] Detected HuggingFace hub cache layout"
-
-        # Step 1: Parallel-sync the large blob files (the actual model weights)
-        mkdir -p "$dst/blobs"
-        find "$src/blobs" -type f -printf '%f\n' | \
-            xargs -P "$CACHE_PARALLEL_JOBS" -I{} \
-            timeout "$CACHE_TIMEOUT" rsync "${rsync_opts[@]}" "$src/blobs/{}" "$dst/blobs/{}"
-
-        # Step 2: Sync everything else (symlinks in snapshots/, refs/, etc.) — fast
-        rsync "${rsync_opts[@]}" --exclude='blobs/' "$src/" "$dst/"
-    else
-        # Flat model directory: parallel-sync large files, then the rest
-        echo "[cache] Detected flat model directory"
-
-        # Sync large files (>100MB) in parallel
-        find "$src" -type f -size +100M -printf '%P\n' | \
-            xargs -P "$CACHE_PARALLEL_JOBS" -I{} bash -c \
-            'mkdir -p "$(dirname "'"$dst"'/{}")"; timeout '"$CACHE_TIMEOUT"' rsync '"$(printf '%q ' "${rsync_opts[@]}")"' "'"$src"'/{}" "'"$dst"'/{}"'
-
-        # Sync remaining small files and symlinks
-        rsync "${rsync_opts[@]}" "$src/" "$dst/"
+    local rclone_opts=(--transfers "$CACHE_TRANSFERS" --checkers "$CACHE_CHECKERS" --links --progress)
+    if [[ "$CACHE_DRY_RUN" -eq 1 ]]; then
+        rclone_opts+=(--dry-run)
     fi
 
+    rclone sync "$src/" "$dst/" "${rclone_opts[@]}"
+
     local elapsed=$(( $(date +%s) - start_time ))
     local size
     size=$(du -sh "$dst" 2>/dev/null | cut -f1)
@@ -102,7 +67,7 @@ cache_model_locally() {
 if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then
     if [[ $# -lt 2 ]]; then
         echo "Usage: $0 <source_model_path> <dest_model_path>" >&2
-        echo "  Env: CACHE_PARALLEL_JOBS=$CACHE_PARALLEL_JOBS CACHE_TIMEOUT=$CACHE_TIMEOUT" >&2
+        echo "  Env: CACHE_TRANSFERS=$CACHE_TRANSFERS CACHE_CHECKERS=$CACHE_CHECKERS" >&2
         exit 1
     fi
     cache_model_locally "$1" "$2"

From 0a485de74d254f2dab6445b4891f69630a816872 Mon Sep 17 00:00:00 2001
From: JordanNanos <jordan@semianalysis.com>
Date: Wed, 1 Apr 2026 07:30:13 +0000
Subject: [PATCH 8/9] Add MTP baseline to single-node MI325X DeepSeek-R1 FP8
 config

Add spec-decoding: mtp search space entries alongside the existing
non-MTP entries for both 1k/1k and 8k/1k sequence length configs.
This provides a single-node MTP baseline for comparison with the
disaggregated multi-node MTP results.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
---
 .github/configs/amd-master.yaml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.github/configs/amd-master.yaml b/.github/configs/amd-master.yaml
index 00b6a26de..1ab86b8af 100644
--- a/.github/configs/amd-master.yaml
+++ b/.github/configs/amd-master.yaml
@@ -89,10 +89,12 @@ dsr1-fp8-mi325x-sglang:
     osl: 1024
     search-space:
     - { tp: 8, conc-start: 4, conc-end: 64 }
+    - { tp: 8, conc-start: 4, conc-end: 64, spec-decoding: mtp }
   - isl: 8192
     osl: 1024
     search-space:
     - { tp: 8, conc-start: 4, conc-end: 64 }
+    - { tp: 8, conc-start: 4, conc-end: 64, spec-decoding: mtp }
 
 dsr1-fp8-mi355x-sglang:
   image: lmsysorg/sglang:v0.5.9-rocm700-mi35x

From 67dec7cfd088e7f8fb82afec88e1a3a21190c3b7 Mon Sep 17 00:00:00 2001
From: JordanNanos <jordan@semianalysis.com>
Date: Wed, 1 Apr 2026 07:36:09 +0000
Subject: [PATCH 9/9] Split MI325X single-node MTP into separate config key

Separate dsr1-fp8-mi325x-sglang-mtp from the base config so it can
be swept independently. Full sweeps still cover both via their
respective config keys.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
---
 .github/configs/amd-master.yaml | 18 +++++++++++++++++-
 1 file changed, 17 insertions(+), 1 deletion(-)

diff --git a/.github/configs/amd-master.yaml b/.github/configs/amd-master.yaml
index 1ab86b8af..9fb5c53a0 100644
--- a/.github/configs/amd-master.yaml
+++ b/.github/configs/amd-master.yaml
@@ -89,11 +89,27 @@ dsr1-fp8-mi325x-sglang:
     osl: 1024
     search-space:
     - { tp: 8, conc-start: 4, conc-end: 64 }
-    - { tp: 8, conc-start: 4, conc-end: 64, spec-decoding: mtp }
   - isl: 8192
     osl: 1024
     search-space:
     - { tp: 8, conc-start: 4, conc-end: 64 }
+
+dsr1-fp8-mi325x-sglang-mtp:
+  image: lmsysorg/sglang:v0.5.9-rocm700-mi30x
+  model: deepseek-ai/DeepSeek-R1-0528
+  model-prefix: dsr1
+  runner: mi325x
+  precision: fp8
+  framework: sglang
+  multinode: false
+  seq-len-configs:
+  - isl: 1024
+    osl: 1024
+    search-space:
+    - { tp: 8, conc-start: 4, conc-end: 64, spec-decoding: mtp }
+  - isl: 8192
+    osl: 1024
+    search-space:
     - { tp: 8, conc-start: 4, conc-end: 64, spec-decoding: mtp }
 
 dsr1-fp8-mi355x-sglang: