diff --git a/.github/configs/amd-master.yaml b/.github/configs/amd-master.yaml index e84fc0da5..00b6a26de 100644 --- a/.github/configs/amd-master.yaml +++ b/.github/configs/amd-master.yaml @@ -1231,3 +1231,304 @@ dsr1-fp4-mi355x-sglang-disagg-mtp: - "DECODE_NODES=1" - "DECODE_MTP_SIZE=1" + +dsr1-fp8-mi325x-sglang-disagg: + image: ghcr.io/jordannanos/sgl-mi325x-mori:v0.5.9-bnxt-good + model: deepseek-ai/DeepSeek-R1-0528 + model-prefix: dsr1 + runner: mi325x-disagg + precision: fp8 + framework: sglang-disagg + multinode: true + disagg: true + seq-len-configs: + - isl: 1024 + osl: 1024 + search-space: + # "Top of curve" (1 prefill worker at TP8, 1 decode worker at DEP8) + - spec-decoding: "none" + conc-list: [ 512, 1024 ] + prefill: + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: false + additional-settings: + - "PREFILL_NODES=1" + decode: + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: true + additional-settings: + - "DECODE_NODES=2" + - "DECODE_MTP_SIZE=0" + + # "Middle of curve" (1 prefill worker at TP8, 2 decode workers at DEP8) + - spec-decoding: "none" + conc-list: [ 768, 512, 256 ] + prefill: + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: false + additional-settings: + - "PREFILL_NODES=1" + decode: + num-worker: 2 + tp: 8 + ep: 8 + dp-attn: true + additional-settings: + - "DECODE_NODES=2" + - "DECODE_MTP_SIZE=0" + + # "Bottom of curve" (1 prefill worker at TP8, 2 decode workers at TP8) + - spec-decoding: "none" + conc-list: [ 256, 128, 64, 32, 16, 8, 4 ] + prefill: + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: false + additional-settings: + - "PREFILL_NODES=1" + decode: + num-worker: 2 + tp: 8 + ep: 1 + dp-attn: false + additional-settings: + - "DECODE_NODES=2" + - "DECODE_MTP_SIZE=0" + + # "Low concurrency" (1 prefill worker at TP4, 1 decode worker at TP8) + - spec-decoding: "none" + conc-list: [ 64, 32, 16, 8, 4, 2, 1 ] + prefill: + num-worker: 1 + tp: 4 + ep: 1 + dp-attn: false + additional-settings: + - "PREFILL_NODES=1" + decode: + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: false + additional-settings: + - "DECODE_NODES=1" + - "DECODE_MTP_SIZE=0" + + - isl: 8192 + osl: 1024 + search-space: + # "Top of curve" (2 prefill workers at DEP8, 1 decode worker at DEP8) + - spec-decoding: "none" + conc-list: [ 512, 1024 ] + prefill: + num-worker: 2 + tp: 8 + ep: 8 + dp-attn: true + additional-settings: + - "PREFILL_NODES=2" + decode: + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: true + additional-settings: + - "DECODE_NODES=1" + - "DECODE_MTP_SIZE=0" + + # "Bottom of curve" (1 prefill worker at TP8, 2 decode workers at TP8) + - spec-decoding: "none" + conc-list: [ 256, 128, 64, 32, 16, 8, 4 ] + prefill: + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: false + additional-settings: + - "PREFILL_NODES=1" + decode: + num-worker: 2 + tp: 8 + ep: 1 + dp-attn: false + additional-settings: + - "DECODE_NODES=2" + - "DECODE_MTP_SIZE=0" + + # "Low concurrency" (1 prefill worker at TP4, 1 decode worker at TP8) + - spec-decoding: "none" + conc-list: [ 64, 32, 16, 8, 4, 2, 1 ] + prefill: + num-worker: 1 + tp: 4 + ep: 1 + dp-attn: false + additional-settings: + - "PREFILL_NODES=1" + decode: + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: false + additional-settings: + - "DECODE_NODES=1" + - "DECODE_MTP_SIZE=0" + + +dsr1-fp8-mi325x-sglang-disagg-mtp: + image: ghcr.io/jordannanos/sgl-mi325x-mori:v0.5.9-bnxt-good + model: deepseek-ai/DeepSeek-R1-0528 + model-prefix: dsr1 + runner: mi325x-disagg + precision: fp8 + framework: sglang-disagg + multinode: true + disagg: true + seq-len-configs: + - isl: 1024 + osl: 1024 + search-space: + # MTP configurations + # "Top of curve" (1 prefill worker at TP8, 1 decode worker at DEP8) + - spec-decoding: "mtp" + conc-list: [ 512, 1024 ] + prefill: + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: false + additional-settings: + - "PREFILL_NODES=1" + decode: + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: true + additional-settings: + - "DECODE_NODES=2" + - "DECODE_MTP_SIZE=1" + + # "Middle of curve" (1 prefill worker at TP8, 2 decode workers at DEP8) + - spec-decoding: "mtp" + conc-list: [ 768, 512, 256 ] + prefill: + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: false + additional-settings: + - "PREFILL_NODES=1" + decode: + num-worker: 2 + tp: 8 + ep: 8 + dp-attn: true + additional-settings: + - "DECODE_NODES=2" + - "DECODE_MTP_SIZE=1" + + # "Bottom of curve" (1 prefill worker at TP8, 2 decode workers at TP8) + - spec-decoding: "mtp" + conc-list: [ 256, 128, 64, 32, 16, 8, 4 ] + prefill: + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: false + additional-settings: + - "PREFILL_NODES=1" + decode: + num-worker: 2 + tp: 8 + ep: 1 + dp-attn: false + additional-settings: + - "DECODE_NODES=2" + - "DECODE_MTP_SIZE=2" + + # "Low concurrency" (1 prefill worker at TP4, 1 decode worker at TP8) + - spec-decoding: "mtp" + conc-list: [ 64, 32, 16, 8, 4, 2, 1 ] + prefill: + num-worker: 1 + tp: 4 + ep: 1 + dp-attn: false + additional-settings: + - "PREFILL_NODES=1" + decode: + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: false + additional-settings: + - "DECODE_NODES=1" + - "DECODE_MTP_SIZE=2" + + - isl: 8192 + osl: 1024 + search-space: + # MTP configurations + # "Top of curve" (2 prefill workers at DEP8, 1 decode worker at DEP8) + - spec-decoding: "mtp" + conc-list: [ 512, 1024 ] + prefill: + num-worker: 2 + tp: 8 + ep: 8 + dp-attn: true + additional-settings: + - "PREFILL_NODES=2" + decode: + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: true + additional-settings: + - "DECODE_NODES=1" + - "DECODE_MTP_SIZE=1" + + # "Bottom of curve" (1 prefill worker at TP8, 2 decode workers at TP8) + - spec-decoding: "mtp" + conc-list: [ 256, 128, 64, 32, 16, 8, 4, 2 ] + prefill: + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: false + additional-settings: + - "PREFILL_NODES=1" + decode: + num-worker: 2 + tp: 8 + ep: 1 + dp-attn: false + additional-settings: + - "DECODE_NODES=2" + - "DECODE_MTP_SIZE=2" + + # "Low concurrency" (1 prefill worker at TP4, 1 decode worker at TP8) + - spec-decoding: "mtp" + conc-list: [ 64, 32, 16, 8, 4, 2, 1 ] + prefill: + num-worker: 1 + tp: 4 + ep: 1 + dp-attn: false + additional-settings: + - "PREFILL_NODES=1" + decode: + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: false + additional-settings: + - "DECODE_NODES=1" + - "DECODE_MTP_SIZE=2" diff --git a/.github/configs/runners.yaml b/.github/configs/runners.yaml index 1251e459d..f61e81e36 100644 --- a/.github/configs/runners.yaml +++ b/.github/configs/runners.yaml @@ -75,6 +75,11 @@ mi325x: - 'mi325x-amd_1' - 'mi325x-amd_2' - 'mi325x-amd_3' +mi325x-disagg: +- 'mi325x-amd_0' +- 'mi325x-amd_1' +- 'mi325x-amd_2' +- 'mi325x-amd_3' mi355x: - 'mi355x-amds_0' - 'mi355x-amds_1' diff --git a/benchmarks/multi_node/amd_utils/env.sh b/benchmarks/multi_node/amd_utils/env.sh index 5565c5b3b..99f2d0238 100755 --- a/benchmarks/multi_node/amd_utils/env.sh +++ b/benchmarks/multi_node/amd_utils/env.sh @@ -20,6 +20,9 @@ if [[ -z "$IBDEVICES" ]]; then export IBDEVICES=ionic_0,ionic_1,ionic_2,ionic_3,ionic_4,ionic_5,ionic_6,ionic_7 elif [[ $NODENAME == mia1* ]]; then export IBDEVICES=rdma0,rdma1,rdma2,rdma3,rdma4,rdma5,rdma6,rdma7 + elif [[ $NODENAME == chi-mi325x* ]]; then + # Vultr/CPE MI325X cluster: Broadcom RoCE (bnxt_re); bnxt_re6 is DOWN, skip it + export IBDEVICES=bnxt_re0,bnxt_re1,bnxt_re2,bnxt_re3,bnxt_re4,bnxt_re5,bnxt_re7,bnxt_re8 else echo "ERROR: Unable to detect cluster from hostname $NODENAME and IBDEVICES not set" >&2 exit 1 @@ -101,6 +104,11 @@ $1 == "DSCP" && $2 == ":" && $NF == p { elif [[ $NODENAME == mia1* ]]; then export MORI_RDMA_TC=104 echo "[INFO] Auto-detected MORI_RDMA_TC=$MORI_RDMA_TC from hostname $NODENAME" + elif [[ $NODENAME == chi-mi325x* ]]; then + # Vultr/CPE MI325X: Broadcom Thor 2, DSCP AF31(26)->prio 3, TC=4*26=104 + export MORI_RDMA_TC=104 + export MORI_RDMA_SL=3 + echo "[INFO] Auto-detected MORI_RDMA_TC=$MORI_RDMA_TC, MORI_RDMA_SL=$MORI_RDMA_SL from hostname $NODENAME" else echo "[INFO] Unable to detect MORI_RDMA_TC from hostname. Skipping RDMA QoS configuration." fi @@ -114,6 +122,11 @@ else elif [[ $NODENAME == mia1* ]]; then export MORI_RDMA_TC=104 echo "[INFO] Auto-detected MORI_RDMA_TC=$MORI_RDMA_TC from hostname $NODENAME" + elif [[ $NODENAME == chi-mi325x* ]]; then + # Vultr/CPE MI325X: Broadcom Thor 2, DSCP AF31(26)->prio 3, TC=4*26=104 + export MORI_RDMA_TC=104 + export MORI_RDMA_SL=3 + echo "[INFO] Auto-detected MORI_RDMA_TC=$MORI_RDMA_TC, MORI_RDMA_SL=$MORI_RDMA_SL from hostname $NODENAME" else echo "[INFO] nicctl not found and unable to detect from hostname. Skipping RDMA QoS configuration." echo " This is normal for clusters without QoS or outside Docker containers." diff --git a/benchmarks/multi_node/amd_utils/job.slurm b/benchmarks/multi_node/amd_utils/job.slurm index 6b0352f24..784161d06 100755 --- a/benchmarks/multi_node/amd_utils/job.slurm +++ b/benchmarks/multi_node/amd_utils/job.slurm @@ -30,14 +30,18 @@ if [[ ! -f "$MODELS_YAML" ]]; then exit 1 fi -# Validate MODEL_NAME exists as a top-level key in models.yaml -if ! grep -q "^${MODEL_NAME}:" "$MODELS_YAML"; then - echo "Error: Model '$MODEL_NAME' not found in models.yaml" +# MODEL_YAML_KEY is the models.yaml lookup key (bare model name, e.g. DeepSeek-R1-0528). +# MODEL_NAME may be a longer HF cache path (e.g. models--org--repo/snapshots/). +_MODEL_YAML_KEY="${MODEL_YAML_KEY:-$MODEL_NAME}" + +# Validate the yaml key exists as a top-level key in models.yaml +if ! grep -q "^${_MODEL_YAML_KEY}:" "$MODELS_YAML"; then + echo "Error: Model '$_MODEL_YAML_KEY' not found in models.yaml" echo "Available models:" grep -E '^[A-Za-z]' "$MODELS_YAML" | sed 's/:.*$//' | sed 's/^/ - /' exit 1 fi -echo "Model found: $MODEL_NAME" +echo "Model found: $_MODEL_YAML_KEY" # All models use server.sh as the entrypoint RUN_FILE="server.sh" @@ -249,10 +253,9 @@ echo "NNODES is ${NNODES}" echo "REPO Directory is ${DI_REPO_DIR}" echo "USER_NAME is ${USER_NAME}" -# Get the RDMA priority and DSCP value from the NIC +# Get the RDMA priority and DSCP value from the NIC (optional - env.sh handles absence gracefully) if ! command -v nicctl >/dev/null 2>&1; then - echo "Error: nicctl command not found. Please ensure nicctl is installed and available." >&2 - exit 1 + echo "[INFO] nicctl not found. RDMA QoS configuration will be skipped inside the container." >&2 fi # Reduce log spam @@ -296,8 +299,8 @@ SELECTED_NODELIST_SRUN=$(echo "$SELECTED_NODES" | paste -sd,) cleanup() { echo "[${SLURM_JOB_ID}] termination received on $(hostname); cleaning stale logs folder..." - # clean up the logs folder - sudo rm -rf ${SLURM_SUBMIT_DIR}/logs 2>/dev/null || true + # NFS-safe cleanup: use timeout to avoid hanging on stale NFS locks + timeout --kill-after=5 30 sudo rm -rf ${SLURM_SUBMIT_DIR}/logs 2>/dev/null || true echo "[${SLURM_JOB_ID}] cleanup done." } @@ -357,7 +360,7 @@ exec sudo docker run --rm \ --privileged \ -v ${MODEL_DIR}:/models \ -v \$HOME/.ssh:/root/.ssh \ - -v $(which nicctl):/usr/sbin/nicctl \ + $(command -v nicctl &>/dev/null && echo "-v $(which nicctl):/usr/sbin/nicctl") \ --shm-size 128G \ -v /tmp:/run_logs \ -v ${BENCHMARK_LOGS_DIR}:/benchmark_logs \ @@ -373,6 +376,7 @@ exec sudo docker run --rm \ -e xP=\$xP \ -e yD=\$yD \ -e MODEL_NAME=\$MODEL_NAME \ + -e MODEL_YAML_KEY=${_MODEL_YAML_KEY} \ -e IPADDRS=\$IPADDRS \ -e PREFILL_TP_SIZE=\$PREFILL_TP_SIZE \ -e PREFILL_ENABLE_EP=\$PREFILL_ENABLE_EP \ diff --git a/benchmarks/multi_node/amd_utils/server.sh b/benchmarks/multi_node/amd_utils/server.sh index 7f174b760..b477790b3 100755 --- a/benchmarks/multi_node/amd_utils/server.sh +++ b/benchmarks/multi_node/amd_utils/server.sh @@ -72,11 +72,12 @@ fi # Load model config via inline Python (PyYAML is available in SGLang containers) # Formula evaluation (e.g. "SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK * TP * xP") # is done here in Python to avoid bash glob-expanding the * characters. +_MODEL_YAML_KEY="${MODEL_YAML_KEY:-$MODEL_NAME}" eval "$(python3 -c " import yaml, sys, os config_path = '${MODELS_YAML}' -model_name = '${MODEL_NAME}' +model_name = '${_MODEL_YAML_KEY}' with open(config_path) as f: models = yaml.safe_load(f) diff --git a/benchmarks/multi_node/dsr1_fp8_mi325x_sglang-disagg.sh b/benchmarks/multi_node/dsr1_fp8_mi325x_sglang-disagg.sh new file mode 100755 index 000000000..6a7314ab4 --- /dev/null +++ b/benchmarks/multi_node/dsr1_fp8_mi325x_sglang-disagg.sh @@ -0,0 +1,82 @@ +#!/usr/bin/env bash + +source "$(dirname "$0")/../benchmark_lib.sh" + +check_env_vars \ + CONC_LIST \ + ISL \ + OSL \ + IMAGE \ + SPEC_DECODING \ + MODEL_PATH \ + PREFILL_NUM_WORKERS \ + PREFILL_TP \ + PREFILL_EP \ + PREFILL_DP_ATTN \ + DECODE_NUM_WORKERS \ + DECODE_TP \ + DECODE_EP \ + DECODE_DP_ATTN \ + PREFILL_NODES \ + DECODE_NODES \ + RANDOM_RANGE_RATIO + +if [[ -n "$SLURM_JOB_ID" ]]; then + echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" +fi + +set -x + +# Use upstreamed multi_node scripts (no external clone needed) +cd "$GITHUB_WORKSPACE/benchmarks/multi_node/amd_utils" || exit 1 + +# Set up SGL launch script-specific environment variables +export TIME_LIMIT="08:00:00" +export MODEL_PATH=$MODEL_PATH +export MODEL_NAME=$MODEL_NAME +export CONTAINER_IMAGE=$IMAGE + +if [[ "${PREFILL_EP:-1}" -eq 1 ]]; then +export PREFILL_ENABLE_EP=false +else +export PREFILL_ENABLE_EP=true +fi + +if [[ "$PREFILL_DP_ATTN" == "true" ]]; then +export PREFILL_ENABLE_DP=true +else +export PREFILL_ENABLE_DP=false +fi + +if [[ "${DECODE_EP:-1}" -eq 1 ]]; then +export DECODE_ENABLE_EP=false +else +export DECODE_ENABLE_EP=true +fi + +if [[ "$DECODE_DP_ATTN" == "true" ]]; then +export DECODE_ENABLE_DP=true +else +export DECODE_ENABLE_DP=false +fi + +# Launch jobs based on ISL/OSL +# Replace ' ' in CONC_LIST with 'x' such that the concurrency list is represented +# by a list of numbers delimited by 'x'. This is because of how the underlying launch script +# expects the concurrencies. +JOB_ID=$(bash ./submit.sh $PREFILL_NODES \ + $PREFILL_NUM_WORKERS \ + $DECODE_NODES \ + $DECODE_NUM_WORKERS \ + $ISL $OSL "${CONC_LIST// /x}" inf \ + ${PREFILL_ENABLE_EP} ${PREFILL_ENABLE_DP} \ + ${DECODE_ENABLE_EP} ${DECODE_ENABLE_DP} \ + ${PREFILL_TP} ${DECODE_TP} \ + ${RANDOM_RANGE_RATIO}) + +if [[ $? -ne 0 ]]; then + echo "Failed to submit job" >&2 + exit 1 +fi + +echo "$JOB_ID" diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 967edc19c..8e8ebc989 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -1,3 +1,14 @@ +- config-keys: + - dsr1-fp8-mi325x-sglang-disagg + - dsr1-fp8-mi325x-sglang-disagg-mtp + description: + - "Add MI325X DeepSeek-R1 FP8 disaggregated inference with Broadcom Thor 2 IBGDA" + - "Custom container image built from akao-amd/sglang with MORI + bnxt_rocelib patches" + - "Image: ghcr.io/jordannanos/sgl-mi325x-mori:v0.5.9-bnxt-good" + - "Full pareto sweep: non-MTP and MTP configs across 4 curve points, ISL 1k/1k and 8k/1k" + - "Dockerfile patches: https://github.com/JordanNanos/sglang/tree/main/docker" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/985 + - config-keys: - kimik2.5-int4-mi300x-vllm description: diff --git a/runners/launch_mi325x-amd.sh b/runners/launch_mi325x-amd.sh index 67f93a309..a21d2fd58 100644 --- a/runners/launch_mi325x-amd.sh +++ b/runners/launch_mi325x-amd.sh @@ -4,37 +4,187 @@ export HF_HUB_CACHE_MOUNT="/nfsdata/sa/gharunner/gharunners/hf-hub-cache/" export PORT=8888 PARTITION="compute" -SQUASH_FILE="/nfsdata/sa/gharunner/gharunners/squash/$(echo "$IMAGE" | sed 's/[\/:@#]/_/g').sqsh" -LOCK_FILE="${SQUASH_FILE}.lock" -set -x - -JOB_ID=$(salloc --partition=$PARTITION --gres=gpu:$TP --cpus-per-task=256 --time=480 --no-shell --job-name="$RUNNER_NAME" 2>&1 | tee /dev/stderr | grep -oP 'Granted job allocation \K[0-9]+') - -if [ -z "$JOB_ID" ]; then - echo "ERROR: salloc failed to allocate a job" +# Detect benchmark subdir from where the script lives +SCRIPT_NAME="${EXP_NAME%%_*}_${PRECISION}_mi325x_${FRAMEWORK}.sh" +if [[ -f "benchmarks/multi_node/${SCRIPT_NAME}" ]]; then + BENCHMARK_SUBDIR="multi_node" +elif [[ -f "benchmarks/single_node/${SCRIPT_NAME}" ]]; then + BENCHMARK_SUBDIR="single_node" +else + echo "ERROR: ${SCRIPT_NAME} not found in benchmarks/multi_node or benchmarks/single_node" exit 1 fi -# Use flock to serialize concurrent imports to the same squash file -srun --jobid=$JOB_ID --job-name="$RUNNER_NAME" bash -c " - exec 9>\"$LOCK_FILE\" - flock -w 600 9 || { echo 'Failed to acquire lock for $SQUASH_FILE'; exit 1; } - if unsquashfs -l \"$SQUASH_FILE\" > /dev/null 2>&1; then - echo 'Squash file already exists and is valid, skipping import' - else - rm -f \"$SQUASH_FILE\" - enroot import -o \"$SQUASH_FILE\" docker://$IMAGE +# ============================================================================= +# Multi-node disaggregated path: sbatch + Docker via submit.sh +# ============================================================================= +if [[ "$BENCHMARK_SUBDIR" == "multi_node" ]]; then + + scancel_sync() { + local jobid=$1 + local timeout=${2:-600} + local interval=10 + local start + start=$(date +%s) + + echo "[scancel_sync] Requesting cancel of job $jobid" + scancel "$jobid" || true + + while [[ -n "$(squeue -j "$jobid" --noheader 2>/dev/null)" ]]; do + local now + now=$(date +%s) + if (( now - start >= timeout )); then + echo "[scancel_sync][WARN] job $jobid still present after ${timeout}s" + return 1 + fi + echo "[scancel_sync] waiting for job $jobid to exit. $((timeout-(now-start))) secs remaining..." + sleep "$interval" + done + echo "[scancel_sync] job $jobid exited" + return 0 + } + + set -x + + export SLURM_ACCOUNT="$USER" + export SLURM_PARTITION="$PARTITION" + export SLURM_JOB_NAME="benchmark-sglang-disagg.job" + + export MODEL_PATH="${HF_HUB_CACHE_MOUNT%/}" + + # MODEL_YAML_KEY: top-level key in models.yaml for server config lookup. + if [[ -z "${MODEL_YAML_KEY:-}" ]]; then + export MODEL_YAML_KEY="${MODEL##*/}" + fi + + # MODEL_NAME: relative path under MODEL_PATH for --model-path inside the container. + # Auto-resolved from HF hub cache layout so no symlink is needed. + if [[ -z "${MODEL_NAME:-}" ]]; then + _HF_DIR="models--$(echo "${MODEL}" | tr '/' '--')" + _SNAPSHOT=$(ls "${MODEL_PATH}/${_HF_DIR}/snapshots/" 2>/dev/null | sort | tail -1) + if [[ -n "${_SNAPSHOT}" ]]; then + export MODEL_NAME="${_HF_DIR}/snapshots/${_SNAPSHOT}" + else + export MODEL_NAME="${MODEL_YAML_KEY}" + fi fi -" -srun --jobid=$JOB_ID \ ---container-image=$SQUASH_FILE \ ---container-mounts=$GITHUB_WORKSPACE:/workspace/,$HF_HUB_CACHE_MOUNT:$HF_HUB_CACHE \ ---container-mount-home \ ---container-writable \ ---container-remap-root \ ---container-workdir=/workspace/ \ ---no-container-entrypoint --export=ALL \ -bash benchmarks/single_node/${EXP_NAME%%_*}_${PRECISION}_mi325x.sh - -scancel $JOB_ID + + export GPUS_PER_NODE=8 + + export BENCHMARK_LOGS_DIR="${BENCHMARK_LOGS_DIR:-$GITHUB_WORKSPACE/benchmark_logs}" + mkdir -p "$BENCHMARK_LOGS_DIR" + # NFS-safe cleanup: use timeout to avoid hanging on stale NFS locks + timeout --kill-after=5 30 sudo rm -rf "$BENCHMARK_LOGS_DIR/logs" 2>/dev/null || true + + JOB_ID=$(bash "benchmarks/${BENCHMARK_SUBDIR}/${SCRIPT_NAME}") + + LOG_FILE="$BENCHMARK_LOGS_DIR/slurm_job-${JOB_ID}.out" + + sleep 10 + + while ! ls "$LOG_FILE" &>/dev/null; do + if ! squeue -u "$USER" --noheader --format='%i' | grep -q "$JOB_ID"; then + echo "ERROR: Job $JOB_ID failed before creating log file" + scontrol show job "$JOB_ID" + exit 1 + fi + sleep 5 + done + + set +x + + ( + while squeue -u $USER --noheader --format='%i' | grep -q "$JOB_ID"; do + sleep 10 + done + ) & + POLL_PID=$! + + tail -F -s 2 -n+1 "$LOG_FILE" --pid=$POLL_PID 2>/dev/null + + wait $POLL_PID + + set -x + + cat > collect_latest_results.py <<'PY' +import os, sys +sgl_job_dir, isl, osl, nexp = sys.argv[1], int(sys.argv[2]), int(sys.argv[3]), int(sys.argv[4]) +for path in sorted([f"{sgl_job_dir}/logs/{name}/sglang_isl_{isl}_osl_{osl}" for name in os.listdir(f"{sgl_job_dir}/logs/") if os.path.isdir(f"{sgl_job_dir}/logs/{name}/sglang_isl_{isl}_osl_{osl}")], key=os.path.getmtime, reverse=True)[:nexp]: + print(path) +PY + + LOGS_DIR=$(python3 collect_latest_results.py "$BENCHMARK_LOGS_DIR" "$ISL" "$OSL" 1) + if [ -z "$LOGS_DIR" ]; then + echo "No logs directory found for ISL=${ISL}, OSL=${OSL}" + exit 1 + fi + + echo "Found logs directory: $LOGS_DIR" + ls -la "$LOGS_DIR" + + for result_file in $(find $LOGS_DIR -type f); do + file_name=$(basename $result_file) + if [ -f $result_file ]; then + WORKSPACE_RESULT_FILE="$GITHUB_WORKSPACE/${RESULT_FILENAME}_${file_name}" + echo "Found result file ${result_file}. Copying it to ${WORKSPACE_RESULT_FILE}" + cp $result_file $WORKSPACE_RESULT_FILE + fi + done + + echo "All result files processed" + set +x + scancel_sync $JOB_ID + set -x + echo "Canceled the slurm job $JOB_ID" + + # NFS-safe cleanup: use timeout to avoid hanging on stale NFS locks + timeout --kill-after=5 30 sudo rm -rf "$BENCHMARK_LOGS_DIR/logs" 2>/dev/null || true + + if [[ -n "${GITHUB_ACTIONS:-}" ]]; then + ARTIFACT_DIR="$GITHUB_WORKSPACE/benchmark_artifacts" + mkdir -p "$ARTIFACT_DIR" + cp -r "$BENCHMARK_LOGS_DIR"/slurm_job-${JOB_ID}.{out,err} "$ARTIFACT_DIR/" 2>/dev/null || true + echo "Logs copied to $ARTIFACT_DIR for artifact upload" + fi + +# ============================================================================= +# Single-node path: enroot via salloc + srun +# ============================================================================= +else + + SQUASH_FILE="/nfsdata/sa/gharunner/gharunners/squash/$(echo "$IMAGE" | sed 's/[\/:@#]/_/g').sqsh" + LOCK_FILE="${SQUASH_FILE}.lock" + + set -x + + JOB_ID=$(salloc --partition=$PARTITION --gres=gpu:$TP --cpus-per-task=256 --time=480 --no-shell --job-name="$RUNNER_NAME" 2>&1 | tee /dev/stderr | grep -oP 'Granted job allocation \K[0-9]+') + + if [ -z "$JOB_ID" ]; then + echo "ERROR: salloc failed to allocate a job" + exit 1 + fi + + srun --jobid=$JOB_ID --job-name="$RUNNER_NAME" bash -c " + exec 9>\"$LOCK_FILE\" + flock -w 600 9 || { echo 'Failed to acquire lock for $SQUASH_FILE'; exit 1; } + if unsquashfs -l \"$SQUASH_FILE\" > /dev/null 2>&1; then + echo 'Squash file already exists and is valid, skipping import' + else + rm -f \"$SQUASH_FILE\" + enroot import -o \"$SQUASH_FILE\" docker://$IMAGE + fi + " + srun --jobid=$JOB_ID \ + --container-image=$SQUASH_FILE \ + --container-mounts=$GITHUB_WORKSPACE:/workspace/,$HF_HUB_CACHE_MOUNT:$HF_HUB_CACHE \ + --container-mount-home \ + --container-writable \ + --container-remap-root \ + --container-workdir=/workspace/ \ + --no-container-entrypoint --export=ALL \ + bash benchmarks/single_node/${SCRIPT_NAME} + + scancel $JOB_ID + +fi diff --git a/scripts/manual-test-mi325x.sh b/scripts/manual-test-mi325x.sh new file mode 100755 index 000000000..30ec87d6a --- /dev/null +++ b/scripts/manual-test-mi325x.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash +set -euo pipefail + +cd "$(dirname "${BASH_SOURCE[0]}")/.." + +export GITHUB_WORKSPACE=$(pwd) +export RUNNER_NAME=mi325x-amd-manual + +export MODEL=deepseek-ai/DeepSeek-R1-0528 +export EXP_NAME=dsr1_1k1k +export PRECISION=fp8 +export FRAMEWORK=sglang-disagg + +export IMAGE=ghcr.io/jordannanos/sgl-mi325x-mori:v0.5.9-bnxt-good + +export ISL=1024 +export OSL=1024 +export CONC_LIST="1024 512 256 128 64 32 16 8 4 2 1" +export SPEC_DECODING=none +export RANDOM_RANGE_RATIO=1 + +export PREFILL_NODES=1 +export PREFILL_NUM_WORKERS=1 +export PREFILL_TP=4 +export PREFILL_EP=1 +export PREFILL_DP_ATTN=false + +export DECODE_NODES=1 +export DECODE_NUM_WORKERS=1 +export DECODE_TP=8 +export DECODE_EP=1 +export DECODE_DP_ATTN=false + +bash runners/launch_mi325x-amd.sh + +#model files are here: +#/nfsdata/sa/gharunner/gharunners/hf-hub-cache/models--deepseek-ai--DeepSeek-R1-0528 \ No newline at end of file