From 9f21af17303d55b9813b022e4f19cc14534603c4 Mon Sep 17 00:00:00 2001 From: "Chen, Todd" Date: Tue, 31 Mar 2026 04:34:34 -0500 Subject: [PATCH 01/10] feat: enhance Qwen benchmark scripts with additional parameters * Added CONTEXT_LENGTH and MAX_PREFILL_TOKENS variables for better configuration. * Updated launch_server command with new options: --tokenizer-worker-num, --enable-aiter-allreduce-fusion, --cuda-graph-max-bs, --context-length, --disable-radix-cache, --max-prefill-tokens, and --scheduler-recv-interval. --- benchmarks/single_node/qwen3.5_bf16_mi355x.sh | 9 +++++++++ benchmarks/single_node/qwen3.5_fp8_mi355x.sh | 9 +++++++++ 2 files changed, 18 insertions(+) diff --git a/benchmarks/single_node/qwen3.5_bf16_mi355x.sh b/benchmarks/single_node/qwen3.5_bf16_mi355x.sh index 701695def..b67ab63cf 100755 --- a/benchmarks/single_node/qwen3.5_bf16_mi355x.sh +++ b/benchmarks/single_node/qwen3.5_bf16_mi355x.sh @@ -19,6 +19,8 @@ hf download "$MODEL" SERVER_LOG=/workspace/server.log PORT=${PORT:-8888} +CONTEXT_LENGTH=$((ISL + OSL + 20)) +MAX_PREFILL_TOKENS=32768 EVAL_CONTEXT_ARGS="" if [ "${EVAL_ONLY}" = "true" ]; then @@ -35,6 +37,13 @@ python3 -m sglang.launch_server \ --port $PORT \ --tensor-parallel-size $TP \ --trust-remote-code \ + --tokenizer-worker-num 6 \ + --enable-aiter-allreduce-fusion \ + --cuda-graph-max-bs $CONC \ + --context-length $CONTEXT_LENGTH \ + --disable-radix-cache \ + --max-prefill-tokens $MAX_PREFILL_TOKENS \ + --scheduler-recv-interval 30 \ --mem-fraction-static 0.8 $EVAL_CONTEXT_ARGS > $SERVER_LOG 2>&1 & SERVER_PID=$! diff --git a/benchmarks/single_node/qwen3.5_fp8_mi355x.sh b/benchmarks/single_node/qwen3.5_fp8_mi355x.sh index 701695def..b67ab63cf 100644 --- a/benchmarks/single_node/qwen3.5_fp8_mi355x.sh +++ b/benchmarks/single_node/qwen3.5_fp8_mi355x.sh @@ -19,6 +19,8 @@ hf download "$MODEL" SERVER_LOG=/workspace/server.log PORT=${PORT:-8888} +CONTEXT_LENGTH=$((ISL + OSL + 20)) +MAX_PREFILL_TOKENS=32768 EVAL_CONTEXT_ARGS="" if [ "${EVAL_ONLY}" = "true" ]; then @@ -35,6 +37,13 @@ python3 -m sglang.launch_server \ --port $PORT \ --tensor-parallel-size $TP \ --trust-remote-code \ + --tokenizer-worker-num 6 \ + --enable-aiter-allreduce-fusion \ + --cuda-graph-max-bs $CONC \ + --context-length $CONTEXT_LENGTH \ + --disable-radix-cache \ + --max-prefill-tokens $MAX_PREFILL_TOKENS \ + --scheduler-recv-interval 30 \ --mem-fraction-static 0.8 $EVAL_CONTEXT_ARGS > $SERVER_LOG 2>&1 & SERVER_PID=$! From 44bc53a41b33610e124fb6ff1acf161ffc6fffa1 Mon Sep 17 00:00:00 2001 From: "Chen, Todd" Date: Tue, 31 Mar 2026 04:34:34 -0500 Subject: [PATCH 02/10] Update perf-changelog.yaml to include new Qwen3.5 FP8 and BF16 SGLang benchmark configurations for MI355X, enhancing performance with updated CLI arguments. --- perf-changelog.yaml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 967edc19c..115fdc40a 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -1220,3 +1220,10 @@ - "Uses nvidia/GLM-5-NVFP4 model with modelopt_fp4 quantization" - "Image: lmsysorg/sglang:nightly-dev-cu13-20260328-a27651d5" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/973 +- config-keys: + - qwen3.5-fp8-mi355x-sglang + - qwen3.5-bf16-mi355x-sglang + description: + - "Update cli args of Qwen3.5 FP8 and BF16 SGLang benchmarks for MI355X to achieve better performance" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/942 + From 128ddfd591b1b4f28ec0d72bc0468b34de583e33 Mon Sep 17 00:00:00 2001 From: "Chen, Todd" Date: Tue, 31 Mar 2026 04:34:34 -0500 Subject: [PATCH 03/10] Update SGLang image versions for Qwen3.5 configurations in amd-master.yaml to v0.5.9, ensuring compatibility with recent changes. --- .github/configs/amd-master.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/configs/amd-master.yaml b/.github/configs/amd-master.yaml index e84fc0da5..ffbb2b1da 100644 --- a/.github/configs/amd-master.yaml +++ b/.github/configs/amd-master.yaml @@ -114,7 +114,7 @@ dsr1-fp8-mi355x-sglang: - { tp: 8, conc-start: 4, conc-end: 64 } qwen3.5-bf16-mi355x-sglang: - image: rocm/sgl-dev:v0.5.8.post1-rocm720-mi35x-20260215 + image: lmsysorg/sglang-rocm:v0.5.9-rocm720-mi35x-20260325 model: Qwen/Qwen3.5-397B-A17B model-prefix: qwen3.5 runner: mi355x @@ -186,7 +186,7 @@ qwen3.5-fp8-mi325x-sglang: - { tp: 8, conc-start: 4, conc-end: 64 } qwen3.5-fp8-mi355x-sglang: - image: rocm/sgl-dev:v0.5.8.post1-rocm720-mi35x-20260218 + image: lmsysorg/sglang-rocm:v0.5.9-rocm720-mi35x-20260325 model: Qwen/Qwen3.5-397B-A17B-FP8 model-prefix: qwen3.5 runner: mi355x From ab7166e0cd461be63242e1dbfb0ceb4d5e5aabd8 Mon Sep 17 00:00:00 2001 From: "Chen, Todd" Date: Tue, 31 Mar 2026 04:34:34 -0500 Subject: [PATCH 04/10] use 0327 build --- .github/configs/amd-master.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/configs/amd-master.yaml b/.github/configs/amd-master.yaml index ffbb2b1da..f504c1368 100644 --- a/.github/configs/amd-master.yaml +++ b/.github/configs/amd-master.yaml @@ -114,7 +114,7 @@ dsr1-fp8-mi355x-sglang: - { tp: 8, conc-start: 4, conc-end: 64 } qwen3.5-bf16-mi355x-sglang: - image: lmsysorg/sglang-rocm:v0.5.9-rocm720-mi35x-20260325 + image: lmsysorg/sglang-rocm:v0.5.9-rocm720-mi35x-20260327 model: Qwen/Qwen3.5-397B-A17B model-prefix: qwen3.5 runner: mi355x @@ -186,7 +186,7 @@ qwen3.5-fp8-mi325x-sglang: - { tp: 8, conc-start: 4, conc-end: 64 } qwen3.5-fp8-mi355x-sglang: - image: lmsysorg/sglang-rocm:v0.5.9-rocm720-mi35x-20260325 + image: lmsysorg/sglang-rocm:v0.5.9-rocm720-mi35x-20260327 model: Qwen/Qwen3.5-397B-A17B-FP8 model-prefix: qwen3.5 runner: mi355x From 036a487a95b04382dc9fa6cf62cd0c0703554a1f Mon Sep 17 00:00:00 2001 From: "Chen, Todd" Date: Tue, 31 Mar 2026 04:34:34 -0500 Subject: [PATCH 05/10] Update perf-changelog.yaml to reflect the new PR link for Qwen3.5 FP8 and BF16 SGLang benchmarks on MI355X, ensuring accurate tracking of performance enhancements. --- perf-changelog.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 115fdc40a..85bfcd95e 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -1225,5 +1225,5 @@ - qwen3.5-bf16-mi355x-sglang description: - "Update cli args of Qwen3.5 FP8 and BF16 SGLang benchmarks for MI355X to achieve better performance" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/942 + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/980 From 5077256b0d23a95420fecf9e20ddc2da4daa0dbd Mon Sep 17 00:00:00 2001 From: "Chen, Todd" Date: Tue, 31 Mar 2026 04:34:34 -0500 Subject: [PATCH 06/10] Update Qwen3.5 image tags in amd-master.yaml to v0.5.10rc0 for MI355X configurations and adjust perf-changelog.yaml to reflect the changes, ensuring accurate performance tracking and compatibility. --- .github/configs/amd-master.yaml | 4 ++-- perf-changelog.yaml | 5 +++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/configs/amd-master.yaml b/.github/configs/amd-master.yaml index f504c1368..77ff8de80 100644 --- a/.github/configs/amd-master.yaml +++ b/.github/configs/amd-master.yaml @@ -114,7 +114,7 @@ dsr1-fp8-mi355x-sglang: - { tp: 8, conc-start: 4, conc-end: 64 } qwen3.5-bf16-mi355x-sglang: - image: lmsysorg/sglang-rocm:v0.5.9-rocm720-mi35x-20260327 + image: lmsysorg/sglang-rocm:v0.5.10rc0-rocm720-mi35x-20260327 model: Qwen/Qwen3.5-397B-A17B model-prefix: qwen3.5 runner: mi355x @@ -186,7 +186,7 @@ qwen3.5-fp8-mi325x-sglang: - { tp: 8, conc-start: 4, conc-end: 64 } qwen3.5-fp8-mi355x-sglang: - image: lmsysorg/sglang-rocm:v0.5.9-rocm720-mi35x-20260327 + image: lmsysorg/sglang-rocm:v0.5.10rc0-rocm720-mi35x-20260327 model: Qwen/Qwen3.5-397B-A17B-FP8 model-prefix: qwen3.5 runner: mi355x diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 85bfcd95e..0e5bd6952 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -1221,9 +1221,10 @@ - "Image: lmsysorg/sglang:nightly-dev-cu13-20260328-a27651d5" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/973 - config-keys: - - qwen3.5-fp8-mi355x-sglang - qwen3.5-bf16-mi355x-sglang + - qwen3.5-fp8-mi355x-sglang description: - - "Update cli args of Qwen3.5 FP8 and BF16 SGLang benchmarks for MI355X to achieve better performance" + - "Fix MI355X Qwen 3.5 image tag: v0.5.9-rocm720-mi35x-20260327 is not on Docker Hub (404)" + - "Use lmsysorg/sglang-rocm:v0.5.10rc0-rocm720-mi35x-20260329" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/980 From 46e763fb266ffc17dcdc43345038fe4518d3f0cf Mon Sep 17 00:00:00 2001 From: "Chen, Todd" Date: Tue, 31 Mar 2026 04:34:34 -0500 Subject: [PATCH 07/10] Update Qwen3.5 FP8 and BF16 SGLang benchmark descriptions in perf-changelog.yaml to reflect improved CLI arguments for MI355X, ensuring better performance tracking. --- perf-changelog.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 0e5bd6952..68d38ea70 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -1224,7 +1224,7 @@ - qwen3.5-bf16-mi355x-sglang - qwen3.5-fp8-mi355x-sglang description: - - "Fix MI355X Qwen 3.5 image tag: v0.5.9-rocm720-mi35x-20260327 is not on Docker Hub (404)" - - "Use lmsysorg/sglang-rocm:v0.5.10rc0-rocm720-mi35x-20260329" + - "Update cli args of Qwen3.5 FP8 and BF16 SGLang benchmarks for MI355X to achieve better performance" + - "Use lmsysorg/sglang-rocm:v0.5.10rc0-rocm720-mi35x-20260327" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/980 From bf316daee69b68028c799add52ea84eeb22910cb Mon Sep 17 00:00:00 2001 From: "Chen, Todd" Date: Tue, 31 Mar 2026 04:34:34 -0500 Subject: [PATCH 08/10] Enhance Qwen3.5 benchmark scripts for MI355X by adding EP_SIZE parameter and adjusting memory fraction. Updated launch_server command to include data-parallel-size and improved context length handling for better performance. --- benchmarks/single_node/qwen3.5_bf16_mi355x.sh | 8 ++++++-- benchmarks/single_node/qwen3.5_fp8_mi355x.sh | 8 ++++++-- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/benchmarks/single_node/qwen3.5_bf16_mi355x.sh b/benchmarks/single_node/qwen3.5_bf16_mi355x.sh index b67ab63cf..f7a01a6d1 100755 --- a/benchmarks/single_node/qwen3.5_bf16_mi355x.sh +++ b/benchmarks/single_node/qwen3.5_bf16_mi355x.sh @@ -9,7 +9,8 @@ check_env_vars \ ISL \ OSL \ RANDOM_RANGE_RATIO \ - RESULT_FILENAME + RESULT_FILENAME \ + EP_SIZE if [[ -n "$SLURM_JOB_ID" ]]; then echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" @@ -26,6 +27,7 @@ EVAL_CONTEXT_ARGS="" if [ "${EVAL_ONLY}" = "true" ]; then setup_eval_context EVAL_CONTEXT_ARGS="--context-length $EVAL_MAX_MODEL_LEN" +else EVAL_CONTEXT_ARGS="--context-length $CONTEXT_LENGTH" fi # Start GPU monitoring (power, temperature, clocks every second) start_gpu_monitor @@ -36,6 +38,8 @@ python3 -m sglang.launch_server \ --host=0.0.0.0 \ --port $PORT \ --tensor-parallel-size $TP \ + --ep-size $EP_SIZE \ + --data-parallel-size 1 \ --trust-remote-code \ --tokenizer-worker-num 6 \ --enable-aiter-allreduce-fusion \ @@ -44,7 +48,7 @@ python3 -m sglang.launch_server \ --disable-radix-cache \ --max-prefill-tokens $MAX_PREFILL_TOKENS \ --scheduler-recv-interval 30 \ - --mem-fraction-static 0.8 $EVAL_CONTEXT_ARGS > $SERVER_LOG 2>&1 & + --mem-fraction-static 0.75 $EVAL_CONTEXT_ARGS > $SERVER_LOG 2>&1 & SERVER_PID=$! diff --git a/benchmarks/single_node/qwen3.5_fp8_mi355x.sh b/benchmarks/single_node/qwen3.5_fp8_mi355x.sh index b67ab63cf..f7a01a6d1 100644 --- a/benchmarks/single_node/qwen3.5_fp8_mi355x.sh +++ b/benchmarks/single_node/qwen3.5_fp8_mi355x.sh @@ -9,7 +9,8 @@ check_env_vars \ ISL \ OSL \ RANDOM_RANGE_RATIO \ - RESULT_FILENAME + RESULT_FILENAME \ + EP_SIZE if [[ -n "$SLURM_JOB_ID" ]]; then echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" @@ -26,6 +27,7 @@ EVAL_CONTEXT_ARGS="" if [ "${EVAL_ONLY}" = "true" ]; then setup_eval_context EVAL_CONTEXT_ARGS="--context-length $EVAL_MAX_MODEL_LEN" +else EVAL_CONTEXT_ARGS="--context-length $CONTEXT_LENGTH" fi # Start GPU monitoring (power, temperature, clocks every second) start_gpu_monitor @@ -36,6 +38,8 @@ python3 -m sglang.launch_server \ --host=0.0.0.0 \ --port $PORT \ --tensor-parallel-size $TP \ + --ep-size $EP_SIZE \ + --data-parallel-size 1 \ --trust-remote-code \ --tokenizer-worker-num 6 \ --enable-aiter-allreduce-fusion \ @@ -44,7 +48,7 @@ python3 -m sglang.launch_server \ --disable-radix-cache \ --max-prefill-tokens $MAX_PREFILL_TOKENS \ --scheduler-recv-interval 30 \ - --mem-fraction-static 0.8 $EVAL_CONTEXT_ARGS > $SERVER_LOG 2>&1 & + --mem-fraction-static 0.75 $EVAL_CONTEXT_ARGS > $SERVER_LOG 2>&1 & SERVER_PID=$! From 251c54e76beb4df2cd2eb7f2163f1d360b607bfa Mon Sep 17 00:00:00 2001 From: "Chen, Todd" Date: Tue, 31 Mar 2026 04:34:34 -0500 Subject: [PATCH 09/10] Update search-space configurations in amd-master.yaml for Qwen3.5 benchmarks, increasing conc-end values and adding new entries for improved performance tuning on MI355X and MI300X. --- .github/configs/amd-master.yaml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/configs/amd-master.yaml b/.github/configs/amd-master.yaml index 77ff8de80..58323b6a8 100644 --- a/.github/configs/amd-master.yaml +++ b/.github/configs/amd-master.yaml @@ -125,11 +125,13 @@ qwen3.5-bf16-mi355x-sglang: - isl: 1024 osl: 1024 search-space: - - { tp: 8, conc-start: 4, conc-end: 64 } + - { tp: 8, conc-start: 4, conc-end: 256 } + - { tp: 8, ep: 8, conc-start: 64, conc-end: 256 } - isl: 8192 osl: 1024 search-space: - - { tp: 8, conc-start: 4, conc-end: 64 } + - { tp: 8, conc-start: 4, conc-end: 256 } + - { tp: 8, ep: 8, conc-start: 64, conc-end: 256 } qwen3.5-bf16-mi300x-sglang: image: lmsysorg/sglang:v0.5.9-rocm720-mi30x @@ -198,10 +200,12 @@ qwen3.5-fp8-mi355x-sglang: osl: 1024 search-space: - { tp: 8, conc-start: 4, conc-end: 64 } + - { tp: 4, ep: 4, conc-start: 16, conc-end: 256 } - isl: 8192 osl: 1024 search-space: - { tp: 8, conc-start: 4, conc-end: 64 } + - { tp: 4, ep: 4, conc-start: 16, conc-end: 256 } qwen3.5-fp8-mi300x-sglang: image: lmsysorg/sglang:v0.5.9-rocm720-mi30x From 6fe84226af582d43658ece06226bc3e7d8ca0aae Mon Sep 17 00:00:00 2001 From: "Chen, Todd" Date: Tue, 31 Mar 2026 04:34:34 -0500 Subject: [PATCH 10/10] Remove context length parameter from Qwen3.5 BF16 and FP8 benchmark scripts for MI355X to streamline configuration and improve performance. --- benchmarks/single_node/qwen3.5_bf16_mi355x.sh | 1 - benchmarks/single_node/qwen3.5_fp8_mi355x.sh | 1 - 2 files changed, 2 deletions(-) diff --git a/benchmarks/single_node/qwen3.5_bf16_mi355x.sh b/benchmarks/single_node/qwen3.5_bf16_mi355x.sh index f7a01a6d1..ce82b9a53 100755 --- a/benchmarks/single_node/qwen3.5_bf16_mi355x.sh +++ b/benchmarks/single_node/qwen3.5_bf16_mi355x.sh @@ -44,7 +44,6 @@ python3 -m sglang.launch_server \ --tokenizer-worker-num 6 \ --enable-aiter-allreduce-fusion \ --cuda-graph-max-bs $CONC \ - --context-length $CONTEXT_LENGTH \ --disable-radix-cache \ --max-prefill-tokens $MAX_PREFILL_TOKENS \ --scheduler-recv-interval 30 \ diff --git a/benchmarks/single_node/qwen3.5_fp8_mi355x.sh b/benchmarks/single_node/qwen3.5_fp8_mi355x.sh index f7a01a6d1..ce82b9a53 100644 --- a/benchmarks/single_node/qwen3.5_fp8_mi355x.sh +++ b/benchmarks/single_node/qwen3.5_fp8_mi355x.sh @@ -44,7 +44,6 @@ python3 -m sglang.launch_server \ --tokenizer-worker-num 6 \ --enable-aiter-allreduce-fusion \ --cuda-graph-max-bs $CONC \ - --context-length $CONTEXT_LENGTH \ --disable-radix-cache \ --max-prefill-tokens $MAX_PREFILL_TOKENS \ --scheduler-recv-interval 30 \