From 044a1d6d5757269fa80c97137a8654059e101f2d Mon Sep 17 00:00:00 2001 From: tannerbruhn Date: Mon, 31 Aug 2026 19:01:32 +0000 Subject: [PATCH 1/2] CUDA: Allow CUDA optimization per split for multi-GPU. Previous guard caused multi-GPU to skip the graph optimization. The graph is already split per device and the optimization doesnt run over the whole model but once per split, and thus should be allowed. However, the CUDA event ggml_cuda_concurrent_event belongs to whichever GPU was "current" when created. If the pass ran while GPU 0 was current, it would stick and during event creation for the second GPU it would land on GPU 0. The fix: set the device explicitly ggml_cuda_set_device(cuda_ctx->device); Default behaviour remains unchanged, only active for GGML_CUDA_GRAPH_OPT=1. Explicit device setting pattern re-used from ggml_backend_cuda_graph_compute. --- ggml/src/ggml-cuda/ggml-cuda.cu | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu index 31f5aeeacc38..8f0df5ab835d 100644 --- a/ggml/src/ggml-cuda/ggml-cuda.cu +++ b/ggml/src/ggml-cuda/ggml-cuda.cu @@ -4365,10 +4365,16 @@ static void ggml_backend_cuda_graph_optimize(ggml_backend_t backend, ggml_cgraph ggml_cuda_stream_context & stream_context = cuda_ctx->stream_context(); stream_context.reset(); - if (!use_cuda_graph || ggml_backend_cuda_get_device_count() != 1) { + if (!use_cuda_graph) { return; } + // This pass runs per scheduler split, and a split's nodes are all allocated on this + // backend's device, so multiple CUDA devices are safe as long as the fork/join events + // created below land on this device: make it current explicitly, since the scheduler + // thread may have a different device current when optimizing this split. + ggml_cuda_set_device(cuda_ctx->device); + // number of out-degrees for a particular node std::unordered_map fan_out; // reverse mapping of node to index in the cgraph From 3f4500f9bcabd1e60b5d0f2f0d6de2b58bedebf0 Mon Sep 17 00:00:00 2001 From: Tanner Bruhn <66120666+tannerbruhn@users.noreply.github.com> Date: Wed, 2 Sep 2026 18:18:27 +0200 Subject: [PATCH 2/2] Update ggml/src/ggml-cuda/ggml-cuda.cu Co-authored-by: Aman Gupta --- ggml/src/ggml-cuda/ggml-cuda.cu | 4 ---- 1 file changed, 4 deletions(-) diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu index 8f0df5ab835d..94c7d4060566 100644 --- a/ggml/src/ggml-cuda/ggml-cuda.cu +++ b/ggml/src/ggml-cuda/ggml-cuda.cu @@ -4369,10 +4369,6 @@ static void ggml_backend_cuda_graph_optimize(ggml_backend_t backend, ggml_cgraph return; } - // This pass runs per scheduler split, and a split's nodes are all allocated on this - // backend's device, so multiple CUDA devices are safe as long as the fork/join events - // created below land on this device: make it current explicitly, since the scheduler - // thread may have a different device current when optimizing this split. ggml_cuda_set_device(cuda_ctx->device); // number of out-degrees for a particular node