From 881f974fd03644a7d8ed8d89c2c34a306fd87b0b Mon Sep 17 00:00:00 2001 From: Aditya Ukarande Date: Tue, 14 Apr 2026 23:47:25 -0700 Subject: [PATCH] feat: GPU/CPU roofline profiler for per-op benchmarking --- examples/CMakeLists.txt | 1 + examples/llama-profiler/CMakeLists.txt | 19 + examples/llama-profiler/profiler-common.h | 161 ++++++ examples/llama-profiler/profiler-cpu.cpp | 595 ++++++++++++++++++++++ examples/llama-profiler/profiler-gpu.cpp | 424 +++++++++++++++ 5 files changed, 1200 insertions(+) create mode 100644 examples/llama-profiler/CMakeLists.txt create mode 100644 examples/llama-profiler/profiler-common.h create mode 100644 examples/llama-profiler/profiler-cpu.cpp create mode 100644 examples/llama-profiler/profiler-gpu.cpp diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt index a29dc707c3dc..4f3247bbf819 100644 --- a/examples/CMakeLists.txt +++ b/examples/CMakeLists.txt @@ -35,6 +35,7 @@ else() add_subdirectory(gen-docs) add_subdirectory(training) add_subdirectory(diffusion) + add_subdirectory(llama-profiler) if (NOT GGML_BACKEND_DL) add_subdirectory(convert-llama2c-to-ggml) # these examples use the backends directly and cannot be built with dynamic loading diff --git a/examples/llama-profiler/CMakeLists.txt b/examples/llama-profiler/CMakeLists.txt new file mode 100644 index 000000000000..1e4b88eb5642 --- /dev/null +++ b/examples/llama-profiler/CMakeLists.txt @@ -0,0 +1,19 @@ +set(TARGET_CPU llama-profiler-cpu) +add_executable(${TARGET_CPU} profiler-cpu.cpp) +install(TARGETS ${TARGET_CPU} RUNTIME) +target_link_libraries(${TARGET_CPU} PRIVATE ggml ggml-base common ${CMAKE_THREAD_LIBS_INIT}) +target_compile_features(${TARGET_CPU} PRIVATE cxx_std_17) +if (GGML_CUDA) + target_link_libraries(${TARGET_CPU} PRIVATE ggml-cuda) + target_compile_definitions(${TARGET_CPU} PRIVATE GGML_USE_CUDA=1) +endif() + +set(TARGET_GPU llama-profiler-gpu) +add_executable(${TARGET_GPU} profiler-gpu.cpp) +install(TARGETS ${TARGET_GPU} RUNTIME) +target_link_libraries(${TARGET_GPU} PRIVATE ggml ggml-base ${CMAKE_THREAD_LIBS_INIT}) +if (GGML_CUDA) + target_link_libraries(${TARGET_GPU} PRIVATE ggml-cuda) + target_compile_definitions(${TARGET_GPU} PRIVATE GGML_USE_CUDA=1) +endif() +target_compile_features(${TARGET_GPU} PRIVATE cxx_std_17) diff --git a/examples/llama-profiler/profiler-common.h b/examples/llama-profiler/profiler-common.h new file mode 100644 index 000000000000..ae22f7c32f79 --- /dev/null +++ b/examples/llama-profiler/profiler-common.h @@ -0,0 +1,161 @@ +#pragma once + +#include "ggml.h" +#include "ggml-backend.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +struct bench_timer { + using clk = std::chrono::high_resolution_clock; + clk::time_point t0; + void start() { t0 = clk::now(); } + double stop() { return std::chrono::duration(clk::now() - t0).count(); } +}; + +struct bench_result { + std::string op_name; + std::string quant_type; + + int N = 0, K = 0, B = 0; + int n_tokens = 0, ctx_len = 0, n_heads = 0, head_dim = 0; + int64_t n_elements = 0; + + double ops = 0.0; + double bytes = 0.0; + double time_s = 0.0; + + float arithmetic_intensity = 0.0f; + float effective_gflops = 0.0f; + float effective_bw_gb_s = 0.0f; + + void calculate_derived() { + arithmetic_intensity = (bytes > 0.0) ? (float)(ops / bytes) : 0.0f; + effective_gflops = (time_s > 0.0) ? (float)(ops / time_s / 1e9) : 0.0f; + effective_bw_gb_s = (time_s > 0.0) ? (float)(bytes / time_s / 1e9) : 0.0f; + } + + void print_dims() const { + if (op_name.find("MUL_MAT_ID") != std::string::npos) { + printf(" [N=%d K=%d experts=%d/%d B=%d]", N, K, n_tokens, ctx_len, B); + } else if (op_name.find("MUL_MAT") != std::string::npos) { + printf(" [N=%d K=%d B=%d]", N, K, B); + } else if (op_name.find("FLASH_ATTN") != std::string::npos) { + printf(" [tokens=%d ctx=%d heads=%d dim=%d]", n_tokens, ctx_len, n_heads, head_dim); + } else { + printf(" [n=%lld]", (long long)n_elements); + } + } +}; + +struct matmul_size { int32_t N; int32_t K; }; +struct moe_config { int32_t N; int32_t K; int32_t n_experts; int32_t n_experts_used; }; +struct attn_config { const char * name; int32_t n_q_heads; int32_t n_kv_heads; int32_t head_dim; }; + +inline std::vector get_matmul_sizes(bool fast) { + if (fast) { + return { + { 1024, 1024}, { 2048, 2048}, { 4096, 4096}, { 8192, 8192}, + { 512, 2048}, { 8192, 4096}, { 14336, 4096}, { 4096, 14336}, + {128256, 4096}, + }; + } + return { + { 1024, 1024}, { 2048, 2048}, { 4096, 4096}, { 8192, 8192}, { 16384, 16384}, + { 2048, 1024}, { 4096, 2048}, { 8192, 4096}, { 14336, 4096}, { 16384, 8192}, + { 22016, 4096}, { 28672, 8192}, + { 1024, 2048}, { 2048, 4096}, { 4096, 8192}, { 4096, 14336}, { 4096, 22016}, + { 8192, 16384}, { 8192, 28672}, + { 1024, 512}, { 1024, 4096}, + { 1536, 4096}, { 2048, 7168}, { 8192, 5120}, + { 32000, 4096}, {128256, 4096}, {128256, 8192}, {151936, 1024}, {151936, 8192}, + {248320, 4096}, + }; +} + +inline std::vector get_moe_configs(bool fast) { + if (fast) { + return { + { 2048, 7168, 256, 8}, { 1536, 4096, 128, 8}, {14336, 4096, 8, 2}, + { 8192, 5120, 16, 1}, { 1024, 1024, 128, 8}, { 2048, 1024, 128, 8}, + { 4096, 4096, 128, 8}, + }; + } + return { + { 1024, 4096, 512, 10}, { 1536, 4096, 128, 8}, { 2048, 7168, 256, 8}, + { 8192, 5120, 16, 1}, { 8192, 5120, 128, 1}, {14336, 4096, 8, 2}, + { 1024, 1024, 128, 8}, { 2048, 1024, 128, 8}, { 4096, 4096, 128, 8}, + }; +} + +inline std::vector get_attn_configs(bool fast) { + if (fast) { + return { + {"MHA", 32, 32, 128}, {"GQA-8", 32, 8, 128}, + {"GQA-4", 32, 4, 128}, {"MQA", 32, 1, 128}, + }; + } + return { + {"MHA", 32, 32, 128}, {"GQA-16", 32, 16, 128}, {"GQA-8", 32, 8, 128}, + {"GQA-4", 64, 4, 128}, {"GQA-2", 32, 2, 256}, {"MQA", 32, 1, 128}, + }; +} + +inline std::vector get_attn_ctx_lens(bool fast) { + if (fast) return { 1024, 4096, 8192, 16384 }; + return { 1024, 2048, 4096, 8192, 16384, 32768, 65536 }; +} + +inline std::vector get_matmul_quants(bool fast) { + if (fast) { + return { GGML_TYPE_F16, GGML_TYPE_Q8_0, GGML_TYPE_Q4_0, GGML_TYPE_Q5_0, GGML_TYPE_Q2_K, GGML_TYPE_MXFP4 }; + } + return { GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_Q8_0, GGML_TYPE_Q4_0, GGML_TYPE_Q4_1, GGML_TYPE_Q5_0, GGML_TYPE_Q2_K, GGML_TYPE_MXFP4 }; +} + +inline bool parse_int_arg(const char * s, int32_t & out) { + char * end = nullptr; + long val = strtol(s, &end, 10); + if (end == s || *end != '\0' || val < INT32_MIN || val > INT32_MAX) { + return false; + } + out = (int32_t)val; + return true; +} + +struct ridge_result { + std::string key; + double peak_gflops; + double measured_bw; + double ridge; +}; + +template +inline std::vector compute_ridge_points( + const std::vector & results, double measured_bw) { + std::map groups; + + for (const auto & r : results) { + std::string key = r.op_name + "_" + r.quant_type; + groups[key] = std::max(groups[key], (double)r.effective_gflops); + } + + std::vector out; + for (const auto & [key, peak] : groups) { + double ridge = (measured_bw > 0.0) ? (peak / measured_bw) : 0.0; + out.push_back({key, peak, measured_bw, ridge}); + } + return out; +} + +inline std::vector create_quantized_data(ggml_type type, int64_t n_elements) { + size_t quant_size = ggml_row_size(type, n_elements); + return std::vector(quant_size, 0); +} diff --git a/examples/llama-profiler/profiler-cpu.cpp b/examples/llama-profiler/profiler-cpu.cpp new file mode 100644 index 000000000000..37943570c2be --- /dev/null +++ b/examples/llama-profiler/profiler-cpu.cpp @@ -0,0 +1,595 @@ +#define CPU_WARMUP_ITERS 2 +#define CPU_TIMED_ITERS 2 + +#include "profiler-common.h" + +#include "common.h" +#include "ggml-cpu.h" + +#ifdef GGML_USE_CUDA +#include "ggml-cuda.h" +#endif + +#include +#include +#include +#include + +#if defined(_MSC_VER) +#include +#include +#endif + +static std::vector g_flush_buffer; + +static void flush_caches() { + if (g_flush_buffer.empty()) return; + + volatile char sum = 0; + for (size_t i = 0; i < g_flush_buffer.size(); i += 64) { + sum += g_flush_buffer[i]; + g_flush_buffer[i] = (char)(i & 0xFF); + } + g_flush_buffer[0] = sum; + +#if defined(_MSC_VER) + _mm_mfence(); +#elif defined(__GNUC__) || defined(__clang__) + __sync_synchronize(); +#endif +} + +static void init_flush_buffer() { + const size_t flush_size = 256 * 1024 * 1024; + g_flush_buffer.resize(flush_size); + for (size_t i = 0; i < flush_size; i += 4096) { + g_flush_buffer[i] = (char)(i & 0xFF); + } +} + +static double benchmark_cpu_dram_bandwidth(int threads) { + const size_t pool_bytes = 1024ULL * 1024 * 1024; + const size_t chunk_per_thread = pool_bytes / threads; + const int iterations = 10; + + std::vector pool(pool_bytes); + for (size_t i = 0; i < pool.size(); i += 4096) { + pool[i] = (uint8_t)(i & 0xFF); + } + + std::vector workers; + std::vector thread_bytes(threads, 0.0); + + bench_timer t; + t.start(); + + for (int tid = 0; tid < threads; ++tid) { + workers.emplace_back([&pool, &thread_bytes, tid, chunk_per_thread, iterations, pool_bytes]() { + const size_t start = tid * chunk_per_thread; + const size_t end_pos = (tid == (int)(pool_bytes / chunk_per_thread) - 1) ? pool_bytes : (start + chunk_per_thread); + volatile uint64_t local_sink = 0; + double local_bytes = 0.0; + for (int iter = 0; iter < iterations; ++iter) { + const size_t limit = end_pos - sizeof(uint64_t); + for (size_t offset = start; offset + 64 <= limit; offset += 64) { + local_sink += *(const uint64_t *)(pool.data() + offset); + local_bytes += 64.0; + } + } + thread_bytes[tid] = local_bytes; + (void)local_sink; + }); + } + for (auto & w : workers) w.join(); + + double elapsed = t.stop(); + double total_bytes = 0.0; + for (int i = 0; i < threads; i++) total_bytes += thread_bytes[i]; + return total_bytes / elapsed / 1e9; +} + +struct pcie_stress_ctx { + std::atomic active{false}; + std::atomic stop{false}; + + ggml_backend_t gpu_backend = nullptr; + ggml_backend_buffer_t host_buf = nullptr; + ggml_backend_buffer_t dev_buf = nullptr; + ggml_tensor * h_tensor = nullptr; + ggml_tensor * d_tensor = nullptr; + ggml_context * ctx = nullptr; + size_t transfer_size = 256 * 1024 * 1024; + double calibrated_bw_gb_s = 0.0; +}; + +static void pcie_stress_loop(pcie_stress_ctx * pcie) { + pcie->active.store(true, std::memory_order_release); + while (!pcie->stop.load(std::memory_order_acquire)) { + ggml_backend_tensor_set_async(pcie->gpu_backend, pcie->d_tensor, + pcie->h_tensor->data, 0, pcie->transfer_size); + ggml_backend_synchronize(pcie->gpu_backend); + ggml_backend_tensor_get_async(pcie->gpu_backend, pcie->d_tensor, + pcie->h_tensor->data, 0, pcie->transfer_size); + ggml_backend_synchronize(pcie->gpu_backend); + } + pcie->active.store(false, std::memory_order_release); +} + +static void calibrate_pcie(pcie_stress_ctx * pcie) { + printf("Calibrating standalone PCIe bandwidth...\n"); + bench_timer t; + t.start(); + const int cal_iterations = 20; + for (int i = 0; i < cal_iterations; ++i) { + ggml_backend_tensor_set_async(pcie->gpu_backend, pcie->d_tensor, + pcie->h_tensor->data, 0, pcie->transfer_size); + ggml_backend_synchronize(pcie->gpu_backend); + ggml_backend_tensor_get_async(pcie->gpu_backend, pcie->d_tensor, + pcie->h_tensor->data, 0, pcie->transfer_size); + ggml_backend_synchronize(pcie->gpu_backend); + } + double elapsed = t.stop(); + double bytes_moved = (double)cal_iterations * pcie->transfer_size * 2.0; + pcie->calibrated_bw_gb_s = bytes_moved / elapsed / 1e9; + printf(" Standalone PCIe BW: %.1f GB/s\n\n", pcie->calibrated_bw_gb_s); +} + +struct bench_result_cpu : bench_result { + int threads = 0; + float standalone_gflops = 0.0f; + float concurrent_gflops = 0.0f; + float concurrent_efficiency_pct = 0.0f; + float pcie_standalone_bw_gb_s = 0.0f; + + void print(double pcie_bw_ref = 0.0) const { + printf("%-20s quant=%-6s threads=%d AI=%.3f FLOP/byte BW=%.2f GB/s Perf=%.2f GFLOP/s", + op_name.c_str(), quant_type.c_str(), threads, + arithmetic_intensity, effective_bw_gb_s, effective_gflops); + if (concurrent_gflops > 0) { + printf(" | Concur=%.2f (%.1f%%)", concurrent_gflops, concurrent_efficiency_pct); + if (pcie_bw_ref > 0) { + double est_pcie_bw = pcie_bw_ref * (concurrent_efficiency_pct / 100.0) * 0.9; + printf(" PCIe~%.1f GB/s (%.1f%%)", est_pcie_bw, 100.0 * est_pcie_bw / pcie_bw_ref); + } + } + print_dims(); + printf("\n"); + } +}; + +static double benchmark_mul_mat_raw( + ggml_backend_t be, int N, int K, int batch_size, + ggml_type quant, int threads, + double * out_time_s, double * out_ops, double * out_bytes) { + + ggml_init_params params = { 4096ULL * 1024 * 1024, NULL, true }; + ggml_context * ctx = ggml_init(params); + ggml_backend_cpu_set_n_threads(be, threads); + + ggml_tensor * A = ggml_new_tensor_2d(ctx, quant, K, N); + ggml_tensor * B_tensor = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, K, batch_size); + + ggml_cgraph * gf = ggml_new_graph(ctx); + ggml_tensor * C = ggml_mul_mat(ctx, A, B_tensor); + ggml_build_forward_expand(gf, C); + + ggml_backend_buffer_t buffer = ggml_backend_alloc_ctx_tensors(ctx, be); + if (!buffer) { + printf("SKIPPED: MUL_MAT N=%d K=%d B=%d %s (alloc failed)\n", N, K, batch_size, ggml_type_name(quant)); + ggml_free(ctx); return 0.0; + } + + std::vector A_data = create_quantized_data(quant, (int64_t)K * N); + std::vector B_data(K * batch_size, 1.0f); + ggml_backend_tensor_set(A, A_data.data(), 0, ggml_nbytes(A)); + ggml_backend_tensor_set(B_tensor, B_data.data(), 0, ggml_nbytes(B_tensor)); + + for (int i = 0; i < CPU_WARMUP_ITERS; ++i) { flush_caches(); ggml_backend_graph_compute_async(be, gf); } + + double total_time = 0.0; + bench_timer t; + for (int i = 0; i < CPU_TIMED_ITERS; ++i) { + flush_caches(); + t.start(); + ggml_backend_graph_compute_async(be, gf); + total_time += t.stop(); + } + + double time_per_iter = total_time / CPU_TIMED_ITERS; + double ops_total = 2.0 * N * K * batch_size; + double bytes_total = (double)(ggml_nbytes(A) + ggml_nbytes(B_tensor) + ggml_nbytes(C)); + + if (out_time_s) *out_time_s = time_per_iter; + if (out_ops) *out_ops = ops_total; + if (out_bytes) *out_bytes = bytes_total; + + ggml_backend_buffer_free(buffer); + ggml_free(ctx); + return ops_total / time_per_iter / 1e9; +} + +static double benchmark_mul_mat_id_raw( + ggml_backend_t be, int N, int K, int n_experts, int n_experts_used, + int batch_size, ggml_type quant, int threads, + double * out_time_s, double * out_ops, double * out_bytes) { + + ggml_init_params params = { 8192ULL * 1024 * 1024, NULL, true }; + ggml_context * ctx = ggml_init(params); + ggml_backend_cpu_set_n_threads(be, threads); + + ggml_tensor * A = ggml_new_tensor_3d(ctx, quant, K, N, n_experts); + ggml_tensor * B = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, K, 1, batch_size); + ggml_tensor * ids = ggml_new_tensor_2d(ctx, GGML_TYPE_I32, n_experts_used, batch_size); + + ggml_cgraph * gf = ggml_new_graph(ctx); + ggml_build_forward_expand(gf, ggml_mul_mat_id(ctx, A, B, ids)); + + ggml_backend_buffer_t buffer = ggml_backend_alloc_ctx_tensors(ctx, be); + if (!buffer) { + printf("SKIPPED: MUL_MAT_ID N=%d K=%d B=%d (alloc failed)\n", N, K, batch_size); + ggml_free(ctx); return 0.0; + } + + std::vector A_data = create_quantized_data(quant, (int64_t)K * N * n_experts); + ggml_backend_tensor_set(A, A_data.data(), 0, ggml_nbytes(A)); + std::vector B_data(K * batch_size, 1.0f); + ggml_backend_tensor_set(B, B_data.data(), 0, ggml_nbytes(B)); + std::vector ids_data(n_experts_used * batch_size); + for (int i = 0; i < n_experts_used * batch_size; i++) ids_data[i] = i % n_experts; + ggml_backend_tensor_set(ids, ids_data.data(), 0, ggml_nbytes(ids)); + + for (int i = 0; i < CPU_WARMUP_ITERS; ++i) { flush_caches(); ggml_backend_graph_compute_async(be, gf); } + + double total_time = 0.0; + bench_timer t; + for (int i = 0; i < CPU_TIMED_ITERS; ++i) { + flush_caches(); + t.start(); + ggml_backend_graph_compute_async(be, gf); + total_time += t.stop(); + } + + double time_per_iter = total_time / CPU_TIMED_ITERS; + double ops_total = 2.0 * N * K * batch_size * n_experts_used; + double bytes_total = (double)(ggml_nbytes(A) * n_experts_used / n_experts) + ggml_nbytes(B) + (double)(N * batch_size * n_experts_used * 4); + + if (out_time_s) *out_time_s = time_per_iter; + if (out_ops) *out_ops = ops_total; + if (out_bytes) *out_bytes = bytes_total; + + ggml_backend_buffer_free(buffer); + ggml_free(ctx); + return ops_total / time_per_iter / 1e9; +} + +static double benchmark_flash_attn_raw( + ggml_backend_t be, int n_tokens, int ctx_len, + int n_q_heads, int n_kv_heads, int head_dim, + ggml_type kv_quant, int threads, + double * out_time_s, double * out_ops, double * out_bytes) { + + ggml_init_params params = { 8192ULL * 1024 * 1024, NULL, true }; + ggml_context * ctx = ggml_init(params); + ggml_backend_cpu_set_n_threads(be, threads); + + ggml_tensor * Q = ggml_new_tensor_4d(ctx, GGML_TYPE_F32, head_dim, n_tokens, n_q_heads, 1); + ggml_tensor * K = ggml_new_tensor_4d(ctx, kv_quant, head_dim, ctx_len, n_kv_heads, 1); + ggml_tensor * V = ggml_new_tensor_4d(ctx, kv_quant, head_dim, ctx_len, n_kv_heads, 1); + + ggml_cgraph * gf = ggml_new_graph(ctx); + ggml_tensor * out = ggml_flash_attn_ext(ctx, Q, K, V, nullptr, 1.0f / sqrtf((float)head_dim), 0.0f, 0.0f); + ggml_build_forward_expand(gf, out); + + ggml_backend_buffer_t buffer = ggml_backend_alloc_ctx_tensors(ctx, be); + if (!buffer) { + printf("SKIPPED: FLASH_ATTN (alloc failed)\n"); + ggml_free(ctx); return 0.0; + } + + std::vector Q_data(ggml_nelements(Q), 1.0f); + ggml_backend_tensor_set(Q, Q_data.data(), 0, ggml_nbytes(Q)); + std::vector KV_data = create_quantized_data(kv_quant, ggml_nelements(K)); + ggml_backend_tensor_set(K, KV_data.data(), 0, ggml_nbytes(K)); + ggml_backend_tensor_set(V, KV_data.data(), 0, ggml_nbytes(V)); + + for (int i = 0; i < CPU_WARMUP_ITERS; ++i) { flush_caches(); ggml_backend_graph_compute_async(be, gf); } + + double total_time = 0.0; + bench_timer t; + for (int i = 0; i < CPU_TIMED_ITERS; ++i) { + flush_caches(); + t.start(); + ggml_backend_graph_compute_async(be, gf); + total_time += t.stop(); + } + + double time_per_iter = total_time / CPU_TIMED_ITERS; + double ops_total = 2.0 * n_tokens * head_dim * ctx_len * n_q_heads * 2; + double bytes_total = (double)(ggml_nbytes(Q) + ggml_nbytes(K) + ggml_nbytes(V) + ggml_nbytes(out)); + + if (out_time_s) *out_time_s = time_per_iter; + if (out_ops) *out_ops = ops_total; + if (out_bytes) *out_bytes = bytes_total; + + ggml_backend_buffer_free(buffer); + ggml_free(ctx); + return ops_total / time_per_iter / 1e9; +} + +static bench_result_cpu run_concurrent( + std::function bench_fn, + const std::string & op_name, const char * quant_name, int threads, + pcie_stress_ctx * pcie) { + + bench_result_cpu result; + result.op_name = op_name; + result.quant_type = quant_name; + result.threads = threads; + result.pcie_standalone_bw_gb_s = pcie ? (float)pcie->calibrated_bw_gb_s : 0.0f; + + double standalone = bench_fn(&result.time_s, &result.ops, &result.bytes); + result.calculate_derived(); + result.standalone_gflops = result.effective_gflops; + + if (pcie && pcie->gpu_backend) { + pcie->stop.store(false, std::memory_order_release); + std::thread pcie_thread(pcie_stress_loop, pcie); + while (!pcie->active.load(std::memory_order_acquire)) std::this_thread::yield(); + + result.concurrent_gflops = (float)bench_fn(nullptr, nullptr, nullptr); + + pcie->stop.store(true, std::memory_order_release); + pcie_thread.join(); + } else { + result.concurrent_gflops = result.standalone_gflops; + } + + result.concurrent_efficiency_pct = (standalone > 0.0) + ? (float)(std::min)(100.0, 100.0 * result.concurrent_gflops / result.standalone_gflops) + : 100.0f; + + return result; +} + +static void run_matmul_benchmarks( + ggml_backend_t be, int threads, const std::vector & batch_sizes, + bool fast, pcie_stress_ctx * pcie, + std::vector & results) { + + auto sizes = get_matmul_sizes(fast); + auto quants = get_matmul_quants(fast); + + printf("=== MUL_MAT Operations ===\n\n"); + for (ggml_type qt : quants) { + printf("--- Quantization: %s ---\n", ggml_type_name(qt)); + for (int32_t bs : batch_sizes) { + printf(" [Batch=%d]\n", bs); + for (const auto & sz : sizes) { + if (qt == GGML_TYPE_Q2_K && (sz.K % 256 != 0)) continue; + + auto res = run_concurrent( + [&](double * t, double * o, double * b) { + return benchmark_mul_mat_raw(be, sz.N, sz.K, bs, qt, threads, t, o, b); + }, "MUL_MAT", ggml_type_name(qt), threads, pcie); + res.N = sz.N; res.K = sz.K; res.B = bs; + res.print(pcie ? pcie->calibrated_bw_gb_s : 0.0); + results.push_back(res); + } + } + printf("\n"); + } +} + +static void run_moe_benchmarks( + ggml_backend_t be, int threads, const std::vector & batch_sizes, + bool fast, pcie_stress_ctx * pcie, + std::vector & results) { + + auto configs = get_moe_configs(fast); + auto quants = get_matmul_quants(fast); + + printf("=== MUL_MAT_ID Operations (MoE) ===\n\n"); + for (ggml_type qt : quants) { + printf("--- MoE Quantization: %s ---\n", ggml_type_name(qt)); + for (int32_t bs : batch_sizes) { + printf(" [Batch=%d]\n", bs); + for (const auto & cfg : configs) { + if (qt == GGML_TYPE_Q2_K && (cfg.K % 256 != 0)) continue; + + auto res = run_concurrent( + [&](double * t, double * o, double * b) { + return benchmark_mul_mat_id_raw(be, cfg.N, cfg.K, cfg.n_experts, cfg.n_experts_used, bs, qt, threads, t, o, b); + }, "MUL_MAT_ID", ggml_type_name(qt), threads, pcie); + res.N = cfg.N; res.K = cfg.K; res.B = bs; + res.n_tokens = cfg.n_experts_used; res.ctx_len = cfg.n_experts; + res.print(pcie ? pcie->calibrated_bw_gb_s : 0.0); + results.push_back(res); + } + } + printf("\n"); + } +} + +static void run_attention_benchmarks( + ggml_backend_t be, int threads, const std::vector & batch_sizes, + bool fast, pcie_stress_ctx * pcie, + std::vector & results) { + + auto configs = get_attn_configs(fast); + auto ctx_lens = get_attn_ctx_lens(fast); + + printf("=== FLASH_ATTN Operations ===\n\n"); + for (const auto & cfg : configs) { + printf("--- %s (n_q=%d, n_kv=%d, head_dim=%d) ---\n", cfg.name, cfg.n_q_heads, cfg.n_kv_heads, cfg.head_dim); + for (int32_t n_tok : batch_sizes) { + printf(" [n_tokens=%d]\n", n_tok); + for (int32_t cl : ctx_lens) { + auto res = run_concurrent( + [&](double * t, double * o, double * b) { + return benchmark_flash_attn_raw(be, n_tok, cl, cfg.n_q_heads, cfg.n_kv_heads, cfg.head_dim, GGML_TYPE_F16, threads, t, o, b); + }, std::string("FLASH_ATTN_") + cfg.name, ggml_type_name(GGML_TYPE_F16), threads, pcie); + res.n_tokens = n_tok; res.ctx_len = cl; res.n_heads = cfg.n_kv_heads; res.head_dim = cfg.head_dim; + res.print(pcie ? pcie->calibrated_bw_gb_s : 0.0); + results.push_back(res); + } + } + printf("\n"); + } +} + +static void save_results_cpu( + const char * path, + const std::vector & results, + const std::vector & batch_sizes, + int threads, double dram_bw, double pcie_standalone_bw, double pcie_concurrent_bw, double cpu_eff, + bool has_gpu) { + + FILE * f = fopen(path, "w"); + if (!f) { fprintf(stderr, "Failed to open %s for writing\n", path); return; } + + fprintf(f, "# Concurrent Profiling (threads=%d, batch_sizes=[", threads); + for (size_t i = 0; i < batch_sizes.size(); i++) + fprintf(f, "%d%s", batch_sizes[i], i + 1 < batch_sizes.size() ? "," : ""); + fprintf(f, "])\n"); + + fprintf(f, "# Measured Bandwidths Per Thread Count:\n"); + if (has_gpu) { + fprintf(f, "# Threads=%d: DRAM_BW=%.1f GB/s, PCIe_Standalone=%.1f GB/s, PCIe_Concurrent=%.1f GB/s (CPU_Eff=%.1f%%)\n", + threads, dram_bw, pcie_standalone_bw, pcie_concurrent_bw, cpu_eff); + } else { + fprintf(f, "# Threads=%d: DRAM_BW=%.1f GB/s\n", threads, dram_bw); + } + + fprintf(f, "# op_name quant threads AI(FLOP/byte) BW(GB/s) GFLOP/s Ridge(FLOP/byte) Concurrent_GFLOP/s PCIe_Concurrent_BW N K B n_tokens ctx_len n_heads head_dim n_elements\n"); + + auto ridges = compute_ridge_points(results, dram_bw); + std::map ridge_map; + for (const auto & rr : ridges) ridge_map[rr.key] = rr.ridge; + + for (const auto & r : results) { + std::string key = r.op_name + "_" + r.quant_type; + double ridge = ridge_map.count(key) ? ridge_map[key] : 0.0; + double est_pcie = pcie_standalone_bw * (r.standalone_gflops > 0 ? r.concurrent_gflops / r.standalone_gflops : 1.0) * 0.9; + + fprintf(f, "%s %s %d %.4f %.2f %.2f %.4f %.2f %.2f %d %d %d %d %d %d %d %lld\n", + r.op_name.c_str(), r.quant_type.c_str(), r.threads, + r.arithmetic_intensity, r.effective_bw_gb_s, r.effective_gflops, ridge, + r.concurrent_gflops, est_pcie, + r.N, r.K, r.B, r.n_tokens, r.ctx_len, r.n_heads, r.head_dim, (long long)r.n_elements); + } + + fclose(f); + printf("Results saved to %s (%zu benchmarks)\n", path, results.size()); +} + +int main(int argc, char ** argv) { + int32_t fixed_threads = -1; + bool fast_mode = true; + const char * output_path = "cpu_profile.txt"; + + for (int i = 1; i < argc; ++i) { + if (!strcmp(argv[i], "--threads") && i + 1 < argc) { + if (!parse_int_arg(argv[++i], fixed_threads) || fixed_threads <= 0) { + fprintf(stderr, "Invalid --threads value: %s\n", argv[i]); + return 1; + } + } else if (!strcmp(argv[i], "--fast")) { + fast_mode = true; + } else if (!strcmp(argv[i], "--full")) { + fast_mode = false; + } else if (!strcmp(argv[i], "-h") || !strcmp(argv[i], "--help")) { + printf("usage: %s [options]\n", argv[0]); + printf("\n"); + printf("options:\n"); + printf(" -h, --help\n"); + printf(" --fast fast mode with fewer configs (default)\n"); + printf(" --full full mode with all configs\n"); + printf(" --threads number of CPU threads (default: auto)\n"); + printf(" --output output file (default: cpu_profile.txt)\n"); + return 0; + } else if (!strcmp(argv[i], "--output") && i + 1 < argc) { + output_path = argv[++i]; + } + } + + int32_t default_threads = cpu_get_num_math(); + int threads = (fixed_threads > 0) ? fixed_threads : default_threads; + std::vector batch_sizes = { 1, 64, 512 }; + + printf("=== CPU Profiler (cold-cache) ===\n"); + printf("Threads: %d%s\n", threads, fixed_threads > 0 ? " (user)" : " (auto)"); + printf("Mode: %s\n\n", fast_mode ? "FAST" : "FULL"); + + init_flush_buffer(); + + ggml_backend_t cpu_be = ggml_backend_init_by_type(GGML_BACKEND_DEVICE_TYPE_CPU, nullptr); + if (!cpu_be) { fprintf(stderr, "Failed to initialize CPU backend\n"); return 1; } + + ggml_quantize_init(GGML_TYPE_Q2_K); + ggml_quantize_init(GGML_TYPE_Q4_0); + ggml_quantize_init(GGML_TYPE_Q4_1); + ggml_quantize_init(GGML_TYPE_Q5_0); + ggml_quantize_init(GGML_TYPE_Q8_0); + ggml_quantize_init(GGML_TYPE_MXFP4); + + pcie_stress_ctx pcie; + pcie.gpu_backend = ggml_backend_init_by_type(GGML_BACKEND_DEVICE_TYPE_GPU, nullptr); + bool has_gpu = (pcie.gpu_backend != nullptr); + + if (has_gpu) { + ggml_backend_buffer_type_t host_buft = ggml_backend_dev_host_buffer_type( + ggml_backend_get_device(pcie.gpu_backend)); + if (host_buft) { + pcie.host_buf = ggml_backend_buft_alloc_buffer(host_buft, pcie.transfer_size); + pcie.dev_buf = ggml_backend_alloc_buffer(pcie.gpu_backend, pcie.transfer_size); + ggml_init_params p = { pcie.transfer_size + 8 * 1024 * 1024, NULL, true }; + pcie.ctx = ggml_init(p); + pcie.h_tensor = ggml_new_tensor_1d(pcie.ctx, GGML_TYPE_F32, pcie.transfer_size / 4); + pcie.d_tensor = ggml_new_tensor_1d(pcie.ctx, GGML_TYPE_F32, pcie.transfer_size / 4); + ggml_backend_tensor_alloc(pcie.host_buf, pcie.h_tensor, ggml_backend_buffer_get_base(pcie.host_buf)); + ggml_backend_tensor_alloc(pcie.dev_buf, pcie.d_tensor, ggml_backend_buffer_get_base(pcie.dev_buf)); + std::vector init_data(pcie.transfer_size / 4, 1.0f); + ggml_backend_tensor_set(pcie.h_tensor, init_data.data(), 0, pcie.transfer_size); + printf("GPU: %s\n", ggml_backend_name(pcie.gpu_backend)); + } else { + has_gpu = false; + } + } + if (!has_gpu) printf("No GPU — standalone mode\n\n"); + + printf("Measuring DRAM bandwidth...\n"); + double dram_bw = benchmark_cpu_dram_bandwidth(threads); + printf(" DRAM BW: %.1f GB/s\n", dram_bw); + if (has_gpu) calibrate_pcie(&pcie); + + std::vector all_results; + bench_timer overall; + overall.start(); + + run_matmul_benchmarks(cpu_be, threads, batch_sizes, fast_mode, has_gpu ? &pcie : nullptr, all_results); + run_moe_benchmarks(cpu_be, threads, batch_sizes, fast_mode, has_gpu ? &pcie : nullptr, all_results); + run_attention_benchmarks(cpu_be, threads, batch_sizes, fast_mode, has_gpu ? &pcie : nullptr, all_results); + + double pcie_concurrent_bw = 0.0, cpu_eff = 100.0; + if (has_gpu && !all_results.empty()) { + double sum_s = 0.0, sum_c = 0.0; + for (const auto & r : all_results) { sum_s += r.standalone_gflops; sum_c += r.concurrent_gflops; } + cpu_eff = (sum_s > 0) ? 100.0 * sum_c / sum_s : 100.0; + pcie_concurrent_bw = pcie.calibrated_bw_gb_s * (cpu_eff / 100.0) * 0.9; + } + + printf("\nTotal time: %.1f s, %zu benchmarks\n", overall.stop(), all_results.size()); + + save_results_cpu(output_path, all_results, batch_sizes, threads, dram_bw, + pcie.calibrated_bw_gb_s, pcie_concurrent_bw, cpu_eff, has_gpu); + + if (has_gpu) { + if (pcie.ctx) ggml_free(pcie.ctx); + if (pcie.host_buf) ggml_backend_buffer_free(pcie.host_buf); + if (pcie.dev_buf) ggml_backend_buffer_free(pcie.dev_buf); + ggml_backend_free(pcie.gpu_backend); + } + ggml_backend_free(cpu_be); + ggml_quantize_free(); + return 0; +} diff --git a/examples/llama-profiler/profiler-gpu.cpp b/examples/llama-profiler/profiler-gpu.cpp new file mode 100644 index 000000000000..127f4700d728 --- /dev/null +++ b/examples/llama-profiler/profiler-gpu.cpp @@ -0,0 +1,424 @@ +#define GPU_WARMUP_ITERS 3 +#define GPU_TIMED_ITERS 5 + +#include "profiler-common.h" + +#ifdef GGML_USE_CUDA +#include "ggml-cuda.h" +#endif + +#include +#include +#include + +static double benchmark_mul_mat_raw( + ggml_backend_t be, int N, int K, int batch_size, ggml_type quant, + double * out_time_s, double * out_ops, double * out_bytes) { + + ggml_init_params params = { 4096ULL * 1024 * 1024, NULL, true }; + ggml_context * ctx = ggml_init(params); + + ggml_tensor * A = ggml_new_tensor_2d(ctx, quant, K, N); + ggml_tensor * B_tensor = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, K, batch_size); + + ggml_cgraph * gf = ggml_new_graph(ctx); + ggml_tensor * C = ggml_mul_mat(ctx, A, B_tensor); + ggml_build_forward_expand(gf, C); + + if ((int64_t)N * batch_size > INT_MAX) { + printf("SKIPPED: MUL_MAT N=%d K=%d B=%d (output exceeds INT_MAX)\n", N, K, batch_size); + ggml_free(ctx); return 0.0; + } + + size_t gpu_free = 0, gpu_total = 0; + ggml_backend_dev_memory(ggml_backend_get_device(be), &gpu_free, &gpu_total); + size_t need = ggml_nbytes(A) + ggml_nbytes(B_tensor) + ggml_nbytes(C); + if (need > gpu_free * 0.9) { + printf("SKIPPED: MUL_MAT N=%d K=%d B=%d (need %.2f GB, free %.2f GB)\n", + N, K, batch_size, need / 1e9, gpu_free / 1e9); + ggml_free(ctx); return 0.0; + } + + ggml_backend_buffer_t buffer = ggml_backend_alloc_ctx_tensors(ctx, be); + if (!buffer) { fprintf(stderr, "Failed to allocate GPU buffer\n"); ggml_free(ctx); return 0.0; } + + std::vector A_float((size_t)K * N, 0.5f); + if (quant == GGML_TYPE_F32) { + ggml_backend_tensor_set(A, A_float.data(), 0, ggml_nbytes(A)); + } else { + std::vector A_q(ggml_nbytes(A)); + ggml_quantize_chunk(quant, A_float.data(), A_q.data(), 0, + (int64_t)K * N / ggml_blck_size(quant), 1, nullptr); + ggml_backend_tensor_set(A, A_q.data(), 0, ggml_nbytes(A)); + } + std::vector B_data((size_t)K * batch_size, 1.0f); + ggml_backend_tensor_set(B_tensor, B_data.data(), 0, ggml_nbytes(B_tensor)); + + for (int i = 0; i < GPU_WARMUP_ITERS; ++i) ggml_backend_graph_compute(be, gf); + ggml_backend_synchronize(be); + + bench_timer t; t.start(); + for (int i = 0; i < GPU_TIMED_ITERS; ++i) ggml_backend_graph_compute_async(be, gf); + ggml_backend_synchronize(be); + double total_time = t.stop(); + + double time_per_iter = total_time / GPU_TIMED_ITERS; + double ops_total = 2.0 * N * K * batch_size; + double bytes_total = (double)(ggml_nbytes(A) + ggml_nbytes(B_tensor) + ggml_nbytes(C)); + + if (out_time_s) *out_time_s = time_per_iter; + if (out_ops) *out_ops = ops_total; + if (out_bytes) *out_bytes = bytes_total; + + ggml_backend_buffer_free(buffer); + ggml_free(ctx); + return ops_total / time_per_iter / 1e9; +} + +static double benchmark_mul_mat_id_raw( + ggml_backend_t be, int N, int K, int n_experts, int n_experts_used, + int batch_size, ggml_type quant, + double * out_time_s, double * out_ops, double * out_bytes) { + + ggml_init_params params = { 8192ULL * 1024 * 1024, NULL, true }; + ggml_context * ctx = ggml_init(params); + if (!ctx) { fprintf(stderr, "Failed to init context\n"); return 0.0; } + + ggml_tensor * A = ggml_new_tensor_3d(ctx, quant, K, N, n_experts); + ggml_tensor * B = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, K, 1, batch_size); + ggml_tensor * ids = ggml_new_tensor_2d(ctx, GGML_TYPE_I32, n_experts_used, batch_size); + + if ((int64_t)N * batch_size * n_experts_used > INT_MAX) { + printf("SKIPPED: MUL_MAT_ID N=%d K=%d B=%d (output exceeds INT_MAX)\n", N, K, batch_size); + ggml_free(ctx); return 0.0; + } + + size_t gpu_free = 0, gpu_total = 0; + ggml_backend_dev_memory(ggml_backend_get_device(be), &gpu_free, &gpu_total); + size_t need = ggml_nbytes(A) + ggml_nbytes(B) + ggml_nbytes(ids); + if (need > gpu_free * 0.9) { + printf("SKIPPED: MUL_MAT_ID N=%d K=%d B=%d (need %.2f GB, free %.2f GB)\n", + N, K, batch_size, need / 1e9, gpu_free / 1e9); + ggml_free(ctx); return 0.0; + } + + ggml_cgraph * gf = ggml_new_graph(ctx); + ggml_tensor * C = ggml_mul_mat_id(ctx, A, B, ids); + ggml_build_forward_expand(gf, C); + + ggml_backend_buffer_t buffer = ggml_backend_alloc_ctx_tensors(ctx, be); + if (!buffer) { fprintf(stderr, "Failed to allocate GPU buffer\n"); ggml_free(ctx); return 0.0; } + + std::vector A_data(ggml_nbytes(A), 0); + ggml_backend_tensor_set(A, A_data.data(), 0, ggml_nbytes(A)); + std::vector B_data((size_t)K * batch_size, 1.0f); + ggml_backend_tensor_set(B, B_data.data(), 0, ggml_nbytes(B)); + std::vector ids_data(n_experts_used * batch_size); + for (int i = 0; i < n_experts_used * batch_size; i++) ids_data[i] = i % n_experts; + ggml_backend_tensor_set(ids, ids_data.data(), 0, ggml_nbytes(ids)); + + for (int i = 0; i < GPU_WARMUP_ITERS; ++i) ggml_backend_graph_compute(be, gf); + ggml_backend_synchronize(be); + + bench_timer t; t.start(); + for (int i = 0; i < GPU_TIMED_ITERS; ++i) ggml_backend_graph_compute_async(be, gf); + ggml_backend_synchronize(be); + double total_time = t.stop(); + + double time_per_iter = total_time / GPU_TIMED_ITERS; + double ops_total = 2.0 * N * K * batch_size * n_experts_used; + double bytes_total = (double)(ggml_nbytes(A) * n_experts_used / n_experts) + ggml_nbytes(B) + ggml_nbytes(C); + + if (out_time_s) *out_time_s = time_per_iter; + if (out_ops) *out_ops = ops_total; + if (out_bytes) *out_bytes = bytes_total; + + ggml_backend_buffer_free(buffer); + ggml_free(ctx); + return ops_total / time_per_iter / 1e9; +} + +static double benchmark_flash_attn_raw( + ggml_backend_t be, int n_tokens, int ctx_len, + int n_q_heads, int n_kv_heads, int head_dim, ggml_type kv_quant, + double * out_time_s, double * out_ops, double * out_bytes) { + + ggml_init_params params = { 8192ULL * 1024 * 1024, NULL, true }; + ggml_context * ctx = ggml_init(params); + + ggml_tensor * Q = ggml_new_tensor_4d(ctx, GGML_TYPE_F32, head_dim, n_tokens, n_q_heads, 1); + ggml_tensor * K = ggml_new_tensor_4d(ctx, kv_quant, head_dim, ctx_len, n_kv_heads, 1); + ggml_tensor * V = ggml_new_tensor_4d(ctx, kv_quant, head_dim, ctx_len, n_kv_heads, 1); + + ggml_cgraph * gf = ggml_new_graph(ctx); + ggml_tensor * out = ggml_flash_attn_ext(ctx, Q, K, V, nullptr, 1.0f / sqrtf((float)head_dim), 0.0f, 0.0f); + ggml_build_forward_expand(gf, out); + + if ((int64_t)head_dim * n_tokens * n_q_heads > INT_MAX) { + printf("SKIPPED: FLASH_ATTN (output exceeds INT_MAX)\n"); + ggml_free(ctx); return 0.0; + } + + size_t gpu_free = 0, gpu_total = 0; + ggml_backend_dev_memory(ggml_backend_get_device(be), &gpu_free, &gpu_total); + size_t need = ggml_nbytes(Q) + ggml_nbytes(K) + ggml_nbytes(V) + ggml_nbytes(out); + if (need > gpu_free * 0.9) { + printf("SKIPPED: FLASH_ATTN (need %.2f GB, free %.2f GB)\n", need / 1e9, gpu_free / 1e9); + ggml_free(ctx); return 0.0; + } + + ggml_backend_buffer_t buffer = ggml_backend_alloc_ctx_tensors(ctx, be); + if (!buffer) { fprintf(stderr, "Failed to allocate GPU buffer\n"); ggml_free(ctx); return 0.0; } + + std::vector Q_data(ggml_nelements(Q), 1.0f); + ggml_backend_tensor_set(Q, Q_data.data(), 0, ggml_nbytes(Q)); + int64_t kv_elems = ggml_nelements(K); + std::vector KV_float(kv_elems, 0.5f); + if (kv_quant == GGML_TYPE_F32) { + ggml_backend_tensor_set(K, KV_float.data(), 0, ggml_nbytes(K)); + ggml_backend_tensor_set(V, KV_float.data(), 0, ggml_nbytes(V)); + } else { + std::vector Kq(ggml_nbytes(K)), Vq(ggml_nbytes(V)); + ggml_quantize_chunk(kv_quant, KV_float.data(), Kq.data(), 0, kv_elems / ggml_blck_size(kv_quant), 1, nullptr); + ggml_quantize_chunk(kv_quant, KV_float.data(), Vq.data(), 0, kv_elems / ggml_blck_size(kv_quant), 1, nullptr); + ggml_backend_tensor_set(K, Kq.data(), 0, ggml_nbytes(K)); + ggml_backend_tensor_set(V, Vq.data(), 0, ggml_nbytes(V)); + } + + for (int i = 0; i < GPU_WARMUP_ITERS; ++i) ggml_backend_graph_compute(be, gf); + ggml_backend_synchronize(be); + + bench_timer t; t.start(); + for (int i = 0; i < GPU_TIMED_ITERS; ++i) ggml_backend_graph_compute_async(be, gf); + ggml_backend_synchronize(be); + double total_time = t.stop(); + + double time_per_iter = total_time / GPU_TIMED_ITERS; + double ops_total = 2.0 * n_tokens * head_dim * ctx_len * n_q_heads * 2; + double bytes_total = (double)(ggml_nbytes(Q) + ggml_nbytes(K) + ggml_nbytes(V) + ggml_nbytes(out)); + + if (out_time_s) *out_time_s = time_per_iter; + if (out_ops) *out_ops = ops_total; + if (out_bytes) *out_bytes = bytes_total; + + ggml_backend_buffer_free(buffer); + ggml_free(ctx); + return ops_total / time_per_iter / 1e9; +} + +static void run_matmul_benchmarks( + ggml_backend_t be, const std::vector & batch_sizes, + int32_t filter_batch, bool fast, + std::vector & results) { + + auto sizes = get_matmul_sizes(fast); + auto quants = get_matmul_quants(fast); + + printf("=== MUL_MAT Operations ===\n\n"); + for (ggml_type qt : quants) { + printf("--- Quantization: %s ---\n", ggml_type_name(qt)); + for (int32_t bs : batch_sizes) { + if (filter_batch >= 0 && bs != filter_batch) continue; + printf(" [Batch=%d]\n", bs); + for (const auto & sz : sizes) { + if (qt == GGML_TYPE_Q2_K && (sz.K % 256 != 0)) continue; + + bench_result res; + res.op_name = "MUL_MAT"; + res.quant_type = ggml_type_name(qt); + res.N = sz.N; res.K = sz.K; res.B = bs; + benchmark_mul_mat_raw(be, sz.N, sz.K, bs, qt, &res.time_s, &res.ops, &res.bytes); + res.calculate_derived(); + printf("%-20s quant=%-6s AI=%.3f BW=%.2f GB/s Perf=%.2f GFLOP/s", + res.op_name.c_str(), res.quant_type.c_str(), + res.arithmetic_intensity, res.effective_bw_gb_s, res.effective_gflops); + res.print_dims(); + printf("\n"); + results.push_back(res); + } + } + printf("\n"); + } +} + +static void run_moe_benchmarks( + ggml_backend_t be, const std::vector & batch_sizes, + int32_t filter_batch, bool fast, + std::vector & results) { + + auto configs = get_moe_configs(fast); + auto quants = get_matmul_quants(fast); + + printf("=== MUL_MAT_ID Operations (MoE) ===\n\n"); + for (ggml_type qt : quants) { + printf("--- MoE Quantization: %s ---\n", ggml_type_name(qt)); + for (int32_t bs : batch_sizes) { + if (filter_batch >= 0 && bs != filter_batch) continue; + printf(" [Batch=%d]\n", bs); + for (const auto & cfg : configs) { + if (qt == GGML_TYPE_Q2_K && (cfg.K % 256 != 0)) continue; + try { + bench_result res; + res.op_name = "MUL_MAT_ID"; + res.quant_type = ggml_type_name(qt); + res.N = cfg.N; res.K = cfg.K; res.B = bs; + res.n_tokens = cfg.n_experts_used; res.ctx_len = cfg.n_experts; + double gflops = benchmark_mul_mat_id_raw(be, cfg.N, cfg.K, cfg.n_experts, cfg.n_experts_used, bs, qt, + &res.time_s, &res.ops, &res.bytes); + if (gflops == 0.0) continue; + res.calculate_derived(); + printf("%-20s quant=%-6s AI=%.3f BW=%.2f GB/s Perf=%.2f GFLOP/s", + res.op_name.c_str(), res.quant_type.c_str(), + res.arithmetic_intensity, res.effective_bw_gb_s, res.effective_gflops); + res.print_dims(); + printf("\n"); + results.push_back(res); + } catch (const std::exception & e) { + printf("SKIPPED: MUL_MAT_ID N=%d K=%d %s (%s)\n", cfg.N, cfg.K, ggml_type_name(qt), e.what()); + } + } + } + printf("\n"); + } +} + +static void run_attention_benchmarks( + ggml_backend_t be, const std::vector & batch_sizes, + int32_t filter_batch, bool fast, + std::vector & results) { + + auto configs = get_attn_configs(fast); + auto ctx_lens = get_attn_ctx_lens(fast); + + printf("=== FLASH_ATTN Operations ===\n\n"); + for (const auto & cfg : configs) { + printf("--- %s (n_q=%d, n_kv=%d, head_dim=%d) ---\n", cfg.name, cfg.n_q_heads, cfg.n_kv_heads, cfg.head_dim); + for (int32_t n_tok : batch_sizes) { + if (filter_batch >= 0 && n_tok != filter_batch) continue; + printf(" [n_tokens=%d]\n", n_tok); + for (int32_t cl : ctx_lens) { + bench_result res; + res.op_name = std::string("FLASH_ATTN_") + cfg.name; + res.quant_type = ggml_type_name(GGML_TYPE_F16); + res.n_tokens = n_tok; res.ctx_len = cl; res.n_heads = cfg.n_kv_heads; res.head_dim = cfg.head_dim; + benchmark_flash_attn_raw(be, n_tok, cl, cfg.n_q_heads, cfg.n_kv_heads, cfg.head_dim, GGML_TYPE_F16, + &res.time_s, &res.ops, &res.bytes); + res.calculate_derived(); + printf("%-20s quant=%-6s AI=%.3f BW=%.2f GB/s Perf=%.2f GFLOP/s", + res.op_name.c_str(), res.quant_type.c_str(), + res.arithmetic_intensity, res.effective_bw_gb_s, res.effective_gflops); + res.print_dims(); + printf("\n"); + results.push_back(res); + } + } + printf("\n"); + } +} + +static void save_results_gpu( + const char * path, + const std::vector & results, + const char * backend_name, + const std::vector & batch_sizes, + double peak_gpu_bw, double peak_gpu_compute) { + + FILE * f = fopen(path, "w"); + if (!f) { fprintf(stderr, "Failed to open %s for writing\n", path); return; } + + fprintf(f, "# GPU Profiling (backend=%s, batch_sizes=[", backend_name); + for (size_t i = 0; i < batch_sizes.size(); i++) + fprintf(f, "%d%s", batch_sizes[i], i + 1 < batch_sizes.size() ? "," : ""); + fprintf(f, "], GPU_Memory_BW=%.1f GB/s, GPU_Peak_Compute=%.1f GFLOP/s)\n", peak_gpu_bw, peak_gpu_compute); + fprintf(f, "# op_name quant AI(FLOP/byte) BW(GB/s) GFLOP/s Ridge(FLOP/byte) N K B n_tokens ctx_len n_heads head_dim n_elements\n"); + + auto ridges = compute_ridge_points(results, peak_gpu_bw); + std::map ridge_map; + for (const auto & rr : ridges) ridge_map[rr.key] = rr.ridge; + + for (const auto & r : results) { + std::string key = r.op_name + "_" + r.quant_type; + double ridge = ridge_map.count(key) ? ridge_map[key] : 0.0; + fprintf(f, "%s %s %.4f %.2f %.2f %.4f %d %d %d %d %d %d %d %lld\n", + r.op_name.c_str(), r.quant_type.c_str(), + r.arithmetic_intensity, r.effective_bw_gb_s, r.effective_gflops, ridge, + r.N, r.K, r.B, r.n_tokens, r.ctx_len, r.n_heads, r.head_dim, (long long)r.n_elements); + } + + fclose(f); + printf("Results saved to %s (%zu benchmarks)\n", path, results.size()); +} + +int main(int argc, char ** argv) { + bool fast_mode = true; + int32_t filter_batch = -1; + const char * output_path = "gpu_profile.txt"; + + for (int i = 1; i < argc; ++i) { + if (!strcmp(argv[i], "--fast")) { + fast_mode = true; + } else if (!strcmp(argv[i], "--full")) { + fast_mode = false; + } else if (!strcmp(argv[i], "-h") || !strcmp(argv[i], "--help")) { + printf("usage: %s [options]\n", argv[0]); + printf("\n"); + printf("options:\n"); + printf(" -h, --help\n"); + printf(" --fast fast mode with fewer configs (default)\n"); + printf(" --full full mode with all configs\n"); + printf(" --batch only run batch size N\n"); + printf(" --output output file (default: gpu_profile.txt)\n"); + return 0; + } else if (!strcmp(argv[i], "--batch") && i + 1 < argc) { + if (!parse_int_arg(argv[++i], filter_batch)) { + fprintf(stderr, "Invalid --batch value: %s\n", argv[i]); + return 1; + } + } else if (!strcmp(argv[i], "--output") && i + 1 < argc) { + output_path = argv[++i]; + } + } + + std::vector batch_sizes = { 1, 64, 512, 1024, 2048, 4096, 8192, 16384 }; + + printf("=== GPU Profiler ===\n"); + printf("Mode: %s\n\n", fast_mode ? "FAST" : "FULL"); + + ggml_backend_t gpu_be = ggml_backend_init_by_type(GGML_BACKEND_DEVICE_TYPE_GPU, nullptr); + if (!gpu_be) { + fprintf(stderr, "No GPU backend available. Cannot run GPU profiler.\n"); + return 1; + } + printf("GPU: %s\n\n", ggml_backend_name(gpu_be)); + + ggml_quantize_init(GGML_TYPE_Q2_K); + ggml_quantize_init(GGML_TYPE_Q4_0); + ggml_quantize_init(GGML_TYPE_Q4_1); + ggml_quantize_init(GGML_TYPE_Q5_0); + ggml_quantize_init(GGML_TYPE_Q8_0); + ggml_quantize_init(GGML_TYPE_MXFP4); + + std::vector all_results; + bench_timer overall; overall.start(); + + run_matmul_benchmarks(gpu_be, batch_sizes, filter_batch, fast_mode, all_results); + run_moe_benchmarks(gpu_be, batch_sizes, filter_batch, fast_mode, all_results); + run_attention_benchmarks(gpu_be, batch_sizes, filter_batch, fast_mode, all_results); + + double peak_gpu_bw = 0.0, peak_gpu_compute = 0.0; + for (const auto & r : all_results) { + if (r.arithmetic_intensity < 2.0) + peak_gpu_bw = std::max(peak_gpu_bw, (double)r.effective_bw_gb_s); + if (r.arithmetic_intensity > 10.0 || (r.op_name == "MUL_MAT" && r.N >= 4096 && r.K >= 4096)) + peak_gpu_compute = std::max(peak_gpu_compute, (double)r.effective_gflops); + } + printf("\nEstimated GPU Memory BW: %.1f GB/s\n", peak_gpu_bw); + printf("Estimated GPU Compute: %.1f GFLOP/s\n", peak_gpu_compute); + printf("Total time: %.1f s, %zu benchmarks\n", overall.stop(), all_results.size()); + + save_results_gpu(output_path, all_results, ggml_backend_name(gpu_be), batch_sizes, peak_gpu_bw, peak_gpu_compute); + + ggml_backend_free(gpu_be); + ggml_quantize_free(); + return 0; +}