From 739cd0863303ffbcb26a83de4717ae628dfc5919 Mon Sep 17 00:00:00 2001 From: Matt Leon Date: Thu, 28 May 2026 21:53:22 -0400 Subject: [PATCH 1/7] Add benchmark library as a dependency Signed-off-by: Matt Leon --- bazel/dependencies.bzl | 2 ++ bazel/repositories.bzl | 8 ++++++++ 2 files changed, 10 insertions(+) diff --git a/bazel/dependencies.bzl b/bazel/dependencies.bzl index e0439bb07..9e80e355a 100644 --- a/bazel/dependencies.bzl +++ b/bazel/dependencies.bzl @@ -14,6 +14,7 @@ load("@aspect_rules_lint//format:repositories.bzl", "rules_lint_dependencies") load("@bazel_lib//lib:repositories.bzl", "bazel_lib_dependencies", "bazel_lib_register_toolchains") +load("@com_google_benchmark//:bazel/benchmark_deps.bzl", "benchmark_deps") load("@com_google_googletest//:googletest_deps.bzl", "googletest_deps") load("@com_google_protobuf//:protobuf_deps.bzl", "protobuf_deps") load("@envoy_toolshed//sysroot:sysroot.bzl", "setup_sysroots") @@ -31,6 +32,7 @@ def proxy_wasm_cpp_host_dependencies(): # Bazel extensions. googletest_deps() rules_foreign_cc_dependencies() + benchmark_deps() rules_lint_dependencies() diff --git a/bazel/repositories.bzl b/bazel/repositories.bzl index 4f0937c2a..2c0930545 100644 --- a/bazel/repositories.bzl +++ b/bazel/repositories.bzl @@ -193,6 +193,14 @@ def proxy_wasm_cpp_host_repositories(): }, ) + maybe( + http_archive, + name = "com_google_benchmark", + sha256 = "9631341c82bac4a288bef951f8b26b41f69021794184ece969f8473977eaa340", + strip_prefix = "benchmark-1.9.5", + urls = ["https://github.com/google/benchmark/archive/refs/tags/v1.9.5.tar.gz"], + ) + # NullVM dependencies. maybe( From 996e6a7b256804dc6879b2765521ad7d02ff39d1 Mon Sep 17 00:00:00 2001 From: Matt Leon Date: Thu, 28 May 2026 21:53:58 -0400 Subject: [PATCH 2/7] Add benchmark for warm VM init Signed-off-by: Matt Leon --- test/BUILD | 1 + test/wasm_vm_test.cc | 79 ++++++++++++++++++++++++++++++++++++-------- 2 files changed, 66 insertions(+), 14 deletions(-) diff --git a/test/BUILD b/test/BUILD index 1b45b1318..17cb81b6b 100644 --- a/test/BUILD +++ b/test/BUILD @@ -225,6 +225,7 @@ cc_test( "//:lib", "@com_google_googletest//:gtest", "@com_google_googletest//:gtest_main", + "@com_google_benchmark//:benchmark", ], ) diff --git a/test/wasm_vm_test.cc b/test/wasm_vm_test.cc index d792c63df..cc2daadc6 100644 --- a/test/wasm_vm_test.cc +++ b/test/wasm_vm_test.cc @@ -13,6 +13,7 @@ // limitations under the License. #include "gtest/gtest.h" +#include "benchmark/benchmark.h" #include #include @@ -31,10 +32,12 @@ INSTANTIATE_TEST_SUITE_P(WasmEngines, TestVm, testing::ValuesIn(getWasmEngines() }); TEST_P(TestVm, Init) { + std::unique_ptr vm1 = makeVm(engine_); + std::unique_ptr vm2 = makeVm(engine_); auto time1 = std::chrono::steady_clock::now(); - vm_->warm(); + vm1->warm(); auto time2 = std::chrono::steady_clock::now(); - vm_->warm(); + vm2->warm(); auto time3 = std::chrono::steady_clock::now(); auto cold = std::chrono::duration_cast(time2 - time1).count(); @@ -43,24 +46,37 @@ TEST_P(TestVm, Init) { std::cout << "[" << engine_ << "] \"cold\" engine time: " << cold << "ns" << std::endl; std::cout << "[" << engine_ << "] \"warm\" engine time: " << warm << "ns" << std::endl; - // Default warm time in nanoseconds. - int warm_time_ns_limit = 10000; - -#if defined(__linux__) && defined(__s390x__) - // Linux 390x is significantly slower, so we use a more lenient limit. - warm_time_ns_limit = 75000; -#endif - - // Verify that getting a "warm" engine takes less than 10us. - EXPECT_LE(warm, warm_time_ns_limit); - // Verify that getting a "warm" engine takes at least 50x less time than getting a "cold" one. // We skip NullVM because warm() is a noop. if (engine_ == "null") { std::cout << "Skipping warm() performance assertions for NullVM." << std::endl; return; } - EXPECT_LE(warm * 50, cold); + int expected_warm_time_ns = 0; + double expected_warm_time_speedup_factor = 0; + if (engine_ == "v8") { + expected_warm_time_ns = 20000000; + expected_warm_time_speedup_factor = 1.5; + } else if (engine_ == "wamr") { + expected_warm_time_ns = 4000; + expected_warm_time_speedup_factor = 2000; + } else if (engine_ == "wasmedge") { + expected_warm_time_ns = 2000; + expected_warm_time_speedup_factor = 1.5; + } else if (engine_ == "wasmtime") { + expected_warm_time_ns = 8000; + expected_warm_time_speedup_factor = 6; + } + // Linux 390x is significantly slower, so we use a more lenient limit. +#if defined(__linux__) && defined(__s390x__) + expected_warm_time_ns *= 10; +#endif +#if defined(THREAD_SANITIZER) || defined(MEMORY_SANITIZER) || defined(ADDRESS_SANITIZER) || \ + defined(HWADDRESS_SANITIZER) || defined(THREAD_SANITIZER) + expected_warm_time_ns *= 10; +#endif + EXPECT_LE(warm * expected_warm_time_speedup_factor, cold); + EXPECT_LT(warm, expected_warm_time_ns); } TEST_P(TestVm, Basic) { @@ -164,5 +180,40 @@ TEST_P(TestVm, DISABLED_CloneUntilOutOfMemory) { #endif +void BM_WarmVmStart(benchmark::State &state, auto makeVm) { + std::unique_ptr cold_vm = makeVm(); + cold_vm->warm(); + for (auto _ : state) { + std::unique_ptr warm_vm = makeVm(); + warm_vm->warm(); + } +} +#ifdef PROXY_WASM_HOST_ENGINE_V8 +static void BM_V8WarmVmStart(benchmark::State &state) { + BM_WarmVmStart(state, []() { return proxy_wasm::createV8Vm(); }); +} +BENCHMARK(BM_V8WarmVmStart); +#endif +#ifdef PROXY_WASM_HOST_ENGINE_WASMTIME +static void BM_WasmtimeWarmVmStart(benchmark::State &state) { + BM_WarmVmStart(state, []() { return proxy_wasm::createWasmtimeVm(); }); +} +BENCHMARK(BM_WasmtimeWarmVmStart); +#endif +#ifdef PROXY_WASM_HOST_ENGINE_WASMEDGE +static void BM_WasmEdgeWarmVmStart(benchmark::State &state) { + BM_WarmVmStart(state, []() { return proxy_wasm::createWasmEdgeVm(); }); +} +BENCHMARK(BM_WasmEdgeWarmVmStart); +#endif +#ifdef PROXY_WASM_HOST_ENGINE_WAMR +static void BM_WamrWarmVmStart(benchmark::State &state) { + BM_WarmVmStart(state, []() { return proxy_wasm::createWamrVm(); }); +} +BENCHMARK(BM_WamrWarmVmStart); +#endif + +TEST(TestVm, Benchmarks) { benchmark::RunSpecifiedBenchmarks(); } + } // namespace } // namespace proxy_wasm From 1c7a40c19a37077817d8c9d5b96c1ef1f2f56e06 Mon Sep 17 00:00:00 2001 From: Matt Leon Date: Thu, 11 Jun 2026 10:28:18 -0400 Subject: [PATCH 3/7] Simplify running benchmarks with different runtimes Signed-off-by: Matt Leon --- test/wasm_vm_test.cc | 35 ++++++++++------------------------- 1 file changed, 10 insertions(+), 25 deletions(-) diff --git a/test/wasm_vm_test.cc b/test/wasm_vm_test.cc index cc2daadc6..59dee5a78 100644 --- a/test/wasm_vm_test.cc +++ b/test/wasm_vm_test.cc @@ -188,32 +188,17 @@ void BM_WarmVmStart(benchmark::State &state, auto makeVm) { warm_vm->warm(); } } -#ifdef PROXY_WASM_HOST_ENGINE_V8 -static void BM_V8WarmVmStart(benchmark::State &state) { - BM_WarmVmStart(state, []() { return proxy_wasm::createV8Vm(); }); -} -BENCHMARK(BM_V8WarmVmStart); -#endif -#ifdef PROXY_WASM_HOST_ENGINE_WASMTIME -static void BM_WasmtimeWarmVmStart(benchmark::State &state) { - BM_WarmVmStart(state, []() { return proxy_wasm::createWasmtimeVm(); }); -} -BENCHMARK(BM_WasmtimeWarmVmStart); -#endif -#ifdef PROXY_WASM_HOST_ENGINE_WASMEDGE -static void BM_WasmEdgeWarmVmStart(benchmark::State &state) { - BM_WarmVmStart(state, []() { return proxy_wasm::createWasmEdgeVm(); }); -} -BENCHMARK(BM_WasmEdgeWarmVmStart); -#endif -#ifdef PROXY_WASM_HOST_ENGINE_WAMR -static void BM_WamrWarmVmStart(benchmark::State &state) { - BM_WarmVmStart(state, []() { return proxy_wasm::createWamrVm(); }); -} -BENCHMARK(BM_WamrWarmVmStart); -#endif -TEST(TestVm, Benchmarks) { benchmark::RunSpecifiedBenchmarks(); } +TEST(TestVm, Benchmarks) { + for (std::string engine : getWasmEngines()) { + benchmark::RegisterBenchmark( + "BM_WarmVmStart/" + engine, + [](benchmark::State &state, auto makeVm) { BM_WarmVmStart(state, makeVm); }, + [engine]() { return TestVm::makeVm(engine); }); + } + benchmark::RunSpecifiedBenchmarks(); + benchmark::Shutdown(); +} } // namespace } // namespace proxy_wasm From 6e6b26a21eb909a66a0fc2b643598ec7323c474b Mon Sep 17 00:00:00 2001 From: Matt Leon Date: Thu, 11 Jun 2026 10:31:28 -0400 Subject: [PATCH 4/7] Format test/BUILD Signed-off-by: Matt Leon --- test/BUILD | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/BUILD b/test/BUILD index 17cb81b6b..d653f579b 100644 --- a/test/BUILD +++ b/test/BUILD @@ -223,9 +223,9 @@ cc_test( deps = [ ":utility_lib", "//:lib", + "@com_google_benchmark//:benchmark", "@com_google_googletest//:gtest", "@com_google_googletest//:gtest_main", - "@com_google_benchmark//:benchmark", ], ) From 967e55b9ef155a0088e32ed5b1348fe14ba3df4e Mon Sep 17 00:00:00 2001 From: Matt Leon Date: Thu, 11 Jun 2026 10:38:02 -0400 Subject: [PATCH 5/7] Disable speedup factor test when sanitizer builds are enabled Signed-off-by: Matt Leon --- test/wasm_vm_test.cc | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/test/wasm_vm_test.cc b/test/wasm_vm_test.cc index 59dee5a78..a549cf919 100644 --- a/test/wasm_vm_test.cc +++ b/test/wasm_vm_test.cc @@ -74,8 +74,9 @@ TEST_P(TestVm, Init) { #if defined(THREAD_SANITIZER) || defined(MEMORY_SANITIZER) || defined(ADDRESS_SANITIZER) || \ defined(HWADDRESS_SANITIZER) || defined(THREAD_SANITIZER) expected_warm_time_ns *= 10; -#endif +#else EXPECT_LE(warm * expected_warm_time_speedup_factor, cold); +#endif EXPECT_LT(warm, expected_warm_time_ns); } @@ -190,6 +191,10 @@ void BM_WarmVmStart(benchmark::State &state, auto makeVm) { } TEST(TestVm, Benchmarks) { +#if defined(THREAD_SANITIZER) || defined(MEMORY_SANITIZER) || defined(ADDRESS_SANITIZER) || \ + defined(HWADDRESS_SANITIZER) || defined(THREAD_SANITIZER) + GTEST_SKIP() << "Disabled due to sanitizer."; +#endif for (std::string engine : getWasmEngines()) { benchmark::RegisterBenchmark( "BM_WarmVmStart/" + engine, From 6e4670e91681c590cdf2bca192ba96e18dbda1ce Mon Sep 17 00:00:00 2001 From: Matt Leon Date: Thu, 11 Jun 2026 10:45:51 -0400 Subject: [PATCH 6/7] Drop comparison to cold start in Init test Signed-off-by: Matt Leon --- test/wasm_vm_test.cc | 20 ++++++-------------- 1 file changed, 6 insertions(+), 14 deletions(-) diff --git a/test/wasm_vm_test.cc b/test/wasm_vm_test.cc index a549cf919..22158c946 100644 --- a/test/wasm_vm_test.cc +++ b/test/wasm_vm_test.cc @@ -53,31 +53,23 @@ TEST_P(TestVm, Init) { return; } int expected_warm_time_ns = 0; - double expected_warm_time_speedup_factor = 0; if (engine_ == "v8") { - expected_warm_time_ns = 20000000; - expected_warm_time_speedup_factor = 1.5; + expected_warm_time_ns = 30000000; } else if (engine_ == "wamr") { - expected_warm_time_ns = 4000; - expected_warm_time_speedup_factor = 2000; + expected_warm_time_ns = 10000; } else if (engine_ == "wasmedge") { expected_warm_time_ns = 2000; - expected_warm_time_speedup_factor = 1.5; } else if (engine_ == "wasmtime") { expected_warm_time_ns = 8000; - expected_warm_time_speedup_factor = 6; } // Linux 390x is significantly slower, so we use a more lenient limit. #if defined(__linux__) && defined(__s390x__) - expected_warm_time_ns *= 10; -#endif -#if defined(THREAD_SANITIZER) || defined(MEMORY_SANITIZER) || defined(ADDRESS_SANITIZER) || \ - defined(HWADDRESS_SANITIZER) || defined(THREAD_SANITIZER) - expected_warm_time_ns *= 10; -#else - EXPECT_LE(warm * expected_warm_time_speedup_factor, cold); + expected_warm_time_ns *= 30; #endif +#if !defined(THREAD_SANITIZER) && !defined(MEMORY_SANITIZER) && !defined(ADDRESS_SANITIZER) && \ + !defined(HWADDRESS_SANITIZER) && !defined(THREAD_SANITIZER) EXPECT_LT(warm, expected_warm_time_ns); +#endif } TEST_P(TestVm, Basic) { From 2a15d7752314b4b2e14fc2ca4dd468c807a5d49f Mon Sep 17 00:00:00 2001 From: Matt Leon Date: Thu, 11 Jun 2026 11:11:41 -0400 Subject: [PATCH 7/7] Adjust wasmtime warm init time for macos Signed-off-by: Matt Leon --- test/wasm_vm_test.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/wasm_vm_test.cc b/test/wasm_vm_test.cc index 22158c946..c77c4cc79 100644 --- a/test/wasm_vm_test.cc +++ b/test/wasm_vm_test.cc @@ -60,7 +60,7 @@ TEST_P(TestVm, Init) { } else if (engine_ == "wasmedge") { expected_warm_time_ns = 2000; } else if (engine_ == "wasmtime") { - expected_warm_time_ns = 8000; + expected_warm_time_ns = 20000; } // Linux 390x is significantly slower, so we use a more lenient limit. #if defined(__linux__) && defined(__s390x__)