diff --git a/bazel/dependencies.bzl b/bazel/dependencies.bzl index e0439bb07..9e80e355a 100644 --- a/bazel/dependencies.bzl +++ b/bazel/dependencies.bzl @@ -14,6 +14,7 @@ load("@aspect_rules_lint//format:repositories.bzl", "rules_lint_dependencies") load("@bazel_lib//lib:repositories.bzl", "bazel_lib_dependencies", "bazel_lib_register_toolchains") +load("@com_google_benchmark//:bazel/benchmark_deps.bzl", "benchmark_deps") load("@com_google_googletest//:googletest_deps.bzl", "googletest_deps") load("@com_google_protobuf//:protobuf_deps.bzl", "protobuf_deps") load("@envoy_toolshed//sysroot:sysroot.bzl", "setup_sysroots") @@ -31,6 +32,7 @@ def proxy_wasm_cpp_host_dependencies(): # Bazel extensions. googletest_deps() rules_foreign_cc_dependencies() + benchmark_deps() rules_lint_dependencies() diff --git a/bazel/repositories.bzl b/bazel/repositories.bzl index 4f0937c2a..2c0930545 100644 --- a/bazel/repositories.bzl +++ b/bazel/repositories.bzl @@ -193,6 +193,14 @@ def proxy_wasm_cpp_host_repositories(): }, ) + maybe( + http_archive, + name = "com_google_benchmark", + sha256 = "9631341c82bac4a288bef951f8b26b41f69021794184ece969f8473977eaa340", + strip_prefix = "benchmark-1.9.5", + urls = ["https://github.com/google/benchmark/archive/refs/tags/v1.9.5.tar.gz"], + ) + # NullVM dependencies. maybe( diff --git a/test/BUILD b/test/BUILD index 1b45b1318..d653f579b 100644 --- a/test/BUILD +++ b/test/BUILD @@ -223,6 +223,7 @@ cc_test( deps = [ ":utility_lib", "//:lib", + "@com_google_benchmark//:benchmark", "@com_google_googletest//:gtest", "@com_google_googletest//:gtest_main", ], diff --git a/test/wasm_vm_test.cc b/test/wasm_vm_test.cc index d792c63df..c77c4cc79 100644 --- a/test/wasm_vm_test.cc +++ b/test/wasm_vm_test.cc @@ -13,6 +13,7 @@ // limitations under the License. #include "gtest/gtest.h" +#include "benchmark/benchmark.h" #include #include @@ -31,10 +32,12 @@ INSTANTIATE_TEST_SUITE_P(WasmEngines, TestVm, testing::ValuesIn(getWasmEngines() }); TEST_P(TestVm, Init) { + std::unique_ptr vm1 = makeVm(engine_); + std::unique_ptr vm2 = makeVm(engine_); auto time1 = std::chrono::steady_clock::now(); - vm_->warm(); + vm1->warm(); auto time2 = std::chrono::steady_clock::now(); - vm_->warm(); + vm2->warm(); auto time3 = std::chrono::steady_clock::now(); auto cold = std::chrono::duration_cast(time2 - time1).count(); @@ -43,24 +46,30 @@ TEST_P(TestVm, Init) { std::cout << "[" << engine_ << "] \"cold\" engine time: " << cold << "ns" << std::endl; std::cout << "[" << engine_ << "] \"warm\" engine time: " << warm << "ns" << std::endl; - // Default warm time in nanoseconds. - int warm_time_ns_limit = 10000; - -#if defined(__linux__) && defined(__s390x__) - // Linux 390x is significantly slower, so we use a more lenient limit. - warm_time_ns_limit = 75000; -#endif - - // Verify that getting a "warm" engine takes less than 10us. - EXPECT_LE(warm, warm_time_ns_limit); - // Verify that getting a "warm" engine takes at least 50x less time than getting a "cold" one. // We skip NullVM because warm() is a noop. if (engine_ == "null") { std::cout << "Skipping warm() performance assertions for NullVM." << std::endl; return; } - EXPECT_LE(warm * 50, cold); + int expected_warm_time_ns = 0; + if (engine_ == "v8") { + expected_warm_time_ns = 30000000; + } else if (engine_ == "wamr") { + expected_warm_time_ns = 10000; + } else if (engine_ == "wasmedge") { + expected_warm_time_ns = 2000; + } else if (engine_ == "wasmtime") { + expected_warm_time_ns = 20000; + } + // Linux 390x is significantly slower, so we use a more lenient limit. +#if defined(__linux__) && defined(__s390x__) + expected_warm_time_ns *= 30; +#endif +#if !defined(THREAD_SANITIZER) && !defined(MEMORY_SANITIZER) && !defined(ADDRESS_SANITIZER) && \ + !defined(HWADDRESS_SANITIZER) && !defined(THREAD_SANITIZER) + EXPECT_LT(warm, expected_warm_time_ns); +#endif } TEST_P(TestVm, Basic) { @@ -164,5 +173,29 @@ TEST_P(TestVm, DISABLED_CloneUntilOutOfMemory) { #endif +void BM_WarmVmStart(benchmark::State &state, auto makeVm) { + std::unique_ptr cold_vm = makeVm(); + cold_vm->warm(); + for (auto _ : state) { + std::unique_ptr warm_vm = makeVm(); + warm_vm->warm(); + } +} + +TEST(TestVm, Benchmarks) { +#if defined(THREAD_SANITIZER) || defined(MEMORY_SANITIZER) || defined(ADDRESS_SANITIZER) || \ + defined(HWADDRESS_SANITIZER) || defined(THREAD_SANITIZER) + GTEST_SKIP() << "Disabled due to sanitizer."; +#endif + for (std::string engine : getWasmEngines()) { + benchmark::RegisterBenchmark( + "BM_WarmVmStart/" + engine, + [](benchmark::State &state, auto makeVm) { BM_WarmVmStart(state, makeVm); }, + [engine]() { return TestVm::makeVm(engine); }); + } + benchmark::RunSpecifiedBenchmarks(); + benchmark::Shutdown(); +} + } // namespace } // namespace proxy_wasm