diff --git a/examples/vulkan/CarControlDemo/main.cpp b/examples/vulkan/CarControlDemo/main.cpp index df7e778..0530bd8 100644 --- a/examples/vulkan/CarControlDemo/main.cpp +++ b/examples/vulkan/CarControlDemo/main.cpp @@ -5,6 +5,7 @@ #include #include #include +#include #include #include #include @@ -15,6 +16,7 @@ #include "VulkanCommandBuffer.h" #include "VulkanDynamicRendering.h" #include "VulkanImage.h" +#include "VulkanPipelineCache.h" #include "VulkanSwapChain.h" #include "VulkanSync.h" #include "VulkanUtils.h" @@ -126,9 +128,15 @@ int main() { render_finished_semaphores.push_back(std::make_unique(&context)); } + core::vulkan::VulkanPipelineCache pipeline_cache( + &context, std::filesystem::path(PIPELINE_CACHE_DIR) / "car_control_demo.cache"); + skybox->SetPipelineCache(&pipeline_cache); + ground->SetPipelineCache(&pipeline_cache); + car->SetPipelineCache(&pipeline_cache); skybox->Init(kSkyboxFacePaths); ground->Init(kGroundTexturePath); car->Init(); + pipeline_cache.Save(); auto last_time = std::chrono::high_resolution_clock::now(); diff --git a/examples/vulkan/FPSShooterDemo/main.cpp b/examples/vulkan/FPSShooterDemo/main.cpp index 42eebfb..c33db01 100644 --- a/examples/vulkan/FPSShooterDemo/main.cpp +++ b/examples/vulkan/FPSShooterDemo/main.cpp @@ -5,6 +5,7 @@ #include #include #include +#include #include #include #include @@ -17,6 +18,7 @@ #include "VulkanCommandBuffer.h" #include "VulkanDynamicRendering.h" #include "VulkanImage.h" +#include "VulkanPipelineCache.h" #include "VulkanSwapChain.h" #include "VulkanSync.h" #include "VulkanUtils.h" @@ -137,10 +139,17 @@ int main() { render_finished_semaphores.push_back(std::make_unique(&context)); } + core::vulkan::VulkanPipelineCache pipeline_cache( + &context, std::filesystem::path(PIPELINE_CACHE_DIR) / "fps_shooter_demo.cache"); + floor->SetPipelineCache(&pipeline_cache); + targets->SetPipelineCache(&pipeline_cache); + bullets->SetPipelineCache(&pipeline_cache); + gun->SetPipelineCache(&pipeline_cache); floor->Init(); targets->Init(); bullets->Init(); gun->Init(kGunTexturePath, kGunModelPath); + pipeline_cache.Save(); auto last_time = std::chrono::high_resolution_clock::now(); diff --git a/examples/vulkan/PhysicsDemo/main.cpp b/examples/vulkan/PhysicsDemo/main.cpp index 9655033..4ceaa45 100644 --- a/examples/vulkan/PhysicsDemo/main.cpp +++ b/examples/vulkan/PhysicsDemo/main.cpp @@ -3,6 +3,7 @@ #include #include +#include #include #include @@ -13,6 +14,7 @@ #include "VulkanCommandBuffer.h" #include "VulkanDynamicRendering.h" #include "VulkanImage.h" +#include "VulkanPipelineCache.h" #include "VulkanSwapChain.h" #include "VulkanSync.h" #include "VulkanUtils.h" @@ -111,8 +113,13 @@ int main() { render_finished_semaphores.push_back(std::make_unique(&context)); } + core::vulkan::VulkanPipelineCache pipeline_cache( + &context, std::filesystem::path(PIPELINE_CACHE_DIR) / "physics_demo.cache"); + floor->SetPipelineCache(&pipeline_cache); + cube->SetPipelineCache(&pipeline_cache); floor->Init(); cube->Init(); + pipeline_cache.Save(); auto last_time = std::chrono::high_resolution_clock::now(); diff --git a/tests/vulkan/CMakeLists.txt b/tests/vulkan/CMakeLists.txt index b5bf10e..5ac6ccc 100644 --- a/tests/vulkan/CMakeLists.txt +++ b/tests/vulkan/CMakeLists.txt @@ -48,3 +48,19 @@ target_include_directories(vulkan_gaussian_benchmark PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/shaders) add_dependencies(vulkan_gaussian_benchmark vulkan_shaders) target_link_libraries(vulkan_gaussian_benchmark PRIVATE mat vulkan) + +add_executable(vulkan_pipeline_cache_benchmark + benchmarks/PipelineCacheBenchmark.cpp + ComputeGaussianBlur/ComputeGaussianBlur.cpp + ComputeSum/ComputeSum.cpp + Roofline/ComputeCopy.cpp + Roofline/ComputeFma.cpp + ${PROJECT_SOURCE_DIR}/examples/vulkan/ComputeBarycentricRasterizer/ComputeBarycentric.cpp) +target_include_directories(vulkan_pipeline_cache_benchmark PRIVATE + ComputeGaussianBlur + ComputeSum + Roofline + ${PROJECT_SOURCE_DIR}/examples/vulkan/ComputeBarycentricRasterizer + ${CMAKE_CURRENT_BINARY_DIR}/shaders) +add_dependencies(vulkan_pipeline_cache_benchmark vulkan_shaders vulkan_rasterizer_test_shader) +target_link_libraries(vulkan_pipeline_cache_benchmark PRIVATE mat vulkan) diff --git a/tests/vulkan/PipelineCacheProbe.frag b/tests/vulkan/PipelineCacheProbe.frag new file mode 100644 index 0000000..305a9d5 --- /dev/null +++ b/tests/vulkan/PipelineCacheProbe.frag @@ -0,0 +1,8 @@ +#version 450 + +layout(location = 0) in vec2 uv; +layout(location = 0) out vec4 color; + +void main() { + color = vec4(uv, 0.25, 1.0); +} diff --git a/tests/vulkan/PipelineCacheProbe.vert b/tests/vulkan/PipelineCacheProbe.vert new file mode 100644 index 0000000..be68a9e --- /dev/null +++ b/tests/vulkan/PipelineCacheProbe.vert @@ -0,0 +1,9 @@ +#version 450 + +layout(location = 0) out vec2 uv; + +void main() { + const vec2 positions[3] = vec2[](vec2(-1.0, -1.0), vec2(3.0, -1.0), vec2(-1.0, 3.0)); + uv = positions[gl_VertexIndex] * 0.5 + 0.5; + gl_Position = vec4(positions[gl_VertexIndex], 0.0, 1.0); +} diff --git a/tests/vulkan/VulkanPipelineCacheTest.cpp b/tests/vulkan/VulkanPipelineCacheTest.cpp new file mode 100644 index 0000000..4f0a014 --- /dev/null +++ b/tests/vulkan/VulkanPipelineCacheTest.cpp @@ -0,0 +1,379 @@ +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "VulkanCommandBuffer.h" +#include "VulkanCompute.h" +#include "VulkanPipelineCache.h" +#include "VulkanRender.h" +#include "VulkanRenderPass.h" +#include "VulkanSync.h" + +namespace { + +VkPipelineCache last_graphics_pipeline_cache = VK_NULL_HANDLE; +int graphics_pipeline_calls = 0; + +} + +extern "C" VKAPI_ATTR VkResult VKAPI_CALL vkCreateGraphicsPipelines( + VkDevice device, VkPipelineCache pipeline_cache, uint32_t create_info_count, + const VkGraphicsPipelineCreateInfo* create_infos, const VkAllocationCallbacks* allocator, + VkPipeline* pipelines) { + last_graphics_pipeline_cache = pipeline_cache; + ++graphics_pipeline_calls; + const PFN_vkCreateGraphicsPipelines create = reinterpret_cast( + vkGetDeviceProcAddr(device, "vkCreateGraphicsPipelines")); + return create(device, pipeline_cache, create_info_count, create_infos, allocator, pipelines); +} + +namespace core { +namespace test { +namespace { + +class CacheComputeProbe : public vulkan::VulkanCompute { + public: + explicit CacheComputeProbe(vulkan::VulkanContext* context, std::string legacy_path = "") + : VulkanCompute(context), legacy_path_(std::move(legacy_path)) {} + + void Bind(const vulkan::VulkanBuffer& output) { + CreateStorageBufferDescriptorSet(0, output); + vkUpdateDescriptorSets(context_->logical_device, static_cast(writes_.size()), + writes_.data(), 0, nullptr); + } + + void Run(VkCommandBuffer command) { + vkCmdBindPipeline(command, VK_PIPELINE_BIND_POINT_COMPUTE, pipeline); + vkCmdBindDescriptorSets(command, VK_PIPELINE_BIND_POINT_COMPUTE, pipeline_layout, 0, 1, + &descriptor_set_, 0, nullptr); + vkCmdDispatch(command, 1, 1, 1); + } + + void SaveLegacy(const std::string& path) const { SavePipelineCache(path); } + VkPipelineCache used_cache() const { return pipeline_cache_; } + + mutable bool legacy_path_queried = false; + + private: + std::vector GetBindingInfo() const override { + return {{.binding = 0, + .descriptor_type = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, + .shader_stage = VK_SHADER_STAGE_COMPUTE_BIT}}; + } + + const std::string GetPipelineCache() const override { + legacy_path_queried = true; + return legacy_path_; + } + + const std::vector& LoadShaderCode() const override { + static const std::vector code = +#include "ComputeSpecialization.comp.spv" + ; + return code; + } + + std::string legacy_path_; +}; + +class CacheRenderProbe : public vulkan::VulkanRender { + public: + CacheRenderProbe(vulkan::VulkanContext* context, vulkan::VulkanRenderPass* render_pass) + : VulkanRender(context, render_pass) {} + + private: + std::vector GetBindingInfo() const override { + return {{.binding = 0, + .descriptor_type = VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, + .shader_stage = VK_SHADER_STAGE_VERTEX_BIT}}; + } + + const std::vector LoadVertexShader() const override { + return +#include "PipelineCacheProbe.vert.spv" + ; + } + + const std::vector LoadFragmentShader() const override { + return +#include "PipelineCacheProbe.frag.spv" + ; + } + + std::vector GetVertexBindingDescriptions() const override { + return {}; + } + + std::vector GetVertexAttributeDescriptions() const override { + return {}; + } +}; + +std::vector ReadBytes(const std::filesystem::path& path) { + std::ifstream file(path, std::ios::binary); + const std::vector chars((std::istreambuf_iterator(file)), + std::istreambuf_iterator()); + std::vector bytes(chars.size()); + std::memcpy(bytes.data(), chars.data(), chars.size()); + return bytes; +} + +void WriteBytes(const std::filesystem::path& path, const std::vector& bytes) { + std::ofstream file(path, std::ios::binary | std::ios::trunc); + file.write(reinterpret_cast(bytes.data()), + static_cast(bytes.size())); +} + +VkPhysicalDeviceProperties DeviceProperties(const vulkan::VulkanContext& context) { + VkPhysicalDeviceProperties properties{}; + vkGetPhysicalDeviceProperties(context.physical_device, &properties); + return properties; +} + +void ExpectDefaultDispatch(vulkan::VulkanContext& context, CacheComputeProbe& probe) { + vulkan::VulkanBuffer output( + &context, 5 * sizeof(uint32_t), VK_BUFFER_USAGE_STORAGE_BUFFER_BIT, + VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT); + output.MapData([](void* bytes) { std::memset(bytes, 0xff, 5 * sizeof(uint32_t)); }); + probe.Bind(output); + vulkan::VulkanCommandBuffer command(&context); + command.BeginCommandBufferWithReset(); + probe.Run(command.buffer()); + const VkMemoryBarrier barrier{ + .sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER, + .pNext = nullptr, + .srcAccessMask = VK_ACCESS_SHADER_WRITE_BIT, + .dstAccessMask = VK_ACCESS_HOST_READ_BIT, + }; + vkCmdPipelineBarrier(command.buffer(), VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, + VK_PIPELINE_STAGE_HOST_BIT, 0, 1, &barrier, 0, nullptr, 0, nullptr); + vulkan::VulkanFence fence(&context); + fence.Reset(); + command.Submit(fence.fence, vulkan::SubmitSyncInfo{}); + VK_CHECK(vkWaitForFences(context.logical_device, 1, &fence.fence, VK_TRUE, UINT64_MAX)); + const uint32_t expected[] = {std::bit_cast(int32_t{-3}), std::bit_cast(1.5F), + 0U, 1U, 1U}; + output.MapData([&expected](void* bytes) { + const uint32_t* actual = static_cast(bytes); + for (size_t index = 0; index < 5; ++index) { + EXPECT_EQ(actual[index], expected[index]) << index; + } + }); +} + +class VulkanPipelineCacheTest : public ::testing::Test { + protected: + void SetUp() override { + const std::string cache_dir = PIPELINE_CACHE_DIR; + const std::filesystem::path root = cache_dir.empty() ? std::filesystem::temp_directory_path() + : std::filesystem::path(cache_dir); + directory_ = root / "vulkan_pipeline_cache_test"; + std::filesystem::remove_all(directory_); + std::filesystem::create_directories(directory_); + } + + void TearDown() override { + std::error_code error; + std::filesystem::remove_all(directory_, error); + } + + size_t FileCount() const { + size_t count = 0; + for (const std::filesystem::directory_entry& entry : + std::filesystem::directory_iterator(directory_)) { + if (entry.is_regular_file()) { + ++count; + } + } + return count; + } + + std::filesystem::path directory_; +}; + +TEST_F(VulkanPipelineCacheTest, SharesCacheAcrossComputeAndGraphicsAndRestoresOnNewDevice) { + const std::filesystem::path path = directory_ / "shared.cache"; + { + vulkan::VulkanContext context(true); + context.Init(); + vulkan::VulkanPipelineCache cache(&context, path); + EXPECT_FALSE(cache.restored()); + ASSERT_NE(cache.cache(), VK_NULL_HANDLE); + + CacheComputeProbe compute(&context, (directory_ / "legacy.cache").string()); + compute.SetPipelineCache(&cache); + compute.Init(); + EXPECT_FALSE(compute.legacy_path_queried); + EXPECT_EQ(compute.used_cache(), cache.cache()); + + vulkan::VulkanRenderPass render_pass(&context, VK_FORMAT_R8G8B8A8_UNORM); + CacheRenderProbe render(&context, &render_pass); + render.SetPipelineCache(&cache); + render.Init(); + ASSERT_NE(render.pipeline, VK_NULL_HANDLE); + + ASSERT_TRUE(cache.Save()); + EXPECT_EQ(FileCount(), 1U); + EXPECT_TRUE( + vulkan::VulkanPipelineCache::IsCompatible(ReadBytes(path), DeviceProperties(context))); + ExpectDefaultDispatch(context, compute); + } + + vulkan::VulkanContext context(true); + context.Init(); + vulkan::VulkanPipelineCache restored(&context, path); + EXPECT_TRUE(restored.restored()); + CacheComputeProbe compute(&context); + compute.SetPipelineCache(&restored); + compute.Init(); + vulkan::VulkanRenderPass render_pass(&context, VK_FORMAT_R8G8B8A8_UNORM); + CacheRenderProbe render(&context, &render_pass); + render.SetPipelineCache(&restored); + render.Init(); + ASSERT_NE(render.pipeline, VK_NULL_HANDLE); + ExpectDefaultDispatch(context, compute); + EXPECT_TRUE(restored.Save()); + EXPECT_EQ(FileCount(), 1U); +} + +TEST_F(VulkanPipelineCacheTest, RejectsIncompatibleDataAndReplacesItOnSave) { + vulkan::VulkanContext context(true); + context.Init(); + const VkPhysicalDeviceProperties properties = DeviceProperties(context); + const std::filesystem::path path = directory_ / "pipeline.cache"; + { + vulkan::VulkanPipelineCache cache(&context, path); + CacheComputeProbe compute(&context); + compute.SetPipelineCache(&cache); + compute.Init(); + ASSERT_TRUE(cache.Save()); + } + const std::vector valid = ReadBytes(path); + ASSERT_TRUE(vulkan::VulkanPipelineCache::IsCompatible(valid, properties)); + + const auto flip = [&valid](size_t offset) { + std::vector bytes = valid; + bytes[offset] ^= std::byte{0x01}; + return bytes; + }; + std::vector zero_header_size = valid; + std::fill_n(zero_header_size.begin(), 4, std::byte{0x00}); + const std::vector> invalid = { + {}, + std::vector(valid.begin(), valid.begin() + 31), + std::vector(valid.size(), std::byte{0x5a}), + zero_header_size, + flip(0), + flip(1), + flip(4), + flip(8), + flip(12), + flip(16), + flip(31), + }; + for (size_t index = 0; index < invalid.size(); ++index) { + SCOPED_TRACE(index); + EXPECT_FALSE(vulkan::VulkanPipelineCache::IsCompatible(invalid[index], properties)); + WriteBytes(path, invalid[index]); + vulkan::VulkanPipelineCache cache(&context, path); + EXPECT_FALSE(cache.restored()); + CacheComputeProbe compute(&context); + compute.SetPipelineCache(&cache); + compute.Init(); + ExpectDefaultDispatch(context, compute); + ASSERT_TRUE(cache.Save()); + EXPECT_TRUE(vulkan::VulkanPipelineCache::IsCompatible(ReadBytes(path), properties)); + EXPECT_EQ(FileCount(), 1U); + } + + for (const std::vector& header : + {std::vector(valid.begin(), valid.begin() + 32), flip(8)}) { + WriteBytes(path, header); + std::filesystem::resize_file(path, vulkan::VulkanPipelineCache::kMaxDataSize + 1); + vulkan::VulkanPipelineCache oversized(&context, path); + EXPECT_FALSE(oversized.restored()); + } + std::filesystem::remove(path); +} + +TEST_F(VulkanPipelineCacheTest, GraphicsPipelinesReceiveSharedCache) { + vulkan::VulkanContext context(true); + context.Init(); + vulkan::VulkanPipelineCache cache(&context, directory_ / "graphics.cache"); + vulkan::VulkanRenderPass render_pass(&context, VK_FORMAT_R8G8B8A8_UNORM); + + CacheRenderProbe uncached(&context, &render_pass); + const int calls_before = graphics_pipeline_calls; + uncached.Init(); + ASSERT_EQ(graphics_pipeline_calls, calls_before + 1); + EXPECT_EQ(last_graphics_pipeline_cache, VK_NULL_HANDLE); + ASSERT_NE(uncached.pipeline, VK_NULL_HANDLE); + + CacheRenderProbe cached(&context, &render_pass); + cached.SetPipelineCache(&cache); + cached.Init(); + ASSERT_EQ(graphics_pipeline_calls, calls_before + 2); + EXPECT_EQ(last_graphics_pipeline_cache, cache.cache()); + ASSERT_NE(cached.pipeline, VK_NULL_HANDLE); +} + +TEST_F(VulkanPipelineCacheTest, LegacyComputePathValidatesRestoredData) { + vulkan::VulkanContext context(true); + context.Init(); + const VkPhysicalDeviceProperties properties = DeviceProperties(context); + const std::filesystem::path path = directory_ / "legacy.cache"; + WriteBytes(path, std::vector(256, std::byte{0xa5})); + + CacheComputeProbe compute(&context, path.string()); + compute.Init(); + EXPECT_TRUE(compute.legacy_path_queried); + EXPECT_NE(compute.used_cache(), VK_NULL_HANDLE); + ExpectDefaultDispatch(context, compute); + compute.SaveLegacy(path.string()); + EXPECT_TRUE(vulkan::VulkanPipelineCache::IsCompatible(ReadBytes(path), properties)); + + vulkan::VulkanPipelineCache restored(&context, path); + EXPECT_TRUE(restored.restored()); + EXPECT_EQ(FileCount(), 1U); +} + +TEST_F(VulkanPipelineCacheTest, SaveFailsWithoutLeavingFiles) { + vulkan::VulkanContext context(true); + context.Init(); + + vulkan::VulkanPipelineCache in_memory(&context, std::filesystem::path()); + EXPECT_FALSE(in_memory.restored()); + CacheComputeProbe compute(&context); + compute.SetPipelineCache(&in_memory); + compute.Init(); + EXPECT_FALSE(in_memory.Save()); + + const std::filesystem::path missing = directory_ / "missing" / "pipeline.cache"; + vulkan::VulkanPipelineCache unwritable(&context, missing); + EXPECT_FALSE(unwritable.restored()); + CacheComputeProbe second(&context); + second.SetPipelineCache(&unwritable); + second.Init(); + EXPECT_FALSE(unwritable.Save()); + EXPECT_FALSE(std::filesystem::exists(missing.parent_path())); + EXPECT_EQ(FileCount(), 0U); + EXPECT_FALSE(vulkan::VulkanPipelineCache::Write(context.logical_device, VK_NULL_HANDLE, + directory_ / "null.cache")); + EXPECT_EQ(FileCount(), 0U); +} + +} +} +} diff --git a/tests/vulkan/benchmarks/PipelineCacheBenchmark.cpp b/tests/vulkan/benchmarks/PipelineCacheBenchmark.cpp new file mode 100644 index 0000000..1a772c1 --- /dev/null +++ b/tests/vulkan/benchmarks/PipelineCacheBenchmark.cpp @@ -0,0 +1,218 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "ComputeBarycentric.h" +#include "ComputeCopy.h" +#include "ComputeFma.h" +#include "ComputeGaussianBlur.h" +#include "ComputeSum.h" +#include "VulkanCommandBuffer.h" +#include "VulkanPipelineCache.h" +#include "VulkanSync.h" + +namespace { + +using core::vulkan::VulkanBuffer; +using core::vulkan::VulkanContext; +using core::vulkan::VulkanPipelineCache; + +constexpr int kImageSize = 256; +constexpr int kCopyCount = 1024; +constexpr size_t kCopyWords = static_cast(kCopyCount) * 4; +constexpr size_t kPipelineCount = 5; +constexpr std::array kPipelineNames = {"gaussian_blur", "sum", "copy", + "fma", "barycentric"}; +constexpr VkMemoryPropertyFlags kHostMemory = + VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT; +constexpr VkDeviceSize kImageBytes = static_cast(kImageSize) * kImageSize * 4; +constexpr VkDeviceSize kCopyBytes = static_cast(kCopyWords) * sizeof(uint32_t); + +struct Sample { + double total_us = 0.0; + std::array pipeline_us{}; +}; + +double Elapsed(std::chrono::steady_clock::time_point start) { + return std::chrono::duration(std::chrono::steady_clock::now() - start) + .count(); +} + +uint32_t Pattern(size_t index) { return static_cast(index) * 2654435761U; } + +class PipelineSet { + public: + explicit PipelineSet(VulkanContext* context) + : context_(context), + image_source_(context, kImageBytes, VK_BUFFER_USAGE_STORAGE_BUFFER_BIT, kHostMemory), + image_destination_(context, kImageBytes, VK_BUFFER_USAGE_STORAGE_BUFFER_BIT, kHostMemory), + copy_source_(context, kCopyBytes, VK_BUFFER_USAGE_STORAGE_BUFFER_BIT, kHostMemory), + copy_destination_(context, kCopyBytes, VK_BUFFER_USAGE_STORAGE_BUFFER_BIT, kHostMemory), + fma_output_(context, 64 * 16, VK_BUFFER_USAGE_STORAGE_BUFFER_BIT, kHostMemory), + blur_(context, image_source_, image_destination_, kImageSize, kImageSize), + sum_(context, image_source_, image_destination_, kImageSize, kImageSize), + copy_(context, copy_source_, copy_destination_, kCopyCount), + fma_(context, fma_output_, 1, 1), + barycentric_(context, kImageSize, kImageSize) {} + + Sample Create(const VulkanPipelineCache& cache) { + const std::array pipelines = { + &blur_, &sum_, ©_, &fma_, &barycentric_}; + Sample sample{}; + for (size_t index = 0; index < kPipelineCount; ++index) { + const std::chrono::steady_clock::time_point start = std::chrono::steady_clock::now(); + pipelines[index]->SetPipelineCache(&cache); + pipelines[index]->Init(); + sample.pipeline_us[index] = Elapsed(start); + if (pipelines[index]->pipeline == VK_NULL_HANDLE) { + throw std::runtime_error("Pipeline cache benchmark created a null pipeline"); + } + } + return sample; + } + + void VerifyCopy() { + copy_source_.MapData([](void* data) { + uint32_t* values = static_cast(data); + for (size_t index = 0; index < kCopyWords; ++index) { + values[index] = Pattern(index); + } + }); + copy_destination_.MapData( + [](void* data) { std::fill_n(static_cast(data), kCopyWords, 0U); }); + core::vulkan::VulkanCommandBuffer command(context_); + command.BeginCommandBufferWithReset(); + copy_.Run(command.buffer()); + const VkMemoryBarrier barrier{ + .sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER, + .pNext = nullptr, + .srcAccessMask = VK_ACCESS_SHADER_WRITE_BIT, + .dstAccessMask = VK_ACCESS_HOST_READ_BIT, + }; + vkCmdPipelineBarrier(command.buffer(), VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, + VK_PIPELINE_STAGE_HOST_BIT, 0, 1, &barrier, 0, nullptr, 0, nullptr); + core::vulkan::VulkanFence fence(context_); + fence.Reset(); + command.Submit(fence.fence); + VK_CHECK(vkWaitForFences(context_->logical_device, 1, &fence.fence, VK_TRUE, UINT64_MAX)); + copy_destination_.MapData([](void* data) { + const uint32_t* values = static_cast(data); + for (size_t index = 0; index < kCopyWords; ++index) { + if (values[index] != Pattern(index)) { + throw std::runtime_error("Pipeline cache benchmark copy mismatch"); + } + } + }); + } + + private: + VulkanContext* context_; + VulkanBuffer image_source_; + VulkanBuffer image_destination_; + VulkanBuffer copy_source_; + VulkanBuffer copy_destination_; + VulkanBuffer fma_output_; + core::vulkan::ComputeGaussianBlur blur_; + core::vulkan::ComputeSum sum_; + core::vulkan::ComputeCopy copy_; + core::vulkan::ComputeFma fma_; + core::example::ComputeBarycentric barycentric_; +}; + +Sample Run(const std::filesystem::path& cache_path, bool warm) { + VulkanContext context(false); + context.Init(); + PipelineSet pipelines(&context); + const std::chrono::steady_clock::time_point start = std::chrono::steady_clock::now(); + const VulkanPipelineCache cache(&context, warm ? cache_path : std::filesystem::path()); + Sample sample = pipelines.Create(cache); + sample.total_us = Elapsed(start); + if (cache.restored() != warm) { + throw std::runtime_error("Unexpected pipeline cache restore state"); + } + pipelines.VerifyCopy(); + return sample; +} + +void Prime(const std::filesystem::path& cache_path) { + std::filesystem::remove(cache_path); + VulkanContext context(false); + context.Init(); + PipelineSet pipelines(&context); + const VulkanPipelineCache cache(&context, cache_path); + pipelines.Create(cache); + if (!cache.Save()) { + throw std::runtime_error("Failed to save pipeline cache"); + } +} + +double Percentile(std::vector values, double fraction) { + std::sort(values.begin(), values.end()); + const size_t index = static_cast(fraction * static_cast(values.size() - 1)); + return values[index]; +} + +std::vector Totals(const std::vector& samples) { + std::vector totals; + for (const Sample& sample : samples) { + totals.push_back(sample.total_us); + } + return totals; +} + +void Report(const char* label, const std::vector& samples) { + const std::vector totals = Totals(samples); + std::printf("%-5s total_us n=%zu median=%.1f p10=%.1f p90=%.1f min=%.1f max=%.1f\n", label, + totals.size(), Percentile(totals, 0.5), Percentile(totals, 0.1), + Percentile(totals, 0.9), Percentile(totals, 0.0), Percentile(totals, 1.0)); + for (size_t index = 0; index < kPipelineCount; ++index) { + std::vector values; + for (const Sample& sample : samples) { + values.push_back(sample.pipeline_us[index]); + } + std::printf("%-5s %-13s median_us=%.1f\n", label, kPipelineNames[index], + Percentile(values, 0.5)); + } +} + +} + +int main(int argc, char** argv) { + const std::string cache_dir = PIPELINE_CACHE_DIR; + const std::filesystem::path default_dir = + cache_dir.empty() ? std::filesystem::temp_directory_path() : std::filesystem::path(cache_dir); + const std::filesystem::path cache_path = + argc > 1 ? std::filesystem::path(argv[1]) : default_dir / "pipeline_cache_benchmark.cache"; + const int samples = argc > 2 ? std::stoi(argv[2]) : 15; + if (samples <= 0) { + throw std::invalid_argument("Sample count must be positive"); + } + constexpr int kWarmups = 2; + + Prime(cache_path); + std::vector cold; + std::vector warm; + for (int sample = -kWarmups; sample < samples; ++sample) { + for (int turn = 0; turn < 2; ++turn) { + const bool use_warm = ((sample + kWarmups + turn) % 2) == 0; + const Sample result = Run(cache_path, use_warm); + if (sample >= 0) { + (use_warm ? warm : cold).push_back(result); + } + } + } + std::printf("cache=%s bytes=%ju\n", cache_path.string().c_str(), + static_cast(std::filesystem::file_size(cache_path))); + Report("cold", cold); + Report("warm", warm); + std::printf("median_speedup=%.2fx\n", + Percentile(Totals(cold), 0.5) / Percentile(Totals(warm), 0.5)); + std::filesystem::remove(cache_path); + return 0; +} diff --git a/vulkan/include/VulkanBase.h b/vulkan/include/VulkanBase.h index f7a5dee..9ec2f57 100644 --- a/vulkan/include/VulkanBase.h +++ b/vulkan/include/VulkanBase.h @@ -7,6 +7,7 @@ #include "VulkanBuffer.h" #include "VulkanContext.h" #include "VulkanImage.h" +#include "VulkanPipelineCache.h" namespace core { namespace vulkan { @@ -29,6 +30,8 @@ class VulkanBase { virtual void Init(); + void SetPipelineCache(const VulkanPipelineCache* cache); + VkPipelineLayout pipeline_layout = VK_NULL_HANDLE; VkPipeline pipeline = VK_NULL_HANDLE; @@ -41,6 +44,7 @@ class VulkanBase { std::vector CreateDescriptorLayoutBindings() const; std::vector CreateDescriptorPoolSizes() const; VkShaderModule CreateShaderModule(const std::vector& shader_code) const; + VkPipelineCache shared_pipeline_cache() const { return shared_pipeline_cache_; } void CreateUniformBufferDescriptorSet(const uint32_t binding, const VulkanBuffer& buffer); void CreateStorageBufferDescriptorSet(const uint32_t binding, const VulkanBuffer& buffer); @@ -59,6 +63,7 @@ class VulkanBase { private: std::mutex write_mutex_; + VkPipelineCache shared_pipeline_cache_ = VK_NULL_HANDLE; int buffer_idx_ = 0; int image_idx_ = 0; diff --git a/vulkan/include/VulkanCompute.h b/vulkan/include/VulkanCompute.h index 62c2e3e..a3aeb9d 100644 --- a/vulkan/include/VulkanCompute.h +++ b/vulkan/include/VulkanCompute.h @@ -1,12 +1,15 @@ #pragma once #include +#include +#include #include #include "VulkanBase.h" #include "VulkanBuffer.h" #include "VulkanContext.h" #include "VulkanImage.h" +#include "VulkanPipelineCache.h" namespace core { namespace vulkan { @@ -34,9 +37,7 @@ class VulkanCompute : public VulkanBase { private: virtual const std::vector& LoadShaderCode() const = 0; - void CreatePipelineCache(const std::string& cache); - - static std::vector LoadPipelineCache(const std::string& cache); + std::unique_ptr owned_pipeline_cache_; protected: VkPipelineCache pipeline_cache_ = VK_NULL_HANDLE; diff --git a/vulkan/include/VulkanPipelineCache.h b/vulkan/include/VulkanPipelineCache.h new file mode 100644 index 0000000..2a6de86 --- /dev/null +++ b/vulkan/include/VulkanPipelineCache.h @@ -0,0 +1,43 @@ +#pragma once + +#include + +#include +#include +#include +#include + +#include "VulkanContext.h" + +namespace core { +namespace vulkan { + +class VulkanPipelineCache { + public: + static constexpr std::uintmax_t kMaxDataSize = std::uintmax_t{64} << 20U; + + VulkanPipelineCache(VulkanContext* context, std::filesystem::path path); + ~VulkanPipelineCache(); + + VulkanPipelineCache(const VulkanPipelineCache&) = delete; + VulkanPipelineCache& operator=(const VulkanPipelineCache&) = delete; + + VkPipelineCache cache() const { return cache_; } + const std::filesystem::path& path() const { return path_; } + bool restored() const { return restored_; } + + bool Save() const; + + static bool IsCompatible(const std::vector& data, + const VkPhysicalDeviceProperties& properties); + static bool Write(VkDevice device, VkPipelineCache cache, const std::filesystem::path& path); + + private: + VulkanContext* context_ = nullptr; + std::filesystem::path path_; + VkPipelineCache cache_ = VK_NULL_HANDLE; + bool restored_ = false; +}; + +} +} diff --git a/vulkan/src/VulkanBase.cpp b/vulkan/src/VulkanBase.cpp index c250774..e914806 100644 --- a/vulkan/src/VulkanBase.cpp +++ b/vulkan/src/VulkanBase.cpp @@ -16,6 +16,10 @@ VulkanBase::~VulkanBase() { vkDestroyDescriptorSetLayout(context_->logical_device, descriptor_set_layout_, nullptr); } +void VulkanBase::SetPipelineCache(const VulkanPipelineCache* cache) { + shared_pipeline_cache_ = cache == nullptr ? VK_NULL_HANDLE : cache->cache(); +} + void VulkanBase::Init() { const std::vector push_constant_ranges = GetPushConstantRanges(); diff --git a/vulkan/src/VulkanCompute.cpp b/vulkan/src/VulkanCompute.cpp index bd5e224..e669fe1 100644 --- a/vulkan/src/VulkanCompute.cpp +++ b/vulkan/src/VulkanCompute.cpp @@ -1,6 +1,5 @@ #include "VulkanCompute.h" -#include #include #include #include @@ -10,9 +9,7 @@ namespace vulkan { VulkanCompute::VulkanCompute(VulkanContext* context) : VulkanBase(context) {} -VulkanCompute::~VulkanCompute() { - vkDestroyPipelineCache(context_->logical_device, pipeline_cache_, nullptr); -} +VulkanCompute::~VulkanCompute() = default; void VulkanCompute::CreatePipeline() { const ComputeSpecialization specialization = GetSpecialization(); @@ -41,7 +38,13 @@ void VulkanCompute::CreatePipeline() { ~ShaderModuleScope() { vkDestroyShaderModule(device, module, nullptr); } } shader_scope{.device = context_->logical_device, .module = shader_module}; - CreatePipelineCache(GetPipelineCache()); + if (shared_pipeline_cache() != VK_NULL_HANDLE) { + owned_pipeline_cache_.reset(); + pipeline_cache_ = shared_pipeline_cache(); + } else { + owned_pipeline_cache_ = std::make_unique(context_, GetPipelineCache()); + pipeline_cache_ = owned_pipeline_cache_->cache(); + } VkPipelineShaderStageCreateInfo shader_stage_info{}; shader_stage_info.sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO; @@ -64,56 +67,8 @@ ComputeSpecialization VulkanCompute::GetSpecialization() const { return {}; } const std::string VulkanCompute::GetPipelineCache() const { return ""; } -std::vector VulkanCompute::LoadPipelineCache(const std::string& cache) { - // Load the pipeline cache from the specified file - std::ifstream file(cache, std::ios::ate | std::ios::binary); - if (!file.is_open()) { - return {}; - } - - size_t file_size = static_cast(file.tellg()); - file.seekg(0); - - std::vector buffer(file_size); - file.read(buffer.data(), file_size); - return buffer; -} - -void VulkanCompute::CreatePipelineCache(const std::string& cache) { - std::vector cache_data = LoadPipelineCache(cache); - VkPipelineCacheCreateInfo cache_info{}; - cache_info.sType = VK_STRUCTURE_TYPE_PIPELINE_CACHE_CREATE_INFO; - - if (cache_data.empty()) { - cache_info.initialDataSize = 0; - cache_info.pInitialData = nullptr; - } else { - cache_info.initialDataSize = cache_data.size(); - cache_info.pInitialData = cache_data.data(); - } - - VK_CHECK(vkCreatePipelineCache(context_->logical_device, &cache_info, nullptr, &pipeline_cache_)); -} - void VulkanCompute::SavePipelineCache(const std::string& cache) const { - if (pipeline_cache_ == VK_NULL_HANDLE) { - return; - } - - size_t cache_size = 0; - VK_CHECK(vkGetPipelineCacheData(context_->logical_device, pipeline_cache_, &cache_size, nullptr)); - if (cache_size == 0) { - return; - } - - std::vector cache_data(cache_size); - VK_CHECK(vkGetPipelineCacheData(context_->logical_device, pipeline_cache_, &cache_size, - cache_data.data())); - - std::ofstream file(cache, std::ios::binary); - if (file.is_open()) { - file.write(cache_data.data(), cache_data.size()); - } + VulkanPipelineCache::Write(context_->logical_device, pipeline_cache_, cache); } } // namespace vulkan diff --git a/vulkan/src/VulkanPipelineCache.cpp b/vulkan/src/VulkanPipelineCache.cpp new file mode 100644 index 0000000..a2c9882 --- /dev/null +++ b/vulkan/src/VulkanPipelineCache.cpp @@ -0,0 +1,151 @@ +#include "VulkanPipelineCache.h" + +#include +#include +#include +#include +#include +#include +#include + +namespace core { +namespace vulkan { + +namespace { + +constexpr size_t kHeaderSize = 32; + +uint32_t ReadLittleEndian32(const std::vector& data, size_t offset) { + uint32_t value = 0; + for (size_t index = 0; index < sizeof(uint32_t); ++index) { + value |= static_cast(std::to_integer(data[offset + index])) + << (8U * static_cast(index)); + } + return value; +} + +std::vector ReadCompatibleFile(const std::filesystem::path& path, + const VkPhysicalDeviceProperties& properties) { + std::error_code error; + const std::uintmax_t size = std::filesystem::file_size(path, error); + if (error || size < kHeaderSize || size > VulkanPipelineCache::kMaxDataSize) { + return {}; + } + std::ifstream file(path, std::ios::binary); + if (!file.is_open()) { + return {}; + } + std::vector data(kHeaderSize); + file.read(reinterpret_cast(data.data()), static_cast(kHeaderSize)); + if (!file || !VulkanPipelineCache::IsCompatible(data, properties)) { + return {}; + } + data.resize(static_cast(size)); + file.read(reinterpret_cast(data.data() + kHeaderSize), + static_cast(size - kHeaderSize)); + if (!file || file.peek() != std::ifstream::traits_type::eof()) { + return {}; + } + return data; +} + +} + +VulkanPipelineCache::VulkanPipelineCache(VulkanContext* context, std::filesystem::path path) + : context_(context), path_(std::move(path)) { + std::vector data; + if (!path_.empty()) { + VkPhysicalDeviceProperties properties{}; + vkGetPhysicalDeviceProperties(context_->physical_device, &properties); + data = ReadCompatibleFile(path_, properties); + } + restored_ = !data.empty(); + + const VkPipelineCacheCreateInfo create_info{ + .sType = VK_STRUCTURE_TYPE_PIPELINE_CACHE_CREATE_INFO, + .pNext = nullptr, + .flags = 0, + .initialDataSize = data.size(), + .pInitialData = restored_ ? data.data() : nullptr, + }; + VK_CHECK(vkCreatePipelineCache(context_->logical_device, &create_info, nullptr, &cache_)); +} + +VulkanPipelineCache::~VulkanPipelineCache() { + vkDestroyPipelineCache(context_->logical_device, cache_, nullptr); +} + +bool VulkanPipelineCache::Save() const { return Write(context_->logical_device, cache_, path_); } + +bool VulkanPipelineCache::IsCompatible(const std::vector& data, + const VkPhysicalDeviceProperties& properties) { + if (data.size() < kHeaderSize) { + return false; + } + for (size_t index = 0; index < VK_UUID_SIZE; ++index) { + if (std::to_integer(data[16 + index]) != properties.pipelineCacheUUID[index]) { + return false; + } + } + return ReadLittleEndian32(data, 0) == static_cast(kHeaderSize) && + ReadLittleEndian32(data, 4) == + static_cast(VK_PIPELINE_CACHE_HEADER_VERSION_ONE) && + ReadLittleEndian32(data, 8) == properties.vendorID && + ReadLittleEndian32(data, 12) == properties.deviceID; +} + +bool VulkanPipelineCache::Write(VkDevice device, VkPipelineCache cache, + const std::filesystem::path& path) { + if (cache == VK_NULL_HANDLE || path.empty()) { + return false; + } + + std::vector data; + VkResult result = VK_INCOMPLETE; + while (result == VK_INCOMPLETE) { + size_t size = 0; + VK_CHECK(vkGetPipelineCacheData(device, cache, &size, nullptr)); + if (static_cast(size) > kMaxDataSize) { + return false; + } + data.resize(size); + result = vkGetPipelineCacheData(device, cache, &size, data.data()); + if (result != VK_INCOMPLETE) { + VK_CHECK(result); + data.resize(size); + } + } + if (data.empty()) { + return false; + } + + std::random_device random; + const uint64_t suffix = + (static_cast(random()) << 32U) | static_cast(random()); + std::filesystem::path temp_path = path; + temp_path += ".tmp" + std::to_string(suffix); + + std::error_code error; + { + std::ofstream file(temp_path, std::ios::binary | std::ios::trunc); + if (!file.is_open()) { + return false; + } + file.write(reinterpret_cast(data.data()), + static_cast(data.size())); + file.close(); + if (!file) { + std::filesystem::remove(temp_path, error); + return false; + } + } + std::filesystem::rename(temp_path, path, error); + if (error) { + std::filesystem::remove(temp_path, error); + return false; + } + return true; +} + +} +} diff --git a/vulkan/src/VulkanRender.cpp b/vulkan/src/VulkanRender.cpp index 944dc90..ebd23cf 100644 --- a/vulkan/src/VulkanRender.cpp +++ b/vulkan/src/VulkanRender.cpp @@ -179,8 +179,8 @@ void VulkanRender::CreatePipeline() { pipeline_info.basePipelineHandle = VK_NULL_HANDLE; pipeline_info.basePipelineIndex = -1; - VK_CHECK(vkCreateGraphicsPipelines(context_->logical_device, VK_NULL_HANDLE, 1, &pipeline_info, - nullptr, &pipeline)); + VK_CHECK(vkCreateGraphicsPipelines(context_->logical_device, shared_pipeline_cache(), 1, + &pipeline_info, nullptr, &pipeline)); vkDestroyShaderModule(context_->logical_device, fragment_shader_module, nullptr); vkDestroyShaderModule(context_->logical_device, vertex_shader_module, nullptr); }