diff --git a/unified-runtime/source/adapters/level_zero/device.cpp b/unified-runtime/source/adapters/level_zero/device.cpp index 638df58eea1c3..c46153e05686c 100644 --- a/unified-runtime/source/adapters/level_zero/device.cpp +++ b/unified-runtime/source/adapters/level_zero/device.cpp @@ -1388,6 +1388,13 @@ ur_result_t urDeviceGetInfo( return ReturnValue(true); #else return ReturnValue(false); +#endif + case UR_DEVICE_INFO_IPC_EVENT_SUPPORT_EXP: +#if defined(UR_ADAPTER_LEVEL_ZERO_V2) && defined(__linux__) + return ReturnValue( + static_cast(Device->Platform->loadIpcEventExtension())); +#else + return ReturnValue(false); #endif case UR_DEVICE_INFO_ASYNC_BARRIER: return ReturnValue(false); diff --git a/unified-runtime/source/adapters/level_zero/platform.cpp b/unified-runtime/source/adapters/level_zero/platform.cpp index 574365c92df97..9ca15e1c481a4 100644 --- a/unified-runtime/source/adapters/level_zero/platform.cpp +++ b/unified-runtime/source/adapters/level_zero/platform.cpp @@ -659,6 +659,25 @@ ur_result_t ur_platform_handle_t_::initialize() { return UR_RESULT_SUCCESS; } +bool ur_platform_handle_t_::loadIpcEventExtension() { + std::call_once(ZeIpcEventExt.InitFlag, [this] { + auto load = [this](const char *name, auto &slot) { + void *ptr = nullptr; + if (ZE_CALL_NOCHECK(zeDriverGetExtensionFunctionAddress, + (ZeDriver, name, &ptr)) == ZE_RESULT_SUCCESS) { + slot = reinterpret_cast>(ptr); + } + }; + load("zexCounterBasedEventGetIpcHandle", ZeIpcEventExt.pfnGetIpcHandle); + load("zexCounterBasedEventOpenIpcHandle", ZeIpcEventExt.pfnOpenIpcHandle); + load("zexCounterBasedEventCloseIpcHandle", ZeIpcEventExt.pfnCloseIpcHandle); + ZeIpcEventExt.Supported = ZeIpcEventExt.pfnGetIpcHandle != nullptr && + ZeIpcEventExt.pfnOpenIpcHandle != nullptr && + ZeIpcEventExt.pfnCloseIpcHandle != nullptr; + }); + return ZeIpcEventExt.Supported; +} + bool ur_platform_handle_t_::allowDriverInOrderLists(bool OnlyIfRequested) { // Use in-order lists implementation from L0 driver instead // of adapter's implementation. diff --git a/unified-runtime/source/adapters/level_zero/platform.hpp b/unified-runtime/source/adapters/level_zero/platform.hpp index e6f582c78eee0..f4925ff74306d 100644 --- a/unified-runtime/source/adapters/level_zero/platform.hpp +++ b/unified-runtime/source/adapters/level_zero/platform.hpp @@ -15,6 +15,10 @@ #include "ze_ddi.h" #include "zes_api.h" +#include + +#include + struct ur_device_handle_t_; typedef size_t DeviceId; @@ -220,4 +224,22 @@ struct ur_platform_handle_t_ : ur::handle_base, // Some platforms may not support this API due to frozen driver, eg. gen12 on // Windows. For details, see https://github.com/intel/llvm/issues/20927. bool ZeDeviceSynchronizeSupported{false}; + + // Counter-based event IPC entry points, populated by + // loadIpcEventExtension(). + struct ZeIpcEventExtension { + ze_result_t (*pfnGetIpcHandle)( + ze_event_handle_t hEvent, + zex_ipc_counter_based_event_handle_t *phIpc) = nullptr; + ze_result_t (*pfnOpenIpcHandle)(ze_context_handle_t hContext, + zex_ipc_counter_based_event_handle_t hIpc, + ze_event_handle_t *phEvent) = nullptr; + ze_result_t (*pfnCloseIpcHandle)(ze_event_handle_t hEvent) = nullptr; + bool Supported = false; + std::once_flag InitFlag; + } ZeIpcEventExt; + + // Lazily resolves the counter-based event IPC entry points; returns true + // if the driver exposes all three. Thread-safe. + bool loadIpcEventExtension(); }; diff --git a/unified-runtime/source/adapters/level_zero/v2/event.cpp b/unified-runtime/source/adapters/level_zero/v2/event.cpp index ad94374ff14f8..049eaa97b6011 100644 --- a/unified-runtime/source/adapters/level_zero/v2/event.cpp +++ b/unified-runtime/source/adapters/level_zero/v2/event.cpp @@ -14,9 +14,11 @@ #include "event.hpp" #include "event_pool.hpp" #include "event_provider.hpp" +#include "event_provider_counter.hpp" #include "queue_api.hpp" #include "queue_handle.hpp" +#include "../device.hpp" #include "../ur_interface_loader.hpp" static uint64_t adjustEndEventTimestamp(uint64_t adjustedStartTimestamp, @@ -164,11 +166,7 @@ void ur_event_handle_t_::reset() { } ze_event_handle_t ur_event_handle_t_::getZeEvent() const { - if (event_pool) { - return std::get(hZeEvent).get(); - } else { - return std::get(hZeEvent).get(); - } + return std::visit([](const auto &h) { return h.get(); }, hZeEvent); } ur_result_t ur_event_handle_t_::retain() { @@ -182,10 +180,17 @@ ur_result_t ur_event_handle_t_::release() { if (event_pool) { event_pool->free(this); - } else { - std::get(hZeEvent).release(); - delete this; + return UR_RESULT_SUCCESS; + } + + // External native events (urEventCreateWithNativeHandle): detach so the + // variant destructor doesn't destroy a handle this adapter doesn't own. + // For adapter_owned/ipc_imported events the variant destructor runs the + // correct teardown. + if (auto *native = std::get_if(&hZeEvent)) { + native->release(); } + delete this; return UR_RESULT_SUCCESS; } @@ -197,6 +202,14 @@ bool ur_event_handle_t_::isProfilingEnabled() const { return flags & v2::EVENT_FLAGS_PROFILING_ENABLED; } +bool ur_event_handle_t_::isIpcCapable() const { + return flags & v2::EVENT_FLAGS_IPC; +} + +bool ur_event_handle_t_::isIpcImported() const { + return flags & v2::EVENT_FLAGS_IPC_IMPORTED; +} + std::pair ur_event_handle_t_::getEventEndTimestampAndHandle() { return {profilingData.eventEndTimestampAddr(), getZeEvent()}; @@ -234,6 +247,11 @@ ur_event_handle_t_::ur_event_handle_t_( , nullptr) {} +ur_event_handle_t_::ur_event_handle_t_(ur_context_handle_t hContext, + event_variant hZeEvent, + v2::event_flags_t flags) + : ur_event_handle_t_(hContext, std::move(hZeEvent), flags, nullptr) {} + namespace ur::level_zero { ur_result_t urEventRetain(ur_event_handle_t hEvent) try { return hEvent->retain(); @@ -416,12 +434,49 @@ urEventCreateWithNativeHandle(ur_native_handle_t hNativeEvent, return exceptionToResult(std::current_exception()); } -ur_result_t urEventCreateExp(ur_context_handle_t /*hContext*/, - ur_device_handle_t /*hDevice*/, - const ur_exp_event_desc_t * /*pEventDesc*/, - ur_event_handle_t * /*phEvent*/) { - UR_LOG(ERR, "{} function not implemented!", __FUNCTION__); - return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; +ur_result_t urEventCreateExp(ur_context_handle_t hContext, + ur_device_handle_t hDevice, + const ur_exp_event_desc_t *pEventDesc, + ur_event_handle_t *phEvent) try { + // Local check (instead of context.cpp::isValidDevice) so unit-test link + // targets that compile a subset of adapter sources don't pull in context.cpp. + bool validDevice = false; + for (ur_device_handle_t Device = hDevice; Device != nullptr && !validDevice; + Device = Device->RootDevice) { + for (const auto &ContextDevice : hContext->getDevices()) { + if (ContextDevice == Device) { + validDevice = true; + break; + } + } + } + if (!validDevice) { + return UR_RESULT_ERROR_INVALID_DEVICE; + } + + const ur_exp_event_flags_t flags = pEventDesc->flags; + + // Only the IPC creation path is implemented; the non-IPC reusable-event + // path is a separate work item. + if (!(flags & UR_EXP_EVENT_FLAG_IPC_EXP)) { + UR_LOG(ERR, "{}: non-IPC reusable events not implemented!", __FUNCTION__); + return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; + } + + // IPC and profiling are mutually exclusive. + if (flags & UR_EXP_EVENT_FLAG_ENABLE_PROFILING) { + return UR_RESULT_ERROR_INVALID_VALUE; + } + + ze_event_handle_t hZeEvent = nullptr; + UR_CALL(v2::createIpcCounterBasedEvent(hContext, hDevice, &hZeEvent)); + + *phEvent = new ur_event_handle_t_( + hContext, v2::raii::adapter_owned_event_handle_t{hZeEvent}, + v2::EVENT_FLAGS_COUNTER | v2::EVENT_FLAGS_IPC); + return UR_RESULT_SUCCESS; +} catch (...) { + return exceptionToResult(std::current_exception()); } } // namespace ur::level_zero diff --git a/unified-runtime/source/adapters/level_zero/v2/event.hpp b/unified-runtime/source/adapters/level_zero/v2/event.hpp index d4d4f8b65e986..19a660e2ea4dc 100644 --- a/unified-runtime/source/adapters/level_zero/v2/event.hpp +++ b/unified-runtime/source/adapters/level_zero/v2/event.hpp @@ -18,6 +18,7 @@ #include "common.hpp" #include "common/ur_ref_count.hpp" #include "event_provider.hpp" +#include "ipc_event_handle.hpp" using ur_event_generation_t = int64_t; @@ -56,10 +57,15 @@ struct event_profiling_data_t { struct ur_event_handle_t_ : ur_object { public: - // cache_borrowed_event is used for pooled events, whilst ze_event_handle_t is - // used for native events + // The variant alternative encodes how the L0 event handle is torn down: + // - cache_borrowed_event: returned to the pool. + // - ze_event_handle_t: external (urEventCreateWithNativeHandle), detached. + // - adapter_owned_event_handle_t: zeEventDestroy. + // - ipc_imported_event_handle_t: zexCounterBasedEventCloseIpcHandle. using event_variant = - std::variant; + std::variant; ur_event_handle_t_(ur_context_handle_t hContext, v2::raii::cache_borrowed_event eventAllocation, @@ -69,6 +75,11 @@ struct ur_event_handle_t_ : ur_object { ur_native_handle_t hNativeEvent, const ur_event_native_properties_t *pProperties); + // Wraps a caller-built event_variant with explicit flags. Used by the IPC + // producer (urEventCreateExp) and consumer (urIPCOpenEventHandleExp) paths. + ur_event_handle_t_(ur_context_handle_t hContext, event_variant hZeEvent, + v2::event_flags_t flags); + // Set the queue and command that this event is associated with void setQueue(ur_queue_t_ *hQueue); void setCommandType(ur_command_t commandType); @@ -99,6 +110,12 @@ struct ur_event_handle_t_ : ur_object { // Tells if this event comes from a pool that has profiling enabled. bool isProfilingEnabled() const; + // True for IPC-shareable events (both producer and consumer side). + bool isIpcCapable() const; + + // True for events opened via urIPCOpenEventHandleExp. + bool isIpcImported() const; + // Queue associated with this event. Can be nullptr (for native events) ur_queue_t_ *getQueue() const; diff --git a/unified-runtime/source/adapters/level_zero/v2/event_provider.hpp b/unified-runtime/source/adapters/level_zero/v2/event_provider.hpp index 2be5e9058b084..d9e44ea2254a8 100644 --- a/unified-runtime/source/adapters/level_zero/v2/event_provider.hpp +++ b/unified-runtime/source/adapters/level_zero/v2/event_provider.hpp @@ -24,7 +24,13 @@ using event_flags_t = uint32_t; enum event_flag_t { EVENT_FLAGS_COUNTER = UR_BIT(0), EVENT_FLAGS_PROFILING_ENABLED = UR_BIT(1), + // IPC-shareable producer event. + EVENT_FLAGS_IPC = UR_BIT(2), + // Event opened from an IPC handle. + EVENT_FLAGS_IPC_IMPORTED = UR_BIT(3), }; +// Bits used to index pooled events in event_pool_cache. IPC events are never +// pool-allocated, so EVENT_FLAGS_IPC* are excluded from this count. static constexpr size_t EVENT_FLAGS_USED_BITS = 2; enum queue_type { diff --git a/unified-runtime/source/adapters/level_zero/v2/event_provider_counter.cpp b/unified-runtime/source/adapters/level_zero/v2/event_provider_counter.cpp index 5fe7162f3ae76..1c040220fc064 100644 --- a/unified-runtime/source/adapters/level_zero/v2/event_provider_counter.cpp +++ b/unified-runtime/source/adapters/level_zero/v2/event_provider_counter.cpp @@ -113,4 +113,35 @@ std::unique_ptr createProvider(ur_platform_handle_t platform, return std::make_unique(context, queueType, flags); } +ur_result_t createIpcCounterBasedEvent(ur_context_handle_t context, + ur_device_handle_t device, + ze_event_handle_t *phEvent) { + ur_platform_handle_t platform = device->Platform; + + // Both the create and IPC entry points must be available; otherwise the + // event would not be exportable. + zexCounterBasedEventCreate pfnCreate = nullptr; + if (ZE_CALL_NOCHECK(zeDriverGetExtensionFunctionAddress, + (platform->ZeDriver, "zexCounterBasedEventCreate2", + reinterpret_cast(&pfnCreate))) != + ZE_RESULT_SUCCESS || + !pfnCreate || !platform->loadIpcEventExtension()) { + return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; + } + + zex_counter_based_event_desc_t desc = {}; + desc.stype = ZEX_STRUCTURE_COUNTER_BASED_EVENT_DESC; + desc.flags = ZEX_COUNTER_BASED_EVENT_FLAG_HOST_VISIBLE | + ZEX_COUNTER_BASED_EVENT_FLAG_IMMEDIATE | + ZEX_COUNTER_BASED_EVENT_FLAG_NON_IMMEDIATE | + ZEX_COUNTER_BASED_EVENT_FLAG_IPC; + desc.signalScope = ZE_EVENT_SCOPE_FLAG_HOST; + + // The IPC entry points consume loader-level handles, so the producer event + // is created from loader-level handles too (no zelLoaderTranslateHandle). + ZE2UR_CALL(pfnCreate, + (context->getZeHandle(), device->ZeDevice, &desc, phEvent)); + return UR_RESULT_SUCCESS; +} + } // namespace v2 diff --git a/unified-runtime/source/adapters/level_zero/v2/event_provider_counter.hpp b/unified-runtime/source/adapters/level_zero/v2/event_provider_counter.hpp index 760fe55ce8336..c9a528b52d051 100644 --- a/unified-runtime/source/adapters/level_zero/v2/event_provider_counter.hpp +++ b/unified-runtime/source/adapters/level_zero/v2/event_provider_counter.hpp @@ -61,4 +61,11 @@ std::unique_ptr createProvider(ur_platform_handle_t platform, ur_device_handle_t device, event_flags_t flags); +// Creates an IPC-shareable counter-based ze_event_handle_t for the producer +// side of urEventCreateExp. Returns UR_RESULT_ERROR_UNSUPPORTED_FEATURE if the +// driver does not expose counter-based event creation or the IPC entry points. +ur_result_t createIpcCounterBasedEvent(ur_context_handle_t context, + ur_device_handle_t device, + ze_event_handle_t *phEvent); + } // namespace v2 diff --git a/unified-runtime/source/adapters/level_zero/v2/ipc_event.cpp b/unified-runtime/source/adapters/level_zero/v2/ipc_event.cpp index a5a0cc0ec0f22..a5bac069a44c9 100644 --- a/unified-runtime/source/adapters/level_zero/v2/ipc_event.cpp +++ b/unified-runtime/source/adapters/level_zero/v2/ipc_event.cpp @@ -7,28 +7,100 @@ // //===----------------------------------------------------------------------===// +#include + #include +#include "../platform.hpp" #include "../ur_interface_loader.hpp" +#include "context.hpp" +#include "event.hpp" +#include "ipc_event_handle.hpp" namespace ur::level_zero { -ur_result_t urIPCGetEventHandleExp(ur_event_handle_t /*hEvent*/, - void ** /*ppIPCEventHandleData*/, - size_t * /*pIPCEventHandleDataSizeRet*/) { - return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; +namespace { + +// Fixed L0 IPC handle size; reported by Get and validated by Open. +constexpr size_t kIpcEventHandleDataSize = + sizeof(zex_ipc_counter_based_event_handle_t); + +} // namespace + +ur_result_t urIPCGetEventHandleExp(ur_event_handle_t hEvent, + void **ppIPCEventHandleData, + size_t *pIPCEventHandleDataSizeRet) try { + if (!ppIPCEventHandleData || !pIPCEventHandleDataSizeRet) { + return UR_RESULT_ERROR_INVALID_NULL_POINTER; + } + // Only producer-side IPC events can be exported. + if (!hEvent->isIpcCapable() || hEvent->isIpcImported()) { + return UR_RESULT_ERROR_INVALID_EVENT; + } + if (hEvent->isProfilingEnabled() || hEvent->isTimestamped()) { + return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; + } + ur_platform_handle_t platform = hEvent->getContext()->getPlatform(); + if (!platform->loadIpcEventExtension()) { + return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; + } + + auto handle = std::make_unique(); + ZE2UR_CALL(platform->ZeIpcEventExt.pfnGetIpcHandle, + (hEvent->getZeEvent(), handle.get())); + + // Caller releases the buffer via urIPCPutEventHandleExp. + *ppIPCEventHandleData = handle.release(); + *pIPCEventHandleDataSizeRet = kIpcEventHandleDataSize; + return UR_RESULT_SUCCESS; +} catch (...) { + return exceptionToResult(std::current_exception()); } ur_result_t urIPCPutEventHandleExp(ur_context_handle_t /*hContext*/, - void * /*pIPCEventHandleData*/) { - return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; + void *pIPCEventHandleData) try { + if (!pIPCEventHandleData) { + return UR_RESULT_ERROR_INVALID_NULL_POINTER; + } + // Free the buffer allocated by urIPCGetEventHandleExp via RAII. + std::unique_ptr owner( + static_cast(pIPCEventHandleData)); + return UR_RESULT_SUCCESS; +} catch (...) { + return exceptionToResult(std::current_exception()); } -ur_result_t urIPCOpenEventHandleExp(ur_context_handle_t /*hContext*/, - const void * /*pIPCEventHandleData*/, - size_t /*ipcEventHandleDataSize*/, - ur_event_handle_t * /*phEvent*/) { - return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; +ur_result_t urIPCOpenEventHandleExp(ur_context_handle_t hContext, + const void *pIPCEventHandleData, + size_t ipcEventHandleDataSize, + ur_event_handle_t *phEvent) try { + if (!pIPCEventHandleData || !phEvent) { + return UR_RESULT_ERROR_INVALID_NULL_POINTER; + } + if (ipcEventHandleDataSize != kIpcEventHandleDataSize) { + return UR_RESULT_ERROR_INVALID_VALUE; + } + ur_platform_handle_t platform = hContext->getPlatform(); + if (!platform->loadIpcEventExtension()) { + return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; + } + + // The driver consumes the handle by value. + zex_ipc_counter_based_event_handle_t handle = + *static_cast( + pIPCEventHandleData); + ze_event_handle_t hZeEvent = nullptr; + ZE2UR_CALL(platform->ZeIpcEventExt.pfnOpenIpcHandle, + (hContext->getZeHandle(), handle, &hZeEvent)); + + // The ipc_imported variant tears the handle down on urEventRelease. + *phEvent = new ur_event_handle_t_( + hContext, v2::raii::ipc_imported_event_handle_t{platform, hZeEvent}, + v2::EVENT_FLAGS_COUNTER | v2::EVENT_FLAGS_IPC | + v2::EVENT_FLAGS_IPC_IMPORTED); + return UR_RESULT_SUCCESS; +} catch (...) { + return exceptionToResult(std::current_exception()); } } // namespace ur::level_zero diff --git a/unified-runtime/source/adapters/level_zero/v2/ipc_event_handle.hpp b/unified-runtime/source/adapters/level_zero/v2/ipc_event_handle.hpp new file mode 100644 index 0000000000000..4a4f8899c5cd0 --- /dev/null +++ b/unified-runtime/source/adapters/level_zero/v2/ipc_event_handle.hpp @@ -0,0 +1,137 @@ +//===--------- ipc_event_handle.hpp - Level Zero Adapter ------------------===// +// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM +// Exceptions. See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +#pragma once + +#include + +#include "../platform.hpp" +#include "common.hpp" + +namespace v2 { +namespace raii { + +// Owns an adapter-allocated ze_event_handle_t (e.g. an IPC producer event). +// Destructor calls zeEventDestroy. A distinct type from ze_event_handle_t so +// the variant dispatch in ur_event_handle_t_ stays unambiguous: that +// alternative is reserved for externally-owned native events. +class adapter_owned_event_handle_t { +public: + adapter_owned_event_handle_t() noexcept = default; + + explicit adapter_owned_event_handle_t(::ze_event_handle_t handle) noexcept + : handle(handle) {} + + adapter_owned_event_handle_t(const adapter_owned_event_handle_t &) = delete; + adapter_owned_event_handle_t & + operator=(const adapter_owned_event_handle_t &) = delete; + + adapter_owned_event_handle_t(adapter_owned_event_handle_t &&other) noexcept + : handle(other.handle) { + other.handle = nullptr; + } + + adapter_owned_event_handle_t & + operator=(adapter_owned_event_handle_t &&other) noexcept { + if (this == &other) { + return *this; + } + reset(); + handle = other.handle; + other.handle = nullptr; + return *this; + } + + ~adapter_owned_event_handle_t() { + try { + reset(); + } catch (...) { + // Swallow teardown failures; destructor must not throw. + } + } + + void reset() { + if (!handle) { + return; + } + if (checkL0LoaderTeardown()) { + ZE_CALL_NOCHECK(zeEventDestroy, (handle)); + } + handle = nullptr; + } + + ::ze_event_handle_t get() const noexcept { return handle; } + +private: + ::ze_event_handle_t handle = nullptr; +}; + +// Owns a ze_event_handle_t opened via zexCounterBasedEventOpenIpcHandle. +// Destructor calls zexCounterBasedEventCloseIpcHandle (not zeEventDestroy). +// Captures the platform so the close pointer is reachable from the destructor. +class ipc_imported_event_handle_t { +public: + ipc_imported_event_handle_t() noexcept = default; + + ipc_imported_event_handle_t(ur_platform_handle_t platform, + ::ze_event_handle_t handle) noexcept + : platform(platform), handle(handle) {} + + ipc_imported_event_handle_t(const ipc_imported_event_handle_t &) = delete; + ipc_imported_event_handle_t & + operator=(const ipc_imported_event_handle_t &) = delete; + + ipc_imported_event_handle_t(ipc_imported_event_handle_t &&other) noexcept + : platform(other.platform), handle(other.handle) { + other.platform = nullptr; + other.handle = nullptr; + } + + ipc_imported_event_handle_t & + operator=(ipc_imported_event_handle_t &&other) noexcept { + if (this == &other) { + return *this; + } + reset(); + platform = other.platform; + handle = other.handle; + other.platform = nullptr; + other.handle = nullptr; + return *this; + } + + ~ipc_imported_event_handle_t() { + try { + reset(); + } catch (...) { + // Swallow teardown failures; destructor must not throw. + } + } + + void reset() { + if (!handle) { + platform = nullptr; + return; + } + if (platform && platform->ZeIpcEventExt.pfnCloseIpcHandle && + checkL0LoaderTeardown()) { + ZE_CALL_NOCHECK(platform->ZeIpcEventExt.pfnCloseIpcHandle, (handle)); + } + handle = nullptr; + platform = nullptr; + } + + ::ze_event_handle_t get() const noexcept { return handle; } + +private: + ur_platform_handle_t platform = nullptr; + ::ze_event_handle_t handle = nullptr; +}; + +} // namespace raii +} // namespace v2 diff --git a/unified-runtime/test/conformance/event/CMakeLists.txt b/unified-runtime/test/conformance/event/CMakeLists.txt index ada955fe4d9b4..413012d5afacd 100644 --- a/unified-runtime/test/conformance/event/CMakeLists.txt +++ b/unified-runtime/test/conformance/event/CMakeLists.txt @@ -11,3 +11,13 @@ add_conformance_devices_test(event urEventGetNativeHandle.cpp urEventCreateWithNativeHandle.cpp urEventSetCallback.cpp) + +# The IPC event tests signal the producer event via the native Level Zero API, +# so they are only built and linked when a Level Zero adapter is present. +if(UR_BUILD_ADAPTER_L0 OR UR_BUILD_ADAPTER_L0_V2) + target_sources(event-test PRIVATE + urIPCEventExp.cpp + urIPCEventExpNegative.cpp) + target_link_libraries(event-test PRIVATE + LevelZeroLoader LevelZeroLoader-Headers) +endif() diff --git a/unified-runtime/test/conformance/event/ipc_event_fixtures.h b/unified-runtime/test/conformance/event/ipc_event_fixtures.h new file mode 100644 index 0000000000000..833313b5b5ecb --- /dev/null +++ b/unified-runtime/test/conformance/event/ipc_event_fixtures.h @@ -0,0 +1,154 @@ +// Part of the LLVM Project, under the Apache License v2.0 with LLVM +// Exceptions. See https://llvm.org/LICENSE.txt for license information. +// +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +#ifndef UR_CONFORMANCE_EVENT_IPC_EVENT_FIXTURES_H_INCLUDED +#define UR_CONFORMANCE_EVENT_IPC_EVENT_FIXTURES_H_INCLUDED + +#include + +#include + +namespace uur { +namespace event { + +// Signals an IPC-shareable producer event using the native Level Zero API. +// +// A counter-based event has no underlying counter allocation until it is used +// as the signal target of an append call, so urIPCGetEventHandleExp cannot +// serialize it beforehand. The SYCL producer-side signal API +// (enqueue_signal_event) is part of a separate extension that has not landed +// yet, so here -- as in the SYCL E2E tests -- we drop into Level Zero: append +// a barrier that signals the event on a fresh immediate command list and host +// synchronize. After this the event carries a real allocation/value the driver +// can share. +inline void signalEventViaLevelZero(ur_event_handle_t event, + ur_context_handle_t context, + ur_device_handle_t device) { + ze_event_handle_t zeEvent = nullptr; + ze_context_handle_t zeContext = nullptr; + ze_device_handle_t zeDevice = nullptr; + ASSERT_SUCCESS(urEventGetNativeHandle( + event, reinterpret_cast(&zeEvent))); + ASSERT_SUCCESS(urContextGetNativeHandle( + context, reinterpret_cast(&zeContext))); + ASSERT_SUCCESS(urDeviceGetNativeHandle( + device, reinterpret_cast(&zeDevice))); + ASSERT_NE(zeEvent, nullptr); + + uint32_t numGroups = 0; + ASSERT_EQ( + zeDeviceGetCommandQueueGroupProperties(zeDevice, &numGroups, nullptr), + ZE_RESULT_SUCCESS); + ze_command_queue_group_properties_t propsInit{}; + propsInit.stype = ZE_STRUCTURE_TYPE_COMMAND_QUEUE_GROUP_PROPERTIES; + std::vector props(numGroups, propsInit); + ASSERT_EQ(zeDeviceGetCommandQueueGroupProperties(zeDevice, &numGroups, + props.data()), + ZE_RESULT_SUCCESS); + + uint32_t computeOrdinal = 0; + for (uint32_t i = 0; i < numGroups; ++i) { + if (props[i].flags & ZE_COMMAND_QUEUE_GROUP_PROPERTY_FLAG_COMPUTE) { + computeOrdinal = i; + break; + } + } + + ze_command_queue_desc_t qDesc{}; + qDesc.stype = ZE_STRUCTURE_TYPE_COMMAND_QUEUE_DESC; + qDesc.ordinal = computeOrdinal; + qDesc.mode = ZE_COMMAND_QUEUE_MODE_ASYNCHRONOUS; + qDesc.flags = ZE_COMMAND_QUEUE_FLAG_IN_ORDER; + + ze_command_list_handle_t cmdList = nullptr; + ASSERT_EQ(zeCommandListCreateImmediate(zeContext, zeDevice, &qDesc, &cmdList), + ZE_RESULT_SUCCESS); + ASSERT_EQ(zeCommandListAppendBarrier(cmdList, zeEvent, 0, nullptr), + ZE_RESULT_SUCCESS); + ASSERT_EQ(zeEventHostSynchronize(zeEvent, UINT64_MAX), ZE_RESULT_SUCCESS); + zeCommandListDestroy(cmdList); +} + +/// Fixture for the inter-process event sharing experimental APIs +/// (urIPC{Get,Put,Open}EventHandleExp). +/// +/// SetUp: +/// - skips on backends other than Level Zero (the producer event is signaled +/// via the native Level Zero API), +/// - skips on devices that don't advertise +/// UR_DEVICE_INFO_IPC_EVENT_SUPPORT_EXP, +/// - creates an IPC-shareable event via urEventCreateExp with +/// UR_EXP_EVENT_FLAG_IPC_EXP set, and signals it so it has a concrete +/// counter allocation backing it before any IPC Get call. +/// +/// Derives from urQueueTest (no profiling) since IPC and per-event profiling +/// are mutually exclusive. +struct urIPCEventTest : uur::urQueueTest { + void SetUp() override { + UUR_RETURN_ON_FATAL_FAILURE(uur::urQueueTest::SetUp()); + + ur_backend_t backend = UR_BACKEND_UNKNOWN; + ASSERT_SUCCESS(urPlatformGetInfo(platform, UR_PLATFORM_INFO_BACKEND, + sizeof(backend), &backend, nullptr)); + if (backend != UR_BACKEND_LEVEL_ZERO) { + GTEST_SKIP() << "IPC event tests are only supported on Level Zero."; + } + + ur_bool_t ipcEventSupport = false; + ASSERT_SUCCESS(urDeviceGetInfo(device, UR_DEVICE_INFO_IPC_EVENT_SUPPORT_EXP, + sizeof(ipcEventSupport), &ipcEventSupport, + nullptr)); + if (!ipcEventSupport) { + GTEST_SKIP() << "IPC event feature is not supported on this device."; + } + + // Initializing the Level Zero driver is required when the test is linked + // statically with the Level Zero loader, otherwise the driver will not be + // initialized. + zeInit(ZE_INIT_FLAG_GPU_ONLY); + + ur_exp_event_desc_t desc{UR_STRUCTURE_TYPE_EXP_EVENT_DESC, nullptr, + UR_EXP_EVENT_FLAG_IPC_EXP}; + ASSERT_SUCCESS(urEventCreateExp(context, device, &desc, &event)); + ASSERT_NE(event, nullptr); + + UUR_RETURN_ON_FATAL_FAILURE( + signalEventViaLevelZero(event, context, device)); + } + + void TearDown() override { + if (event) { + EXPECT_SUCCESS(urEventRelease(event)); + event = nullptr; + } + uur::urQueueTest::TearDown(); + } + + ur_event_handle_t event = nullptr; +}; + +// Opens an IPC event handle, skipping the test if the installed Level Zero +// driver does not reclaim imported counter-based IPC event allocations: such +// drivers return UR_RESULT_ERROR_OUT_OF_DEVICE_MEMORY from a subsequent open +// once an earlier import in the same process has been closed. Cycling +// open/close repeatedly requires a driver with 2-way IPC event support. The +// handle buffer is released before skipping so it is not leaked. +#define UUR_IPC_OPEN_OR_SKIP(ctx, data, size, imported) \ + do { \ + ur_result_t _open_rc = \ + urIPCOpenEventHandleExp((ctx), (data), (size), &(imported)); \ + if (_open_rc == UR_RESULT_ERROR_OUT_OF_DEVICE_MEMORY) { \ + EXPECT_SUCCESS(urIPCPutEventHandleExp((ctx), (data))); \ + GTEST_SKIP() << "Level Zero driver did not reclaim a previously closed " \ + "imported IPC event; open/close cycling requires 2-way " \ + "IPC event driver support."; \ + } \ + ASSERT_SUCCESS(_open_rc); \ + } while (0) + +} // namespace event +} // namespace uur + +#endif // UR_CONFORMANCE_EVENT_IPC_EVENT_FIXTURES_H_INCLUDED diff --git a/unified-runtime/test/conformance/event/urIPCEventExp.cpp b/unified-runtime/test/conformance/event/urIPCEventExp.cpp new file mode 100644 index 0000000000000..86fd399213393 --- /dev/null +++ b/unified-runtime/test/conformance/event/urIPCEventExp.cpp @@ -0,0 +1,123 @@ +// Part of the LLVM Project, under the Apache License v2.0 with LLVM +// Exceptions. See https://llvm.org/LICENSE.txt for license information. +// +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +#include "ipc_event_fixtures.h" + +using urIPCEventExpTest = uur::event::urIPCEventTest; +UUR_INSTANTIATE_DEVICE_TEST_SUITE(urIPCEventExpTest); + +// Get returns a non-null buffer and a non-zero size. +TEST_P(urIPCEventExpTest, GetReturnsBuffer) { + void *data = nullptr; + size_t size = 0; + ASSERT_SUCCESS(urIPCGetEventHandleExp(event, &data, &size)); + ASSERT_NE(data, nullptr); + ASSERT_GT(size, 0u); + EXPECT_SUCCESS(urIPCPutEventHandleExp(context, data)); +} + +// Producer Get -> consumer Open in the same process succeeds and yields a +// non-null event. +TEST_P(urIPCEventExpTest, OpenSucceedsSameProcess) { + void *data = nullptr; + size_t size = 0; + ASSERT_SUCCESS(urIPCGetEventHandleExp(event, &data, &size)); + + ur_event_handle_t imported = nullptr; + UUR_IPC_OPEN_OR_SKIP(context, data, size, imported); + ASSERT_NE(imported, nullptr); + + EXPECT_SUCCESS(urEventRelease(imported)); + EXPECT_SUCCESS(urIPCPutEventHandleExp(context, data)); +} + +// The producer event was signaled in SetUp, so a wait on the imported event +// must complete. +TEST_P(urIPCEventExpTest, WaitOnImportedSucceeds) { + void *data = nullptr; + size_t size = 0; + ASSERT_SUCCESS(urIPCGetEventHandleExp(event, &data, &size)); + + ur_event_handle_t imported = nullptr; + UUR_IPC_OPEN_OR_SKIP(context, data, size, imported); + + EXPECT_SUCCESS(urEventWait(1, &imported)); + + EXPECT_SUCCESS(urEventRelease(imported)); + EXPECT_SUCCESS(urIPCPutEventHandleExp(context, data)); +} + +// The imported event is a normal ur_event_handle_t and participates in the +// usual reference counting. +TEST_P(urIPCEventExpTest, ImportedEventRetainRelease) { + void *data = nullptr; + size_t size = 0; + ASSERT_SUCCESS(urIPCGetEventHandleExp(event, &data, &size)); + + ur_event_handle_t imported = nullptr; + UUR_IPC_OPEN_OR_SKIP(context, data, size, imported); + + ASSERT_SUCCESS(urEventRetain(imported)); + EXPECT_SUCCESS(urEventRelease(imported)); + EXPECT_SUCCESS(urEventRelease(imported)); + + EXPECT_SUCCESS(urIPCPutEventHandleExp(context, data)); +} + +// Multiple Get/Open/Release/Put round trips on the same producer event each +// yield a fresh non-null imported event. +TEST_P(urIPCEventExpTest, MultipleRoundTrips) { + for (int i = 0; i < 3; ++i) { + void *data = nullptr; + size_t size = 0; + ASSERT_SUCCESS(urIPCGetEventHandleExp(event, &data, &size)); + + ur_event_handle_t imported = nullptr; + UUR_IPC_OPEN_OR_SKIP(context, data, size, imported); + ASSERT_NE(imported, nullptr); + + EXPECT_SUCCESS(urEventRelease(imported)); + EXPECT_SUCCESS(urIPCPutEventHandleExp(context, data)); + } +} + +// The same handle can be opened more than once; each open yields an event that +// shares state with the producer (so each completes a wait). +TEST_P(urIPCEventExpTest, MultipleOpensOfSameHandle) { + void *data = nullptr; + size_t size = 0; + ASSERT_SUCCESS(urIPCGetEventHandleExp(event, &data, &size)); + + ur_event_handle_t first = nullptr; + ur_event_handle_t second = nullptr; + UUR_IPC_OPEN_OR_SKIP(context, data, size, first); + UUR_IPC_OPEN_OR_SKIP(context, data, size, second); + ASSERT_NE(first, nullptr); + ASSERT_NE(second, nullptr); + ASSERT_NE(first, second); + + EXPECT_SUCCESS(urEventWait(1, &first)); + EXPECT_SUCCESS(urEventWait(1, &second)); + + EXPECT_SUCCESS(urEventRelease(first)); + EXPECT_SUCCESS(urEventRelease(second)); + EXPECT_SUCCESS(urIPCPutEventHandleExp(context, data)); +} + +// Releasing the imported event before Put on the handle is fine, and Put does +// not affect the still-alive producer event. +TEST_P(urIPCEventExpTest, PutLeavesProducerAlive) { + void *data = nullptr; + size_t size = 0; + ASSERT_SUCCESS(urIPCGetEventHandleExp(event, &data, &size)); + + ur_event_handle_t imported = nullptr; + UUR_IPC_OPEN_OR_SKIP(context, data, size, imported); + EXPECT_SUCCESS(urEventRelease(imported)); + EXPECT_SUCCESS(urIPCPutEventHandleExp(context, data)); + + // The producer event (released by the fixture TearDown) is still usable. + EXPECT_SUCCESS(urEventWait(1, &event)); +} diff --git a/unified-runtime/test/conformance/event/urIPCEventExpNegative.cpp b/unified-runtime/test/conformance/event/urIPCEventExpNegative.cpp new file mode 100644 index 0000000000000..2a2a1f8fec0f8 --- /dev/null +++ b/unified-runtime/test/conformance/event/urIPCEventExpNegative.cpp @@ -0,0 +1,88 @@ +// Part of the LLVM Project, under the Apache License v2.0 with LLVM +// Exceptions. See https://llvm.org/LICENSE.txt for license information. +// +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +#include "ipc_event_fixtures.h" + +using urIPCEventExpNegativeTest = uur::event::urIPCEventTest; +UUR_INSTANTIATE_DEVICE_TEST_SUITE(urIPCEventExpNegativeTest); + +TEST_P(urIPCEventExpNegativeTest, GetWithNullOutData) { + size_t size = 0; + EXPECT_EQ_RESULT(urIPCGetEventHandleExp(event, nullptr, &size), + UR_RESULT_ERROR_INVALID_NULL_POINTER); +} + +TEST_P(urIPCEventExpNegativeTest, GetWithNullOutSize) { + void *data = nullptr; + EXPECT_EQ_RESULT(urIPCGetEventHandleExp(event, &data, nullptr), + UR_RESULT_ERROR_INVALID_NULL_POINTER); +} + +TEST_P(urIPCEventExpNegativeTest, GetWithNullEvent) { + void *data = nullptr; + size_t size = 0; + EXPECT_EQ_RESULT(urIPCGetEventHandleExp(nullptr, &data, &size), + UR_RESULT_ERROR_INVALID_NULL_HANDLE); +} + +TEST_P(urIPCEventExpNegativeTest, OpenWithMismatchedSize) { + void *data = nullptr; + size_t size = 0; + ASSERT_SUCCESS(urIPCGetEventHandleExp(event, &data, &size)); + + ur_event_handle_t imported = nullptr; + EXPECT_EQ_RESULT(urIPCOpenEventHandleExp(context, data, size + 1, &imported), + UR_RESULT_ERROR_INVALID_VALUE); + EXPECT_EQ_RESULT(urIPCOpenEventHandleExp(context, data, size - 1, &imported), + UR_RESULT_ERROR_INVALID_VALUE); + + EXPECT_SUCCESS(urIPCPutEventHandleExp(context, data)); +} + +TEST_P(urIPCEventExpNegativeTest, OpenWithNullData) { + ur_event_handle_t imported = nullptr; + EXPECT_EQ_RESULT(urIPCOpenEventHandleExp(context, nullptr, 64, &imported), + UR_RESULT_ERROR_INVALID_NULL_POINTER); +} + +TEST_P(urIPCEventExpNegativeTest, OpenWithNullPhEvent) { + void *data = nullptr; + size_t size = 0; + ASSERT_SUCCESS(urIPCGetEventHandleExp(event, &data, &size)); + + EXPECT_EQ_RESULT(urIPCOpenEventHandleExp(context, data, size, nullptr), + UR_RESULT_ERROR_INVALID_NULL_POINTER); + EXPECT_SUCCESS(urIPCPutEventHandleExp(context, data)); +} + +TEST_P(urIPCEventExpNegativeTest, OpenWithNullContext) { + void *data = nullptr; + size_t size = 0; + ASSERT_SUCCESS(urIPCGetEventHandleExp(event, &data, &size)); + + ur_event_handle_t imported = nullptr; + EXPECT_EQ_RESULT(urIPCOpenEventHandleExp(nullptr, data, size, &imported), + UR_RESULT_ERROR_INVALID_NULL_HANDLE); + EXPECT_SUCCESS(urIPCPutEventHandleExp(context, data)); +} + +TEST_P(urIPCEventExpNegativeTest, PutWithNullData) { + EXPECT_EQ_RESULT(urIPCPutEventHandleExp(context, nullptr), + UR_RESULT_ERROR_INVALID_NULL_POINTER); +} + +// Profiling + IPC at create time is mutually exclusive. +TEST_P(urIPCEventExpNegativeTest, CreateRejectsProfilingPlusIPC) { + ur_exp_event_desc_t desc{UR_STRUCTURE_TYPE_EXP_EVENT_DESC, nullptr, + UR_EXP_EVENT_FLAG_IPC_EXP | + UR_EXP_EVENT_FLAG_ENABLE_PROFILING}; + + ur_event_handle_t evt = nullptr; + EXPECT_EQ_RESULT(urEventCreateExp(context, device, &desc, &evt), + UR_RESULT_ERROR_INVALID_VALUE); + if (evt) { + EXPECT_SUCCESS(urEventRelease(evt)); + } +}