[UR][L0v2] Implement IPC event sharing

unified-runtime/source/adapters/level_zero/device.cpp

@@ -1388,6 +1388,13 @@ ur_result_t urDeviceGetInfo(
     return ReturnValue(true);
 #else
     return ReturnValue(false);
+#endif
+  case UR_DEVICE_INFO_IPC_EVENT_SUPPORT_EXP:
+#if defined(UR_ADAPTER_LEVEL_ZERO_V2) && defined(__linux__)
+    return ReturnValue(
+        static_cast<ur_bool_t>(Device->Platform->loadIpcEventExtension()));
+#else
+    return ReturnValue(false);
 #endif
   case UR_DEVICE_INFO_ASYNC_BARRIER:
     return ReturnValue(false);

unified-runtime/source/adapters/level_zero/platform.cpp

@@ -659,6 +659,25 @@ ur_result_t ur_platform_handle_t_::initialize() {
   return UR_RESULT_SUCCESS;
 }
 
+bool ur_platform_handle_t_::loadIpcEventExtension() {
+  std::call_once(ZeIpcEventExt.InitFlag, [this] {
+    auto load = [this](const char *name, auto &slot) {
+      void *ptr = nullptr;
+      if (ZE_CALL_NOCHECK(zeDriverGetExtensionFunctionAddress,
+                          (ZeDriver, name, &ptr)) == ZE_RESULT_SUCCESS) {
+        slot = reinterpret_cast<std::remove_reference_t<decltype(slot)>>(ptr);
+      }
+    };
+    load("zexCounterBasedEventGetIpcHandle", ZeIpcEventExt.pfnGetIpcHandle);
+    load("zexCounterBasedEventOpenIpcHandle", ZeIpcEventExt.pfnOpenIpcHandle);
+    load("zexCounterBasedEventCloseIpcHandle", ZeIpcEventExt.pfnCloseIpcHandle);
+    ZeIpcEventExt.Supported = ZeIpcEventExt.pfnGetIpcHandle != nullptr &&
+                              ZeIpcEventExt.pfnOpenIpcHandle != nullptr &&
+                              ZeIpcEventExt.pfnCloseIpcHandle != nullptr;
+  });
+  return ZeIpcEventExt.Supported;
+}
+
 bool ur_platform_handle_t_::allowDriverInOrderLists(bool OnlyIfRequested) {
   // Use in-order lists implementation from L0 driver instead
   // of adapter's implementation.

unified-runtime/source/adapters/level_zero/platform.hpp

@@ -15,6 +15,10 @@
 #include "ze_ddi.h"
 #include "zes_api.h"
 
+#include <level_zero/driver_experimental/zex_common.h>
+
+#include <mutex>
+
 struct ur_device_handle_t_;
 
 typedef size_t DeviceId;
@@ -220,4 +224,22 @@ struct ur_platform_handle_t_ : ur::handle_base<ur::level_zero::ddi_getter>,
   // Some platforms may not support this API due to frozen driver, eg. gen12 on
   // Windows. For details, see https://github.com/intel/llvm/issues/20927.
   bool ZeDeviceSynchronizeSupported{false};
+
+  // Counter-based event IPC entry points, populated by
+  // loadIpcEventExtension().
+  struct ZeIpcEventExtension {
+    ze_result_t (*pfnGetIpcHandle)(
+        ze_event_handle_t hEvent,
+        zex_ipc_counter_based_event_handle_t *phIpc) = nullptr;
+    ze_result_t (*pfnOpenIpcHandle)(ze_context_handle_t hContext,
+                                    zex_ipc_counter_based_event_handle_t hIpc,
+                                    ze_event_handle_t *phEvent) = nullptr;
+    ze_result_t (*pfnCloseIpcHandle)(ze_event_handle_t hEvent) = nullptr;
+    bool Supported = false;
+    std::once_flag InitFlag;
+  } ZeIpcEventExt;
+
+  // Lazily resolves the counter-based event IPC entry points; returns true
+  // if the driver exposes all three. Thread-safe.
+  bool loadIpcEventExtension();
 };

unified-runtime/source/adapters/level_zero/v2/event.cpp

@@ -14,9 +14,11 @@
 #include "event.hpp"
 #include "event_pool.hpp"
 #include "event_provider.hpp"
+#include "event_provider_counter.hpp"
 #include "queue_api.hpp"
 #include "queue_handle.hpp"
 
+#include "../device.hpp"
 #include "../ur_interface_loader.hpp"
 
 static uint64_t adjustEndEventTimestamp(uint64_t adjustedStartTimestamp,
@@ -164,11 +166,7 @@ void ur_event_handle_t_::reset() {
 }
 
 ze_event_handle_t ur_event_handle_t_::getZeEvent() const {
-  if (event_pool) {
-    return std::get<v2::raii::cache_borrowed_event>(hZeEvent).get();
-  } else {
-    return std::get<v2::raii::ze_event_handle_t>(hZeEvent).get();
-  }
+  return std::visit([](const auto &h) { return h.get(); }, hZeEvent);
 }
 
 ur_result_t ur_event_handle_t_::retain() {
@@ -182,10 +180,17 @@ ur_result_t ur_event_handle_t_::release() {
 
   if (event_pool) {
     event_pool->free(this);
-  } else {
-    std::get<v2::raii::ze_event_handle_t>(hZeEvent).release();
-    delete this;
+    return UR_RESULT_SUCCESS;
+  }
+
+  // External native events (urEventCreateWithNativeHandle): detach so the
+  // variant destructor doesn't destroy a handle this adapter doesn't own.
+  // For adapter_owned/ipc_imported events the variant destructor runs the
+  // correct teardown.
+  if (auto *native = std::get_if<v2::raii::ze_event_handle_t>(&hZeEvent)) {
+    native->release();
   }
+  delete this;
   return UR_RESULT_SUCCESS;
 }
 
@@ -197,6 +202,14 @@ bool ur_event_handle_t_::isProfilingEnabled() const {
   return flags & v2::EVENT_FLAGS_PROFILING_ENABLED;
 }
 
+bool ur_event_handle_t_::isIpcCapable() const {
+  return flags & v2::EVENT_FLAGS_IPC;
+}
+
+bool ur_event_handle_t_::isIpcImported() const {
+  return flags & v2::EVENT_FLAGS_IPC_IMPORTED;
+}
+
 std::pair<uint64_t *, ze_event_handle_t>
 ur_event_handle_t_::getEventEndTimestampAndHandle() {
   return {profilingData.eventEndTimestampAddr(), getZeEvent()};
@@ -234,6 +247,11 @@ ur_event_handle_t_::ur_event_handle_t_(
           ,
           nullptr) {}
 
+ur_event_handle_t_::ur_event_handle_t_(ur_context_handle_t hContext,
+                                       event_variant hZeEvent,
+                                       v2::event_flags_t flags)
+    : ur_event_handle_t_(hContext, std::move(hZeEvent), flags, nullptr) {}
+
 namespace ur::level_zero {
 ur_result_t urEventRetain(ur_event_handle_t hEvent) try {
   return hEvent->retain();
@@ -416,12 +434,49 @@ urEventCreateWithNativeHandle(ur_native_handle_t hNativeEvent,
   return exceptionToResult(std::current_exception());
 }
 
-ur_result_t urEventCreateExp(ur_context_handle_t /*hContext*/,
-                             ur_device_handle_t /*hDevice*/,
-                             const ur_exp_event_desc_t * /*pEventDesc*/,
-                             ur_event_handle_t * /*phEvent*/) {
-  UR_LOG(ERR, "{} function not implemented!", __FUNCTION__);
-  return UR_RESULT_ERROR_UNSUPPORTED_FEATURE;
+ur_result_t urEventCreateExp(ur_context_handle_t hContext,
+                             ur_device_handle_t hDevice,
+                             const ur_exp_event_desc_t *pEventDesc,
+                             ur_event_handle_t *phEvent) try {
+  // Local check (instead of context.cpp::isValidDevice) so unit-test link
+  // targets that compile a subset of adapter sources don't pull in context.cpp.
+  bool validDevice = false;
+  for (ur_device_handle_t Device = hDevice; Device != nullptr && !validDevice;
+       Device = Device->RootDevice) {
+    for (const auto &ContextDevice : hContext->getDevices()) {
+      if (ContextDevice == Device) {
+        validDevice = true;
+        break;
+      }
+    }
+  }
+  if (!validDevice) {
+    return UR_RESULT_ERROR_INVALID_DEVICE;
+  }
+
+  const ur_exp_event_flags_t flags = pEventDesc->flags;
+
+  // Only the IPC creation path is implemented; the non-IPC reusable-event
+  // path is a separate work item.
+  if (!(flags & UR_EXP_EVENT_FLAG_IPC_EXP)) {
+    UR_LOG(ERR, "{}: non-IPC reusable events not implemented!", __FUNCTION__);
+    return UR_RESULT_ERROR_UNSUPPORTED_FEATURE;
+  }
+
+  // IPC and profiling are mutually exclusive.
+  if (flags & UR_EXP_EVENT_FLAG_ENABLE_PROFILING) {
+    return UR_RESULT_ERROR_INVALID_VALUE;
+  }
+
+  ze_event_handle_t hZeEvent = nullptr;
+  UR_CALL(v2::createIpcCounterBasedEvent(hContext, hDevice, &hZeEvent));
+
+  *phEvent = new ur_event_handle_t_(
+      hContext, v2::raii::adapter_owned_event_handle_t{hZeEvent},
+      v2::EVENT_FLAGS_COUNTER | v2::EVENT_FLAGS_IPC);
+  return UR_RESULT_SUCCESS;
+} catch (...) {
+  return exceptionToResult(std::current_exception());
 }
 
 } // namespace ur::level_zero

unified-runtime/source/adapters/level_zero/v2/event.hpp

@@ -18,6 +18,7 @@
 #include "common.hpp"
 #include "common/ur_ref_count.hpp"
 #include "event_provider.hpp"
+#include "ipc_event_handle.hpp"
 
 using ur_event_generation_t = int64_t;
 
@@ -56,10 +57,15 @@ struct event_profiling_data_t {
 
 struct ur_event_handle_t_ : ur_object {
 public:
-  // cache_borrowed_event is used for pooled events, whilst ze_event_handle_t is
-  // used for native events
+  // The variant alternative encodes how the L0 event handle is torn down:
+  //  - cache_borrowed_event: returned to the pool.
+  //  - ze_event_handle_t: external (urEventCreateWithNativeHandle), detached.
+  //  - adapter_owned_event_handle_t: zeEventDestroy.
+  //  - ipc_imported_event_handle_t: zexCounterBasedEventCloseIpcHandle.
   using event_variant =
-      std::variant<v2::raii::cache_borrowed_event, v2::raii::ze_event_handle_t>;
+      std::variant<v2::raii::cache_borrowed_event, v2::raii::ze_event_handle_t,
+                   v2::raii::adapter_owned_event_handle_t,
+                   v2::raii::ipc_imported_event_handle_t>;
 
   ur_event_handle_t_(ur_context_handle_t hContext,
                      v2::raii::cache_borrowed_event eventAllocation,
@@ -69,6 +75,11 @@ struct ur_event_handle_t_ : ur_object {
                      ur_native_handle_t hNativeEvent,
                      const ur_event_native_properties_t *pProperties);
 
+  // Wraps a caller-built event_variant with explicit flags. Used by the IPC
+  // producer (urEventCreateExp) and consumer (urIPCOpenEventHandleExp) paths.
+  ur_event_handle_t_(ur_context_handle_t hContext, event_variant hZeEvent,
+                     v2::event_flags_t flags);
+
   // Set the queue and command that this event is associated with
   void setQueue(ur_queue_t_ *hQueue);
   void setCommandType(ur_command_t commandType);
@@ -99,6 +110,12 @@ struct ur_event_handle_t_ : ur_object {
   // Tells if this event comes from a pool that has profiling enabled.
   bool isProfilingEnabled() const;
 
+  // True for IPC-shareable events (both producer and consumer side).
+  bool isIpcCapable() const;
+
+  // True for events opened via urIPCOpenEventHandleExp.
+  bool isIpcImported() const;
+
   // Queue associated with this event. Can be nullptr (for native events)
   ur_queue_t_ *getQueue() const;
 

unified-runtime/source/adapters/level_zero/v2/event_provider.hpp

@@ -24,7 +24,13 @@ using event_flags_t = uint32_t;
 enum event_flag_t {
   EVENT_FLAGS_COUNTER = UR_BIT(0),
   EVENT_FLAGS_PROFILING_ENABLED = UR_BIT(1),
+  // IPC-shareable producer event.
+  EVENT_FLAGS_IPC = UR_BIT(2),
+  // Event opened from an IPC handle.
+  EVENT_FLAGS_IPC_IMPORTED = UR_BIT(3),
 };
+// Bits used to index pooled events in event_pool_cache. IPC events are never
+// pool-allocated, so EVENT_FLAGS_IPC* are excluded from this count.
 static constexpr size_t EVENT_FLAGS_USED_BITS = 2;
 
 enum queue_type {

unified-runtime/source/adapters/level_zero/v2/event_provider_counter.cpp

@@ -113,4 +113,35 @@ std::unique_ptr<event_provider> createProvider(ur_platform_handle_t platform,
   return std::make_unique<provider_normal>(context, queueType, flags);
 }
 
+ur_result_t createIpcCounterBasedEvent(ur_context_handle_t context,
+                                       ur_device_handle_t device,
+                                       ze_event_handle_t *phEvent) {
+  ur_platform_handle_t platform = device->Platform;
+
+  // Both the create and IPC entry points must be available; otherwise the
+  // event would not be exportable.
+  zexCounterBasedEventCreate pfnCreate = nullptr;
+  if (ZE_CALL_NOCHECK(zeDriverGetExtensionFunctionAddress,
+                      (platform->ZeDriver, "zexCounterBasedEventCreate2",
+                       reinterpret_cast<void **>(&pfnCreate))) !=
+          ZE_RESULT_SUCCESS ||
+      !pfnCreate || !platform->loadIpcEventExtension()) {
+    return UR_RESULT_ERROR_UNSUPPORTED_FEATURE;
+  }
+
+  zex_counter_based_event_desc_t desc = {};
+  desc.stype = ZEX_STRUCTURE_COUNTER_BASED_EVENT_DESC;
+  desc.flags = ZEX_COUNTER_BASED_EVENT_FLAG_HOST_VISIBLE |
+               ZEX_COUNTER_BASED_EVENT_FLAG_IMMEDIATE |
+               ZEX_COUNTER_BASED_EVENT_FLAG_NON_IMMEDIATE |
+               ZEX_COUNTER_BASED_EVENT_FLAG_IPC;
+  desc.signalScope = ZE_EVENT_SCOPE_FLAG_HOST;
+
+  // The IPC entry points consume loader-level handles, so the producer event
+  // is created from loader-level handles too (no zelLoaderTranslateHandle).
+  ZE2UR_CALL(pfnCreate,
+             (context->getZeHandle(), device->ZeDevice, &desc, phEvent));
+  return UR_RESULT_SUCCESS;
+}
+
 } // namespace v2

unified-runtime/source/adapters/level_zero/v2/event_provider_counter.hpp

@@ -61,4 +61,11 @@ std::unique_ptr<event_provider> createProvider(ur_platform_handle_t platform,
                                                ur_device_handle_t device,
                                                event_flags_t flags);
 
+// Creates an IPC-shareable counter-based ze_event_handle_t for the producer
+// side of urEventCreateExp. Returns UR_RESULT_ERROR_UNSUPPORTED_FEATURE if the
+// driver does not expose counter-based event creation or the IPC entry points.
+ur_result_t createIpcCounterBasedEvent(ur_context_handle_t context,
+                                       ur_device_handle_t device,
+                                       ze_event_handle_t *phEvent);
+
 } // namespace v2