From 85da2d98e885ad594deb56127c3cbe5cef78917c Mon Sep 17 00:00:00 2001 From: ccyywwen <75376396+ccyywwen@users.noreply.github.com> Date: Wed, 2 Sep 2026 13:00:29 +0800 Subject: [PATCH 1/7] Add: SPSC queue template and switch worker-chip queue to SPSQ 1.0 Introduce the region-template coordinator, Python bound queue, and common native endpoint with a 10-scalar SPSQ binding. Route create_worker_chip_queue through that path and correlate native queue fatals by allocation identity so L3 no longer ships a 12-scalar L3Q2 layout. --- .../l3/worker_chip_message_queue/README.md | 9 +- .../worker_chip_message_queue_orch.cpp | 83 +- .../test_worker_chip_message_queue.py | 1 + python/simpler/comm_region.py | 35 + python/simpler/comm_region_template.py | 1016 ++++++++++++++ python/simpler/orchestrator.py | 21 +- python/simpler/worker.py | 42 + python/simpler/worker_chip_message_queue.py | 670 ++------- .../include/aicpu/region_instance_view.h | 18 +- .../platform/include/common/region_template.h | 1211 +++++++++++++++++ tests/ut/cpp/CMakeLists.txt | 15 + tests/ut/cpp/common/test_region_template.cpp | 901 ++++++++++++ tests/ut/py/test_worker/test_comm_region.py | 174 +++ .../test_worker/test_comm_region_template.py | 1027 ++++++++++++++ .../test_worker_chip_message_queue.py | 204 ++- 15 files changed, 4660 insertions(+), 767 deletions(-) create mode 100644 python/simpler/comm_region_template.py create mode 100644 src/common/platform/include/common/region_template.h create mode 100644 tests/ut/cpp/common/test_region_template.cpp create mode 100644 tests/ut/py/test_worker/test_comm_region_template.py diff --git a/examples/workers/l3/worker_chip_message_queue/README.md b/examples/workers/l3/worker_chip_message_queue/README.md index a6f2e8ca1d..2d04de758e 100644 --- a/examples/workers/l3/worker_chip_message_queue/README.md +++ b/examples/workers/l3/worker_chip_message_queue/README.md @@ -5,8 +5,11 @@ submits it, and waits. Here the host submits **one long-lived L2 task** and then feeds it a stream of requests while it runs, reading results back as they appear — a serving loop, not a batch. -The transport is the L3-L2 message queue: two arenas (input and output) plus a -descriptor the L2 orchestration receives as plain scalars. See +The transport is the L3-L2 SPSC queue (`SPSQ` ABI 1.0): two arenas (input and +output) plus a 10-scalar endpoint binding the L2 orchestration receives as +plain `TaskArgs` scalars starting at offset 0. Binaries built against the +previous `L3Q2` 12-scalar layout must be recompiled. This example is a smoke +path, not the formal Queue Acceptance record. See [`docs/l3-l2-message-queue.md`](../../../../docs/l3-l2-message-queue.md) for the channel's design. @@ -15,7 +18,7 @@ the channel's design. | Concept | How | | ------- | --- | | **Creating the channel** | `orch.create_worker_chip_queue(worker_id=0, depth=8, input_arena_bytes=..., output_arena_bytes=...)`. | -| **Handing it to L2** | `queue.chip_task_arg_scalars()` packs the descriptor into `TaskArgs` as scalars — the L2 side needs no other setup. | +| **Handing it to L2** | `queue.chip_task_arg_scalars()` packs the `SPSQ` 1.0 binding into `TaskArgs` as exactly 10 scalars — the L2 side needs no other setup. | | **Full-duplex, decoupled** | The host enqueues two requests, drains **three** responses, then enqueues two more. Requests and responses are not paired one-to-one. | | **Zero-copy reads** | `queue.output.peek(timeout)` → `read_into(message, buf)` → `release(message)`. `release` is what returns arena space; skipping it stalls the producer once the queue fills. | | **Cooperative shutdown** | `queue.request_stop(timeout)` lets the L2 side finish work already accepted. Responses queued before the stop are still drained afterwards. | diff --git a/examples/workers/l3/worker_chip_message_queue/kernels/orchestration/worker_chip_message_queue_orch.cpp b/examples/workers/l3/worker_chip_message_queue/kernels/orchestration/worker_chip_message_queue_orch.cpp index c48c745b53..1148d3ae06 100644 --- a/examples/workers/l3/worker_chip_message_queue/kernels/orchestration/worker_chip_message_queue_orch.cpp +++ b/examples/workers/l3/worker_chip_message_queue/kernels/orchestration/worker_chip_message_queue_orch.cpp @@ -11,13 +11,17 @@ #include #include +#include -#include "aicpu/worker_chip_message_queue.h" +#include "aicpu/cache_maintenance.h" +#include "aicpu/device_time.h" +#include "aicpu/region_instance_view.h" +#include "common/region_template.h" #include "orchestration_api.h" // NOLINT(build/include_subdir) namespace { -constexpr int kExpectedArgCount = 12; +constexpr int kExpectedArgCount = static_cast(spsc_queue::kSpscQueueEndpointBindingScalarCount); constexpr uint32_t kInputWindowComputeFuncId = 0; constexpr uint64_t kQueueTimeoutNs = 5000000000ULL; constexpr uint64_t kInputWindow = 4; @@ -46,54 +50,50 @@ struct OutputHeader { uint64_t aux; }; +using QueueEndpoint = spsc_queue::SpscQueueEndpoint; + struct ActiveRequest { - WorkerChipQueueInputHandle handle; + spsc_queue::SpscQueueInputHandle handle; InputHeader header; }; -using QueueEndpoint = WorkerChipQueueEndpoint; +uint64_t spsc_queue_now_ns() { return sys_cnt_ticks_to_ns(device_time_now_ticks(), device_time_frequency_hz()); } void report_queue_error(const QueueEndpoint &queue) { - const WorkerChipQueueError &err = queue.error(); - rt_report_fatal( - SIMPLER_ERROR_EXPLICIT_ORCH_FATAL, "L3-L2 queue error op=%s kind=%u region=%llu msg=%s", - worker_chip_queue_op_to_string(err.op), static_cast(err.kind), - static_cast(err.region_id), err.message - ); + rt_report_fatal(SIMPLER_ERROR_EXPLICIT_ORCH_FATAL, "%s", queue.error().message); } -bool has_queue_error(const QueueEndpoint &queue) { return queue.error().kind != WorkerChipQueueErrorKind::NONE; } +bool has_queue_error(const QueueEndpoint &queue) { return queue.error().kind != spsc_queue::SpscQueueErrorKind::NONE; } -bool parse_input_header(const WorkerChipQueueInputHandle &input, InputHeader *header) { +bool parse_input_header(const spsc_queue::SpscQueueInputHandle &input, InputHeader *header) { if (header == nullptr || input.payload_nbytes != kInputHeaderBytes + kTileBytes) { return false; } - memcpy(header, reinterpret_cast(static_cast(input.payload.gm_addr)), sizeof(*header)); + memcpy(header, reinterpret_cast(static_cast(input.payload.local_addr)), sizeof(*header)); return true; } -simpler::tmr::Tensor make_input_values_tensor(const WorkerChipQueueInputHandle &input) { +simpler::tmr::Tensor make_input_values_tensor(const spsc_queue::SpscQueueInputHandle &input) { uint32_t shape[2] = {kTileRows, kTileCols}; - void *values = reinterpret_cast(static_cast(input.payload.gm_addr + kInputHeaderBytes)); + void *values = reinterpret_cast(static_cast(input.payload.local_addr + kInputHeaderBytes)); return simpler::tmr::make_tensor_external(values, shape, 2, DataType::FLOAT32); } bool publish_aiv_output( - QueueEndpoint &queue, const WorkerChipQueueInputHandle &first, const WorkerChipQueueInputHandle &second, + QueueEndpoint &queue, const spsc_queue::SpscQueueInputHandle &first, const spsc_queue::SpscQueueInputHandle &second, uint64_t request_id, uint64_t kind, uint64_t aux, InputWindowOp op, float scalar ) { uint64_t nbytes = kOutputHeaderBytes + kTileBytes; - WorkerChipQueueOutputReservation output{}; + spsc_queue::SpscQueueOutputReservation output{}; if (!queue.output().reserve(nbytes, kQueueTimeoutNs, output)) { report_queue_error(queue); return false; } OutputHeader header{request_id, kind, aux}; - uint8_t *dst = reinterpret_cast(static_cast(output.payload.gm_addr)); + uint8_t *dst = reinterpret_cast(static_cast(output.payload.local_addr)); memset(dst, 0, kOutputHeaderBytes); memcpy(dst, &header, sizeof(header)); - cache_flush_range(dst, kOutputHeaderBytes); simpler::tmr::Tensor first_tensor = make_input_values_tensor(first); simpler::tmr::Tensor second_tensor = make_input_values_tensor(second); @@ -111,15 +111,16 @@ bool publish_aiv_output( uint32_t first_output_index[2] = {0, 0}; (void)get_tensor_data(output_tensor, 2, first_output_index); + cache_flush_range(dst, nbytes); - if (!queue.output().publish(output, WorkerChipQueueOpcode::DATA)) { + if (!queue.output().publish(output, spsc_queue::SpscQueueOpcode::DATA)) { report_queue_error(queue); return false; } return true; } -bool release_input(QueueEndpoint &queue, const WorkerChipQueueInputHandle &input) { +bool release_input(QueueEndpoint &queue, const spsc_queue::SpscQueueInputHandle &input) { if (!queue.input().release(input)) { report_queue_error(queue); return false; @@ -127,7 +128,7 @@ bool release_input(QueueEndpoint &queue, const WorkerChipQueueInputHandle &input return true; } -bool process_first_pair(QueueEndpoint &queue, ActiveRequest *active, const WorkerChipQueueInputHandle &input) { +bool process_first_pair(QueueEndpoint &queue, ActiveRequest *active, const spsc_queue::SpscQueueInputHandle &input) { active[1].handle = input; if (!parse_input_header(input, &active[1].header) || active[0].header.request_id == 0) { rt_report_fatal(SIMPLER_ERROR_EXPLICIT_ORCH_FATAL, "invalid L3-L2 queue example request"); @@ -158,7 +159,7 @@ bool process_first_pair(QueueEndpoint &queue, ActiveRequest *active, const Worke } bool remember_input_for_pair( - ActiveRequest *active, const WorkerChipQueueInputHandle &input, const InputHeader &header + ActiveRequest *active, const spsc_queue::SpscQueueInputHandle &input, const InputHeader &header ) { if (active[2].header.request_id == 0) { active[2].handle = input; @@ -174,7 +175,7 @@ bool remember_input_for_pair( return false; } -bool process_data_message(QueueEndpoint &queue, const WorkerChipQueueInputHandle &input, ActiveRequest *active) { +bool process_data_message(QueueEndpoint &queue, const spsc_queue::SpscQueueInputHandle &input, ActiveRequest *active) { InputHeader header{}; if (!parse_input_header(input, &header)) { rt_report_fatal(SIMPLER_ERROR_EXPLICIT_ORCH_FATAL, "invalid L3-L2 queue example request"); @@ -239,23 +240,31 @@ __attribute__((visibility("default"))) OrchestrationConfig aicpu_orchestration_c } __attribute__((visibility("default"))) void worker_chip_message_queue_orchestration(const ChipTaskArgs &orch_args) { - WorkerChipOrchRegionDesc desc{ - orch_args.scalar(0), orch_args.scalar(1), orch_args.scalar(2), - orch_args.scalar(3), orch_args.scalar(4), orch_args.scalar(5), - }; - WorkerChipQueueArgs queue_args{ - orch_args.scalar(6), orch_args.scalar(7), orch_args.scalar(8), - orch_args.scalar(9), orch_args.scalar(10), orch_args.scalar(11), - }; - QueueEndpoint queue(desc, queue_args); - if (has_queue_error(queue)) { + uint64_t scalars[spsc_queue::kSpscQueueEndpointBindingScalarCount]; + for (size_t i = 0; i < spsc_queue::kSpscQueueEndpointBindingScalarCount; ++i) { + scalars[i] = orch_args.scalar(static_cast(i)); + } + spsc_queue::SpscQueueEndpointBinding binding{}; + if (!spsc_queue::decode_endpoint_binding(scalars, spsc_queue::kSpscQueueEndpointBindingScalarCount, &binding)) { + rt_report_fatal( + SIMPLER_ERROR_EXPLICIT_ORCH_FATAL, "SPSC queue endpoint error op=init kind=2 msg=invalid queue binding" + ); + return; + } + RegionInstanceView view( + RegionPartLocalSpan{binding.payload_base, binding.payload_bytes}, + RegionPartLocalSpan{binding.counter_base, binding.counter_bytes} + ); + spsc_queue::MonotonicClock clock{&spsc_queue_now_ns}; + QueueEndpoint queue(binding, std::move(view), clock); + if (!queue.live()) { report_queue_error(queue); return; } ActiveRequest active[kInputWindow]{}; for (;;) { - WorkerChipQueueInputHandle input{}; + spsc_queue::SpscQueueInputHandle input{}; if (!queue.input().peek(kQueueTimeoutNs, input)) { if (has_queue_error(queue)) { report_queue_error(queue); @@ -263,7 +272,7 @@ __attribute__((visibility("default"))) void worker_chip_message_queue_orchestrat } continue; } - if (input.opcode == WorkerChipQueueOpcode::STOP) { + if (input.opcode == spsc_queue::SpscQueueOpcode::STOP) { if (!finish_pending_inputs(queue, active)) { return; } @@ -276,7 +285,7 @@ __attribute__((visibility("default"))) void worker_chip_message_queue_orchestrat } return; } - if (input.opcode != WorkerChipQueueOpcode::DATA) { + if (input.opcode != spsc_queue::SpscQueueOpcode::DATA) { rt_report_fatal( SIMPLER_ERROR_EXPLICIT_ORCH_FATAL, "L3-L2 queue example unexpected input opcode=%llu", static_cast(input.opcode) diff --git a/examples/workers/l3/worker_chip_message_queue/test_worker_chip_message_queue.py b/examples/workers/l3/worker_chip_message_queue/test_worker_chip_message_queue.py index a08c0d311b..9bd4110ff6 100644 --- a/examples/workers/l3/worker_chip_message_queue/test_worker_chip_message_queue.py +++ b/examples/workers/l3/worker_chip_message_queue/test_worker_chip_message_queue.py @@ -174,6 +174,7 @@ def orch(orch_handle, _args, cfg): ) task_args = TaskArgs() + # Binding occupies TaskArgs scalars [0, 10). for scalar in queue.chip_task_arg_scalars(): task_args.add_scalar(int(scalar)) orch_handle.submit_next_level(handle, task_args, cfg, worker=0) diff --git a/python/simpler/comm_region.py b/python/simpler/comm_region.py index ccfd64a886..ab4dc4b4a4 100644 --- a/python/simpler/comm_region.py +++ b/python/simpler/comm_region.py @@ -456,6 +456,31 @@ def record_data_plane_failure(self, run_scope: Any, resource_id: int, error: Bas if instance._data_plane_error is None: instance._data_plane_error = error + def record_data_plane_failure_by_allocation_identity( + self, + run_scope: object, + session_instance_id: bytes, + transaction_id: int, + error: BaseException, + ) -> None: + session = bytes(session_instance_id) + transaction = int(transaction_id) + matches: list[RegionInstance] = [] + for instance in self._iter_run(run_scope): + try: + identity = instance._allocation_identity + except MaterializationError: + continue + if identity == (session, transaction): + matches.append(instance) + if not matches: + raise MaterializationError("no region instance for allocation identity") + if len(matches) > 1: + raise MaterializationError("duplicate region instances for allocation identity") + instance = matches[0] + if instance._data_plane_error is None: + instance._data_plane_error = error + def _iter_run(self, run_scope: Any) -> tuple[RegionInstance, ...]: return tuple(instance for key, instance in self._instances.items() if self._run_scopes[key] is run_scope) @@ -516,6 +541,16 @@ def state(self) -> RegionInstanceState: def provider_resource_id(self) -> int: return int(self._provider_resource_id) + @property + def _allocation_identity(self) -> tuple[bytes, int]: + session = self._delegated_session_instance_id + if not isinstance(session, (bytes, bytearray)) or len(bytes(session)) != _SESSION_INSTANCE_ID_BYTES: + raise MaterializationError("region allocation identity is incomplete") + transaction_id = self._delegated_transaction_id + if type(transaction_id) is not int or transaction_id < 1 or transaction_id > _UINT64_MAX: + raise MaterializationError("region allocation identity is incomplete") + return bytes(session), int(transaction_id) + @property def data_plane_error(self) -> BaseException | None: return self._data_plane_error diff --git a/python/simpler/comm_region_template.py b/python/simpler/comm_region_template.py new file mode 100644 index 0000000000..5e0ff3fa2a --- /dev/null +++ b/python/simpler/comm_region_template.py @@ -0,0 +1,1016 @@ +# Copyright (c) PyPTO Contributors. +# This program is free software, you can redistribute it and/or modify it under the terms and conditions of +# CANN Open Software License Agreement Version 2.0 (the "License"). +# Please refer to the License for details. You may not use this file except in compliance with the License. +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, +# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. +# See LICENSE in the root of the software repository for the full text of the License. +# ----------------------------------------------------------------------------------------------------------- +"""Internal Region Template types and the duplex SPSC queue ABI.""" + +from __future__ import annotations + +import ctypes +import struct +import threading +import time +from collections.abc import Callable, Sequence +from dataclasses import dataclass, field +from enum import Enum, IntEnum +from typing import Any, NoReturn, Protocol, TypeVar + +from .comm_endpoints import ( + BackendPlan, + BackendResolver, + EndpointRecord, + EndpointSelector, + RegionLayoutSpec, + SingleOwner, + UnsupportedRegionPlan, +) +from .comm_provider import RegionPartKind, validate_independent_local_views +from .comm_region import ( + MaterializationContext, + MaterializationRefusal, + NotifyOp, + RefusalReason, + RegionInstanceState, + WaitCmp, + materialize_region_instance, +) + +_UINT64_MAX = (1 << 64) - 1 +_SESSION_INSTANCE_ID_BYTES = 8 + +_SPSC_QUEUE_MAGIC = 0x53505351 +_SPSC_QUEUE_ABI_MAJOR = 1 +_SPSC_QUEUE_ABI_MINOR = 0 +_SPSC_QUEUE_MAGIC_VERSION = (_SPSC_QUEUE_MAGIC << 32) | (_SPSC_QUEUE_ABI_MAJOR << 16) | _SPSC_QUEUE_ABI_MINOR +_SPSC_QUEUE_ENDPOINT_BINDING_SCALAR_COUNT = 10 +_SPSC_QUEUE_DESCRIPTOR_BYTES = 32 +_SPSC_QUEUE_ARENA_ALIGNMENT = 64 +_SPSC_QUEUE_DESCRIPTOR_RING_ALIGNMENT = 8 +_SPSC_QUEUE_COUNTER_STRIDE = 64 +_SPSC_QUEUE_COUNTER_BYTES = 384 +_SPSC_QUEUE_MAX_DEPTH = 1 << 30 +_SPSC_QUEUE_INPUT_DESC_TAIL_OFFSET = 0 +_SPSC_QUEUE_INPUT_DESC_HEAD_OFFSET = 64 +_SPSC_QUEUE_OUTPUT_DESC_TAIL_OFFSET = 128 +_SPSC_QUEUE_OUTPUT_DESC_HEAD_OFFSET = 192 +_SPSC_QUEUE_INITIATOR_ABORT_OFFSET = 256 +_SPSC_QUEUE_PEER_ABORT_OFFSET = 320 + +_DESCRIPTOR_STRUCT = struct.Struct(" int: + if type(value) is not int: + raise TypeError(f"{name} must be an exact int") + if value < 0 or value > _UINT64_MAX: + raise ValueError(f"{name} overflowed uint64") + return value + + +def _checked_add_u64(lhs: int, rhs: int) -> int: + result = lhs + rhs + if lhs < 0 or rhs < 0 or result > _UINT64_MAX: + raise ValueError("SPSC queue layout calculation overflowed uint64") + return result + + +def _checked_mul_u64(lhs: int, rhs: int) -> int: + if lhs < 0 or rhs < 0: + raise ValueError("SPSC queue layout calculation overflowed uint64") + result = lhs * rhs + if result > _UINT64_MAX: + raise ValueError("SPSC queue layout calculation overflowed uint64") + return result + + +def _align_up_u64(value: int, align: int) -> int: + if align <= 0: + raise ValueError("SPSC queue layout calculation overflowed uint64") + remainder = value % align + bump = 0 if remainder == 0 else align - remainder + return _checked_add_u64(value, bump) + + +def session_instance_id_to_bits(session_instance_id: bytes) -> int: + if not isinstance(session_instance_id, (bytes, bytearray, memoryview)): + raise TypeError("session_instance_id must be 8 opaque bytes") + raw = bytes(session_instance_id) + if len(raw) != _SESSION_INSTANCE_ID_BYTES: + raise ValueError("session_instance_id must be 8 opaque bytes") + return int(_SESSION_STRUCT.unpack(raw)[0]) + + +def session_instance_id_from_bits(bits: object) -> bytes: + return _SESSION_STRUCT.pack(_require_exact_u64("session_instance_id_bits", bits)) + + +def encode_spsc_queue_descriptor( + seq: object, + opcode: object, + payload_offset: object, + payload_nbytes: object, +) -> bytes: + opcode_value = int(opcode) if isinstance(opcode, SpscQueueOpcode) else opcode + return _DESCRIPTOR_STRUCT.pack( + _require_exact_u64("seq", seq), + _require_exact_u64("opcode", opcode_value), + _require_exact_u64("payload_offset", payload_offset), + _require_exact_u64("payload_nbytes", payload_nbytes), + ) + + +def decode_spsc_queue_descriptor(data: bytes) -> tuple[int, int, int, int]: + raw = bytes(data) + if len(raw) != _SPSC_QUEUE_DESCRIPTOR_BYTES: + raise ValueError("descriptor requires exactly 32 bytes") + seq, opcode, payload_offset, payload_nbytes = _DESCRIPTOR_STRUCT.unpack(raw) + return int(seq), int(opcode), int(payload_offset), int(payload_nbytes) + + +@dataclass(frozen=True) +class _SpscQueueConfig: + depth: int + input_arena_bytes: int + output_arena_bytes: int + + +@dataclass(frozen=True) +class _SpscQueueLayout: + depth: int + input_arena_bytes: int + output_arena_bytes: int + input_desc_offset: int + output_desc_offset: int + input_arena_offset: int + output_arena_offset: int + payload_bytes: int + input_desc_tail_offset: int + input_desc_head_offset: int + output_desc_tail_offset: int + output_desc_head_offset: int + initiator_abort_offset: int + peer_abort_offset: int + counter_bytes: int + + @classmethod + def create(cls, config: _SpscQueueConfig) -> _SpscQueueLayout: + depth = _require_exact_u64("depth", config.depth) + input_arena_bytes = _require_exact_u64("input_arena_bytes", config.input_arena_bytes) + output_arena_bytes = _require_exact_u64("output_arena_bytes", config.output_arena_bytes) + if depth < 1 or depth & (depth - 1) != 0 or depth > _SPSC_QUEUE_MAX_DEPTH: + raise ValueError("depth must be a power of two and <= 2^30") + if input_arena_bytes < 1 or input_arena_bytes % _SPSC_QUEUE_ARENA_ALIGNMENT != 0: + raise ValueError("input_arena_bytes must be a positive 64-byte multiple") + if output_arena_bytes < 1 or output_arena_bytes % _SPSC_QUEUE_ARENA_ALIGNMENT != 0: + raise ValueError("output_arena_bytes must be a positive 64-byte multiple") + + desc_ring_bytes = _checked_mul_u64(depth, _SPSC_QUEUE_DESCRIPTOR_BYTES) + input_desc_offset = 0 + output_desc_offset = _checked_add_u64(input_desc_offset, desc_ring_bytes) + desc_end = _checked_add_u64(output_desc_offset, desc_ring_bytes) + input_arena_offset = _align_up_u64(desc_end, _SPSC_QUEUE_ARENA_ALIGNMENT) + input_arena_end = _checked_add_u64(input_arena_offset, input_arena_bytes) + output_arena_offset = _align_up_u64(input_arena_end, _SPSC_QUEUE_ARENA_ALIGNMENT) + payload_bytes = _checked_add_u64(output_arena_offset, output_arena_bytes) + if ( + output_desc_offset % _SPSC_QUEUE_DESCRIPTOR_RING_ALIGNMENT != 0 + or input_arena_offset % _SPSC_QUEUE_ARENA_ALIGNMENT != 0 + or output_arena_offset % _SPSC_QUEUE_ARENA_ALIGNMENT != 0 + ): + raise ValueError("SPSC queue layout alignment is invalid") + return cls( + depth=depth, + input_arena_bytes=input_arena_bytes, + output_arena_bytes=output_arena_bytes, + input_desc_offset=input_desc_offset, + output_desc_offset=output_desc_offset, + input_arena_offset=input_arena_offset, + output_arena_offset=output_arena_offset, + payload_bytes=payload_bytes, + input_desc_tail_offset=_SPSC_QUEUE_INPUT_DESC_TAIL_OFFSET, + input_desc_head_offset=_SPSC_QUEUE_INPUT_DESC_HEAD_OFFSET, + output_desc_tail_offset=_SPSC_QUEUE_OUTPUT_DESC_TAIL_OFFSET, + output_desc_head_offset=_SPSC_QUEUE_OUTPUT_DESC_HEAD_OFFSET, + initiator_abort_offset=_SPSC_QUEUE_INITIATOR_ABORT_OFFSET, + peer_abort_offset=_SPSC_QUEUE_PEER_ABORT_OFFSET, + counter_bytes=_SPSC_QUEUE_COUNTER_BYTES, + ) + + +@dataclass(frozen=True) +class SpscQueueEndpointBinding: + magic_version: int + session_instance_id_bits: int + transaction_id: int + payload_base: int + payload_bytes: int + counter_base: int + counter_bytes: int + depth: int + input_arena_bytes: int + output_arena_bytes: int + + def __post_init__(self) -> None: + object.__setattr__(self, "magic_version", _require_exact_u64("magic_version", self.magic_version)) + object.__setattr__( + self, + "session_instance_id_bits", + _require_exact_u64("session_instance_id_bits", self.session_instance_id_bits), + ) + object.__setattr__(self, "transaction_id", _require_exact_u64("transaction_id", self.transaction_id)) + object.__setattr__(self, "payload_base", _require_exact_u64("payload_base", self.payload_base)) + object.__setattr__(self, "payload_bytes", _require_exact_u64("payload_bytes", self.payload_bytes)) + object.__setattr__(self, "counter_base", _require_exact_u64("counter_base", self.counter_base)) + object.__setattr__(self, "counter_bytes", _require_exact_u64("counter_bytes", self.counter_bytes)) + object.__setattr__(self, "depth", _require_exact_u64("depth", self.depth)) + object.__setattr__(self, "input_arena_bytes", _require_exact_u64("input_arena_bytes", self.input_arena_bytes)) + object.__setattr__( + self, "output_arena_bytes", _require_exact_u64("output_arena_bytes", self.output_arena_bytes) + ) + if self.magic_version != _SPSC_QUEUE_MAGIC_VERSION: + raise ValueError("binding magic_version is not SPSQ ABI 1.0") + + def to_scalars(self) -> tuple[int, ...]: + return ( + self.magic_version, + self.session_instance_id_bits, + self.transaction_id, + self.payload_base, + self.payload_bytes, + self.counter_base, + self.counter_bytes, + self.depth, + self.input_arena_bytes, + self.output_arena_bytes, + ) + + @classmethod + def from_scalars(cls, scalars: Sequence[object]) -> SpscQueueEndpointBinding: + try: + count = len(scalars) + except TypeError as exc: + raise TypeError("binding scalars must be a sequence of 10 uint64 values") from exc + if count != _SPSC_QUEUE_ENDPOINT_BINDING_SCALAR_COUNT: + raise ValueError("binding requires exactly 10 uint64 scalars") + values = tuple(_require_exact_u64(f"binding[{index}]", scalars[index]) for index in range(count)) + if values[0] != _SPSC_QUEUE_MAGIC_VERSION: + raise ValueError("binding magic_version is not SPSQ ABI 1.0") + return cls( + magic_version=values[0], + session_instance_id_bits=values[1], + transaction_id=values[2], + payload_base=values[3], + payload_bytes=values[4], + counter_base=values[5], + counter_bytes=values[6], + depth=values[7], + input_arena_bytes=values[8], + output_arena_bytes=values[9], + ) + + +_DESCRIPTOR_FIELDS_STRUCT = struct.Struct(" EndpointRecord: + matches = [binding.endpoint for binding in self.bindings if binding.slot == slot] + if len(matches) != 1: + raise ValueError(f"template slot {slot!r} is not uniquely bound") + return matches[0] + + +class _RegionTemplate(Protocol): + def plan(self, config: object) -> _RegionTemplatePlan: ... + + +class _RegionTemplatePlan(Protocol): + @property + def region_layout(self) -> RegionLayoutSpec: ... + + def _bind(self, instance: object, slots: _ResolvedTemplateSlots) -> object: ... + + +def _reject_copy(obj: object) -> NoReturn: + raise TypeError(f"{type(obj).__name__} cannot be copied") + + +def _resolve_template_slots( + registry: object, + resolved_members: Sequence[EndpointRecord], + slot_bindings: Sequence[_TemplateSlotBindingRequest], + required_slots: Sequence[Enum], +) -> _ResolvedTemplateSlots: + required = tuple(required_slots) + member_by_identity = {record.identity: record for record in resolved_members} + seen: dict[Enum, EndpointRecord] = {} + resolved: list[_ResolvedTemplateSlot] = [] + for binding in slot_bindings: + if not isinstance(binding, _TemplateSlotBindingRequest): + raise TypeError("slot_bindings must contain _TemplateSlotBindingRequest values") + slot = binding.slot + if slot not in required: + raise ValueError(f"unknown template slot: {slot!r}") + if slot in seen: + raise ValueError(f"duplicate template slot: {slot!r}") + records = registry.resolve_members((binding.endpoint,)) + if len(records) != 1: + raise ValueError(f"template slot {slot!r} must resolve to exactly one endpoint") + record = records[0] + member = member_by_identity.get(record.identity) + if member is None: + raise ValueError(f"template slot {slot!r} endpoint is not a region member") + seen[slot] = member + resolved.append(_ResolvedTemplateSlot(slot=slot, endpoint=member)) + for slot in required: + if slot not in seen: + raise ValueError(f"missing template slot: {slot!r}") + return _ResolvedTemplateSlots(bindings=tuple(resolved)) + + +def _require_distinct_initiator_peer(slots: _ResolvedTemplateSlots) -> tuple[EndpointRecord, EndpointRecord]: + initiator = slots.endpoint(_SpscQueueSlot.INITIATOR) + peer = slots.endpoint(_SpscQueueSlot.PEER) + if initiator.identity == peer.identity: + raise ValueError("INITIATOR and PEER must bind different endpoint identities") + return initiator, peer + + +class _SpscQueueTemplate: + required_slots = (_SpscQueueSlot.INITIATOR, _SpscQueueSlot.PEER) + + def plan(self, config: object) -> _SpscQueuePlan: + if not isinstance(config, _SpscQueueConfig): + raise TypeError("SPSC queue template.plan requires _SpscQueueConfig") + layout = _SpscQueueLayout.create(config) + return _SpscQueuePlan( + config=config, + layout=layout, + region_layout=RegionLayoutSpec(payload_bytes=layout.payload_bytes, counter_bytes=layout.counter_bytes), + ) + + +class _SpscQueuePlan: + def __init__(self, config: _SpscQueueConfig, layout: _SpscQueueLayout, region_layout: RegionLayoutSpec) -> None: + self._config = config + self._layout = layout + self._region_layout = region_layout + self._state = _SpscQueuePlanState.AVAILABLE + self._bind_lock = threading.Lock() + + @property + def region_layout(self) -> RegionLayoutSpec: + return self._region_layout + + def _consume_for_bind(self) -> None: + with self._bind_lock: + if self._state is not _SpscQueuePlanState.AVAILABLE: + raise RuntimeError("queue plan bind token is already consumed") + self._state = _SpscQueuePlanState.CONSUMED + + def _bind(self, instance: object, slots: _ResolvedTemplateSlots) -> _BoundSpscQueue: + self._consume_for_bind() + if instance.state is not RegionInstanceState.LIVE: + raise ValueError("queue bind requires a LIVE region instance") + layout = instance.layout + if int(layout.payload_bytes) != int(self._layout.payload_bytes): + raise ValueError("region payload_bytes do not match the queue plan") + if int(layout.counter_bytes) != int(self._layout.counter_bytes): + raise ValueError("region counter_bytes do not match the queue plan") + _require_distinct_initiator_peer(slots) + payload_view = instance.local_view(RegionPartKind.PAYLOAD) + counter_view = instance.local_view(RegionPartKind.COUNTER) + if payload_view is None or counter_view is None: + raise ValueError("queue bind requires PAYLOAD and COUNTER local views") + validate_independent_local_views(payload_view, counter_view) + if int(payload_view.logical_bytes) != int(self._layout.payload_bytes): + raise ValueError("PAYLOAD local view does not match the queue plan") + if int(counter_view.logical_bytes) != int(self._layout.counter_bytes): + raise ValueError("COUNTER local view does not match the queue plan") + session, transaction_id = instance._allocation_identity + binding = SpscQueueEndpointBinding( + magic_version=_SPSC_QUEUE_MAGIC_VERSION, + session_instance_id_bits=session_instance_id_to_bits(session), + transaction_id=int(transaction_id), + payload_base=int(payload_view.local_base), + payload_bytes=int(payload_view.logical_bytes), + counter_base=int(counter_view.local_base), + counter_bytes=int(counter_view.logical_bytes), + depth=int(self._layout.depth), + input_arena_bytes=int(self._layout.input_arena_bytes), + output_arena_bytes=int(self._layout.output_arena_bytes), + ) + return _BoundSpscQueue(instance, self._layout, binding) + + def __copy__(self) -> _SpscQueuePlan: + _reject_copy(self) + + def __deepcopy__(self, memo: dict[int, object]) -> _SpscQueuePlan: + _reject_copy(self) + + def __reduce__(self) -> tuple[object, ...]: + _reject_copy(self) + + def __getstate__(self) -> object: + _reject_copy(self) + + +class _RegionTemplateCoordinator: + def __init__(self, worker: object) -> None: + self._worker = worker + + def create( + self, + *, + template: _RegionTemplate, + config: object, + placement: _RegionTemplatePlacementRequest, + result_projector: Callable[[object], _TProjected], + ) -> _TProjected: + if not isinstance(placement, _RegionTemplatePlacementRequest): + raise TypeError("region template create requires _RegionTemplatePlacementRequest") + if not callable(result_projector): + raise TypeError("region template create requires a result projector") + worker = self._worker + with worker._operation_lease("region_template.create"), worker._control_admission("region_template.create"): + return self._create_in_transaction(template, config, placement, result_projector) + + def _create_in_transaction( + self, + template: _RegionTemplate, + config: object, + placement: _RegionTemplatePlacementRequest, + result_projector: Callable[[object], _TProjected], + ) -> _TProjected: + worker = self._worker + instance = None + published = False + try: + plan = template.plan(config) + registry = worker._get_endpoint_registry() + resolved_region = registry.resolve_region_spec(placement.members, placement.topology) + required = tuple(getattr(template, "required_slots", ())) + if not required: + required = tuple(binding.slot for binding in placement.slot_bindings) + slots = _resolve_template_slots(registry, resolved_region.members, placement.slot_bindings, required) + _require_distinct_initiator_peer(slots) + backend_plan = BackendResolver(registry, worker._get_region_access_service()).plan( + resolved_region, plan.region_layout + ) + if isinstance(backend_plan, UnsupportedRegionPlan): + raise MaterializationRefusal(RefusalReason.UNSUPPORTED_PLAN, backend_plan.message) + if not isinstance(backend_plan, BackendPlan): + raise MaterializationRefusal( + RefusalReason.UNSUPPORTED_PLAN, + "materialized region requires a BackendPlan", + ) + instance = materialize_region_instance( + MaterializationContext( + worker=worker, + registry=registry, + plan=backend_plan, + layout=plan.region_layout, + ) + ) + self._prove_initiator_peer_access(instance, slots) + bound = plan._bind(instance, slots) + projected = result_projector(bound) + published = True + return projected + except BaseException: + if instance is not None and not published: + self._rollback_unpublished(instance) + raise + + def _prove_initiator_peer_access(self, instance: object, slots: _ResolvedTemplateSlots) -> None: + initiator, peer = _require_distinct_initiator_peer(slots) + if instance.consumer.identity != initiator.identity: + raise ValueError("INITIATOR slot does not match the materialized consumer access endpoint") + if instance.provider.identity != peer.identity: + raise ValueError("PEER slot does not match the materialized provider-local view endpoint") + + def _rollback_unpublished(self, instance: object) -> None: + try: + live = instance.state is RegionInstanceState.LIVE + except BaseException: + live = False + if not live: + return + try: + self._worker._region_instance_registry.close(instance) + except BaseException as cleanup_error: + self._worker._record_unreclaimable( + "region template create: materialized instance could not be reclaimed; no further work is admitted", + cleanup_error, + ) + + +@dataclass(frozen=True) +class _SpscQueueMessage: + seq: int + opcode: SpscQueueOpcode + payload_offset: int + payload_nbytes: int + _owner_token: object | None = field(default=None, repr=False, compare=False) + + +class _BoundSpscQueue: + def __init__(self, instance: object, layout: _SpscQueueLayout, binding: SpscQueueEndpointBinding) -> None: + self._instance = instance + self._layout = layout + self._endpoint_binding = binding + self._state = _SpscQueueState.LIVE + self._first_error: BaseException | None = None + self._owner_token = object() + self._input_head = 0 + self._input_tail = 0 + self._input_payload_head = 0 + self._input_payload_tail = 0 + self._input_stop_published = False + self._output_head = 0 + self._output_tail = 0 + self._output_payload_head = 0 + self._output_active: _SpscQueueMessage | None = None + self._desc_fields = bytearray(24) + self._desc_seq = bytearray(8) + self._desc_read = bytearray(_SPSC_QUEUE_DESCRIPTOR_BYTES) + self.input = _SpscQueueInitiatorInput(self) + self.output = _SpscQueueInitiatorOutput(self) + + @property + def layout(self) -> _SpscQueueLayout: + return self._layout + + @property + def endpoint_binding(self) -> SpscQueueEndpointBinding: + return self._endpoint_binding + + def try_request_stop(self) -> bool: + return self.input._try_enqueue(None, 0, SpscQueueOpcode.STOP) + + def request_stop(self, timeout: float) -> None: + self.input._enqueue(None, 0, SpscQueueOpcode.STOP, timeout) + + def free(self) -> None: + if self._state is _SpscQueueState.RELEASED: + return + self._state = _SpscQueueState.RELEASED + + def _ensure_usable(self) -> None: + if self._state is _SpscQueueState.RELEASED: + raise RuntimeError("SPSC queue has been released") + if self._state is _SpscQueueState.POISONED_REMOTE: + raise RuntimeError("SPSC queue is remote-aborted") + if self._state is _SpscQueueState.POISONED_LOCAL: + if self._first_error is not None: + raise self._first_error + raise RuntimeError("SPSC queue is poisoned") + if self._state is _SpscQueueState.EXPIRED: + raise RuntimeError("SPSC queue expired after the region instance left LIVE") + try: + live = self._instance.state is RegionInstanceState.LIVE + except BaseException: + live = False + if not live: + self._state = _SpscQueueState.EXPIRED + raise RuntimeError("SPSC queue expired after the region instance left LIVE") + + def _poison_local(self, error: BaseException) -> None: + if self._state is not _SpscQueueState.LIVE: + return + self._first_error = error + self._state = _SpscQueueState.POISONED_LOCAL + try: + self._instance.counter(self._layout.initiator_abort_offset).notify(1, NotifyOp.Set) + except BaseException as abort_error: + adder = getattr(error, "add_note", None) + if callable(adder): + adder(f"initiator abort notify failed: {abort_error}") + + def _run_primitive(self, fn: Callable[..., Any], *args: Any, **kwargs: Any) -> Any: + self._ensure_usable() + try: + return fn(*args, **kwargs) + except TimeoutError: + raise + except BaseException as exc: + self._poison_local(exc) + raise + + def _signal_test(self, offset: int, cmp_value: int, cmp: WaitCmp) -> Any: + return self._run_primitive(lambda: self._instance.counter(offset).test(int(cmp_value), cmp)) + + def _signal_notify(self, offset: int, value: int) -> None: + self._run_primitive(lambda: self._instance.counter(offset).notify(int(value), NotifyOp.Set)) + + def _sample_peer_abort(self) -> None: + result = self._signal_test(self._layout.peer_abort_offset, 1, WaitCmp.GE) + if result.matched: + self._state = _SpscQueueState.POISONED_REMOTE + raise RuntimeError("SPSC queue remote abort observed") + + def _refresh_counter(self, offset: int, local_value: int) -> int: + result = self._signal_test(offset, local_value & 0xFFFF_FFFF, WaitCmp.NE) + if not result.matched: + return local_value + observed = int(result.observed) & 0xFFFF_FFFF + local_low = local_value & 0xFFFF_FFFF + delta = ctypes.c_int32((observed - local_low) & 0xFFFF_FFFF).value + if delta < 0 or delta > int(self._layout.depth): + error = RuntimeError("SPSC queue counter reconstruction failed") + self._poison_local(error) + raise error + return local_value + delta + + def _wait_remaining_seconds(self, deadline_ns: int) -> float: + remaining_ns = int(deadline_ns) - time.monotonic_ns() + if remaining_ns <= 0: + return 0.0 + return remaining_ns / 1_000_000_000 + + def _wait_progress(self, offset: int, local_value: int, deadline_ns: int) -> None: + self._sample_peer_abort() + remaining = self._wait_remaining_seconds(deadline_ns) + if remaining <= 0: + self._sample_peer_abort() + raise TimeoutError("SPSC queue operation timed out") + try: + self._run_primitive( + lambda: self._instance.counter(offset).wait(local_value & 0xFFFF_FFFF, WaitCmp.NE, remaining) + ) + except TimeoutError: + self._sample_peer_abort() + raise TimeoutError("SPSC queue operation timed out") from None + self._sample_peer_abort() + + def _blocking_deadline(self, timeout: float) -> int: + if timeout is None or float(timeout) <= 0: + raise ValueError("SPSC queue blocking operations require a positive timeout") + return time.monotonic_ns() + int(float(timeout) * 1_000_000_000) + + def _write_descriptor( + self, offset: int, seq: int, opcode: SpscQueueOpcode, payload_offset: int, nbytes: int + ) -> None: + self._desc_fields[:] = _DESCRIPTOR_FIELDS_STRUCT.pack(int(opcode), int(payload_offset), int(nbytes)) + self._desc_seq[:] = struct.pack(" _SpscQueueMessage: + self._run_primitive(self._instance.payload_read, offset, self._desc_read, _SPSC_QUEUE_DESCRIPTOR_BYTES) + seq, opcode_value, payload_offset, payload_nbytes = decode_spsc_queue_descriptor(bytes(self._desc_read)) + try: + opcode = SpscQueueOpcode(opcode_value) + except ValueError: + error = RuntimeError("SPSC queue observed invalid descriptor opcode") + self._poison_local(error) + raise error from None + return _SpscQueueMessage( + seq=int(seq), + opcode=opcode, + payload_offset=int(payload_offset), + payload_nbytes=int(payload_nbytes), + ) + + def _advance_payload_head( + self, + cursor: int, + payload_offset: int, + payload_nbytes: int, + arena_offset: int, + arena_bytes: int, + ) -> int: + if payload_nbytes == 0: + return cursor + expected_offset = arena_offset + (cursor % arena_bytes) + if expected_offset != payload_offset: + if payload_offset != arena_offset: + error = RuntimeError("SPSC queue payload replay offset mismatch") + self._poison_local(error) + raise error + cursor += arena_bytes - (cursor % arena_bytes) + return cursor + payload_nbytes + + def _replay_released_input_descriptors(self, old_head: int, new_head: int) -> None: + cursor = old_head + while cursor < new_head: + slot_index = cursor & (self._layout.depth - 1) + slot_offset = self._layout.input_desc_offset + slot_index * _SPSC_QUEUE_DESCRIPTOR_BYTES + message = self._read_descriptor(slot_offset) + if message.seq != cursor + 1: + error = RuntimeError("SPSC queue input release replay seq mismatch") + self._poison_local(error) + raise error + self._input_payload_head = self._advance_payload_head( + self._input_payload_head, + message.payload_offset, + message.payload_nbytes, + self._layout.input_arena_offset, + self._layout.input_arena_bytes, + ) + cursor += 1 + + def _reserve_input_payload(self, nbytes: int, next_payload_tail: int) -> tuple[int, int] | None: + arena_pos = next_payload_tail % self._layout.input_arena_bytes + if arena_pos + nbytes > self._layout.input_arena_bytes: + next_payload_tail += self._layout.input_arena_bytes - arena_pos + arena_pos = 0 + if next_payload_tail + nbytes - self._input_payload_head > self._layout.input_arena_bytes: + return None + return self._layout.input_arena_offset + arena_pos, next_payload_tail + + def __copy__(self) -> _BoundSpscQueue: + _reject_copy(self) + + def __deepcopy__(self, memo: dict[int, object]) -> _BoundSpscQueue: + _reject_copy(self) + + def __reduce__(self) -> tuple[object, ...]: + _reject_copy(self) + + def __getstate__(self) -> object: + _reject_copy(self) + + +class _SpscQueueInitiatorInput: + def __init__(self, queue: _BoundSpscQueue) -> None: + self._queue = queue + + def try_enqueue(self, buffer_or_none: object, nbytes: int) -> bool: + return self._try_enqueue(buffer_or_none, nbytes, SpscQueueOpcode.DATA) + + def enqueue(self, buffer_or_none: object, nbytes: int, timeout: float) -> None: + self._enqueue(buffer_or_none, nbytes, SpscQueueOpcode.DATA, timeout) + + def _enqueue(self, buffer_or_none: object, nbytes: int, opcode: SpscQueueOpcode, timeout: float) -> None: + nbytes = int(nbytes) + if nbytes > int(self._queue._layout.input_arena_bytes): + raise ValueError("SPSC queue payload exceeds input arena capacity") + deadline_ns = self._queue._blocking_deadline(timeout) + while True: + if self._try_enqueue(buffer_or_none, nbytes, opcode): + return + if self._queue._input_stop_published: + raise RuntimeError("SPSC queue input is stopped") + self._queue._wait_progress(self._queue._layout.input_desc_head_offset, self._queue._input_head, deadline_ns) + + def _try_enqueue(self, buffer_or_none: object, nbytes: int, opcode: SpscQueueOpcode) -> bool: + queue = self._queue + nbytes = int(nbytes) + if nbytes < 0: + raise ValueError("SPSC queue nbytes must be non-negative") + try: + opcode = SpscQueueOpcode(opcode) + except ValueError as exc: + raise ValueError("SPSC queue input opcode must be DATA or STOP") from exc + if opcode not in (SpscQueueOpcode.DATA, SpscQueueOpcode.STOP): + raise ValueError("SPSC queue input opcode must be DATA or STOP") + if opcode is SpscQueueOpcode.STOP and nbytes != 0: + raise ValueError("SPSC queue STOP must be zero-byte") + if nbytes == 0: + if buffer_or_none is not None: + raise ValueError("SPSC queue zero-byte enqueue requires buffer_or_none == None") + elif buffer_or_none is None: + raise ValueError("SPSC queue nonzero enqueue requires a host buffer") + queue._ensure_usable() + if nbytes > int(queue._layout.input_arena_bytes): + return False + if queue._input_stop_published: + return False + queue._sample_peer_abort() + if nbytes != 0: + self._require_enqueue_source(buffer_or_none, nbytes) + old_head = queue._input_head + queue._input_head = queue._refresh_counter(queue._layout.input_desc_head_offset, queue._input_head) + if queue._input_head != old_head: + queue._replay_released_input_descriptors(old_head, queue._input_head) + if queue._input_tail - queue._input_head >= queue._layout.depth: + return False + payload_offset = 0 + next_payload_tail = queue._input_payload_tail + if nbytes != 0: + reserved = queue._reserve_input_payload(nbytes, next_payload_tail) + if reserved is None: + return False + payload_offset, next_payload_tail = reserved + queue._run_primitive(queue._instance.payload_write, payload_offset, buffer_or_none, nbytes) + seq = queue._input_tail + 1 + slot_index = queue._input_tail & (queue._layout.depth - 1) + slot_offset = queue._layout.input_desc_offset + slot_index * _SPSC_QUEUE_DESCRIPTOR_BYTES + queue._write_descriptor(slot_offset, seq, opcode, payload_offset, nbytes) + queue._input_tail += 1 + queue._input_payload_tail = next_payload_tail + nbytes + queue._signal_notify(queue._layout.input_desc_tail_offset, queue._input_tail) + if opcode is SpscQueueOpcode.STOP: + queue._input_stop_published = True + return True + + def _require_enqueue_source(self, buffer_or_none: Any, nbytes: int) -> None: + try: + view = memoryview(buffer_or_none) + except TypeError as exc: + if hasattr(buffer_or_none, "nbytes") and hasattr(buffer_or_none, "base"): + available = int(buffer_or_none.nbytes) + if available < int(nbytes): + raise ValueError(f"SPSC queue nbytes={nbytes} exceeds registered buffer size {available}") from None + return + raise ValueError("SPSC queue requires a registered Buffer or contiguous host buffer") from exc + if not view.c_contiguous: + raise ValueError("SPSC queue ordinary host buffer must be C-contiguous") + if int(view.nbytes) < int(nbytes): + raise ValueError(f"SPSC queue nbytes={nbytes} exceeds ordinary host buffer size {int(view.nbytes)}") + + +class _SpscQueueInitiatorOutput: + def __init__(self, queue: _BoundSpscQueue) -> None: + self._queue = queue + + def try_peek(self) -> _SpscQueueMessage | None: + queue = self._queue + queue._ensure_usable() + queue._sample_peer_abort() + if queue._output_active is not None: + return queue._output_active + queue._output_tail = queue._refresh_counter(queue._layout.output_desc_tail_offset, queue._output_tail) + if queue._output_tail == queue._output_head: + return None + slot_index = queue._output_head & (queue._layout.depth - 1) + slot_offset = queue._layout.output_desc_offset + slot_index * _SPSC_QUEUE_DESCRIPTOR_BYTES + message = queue._read_descriptor(slot_offset) + if message.seq != queue._output_head + 1: + error = RuntimeError("SPSC queue output descriptor seq mismatch") + queue._poison_local(error) + raise error + if message.opcode not in (SpscQueueOpcode.DATA, SpscQueueOpcode.ERROR): + error = RuntimeError("SPSC queue output descriptor must be DATA or ERROR") + queue._poison_local(error) + raise error + if message.payload_nbytes == 0: + if message.payload_offset != 0: + error = RuntimeError("SPSC queue zero-byte output descriptor has nonzero offset") + queue._poison_local(error) + raise error + else: + begin = queue._layout.output_arena_offset + end = begin + queue._layout.output_arena_bytes + if message.payload_offset < begin or message.payload_offset + message.payload_nbytes > end: + error = RuntimeError("SPSC queue output payload outside output arena") + queue._poison_local(error) + raise error + queue._advance_payload_head( + queue._output_payload_head, + message.payload_offset, + message.payload_nbytes, + queue._layout.output_arena_offset, + queue._layout.output_arena_bytes, + ) + owned = _SpscQueueMessage( + seq=message.seq, + opcode=message.opcode, + payload_offset=message.payload_offset, + payload_nbytes=message.payload_nbytes, + _owner_token=queue._owner_token, + ) + queue._output_active = owned + return owned + + def peek(self, timeout: float) -> _SpscQueueMessage: + queue = self._queue + deadline_ns = queue._blocking_deadline(timeout) + while True: + message = self.try_peek() + if message is not None: + return message + queue._wait_progress(queue._layout.output_desc_tail_offset, queue._output_tail, deadline_ns) + + def read_into(self, handle: _SpscQueueMessage, buffer: object) -> None: + queue = self._queue + queue._ensure_usable() + self._require_active_handle(handle, ownership_violation=True) + if handle.payload_nbytes == 0: + if buffer is not None: + raise ValueError("SPSC queue zero-byte output read requires buffer == None") + return + if buffer is None: + raise ValueError("SPSC queue nonzero output read requires a writable host buffer") + self._require_read_destination(buffer, handle.payload_nbytes) + queue._run_primitive(queue._instance.payload_read, handle.payload_offset, buffer, handle.payload_nbytes) + + def release(self, handle: _SpscQueueMessage) -> None: + queue = self._queue + queue._ensure_usable() + self._require_active_handle(handle, ownership_violation=True) + queue._output_payload_head = queue._advance_payload_head( + queue._output_payload_head, + handle.payload_offset, + handle.payload_nbytes, + queue._layout.output_arena_offset, + queue._layout.output_arena_bytes, + ) + queue._output_head += 1 + queue._output_active = None + queue._signal_notify(queue._layout.output_desc_head_offset, queue._output_head) + + def dequeue_into(self, buffer: object, timeout: float) -> _SpscQueueMessage: + handle = self.peek(timeout) + self.read_into(handle, buffer) + self.release(handle) + return handle + + def try_dequeue_into(self, buffer: object) -> _SpscQueueMessage | None: + handle = self.try_peek() + if handle is None: + return None + self.read_into(handle, buffer) + self.release(handle) + return handle + + def _require_active_handle(self, handle: _SpscQueueMessage, *, ownership_violation: bool) -> None: + queue = self._queue + active = queue._output_active + if ( + active is not None + and active is handle + and handle._owner_token is queue._owner_token + and handle.seq == active.seq + and handle.opcode == active.opcode + and handle.payload_offset == active.payload_offset + and handle.payload_nbytes == active.payload_nbytes + ): + return + if ownership_violation: + error = RuntimeError("SPSC queue output handle is not active") + queue._poison_local(error) + raise error + raise RuntimeError("SPSC queue output handle is not active") + + def _require_read_destination(self, buffer: Any, nbytes: int) -> None: + try: + view = memoryview(buffer) + except TypeError as exc: + if hasattr(buffer, "nbytes") and hasattr(buffer, "base"): + available = int(buffer.nbytes) + if available < int(nbytes): + raise ValueError(f"SPSC queue nbytes={nbytes} exceeds registered buffer size {available}") from None + return + raise ValueError("SPSC queue requires a registered Buffer or writable contiguous host buffer") from exc + if not view.c_contiguous: + raise ValueError("SPSC queue ordinary host buffer must be C-contiguous") + if view.readonly: + raise ValueError("SPSC queue output target must be a writable ordinary host buffer") + if int(view.nbytes) < int(nbytes): + raise ValueError(f"SPSC queue nbytes={nbytes} exceeds ordinary host buffer size {int(view.nbytes)}") diff --git a/python/simpler/orchestrator.py b/python/simpler/orchestrator.py index 6443711159..4d01b6fac5 100644 --- a/python/simpler/orchestrator.py +++ b/python/simpler/orchestrator.py @@ -669,23 +669,18 @@ def create_worker_chip_region(self, *, worker_id: int, payload_bytes: int, count return self._worker._create_worker_chip_region(int(worker_id), int(payload_bytes), int(counter_bytes)) def create_worker_chip_queue(self, *, worker_id: int, depth: int, input_arena_bytes: int, output_arena_bytes: int): - """Create an L3-L2 message queue backed by one L3-L2 communication region.""" + """Create an L3-L2 message queue on one NEXT_LEVEL chip worker.""" if self._worker is None: raise RuntimeError("create_worker_chip_queue requires an Orchestrator bound to a Worker") from .worker_chip_message_queue import create_worker_chip_queue # noqa: PLC0415 - # Reserved across the whole build, not just the region creation it - # nests: the descriptor writes that follow are device effects too. The - # reservation is re-entrant, so the inner create_worker_chip_region joins this - # one rather than deadlocking on it. - with self._control_admission("create_worker_chip_queue"): - return create_worker_chip_queue( - self, - worker_id=int(worker_id), - depth=int(depth), - input_arena_bytes=int(input_arena_bytes), - output_arena_bytes=int(output_arena_bytes), - ) + return create_worker_chip_queue( + self, + worker_id=int(worker_id), + depth=int(depth), + input_arena_bytes=int(input_arena_bytes), + output_arena_bytes=int(output_arena_bytes), + ) # ------------------------------------------------------------------ # Nested scope (Strict-1 per-scope rings) diff --git a/python/simpler/worker.py b/python/simpler/worker.py index 785bb74cc5..5046d359cf 100644 --- a/python/simpler/worker.py +++ b/python/simpler/worker.py @@ -299,6 +299,9 @@ def my_l4_orch(orch, args, config): # text emitted by the orchestration wrapper; keep this pattern in sync with the # wrapper's ``L3-L2 endpoint error ... region=`` format. _WORKER_CHIP_ENDPOINT_ERROR_REGION_RE = re.compile(r"\bL3-L2 endpoint error\b[^\n]*\bregion=(\d+)\b") +_SPSC_QUEUE_ENDPOINT_ERROR_RE = re.compile( + r"SPSC queue endpoint error op=\S+ kind=\d+ session=0x([0-9A-Fa-f]{16}) transaction=(\d+) msg=" +) def _host_spans_active() -> bool: @@ -8361,6 +8364,43 @@ def _validate_worker_chip_id(self, worker_id: int) -> None: if worker_id < 0 or worker_id >= len(device_ids): raise ValueError(f"create_worker_chip_region: worker_id {worker_id} outside [0, {len(device_ids)})") + def _poison_spsc_queue_from_endpoint_error( + self, exc: BaseException, resources: _RunResources | None = None + ) -> bool: + text = str(exc) + if "SPSC queue endpoint error" not in text: + return False + match = _SPSC_QUEUE_ENDPOINT_ERROR_RE.search(text) + if match is None: + self._record_unreclaimable( + "spsc queue: endpoint error marker is malformed; no further work is admitted", + exc, + ) + return True + session = struct.pack(" bool: @@ -11438,6 +11478,8 @@ def _finalize_run_handle_unlocked( # noqa: PLR0912 -- one extra branch for the def _poison_endpoint() -> None: if native_error is not None: + if self._poison_spsc_queue_from_endpoint_error(native_error, resources): + return self._poison_worker_chip_region_from_endpoint_error(native_error, resources) def _release_native_run() -> None: diff --git a/python/simpler/worker_chip_message_queue.py b/python/simpler/worker_chip_message_queue.py index a964477521..64939aee41 100644 --- a/python/simpler/worker_chip_message_queue.py +++ b/python/simpler/worker_chip_message_queue.py @@ -6,214 +6,49 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. # ----------------------------------------------------------------------------------------------------------- -"""L3-side L3-L2 SPSC message queue wrapper.""" +"""L3-side L3-L2 SPSC message queue compatibility facade.""" from __future__ import annotations -import ctypes -import struct -import time -from dataclasses import dataclass -from enum import IntEnum from typing import Any -from .buffer import Buffer -from .task_interface import DataType -from .worker_chip_orch_comm import ( - NotifyOp, - WaitCmp, - WorkerChipOrchRegion, +from .comm_endpoints import DEVICE_AICPU, HOST_CPU, SingleOwner, _format_worker_path, at +from .comm_provider import RegionPartKind +from .comm_region_template import ( + _SPSC_QUEUE_COUNTER_BYTES, + _SPSC_QUEUE_DESCRIPTOR_BYTES, + _SPSC_QUEUE_INITIATOR_ABORT_OFFSET, + _SPSC_QUEUE_PEER_ABORT_OFFSET, + SpscQueueOpcode, + _BoundSpscQueue, + _RegionTemplateCoordinator, + _RegionTemplatePlacementRequest, + _SpscQueueConfig, + _SpscQueueLayout, + _SpscQueueMessage, + _SpscQueueSlot, + _SpscQueueTemplate, + _TemplateSlotBindingRequest, ) +from .worker_chip_orch_comm import WorkerChipOrchRegion, worker_chip_orch_region_desc_from_local_views -WORKER_CHIP_QUEUE_MAGIC = 0x4C335132 -WORKER_CHIP_QUEUE_ABI_MAJOR = 1 -WORKER_CHIP_QUEUE_ABI_MINOR = 1 -WORKER_CHIP_QUEUE_DESC_SLOT_BYTES = 32 -WORKER_CHIP_QUEUE_PAYLOAD_ARENA_ALIGNMENT = 64 -WORKER_CHIP_QUEUE_COUNTER_STRIDE = 64 -WORKER_CHIP_QUEUE_INPUT_DESC_TAIL_OFFSET = 0 -WORKER_CHIP_QUEUE_INPUT_DESC_HEAD_OFFSET = 64 -WORKER_CHIP_QUEUE_OUTPUT_DESC_TAIL_OFFSET = 128 -WORKER_CHIP_QUEUE_OUTPUT_DESC_HEAD_OFFSET = 192 -WORKER_CHIP_QUEUE_WORKER_ABORT_FLAG_OFFSET = 256 -WORKER_CHIP_QUEUE_CHIP_ABORT_FLAG_OFFSET = 320 -WORKER_CHIP_QUEUE_COUNTER_BYTES = 384 -WORKER_CHIP_QUEUE_MAX_DEPTH = 1 << 30 -_UINT64_MAX = (1 << 64) - 1 +WORKER_CHIP_QUEUE_DESC_SLOT_BYTES = _SPSC_QUEUE_DESCRIPTOR_BYTES +WORKER_CHIP_QUEUE_COUNTER_BYTES = _SPSC_QUEUE_COUNTER_BYTES +WORKER_CHIP_QUEUE_WORKER_ABORT_FLAG_OFFSET = _SPSC_QUEUE_INITIATOR_ABORT_OFFSET +WORKER_CHIP_QUEUE_CHIP_ABORT_FLAG_OFFSET = _SPSC_QUEUE_PEER_ABORT_OFFSET -_DESC = struct.Struct("<4Q") -_POLL_INTERVAL_S = 0.00005 +WorkerChipQueueOpcode = SpscQueueOpcode +WorkerChipQueueMessage = _SpscQueueMessage +WorkerChipQueueLayout = _SpscQueueLayout -@dataclass(frozen=True) -class _HostByteSpan: - nbytes: int - ptr: int | None - view: memoryview | None - - -class WorkerChipQueueOpcode(IntEnum): - INVALID = 0 - DATA = 1 - STOP = 2 - ERROR = 3 - - -class _QueueState(IntEnum): - LIVE = 0 - RELEASED = 1 - POISONED_LOCAL = 2 - POISONED_REMOTE = 3 - EXPIRED = 4 - - -@dataclass(frozen=True) -class WorkerChipQueueLayout: - depth: int - input_desc_offset: int - output_desc_offset: int - input_arena_offset: int - output_arena_offset: int - input_arena_bytes: int - output_arena_bytes: int - payload_bytes: int - input_desc_tail_offset: int - input_desc_head_offset: int - output_desc_tail_offset: int - output_desc_head_offset: int - worker_abort_flag_offset: int - chip_abort_flag_offset: int - counter_bytes: int - - -@dataclass(frozen=True) -class WorkerChipQueueMessage: - seq: int - opcode: WorkerChipQueueOpcode - payload_offset: int - payload_nbytes: int - - -def worker_chip_queue_magic_version() -> int: - return (WORKER_CHIP_QUEUE_MAGIC << 32) | (WORKER_CHIP_QUEUE_ABI_MAJOR << 16) | WORKER_CHIP_QUEUE_ABI_MINOR - - -def _align_up(value: int, align: int) -> int: - if value < 0 or value > _UINT64_MAX: - raise ValueError("L3-L2 queue layout calculation overflowed uint64") - remainder = value % align - bump = 0 if remainder == 0 else align - remainder - result = value + bump - if result > _UINT64_MAX: - raise ValueError("L3-L2 queue layout calculation overflowed uint64") - return result - - -def _checked_add_u64(lhs: int, rhs: int) -> int: - result = lhs + rhs - if lhs < 0 or rhs < 0 or result > _UINT64_MAX: - raise ValueError("L3-L2 queue layout calculation overflowed uint64") - return result - - -def _tensor_like_nbytes(buffer: Any) -> int | None: - nbytes_attr: Any = getattr(buffer, "nbytes", None) - if nbytes_attr is not None: - nbytes_value: Any = nbytes_attr() if callable(nbytes_attr) else nbytes_attr - return int(nbytes_value) - numel: Any = getattr(buffer, "numel", None) - element_size: Any = getattr(buffer, "element_size", None) - if callable(numel) and callable(element_size): - numel_value: Any = numel() - element_size_value: Any = element_size() - return int(numel_value) * int(element_size_value) - return None - - -def _host_byte_span(buffer: Any, nbytes: int, *, writable: bool) -> _HostByteSpan: - nbytes = int(nbytes) - try: - view = memoryview(buffer) - except TypeError: - view = None - - if view is not None: - if not view.c_contiguous: - raise ValueError("L3-L2 queue ordinary host buffer must be C-contiguous") - if int(view.nbytes) < nbytes: - raise ValueError(f"L3-L2 queue nbytes={nbytes} exceeds ordinary host buffer size {int(view.nbytes)}") - try: - byte_view = view if view.itemsize == 1 and view.format in {"B", "b", "c"} else view.cast("B") - except (TypeError, ValueError) as exc: - raise ValueError("L3-L2 queue ordinary host buffer must be viewable as bytes") from exc - if writable and byte_view.readonly: - raise ValueError("L3-L2 queue output target must be a writable ordinary host buffer") - ptr = None - if not byte_view.readonly: - ptr = ctypes.addressof(ctypes.c_char.from_buffer(byte_view)) - return _HostByteSpan(nbytes=nbytes, ptr=ptr, view=byte_view) - - data_ptr: Any = getattr(buffer, "data_ptr", None) - if callable(data_ptr): - is_contiguous = getattr(buffer, "is_contiguous", None) - if callable(is_contiguous) and not bool(is_contiguous()): - raise ValueError("L3-L2 queue ordinary host tensor-like buffer must be contiguous") - device = getattr(buffer, "device", None) - device_type = getattr(device, "type", device) - if device_type is not None and str(device_type) != "cpu": - raise ValueError("L3-L2 queue ordinary host tensor-like buffer must be on CPU") - available = _tensor_like_nbytes(buffer) - if available is None: - raise ValueError("L3-L2 queue ordinary host tensor-like buffer must expose nbytes") - if available < nbytes: - raise ValueError(f"L3-L2 queue nbytes={nbytes} exceeds ordinary host buffer size {available}") - ptr_value: Any = data_ptr() - ptr = int(ptr_value) - if ptr <= 0 and nbytes > 0: - raise ValueError("L3-L2 queue ordinary host tensor-like buffer must have a nonzero data_ptr") - return _HostByteSpan(nbytes=nbytes, ptr=ptr, view=None) - - access = "writable" if writable else "readable" - raise ValueError(f"L3-L2 queue requires a registered Buffer or {access} contiguous ordinary host buffer") - - -def make_worker_chip_queue_layout(depth: int, input_arena_bytes: int, output_arena_bytes: int) -> WorkerChipQueueLayout: - depth = int(depth) - input_arena_bytes = int(input_arena_bytes) - output_arena_bytes = int(output_arena_bytes) - if depth <= 0 or depth & (depth - 1) != 0 or depth > WORKER_CHIP_QUEUE_MAX_DEPTH: - raise ValueError("L3-L2 queue depth must be a power of two and <= 2^30") - if input_arena_bytes <= 0 or input_arena_bytes % WORKER_CHIP_QUEUE_PAYLOAD_ARENA_ALIGNMENT != 0: - raise ValueError("L3-L2 queue input_arena_bytes must be a positive 64-byte multiple") - if output_arena_bytes <= 0 or output_arena_bytes % WORKER_CHIP_QUEUE_PAYLOAD_ARENA_ALIGNMENT != 0: - raise ValueError("L3-L2 queue output_arena_bytes must be a positive 64-byte multiple") - - desc_ring_bytes = depth * WORKER_CHIP_QUEUE_DESC_SLOT_BYTES - if desc_ring_bytes > _UINT64_MAX: - raise ValueError("L3-L2 queue layout calculation overflowed uint64") - input_desc_offset = 0 - output_desc_offset = _checked_add_u64(input_desc_offset, desc_ring_bytes) - desc_end = _checked_add_u64(output_desc_offset, desc_ring_bytes) - input_arena_offset = _align_up(desc_end, WORKER_CHIP_QUEUE_PAYLOAD_ARENA_ALIGNMENT) - input_arena_end = _checked_add_u64(input_arena_offset, input_arena_bytes) - output_arena_offset = _align_up(input_arena_end, WORKER_CHIP_QUEUE_PAYLOAD_ARENA_ALIGNMENT) - payload_bytes = _checked_add_u64(output_arena_offset, output_arena_bytes) - return WorkerChipQueueLayout( - depth=depth, - input_desc_offset=input_desc_offset, - output_desc_offset=output_desc_offset, - input_arena_offset=input_arena_offset, - output_arena_offset=output_arena_offset, - input_arena_bytes=input_arena_bytes, - output_arena_bytes=output_arena_bytes, - payload_bytes=payload_bytes, - input_desc_tail_offset=WORKER_CHIP_QUEUE_INPUT_DESC_TAIL_OFFSET, - input_desc_head_offset=WORKER_CHIP_QUEUE_INPUT_DESC_HEAD_OFFSET, - output_desc_tail_offset=WORKER_CHIP_QUEUE_OUTPUT_DESC_TAIL_OFFSET, - output_desc_head_offset=WORKER_CHIP_QUEUE_OUTPUT_DESC_HEAD_OFFSET, - worker_abort_flag_offset=WORKER_CHIP_QUEUE_WORKER_ABORT_FLAG_OFFSET, - chip_abort_flag_offset=WORKER_CHIP_QUEUE_CHIP_ABORT_FLAG_OFFSET, - counter_bytes=WORKER_CHIP_QUEUE_COUNTER_BYTES, +def make_worker_chip_queue_layout(depth: int, input_arena_bytes: int, output_arena_bytes: int) -> _SpscQueueLayout: + return _SpscQueueLayout.create( + _SpscQueueConfig( + depth=int(depth), + input_arena_bytes=int(input_arena_bytes), + output_arena_bytes=int(output_arena_bytes), + ) ) @@ -225,416 +60,75 @@ def create_worker_chip_queue( input_arena_bytes: int, output_arena_bytes: int, ) -> WorkerChipQueue: - layout = make_worker_chip_queue_layout(depth, input_arena_bytes, output_arena_bytes) - region = orch.create_worker_chip_region( - worker_id=int(worker_id), - payload_bytes=layout.payload_bytes, - counter_bytes=layout.counter_bytes, + worker = orch._worker + worker._validate_worker_chip_id(int(worker_id)) + root_path = _format_worker_path(int(worker.level)) + provider_path = _format_worker_path(2, parent_path=root_path, index=int(worker_id)) + host = at(root_path, HOST_CPU) + peer = at(provider_path, DEVICE_AICPU) + placement = _RegionTemplatePlacementRequest( + members=(host, peer), + topology=SingleOwner(provider=peer), + slot_bindings=( + _TemplateSlotBindingRequest(slot=_SpscQueueSlot.INITIATOR, endpoint=host), + _TemplateSlotBindingRequest(slot=_SpscQueueSlot.PEER, endpoint=peer), + ), ) - try: - desc_fields = orch.alloc([24], DataType.UINT8) - desc_seq = orch.alloc([8], DataType.UINT8) - desc_read = orch.alloc([WORKER_CHIP_QUEUE_DESC_SLOT_BYTES], DataType.UINT8) - for offset in ( - layout.input_desc_tail_offset, - layout.input_desc_head_offset, - layout.output_desc_tail_offset, - layout.output_desc_head_offset, - layout.worker_abort_flag_offset, - layout.chip_abort_flag_offset, - ): - region.counter(offset).notify(0, NotifyOp.Set) - except Exception: - try: - region.free() - except Exception: - pass - raise - return WorkerChipQueue(orch, region, layout, desc_fields, desc_seq, desc_read) + config = _SpscQueueConfig( + depth=int(depth), + input_arena_bytes=int(input_arena_bytes), + output_arena_bytes=int(output_arena_bytes), + ) + return _RegionTemplateCoordinator(worker).create( + template=_SpscQueueTemplate(), + config=config, + placement=placement, + result_projector=lambda bound: _project_worker_chip_queue(worker, bound), + ) + + +def _project_worker_chip_queue(worker: Any, bound: object) -> WorkerChipQueue: + if not isinstance(bound, _BoundSpscQueue): + raise TypeError("worker-chip queue projector requires a bound SPSC queue") + instance = bound._instance + payload_view = instance.local_view(RegionPartKind.PAYLOAD) + counter_view = instance.local_view(RegionPartKind.COUNTER) + if payload_view is None or counter_view is None: + raise RuntimeError("worker-chip queue projector requires PAYLOAD and COUNTER local views") + desc = worker_chip_orch_region_desc_from_local_views(instance.provider_resource_id, payload_view, counter_view) + region = WorkerChipOrchRegion(worker, instance, desc) + return WorkerChipQueue(bound, region) class WorkerChipQueue: - def __init__( - self, - orch: Any, - region: WorkerChipOrchRegion, - layout: WorkerChipQueueLayout, - desc_fields: Buffer, - desc_seq: Buffer, - desc_read: Buffer, - ) -> None: - self._orch = orch + def __init__(self, bound: _BoundSpscQueue, region: WorkerChipOrchRegion) -> None: + self._bound = bound self._region = region - self._layout = layout - self._desc_fields = desc_fields - self._desc_seq = desc_seq - self._desc_read = desc_read - self._state = _QueueState.LIVE - self._input_head = 0 - self._input_tail = 0 - self._output_head = 0 - self._output_tail = 0 - self._input_payload_tail = 0 - self._input_payload_head = 0 - self._output_payload_head = 0 - self._output_active: WorkerChipQueueMessage | None = None - self._stop_published = False - self.input = _L3InputQueue(self) - self.output = _L3OutputQueue(self) + self.input = bound.input + self.output = bound.output @property def region(self) -> WorkerChipOrchRegion: return self._region @property - def layout(self) -> WorkerChipQueueLayout: - return self._layout + def layout(self) -> _SpscQueueLayout: + return self._bound.layout @property def magic_version(self) -> int: - return worker_chip_queue_magic_version() + return int(self._bound.endpoint_binding.magic_version) def chip_task_arg_scalars(self) -> list[int]: - self._ensure_live() - return [ - *self._region.descriptor_scalars(), - self.magic_version, - self._layout.depth, - self._layout.input_arena_bytes, - self._layout.output_arena_bytes, - self._layout.payload_bytes, - self._layout.counter_bytes, - ] + self._bound._ensure_usable() + return list(self._bound.endpoint_binding.to_scalars()) def try_request_stop(self) -> bool: - return self.input._try_enqueue(None, 0, WorkerChipQueueOpcode.STOP) + return self._bound.try_request_stop() def request_stop(self, timeout: float) -> None: - self.input._enqueue(None, 0, WorkerChipQueueOpcode.STOP, timeout) + self._bound.request_stop(timeout) def free(self) -> None: - if self._state == _QueueState.RELEASED: - return - self._state = _QueueState.RELEASED + self._bound.free() self._region.free() - - def _ensure_live(self) -> None: - if self._state == _QueueState.RELEASED: - raise RuntimeError("L3-L2 queue has been released") - if self._state == _QueueState.POISONED_REMOTE: - raise RuntimeError("L3-L2 queue is remote-aborted") - if self._state == _QueueState.POISONED_LOCAL: - raise RuntimeError("L3-L2 queue is poisoned") - if self._state == _QueueState.EXPIRED: - raise RuntimeError("L3-L2 queue expired after orchestration run") - if self._region.expired: - self._state = _QueueState.EXPIRED - raise RuntimeError("L3-L2 queue expired after orchestration run") - self._region._ensure_live() - - def _validate_registered_buffer(self, buffer: Any, nbytes: int) -> Buffer: - if not isinstance(buffer, Buffer): - raise ValueError("L3-L2 queue requires a registered Buffer returned by orch.alloc(...)") - self._region._validate_host_buffer(buffer) - if int(nbytes) > int(buffer.nbytes): - raise ValueError(f"L3-L2 queue nbytes={nbytes} exceeds registered buffer size {int(buffer.nbytes)}") - return buffer - - def _registered_buffer_or_none(self, buffer: Any, nbytes: int) -> Buffer | None: - if not isinstance(buffer, Buffer): - return None - return self._validate_registered_buffer(buffer, nbytes) - - def _refresh_counter(self, offset: int, local_value: int, depth: int) -> int: - result = self._signal_test(offset, local_value & 0xFFFF_FFFF, WaitCmp.NE) - if not result.matched: - return local_value - observed = int(result.observed) & 0xFFFF_FFFF - local_low = local_value & 0xFFFF_FFFF - delta = ctypes.c_int32((observed - local_low) & 0xFFFF_FFFF).value - if delta < 0 or delta > depth: - self._poison_local() - raise RuntimeError("L3-L2 queue counter reconstruction failed") - return local_value + delta - - def _sample_peer_abort_after_timeout(self) -> None: - result = self._signal_test(self._layout.chip_abort_flag_offset, 1, WaitCmp.GE) - if result.matched: - self._state = _QueueState.POISONED_REMOTE - raise RuntimeError("L3-L2 queue remote abort observed") - raise TimeoutError("L3-L2 queue operation timed out") - - def _poison_local(self) -> None: - if self._state != _QueueState.LIVE: - return - self._state = _QueueState.POISONED_LOCAL - try: - self._region._direct_counter_notify(self._layout.worker_abort_flag_offset, 1, NotifyOp.Set) - except Exception: - pass - - def _run_primitive(self, fn: Any, *args: Any, **kwargs: Any) -> Any: - try: - return fn(*args, **kwargs) - except Exception: - self._poison_local() - raise - - def _signal_test(self, offset: int, cmp_value: int, cmp: WaitCmp) -> Any: - return self._run_primitive(lambda: self._region.counter(offset).test(cmp_value, cmp)) - - def _signal_notify(self, offset: int, value: int) -> None: - self._run_primitive(lambda: self._region.counter(offset).notify(value, NotifyOp.Set)) - - def _write_descriptor( - self, offset: int, seq: int, opcode: WorkerChipQueueOpcode, payload_offset: int, nbytes: int - ) -> None: - fields_buf = (ctypes.c_uint8 * 24).from_address(int(self._desc_fields.base)) - fields_buf[:] = _DESC.pack(0, int(opcode), int(payload_offset), int(nbytes))[8:] - seq_buf = (ctypes.c_uint8 * 8).from_address(int(self._desc_seq.base)) - seq_buf[:] = struct.pack(" WorkerChipQueueMessage: - self._run_primitive( - self._region.payload_read, offset, self._desc_read, nbytes=WORKER_CHIP_QUEUE_DESC_SLOT_BYTES - ) - raw = ctypes.string_at(int(self._desc_read.base), WORKER_CHIP_QUEUE_DESC_SLOT_BYTES) - seq, opcode_value, payload_offset, payload_nbytes = _DESC.unpack(raw) - try: - opcode = WorkerChipQueueOpcode(opcode_value) - except ValueError: - self._poison_local() - raise RuntimeError("L3-L2 queue observed invalid descriptor opcode") from None - return WorkerChipQueueMessage( - seq=int(seq), - opcode=opcode, - payload_offset=int(payload_offset), - payload_nbytes=int(payload_nbytes), - ) - - def _advance_payload_head( - self, - cursor: int, - payload_offset: int, - payload_nbytes: int, - arena_offset: int, - arena_bytes: int, - ) -> int: - if payload_nbytes == 0: - return cursor - expected_offset = arena_offset + (cursor % arena_bytes) - if expected_offset != payload_offset: - if payload_offset != arena_offset: - self._poison_local() - raise RuntimeError("L3-L2 queue payload replay offset mismatch") - cursor += arena_bytes - (cursor % arena_bytes) - return cursor + payload_nbytes - - def _replay_released_input_descriptors(self, old_head: int, new_head: int) -> None: - cursor = old_head - while cursor < new_head: - slot_index = cursor & (self._layout.depth - 1) - slot_offset = self._layout.input_desc_offset + slot_index * WORKER_CHIP_QUEUE_DESC_SLOT_BYTES - message = self._read_descriptor(slot_offset) - if message.seq != cursor + 1: - self._poison_local() - raise RuntimeError("L3-L2 queue input release replay seq mismatch") - self._input_payload_head = self._advance_payload_head( - self._input_payload_head, - message.payload_offset, - message.payload_nbytes, - self._layout.input_arena_offset, - self._layout.input_arena_bytes, - ) - cursor += 1 - - -class _L3InputQueue: - def __init__(self, queue: WorkerChipQueue) -> None: - self._queue = queue - - def enqueue(self, buffer_or_none: Any, nbytes: int, timeout: float) -> None: - self._enqueue(buffer_or_none, nbytes, WorkerChipQueueOpcode.DATA, timeout) - - def try_enqueue(self, buffer_or_none: Any, nbytes: int) -> bool: - return self._try_enqueue(buffer_or_none, nbytes, WorkerChipQueueOpcode.DATA) - - def _enqueue(self, buffer_or_none: Any, nbytes: int, opcode: WorkerChipQueueOpcode, timeout: float) -> None: - if timeout is None or float(timeout) <= 0: - raise ValueError("L3-L2 queue blocking operations require a positive timeout") - deadline = time.monotonic() + float(timeout) - while True: - if self._try_enqueue(buffer_or_none, nbytes, opcode): - return - if self._queue._stop_published: - raise RuntimeError("L3-L2 queue input is stopped") - if time.monotonic() >= deadline: - self._queue._sample_peer_abort_after_timeout() - time.sleep(_POLL_INTERVAL_S) - - def _try_enqueue(self, buffer_or_none: Any, nbytes: int, opcode: WorkerChipQueueOpcode) -> bool: - queue = self._queue - nbytes = int(nbytes) - if nbytes < 0: - raise ValueError("L3-L2 queue nbytes must be non-negative") - payload_buffer = self._resolve_payload_source(buffer_or_none, nbytes) - - queue._ensure_live() - if queue._stop_published: - return False - if opcode == WorkerChipQueueOpcode.STOP and nbytes != 0: - raise ValueError("L3-L2 queue STOP must be zero-byte") - if nbytes > queue._layout.input_arena_bytes: - return False - old_head = queue._input_head - queue._input_head = queue._refresh_counter( - queue._layout.input_desc_head_offset, queue._input_head, queue._layout.depth - ) - if queue._input_head != old_head: - queue._replay_released_input_descriptors(old_head, queue._input_head) - if queue._input_tail - queue._input_head >= queue._layout.depth: - return False - - payload_offset = 0 - next_payload_tail = queue._input_payload_tail - if nbytes != 0: - payload_offset, next_payload_tail = self._reserve_payload_range(nbytes, next_payload_tail) - if payload_offset < 0: - return False - queue._run_primitive(queue._region.payload_write, payload_offset, payload_buffer, nbytes=nbytes) - queue._input_payload_tail = next_payload_tail + nbytes - - seq = queue._input_tail + 1 - slot_index = queue._input_tail & (queue._layout.depth - 1) - slot_offset = queue._layout.input_desc_offset + slot_index * WORKER_CHIP_QUEUE_DESC_SLOT_BYTES - queue._write_descriptor(slot_offset, seq, opcode, payload_offset, nbytes) - queue._input_tail += 1 - queue._signal_notify(queue._layout.input_desc_tail_offset, queue._input_tail) - if opcode == WorkerChipQueueOpcode.STOP: - queue._stop_published = True - return True - - def _resolve_payload_source(self, buffer_or_none: Any, nbytes: int) -> Any | None: - if nbytes == 0: - if buffer_or_none is not None: - raise ValueError("L3-L2 queue zero-byte enqueue requires buffer_or_none == None") - return None - payload_tensor = self._queue._registered_buffer_or_none(buffer_or_none, nbytes) - if payload_tensor is not None: - return payload_tensor - _host_byte_span(buffer_or_none, nbytes, writable=False) - return buffer_or_none - - def _reserve_payload_range(self, nbytes: int, next_payload_tail: int) -> tuple[int, int]: - queue = self._queue - arena_pos = next_payload_tail % queue._layout.input_arena_bytes - if arena_pos + nbytes > queue._layout.input_arena_bytes: - next_payload_tail += queue._layout.input_arena_bytes - arena_pos - arena_pos = 0 - if next_payload_tail + nbytes - queue._input_payload_head > queue._layout.input_arena_bytes: - return -1, next_payload_tail - return queue._layout.input_arena_offset + arena_pos, next_payload_tail - - -class _L3OutputQueue: - def __init__(self, queue: WorkerChipQueue) -> None: - self._queue = queue - - def try_peek(self) -> WorkerChipQueueMessage | None: - queue = self._queue - queue._ensure_live() - if queue._output_active is not None: - return queue._output_active - queue._output_tail = queue._refresh_counter( - queue._layout.output_desc_tail_offset, queue._output_tail, queue._layout.depth - ) - if queue._output_tail == queue._output_head: - return None - slot_index = queue._output_head & (queue._layout.depth - 1) - slot_offset = queue._layout.output_desc_offset + slot_index * WORKER_CHIP_QUEUE_DESC_SLOT_BYTES - message = queue._read_descriptor(slot_offset) - if message.seq != queue._output_head + 1: - queue._poison_local() - raise RuntimeError("L3-L2 queue output descriptor seq mismatch") - if message.opcode == WorkerChipQueueOpcode.STOP: - queue._poison_local() - raise RuntimeError("L3-L2 queue output descriptor cannot be STOP") - if message.payload_nbytes == 0: - if message.payload_offset != 0: - queue._poison_local() - raise RuntimeError("L3-L2 queue zero-byte output descriptor has nonzero offset") - else: - begin = queue._layout.output_arena_offset - end = begin + queue._layout.output_arena_bytes - if message.payload_offset < begin or message.payload_offset + message.payload_nbytes > end: - queue._poison_local() - raise RuntimeError("L3-L2 queue output payload outside output arena") - queue._advance_payload_head( - queue._output_payload_head, - message.payload_offset, - message.payload_nbytes, - queue._layout.output_arena_offset, - queue._layout.output_arena_bytes, - ) - queue._output_active = message - return message - - def peek(self, timeout: float) -> WorkerChipQueueMessage: - if timeout is None or float(timeout) <= 0: - raise ValueError("L3-L2 queue blocking operations require a positive timeout") - deadline = time.monotonic() + float(timeout) - while True: - message = self.try_peek() - if message is not None: - return message - if time.monotonic() >= deadline: - self._queue._sample_peer_abort_after_timeout() - time.sleep(_POLL_INTERVAL_S) - - def read_into(self, handle: WorkerChipQueueMessage, buffer: Any) -> None: - queue = self._queue - queue._ensure_live() - if queue._output_active != handle: - raise RuntimeError("L3-L2 queue output handle is not active") - if handle.payload_nbytes == 0: - if buffer is not None: - raise ValueError("L3-L2 queue zero-byte output read requires buffer == None") - return - target = queue._registered_buffer_or_none(buffer, handle.payload_nbytes) - if target is None: - _host_byte_span(buffer, handle.payload_nbytes, writable=True) - target = buffer - queue._run_primitive(queue._region.payload_read, handle.payload_offset, target, nbytes=handle.payload_nbytes) - - def release(self, handle: WorkerChipQueueMessage) -> None: - queue = self._queue - queue._ensure_live() - if queue._output_active != handle: - queue._poison_local() - raise RuntimeError("L3-L2 queue output handle is not active") - queue._output_payload_head = queue._advance_payload_head( - queue._output_payload_head, - handle.payload_offset, - handle.payload_nbytes, - queue._layout.output_arena_offset, - queue._layout.output_arena_bytes, - ) - queue._output_head += 1 - queue._output_active = None - queue._signal_notify(queue._layout.output_desc_head_offset, queue._output_head) - - def dequeue_into(self, buffer: Any, timeout: float) -> WorkerChipQueueMessage: - handle = self.peek(timeout) - self.read_into(handle, buffer) - self.release(handle) - return handle - - def try_dequeue_into(self, buffer: Any) -> WorkerChipQueueMessage | None: - handle = self.try_peek() - if handle is None: - return None - self.read_into(handle, buffer) - self.release(handle) - return handle diff --git a/src/common/platform/include/aicpu/region_instance_view.h b/src/common/platform/include/aicpu/region_instance_view.h index 0e8d71b650..8e173abc19 100644 --- a/src/common/platform/include/aicpu/region_instance_view.h +++ b/src/common/platform/include/aicpu/region_instance_view.h @@ -111,8 +111,13 @@ class RegionInstanceViewImpl { public: class PayloadPart { public: - bool read(uint64_t offset, uint64_t nbytes, RegionPayloadView &out) { - return view_->payload_read(offset, nbytes, out); + template + bool read(uint64_t offset, uint64_t nbytes, View &out) { + RegionPayloadView view{}; + bool ok = view_->payload_read(offset, nbytes, view); + out.local_addr = view.local_addr; + out.nbytes = view.nbytes; + return ok; } bool write(uint64_t offset, const void *src, uint64_t nbytes) { @@ -134,8 +139,13 @@ class RegionInstanceViewImpl { bool notify(uint64_t offset, int32_t value, RegionNotifyOp op) { return view_->notify(offset, value, op); } - bool test(uint64_t offset, int32_t cmp_value, RegionWaitCmp cmp, RegionSignalTestResult &out) { - return view_->test(offset, cmp_value, cmp, out); + template + bool test(uint64_t offset, int32_t cmp_value, RegionWaitCmp cmp, Sample &out) { + RegionSignalTestResult sample{}; + bool ok = view_->test(offset, cmp_value, cmp, sample); + out.matched = sample.matched; + out.observed = sample.observed; + return ok; } bool wait(uint64_t offset, int32_t cmp_value, RegionWaitCmp cmp, uint64_t timeout_ns, int32_t &observed) { diff --git a/src/common/platform/include/common/region_template.h b/src/common/platform/include/common/region_template.h new file mode 100644 index 0000000000..4fdd3a9464 --- /dev/null +++ b/src/common/platform/include/common/region_template.h @@ -0,0 +1,1211 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + * ----------------------------------------------------------------------------------------------------------- + */ + +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +#include "common/region_instance_semantics.h" + +namespace spsc_queue { + +inline constexpr uint32_t kSpscQueueMagic = 0x53505351u; +inline constexpr uint16_t kSpscQueueAbiMajor = 1; +inline constexpr uint16_t kSpscQueueAbiMinor = 0; +inline constexpr uint64_t kSpscQueueMagicVersion = (static_cast(kSpscQueueMagic) << 32) | + (static_cast(kSpscQueueAbiMajor) << 16) | + kSpscQueueAbiMinor; +inline constexpr size_t kSpscQueueEndpointBindingScalarCount = 10; +inline constexpr uint64_t kDescriptorBytes = 32; +inline constexpr uint64_t kArenaAlignment = 64; +inline constexpr uint64_t kDescriptorRingAlignment = 8; +inline constexpr uint64_t kCounterStride = 64; +inline constexpr uint64_t kCounterBytes = 384; +inline constexpr uint64_t kMaxDepth = 1ull << 30; +inline constexpr uint64_t kInputDescTailOffset = 0; +inline constexpr uint64_t kInputDescHeadOffset = 64; +inline constexpr uint64_t kOutputDescTailOffset = 128; +inline constexpr uint64_t kOutputDescHeadOffset = 192; +inline constexpr uint64_t kInitiatorAbortOffset = 256; +inline constexpr uint64_t kPeerAbortOffset = 320; + +enum class SpscQueueOpcode : uint64_t { + INVALID = 0, + DATA = 1, + STOP = 2, + ERROR = 3, +}; + +struct SpscQueueDescriptor { + uint64_t seq; + uint64_t opcode; + uint64_t payload_offset; + uint64_t payload_nbytes; +}; + +struct SpscQueueLayout { + uint64_t depth; + uint64_t input_arena_bytes; + uint64_t output_arena_bytes; + uint64_t input_desc_offset; + uint64_t output_desc_offset; + uint64_t input_arena_offset; + uint64_t output_arena_offset; + uint64_t payload_bytes; + uint64_t input_desc_tail_offset; + uint64_t input_desc_head_offset; + uint64_t output_desc_tail_offset; + uint64_t output_desc_head_offset; + uint64_t initiator_abort_offset; + uint64_t peer_abort_offset; + uint64_t counter_bytes; + + static bool create(uint64_t depth, uint64_t input_arena_bytes, uint64_t output_arena_bytes, SpscQueueLayout *out); +}; + +struct SpscQueueEndpointBinding { + uint64_t magic_version; + uint64_t session_instance_id_bits; + uint64_t transaction_id; + uint64_t payload_base; + uint64_t payload_bytes; + uint64_t counter_base; + uint64_t counter_bytes; + uint64_t depth; + uint64_t input_arena_bytes; + uint64_t output_arena_bytes; +}; + +struct SpscQueuePayloadView { + uint64_t local_addr; + uint64_t nbytes; +}; + +struct SpscQueueCounterSample { + bool matched; + int32_t observed; +}; + +struct SpscQueueInputHandle { + uint64_t seq; + SpscQueueOpcode opcode; + uint64_t payload_offset; + uint64_t payload_nbytes; + SpscQueuePayloadView payload; + uint64_t owner_cookie; +}; + +struct SpscQueueOutputReservation { + uint64_t seq; + uint64_t payload_offset; + uint64_t payload_nbytes; + SpscQueuePayloadView payload; + uint64_t owner_cookie; + bool valid; +}; + +enum class SpscQueueErrorKind : uint32_t { + NONE = 0, + BAD_ARGUMENT = 1, + BAD_BINDING = 2, + INVALID_DESCRIPTOR = 3, + OWNERSHIP = 4, + REMOTE_ABORTED = 5, + ENDPOINT_ERROR = 6, +}; + +enum class SpscQueueOp : uint32_t { + INIT = 1, + TIMEOUT = 2, + INPUT_TRY_PEEK = 3, + INPUT_RELEASE = 4, + OUTPUT_TRY_RESERVE = 5, + OUTPUT_PUBLISH = 6, +}; + +struct SpscQueueError { + SpscQueueErrorKind kind; + SpscQueueOp op; + uint64_t session_instance_id_bits; + uint64_t transaction_id; + char message[256]; +}; + +struct MonotonicClock { + uint64_t (*now_ns)(); +}; + +inline bool is_power_of_two(uint64_t value) { return value != 0 && (value & (value - 1)) == 0; } + +inline bool mul_overflows(uint64_t a, uint64_t b) { +#if defined(__clang__) || defined(__GNUC__) + uint64_t result = 0; + return __builtin_mul_overflow(a, b, &result); +#else + return a != 0 && b > UINT64_MAX / a; +#endif +} + +inline bool add_overflows(uint64_t a, uint64_t b) { return region_add_overflows(a, b); } + +inline bool align_up(uint64_t value, uint64_t align, uint64_t *out) { + if (out == nullptr || align == 0) { + return false; + } + uint64_t remainder = value % align; + uint64_t bump = remainder == 0 ? 0 : align - remainder; + if (add_overflows(value, bump)) { + return false; + } + *out = value + bump; + return true; +} + +inline void store_u64_le(uint8_t *dst, uint64_t value) { + for (size_t index = 0; index < 8; ++index) { + dst[index] = static_cast((value >> (8 * index)) & 0xff); + } +} + +inline uint64_t load_u64_le(const uint8_t *src) { + uint64_t value = 0; + for (size_t index = 0; index < 8; ++index) { + value |= static_cast(src[index]) << (8 * index); + } + return value; +} + +inline bool session_instance_id_to_bits(const uint8_t *bytes, size_t nbytes, uint64_t *out) { + if (bytes == nullptr || out == nullptr || nbytes != 8) { + return false; + } + *out = load_u64_le(bytes); + return true; +} + +inline bool session_instance_id_from_bits(uint64_t bits, uint8_t *out, size_t nbytes) { + if (out == nullptr || nbytes != 8) { + return false; + } + store_u64_le(out, bits); + return true; +} + +inline bool encode_descriptor(const SpscQueueDescriptor &descriptor, uint8_t *out, size_t nbytes) { + if (out == nullptr || nbytes != kDescriptorBytes) { + return false; + } + store_u64_le(out + 0, descriptor.seq); + store_u64_le(out + 8, descriptor.opcode); + store_u64_le(out + 16, descriptor.payload_offset); + store_u64_le(out + 24, descriptor.payload_nbytes); + return true; +} + +inline bool decode_descriptor(const uint8_t *src, size_t nbytes, SpscQueueDescriptor *out) { + if (src == nullptr || out == nullptr || nbytes != kDescriptorBytes) { + return false; + } + SpscQueueDescriptor decoded{ + load_u64_le(src + 0), + load_u64_le(src + 8), + load_u64_le(src + 16), + load_u64_le(src + 24), + }; + *out = decoded; + return true; +} + +inline bool encode_endpoint_binding(const SpscQueueEndpointBinding &binding, uint64_t *out, size_t count) { + if (out == nullptr || count != kSpscQueueEndpointBindingScalarCount || + binding.magic_version != kSpscQueueMagicVersion) { + return false; + } + out[0] = binding.magic_version; + out[1] = binding.session_instance_id_bits; + out[2] = binding.transaction_id; + out[3] = binding.payload_base; + out[4] = binding.payload_bytes; + out[5] = binding.counter_base; + out[6] = binding.counter_bytes; + out[7] = binding.depth; + out[8] = binding.input_arena_bytes; + out[9] = binding.output_arena_bytes; + return true; +} + +inline bool decode_endpoint_binding(const uint64_t *scalars, size_t count, SpscQueueEndpointBinding *out) { + if (scalars == nullptr || out == nullptr || count != kSpscQueueEndpointBindingScalarCount || + scalars[0] != kSpscQueueMagicVersion) { + return false; + } + SpscQueueEndpointBinding decoded{ + scalars[0], scalars[1], scalars[2], scalars[3], scalars[4], + scalars[5], scalars[6], scalars[7], scalars[8], scalars[9], + }; + *out = decoded; + return true; +} + +inline const char *spsc_queue_op_name(SpscQueueOp op) { + switch (op) { + case SpscQueueOp::INIT: + return "init"; + case SpscQueueOp::TIMEOUT: + return "timeout"; + case SpscQueueOp::INPUT_TRY_PEEK: + return "input.try_peek"; + case SpscQueueOp::INPUT_RELEASE: + return "input.release"; + case SpscQueueOp::OUTPUT_TRY_RESERVE: + return "output.try_reserve"; + case SpscQueueOp::OUTPUT_PUBLISH: + return "output.publish"; + } + return "unknown"; +} + +inline bool valid_input_opcode(SpscQueueOpcode opcode) { + return opcode == SpscQueueOpcode::DATA || opcode == SpscQueueOpcode::STOP; +} + +inline bool valid_output_opcode(SpscQueueOpcode opcode) { + return opcode == SpscQueueOpcode::DATA || opcode == SpscQueueOpcode::ERROR; +} + +inline int32_t counter_low32(uint64_t value) { return static_cast(static_cast(value)); } + +inline bool reconstruct_counter(int32_t observed_low32, uint64_t depth, uint64_t *local_value) { + if (local_value == nullptr || depth == 0 || depth > kMaxDepth) { + return false; + } + uint32_t local_low32 = static_cast(*local_value); + int32_t delta = static_cast(static_cast(observed_low32) - local_low32); + if (delta < 0 || static_cast(delta) > depth) { + return false; + } + *local_value += static_cast(delta); + return true; +} + +inline void copy_error_text(char *dst, size_t dst_size, const char *src) { + if (dst == nullptr || dst_size == 0) { + return; + } + const char *in = src == nullptr ? "" : src; + size_t n = strnlen(in, dst_size - 1); + memcpy(dst, in, n); + dst[n] = '\0'; +} + +inline void append_error_note(char *dst, size_t dst_size, const char *note) { + if (dst == nullptr || dst_size == 0 || note == nullptr || note[0] == '\0') { + return; + } + size_t len = strnlen(dst, dst_size); + if (len + 4 >= dst_size) { + return; + } + int written = snprintf(dst + len, dst_size - len, " (%s)", note); + (void)written; +} + +inline bool payload_in_arena(uint64_t offset, uint64_t nbytes, uint64_t arena_offset, uint64_t arena_bytes) { + if (nbytes == 0 || add_overflows(offset, nbytes) || add_overflows(arena_offset, arena_bytes)) { + return false; + } + return offset >= arena_offset && offset + nbytes <= arena_offset + arena_bytes; +} + +inline uint64_t payload_expected_offset(uint64_t cursor, uint64_t nbytes, uint64_t arena_offset, uint64_t arena_bytes) { + uint64_t arena_pos = cursor % arena_bytes; + return arena_pos + nbytes > arena_bytes ? arena_offset : arena_offset + arena_pos; +} + +inline bool +SpscQueueLayout::create(uint64_t depth, uint64_t input_arena_bytes, uint64_t output_arena_bytes, SpscQueueLayout *out) { + if (out == nullptr || !is_power_of_two(depth) || depth > kMaxDepth || input_arena_bytes == 0 || + output_arena_bytes == 0 || input_arena_bytes % kArenaAlignment != 0 || + output_arena_bytes % kArenaAlignment != 0) { + return false; + } + if (mul_overflows(depth, kDescriptorBytes)) { + return false; + } + uint64_t desc_ring_bytes = depth * kDescriptorBytes; + uint64_t input_desc_offset = 0; + if (add_overflows(input_desc_offset, desc_ring_bytes)) { + return false; + } + uint64_t output_desc_offset = input_desc_offset + desc_ring_bytes; + if (add_overflows(output_desc_offset, desc_ring_bytes)) { + return false; + } + uint64_t desc_end = output_desc_offset + desc_ring_bytes; + uint64_t input_arena_offset = 0; + if (!align_up(desc_end, kArenaAlignment, &input_arena_offset)) { + return false; + } + if (add_overflows(input_arena_offset, input_arena_bytes)) { + return false; + } + uint64_t input_arena_end = input_arena_offset + input_arena_bytes; + uint64_t output_arena_offset = 0; + if (!align_up(input_arena_end, kArenaAlignment, &output_arena_offset)) { + return false; + } + if (add_overflows(output_arena_offset, output_arena_bytes)) { + return false; + } + uint64_t payload_bytes = output_arena_offset + output_arena_bytes; + if (output_desc_offset % kDescriptorRingAlignment != 0 || input_arena_offset % kArenaAlignment != 0 || + output_arena_offset % kArenaAlignment != 0) { + return false; + } + *out = SpscQueueLayout{ + depth, + input_arena_bytes, + output_arena_bytes, + input_desc_offset, + output_desc_offset, + input_arena_offset, + output_arena_offset, + payload_bytes, + kInputDescTailOffset, + kInputDescHeadOffset, + kOutputDescTailOffset, + kOutputDescHeadOffset, + kInitiatorAbortOffset, + kPeerAbortOffset, + kCounterBytes, + }; + return true; +} + +template +class SpscQueueEndpoint { + static_assert(MaxInflight >= 1, "MaxInflight must be at least 1"); + static_assert(MaxInflight <= kMaxDepth, "MaxInflight exceeds kMaxDepth"); + +public: + static constexpr uint64_t kEntryCapacity = MaxInflight + 1; + + class InputQueue { + struct ActiveInputEntry { + uint64_t seq; + SpscQueueOpcode opcode; + uint64_t payload_offset; + uint64_t payload_nbytes; + SpscQueuePayloadView payload; + bool completed; + }; + + public: + explicit InputQueue(SpscQueueEndpoint *parent) : + parent_(parent) {} + + InputQueue(const InputQueue &) = delete; + InputQueue &operator=(const InputQueue &) = delete; + InputQueue(InputQueue &&) = delete; + InputQueue &operator=(InputQueue &&) = delete; + + bool peek(uint64_t timeout_ns, SpscQueueInputHandle &out) { + out = SpscQueueInputHandle{}; + if (!parent_->ensure_live()) { + return false; + } + if (timeout_ns == 0) { + parent_->set_error( + SpscQueueErrorKind::BAD_ARGUMENT, SpscQueueOp::INPUT_TRY_PEEK, + "blocking operations require a positive timeout" + ); + return false; + } + uint64_t now = parent_->clock_.now_ns(); + uint64_t deadline = add_overflows(now, timeout_ns) ? UINT64_MAX : now + timeout_ns; + while (true) { + if (try_peek(out)) { + return true; + } + if (parent_->error_.kind != SpscQueueErrorKind::NONE) { + return false; + } + if (!parent_->wait_progress( + parent_->layout_.input_desc_tail_offset, input_tail_, deadline, SpscQueueOp::INPUT_TRY_PEEK + )) { + return false; + } + } + } + + bool try_peek(SpscQueueInputHandle &out) { + out = SpscQueueInputHandle{}; + if (!parent_->ensure_live()) { + return false; + } + const SpscQueueLayout &layout = parent_->layout_; + if (!parent_->refresh_counter(layout.input_desc_tail_offset, input_tail_, SpscQueueOp::INPUT_TRY_PEEK)) { + return false; + } + if (stop_observed_) { + if (input_tail_ != input_acquire_) { + parent_->poison( + SpscQueueErrorKind::INVALID_DESCRIPTOR, SpscQueueOp::INPUT_TRY_PEEK, + "input descriptor published after STOP" + ); + } + return false; + } + if (input_tail_ == input_acquire_) { + return false; + } + if (input_tail_ - input_head_ > layout.depth || input_acquire_ < input_head_ || + input_acquire_ > input_tail_) { + parent_->poison( + SpscQueueErrorKind::INVALID_DESCRIPTOR, SpscQueueOp::INPUT_TRY_PEEK, + "input descriptor state invalid" + ); + return false; + } + + SpscQueueDescriptor slot{}; + uint64_t slot_index = input_acquire_ & (layout.depth - 1); + uint64_t slot_offset = layout.input_desc_offset + slot_index * kDescriptorBytes; + if (!parent_->read_descriptor(slot_offset, &slot, SpscQueueOp::INPUT_TRY_PEEK)) { + return false; + } + uint64_t expected_seq = input_acquire_ + 1; + if (slot.seq != expected_seq) { + parent_->poison( + SpscQueueErrorKind::INVALID_DESCRIPTOR, SpscQueueOp::INPUT_TRY_PEEK, "input descriptor seq mismatch" + ); + return false; + } + SpscQueueOpcode opcode = static_cast(slot.opcode); + if (!valid_input_opcode(opcode)) { + parent_->poison( + SpscQueueErrorKind::INVALID_DESCRIPTOR, SpscQueueOp::INPUT_TRY_PEEK, "invalid input opcode" + ); + return false; + } + if (opcode == SpscQueueOpcode::STOP && (slot.payload_offset != 0 || slot.payload_nbytes != 0)) { + parent_->poison( + SpscQueueErrorKind::INVALID_DESCRIPTOR, SpscQueueOp::INPUT_TRY_PEEK, + "STOP descriptor must be zero-byte" + ); + return false; + } + bool counts_against_window = opcode == SpscQueueOpcode::DATA; + if (counts_against_window && active_non_stop_count_ >= MaxInflight) { + return false; + } + if (active_count_ >= kEntryCapacity) { + parent_->poison(SpscQueueErrorKind::OWNERSHIP, SpscQueueOp::INPUT_TRY_PEEK, "input window state full"); + return false; + } + + SpscQueuePayloadView view{0, 0}; + if (slot.payload_nbytes == 0) { + if (slot.payload_offset != 0) { + parent_->poison( + SpscQueueErrorKind::INVALID_DESCRIPTOR, SpscQueueOp::INPUT_TRY_PEEK, + "zero-byte descriptor uses nonzero payload offset" + ); + return false; + } + } else if (!payload_in_arena( + slot.payload_offset, slot.payload_nbytes, layout.input_arena_offset, layout.input_arena_bytes + )) { + parent_->poison( + SpscQueueErrorKind::INVALID_DESCRIPTOR, SpscQueueOp::INPUT_TRY_PEEK, "input payload out of arena" + ); + return false; + } else if (!parent_->payload_matches_head( + input_payload_acquire_head_, slot.payload_offset, slot.payload_nbytes, + layout.input_arena_offset, layout.input_arena_bytes, SpscQueueOp::INPUT_TRY_PEEK + )) { + return false; + } else if (!parent_->payload_read( + slot.payload_offset, slot.payload_nbytes, view, SpscQueueOp::INPUT_TRY_PEEK + )) { + return false; + } else { + parent_->advance_payload_head( + input_payload_acquire_head_, slot.payload_offset, slot.payload_nbytes, layout.input_arena_offset, + layout.input_arena_bytes, SpscQueueOp::INPUT_TRY_PEEK + ); + if (parent_->error_.kind != SpscQueueErrorKind::NONE) { + return false; + } + } + + out = SpscQueueInputHandle{ + slot.seq, opcode, slot.payload_offset, slot.payload_nbytes, view, parent_->owner_cookie() + }; + uint64_t insert_index = (active_head_ + active_count_) % kEntryCapacity; + active_entries_[insert_index] = + ActiveInputEntry{slot.seq, opcode, slot.payload_offset, slot.payload_nbytes, view, false}; + active_count_ += 1; + if (counts_against_window) { + active_non_stop_count_ += 1; + } + input_acquire_ += 1; + if (opcode == SpscQueueOpcode::STOP) { + stop_observed_ = true; + if (input_tail_ != input_acquire_) { + parent_->poison( + SpscQueueErrorKind::INVALID_DESCRIPTOR, SpscQueueOp::INPUT_TRY_PEEK, + "input descriptor published after STOP" + ); + return false; + } + } + return true; + } + + bool release(const SpscQueueInputHandle &handle) { + if (!parent_->ensure_live()) { + return false; + } + if (handle.owner_cookie != parent_->owner_cookie()) { + parent_->poison( + SpscQueueErrorKind::OWNERSHIP, SpscQueueOp::INPUT_RELEASE, "input handle is not active" + ); + return false; + } + ActiveInputEntry *entry = entry_for_seq(handle.seq); + if (entry == nullptr || handle.opcode != entry->opcode || handle.payload_offset != entry->payload_offset || + handle.payload_nbytes != entry->payload_nbytes || + handle.payload.local_addr != entry->payload.local_addr || + handle.payload.nbytes != entry->payload.nbytes) { + parent_->poison( + SpscQueueErrorKind::OWNERSHIP, SpscQueueOp::INPUT_RELEASE, "input handle is not active" + ); + return false; + } + if (entry->completed) { + parent_->poison( + SpscQueueErrorKind::OWNERSHIP, SpscQueueOp::INPUT_RELEASE, "input handle already released" + ); + return false; + } + entry->completed = true; + return release_completed_prefix(); + } + + bool drained() const { return drained_; } + + private: + friend class SpscQueueEndpoint; + + void initialize() { + for (uint64_t i = 0; i < kEntryCapacity; ++i) { + active_entries_[i] = ActiveInputEntry{}; + } + active_head_ = 0; + active_count_ = 0; + active_non_stop_count_ = 0; + input_head_ = 0; + input_tail_ = 0; + input_payload_head_ = 0; + input_payload_acquire_head_ = 0; + input_acquire_ = 0; + stop_observed_ = false; + drained_ = false; + } + + ActiveInputEntry *entry_for_seq(uint64_t seq) { + uint64_t first_seq = input_head_ + 1; + if (seq < first_seq) { + return nullptr; + } + uint64_t ordinal = seq - first_seq; + if (ordinal >= active_count_) { + return nullptr; + } + uint64_t index = (active_head_ + ordinal) % kEntryCapacity; + return active_entries_[index].seq == seq ? &active_entries_[index] : nullptr; + } + + bool release_completed_prefix() { + while (active_count_ != 0 && active_entries_[active_head_].completed) { + ActiveInputEntry entry = active_entries_[active_head_]; + if (entry.payload_nbytes != 0) { + parent_->advance_payload_head( + input_payload_head_, entry.payload_offset, entry.payload_nbytes, + parent_->layout_.input_arena_offset, parent_->layout_.input_arena_bytes, + SpscQueueOp::INPUT_RELEASE + ); + if (parent_->error_.kind != SpscQueueErrorKind::NONE) { + return false; + } + } + input_head_ += 1; + if (entry.opcode == SpscQueueOpcode::DATA) { + active_non_stop_count_ -= 1; + } + if (entry.opcode == SpscQueueOpcode::STOP) { + drained_ = true; + } + active_entries_[active_head_] = ActiveInputEntry{}; + active_head_ = (active_head_ + 1) % kEntryCapacity; + active_count_ -= 1; + if (!parent_->notify_counter( + parent_->layout_.input_desc_head_offset, input_head_, SpscQueueOp::INPUT_RELEASE + )) { + return false; + } + } + return true; + } + + SpscQueueEndpoint *parent_; + ActiveInputEntry active_entries_[kEntryCapacity]{}; + uint64_t active_head_{0}; + uint64_t active_count_{0}; + uint64_t active_non_stop_count_{0}; + uint64_t input_head_{0}; + uint64_t input_tail_{0}; + uint64_t input_payload_head_{0}; + uint64_t input_payload_acquire_head_{0}; + uint64_t input_acquire_{0}; + bool stop_observed_{false}; + bool drained_{false}; + }; + + class OutputQueue { + public: + explicit OutputQueue(SpscQueueEndpoint *parent) : + parent_(parent) {} + + OutputQueue(const OutputQueue &) = delete; + OutputQueue &operator=(const OutputQueue &) = delete; + OutputQueue(OutputQueue &&) = delete; + OutputQueue &operator=(OutputQueue &&) = delete; + + bool reserve(uint64_t nbytes, uint64_t timeout_ns, SpscQueueOutputReservation &out) { + out = SpscQueueOutputReservation{}; + if (!parent_->ensure_live()) { + return false; + } + if (nbytes > parent_->layout_.output_arena_bytes) { + return false; + } + if (timeout_ns == 0) { + parent_->set_error( + SpscQueueErrorKind::BAD_ARGUMENT, SpscQueueOp::OUTPUT_TRY_RESERVE, + "blocking operations require a positive timeout" + ); + return false; + } + uint64_t now = parent_->clock_.now_ns(); + uint64_t deadline = add_overflows(now, timeout_ns) ? UINT64_MAX : now + timeout_ns; + while (true) { + if (try_reserve(nbytes, out)) { + return true; + } + if (parent_->error_.kind != SpscQueueErrorKind::NONE) { + return false; + } + if (!parent_->wait_progress( + parent_->layout_.output_desc_head_offset, output_head_, deadline, + SpscQueueOp::OUTPUT_TRY_RESERVE + )) { + return false; + } + } + } + + bool try_reserve(uint64_t nbytes, SpscQueueOutputReservation &out) { + out = SpscQueueOutputReservation{}; + if (!parent_->ensure_live()) { + return false; + } + const SpscQueueLayout &layout = parent_->layout_; + if (nbytes > layout.output_arena_bytes) { + return false; + } + if (reservation_active_) { + parent_->poison( + SpscQueueErrorKind::OWNERSHIP, SpscQueueOp::OUTPUT_TRY_RESERVE, "output reservation already active" + ); + return false; + } + uint64_t old_head = output_head_; + if (!parent_->refresh_counter( + layout.output_desc_head_offset, output_head_, SpscQueueOp::OUTPUT_TRY_RESERVE + )) { + return false; + } + if (output_head_ != old_head && + !replay_output_releases(old_head, output_head_, SpscQueueOp::OUTPUT_TRY_RESERVE)) { + return false; + } + if (output_tail_ - output_head_ >= layout.depth) { + return false; + } + + uint64_t payload_offset = 0; + SpscQueuePayloadView view{0, 0}; + uint64_t next_payload_tail = output_payload_tail_; + if (nbytes != 0) { + uint64_t arena_base = layout.output_arena_offset; + uint64_t arena_bytes = layout.output_arena_bytes; + uint64_t arena_pos = next_payload_tail % arena_bytes; + if (arena_pos + nbytes > arena_bytes) { + next_payload_tail += arena_bytes - arena_pos; + arena_pos = 0; + } + if (next_payload_tail + nbytes - output_payload_head_ > arena_bytes) { + return false; + } + payload_offset = arena_base + arena_pos; + view = SpscQueuePayloadView{parent_->view_.payload().span().base + payload_offset, nbytes}; + next_payload_tail += nbytes; + } + + reservation_active_ = true; + reservation_seq_ = output_tail_ + 1; + reservation_offset_ = payload_offset; + reservation_nbytes_ = nbytes; + output_payload_tail_ = next_payload_tail; + out = SpscQueueOutputReservation{ + reservation_seq_, payload_offset, nbytes, view, parent_->owner_cookie(), true + }; + return true; + } + + bool publish(const SpscQueueOutputReservation &reservation, SpscQueueOpcode opcode) { + if (!parent_->ensure_live()) { + return false; + } + if (!reservation_active_ || !reservation.valid || reservation.owner_cookie != parent_->owner_cookie() || + reservation.seq != reservation_seq_ || reservation.payload_offset != reservation_offset_ || + reservation.payload_nbytes != reservation_nbytes_) { + parent_->poison( + SpscQueueErrorKind::OWNERSHIP, SpscQueueOp::OUTPUT_PUBLISH, "unknown output reservation" + ); + return false; + } + if (!valid_output_opcode(opcode)) { + parent_->poison( + SpscQueueErrorKind::INVALID_DESCRIPTOR, SpscQueueOp::OUTPUT_PUBLISH, "invalid output opcode" + ); + return false; + } + uint64_t slot_index = output_tail_ & (parent_->layout_.depth - 1); + uint64_t slot_offset = parent_->layout_.output_desc_offset + slot_index * kDescriptorBytes; + if (!parent_->write_descriptor( + slot_offset, reservation.seq, opcode, reservation.payload_offset, reservation.payload_nbytes, + SpscQueueOp::OUTPUT_PUBLISH + )) { + return false; + } + output_tail_ += 1; + reservation_active_ = false; + reservation_seq_ = 0; + reservation_offset_ = 0; + reservation_nbytes_ = 0; + return parent_->notify_counter( + parent_->layout_.output_desc_tail_offset, output_tail_, SpscQueueOp::OUTPUT_PUBLISH + ); + } + + private: + friend class SpscQueueEndpoint; + + void initialize() { + output_head_ = 0; + output_tail_ = 0; + output_payload_head_ = 0; + output_payload_tail_ = 0; + reservation_active_ = false; + reservation_seq_ = 0; + reservation_offset_ = 0; + reservation_nbytes_ = 0; + } + + bool replay_output_releases(uint64_t old_head, uint64_t new_head, SpscQueueOp op) { + uint64_t cursor = old_head; + while (cursor < new_head) { + SpscQueueDescriptor slot{}; + uint64_t slot_index = cursor & (parent_->layout_.depth - 1); + uint64_t slot_offset = parent_->layout_.output_desc_offset + slot_index * kDescriptorBytes; + if (!parent_->read_descriptor(slot_offset, &slot, op)) { + return false; + } + if (slot.seq != cursor + 1) { + parent_->poison(SpscQueueErrorKind::INVALID_DESCRIPTOR, op, "output release replay seq mismatch"); + return false; + } + if (slot.payload_nbytes != 0) { + parent_->advance_payload_head( + output_payload_head_, slot.payload_offset, slot.payload_nbytes, + parent_->layout_.output_arena_offset, parent_->layout_.output_arena_bytes, op + ); + if (parent_->error_.kind != SpscQueueErrorKind::NONE) { + return false; + } + } + cursor += 1; + } + return true; + } + + SpscQueueEndpoint *parent_; + uint64_t output_head_{0}; + uint64_t output_tail_{0}; + uint64_t output_payload_head_{0}; + uint64_t output_payload_tail_{0}; + bool reservation_active_{false}; + uint64_t reservation_seq_{0}; + uint64_t reservation_offset_{0}; + uint64_t reservation_nbytes_{0}; + }; + + SpscQueueEndpoint(const SpscQueueEndpointBinding &binding, RegionView view, MonotonicClock clock) : + view_(std::move(view)), + clock_(clock), + input_queue_(this), + output_queue_(this) { + construct(binding); + } + + SpscQueueEndpoint(const SpscQueueEndpoint &) = delete; + SpscQueueEndpoint &operator=(const SpscQueueEndpoint &) = delete; + SpscQueueEndpoint(SpscQueueEndpoint &&) = delete; + SpscQueueEndpoint &operator=(SpscQueueEndpoint &&) = delete; + + const SpscQueueError &error() const { return error_; } + const SpscQueueLayout &layout() const { return layout_; } + bool live() const { return live_; } + InputQueue &input() { return input_queue_; } + OutputQueue &output() { return output_queue_; } + +private: + uint64_t owner_cookie() const { return static_cast(reinterpret_cast(this)); } + + void construct(const SpscQueueEndpointBinding &binding) { + if (binding.magic_version != kSpscQueueMagicVersion) { + identity_trusted_ = false; + set_error(SpscQueueErrorKind::BAD_BINDING, SpscQueueOp::INIT, "invalid queue binding"); + return; + } + identity_trusted_ = true; + error_.session_instance_id_bits = binding.session_instance_id_bits; + error_.transaction_id = binding.transaction_id; + + if (clock_.now_ns == nullptr) { + set_error(SpscQueueErrorKind::BAD_ARGUMENT, SpscQueueOp::INIT, "clock is null"); + return; + } + if (view_.failed()) { + set_error(SpscQueueErrorKind::ENDPOINT_ERROR, SpscQueueOp::INIT, view_.error().message); + return; + } + + RegionPartLocalSpan payload_span = view_.payload().span(); + RegionPartLocalSpan counter_span = view_.counter().span(); + if (binding.payload_base == 0 || binding.payload_bytes == 0 || binding.counter_base == 0 || + binding.counter_bytes == 0 || payload_span.base != binding.payload_base || + payload_span.logical_bytes != binding.payload_bytes || counter_span.base != binding.counter_base || + counter_span.logical_bytes != binding.counter_bytes) { + set_error(SpscQueueErrorKind::BAD_BINDING, SpscQueueOp::INIT, "view spans do not match binding"); + return; + } + if (!SpscQueueLayout::create(binding.depth, binding.input_arena_bytes, binding.output_arena_bytes, &layout_)) { + set_error(SpscQueueErrorKind::BAD_BINDING, SpscQueueOp::INIT, "invalid queue layout"); + return; + } + if (layout_.payload_bytes != binding.payload_bytes || layout_.counter_bytes != binding.counter_bytes) { + set_error( + SpscQueueErrorKind::BAD_BINDING, SpscQueueOp::INIT, "reconstructed layout does not match binding" + ); + return; + } + if (MaxInflight > layout_.depth) { + set_error(SpscQueueErrorKind::BAD_ARGUMENT, SpscQueueOp::INIT, "invalid input window"); + return; + } + + input_queue_.initialize(); + output_queue_.initialize(); + live_ = true; + } + + bool ensure_live() const { return live_ && error_.kind == SpscQueueErrorKind::NONE; } + + void format_error_message(const char *detail) { + const char *text = detail == nullptr ? "" : detail; + if (identity_trusted_) { + snprintf( + error_.message, sizeof(error_.message), + "SPSC queue endpoint error op=%s kind=%" PRIu32 " session=0x%016" PRIx64 " transaction=%" PRIu64 + " msg=%s", + spsc_queue_op_name(error_.op), static_cast(error_.kind), error_.session_instance_id_bits, + error_.transaction_id, text + ); + } else { + snprintf( + error_.message, sizeof(error_.message), "SPSC queue endpoint error op=%s kind=%" PRIu32 " msg=%s", + spsc_queue_op_name(error_.op), static_cast(error_.kind), text + ); + } + } + + void set_error(SpscQueueErrorKind kind, SpscQueueOp op, const char *message) { + if (error_.kind != SpscQueueErrorKind::NONE) { + return; + } + live_ = false; + error_.kind = kind; + error_.op = op; + format_error_message(message); + } + + void poison(SpscQueueErrorKind kind, SpscQueueOp op, const char *message) { + bool first = error_.kind == SpscQueueErrorKind::NONE; + set_error(kind, op, message); + if (!first || kind == SpscQueueErrorKind::REMOTE_ABORTED) { + return; + } + if (!view_.counter().notify(layout_.peer_abort_offset, 1, RegionNotifyOp::Set)) { + append_error_note(error_.message, sizeof(error_.message), "abort notify failed"); + } + } + + bool sample_peer_abort(SpscQueueOp op) { + SpscQueueCounterSample sample{}; + if (!view_.counter().test(layout_.initiator_abort_offset, 1, RegionWaitCmp::GE, sample)) { + if (view_.failed()) { + poison(SpscQueueErrorKind::ENDPOINT_ERROR, op, view_.error().message); + } else { + poison(SpscQueueErrorKind::ENDPOINT_ERROR, op, "peer abort sample failed"); + } + return false; + } + if (sample.matched) { + poison(SpscQueueErrorKind::REMOTE_ABORTED, SpscQueueOp::TIMEOUT, "remote abort"); + return false; + } + return true; + } + + bool wait_progress(uint64_t offset, uint64_t local_value, uint64_t deadline, SpscQueueOp op) { + if (!sample_peer_abort(op)) { + return false; + } + uint64_t now = clock_.now_ns(); + if (now >= deadline) { + if (!sample_peer_abort(SpscQueueOp::TIMEOUT)) { + return false; + } + return false; + } + uint64_t remaining = deadline - now; + int32_t observed = 0; + bool woke = view_.counter().wait(offset, counter_low32(local_value), RegionWaitCmp::NE, remaining, observed); + if (view_.failed()) { + poison(SpscQueueErrorKind::ENDPOINT_ERROR, op, view_.error().message); + return false; + } + if (woke) { + return sample_peer_abort(op); + } + if (!sample_peer_abort(SpscQueueOp::TIMEOUT)) { + return false; + } + return false; + } + + bool refresh_counter(uint64_t offset, uint64_t &local, SpscQueueOp op) { + SpscQueueCounterSample sample{}; + if (!view_.counter().test(offset, counter_low32(local), RegionWaitCmp::NE, sample)) { + if (view_.failed()) { + poison(SpscQueueErrorKind::ENDPOINT_ERROR, op, view_.error().message); + } else { + poison(SpscQueueErrorKind::ENDPOINT_ERROR, op, "counter test failed"); + } + return false; + } + if (!sample.matched) { + return true; + } + if (!reconstruct_counter(sample.observed, layout_.depth, &local)) { + poison(SpscQueueErrorKind::INVALID_DESCRIPTOR, op, "counter reconstruction failed"); + return false; + } + return true; + } + + bool notify_counter(uint64_t offset, uint64_t value, SpscQueueOp op) { + if (!view_.counter().notify(offset, counter_low32(value), RegionNotifyOp::Set)) { + if (view_.failed()) { + poison(SpscQueueErrorKind::ENDPOINT_ERROR, op, view_.error().message); + } else { + poison(SpscQueueErrorKind::ENDPOINT_ERROR, op, "counter notify failed"); + } + return false; + } + return true; + } + + bool payload_read(uint64_t offset, uint64_t nbytes, SpscQueuePayloadView &out, SpscQueueOp op) { + out = SpscQueuePayloadView{0, 0}; + if (!view_.payload().read(offset, nbytes, out)) { + if (view_.failed()) { + poison(SpscQueueErrorKind::ENDPOINT_ERROR, op, view_.error().message); + } else { + poison(SpscQueueErrorKind::ENDPOINT_ERROR, op, "payload read failed"); + } + return false; + } + return true; + } + + bool payload_write(uint64_t offset, const void *src, uint64_t nbytes, SpscQueueOp op) { + if (!view_.payload().write(offset, src, nbytes)) { + if (view_.failed()) { + poison(SpscQueueErrorKind::ENDPOINT_ERROR, op, view_.error().message); + } else { + poison(SpscQueueErrorKind::ENDPOINT_ERROR, op, "payload write failed"); + } + return false; + } + return true; + } + + bool read_descriptor(uint64_t slot_offset, SpscQueueDescriptor *slot, SpscQueueOp op) { + SpscQueuePayloadView view{}; + if (!payload_read(slot_offset, kDescriptorBytes, view, op)) { + return false; + } + uint8_t bytes[kDescriptorBytes]; + memcpy(bytes, reinterpret_cast(static_cast(view.local_addr)), kDescriptorBytes); + return decode_descriptor(bytes, kDescriptorBytes, slot); + } + + bool write_descriptor( + uint64_t slot_offset, uint64_t seq, SpscQueueOpcode opcode, uint64_t payload_offset, uint64_t payload_nbytes, + SpscQueueOp op + ) { + uint8_t encoded[kDescriptorBytes]; + SpscQueueDescriptor descriptor{0, static_cast(opcode), payload_offset, payload_nbytes}; + if (!encode_descriptor(descriptor, encoded, kDescriptorBytes)) { + poison(SpscQueueErrorKind::INVALID_DESCRIPTOR, op, "descriptor encode failed"); + return false; + } + if (!payload_write(slot_offset + 8, encoded + 8, 24, op)) { + return false; + } + store_u64_le(encoded, seq); + return payload_write(slot_offset, encoded, 8, op); + } + + bool payload_matches_head( + uint64_t cursor, uint64_t payload_offset, uint64_t nbytes, uint64_t arena_offset, uint64_t arena_bytes, + SpscQueueOp op + ) { + if (nbytes == 0) { + return true; + } + uint64_t expected_offset = payload_expected_offset(cursor, nbytes, arena_offset, arena_bytes); + if (payload_offset != expected_offset) { + poison(SpscQueueErrorKind::INVALID_DESCRIPTOR, op, "payload replay offset mismatch"); + return false; + } + return true; + } + + void advance_payload_head( + uint64_t &cursor, uint64_t payload_offset, uint64_t nbytes, uint64_t arena_offset, uint64_t arena_bytes, + SpscQueueOp op + ) { + if (nbytes == 0) { + return; + } + uint64_t expected_offset = payload_expected_offset(cursor, nbytes, arena_offset, arena_bytes); + if (expected_offset != payload_offset) { + poison(SpscQueueErrorKind::INVALID_DESCRIPTOR, op, "payload replay offset mismatch"); + return; + } + uint64_t arena_pos = cursor % arena_bytes; + if (arena_pos + nbytes > arena_bytes) { + cursor += arena_bytes - arena_pos; + } + cursor += nbytes; + } + + RegionView view_; + MonotonicClock clock_{}; + SpscQueueLayout layout_{}; + SpscQueueError error_{SpscQueueErrorKind::NONE, SpscQueueOp::INIT, 0, 0, ""}; + bool identity_trusted_{false}; + bool live_{false}; + InputQueue input_queue_; + OutputQueue output_queue_; +}; + +static_assert(kSpscQueueMagic == 0x53505351u, "SPSQ magic changed"); +static_assert(kSpscQueueAbiMajor == 1, "SPSQ ABI major changed"); +static_assert(kSpscQueueAbiMinor == 0, "SPSQ ABI minor changed"); +static_assert(kSpscQueueMagicVersion == 0x5350535100010000ull, "SPSQ packed magic_version changed"); +static_assert(kSpscQueueEndpointBindingScalarCount == 10, "SPSQ binding scalar count changed"); +static_assert(kDescriptorBytes == 32, "SPSQ descriptor size changed"); +static_assert(kArenaAlignment == 64, "SPSQ arena alignment changed"); +static_assert(kCounterStride == 64, "SPSQ counter stride changed"); +static_assert(kCounterBytes == 384, "SPSQ counter bytes changed"); +static_assert(kMaxDepth == (1ull << 30), "SPSQ max depth changed"); +static_assert(kInputDescTailOffset == 0, "SPSQ input tail offset changed"); +static_assert(kInputDescHeadOffset == 64, "SPSQ input head offset changed"); +static_assert(kOutputDescTailOffset == 128, "SPSQ output tail offset changed"); +static_assert(kOutputDescHeadOffset == 192, "SPSQ output head offset changed"); +static_assert(kInitiatorAbortOffset == 256, "SPSQ initiator abort offset changed"); +static_assert(kPeerAbortOffset == 320, "SPSQ peer abort offset changed"); +static_assert(sizeof(SpscQueueDescriptor) == 32, "SpscQueueDescriptor ABI size changed"); +static_assert(offsetof(SpscQueueDescriptor, seq) == 0, "SpscQueueDescriptor::seq offset changed"); +static_assert(offsetof(SpscQueueDescriptor, opcode) == 8, "SpscQueueDescriptor::opcode offset changed"); +static_assert(offsetof(SpscQueueDescriptor, payload_offset) == 16, "SpscQueueDescriptor::payload_offset changed"); +static_assert(offsetof(SpscQueueDescriptor, payload_nbytes) == 24, "SpscQueueDescriptor::payload_nbytes changed"); +static_assert(sizeof(SpscQueueEndpointBinding) == 80, "SpscQueueEndpointBinding ABI size changed"); +static_assert(kSpscQueueEndpointBindingScalarCount * sizeof(uint64_t) == 80, "SPSQ binding scalar bytes changed"); +static_assert(offsetof(SpscQueueEndpointBinding, magic_version) == 0, "binding magic_version offset changed"); +static_assert(offsetof(SpscQueueEndpointBinding, session_instance_id_bits) == 8, "binding session bits offset changed"); +static_assert(offsetof(SpscQueueEndpointBinding, transaction_id) == 16, "binding transaction_id offset changed"); +static_assert(offsetof(SpscQueueEndpointBinding, payload_base) == 24, "binding payload_base offset changed"); +static_assert(offsetof(SpscQueueEndpointBinding, payload_bytes) == 32, "binding payload_bytes offset changed"); +static_assert(offsetof(SpscQueueEndpointBinding, counter_base) == 40, "binding counter_base offset changed"); +static_assert(offsetof(SpscQueueEndpointBinding, counter_bytes) == 48, "binding counter_bytes offset changed"); +static_assert(offsetof(SpscQueueEndpointBinding, depth) == 56, "binding depth offset changed"); +static_assert(offsetof(SpscQueueEndpointBinding, input_arena_bytes) == 64, "binding input_arena_bytes offset changed"); +static_assert( + offsetof(SpscQueueEndpointBinding, output_arena_bytes) == 72, "binding output_arena_bytes offset changed" +); +static_assert(std::is_standard_layout_v, "SpscQueueDescriptor must be standard layout"); +static_assert(std::is_trivially_copyable_v, "SpscQueueDescriptor must be trivially copyable"); +static_assert(std::is_standard_layout_v, "SpscQueueEndpointBinding must be standard layout"); +static_assert( + std::is_trivially_copyable_v, "SpscQueueEndpointBinding must be trivially copyable" +); +static_assert(std::is_standard_layout_v, "SpscQueueLayout must be standard layout"); +static_assert(std::is_standard_layout_v, "SpscQueuePayloadView must be standard layout"); +static_assert(std::is_trivially_copyable_v, "SpscQueuePayloadView must be trivially copyable"); +static_assert(sizeof(SpscQueuePayloadView) == 16, "SpscQueuePayloadView size changed"); +static_assert(std::is_standard_layout_v, "SpscQueueError must be standard layout"); +static_assert(offsetof(SpscQueueError, session_instance_id_bits) == 8, "SpscQueueError session offset changed"); + +} // namespace spsc_queue diff --git a/tests/ut/cpp/CMakeLists.txt b/tests/ut/cpp/CMakeLists.txt index e6e6c0af38..33b018fc7b 100644 --- a/tests/ut/cpp/CMakeLists.txt +++ b/tests/ut/cpp/CMakeLists.txt @@ -644,6 +644,21 @@ target_link_libraries(test_worker_chip_message_queue PRIVATE add_test(NAME test_worker_chip_message_queue COMMAND test_worker_chip_message_queue) set_tests_properties(test_worker_chip_message_queue PROPERTIES LABELS "no_hardware") +add_executable(test_region_template + common/test_region_template.cpp +) +target_include_directories(test_region_template PRIVATE + ${GTEST_INCLUDE_DIRS} + ${CMAKE_SOURCE_DIR}/../../../src/common/platform/include +) +target_link_libraries(test_region_template PRIVATE + ${GTEST_MAIN_LIB} + ${GTEST_LIB} + pthread +) +add_test(NAME test_region_template COMMAND test_region_template) +set_tests_properties(test_region_template PROPERTIES LABELS "no_hardware") + add_executable(test_worker_chip_orch_endpoint common/test_worker_chip_orch_endpoint.cpp stubs/test_stubs.cpp diff --git a/tests/ut/cpp/common/test_region_template.cpp b/tests/ut/cpp/common/test_region_template.cpp new file mode 100644 index 0000000000..17f6dfa7b6 --- /dev/null +++ b/tests/ut/cpp/common/test_region_template.cpp @@ -0,0 +1,901 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + * ----------------------------------------------------------------------------------------------------------- + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "common/region_template.h" + +namespace { + +struct LayoutCase { + uint64_t depth; + uint64_t input_arena_bytes; + uint64_t output_arena_bytes; + uint64_t output_desc_offset; + uint64_t input_arena_offset; + uint64_t output_arena_offset; + uint64_t payload_bytes; +}; + +// Golden vectors shared with tests/ut/py/test_worker/test_comm_region_template.py. +constexpr std::array kLayoutGolden{{ + {1, 64, 64, 32, 64, 128, 192}, + {4, 128, 192, 128, 256, 384, 576}, + {8, 192, 64, 256, 512, 704, 768}, + {2, 64, 128, 64, 128, 192, 320}, +}}; + +constexpr uint8_t kDescriptorGoldenBytes[32] = { + 0x07, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +}; + +constexpr uint8_t kSessionGoldenBytes[8] = {0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08}; +constexpr uint64_t kSessionGoldenBits = 0x0807060504030201ull; +constexpr std::array kBindingGolden{ + 0x5350535100010000ull, kSessionGoldenBits, 42, 0x1000, 192, 0x2000, 384, 1, 64, 64, +}; + +} // namespace + +TEST(RegionTemplateTest, PackedMagicVersionIsSpsqAbi10) { + EXPECT_EQ(spsc_queue::kSpscQueueMagic, 0x53505351u); + EXPECT_EQ(spsc_queue::kSpscQueueAbiMajor, 1); + EXPECT_EQ(spsc_queue::kSpscQueueAbiMinor, 0); + EXPECT_EQ(spsc_queue::kSpscQueueMagicVersion, 0x5350535100010000ull); + EXPECT_EQ(spsc_queue::kSpscQueueEndpointBindingScalarCount, 10u); + EXPECT_EQ(spsc_queue::kDescriptorBytes, 32u); + EXPECT_EQ(spsc_queue::kArenaAlignment, 64u); + EXPECT_EQ(spsc_queue::kCounterStride, 64u); + EXPECT_EQ(spsc_queue::kCounterBytes, 384u); + EXPECT_EQ(spsc_queue::kMaxDepth, 1ull << 30); +} + +TEST(RegionTemplateTest, LayoutGoldenVectors) { + for (const auto &test_case : kLayoutGolden) { + spsc_queue::SpscQueueLayout layout{}; + ASSERT_TRUE( + spsc_queue::SpscQueueLayout::create( + test_case.depth, test_case.input_arena_bytes, test_case.output_arena_bytes, &layout + ) + ); + EXPECT_EQ(layout.input_desc_offset, 0u); + EXPECT_EQ(layout.output_desc_offset, test_case.output_desc_offset); + EXPECT_EQ(layout.input_arena_offset, test_case.input_arena_offset); + EXPECT_EQ(layout.output_arena_offset, test_case.output_arena_offset); + EXPECT_EQ(layout.payload_bytes, test_case.payload_bytes); + EXPECT_EQ(layout.input_arena_offset % 64u, 0u); + EXPECT_EQ(layout.output_arena_offset % 64u, 0u); + EXPECT_EQ(layout.input_desc_tail_offset, 0u); + EXPECT_EQ(layout.input_desc_head_offset, 64u); + EXPECT_EQ(layout.output_desc_tail_offset, 128u); + EXPECT_EQ(layout.output_desc_head_offset, 192u); + EXPECT_EQ(layout.initiator_abort_offset, 256u); + EXPECT_EQ(layout.peer_abort_offset, 320u); + EXPECT_EQ(layout.counter_bytes, 384u); + } +} + +TEST(RegionTemplateTest, LayoutMaxDepthGoldenVector) { + spsc_queue::SpscQueueLayout layout{}; + ASSERT_TRUE(spsc_queue::SpscQueueLayout::create(spsc_queue::kMaxDepth, 64, 64, &layout)); + EXPECT_EQ(layout.output_desc_offset, spsc_queue::kMaxDepth * 32u); + EXPECT_EQ(layout.input_arena_offset, spsc_queue::kMaxDepth * 64u); + EXPECT_EQ(layout.output_arena_offset, spsc_queue::kMaxDepth * 64u + 64u); + EXPECT_EQ(layout.payload_bytes, spsc_queue::kMaxDepth * 64u + 128u); + EXPECT_EQ(layout.input_arena_offset % 64u, 0u); + EXPECT_EQ(layout.output_arena_offset % 64u, 0u); +} + +TEST(RegionTemplateTest, LayoutRejectsInvalidDepthAndArenaValues) { + spsc_queue::SpscQueueLayout layout{}; + EXPECT_FALSE(spsc_queue::SpscQueueLayout::create(3, 64, 64, &layout)); + EXPECT_FALSE(spsc_queue::SpscQueueLayout::create(0, 64, 64, &layout)); + EXPECT_FALSE(spsc_queue::SpscQueueLayout::create((1ull << 30) + 1, 64, 64, &layout)); + EXPECT_FALSE(spsc_queue::SpscQueueLayout::create(2, 0, 64, &layout)); + EXPECT_FALSE(spsc_queue::SpscQueueLayout::create(2, 65, 64, &layout)); + EXPECT_FALSE(spsc_queue::SpscQueueLayout::create(2, 64, 0, &layout)); + EXPECT_FALSE(spsc_queue::SpscQueueLayout::create(2, 64, 63, &layout)); + EXPECT_FALSE(spsc_queue::SpscQueueLayout::create(2, 64, 64, nullptr)); +} + +TEST(RegionTemplateTest, LayoutOverflowFailsClosedWithoutModifyingOutput) { + spsc_queue::SpscQueueLayout layout{ + 7, 11, 13, 17, 19, 23, 29, 31, 37, 41, 43, 47, 53, 59, 61, + }; + const spsc_queue::SpscQueueLayout original = layout; + + EXPECT_FALSE(spsc_queue::SpscQueueLayout::create(2, std::numeric_limits::max() - 63, 64, &layout)); + EXPECT_EQ(layout.depth, original.depth); + EXPECT_EQ(layout.payload_bytes, original.payload_bytes); + + EXPECT_FALSE(spsc_queue::SpscQueueLayout::create(1, 64, std::numeric_limits::max() - 63, &layout)); + EXPECT_EQ(layout.depth, original.depth); +} + +TEST(RegionTemplateTest, CheckedMulAddAlignOverflowBoundaries) { + EXPECT_TRUE(spsc_queue::mul_overflows(1ull << 63, 2)); + EXPECT_TRUE(spsc_queue::add_overflows(std::numeric_limits::max(), 1)); + uint64_t aligned = 0; + EXPECT_FALSE(spsc_queue::align_up(std::numeric_limits::max() - 31, 64, &aligned)); + EXPECT_FALSE(spsc_queue::mul_overflows(spsc_queue::kMaxDepth, 32)); + EXPECT_TRUE(spsc_queue::align_up(64, 64, &aligned)); + EXPECT_EQ(aligned, 64u); + EXPECT_TRUE(spsc_queue::align_up(65, 64, &aligned)); + EXPECT_EQ(aligned, 128u); +} + +TEST(RegionTemplateTest, OpcodeAndDescriptorBytesGoldenVector) { + EXPECT_EQ(static_cast(spsc_queue::SpscQueueOpcode::INVALID), 0u); + EXPECT_EQ(static_cast(spsc_queue::SpscQueueOpcode::DATA), 1u); + EXPECT_EQ(static_cast(spsc_queue::SpscQueueOpcode::STOP), 2u); + EXPECT_EQ(static_cast(spsc_queue::SpscQueueOpcode::ERROR), 3u); + + spsc_queue::SpscQueueDescriptor descriptor{ + 7, + static_cast(spsc_queue::SpscQueueOpcode::ERROR), + 128, + 16, + }; + uint8_t encoded[32] = {}; + ASSERT_TRUE(spsc_queue::encode_descriptor(descriptor, encoded, sizeof(encoded))); + EXPECT_EQ(std::memcmp(encoded, kDescriptorGoldenBytes, sizeof(kDescriptorGoldenBytes)), 0); + + spsc_queue::SpscQueueDescriptor decoded{}; + ASSERT_TRUE(spsc_queue::decode_descriptor(kDescriptorGoldenBytes, sizeof(kDescriptorGoldenBytes), &decoded)); + EXPECT_EQ(decoded.seq, 7u); + EXPECT_EQ(decoded.opcode, 3u); + EXPECT_EQ(decoded.payload_offset, 128u); + EXPECT_EQ(decoded.payload_nbytes, 16u); + EXPECT_FALSE(spsc_queue::decode_descriptor(kDescriptorGoldenBytes, 31, &decoded)); +} + +TEST(RegionTemplateTest, SessionIdentityLittleEndianRoundTrip) { + uint64_t bits = 0; + ASSERT_TRUE(spsc_queue::session_instance_id_to_bits(kSessionGoldenBytes, sizeof(kSessionGoldenBytes), &bits)); + EXPECT_EQ(bits, kSessionGoldenBits); + + uint8_t restored[8] = {}; + ASSERT_TRUE(spsc_queue::session_instance_id_from_bits(kSessionGoldenBits, restored, sizeof(restored))); + EXPECT_EQ(std::memcmp(restored, kSessionGoldenBytes, sizeof(kSessionGoldenBytes)), 0); +} + +TEST(RegionTemplateTest, BindingExactTenScalarGoldenVector) { + spsc_queue::SpscQueueEndpointBinding binding{}; + ASSERT_TRUE(spsc_queue::decode_endpoint_binding(kBindingGolden.data(), kBindingGolden.size(), &binding)); + EXPECT_EQ(binding.magic_version, spsc_queue::kSpscQueueMagicVersion); + EXPECT_EQ(binding.session_instance_id_bits, kSessionGoldenBits); + EXPECT_EQ(binding.transaction_id, 42u); + EXPECT_EQ(binding.payload_base, 0x1000u); + EXPECT_EQ(binding.payload_bytes, 192u); + EXPECT_EQ(binding.counter_base, 0x2000u); + EXPECT_EQ(binding.counter_bytes, 384u); + EXPECT_EQ(binding.depth, 1u); + EXPECT_EQ(binding.input_arena_bytes, 64u); + EXPECT_EQ(binding.output_arena_bytes, 64u); + + std::array encoded{}; + ASSERT_TRUE(spsc_queue::encode_endpoint_binding(binding, encoded.data(), encoded.size())); + EXPECT_EQ(encoded, kBindingGolden); +} + +TEST(RegionTemplateTest, BindingRejectsCountAndVersionMismatchWithoutWritingOutput) { + spsc_queue::SpscQueueEndpointBinding original{ + 7, 11, 13, 17, 19, 23, 29, 31, 37, 41, + }; + spsc_queue::SpscQueueEndpointBinding decoded = original; + EXPECT_FALSE(spsc_queue::decode_endpoint_binding(kBindingGolden.data(), 9, &decoded)); + EXPECT_EQ(decoded.magic_version, original.magic_version); + EXPECT_FALSE(spsc_queue::decode_endpoint_binding(kBindingGolden.data(), 11, &decoded)); + EXPECT_EQ(decoded.transaction_id, original.transaction_id); + + std::array wrong_magic = kBindingGolden; + wrong_magic[0] = 0x4C33513200010001ull; + EXPECT_FALSE(spsc_queue::decode_endpoint_binding(wrong_magic.data(), wrong_magic.size(), &decoded)); + EXPECT_EQ(decoded.payload_base, original.payload_base); + + std::array wrong_major = kBindingGolden; + wrong_major[0] = (static_cast(spsc_queue::kSpscQueueMagic) << 32) | (2ull << 16); + EXPECT_FALSE(spsc_queue::decode_endpoint_binding(wrong_major.data(), wrong_major.size(), &decoded)); + + std::array wrong_minor = kBindingGolden; + wrong_minor[0] = (static_cast(spsc_queue::kSpscQueueMagic) << 32) | (1ull << 16) | 1ull; + EXPECT_FALSE(spsc_queue::decode_endpoint_binding(wrong_minor.data(), wrong_minor.size(), &decoded)); + EXPECT_FALSE(spsc_queue::decode_endpoint_binding(nullptr, 10, &decoded)); + EXPECT_FALSE(spsc_queue::encode_endpoint_binding(original, wrong_magic.data(), 10)); +} + +TEST(RegionTemplateTest, BindingAndDescriptorStaticAbi) { + static_assert(sizeof(spsc_queue::SpscQueueEndpointBinding) == 80, "binding size"); + static_assert(sizeof(spsc_queue::SpscQueueDescriptor) == 32, "descriptor size"); + static_assert(std::is_standard_layout_v, "binding layout"); + static_assert(std::is_trivially_copyable_v, "binding copy"); + EXPECT_EQ(offsetof(spsc_queue::SpscQueueEndpointBinding, output_arena_bytes), 72u); + EXPECT_EQ(offsetof(spsc_queue::SpscQueueDescriptor, payload_nbytes), 24u); +} + +namespace { + +enum class FakeAccessKind { + PayloadRead = 0, + PayloadWrite = 1, + CounterTest = 2, + CounterWait = 3, + CounterNotify = 4, +}; + +struct FakeAccess { + FakeAccessKind kind; + uint64_t offset; + uint64_t nbytes; + int32_t value; + RegionWaitCmp cmp; + uint64_t timeout_ns; +}; + +struct FakeViewError { + uint32_t kind; + char message[256]; +}; + +struct FakeState { + std::vector payload_mem; + std::vector counter_mem; + std::vector log; + bool sticky_failed = false; + FakeViewError last_error{}; + bool wait_sticky = false; + bool fail_peer_abort_notify = false; + std::function on_wait; +}; + +constexpr uint32_t kFakeTimeoutKind = 4; +constexpr uint64_t kSessionBits = 0x0123456789abcdefull; +constexpr uint64_t kTransactionId = 99; +uint64_t g_now_ns = 1000; +bool g_auto_advance = false; + +uint64_t test_now_ns() { + uint64_t now = g_now_ns; + if (g_auto_advance) { + g_now_ns += 100; + } + return now; +} + +void reset_clock() { + g_now_ns = 1000; + g_auto_advance = false; +} + +void set_error_message(FakeViewError *error, const char *text) { + const char *src = text == nullptr ? "" : text; + size_t n = strnlen(src, sizeof(error->message) - 1); + memcpy(error->message, src, n); + error->message[n] = '\0'; +} + +struct FakeRegionView { + std::shared_ptr state; + + explicit FakeRegionView(uint64_t payload_bytes) : + state(std::make_shared()) { + state->payload_mem.assign(static_cast(payload_bytes), 0); + state->counter_mem.assign(static_cast(spsc_queue::kCounterBytes), 0); + } + + static FakeRegionView prefailed() { + FakeRegionView view(64); + view.state->sticky_failed = true; + view.state->last_error.kind = 1; + set_error_message(&view.state->last_error, "pre-failed view"); + return view; + } + + bool failed() const { return state->sticky_failed; } + const FakeViewError &error() const { return state->last_error; } + + uint64_t payload_base() const { + return static_cast(reinterpret_cast(state->payload_mem.data())); + } + uint64_t counter_base() const { + return static_cast(reinterpret_cast(state->counter_mem.data())); + } + + class PayloadPart { + public: + explicit PayloadPart(FakeRegionView *view) : + view_(view) {} + + RegionPartLocalSpan span() const { + return RegionPartLocalSpan{view_->payload_base(), view_->state->payload_mem.size()}; + } + + bool read(uint64_t offset, uint64_t nbytes, spsc_queue::SpscQueuePayloadView &out) { + out = spsc_queue::SpscQueuePayloadView{0, 0}; + view_->state->log.push_back( + FakeAccess{FakeAccessKind::PayloadRead, offset, nbytes, 0, RegionWaitCmp::EQ, 0} + ); + if (view_->state->sticky_failed) { + return false; + } + if (nbytes == 0 || offset + nbytes > view_->state->payload_mem.size()) { + view_->state->last_error.kind = 2; + set_error_message(&view_->state->last_error, "payload range is out of bounds"); + return false; + } + out = spsc_queue::SpscQueuePayloadView{view_->payload_base() + offset, nbytes}; + return true; + } + + bool write(uint64_t offset, const void *src, uint64_t nbytes) { + view_->state->log.push_back( + FakeAccess{FakeAccessKind::PayloadWrite, offset, nbytes, 0, RegionWaitCmp::EQ, 0} + ); + if (view_->state->sticky_failed) { + return false; + } + if (src == nullptr || nbytes == 0 || offset + nbytes > view_->state->payload_mem.size()) { + view_->state->last_error.kind = 2; + set_error_message(&view_->state->last_error, "payload write out of bounds"); + return false; + } + memcpy(view_->state->payload_mem.data() + offset, src, static_cast(nbytes)); + return true; + } + + private: + FakeRegionView *view_; + }; + + class CounterPart { + public: + explicit CounterPart(FakeRegionView *view) : + view_(view) {} + + RegionPartLocalSpan span() const { + return RegionPartLocalSpan{view_->counter_base(), view_->state->counter_mem.size()}; + } + + bool test(uint64_t offset, int32_t cmp_value, RegionWaitCmp cmp, spsc_queue::SpscQueueCounterSample &out) { + out = spsc_queue::SpscQueueCounterSample{false, 0}; + view_->state->log.push_back(FakeAccess{FakeAccessKind::CounterTest, offset, 4, cmp_value, cmp, 0}); + int32_t observed = 0; + if (!load(offset, observed)) { + return false; + } + out = spsc_queue::SpscQueueCounterSample{region_compare_counter(observed, cmp_value, cmp), observed}; + return true; + } + + bool wait(uint64_t offset, int32_t cmp_value, RegionWaitCmp cmp, uint64_t timeout_ns, int32_t &observed) { + observed = 0; + view_->state->log.push_back(FakeAccess{FakeAccessKind::CounterWait, offset, 4, cmp_value, cmp, timeout_ns}); + if (view_->state->on_wait) { + view_->state->on_wait(); + } + if (view_->state->wait_sticky) { + view_->state->sticky_failed = true; + view_->state->last_error.kind = 5; + set_error_message(&view_->state->last_error, "injected wait failure"); + return false; + } + if (!load(offset, observed)) { + return false; + } + if (region_compare_counter(observed, cmp_value, cmp)) { + return true; + } + view_->state->last_error.kind = kFakeTimeoutKind; + set_error_message(&view_->state->last_error, "wait timed out"); + return false; + } + + bool notify(uint64_t offset, int32_t value, RegionNotifyOp op) { + view_->state->log.push_back( + FakeAccess{FakeAccessKind::CounterNotify, offset, 4, value, RegionWaitCmp::EQ, 0} + ); + (void)op; + if (view_->state->fail_peer_abort_notify && offset == spsc_queue::kPeerAbortOffset) { + return false; + } + if (view_->state->sticky_failed) { + return false; + } + return store(offset, value); + } + + private: + bool load(uint64_t offset, int32_t &out) { + if (view_->state->sticky_failed) { + return false; + } + if (offset + 4 > view_->state->counter_mem.size() || (offset % 4) != 0) { + view_->state->last_error.kind = 2; + set_error_message(&view_->state->last_error, "invalid counter address"); + return false; + } + memcpy(&out, view_->state->counter_mem.data() + offset, sizeof(out)); + return true; + } + + bool store(uint64_t offset, int32_t value) { + if (offset + 4 > view_->state->counter_mem.size() || (offset % 4) != 0) { + view_->state->last_error.kind = 2; + set_error_message(&view_->state->last_error, "invalid counter address"); + return false; + } + memcpy(view_->state->counter_mem.data() + offset, &value, sizeof(value)); + return true; + } + + FakeRegionView *view_; + }; + + PayloadPart payload() { return PayloadPart(this); } + CounterPart counter() { return CounterPart(this); } +}; + +spsc_queue::SpscQueueLayout make_layout(uint64_t depth, uint64_t input_arena, uint64_t output_arena) { + spsc_queue::SpscQueueLayout layout{}; + EXPECT_TRUE(spsc_queue::SpscQueueLayout::create(depth, input_arena, output_arena, &layout)); + return layout; +} + +spsc_queue::SpscQueueEndpointBinding +make_binding(const FakeRegionView &view, const spsc_queue::SpscQueueLayout &layout) { + spsc_queue::SpscQueueEndpointBinding binding{}; + binding.magic_version = spsc_queue::kSpscQueueMagicVersion; + binding.session_instance_id_bits = kSessionBits; + binding.transaction_id = kTransactionId; + binding.payload_base = view.payload_base(); + binding.payload_bytes = layout.payload_bytes; + binding.counter_base = view.counter_base(); + binding.counter_bytes = layout.counter_bytes; + binding.depth = layout.depth; + binding.input_arena_bytes = layout.input_arena_bytes; + binding.output_arena_bytes = layout.output_arena_bytes; + return binding; +} + +void store_counter(FakeState *state, uint64_t offset, int32_t value) { + memcpy(state->counter_mem.data() + offset, &value, sizeof(value)); +} + +int32_t load_counter(const FakeState *state, uint64_t offset) { + int32_t value = 0; + memcpy(&value, state->counter_mem.data() + offset, sizeof(value)); + return value; +} + +void plant_descriptor( + FakeState *state, uint64_t slot_offset, uint64_t seq, spsc_queue::SpscQueueOpcode opcode, uint64_t payload_offset, + uint64_t nbytes +) { + spsc_queue::SpscQueueDescriptor descriptor{seq, static_cast(opcode), payload_offset, nbytes}; + uint8_t encoded[32]; + ASSERT_TRUE(spsc_queue::encode_descriptor(descriptor, encoded, sizeof(encoded))); + memcpy(state->payload_mem.data() + slot_offset, encoded, sizeof(encoded)); +} + +void plant_input( + FakeState *state, const spsc_queue::SpscQueueLayout &layout, uint64_t seq, spsc_queue::SpscQueueOpcode opcode, + uint64_t nbytes, const void *bytes +) { + uint64_t slot_index = (seq - 1) & (layout.depth - 1); + uint64_t slot_offset = layout.input_desc_offset + slot_index * spsc_queue::kDescriptorBytes; + uint64_t payload_offset = nbytes == 0 ? 0 : layout.input_arena_offset; + plant_descriptor(state, slot_offset, seq, opcode, payload_offset, nbytes); + if (nbytes != 0 && bytes != nullptr) { + memcpy(state->payload_mem.data() + payload_offset, bytes, static_cast(nbytes)); + } + store_counter(state, layout.input_desc_tail_offset, static_cast(seq)); +} + +bool has_session_marker(const char *message) { return std::string(message).find("session=0x") != std::string::npos; } + +bool has_sixteen_hex_session(const char *message) { + const char *p = std::strstr(message, "session=0x"); + if (p == nullptr) { + return false; + } + p += 10; + for (int i = 0; i < 16; ++i) { + char c = p[i]; + bool hex = (c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F'); + if (!hex) { + return false; + } + } + return true; +} + +spsc_queue::MonotonicClock test_clock() { return spsc_queue::MonotonicClock{&test_now_ns}; } + +using Queue1 = spsc_queue::SpscQueueEndpoint; +using Queue2 = spsc_queue::SpscQueueEndpoint; + +static_assert(!std::is_copy_constructible_v, "endpoint must not copy"); +static_assert(!std::is_move_constructible_v, "endpoint must not move"); +static_assert(!std::is_copy_assignable_v, "endpoint must not copy-assign"); +static_assert(!std::is_move_assignable_v, "endpoint must not move-assign"); + +size_t count_kind(const std::vector &log, FakeAccessKind kind) { + size_t n = 0; + for (const auto &entry : log) { + if (entry.kind == kind) { + n += 1; + } + } + return n; +} + +bool has_notify(const std::vector &log, uint64_t offset) { + for (const auto &entry : log) { + if (entry.kind == FakeAccessKind::CounterNotify && entry.offset == offset) { + return true; + } + } + return false; +} + +} // namespace + +TEST(RegionTemplateTest, ConstructionRejectsNullClockWithoutSharedAccess) { + reset_clock(); + auto layout = make_layout(2, 64, 64); + FakeRegionView view(layout.payload_bytes); + auto state = view.state; + auto binding = make_binding(view, layout); + Queue1 queue(binding, std::move(view), spsc_queue::MonotonicClock{nullptr}); + EXPECT_FALSE(queue.live()); + EXPECT_EQ(queue.error().kind, spsc_queue::SpscQueueErrorKind::BAD_ARGUMENT); + EXPECT_TRUE(has_session_marker(queue.error().message)); + EXPECT_TRUE(has_sixteen_hex_session(queue.error().message)); + EXPECT_EQ(queue.error().session_instance_id_bits, kSessionBits); + EXPECT_EQ(queue.error().transaction_id, kTransactionId); + EXPECT_TRUE(state->log.empty()); +} + +TEST(RegionTemplateTest, ConstructionRejectsInvalidBindingWithoutSharedAccessOrIdentity) { + reset_clock(); + auto layout = make_layout(2, 64, 64); + FakeRegionView view(layout.payload_bytes); + auto state = view.state; + auto binding = make_binding(view, layout); + binding.magic_version = 0x4C33513200010001ull; + Queue1 queue(binding, std::move(view), test_clock()); + EXPECT_FALSE(queue.live()); + EXPECT_EQ(queue.error().kind, spsc_queue::SpscQueueErrorKind::BAD_BINDING); + EXPECT_FALSE(has_session_marker(queue.error().message)); + EXPECT_EQ(std::string(queue.error().message).find("transaction="), std::string::npos); + EXPECT_TRUE(state->log.empty()); +} + +TEST(RegionTemplateTest, ConstructionRejectsFailedViewWithoutSharedAccess) { + reset_clock(); + auto layout = make_layout(2, 64, 64); + FakeRegionView view = FakeRegionView::prefailed(); + auto state = view.state; + auto binding = make_binding(view, layout); + Queue1 queue(binding, std::move(view), test_clock()); + EXPECT_FALSE(queue.live()); + EXPECT_EQ(queue.error().kind, spsc_queue::SpscQueueErrorKind::ENDPOINT_ERROR); + EXPECT_NE(std::string(queue.error().message).find("pre-failed view"), std::string::npos); + EXPECT_TRUE(has_session_marker(queue.error().message)); + EXPECT_TRUE(state->log.empty()); +} + +TEST(RegionTemplateTest, ConstructionRejectsMaxInflightGreaterThanDepthWithoutSharedAccess) { + reset_clock(); + auto layout = make_layout(2, 64, 64); + FakeRegionView view(layout.payload_bytes); + auto state = view.state; + auto binding = make_binding(view, layout); + spsc_queue::SpscQueueEndpoint queue(binding, std::move(view), test_clock()); + EXPECT_FALSE(queue.live()); + EXPECT_EQ(queue.error().kind, spsc_queue::SpscQueueErrorKind::BAD_ARGUMENT); + EXPECT_TRUE(state->log.empty()); +} + +TEST(RegionTemplateTest, EndpointMaxInflightOneDuplexZeroByteStopAndPublishOrder) { + reset_clock(); + auto layout = make_layout(4, 64, 64); + FakeRegionView view(layout.payload_bytes); + auto state = view.state; + auto binding = make_binding(view, layout); + Queue1 queue(binding, std::move(view), test_clock()); + ASSERT_TRUE(queue.live()); + EXPECT_TRUE(state->log.empty()); + + const char payload[] = "abcdefgh"; + plant_input(state.get(), layout, 1, spsc_queue::SpscQueueOpcode::DATA, 8, payload); + spsc_queue::SpscQueueInputHandle handle{}; + ASSERT_TRUE(queue.input().try_peek(handle)); + EXPECT_EQ(handle.seq, 1u); + EXPECT_EQ(handle.opcode, spsc_queue::SpscQueueOpcode::DATA); + EXPECT_EQ(handle.payload_nbytes, 8u); + EXPECT_EQ( + std::memcmp(reinterpret_cast(static_cast(handle.payload.local_addr)), payload, 8), 0 + ); + EXPECT_FALSE(queue.input().drained()); + ASSERT_TRUE(queue.input().release(handle)); + EXPECT_EQ(load_counter(state.get(), layout.input_desc_head_offset), 1); + EXPECT_FALSE(queue.input().try_peek(handle)); + + plant_input(state.get(), layout, 2, spsc_queue::SpscQueueOpcode::STOP, 0, nullptr); + ASSERT_TRUE(queue.input().try_peek(handle)); + EXPECT_EQ(handle.opcode, spsc_queue::SpscQueueOpcode::STOP); + EXPECT_EQ(handle.payload.local_addr, 0u); + EXPECT_EQ(handle.payload.nbytes, 0u); + ASSERT_TRUE(queue.input().release(handle)); + EXPECT_TRUE(queue.input().drained()); + EXPECT_FALSE(queue.input().try_peek(handle)); + + spsc_queue::SpscQueueOutputReservation zero{}; + ASSERT_TRUE(queue.output().try_reserve(0, zero)); + EXPECT_EQ(zero.payload.local_addr, 0u); + EXPECT_EQ(zero.payload.nbytes, 0u); + ASSERT_TRUE(queue.output().publish(zero, spsc_queue::SpscQueueOpcode::DATA)); + + spsc_queue::SpscQueueOutputReservation reserved{}; + ASSERT_TRUE(queue.output().try_reserve(8, reserved)); + EXPECT_EQ(reserved.payload.nbytes, 8u); + EXPECT_NE(reserved.payload.local_addr, 0u); + memcpy(reinterpret_cast(static_cast(reserved.payload.local_addr)), payload, 8); + size_t writes_before = count_kind(state->log, FakeAccessKind::PayloadWrite); + ASSERT_TRUE(queue.output().publish(reserved, spsc_queue::SpscQueueOpcode::ERROR)); + EXPECT_EQ(queue.error().kind, spsc_queue::SpscQueueErrorKind::NONE); + + std::vector writes; + for (const auto &entry : state->log) { + if (entry.kind == FakeAccessKind::PayloadWrite) { + writes.push_back(entry); + } + } + ASSERT_GE(writes.size(), writes_before + 2); + const FakeAccess &fields = writes[writes.size() - 2]; + const FakeAccess &seq = writes[writes.size() - 1]; + uint64_t slot1 = layout.output_desc_offset + spsc_queue::kDescriptorBytes; + EXPECT_EQ(fields.offset, slot1 + 8); + EXPECT_EQ(fields.nbytes, 24u); + EXPECT_EQ(seq.offset, slot1); + EXPECT_EQ(seq.nbytes, 8u); + ASSERT_FALSE(state->log.empty()); + EXPECT_EQ(state->log.back().kind, FakeAccessKind::CounterNotify); + EXPECT_EQ(state->log.back().offset, layout.output_desc_tail_offset); + for (const auto &entry : writes) { + EXPECT_FALSE(entry.offset >= layout.output_arena_offset && entry.offset < layout.payload_bytes); + } +} + +TEST(RegionTemplateTest, StopExtraSlotAndCompletedPrefix) { + reset_clock(); + auto layout = make_layout(4, 64, 64); + FakeRegionView view(layout.payload_bytes); + auto state = view.state; + auto binding = make_binding(view, layout); + Queue1 queue(binding, std::move(view), test_clock()); + ASSERT_TRUE(queue.live()); + + const char first[] = "12345678"; + plant_input(state.get(), layout, 1, spsc_queue::SpscQueueOpcode::DATA, 8, first); + spsc_queue::SpscQueueInputHandle data{}; + ASSERT_TRUE(queue.input().try_peek(data)); + spsc_queue::SpscQueueInputHandle blocked{}; + EXPECT_FALSE(queue.input().try_peek(blocked)); + + plant_descriptor( + state.get(), layout.input_desc_offset + spsc_queue::kDescriptorBytes, 2, spsc_queue::SpscQueueOpcode::STOP, 0, 0 + ); + store_counter(state.get(), layout.input_desc_tail_offset, 2); + spsc_queue::SpscQueueInputHandle stop{}; + ASSERT_TRUE(queue.input().try_peek(stop)); + EXPECT_EQ(stop.opcode, spsc_queue::SpscQueueOpcode::STOP); + EXPECT_FALSE(queue.input().drained()); + EXPECT_EQ(load_counter(state.get(), layout.input_desc_head_offset), 0); + + ASSERT_TRUE(queue.input().release(stop)); + EXPECT_FALSE(queue.input().drained()); + EXPECT_EQ(load_counter(state.get(), layout.input_desc_head_offset), 0); + ASSERT_TRUE(queue.input().release(data)); + EXPECT_TRUE(queue.input().drained()); + EXPECT_EQ(load_counter(state.get(), layout.input_desc_head_offset), 2); +} + +TEST(RegionTemplateTest, MaxInflightTwoWindowAndStopExtraSlot) { + reset_clock(); + auto layout = make_layout(4, 64, 64); + FakeRegionView view(layout.payload_bytes); + auto state = view.state; + auto binding = make_binding(view, layout); + Queue2 queue(binding, std::move(view), test_clock()); + ASSERT_TRUE(queue.live()); + + plant_input(state.get(), layout, 1, spsc_queue::SpscQueueOpcode::DATA, 0, nullptr); + spsc_queue::SpscQueueInputHandle a{}; + spsc_queue::SpscQueueInputHandle b{}; + spsc_queue::SpscQueueInputHandle extra{}; + ASSERT_TRUE(queue.input().try_peek(a)); + plant_descriptor( + state.get(), layout.input_desc_offset + spsc_queue::kDescriptorBytes, 2, spsc_queue::SpscQueueOpcode::DATA, 0, 0 + ); + store_counter(state.get(), layout.input_desc_tail_offset, 2); + ASSERT_TRUE(queue.input().try_peek(b)); + plant_descriptor( + state.get(), layout.input_desc_offset + 2 * spsc_queue::kDescriptorBytes, 3, spsc_queue::SpscQueueOpcode::DATA, + 0, 0 + ); + store_counter(state.get(), layout.input_desc_tail_offset, 3); + EXPECT_FALSE(queue.input().try_peek(extra)); + EXPECT_EQ(queue.error().kind, spsc_queue::SpscQueueErrorKind::NONE); + plant_descriptor( + state.get(), layout.input_desc_offset + 2 * spsc_queue::kDescriptorBytes, 3, spsc_queue::SpscQueueOpcode::STOP, + 0, 0 + ); + ASSERT_TRUE(queue.input().try_peek(extra)); + EXPECT_EQ(extra.opcode, spsc_queue::SpscQueueOpcode::STOP); + ASSERT_TRUE(queue.input().release(a)); + ASSERT_TRUE(queue.input().release(b)); + ASSERT_TRUE(queue.input().release(extra)); + EXPECT_TRUE(queue.input().drained()); +} + +TEST(RegionTemplateTest, OutputReservationOwnershipLifetimeAndOversize) { + reset_clock(); + auto layout = make_layout(4, 64, 64); + FakeRegionView view(layout.payload_bytes); + auto state = view.state; + auto binding = make_binding(view, layout); + Queue1 queue(binding, std::move(view), test_clock()); + ASSERT_TRUE(queue.live()); + + spsc_queue::SpscQueueOutputReservation oversize{}; + EXPECT_FALSE(queue.output().try_reserve(layout.output_arena_bytes + 1, oversize)); + EXPECT_EQ(queue.error().kind, spsc_queue::SpscQueueErrorKind::NONE); + EXPECT_FALSE(oversize.valid); + EXPECT_FALSE(queue.output().reserve(layout.output_arena_bytes + 1, 50, oversize)); + EXPECT_EQ(queue.error().kind, spsc_queue::SpscQueueErrorKind::NONE); + EXPECT_TRUE(state->log.empty()); + + spsc_queue::SpscQueueOutputReservation reserved{}; + ASSERT_TRUE(queue.output().try_reserve(8, reserved)); + EXPECT_TRUE(reserved.valid); + spsc_queue::SpscQueueOutputReservation second{}; + EXPECT_FALSE(queue.output().try_reserve(8, second)); + EXPECT_EQ(queue.error().kind, spsc_queue::SpscQueueErrorKind::OWNERSHIP); + + FakeRegionView view2(layout.payload_bytes); + auto state2 = view2.state; + auto binding2 = make_binding(view2, layout); + Queue1 queue2(binding2, std::move(view2), test_clock()); + spsc_queue::SpscQueueOutputReservation ok{}; + ASSERT_TRUE(queue2.output().try_reserve(8, ok)); + spsc_queue::SpscQueueOutputReservation stale = ok; + ASSERT_TRUE(queue2.output().publish(ok, spsc_queue::SpscQueueOpcode::DATA)); + EXPECT_FALSE(queue2.output().publish(stale, spsc_queue::SpscQueueOpcode::DATA)); + EXPECT_EQ(queue2.error().kind, spsc_queue::SpscQueueErrorKind::OWNERSHIP); + EXPECT_TRUE(has_session_marker(queue2.error().message)); +} + +TEST(RegionTemplateTest, WaitTimeoutKeepsErrorNoneAndIgnoresStaleViewKind) { + reset_clock(); + auto layout = make_layout(4, 64, 64); + FakeRegionView view(layout.payload_bytes); + auto state = view.state; + auto binding = make_binding(view, layout); + Queue1 queue(binding, std::move(view), test_clock()); + spsc_queue::SpscQueueInputHandle handle{}; + EXPECT_FALSE(queue.input().peek(50, handle)); + EXPECT_EQ(queue.error().kind, spsc_queue::SpscQueueErrorKind::NONE); + EXPECT_EQ(state->last_error.kind, kFakeTimeoutKind); + EXPECT_GE(count_kind(state->log, FakeAccessKind::CounterWait), 1u); + + const char payload[] = "abcdefgh"; + plant_input(state.get(), layout, 1, spsc_queue::SpscQueueOpcode::DATA, 8, payload); + ASSERT_TRUE(queue.input().try_peek(handle)); + EXPECT_EQ(queue.error().kind, spsc_queue::SpscQueueErrorKind::NONE); + EXPECT_EQ(handle.seq, 1u); +} + +TEST(RegionTemplateTest, WaitStickyFailureIsEndpointError) { + reset_clock(); + auto layout = make_layout(4, 64, 64); + FakeRegionView view(layout.payload_bytes); + auto state = view.state; + state->wait_sticky = true; + auto binding = make_binding(view, layout); + Queue1 queue(binding, std::move(view), test_clock()); + spsc_queue::SpscQueueInputHandle handle{}; + EXPECT_FALSE(queue.input().peek(50, handle)); + EXPECT_EQ(queue.error().kind, spsc_queue::SpscQueueErrorKind::ENDPOINT_ERROR); + EXPECT_NE(std::string(queue.error().message).find("injected wait failure"), std::string::npos); + EXPECT_TRUE(has_session_marker(queue.error().message)); + EXPECT_TRUE(has_notify(state->log, layout.peer_abort_offset)); +} + +TEST(RegionTemplateTest, PeerAbortSampledAtDeadlineWithoutLocalAbortNotify) { + reset_clock(); + g_auto_advance = true; + auto layout = make_layout(4, 64, 64); + FakeRegionView view(layout.payload_bytes); + auto state = view.state; + store_counter(state.get(), layout.initiator_abort_offset, 1); + auto binding = make_binding(view, layout); + Queue1 queue(binding, std::move(view), test_clock()); + spsc_queue::SpscQueueInputHandle handle{}; + EXPECT_FALSE(queue.input().peek(10, handle)); + EXPECT_EQ(queue.error().kind, spsc_queue::SpscQueueErrorKind::REMOTE_ABORTED); + EXPECT_TRUE(has_session_marker(queue.error().message)); + EXPECT_FALSE(has_notify(state->log, layout.peer_abort_offset)); +} + +TEST(RegionTemplateTest, FirstErrorWinsWhenAbortNotifyFails) { + reset_clock(); + auto layout = make_layout(4, 64, 64); + FakeRegionView view(layout.payload_bytes); + auto state = view.state; + state->fail_peer_abort_notify = true; + auto binding = make_binding(view, layout); + Queue1 queue(binding, std::move(view), test_clock()); + plant_input(state.get(), layout, 1, spsc_queue::SpscQueueOpcode::ERROR, 0, nullptr); + spsc_queue::SpscQueueInputHandle handle{}; + EXPECT_FALSE(queue.input().try_peek(handle)); + EXPECT_EQ(queue.error().kind, spsc_queue::SpscQueueErrorKind::INVALID_DESCRIPTOR); + EXPECT_NE(std::string(queue.error().message).find("abort notify failed"), std::string::npos); + EXPECT_NE(std::string(queue.error().message).find("invalid input opcode"), std::string::npos); +} + +TEST(RegionTemplateTest, CounterReconstructionInvalidDeltaPoisons) { + reset_clock(); + auto layout = make_layout(4, 64, 64); + FakeRegionView view(layout.payload_bytes); + auto state = view.state; + auto binding = make_binding(view, layout); + Queue1 queue(binding, std::move(view), test_clock()); + store_counter(state.get(), layout.input_desc_tail_offset, 5); + spsc_queue::SpscQueueInputHandle handle{}; + EXPECT_FALSE(queue.input().try_peek(handle)); + EXPECT_EQ(queue.error().kind, spsc_queue::SpscQueueErrorKind::INVALID_DESCRIPTOR); + EXPECT_NE(std::string(queue.error().message).find("counter reconstruction failed"), std::string::npos); +} + +TEST(RegionTemplateTest, OutputWrapReplayDoesNotFlushBorrowedPayload) { + reset_clock(); + auto layout = make_layout(4, 64, 64); + FakeRegionView view(layout.payload_bytes); + auto state = view.state; + auto binding = make_binding(view, layout); + Queue1 queue(binding, std::move(view), test_clock()); + spsc_queue::SpscQueueOutputReservation first{}; + ASSERT_TRUE(queue.output().try_reserve(40, first)); + EXPECT_EQ(first.payload_offset, layout.output_arena_offset); + ASSERT_TRUE(queue.output().publish(first, spsc_queue::SpscQueueOpcode::DATA)); + store_counter(state.get(), layout.output_desc_head_offset, 1); + spsc_queue::SpscQueueOutputReservation wrapped{}; + ASSERT_TRUE(queue.output().try_reserve(40, wrapped)); + EXPECT_EQ(wrapped.payload_offset, layout.output_arena_offset); + for (const auto &entry : state->log) { + if (entry.kind == FakeAccessKind::PayloadWrite) { + EXPECT_LT(entry.offset, layout.output_arena_offset); + } + } +} diff --git a/tests/ut/py/test_worker/test_comm_region.py b/tests/ut/py/test_worker/test_comm_region.py index abdce97efb..dce1b790ff 100644 --- a/tests/ut/py/test_worker/test_comm_region.py +++ b/tests/ut/py/test_worker/test_comm_region.py @@ -1353,6 +1353,180 @@ def test_endpoint_duplicate_match_poisons_worker(region_worker): worker._require_no_ordered_cleanup_failure("test") +def _queue_endpoint_error(session: bytes, transaction_id: int) -> RuntimeError: + bits = int.from_bytes(bytes(session), "little") + return RuntimeError( + "SPSC queue endpoint error op=init kind=6 " + f"session=0x{bits:016x} transaction={int(transaction_id)} msg=issued local operation failed" + ) + + +def test_allocation_identity_returns_bound_session_and_transaction(region_worker): + worker, _calls, _leases = region_worker() + resources = _RunResources() + worker._building_run_resources = resources + try: + instance = _materialize_default_region(worker) + finally: + worker._building_run_resources = None + session, transaction_id = instance._allocation_identity + assert isinstance(session, bytes) + assert len(session) == 8 + assert type(transaction_id) is int + assert transaction_id >= 1 + instance._delegated_session_instance_id = None + with pytest.raises(MaterializationError, match="incomplete"): + _ = instance._allocation_identity + + +def test_record_data_plane_failure_by_allocation_identity_matches_single_region(region_worker): + worker, calls, _leases = region_worker() + resources = _RunResources() + worker._building_run_resources = resources + try: + instance = _materialize_default_region(worker) + finally: + worker._building_run_resources = None + session, transaction_id = instance._allocation_identity + error = RuntimeError("issued local operation failed") + worker._region_instance_registry.record_data_plane_failure_by_allocation_identity( + resources, session, transaction_id, error + ) + assert instance.data_plane_error is error + assert instance._close_attempted is False + assert instance.state is RegionInstanceState.LIVE + assert [item for item in calls if item[0] == "release"] == [] + later = RuntimeError("second failure") + worker._region_instance_registry.record_data_plane_failure_by_allocation_identity( + resources, session, transaction_id, later + ) + assert instance.data_plane_error is error + + +def test_record_data_plane_failure_by_allocation_identity_unknown_is_routing_error(region_worker): + worker, _calls, _leases = region_worker() + resources = _RunResources() + other = _RunResources() + worker._building_run_resources = resources + try: + instance = _materialize_default_region(worker) + finally: + worker._building_run_resources = None + session, transaction_id = instance._allocation_identity + with pytest.raises(MaterializationError, match="no region instance for allocation identity"): + worker._region_instance_registry.record_data_plane_failure_by_allocation_identity( + other, session, transaction_id, RuntimeError("unused") + ) + with pytest.raises(MaterializationError, match="no region instance for allocation identity"): + worker._region_instance_registry.record_data_plane_failure_by_allocation_identity( + resources, b"\x00" * 8, 99, RuntimeError("unused") + ) + assert instance.data_plane_error is None + assert instance._close_attempted is False + + +def test_record_data_plane_failure_by_allocation_identity_duplicate_match_is_invariant_error(region_worker): + worker, _calls, _leases = region_worker() + resources = _RunResources() + worker._building_run_resources = resources + try: + first = _materialize_default_region(worker) + second = _materialize_default_region(worker) + finally: + worker._building_run_resources = None + session, transaction_id = first._allocation_identity + second._delegated_session_instance_id = first._delegated_session_instance_id + second._delegated_transaction_id = first._delegated_transaction_id + with pytest.raises(MaterializationError, match="duplicate region instances for allocation identity"): + worker._region_instance_registry.record_data_plane_failure_by_allocation_identity( + resources, session, transaction_id, RuntimeError("unused") + ) + assert first.data_plane_error is None + assert second.data_plane_error is None + + +def test_queue_endpoint_error_correlates_unique_allocation_identity(region_worker): + worker, calls, _leases = region_worker() + resources = _RunResources() + worker._building_run_resources = resources + try: + instance = _materialize_default_region(worker) + finally: + worker._building_run_resources = None + session, transaction_id = instance._allocation_identity + error = _queue_endpoint_error(session, transaction_id) + assert worker._poison_spsc_queue_from_endpoint_error(error, resources) is True + assert instance.data_plane_error is error + assert instance._close_attempted is False + assert [item for item in calls if item[0] == "release"] == [] + with pytest.raises(RuntimeError, match="no further work is admitted"): + worker._require_no_ordered_cleanup_failure("test") + + +def test_queue_endpoint_error_unknown_identity_fail_closed(region_worker): + worker, _calls, _leases = region_worker() + resources = _RunResources() + worker._building_run_resources = resources + try: + instance = _materialize_default_region(worker) + finally: + worker._building_run_resources = None + error = _queue_endpoint_error(b"\xff" * 8, 1) + assert worker._poison_spsc_queue_from_endpoint_error(error, resources) is True + assert instance.data_plane_error is None + with pytest.raises(RuntimeError, match="no further work is admitted"): + worker._require_no_ordered_cleanup_failure("test") + + +def test_queue_endpoint_error_malformed_does_not_modify_instance_or_fall_back(region_worker): + worker, _calls, _leases = region_worker() + resources = _RunResources() + worker._building_run_resources = resources + try: + instance = _materialize_default_region(worker) + finally: + worker._building_run_resources = None + error = RuntimeError( + "SPSC queue endpoint error op=init kind=2 msg=invalid queue binding " + f"L3-L2 endpoint error op=payload_write kind=5 region={int(instance.provider_resource_id)} " + "msg=issued local operation failed" + ) + assert worker._poison_spsc_queue_from_endpoint_error(error, resources) is True + assert instance.data_plane_error is None + with pytest.raises(RuntimeError, match="no further work is admitted"): + worker._require_no_ordered_cleanup_failure("test") + + +def test_queue_endpoint_error_zero_transaction_does_not_modify_instance(region_worker): + worker, _calls, _leases = region_worker() + resources = _RunResources() + worker._building_run_resources = resources + try: + instance = _materialize_default_region(worker) + finally: + worker._building_run_resources = None + session, _transaction_id = instance._allocation_identity + error = _queue_endpoint_error(session, 0) + assert worker._poison_spsc_queue_from_endpoint_error(error, resources) is True + assert instance.data_plane_error is None + with pytest.raises(RuntimeError, match="no further work is admitted"): + worker._require_no_ordered_cleanup_failure("test") + + +def test_legacy_region_parser_still_handles_independent_region_marker(region_worker): + worker, _calls, _leases = region_worker() + resources = _RunResources() + worker._building_run_resources = resources + try: + instance = _materialize_default_region(worker) + finally: + worker._building_run_resources = None + error = _endpoint_error(instance.provider_resource_id) + assert worker._poison_spsc_queue_from_endpoint_error(error, resources) is False + assert worker._poison_worker_chip_region_from_endpoint_error(error, resources) is True + assert instance.data_plane_error is error + + def test_data_plane_poison_still_completes_registry_cleanup_and_release(region_worker): worker, calls, _leases = region_worker() resources = _RunResources() diff --git a/tests/ut/py/test_worker/test_comm_region_template.py b/tests/ut/py/test_worker/test_comm_region_template.py new file mode 100644 index 0000000000..6d3b5c7a49 --- /dev/null +++ b/tests/ut/py/test_worker/test_comm_region_template.py @@ -0,0 +1,1027 @@ +# Copyright (c) PyPTO Contributors. +# This program is free software, you can redistribute it and/or modify it under the terms and conditions of +# CANN Open Software License Agreement Version 2.0 (the "License"). +# Please refer to the License for details. You may not use this file except in compliance with the License. +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, +# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. +# See LICENSE in the root of the software repository for the full text of the License. +# ----------------------------------------------------------------------------------------------------------- +# ruff: noqa: PLC0415 + +from __future__ import annotations + +import copy +import inspect +import pickle +import struct +import threading +from dataclasses import dataclass +from enum import Enum +from typing import Any + +import pytest +from simpler.comm_endpoints import ( + DEVICE_AICPU, + HOST_CPU, + EndpointIdentity, + EndpointRecord, + EndpointRegistry, + RegionLayoutSpec, + SingleOwner, + at, +) +from simpler.comm_provider import RegionPartKind, RegionPartLocalView +from simpler.comm_region import ( + MaterializationError, + NotifyOp, + RegionInstanceState, + SignalTestResult, + WaitCmp, +) +from simpler.comm_region_template import ( + _SPSC_QUEUE_ABI_MAJOR, + _SPSC_QUEUE_ABI_MINOR, + _SPSC_QUEUE_ARENA_ALIGNMENT, + _SPSC_QUEUE_COUNTER_BYTES, + _SPSC_QUEUE_COUNTER_STRIDE, + _SPSC_QUEUE_DESCRIPTOR_BYTES, + _SPSC_QUEUE_ENDPOINT_BINDING_SCALAR_COUNT, + _SPSC_QUEUE_INITIATOR_ABORT_OFFSET, + _SPSC_QUEUE_INPUT_DESC_HEAD_OFFSET, + _SPSC_QUEUE_INPUT_DESC_TAIL_OFFSET, + _SPSC_QUEUE_MAGIC, + _SPSC_QUEUE_MAGIC_VERSION, + _SPSC_QUEUE_MAX_DEPTH, + _SPSC_QUEUE_OUTPUT_DESC_HEAD_OFFSET, + _SPSC_QUEUE_OUTPUT_DESC_TAIL_OFFSET, + _SPSC_QUEUE_PEER_ABORT_OFFSET, + SpscQueueEndpointBinding, + SpscQueueOpcode, + _align_up_u64, + _BoundSpscQueue, + _checked_add_u64, + _checked_mul_u64, + _RegionTemplateCoordinator, + _RegionTemplatePlacementRequest, + _require_distinct_initiator_peer, + _resolve_template_slots, + _SpscQueueConfig, + _SpscQueueLayout, + _SpscQueueMessage, + _SpscQueuePlan, + _SpscQueuePlanState, + _SpscQueueSlot, + _SpscQueueState, + _SpscQueueTemplate, + _TemplateSlotBindingRequest, + decode_spsc_queue_descriptor, + encode_spsc_queue_descriptor, + session_instance_id_from_bits, + session_instance_id_to_bits, +) +from simpler.worker import Worker, _Lifecycle + +# Golden vectors shared with tests/ut/cpp/common/test_region_template.cpp. +_LAYOUT_GOLDEN = ( + (1, 64, 64, 32, 64, 128, 192), + (4, 128, 192, 128, 256, 384, 576), + (8, 192, 64, 256, 512, 704, 768), + (2, 64, 128, 64, 128, 192, 320), +) +_MAX_DEPTH_LAYOUT = ( + _SPSC_QUEUE_MAX_DEPTH, + 64, + 64, + _SPSC_QUEUE_MAX_DEPTH * 32, + _SPSC_QUEUE_MAX_DEPTH * 64, + _SPSC_QUEUE_MAX_DEPTH * 64 + 64, + _SPSC_QUEUE_MAX_DEPTH * 64 + 128, +) +_DESCRIPTOR_GOLDEN_FIELDS = (7, 3, 128, 16) +_DESCRIPTOR_GOLDEN_BYTES = bytes( + [ + 0x07, + 0x00, + 0x00, + 0x00, + 0x00, + 0x00, + 0x00, + 0x00, + 0x03, + 0x00, + 0x00, + 0x00, + 0x00, + 0x00, + 0x00, + 0x00, + 0x80, + 0x00, + 0x00, + 0x00, + 0x00, + 0x00, + 0x00, + 0x00, + 0x10, + 0x00, + 0x00, + 0x00, + 0x00, + 0x00, + 0x00, + 0x00, + ] +) +_SESSION_GOLDEN_BYTES = bytes([0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08]) +_SESSION_GOLDEN_BITS = 0x0807060504030201 +_BINDING_GOLDEN: tuple[int, ...] = ( + 0x5350535100010000, + _SESSION_GOLDEN_BITS, + 42, + 0x1000, + 192, + 0x2000, + 384, + 1, + 64, + 64, +) + + +def _layout(depth, input_arena_bytes, output_arena_bytes): + return _SpscQueueLayout.create( + _SpscQueueConfig(depth=depth, input_arena_bytes=input_arena_bytes, output_arena_bytes=output_arena_bytes) + ) + + +def test_packed_magic_version_is_spsq_abi_1_0(): + assert _SPSC_QUEUE_MAGIC == 0x53505351 + assert _SPSC_QUEUE_ABI_MAJOR == 1 + assert _SPSC_QUEUE_ABI_MINOR == 0 + assert _SPSC_QUEUE_MAGIC_VERSION == 0x5350535100010000 + packed = (_SPSC_QUEUE_MAGIC << 32) | (_SPSC_QUEUE_ABI_MAJOR << 16) | _SPSC_QUEUE_ABI_MINOR + assert _SPSC_QUEUE_MAGIC_VERSION == packed + assert _SPSC_QUEUE_ENDPOINT_BINDING_SCALAR_COUNT == 10 + assert _SPSC_QUEUE_DESCRIPTOR_BYTES == 32 + assert _SPSC_QUEUE_ARENA_ALIGNMENT == 64 + assert _SPSC_QUEUE_COUNTER_STRIDE == 64 + assert _SPSC_QUEUE_COUNTER_BYTES == 384 + assert _SPSC_QUEUE_MAX_DEPTH == 1 << 30 + + +@pytest.mark.parametrize( + ("depth", "input_arena_bytes", "output_arena_bytes", "output_desc", "input_arena", "output_arena", "payload"), + _LAYOUT_GOLDEN, +) +def test_layout_golden_vectors( + depth, input_arena_bytes, output_arena_bytes, output_desc, input_arena, output_arena, payload +): + layout = _layout(depth, input_arena_bytes, output_arena_bytes) + assert layout.input_desc_offset == 0 + assert layout.output_desc_offset == output_desc + assert layout.input_arena_offset == input_arena + assert layout.output_arena_offset == output_arena + assert layout.payload_bytes == payload + assert layout.input_arena_bytes == input_arena_bytes + assert layout.output_arena_bytes == output_arena_bytes + assert layout.input_arena_offset % 64 == 0 + assert layout.output_arena_offset % 64 == 0 + assert layout.input_desc_tail_offset == _SPSC_QUEUE_INPUT_DESC_TAIL_OFFSET + assert layout.input_desc_head_offset == _SPSC_QUEUE_INPUT_DESC_HEAD_OFFSET + assert layout.output_desc_tail_offset == _SPSC_QUEUE_OUTPUT_DESC_TAIL_OFFSET + assert layout.output_desc_head_offset == _SPSC_QUEUE_OUTPUT_DESC_HEAD_OFFSET + assert layout.initiator_abort_offset == _SPSC_QUEUE_INITIATOR_ABORT_OFFSET + assert layout.peer_abort_offset == _SPSC_QUEUE_PEER_ABORT_OFFSET + assert layout.counter_bytes == _SPSC_QUEUE_COUNTER_BYTES + + +def test_layout_max_depth_golden_vector(): + depth, input_arena_bytes, output_arena_bytes, output_desc, input_arena, output_arena, payload = _MAX_DEPTH_LAYOUT + layout = _layout(depth, input_arena_bytes, output_arena_bytes) + assert layout.depth == depth + assert layout.output_desc_offset == output_desc + assert layout.input_arena_offset == input_arena + assert layout.output_arena_offset == output_arena + assert layout.payload_bytes == payload + assert layout.input_arena_offset % 64 == 0 + assert layout.output_arena_offset % 64 == 0 + + +def test_layout_is_deterministic_and_has_no_runtime_side_effect(): + config = _SpscQueueConfig(depth=4, input_arena_bytes=128, output_arena_bytes=192) + first = _SpscQueueLayout.create(config) + second = _SpscQueueLayout.create(config) + assert first == second + assert copy.copy(first) == first + + +@pytest.mark.parametrize( + ("depth", "input_arena_bytes", "output_arena_bytes"), + [ + (3, 64, 64), + (0, 64, 64), + ((1 << 30) + 1, 64, 64), + (1 << 31, 64, 64), + (2, 0, 64), + (2, 65, 64), + (2, 64, 0), + (2, 64, 63), + (True, 64, 64), + (2, True, 64), + (2, 64, True), + (-1, 64, 64), + (2, -64, 64), + (1.0, 64, 64), + ], +) +def test_layout_rejects_invalid_depth_and_arena_values(depth, input_arena_bytes, output_arena_bytes): + with pytest.raises((TypeError, ValueError)): + _layout(depth, input_arena_bytes, output_arena_bytes) + + +def test_layout_add_overflow_input_arena(): + with pytest.raises(ValueError, match="overflowed uint64"): + _layout(2, (1 << 64) - 64, 64) + + +def test_layout_add_overflow_output_arena(): + with pytest.raises(ValueError, match="overflowed uint64"): + _layout(1, 64, (1 << 64) - 64) + + +def test_checked_mul_add_align_overflow_boundaries(): + with pytest.raises(ValueError, match="overflowed uint64"): + _checked_mul_u64(1 << 63, 2) + with pytest.raises(ValueError, match="overflowed uint64"): + _checked_add_u64((1 << 64) - 1, 1) + with pytest.raises(ValueError, match="overflowed uint64"): + _align_up_u64((1 << 64) - 32, 64) + assert _checked_mul_u64(_SPSC_QUEUE_MAX_DEPTH, 32) == _SPSC_QUEUE_MAX_DEPTH * 32 + assert _align_up_u64(64, 64) == 64 + assert _align_up_u64(65, 64) == 128 + + +def test_opcode_values_are_fixed(): + assert int(SpscQueueOpcode.INVALID) == 0 + assert int(SpscQueueOpcode.DATA) == 1 + assert int(SpscQueueOpcode.STOP) == 2 + assert int(SpscQueueOpcode.ERROR) == 3 + + +def test_descriptor_bytes_golden_vector(): + encoded = encode_spsc_queue_descriptor( + seq=_DESCRIPTOR_GOLDEN_FIELDS[0], + opcode=SpscQueueOpcode.ERROR, + payload_offset=_DESCRIPTOR_GOLDEN_FIELDS[2], + payload_nbytes=_DESCRIPTOR_GOLDEN_FIELDS[3], + ) + assert encoded == _DESCRIPTOR_GOLDEN_BYTES + assert decode_spsc_queue_descriptor(encoded) == _DESCRIPTOR_GOLDEN_FIELDS + assert struct.calcsize(" bool: + if cmp is WaitCmp.EQ: + return observed == operand + if cmp is WaitCmp.NE: + return observed != operand + if cmp is WaitCmp.GT: + return observed > operand + if cmp is WaitCmp.GE: + return observed >= operand + if cmp is WaitCmp.LT: + return observed < operand + if cmp is WaitCmp.LE: + return observed <= operand + return False + + +class _MemoryCounter: + def __init__(self, region: _MemoryRegion, offset: int) -> None: + self._region = region + self._offset = int(offset) + + def test(self, cmp_value: int, cmp: WaitCmp) -> SignalTestResult: + observed = int(self._region.counters.get(self._offset, 0)) + return SignalTestResult(matched=_compare_counter(observed, int(cmp_value), WaitCmp(cmp)), observed=observed) + + def wait(self, cmp_value: int, cmp: WaitCmp, timeout: float) -> int: + if timeout is None or float(timeout) <= 0: + raise ValueError("region counter wait requires a positive timeout") + if self._region.on_wait is not None: + self._region.on_wait(self._offset) + result = self.test(cmp_value, cmp) + if result.matched: + return int(result.observed) + raise TimeoutError(f"queue counter wait timed out; observed={result.observed}") + + def notify(self, value: int, op: NotifyOp = NotifyOp.Set) -> None: + if self._region.fail_notify is not None: + raise self._region.fail_notify + if op is NotifyOp.Add: + self._region.counters[self._offset] = int(self._region.counters.get(self._offset, 0)) + int(value) + else: + self._region.counters[self._offset] = int(value) & 0xFFFF_FFFF + + +@dataclass +class _MemoryRegion: + layout: RegionLayoutSpec + consumer: EndpointRecord + provider: EndpointRecord + session: bytes + transaction_id: int + payload_base: int = 0x1000 + counter_base: int = 0x2000 + payload: bytearray = None # type: ignore[assignment] + counters: dict[int, int] = None # type: ignore[assignment] + on_wait: object = None + fail_notify: BaseException | None = None + fail_payload: BaseException | None = None + _state: RegionInstanceState = RegionInstanceState.LIVE + + def __post_init__(self) -> None: + if self.payload is None: + self.payload = bytearray(int(self.layout.payload_bytes)) + if self.counters is None: + self.counters = {} + + @property + def state(self) -> RegionInstanceState: + return self._state + + @property + def _allocation_identity(self) -> tuple[bytes, int]: + if len(bytes(self.session)) != 8 or type(self.transaction_id) is not int or self.transaction_id < 1: + raise MaterializationError("region allocation identity is incomplete") + return bytes(self.session), int(self.transaction_id) + + def local_view(self, part: RegionPartKind) -> RegionPartLocalView | None: + if self._state is not RegionInstanceState.LIVE: + return None + if part is RegionPartKind.PAYLOAD: + return RegionPartLocalView(RegionPartKind.PAYLOAD, self.payload_base, int(self.layout.payload_bytes)) + if part is RegionPartKind.COUNTER: + return RegionPartLocalView(RegionPartKind.COUNTER, self.counter_base, int(self.layout.counter_bytes)) + raise ValueError("region part must be PAYLOAD or COUNTER") + + def payload_write(self, offset: int, host_buffer: Any, nbytes: int | None = None) -> None: + if self.fail_payload is not None: + raise self.fail_payload + view = memoryview(host_buffer) + size = int(view.nbytes if nbytes is None else nbytes) + self.payload[int(offset) : int(offset) + size] = bytes(view[:size]) + + def payload_read(self, offset: int, host_buffer: Any, nbytes: int | None = None) -> None: + if self.fail_payload is not None: + raise self.fail_payload + view = memoryview(host_buffer) + size = int(view.nbytes if nbytes is None else nbytes) + view[:size] = bytes(self.payload[int(offset) : int(offset) + size]) + + def counter(self, offset: int) -> _MemoryCounter: + return _MemoryCounter(self, offset) + + +def _endpoint_bundle(): + session = bytes(range(8)) + host = EndpointRecord(EndpointIdentity(session, 1, 0), "L3", HOST_CPU, 0) + peer = EndpointRecord(EndpointIdentity(session, 1, 1), "L3/L2[1]", DEVICE_AICPU, 0) + extra = EndpointRecord(EndpointIdentity(session, 1, 2), "L3/L2[0]", DEVICE_AICPU, 0) + registry = EndpointRegistry( + root_level=3, session_instance_id=session, registry_epoch=1, records=(host, peer, extra) + ) + return session, host, peer, extra, registry + + +def _bind_memory_queue(depth=4, input_arena_bytes=128, output_arena_bytes=192, **region_kwargs): + session, host, peer, _extra, registry = _endpoint_bundle() + config = _SpscQueueConfig(depth, input_arena_bytes, output_arena_bytes) + plan = _SpscQueueTemplate().plan(config) + members = registry.resolve_members((at(host.path, HOST_CPU), at(peer.path, DEVICE_AICPU))) + slots = _resolve_template_slots( + registry, members, _queue_placement(host, peer).slot_bindings, _SpscQueueTemplate.required_slots + ) + region = _MemoryRegion( + layout=plan.region_layout, + consumer=host, + provider=peer, + session=session, + transaction_id=42, + **region_kwargs, + ) + return plan, region, slots, plan._bind(region, slots) + + +def _queue_placement(host: EndpointRecord, peer: EndpointRecord) -> _RegionTemplatePlacementRequest: + return _RegionTemplatePlacementRequest( + members=(at(host.path, host.deployment), at(peer.path, peer.deployment)), + topology=SingleOwner(provider=at(peer.path, peer.deployment)), + slot_bindings=( + _TemplateSlotBindingRequest(_SpscQueueSlot.INITIATOR, at(host.path, host.deployment)), + _TemplateSlotBindingRequest(_SpscQueueSlot.PEER, at(peer.path, peer.deployment)), + ), + ) + + +def test_plan_is_deterministic_and_has_no_runtime_side_effect(): + config = _SpscQueueConfig(4, 128, 192) + first = _SpscQueueTemplate().plan(config) + second = _SpscQueueTemplate().plan(config) + assert first.region_layout == second.region_layout + assert first._layout == second._layout + assert first._state is _SpscQueuePlanState.AVAILABLE + assert second._state is _SpscQueuePlanState.AVAILABLE + assert first.region_layout == RegionLayoutSpec(payload_bytes=576, counter_bytes=_SPSC_QUEUE_COUNTER_BYTES) + + +def test_plan_refuses_copy_deepcopy_and_pickle(): + plan = _SpscQueueTemplate().plan(_SpscQueueConfig(4, 128, 128)) + with pytest.raises(TypeError, match="cannot be copied"): + copy.copy(plan) + with pytest.raises(TypeError, match="cannot be copied"): + copy.deepcopy(plan) + with pytest.raises(TypeError, match="cannot be copied"): + pickle.dumps(plan) + + +def test_slot_resolve_rejects_missing_duplicate_unknown_and_non_member(): + _session, host, peer, extra, registry = _endpoint_bundle() + members = registry.resolve_members((at("L3", HOST_CPU), at("L3/L2[1]", DEVICE_AICPU))) + required = _SpscQueueTemplate.required_slots + with pytest.raises(ValueError, match="missing"): + _resolve_template_slots( + registry, + members, + (_TemplateSlotBindingRequest(_SpscQueueSlot.INITIATOR, at("L3", HOST_CPU)),), + required, + ) + with pytest.raises(ValueError, match="duplicate"): + _resolve_template_slots( + registry, + members, + ( + _TemplateSlotBindingRequest(_SpscQueueSlot.INITIATOR, at("L3", HOST_CPU)), + _TemplateSlotBindingRequest(_SpscQueueSlot.INITIATOR, at("L3/L2[1]", DEVICE_AICPU)), + ), + required, + ) + with pytest.raises(ValueError, match="unknown"): + _resolve_template_slots( + registry, + members, + ( + _TemplateSlotBindingRequest(_SpscQueueSlot.INITIATOR, at("L3", HOST_CPU)), + _TemplateSlotBindingRequest(_SpscQueueSlot.PEER, at("L3/L2[1]", DEVICE_AICPU)), + _TemplateSlotBindingRequest(_UnknownTemplateSlot.EXTRA, at("L3/L2[0]", DEVICE_AICPU)), + ), + required, + ) + with pytest.raises(ValueError, match="not a region member"): + _resolve_template_slots( + registry, + members, + ( + _TemplateSlotBindingRequest(_SpscQueueSlot.INITIATOR, at("L3", HOST_CPU)), + _TemplateSlotBindingRequest(_SpscQueueSlot.PEER, at("L3/L2[0]", DEVICE_AICPU)), + ), + required, + ) + with pytest.raises(ValueError, match="different endpoint"): + same = _resolve_template_slots( + registry, + registry.resolve_members((at("L3", HOST_CPU),)), + ( + _TemplateSlotBindingRequest(_SpscQueueSlot.INITIATOR, at("L3", HOST_CPU)), + _TemplateSlotBindingRequest(_SpscQueueSlot.PEER, at("L3", HOST_CPU)), + ), + required, + ) + _require_distinct_initiator_peer(same) + + +def test_concurrent_first_bind_consumes_once_and_second_bind_does_not_touch_instance(): + session, host, peer, _extra, registry = _endpoint_bundle() + plan = _SpscQueueTemplate().plan(_SpscQueueConfig(4, 128, 128)) + members = registry.resolve_members((at("L3", HOST_CPU), at("L3/L2[1]", DEVICE_AICPU))) + slots = _resolve_template_slots( + registry, members, _queue_placement(host, peer).slot_bindings, _SpscQueueTemplate.required_slots + ) + region = _MemoryRegion(plan.region_layout, host, peer, session, 7) + results: list[object] = [] + errors: list[BaseException] = [] + + def attempt() -> None: + try: + results.append(plan._bind(region, slots)) + except BaseException as exc: # noqa: BLE001 + errors.append(exc) + + threads = [threading.Thread(target=attempt) for _ in range(8)] + for thread in threads: + thread.start() + for thread in threads: + thread.join() + assert len(results) == 1 + assert isinstance(results[0], _BoundSpscQueue) + assert plan._state is _SpscQueuePlanState.CONSUMED + assert all("already consumed" in str(error) for error in errors) + + class _Boom: + def __getattribute__(self, name: str): + raise AssertionError(f"second bind touched {name}") + + with pytest.raises(RuntimeError, match="already consumed"): + plan._bind(_Boom(), slots) + + +def test_bind_rejects_layout_mismatch_and_incomplete_identity(): + session, host, peer, _extra, registry = _endpoint_bundle() + plan = _SpscQueueTemplate().plan(_SpscQueueConfig(4, 128, 128)) + members = registry.resolve_members((at("L3", HOST_CPU), at("L3/L2[1]", DEVICE_AICPU))) + slots = _resolve_template_slots( + registry, members, _queue_placement(host, peer).slot_bindings, _SpscQueueTemplate.required_slots + ) + region = _MemoryRegion(RegionLayoutSpec(64, 384), host, peer, session, 7) + with pytest.raises(ValueError, match="payload_bytes"): + plan._bind(region, slots) + assert plan._state is _SpscQueuePlanState.CONSUMED + + plan = _SpscQueueTemplate().plan(_SpscQueueConfig(4, 128, 128)) + region = _MemoryRegion(plan.region_layout, host, peer, session, 0) + with pytest.raises(MaterializationError, match="allocation identity"): + plan._bind(region, slots) + + +def test_bound_queue_refuses_copy_and_does_not_own_physical_cleanup(): + _plan, region, _slots, queue = _bind_memory_queue() + with pytest.raises(TypeError, match="cannot be copied"): + copy.copy(queue) + with pytest.raises(TypeError, match="cannot be copied"): + pickle.dumps(queue) + queue.free() + assert queue._state is _SpscQueueState.RELEASED + assert region.state is RegionInstanceState.LIVE + with pytest.raises(RuntimeError, match="released"): + queue.input.try_enqueue(b"abc", 3) + + +def test_try_enqueue_oversize_and_blocking_oversize_do_not_touch_shared_state(): + _plan, region, _slots, queue = _bind_memory_queue(input_arena_bytes=64, output_arena_bytes=64) + assert queue.input.try_enqueue(b"x" * 128, 128) is False + assert region.counters == {} + assert bytes(region.payload) == bytes(len(region.payload)) + with pytest.raises(ValueError, match="exceeds input arena"): + queue.input.enqueue(None, 128, timeout=0.1) + assert region.counters == {} + + +def test_duplex_data_zero_byte_wrap_replay_stop_and_error(): + _plan, region, _slots, queue = _bind_memory_queue(depth=4, input_arena_bytes=64, output_arena_bytes=64) + assert queue.input.try_enqueue(None, 0) is True + assert queue.input.try_enqueue(b"x" * 48, 48) is True + region.counters[queue._layout.input_desc_head_offset] = 2 + wrap_ok = queue.input.try_enqueue(b"y" * 32, 32) + assert wrap_ok is True + slot = queue._layout.input_desc_offset + 2 * _SPSC_QUEUE_DESCRIPTOR_BYTES + seq, opcode, offset, nbytes = decode_spsc_queue_descriptor(bytes(region.payload[slot : slot + 32])) + assert seq == 3 + assert opcode == int(SpscQueueOpcode.DATA) + assert nbytes == 32 + assert offset == queue._layout.input_arena_offset + + assert queue.try_request_stop() is True + assert queue.input.try_enqueue(b"no", 2) is False + + out_payload = b"err-data" + out_offset = queue._layout.output_arena_offset + region.payload[out_offset : out_offset + 8] = out_payload + desc = encode_spsc_queue_descriptor(1, SpscQueueOpcode.ERROR, out_offset, 8) + region.payload[queue._layout.output_desc_offset : queue._layout.output_desc_offset + 32] = desc + region.counters[queue._layout.output_desc_tail_offset] = 1 + handle = queue.output.try_peek() + assert handle is not None + assert handle.opcode is SpscQueueOpcode.ERROR + dest = bytearray(8) + queue.output.read_into(handle, dest) + assert bytes(dest) == out_payload + queue.output.release(handle) + + data2 = b"more" + desc2 = encode_spsc_queue_descriptor(2, SpscQueueOpcode.DATA, out_offset, 4) + region.payload[out_offset : out_offset + 4] = data2 + region.payload[queue._layout.output_desc_offset + 32 : queue._layout.output_desc_offset + 64] = desc2 + region.counters[queue._layout.output_desc_tail_offset] = 2 + handle = queue.output.try_peek() + assert handle is not None + assert handle.opcode is SpscQueueOpcode.DATA + dest = bytearray(4) + queue.output.read_into(handle, dest) + queue.output.release(handle) + assert bytes(dest) == data2 + + +def test_output_handle_ownership_and_wait_timeout_abort_failure(): + _plan, region, _slots, queue = _bind_memory_queue() + with pytest.raises(TimeoutError, match="timed out"): + queue.output.peek(0.01) + assert queue._state is _SpscQueueState.LIVE + + def set_abort(_offset: int) -> None: + region.counters[queue._layout.peer_abort_offset] = 1 + + region.on_wait = set_abort + with pytest.raises(RuntimeError, match="remote abort"): + queue.output.peek(0.01) + assert queue._state is _SpscQueueState.POISONED_REMOTE + + _plan, region, _slots, queue = _bind_memory_queue() + out_offset = queue._layout.output_arena_offset + region.payload[out_offset : out_offset + 3] = b"abc" + region.payload[queue._layout.output_desc_offset : queue._layout.output_desc_offset + 32] = ( + encode_spsc_queue_descriptor(1, SpscQueueOpcode.DATA, out_offset, 3) + ) + region.counters[queue._layout.output_desc_tail_offset] = 1 + handle = queue.output.try_peek() + assert handle is not None + forged = _SpscQueueMessage(handle.seq, handle.opcode, handle.payload_offset, handle.payload_nbytes) + with pytest.raises(RuntimeError, match="not active"): + queue.output.release(forged) + assert queue._state is _SpscQueueState.POISONED_LOCAL + assert queue._layout.initiator_abort_offset in region.counters + assert region.counters[queue._layout.initiator_abort_offset] == 1 + + _plan, region, _slots, queue = _bind_memory_queue() + region.fail_payload = RuntimeError("payload write failed") + region.fail_notify = RuntimeError("abort notify failed") + with pytest.raises(RuntimeError, match="payload write failed") as excinfo: + queue.input.try_enqueue(b"hi", 2) + assert queue._state is _SpscQueueState.POISONED_LOCAL + assert queue._first_error is excinfo.value + notes = getattr(excinfo.value, "__notes__", []) + assert any("abort notify failed" in note for note in notes) + + +def test_zero_byte_enqueue_rejects_non_none_and_stop_is_zero_byte(): + _plan, _region, _slots, queue = _bind_memory_queue() + with pytest.raises(ValueError, match="buffer_or_none == None"): + queue.input.try_enqueue(b"", 0) + with pytest.raises(ValueError, match="STOP must be zero-byte"): + queue.input._try_enqueue(None, 4, SpscQueueOpcode.STOP) + + +def test_exact_depth_capacity_blocking_wake_dequeue_and_expiration(): + _plan, region, _slots, queue = _bind_memory_queue(depth=1, input_arena_bytes=64, output_arena_bytes=64) + writes: list[tuple[int, int]] = [] + original_write = region.payload_write + + def record_write(offset, host_buffer, nbytes=None): + writes.append((int(offset), int(nbytes if nbytes is not None else len(host_buffer)))) + return original_write(offset, host_buffer, nbytes) + + region.payload_write = record_write # type: ignore[method-assign] + assert queue.input.try_enqueue(b"abcd", 4) is True + assert writes[0][0] == queue._layout.input_arena_offset + assert writes[1][0] == 8 + assert writes[2][0] == 0 + assert queue.input.try_enqueue(b"efgh", 4) is False + + def advance_head(_offset: int) -> None: + region.counters[queue._layout.input_desc_head_offset] = 1 + + region.on_wait = advance_head + queue.input.enqueue(b"ijkl", 4, timeout=1.0) + assert queue._input_tail == 2 + + out_offset = queue._layout.output_arena_offset + region.payload[out_offset : out_offset + 4] = b"wxyz" + region.payload[queue._layout.output_desc_offset : queue._layout.output_desc_offset + 32] = ( + encode_spsc_queue_descriptor(1, SpscQueueOpcode.DATA, out_offset, 4) + ) + region.counters[queue._layout.output_desc_tail_offset] = 1 + dest = bytearray(4) + message = queue.output.try_dequeue_into(dest) + assert message is not None + assert bytes(dest) == b"wxyz" + with pytest.raises(RuntimeError, match="not active"): + queue.output.release(message) + + _plan, region, _slots, queue = _bind_memory_queue() + region._state = RegionInstanceState.CLOSED + with pytest.raises(RuntimeError, match="expired"): + queue.input.try_enqueue(b"ab", 2) + assert queue._state is _SpscQueueState.EXPIRED + assert region.counters == {} + + +def test_output_stop_poisons_and_handle_invalidates_after_release(): + _plan, region, _slots, queue = _bind_memory_queue() + region.payload[queue._layout.output_desc_offset : queue._layout.output_desc_offset + 32] = ( + encode_spsc_queue_descriptor(1, SpscQueueOpcode.STOP, 0, 0) + ) + region.counters[queue._layout.output_desc_tail_offset] = 1 + with pytest.raises(RuntimeError, match="DATA or ERROR"): + queue.output.try_peek() + assert queue._state is _SpscQueueState.POISONED_LOCAL + + +class _FakeLease: + def __init__(self, calls: list[tuple], name: str, handle: int, *, fail_close: bool = False) -> None: + self._calls = calls + self._name = name + self.handle = handle + self.closed = False + self._fail_close = fail_close + + def close(self) -> None: + if self.closed: + return + self.closed = True + self._calls.append(("mapping_close", self._name)) + if self._fail_close: + raise RuntimeError("mapping close failed") + + +class _FakeNativeWorker: + def __init__(self, calls: list[tuple], *, fail_release: bool = False) -> None: + self._calls = calls + self._fail_release = fail_release + self._last_resource_id = 42 + + def control_payload(self, _worker_type, worker_id, sub_cmd, payload, _timeout): + from simpler.comm_provider import ( + PosixShmImport, + ProviderReleaseResult, + ProviderReleaseStatus, + RegionAllocationResult, + RegionExportDescriptor, + RegionPartExportDescriptor, + RegionPartKind, + RegionPartLocalView, + ) + from simpler.comm_provider_control import ( + DelegatedAllocateReply, + DelegatedAllocateReplyTag, + DelegatedRegionOperation, + DelegatedReleaseReply, + DelegatedReleaseReplyTag, + encode_reply, + parse_request, + publish_reply, + ) + from simpler.worker import _CTRL_DELEGATED_REGION + + assert int(sub_cmd) == _CTRL_DELEGATED_REGION + staged = bytearray(payload) + envelope = parse_request(staged) + if envelope.operation is DelegatedRegionOperation.DELEGATED_ALLOCATE: + request = envelope.decode_terminal() + spec = request.spec + self._calls.append(("allocate", int(spec.payload.logical_bytes), int(spec.counter.logical_bytes))) + result = RegionAllocationResult( + provider_resource_id=42, + export_descriptor=RegionExportDescriptor( + payload=RegionPartExportDescriptor( + spec.payload.planned_backing_kind, + int(spec.payload.logical_bytes), + int(spec.payload.logical_bytes), + PosixShmImport("/pto_payload_42"), + ), + counter=RegionPartExportDescriptor( + spec.counter.planned_backing_kind, + int(spec.counter.logical_bytes), + int(spec.counter.logical_bytes), + PosixShmImport("/pto_counter_42"), + ), + ), + ) + payload_view = RegionPartLocalView(RegionPartKind.PAYLOAD, 0x1000, int(spec.payload.logical_bytes)) + counter_view = RegionPartLocalView(RegionPartKind.COUNTER, 0x2000, int(spec.counter.logical_bytes)) + committed = encode_reply( + DelegatedAllocateReply( + tag=DelegatedAllocateReplyTag.ALLOCATED, + session_instance_id=envelope.session_instance_id, + transaction_id=envelope.transaction_id, + result=result, + payload_view=payload_view, + counter_view=counter_view, + ) + ) + publish_reply(memoryview(staged), committed) + return bytes(staged) + self._calls.append(("release", int(worker_id), int(self._last_resource_id))) + request = envelope.decode_terminal() + committed = encode_reply( + DelegatedReleaseReply( + tag=DelegatedReleaseReplyTag.RELEASED, + session_instance_id=envelope.session_instance_id, + transaction_id=request.transaction_id, + result=ProviderReleaseResult( + provider_resource_id=int(self._last_resource_id), + status=ProviderReleaseStatus.RELEASED, + ), + ) + ) + publish_reply(memoryview(staged), committed) + if self._fail_release: + raise RuntimeError("release failed") + return bytes(staged) + + +@pytest.fixture +def region_worker(monkeypatch): + def build(*, fail_mapping_close: bool = False, fail_release: bool = False, device_ids=(8, 9)): + worker = Worker(level=3, device_ids=list(device_ids)) + worker._lifecycle = _Lifecycle.READY + worker._worker = object() + worker._next_level_worker_ids = list(range(len(device_ids))) + worker._config = {**worker._config, "platform": "a2a3sim", "device_ids": list(device_ids)} + calls: list[tuple] = [] + leases: list[_FakeLease] = [] + worker._worker = _FakeNativeWorker(calls, fail_release=fail_release) + monkeypatch.setattr(worker, "_consume_worker_host_mapped_cleanup_error", lambda _api: None) + + def fake_import(_worker_id, _resource_id, export): + name = "payload" if not leases else "counter" + lease = _FakeLease(calls, name, handle=100 + len(leases), fail_close=fail_mapping_close) + calls.append(("import", name, int(export.logical_bytes))) + leases.append(lease) + return lease + + monkeypatch.setattr(worker, "_import_region_part_lease", fake_import) + return worker, calls, leases + + return build + + +def test_coordinator_create_projects_bound_queue_without_escaping_instance(region_worker): + worker, calls, _leases = region_worker() + captured: dict[str, object] = {} + + def projector(bound): + captured["bound"] = bound + captured["instance"] = bound._instance + return bound + + host_sel = at("L3", HOST_CPU) + peer_sel = at("L3/L2[1]", DEVICE_AICPU) + placement = _RegionTemplatePlacementRequest( + members=(host_sel, peer_sel), + topology=SingleOwner(provider=peer_sel), + slot_bindings=( + _TemplateSlotBindingRequest(_SpscQueueSlot.INITIATOR, host_sel), + _TemplateSlotBindingRequest(_SpscQueueSlot.PEER, peer_sel), + ), + ) + result = _RegionTemplateCoordinator(worker).create( + template=_SpscQueueTemplate(), + config=_SpscQueueConfig(4, 128, 192), + placement=placement, + result_projector=projector, + ) + assert result is captured["bound"] + assert isinstance(result, _BoundSpscQueue) + assert result is not captured["instance"] + assert result.endpoint_binding.to_scalars()[0] == _SPSC_QUEUE_MAGIC_VERSION + assert len(result.endpoint_binding.to_scalars()) == 10 + assert result._desc_fields == bytearray(24) + assert result._desc_seq == bytearray(8) + assert result._desc_read == bytearray(32) + assert ("allocate", 576, 384) in calls + assert not any(item[0] == "counter_notify" for item in calls) + + +def test_coordinator_bind_and_projector_failure_close_instance(region_worker): + worker, calls, _leases = region_worker() + host_sel = at("L3", HOST_CPU) + peer_sel = at("L3/L2[1]", DEVICE_AICPU) + placement = _RegionTemplatePlacementRequest( + members=(host_sel, peer_sel), + topology=SingleOwner(provider=peer_sel), + slot_bindings=( + _TemplateSlotBindingRequest(_SpscQueueSlot.INITIATOR, host_sel), + _TemplateSlotBindingRequest(_SpscQueueSlot.PEER, peer_sel), + ), + ) + + def fail_bind(self, instance, slots): + self._consume_for_bind() + raise RuntimeError("injected bind failure") + + original = _SpscQueuePlan._bind + _SpscQueuePlan._bind = fail_bind + try: + with pytest.raises(RuntimeError, match="injected bind failure"): + _RegionTemplateCoordinator(worker).create( + template=_SpscQueueTemplate(), + config=_SpscQueueConfig(4, 64, 64), + placement=placement, + result_projector=lambda bound: bound, + ) + finally: + _SpscQueuePlan._bind = original + assert any(item[0] == "release" for item in calls) + assert worker._region_instance_registry._instances == {} + + +def test_coordinator_cleanup_failure_records_unreclaimable_and_keeps_first_error(region_worker): + worker, calls, _leases = region_worker(fail_mapping_close=True) + host_sel = at("L3", HOST_CPU) + peer_sel = at("L3/L2[1]", DEVICE_AICPU) + placement = _RegionTemplatePlacementRequest( + members=(host_sel, peer_sel), + topology=SingleOwner(provider=peer_sel), + slot_bindings=( + _TemplateSlotBindingRequest(_SpscQueueSlot.INITIATOR, host_sel), + _TemplateSlotBindingRequest(_SpscQueueSlot.PEER, peer_sel), + ), + ) + with pytest.raises(RuntimeError, match="projector failed") as excinfo: + _RegionTemplateCoordinator(worker).create( + template=_SpscQueueTemplate(), + config=_SpscQueueConfig(4, 64, 64), + placement=placement, + result_projector=lambda _bound: (_ for _ in ()).throw(RuntimeError("projector failed")), + ) + assert str(excinfo.value) == "projector failed" + assert worker._ordered_cleanup_error is not None + + +def test_production_facade_uses_coordinator(): + from simpler import worker_chip_message_queue + from simpler.worker_chip_message_queue import create_worker_chip_queue + + source = inspect.getsource(create_worker_chip_queue) + assert "_RegionTemplateCoordinator" in source + assert "create_worker_chip_region" not in source + module_source = inspect.getsource(worker_chip_message_queue) + assert "time.sleep" not in module_source + assert "orch.alloc" not in module_source diff --git a/tests/ut/py/test_worker/test_worker_chip_message_queue.py b/tests/ut/py/test_worker/test_worker_chip_message_queue.py index 451e5b28c7..0316d38b5d 100644 --- a/tests/ut/py/test_worker/test_worker_chip_message_queue.py +++ b/tests/ut/py/test_worker/test_worker_chip_message_queue.py @@ -8,7 +8,6 @@ # ----------------------------------------------------------------------------------------------------------- import ctypes -import itertools import math import struct from dataclasses import dataclass @@ -16,9 +15,10 @@ from typing import Optional import pytest +import simpler.worker_chip_message_queue as queue_mod from simpler import comm_region from simpler import worker as worker_module -from simpler.buffer import AccessMode, BackendKind, CanonicalIdentity, mint_owner_instance_id, wrap_fork_inherited +from simpler.buffer import BackendKind from simpler.comm_provider import ( PosixShmImport, ProviderReleaseResult, @@ -54,7 +54,6 @@ WORKER_CHIP_QUEUE_COUNTER_BYTES, WORKER_CHIP_QUEUE_DESC_SLOT_BYTES, WORKER_CHIP_QUEUE_WORKER_ABORT_FLAG_OFFSET, - WorkerChipQueue, WorkerChipQueueMessage, WorkerChipQueueOpcode, make_worker_chip_queue_layout, @@ -62,54 +61,8 @@ from simpler.worker_chip_orch_comm import ( NotifyOp, WaitCmp, - WorkerChipOrchRegion, - WorkerChipOrchRegionDesc, ) -_DESC_BID = itertools.count(1) - - -class _CompatInstance: - def __init__(self, payload_handle: int, counter_handle: int, payload_bytes: int, counter_bytes: int) -> None: - self.worker_id = 0 - self._data_plane_error = None - self._payload_part = comm_region.PayloadPart( - comm_region.RegionPartSpan(offset=0, nbytes=int(payload_bytes)), - comm_region.HostVmmCopyAccess(payload_handle), - ) - self._counter_part = comm_region.CounterPart( - comm_region.RegionPartSpan(offset=0, nbytes=int(counter_bytes)), - comm_region.HostVmmCopyAccess(counter_handle), - ) - - @property - def state(self): - return comm_region.RegionInstanceState.LIVE - - @property - def data_plane_error(self): - return self._data_plane_error - - def payload_write(self, offset, host_buffer, nbytes=None): - self._payload_part.write(offset, host_buffer, nbytes) - - def payload_read(self, offset, host_buffer, nbytes=None): - self._payload_part.read(offset, host_buffer, nbytes) - - def counter(self, offset): - return self._counter_part.counter(offset) - - -def _fake_alloc_handle(orch, nbytes): - """A FORK_SHM Buffer over a bare _FakeCOrch alloc — mirrors Orchestrator.alloc for the - low-level tests that drive WorkerChipQueue with a fake C orch directly.""" - oid, bid = mint_owner_instance_id(), next(_DESC_BID) - identity = CanonicalIdentity(oid, bid) - va = int(orch.alloc([nbytes], DataType.UINT8, identity)) - return wrap_fork_inherited( - va, nbytes, oid, bid, "L3", access=AccessMode.READWRITE, backend_kind=BackendKind.FORK_SHM - ) - @dataclass(frozen=True) class _FakeRequest: @@ -463,8 +416,8 @@ def test_layout_lockstep_cases_match_cpp_helper_expectations(depth, input_arena_ assert layout.input_desc_head_offset == 64 assert layout.output_desc_tail_offset == 128 assert layout.output_desc_head_offset == 192 - assert layout.worker_abort_flag_offset == WORKER_CHIP_QUEUE_WORKER_ABORT_FLAG_OFFSET - assert layout.chip_abort_flag_offset == WORKER_CHIP_QUEUE_CHIP_ABORT_FLAG_OFFSET + assert layout.initiator_abort_offset == WORKER_CHIP_QUEUE_WORKER_ABORT_FLAG_OFFSET + assert layout.peer_abort_offset == WORKER_CHIP_QUEUE_CHIP_ABORT_FLAG_OFFSET assert layout.counter_bytes == WORKER_CHIP_QUEUE_COUNTER_BYTES @@ -477,42 +430,41 @@ def test_create_worker_chip_queue_allocates_region_and_exposes_l2_task_scalars() assert alloc_req.cmd == "alloc_region" assert alloc_req.payload_bytes == queue.layout.payload_bytes assert alloc_req.counter_bytes == WORKER_CHIP_QUEUE_COUNTER_BYTES - assert queue.chip_task_arg_scalars() == [ - *queue.region.descriptor_scalars(), + session, transaction_id = queue.region._instance._allocation_identity + scalars = queue.chip_task_arg_scalars() + assert len(scalars) == 10 + assert scalars == [ queue.magic_version, + int.from_bytes(session, "little"), + transaction_id, + queue.region.descriptor.payload_base, + queue.region.descriptor.payload_bytes, + queue.region.descriptor.counter_base, + queue.region.descriptor.counter_bytes, 4, 128, 192, - queue.layout.payload_bytes, - queue.layout.counter_bytes, ] - assert fake_client.counters == { - queue.layout.input_desc_tail_offset: 0, - queue.layout.input_desc_head_offset: 0, - queue.layout.output_desc_tail_offset: 0, - queue.layout.output_desc_head_offset: 0, - queue.layout.worker_abort_flag_offset: 0, - queue.layout.chip_abort_flag_offset: 0, - } + assert queue.magic_version == 0x5350535100010000 + assert queue.region.descriptor.magic_version != scalars[0] + assert queue.region._instance is queue._bound._instance + assert all(req.cmd != "counter_notify" for req, _timeout in fake_client.requests) finally: _close(worker, shm) -def test_create_worker_chip_queue_frees_region_on_post_region_alloc_failure(): +def test_create_worker_chip_queue_projector_failure_rolls_back(monkeypatch): orch, worker, shm, _fake_client = _make_orchestrator() - original_alloc_ref = orch._o.alloc - def fail_alloc(_shape, _dtype, _identity): - raise RuntimeError("injected alloc failure") + def fail_desc(*_args, **_kwargs): + raise RuntimeError("injected projector failure") - orch._o.alloc = fail_alloc + monkeypatch.setattr(queue_mod, "worker_chip_orch_region_desc_from_local_views", fail_desc) try: - with pytest.raises(RuntimeError, match="injected alloc failure"): + with pytest.raises(RuntimeError, match="injected projector failure"): orch.create_worker_chip_queue(worker_id=0, depth=4, input_arena_bytes=128, output_arena_bytes=128) - - assert len(worker._region_instance_registry._instances) == 1 + assert worker._region_instance_registry._instances == {} finally: - orch._o.alloc = original_alloc_ref _close(worker, shm) @@ -594,53 +546,23 @@ def test_enqueue_accepts_ordinary_host_bytes_with_direct_payload_write(): _close(worker, shm) -def test_worker_host_mapped_queue_ordinary_input_uses_direct_payload_write(monkeypatch): - orch = _FakeCOrch() - layout = make_worker_chip_queue_layout(4, 128, 128) - desc = WorkerChipOrchRegionDesc( - magic_version=0x4C334C3200030000, - region_id=1, - payload_base=0x1000_0000, - payload_bytes=layout.payload_bytes, - counter_base=0x1000_0000 + ((layout.payload_bytes + 63) // 64) * 64, - counter_bytes=layout.counter_bytes, - ) - region = WorkerChipOrchRegion( - object(), - _CompatInstance(44, 45, desc.payload_bytes, desc.counter_bytes), - desc, - ) - queue = WorkerChipQueue( - orch, - region, - layout, - _fake_alloc_handle(orch, 24), - _fake_alloc_handle(orch, 8), - _fake_alloc_handle(orch, WORKER_CHIP_QUEUE_DESC_SLOT_BYTES), - ) - alloc_count = len(orch._buffers) - payload_writes: list[tuple[int, bytes]] = [] - counters: dict[int, int] = {} - - def payload_write(_handle: int, offset: int, src: int, nbytes: int) -> None: - payload_writes.append((int(offset), ctypes.string_at(int(src), int(nbytes)))) - - def counter_notify(_handle: int, offset: int, value: int, _op: int) -> None: - counters[int(offset)] = int(value) - - monkeypatch.setattr(comm_region, "_host_vmm_copy_to", payload_write) - monkeypatch.setattr( - comm_region, - "_region_counter_test", - lambda _h, off, _v, _cmp: (False, counters.get(off, 0)), - ) - monkeypatch.setattr(comm_region, "_region_counter_notify", counter_notify) +def test_create_worker_chip_queue_does_not_call_create_worker_chip_region(monkeypatch): + orch, worker, shm, _fake_client = _make_orchestrator() + calls: list[str] = [] - queue.input.enqueue(b"ordinary", nbytes=8, timeout=0.001) + def boom(*_args, **_kwargs): + calls.append("legacy") + raise AssertionError("create_worker_chip_region must not be called") - assert (layout.input_arena_offset, b"ordinary") in payload_writes - assert len(orch._buffers) == alloc_count - assert counters[layout.input_desc_tail_offset] == 1 + monkeypatch.setattr(orch, "create_worker_chip_region", boom) + monkeypatch.setattr(worker, "_create_worker_chip_region", boom) + try: + queue = orch.create_worker_chip_queue(worker_id=0, depth=4, input_arena_bytes=128, output_arena_bytes=128) + assert calls == [] + assert queue.region._instance is queue._bound._instance + assert len(queue.chip_task_arg_scalars()) == 10 + finally: + _close(worker, shm) def test_direct_mapped_ordinary_host_bytearray_does_not_allocate_queue_buffer(): @@ -787,7 +709,7 @@ def test_output_release_inactive_handle_poisons_and_sets_worker_abort_flag(): queue.output.release(wrong) assert fake_client.counters[WORKER_CHIP_QUEUE_WORKER_ABORT_FLAG_OFFSET] == 1 - with pytest.raises(RuntimeError, match="poisoned"): + with pytest.raises(RuntimeError, match="not active"): queue.output.try_peek() finally: _close(worker, shm) @@ -799,7 +721,7 @@ def test_output_stop_descriptor_poisons_and_sets_worker_abort_flag(): queue = orch.create_worker_chip_queue(worker_id=0, depth=4, input_arena_bytes=128, output_arena_bytes=128) _publish_output(fake_client, queue, opcode=int(WorkerChipQueueOpcode.STOP)) - with pytest.raises(RuntimeError, match="cannot be STOP"): + with pytest.raises(RuntimeError, match="DATA or ERROR"): queue.output.peek(timeout=0.001) assert fake_client.counters[WORKER_CHIP_QUEUE_WORKER_ABORT_FLAG_OFFSET] == 1 @@ -917,7 +839,7 @@ def test_try_enqueue_wraparound_arena_full_ordinary_buffer_does_not_stage_or_adv first = orch.alloc([112], DataType.UINT8) queue.input.enqueue(first, nbytes=112, timeout=0.001) alloc_count = len(orch._o._buffers) - old_payload_tail = queue._input_payload_tail + old_payload_tail = queue._bound._input_payload_tail fake_client.requests.clear() fake_client.payload_writes.clear() @@ -925,7 +847,7 @@ def test_try_enqueue_wraparound_arena_full_ordinary_buffer_does_not_stage_or_adv assert fake_client.payload_writes == [] assert len(orch._o._buffers) == alloc_count - assert queue._input_payload_tail == old_payload_tail + assert queue._bound._input_payload_tail == old_payload_tail assert fake_client.counters[queue.layout.input_desc_tail_offset] == 1 assert fake_client.counters.get(WORKER_CHIP_QUEUE_WORKER_ABORT_FLAG_OFFSET, 0) == 0 finally: @@ -967,7 +889,7 @@ def test_enqueue_payload_write_failure_sets_worker_abort_flag(): queue.input.enqueue(host, nbytes=16, timeout=0.001) assert fake_client.counters[WORKER_CHIP_QUEUE_WORKER_ABORT_FLAG_OFFSET] == 1 - with pytest.raises(RuntimeError, match="poisoned"): + with pytest.raises(RuntimeError, match="injected failure"): queue.input.try_enqueue(None, nbytes=0) finally: _close(worker, shm) @@ -1035,3 +957,41 @@ def test_expired_queue_rejects_later_operations_without_abort_flag(): assert fake_client.counters.get(WORKER_CHIP_QUEUE_WORKER_ABORT_FLAG_OFFSET, 0) == 0 finally: _close(worker, shm) + + +def test_create_worker_chip_queue_free_is_logical_only(): + orch, worker, shm, fake_client = _make_orchestrator() + try: + queue = orch.create_worker_chip_queue(worker_id=0, depth=4, input_arena_bytes=128, output_arena_bytes=128) + instance = queue.region._instance + fake_client.requests.clear() + queue.free() + queue.free() + assert instance.state is comm_region.RegionInstanceState.LIVE + assert instance._close_attempted is False + with pytest.raises(RuntimeError, match="released"): + queue.chip_task_arg_scalars() + with pytest.raises(RuntimeError, match="released"): + queue.input.try_enqueue(None, nbytes=0) + with pytest.raises(RuntimeError, match="released"): + queue.region.descriptor_scalars() + assert all(req.cmd != "counter_notify" for req, _timeout in fake_client.requests) + finally: + _close(worker, shm) + + +def test_create_worker_chip_queue_publish_survives_later_caller_failure(): + orch, worker, shm, _fake_client = _make_orchestrator() + try: + queue = orch.create_worker_chip_queue(worker_id=0, depth=4, input_arena_bytes=128, output_arena_bytes=128) + instance = queue.region._instance + try: + raise RuntimeError("submit failed") + except RuntimeError: + pass + assert instance.state is comm_region.RegionInstanceState.LIVE + assert len(queue.chip_task_arg_scalars()) == 10 + queue.input.enqueue(None, nbytes=0, timeout=0.001) + assert instance._close_attempted is False + finally: + _close(worker, shm) From 8fddb112801dde5531ae3a3a3ffb1928dfcdaed7 Mon Sep 17 00:00:00 2001 From: ccyywwen <75376396+ccyywwen@users.noreply.github.com> Date: Thu, 3 Sep 2026 15:44:55 +0800 Subject: [PATCH 2/7] Fix: close SPSC queue identity, admission, and replay holes Keep SPSC role proof on the plan, not the generic coordinator. Reject zero transaction IDs before a binding is trusted. Admit exact ints and worker-chip payloads before shared access. Align Python replay and signed low-32 counters with native, treat zero timeout as a live no-attempt, and track example occupancy without using request_id as a sentinel. --- .../worker_chip_message_queue_orch.cpp | 25 +- .../test_worker_chip_message_queue.py | 16 +- python/simpler/comm_region_template.py | 91 ++-- python/simpler/orchestrator.py | 8 +- python/simpler/worker_chip_message_queue.py | 166 ++++++- .../platform/include/common/region_template.h | 18 +- tests/ut/cpp/common/test_region_template.cpp | 269 ++++++++++- .../test_worker/test_comm_region_template.py | 435 +++++++++++++++++- .../test_worker_chip_message_queue.py | 219 ++++++++- 9 files changed, 1146 insertions(+), 101 deletions(-) diff --git a/examples/workers/l3/worker_chip_message_queue/kernels/orchestration/worker_chip_message_queue_orch.cpp b/examples/workers/l3/worker_chip_message_queue/kernels/orchestration/worker_chip_message_queue_orch.cpp index 1148d3ae06..57ffbbc967 100644 --- a/examples/workers/l3/worker_chip_message_queue/kernels/orchestration/worker_chip_message_queue_orch.cpp +++ b/examples/workers/l3/worker_chip_message_queue/kernels/orchestration/worker_chip_message_queue_orch.cpp @@ -55,6 +55,7 @@ using QueueEndpoint = spsc_queue::SpscQueueEndpoint list[float]: def _input_payloads() -> list[bytes]: inputs = _input_tiles() return [ - _pack_input(101, 1, inputs[0]), - _pack_input(102, 2, inputs[1]), - _pack_input(103, 3, inputs[2]), - _pack_input(104, 3, inputs[3]), + _pack_input(0, 1, inputs[0]), + _pack_input(0, 2, inputs[1]), + _pack_input(0, 3, inputs[2]), + _pack_input(7, 3, inputs[3]), ] def _expected_outputs() -> list[bytes]: inputs = _input_tiles() return [ - _pack_output(102, 20, 0, _add_scalar(inputs[1], 20.0)), - _pack_output(101, 10, 0, _add_scalar(inputs[0], 10.0)), - _pack_output(101, 11, 0, _add_scalar(inputs[0], 11.0)), - _pack_output(103, 30, 104, _add_tiles(inputs[2], inputs[3])), + _pack_output(0, 20, 0, _add_scalar(inputs[1], 20.0)), + _pack_output(0, 10, 0, _add_scalar(inputs[0], 10.0)), + _pack_output(0, 11, 0, _add_scalar(inputs[0], 11.0)), + _pack_output(0, 30, 7, _add_tiles(inputs[2], inputs[3])), ] diff --git a/python/simpler/comm_region_template.py b/python/simpler/comm_region_template.py index 5e0ff3fa2a..aec70b3386 100644 --- a/python/simpler/comm_region_template.py +++ b/python/simpler/comm_region_template.py @@ -73,6 +73,17 @@ class SpscQueueOpcode(IntEnum): ERROR = 3 +def _counter_low32(value: int) -> int: + return ctypes.c_int32(int(value) & 0xFFFFFFFF).value + + +def _payload_expected_offset(cursor: int, nbytes: int, arena_offset: int, arena_bytes: int) -> int: + arena_pos = int(cursor) % int(arena_bytes) + if arena_pos + int(nbytes) > int(arena_bytes): + return int(arena_offset) + return int(arena_offset) + arena_pos + + def _require_exact_u64(name: str, value: object) -> int: if type(value) is not int: raise TypeError(f"{name} must be an exact int") @@ -243,6 +254,8 @@ def __post_init__(self) -> None: ) if self.magic_version != _SPSC_QUEUE_MAGIC_VERSION: raise ValueError("binding magic_version is not SPSQ ABI 1.0") + if self.transaction_id == 0: + raise ValueError("transaction_id must be nonzero") def to_scalars(self) -> tuple[int, ...]: return ( @@ -269,6 +282,8 @@ def from_scalars(cls, scalars: Sequence[object]) -> SpscQueueEndpointBinding: values = tuple(_require_exact_u64(f"binding[{index}]", scalars[index]) for index in range(count)) if values[0] != _SPSC_QUEUE_MAGIC_VERSION: raise ValueError("binding magic_version is not SPSQ ABI 1.0") + if values[2] == 0: + raise ValueError("transaction_id must be nonzero") return cls( magic_version=values[0], session_instance_id_bits=values[1], @@ -335,6 +350,8 @@ def endpoint(self, slot: Enum) -> EndpointRecord: class _RegionTemplate(Protocol): + required_slots: Sequence[Enum] + def plan(self, config: object) -> _RegionTemplatePlan: ... @@ -349,6 +366,18 @@ def _reject_copy(obj: object) -> NoReturn: raise TypeError(f"{type(obj).__name__} cannot be copied") +def _attach_exception_note(error: BaseException, note: str) -> None: + adder = getattr(error, "add_note", None) + if callable(adder): + adder(note) + return + notes = getattr(error, "__notes__", None) + if isinstance(notes, list): + notes.append(note) + return + object.__setattr__(error, "__notes__", [note]) + + def _resolve_template_slots( registry: object, resolved_members: Sequence[EndpointRecord], @@ -431,7 +460,11 @@ def _bind(self, instance: object, slots: _ResolvedTemplateSlots) -> _BoundSpscQu raise ValueError("region payload_bytes do not match the queue plan") if int(layout.counter_bytes) != int(self._layout.counter_bytes): raise ValueError("region counter_bytes do not match the queue plan") - _require_distinct_initiator_peer(slots) + initiator, peer = _require_distinct_initiator_peer(slots) + if instance.consumer.identity != initiator.identity: + raise ValueError("INITIATOR slot does not match the materialized consumer access endpoint") + if instance.provider.identity != peer.identity: + raise ValueError("PEER slot does not match the materialized provider-local view endpoint") payload_view = instance.local_view(RegionPartKind.PAYLOAD) counter_view = instance.local_view(RegionPartKind.COUNTER) if payload_view is None or counter_view is None: @@ -503,11 +536,9 @@ def _create_in_transaction( plan = template.plan(config) registry = worker._get_endpoint_registry() resolved_region = registry.resolve_region_spec(placement.members, placement.topology) - required = tuple(getattr(template, "required_slots", ())) - if not required: - required = tuple(binding.slot for binding in placement.slot_bindings) - slots = _resolve_template_slots(registry, resolved_region.members, placement.slot_bindings, required) - _require_distinct_initiator_peer(slots) + slots = _resolve_template_slots( + registry, resolved_region.members, placement.slot_bindings, tuple(template.required_slots) + ) backend_plan = BackendResolver(registry, worker._get_region_access_service()).plan( resolved_region, plan.region_layout ) @@ -526,7 +557,6 @@ def _create_in_transaction( layout=plan.region_layout, ) ) - self._prove_initiator_peer_access(instance, slots) bound = plan._bind(instance, slots) projected = result_projector(bound) published = True @@ -536,13 +566,6 @@ def _create_in_transaction( self._rollback_unpublished(instance) raise - def _prove_initiator_peer_access(self, instance: object, slots: _ResolvedTemplateSlots) -> None: - initiator, peer = _require_distinct_initiator_peer(slots) - if instance.consumer.identity != initiator.identity: - raise ValueError("INITIATOR slot does not match the materialized consumer access endpoint") - if instance.provider.identity != peer.identity: - raise ValueError("PEER slot does not match the materialized provider-local view endpoint") - def _rollback_unpublished(self, instance: object) -> None: try: live = instance.state is RegionInstanceState.LIVE @@ -637,9 +660,7 @@ def _poison_local(self, error: BaseException) -> None: try: self._instance.counter(self._layout.initiator_abort_offset).notify(1, NotifyOp.Set) except BaseException as abort_error: - adder = getattr(error, "add_note", None) - if callable(adder): - adder(f"initiator abort notify failed: {abort_error}") + _attach_exception_note(error, f"initiator abort notify failed: {abort_error}") def _run_primitive(self, fn: Callable[..., Any], *args: Any, **kwargs: Any) -> Any: self._ensure_usable() @@ -652,10 +673,10 @@ def _run_primitive(self, fn: Callable[..., Any], *args: Any, **kwargs: Any) -> A raise def _signal_test(self, offset: int, cmp_value: int, cmp: WaitCmp) -> Any: - return self._run_primitive(lambda: self._instance.counter(offset).test(int(cmp_value), cmp)) + return self._run_primitive(lambda: self._instance.counter(offset).test(_counter_low32(cmp_value), cmp)) def _signal_notify(self, offset: int, value: int) -> None: - self._run_primitive(lambda: self._instance.counter(offset).notify(int(value), NotifyOp.Set)) + self._run_primitive(lambda: self._instance.counter(offset).notify(_counter_low32(value), NotifyOp.Set)) def _sample_peer_abort(self) -> None: result = self._signal_test(self._layout.peer_abort_offset, 1, WaitCmp.GE) @@ -664,7 +685,7 @@ def _sample_peer_abort(self) -> None: raise RuntimeError("SPSC queue remote abort observed") def _refresh_counter(self, offset: int, local_value: int) -> int: - result = self._signal_test(offset, local_value & 0xFFFF_FFFF, WaitCmp.NE) + result = self._signal_test(offset, local_value, WaitCmp.NE) if not result.matched: return local_value observed = int(result.observed) & 0xFFFF_FFFF @@ -690,7 +711,7 @@ def _wait_progress(self, offset: int, local_value: int, deadline_ns: int) -> Non raise TimeoutError("SPSC queue operation timed out") try: self._run_primitive( - lambda: self._instance.counter(offset).wait(local_value & 0xFFFF_FFFF, WaitCmp.NE, remaining) + lambda: self._instance.counter(offset).wait(_counter_low32(local_value), WaitCmp.NE, remaining) ) except TimeoutError: self._sample_peer_abort() @@ -736,13 +757,14 @@ def _advance_payload_head( ) -> int: if payload_nbytes == 0: return cursor - expected_offset = arena_offset + (cursor % arena_bytes) + expected_offset = _payload_expected_offset(cursor, payload_nbytes, arena_offset, arena_bytes) if expected_offset != payload_offset: - if payload_offset != arena_offset: - error = RuntimeError("SPSC queue payload replay offset mismatch") - self._poison_local(error) - raise error - cursor += arena_bytes - (cursor % arena_bytes) + error = RuntimeError("SPSC queue payload replay offset mismatch") + self._poison_local(error) + raise error + arena_pos = cursor % arena_bytes + if arena_pos + payload_nbytes > arena_bytes: + cursor += arena_bytes - arena_pos return cursor + payload_nbytes def _replay_released_input_descriptors(self, old_head: int, new_head: int) -> None: @@ -765,13 +787,16 @@ def _replay_released_input_descriptors(self, old_head: int, new_head: int) -> No cursor += 1 def _reserve_input_payload(self, nbytes: int, next_payload_tail: int) -> tuple[int, int] | None: - arena_pos = next_payload_tail % self._layout.input_arena_bytes - if arena_pos + nbytes > self._layout.input_arena_bytes: - next_payload_tail += self._layout.input_arena_bytes - arena_pos - arena_pos = 0 - if next_payload_tail + nbytes - self._input_payload_head > self._layout.input_arena_bytes: + arena_bytes = self._layout.input_arena_bytes + expected_offset = _payload_expected_offset( + next_payload_tail, nbytes, self._layout.input_arena_offset, arena_bytes + ) + arena_pos = next_payload_tail % arena_bytes + if arena_pos + nbytes > arena_bytes: + next_payload_tail += arena_bytes - arena_pos + if next_payload_tail + nbytes - self._input_payload_head > arena_bytes: return None - return self._layout.input_arena_offset + arena_pos, next_payload_tail + return expected_offset, next_payload_tail def __copy__(self) -> _BoundSpscQueue: _reject_copy(self) diff --git a/python/simpler/orchestrator.py b/python/simpler/orchestrator.py index 4d01b6fac5..c3135521c2 100644 --- a/python/simpler/orchestrator.py +++ b/python/simpler/orchestrator.py @@ -676,10 +676,10 @@ def create_worker_chip_queue(self, *, worker_id: int, depth: int, input_arena_by return create_worker_chip_queue( self, - worker_id=int(worker_id), - depth=int(depth), - input_arena_bytes=int(input_arena_bytes), - output_arena_bytes=int(output_arena_bytes), + worker_id=worker_id, + depth=depth, + input_arena_bytes=input_arena_bytes, + output_arena_bytes=output_arena_bytes, ) # ------------------------------------------------------------------ diff --git a/python/simpler/worker_chip_message_queue.py b/python/simpler/worker_chip_message_queue.py index 64939aee41..3a5ca32df1 100644 --- a/python/simpler/worker_chip_message_queue.py +++ b/python/simpler/worker_chip_message_queue.py @@ -10,8 +10,10 @@ from __future__ import annotations +import operator from typing import Any +from .buffer import AccessMode, Buffer from .comm_endpoints import DEVICE_AICPU, HOST_CPU, SingleOwner, _format_worker_path, at from .comm_provider import RegionPartKind from .comm_region_template import ( @@ -42,12 +44,76 @@ WorkerChipQueueLayout = _SpscQueueLayout +def _require_worker_chip_id(value: object) -> int: + if type(value) is bool: + raise TypeError("worker_id must not be bool") + try: + return operator.index(value) + except TypeError as exc: + raise TypeError("worker_id must support operator.index()") from exc + + +def _admit_bytes_like(obj: object, nbytes: int, *, writable: bool) -> object: + try: + view = memoryview(obj) + except TypeError as exc: + raise ValueError("worker-chip queue requires a registered HOST Buffer or contiguous host buffer") from exc + if not view.c_contiguous: + raise ValueError("worker-chip queue ordinary host buffer must be C-contiguous") + if writable and view.readonly: + raise ValueError("worker-chip queue output target must be writable") + if int(view.nbytes) < int(nbytes): + raise ValueError( + f"worker-chip queue nbytes={int(nbytes)} exceeds ordinary host buffer size {int(view.nbytes)}" + ) + return obj + + +def _admit_registered_buffer(worker: Any, obj: Buffer, nbytes: int, *, writable: bool) -> Buffer: + if obj.closed: + raise ValueError("worker-chip queue buffer is closed") + worker._validate_worker_chip_orch_comm_host_buffer(obj) + needed = AccessMode.WRITE if writable else AccessMode.READ + if obj.access not in (needed, AccessMode.READWRITE): + raise ValueError( + f"worker-chip queue buffer grants {obj.access.name} but this direction needs {needed.name}" + ) + if int(obj.nbytes) < int(nbytes): + raise ValueError(f"worker-chip queue nbytes={int(nbytes)} exceeds registered buffer size {int(obj.nbytes)}") + return obj + + +def _admit_worker_chip_payload( + worker: Any, obj: object, nbytes: int, *, writable: bool, allow_none: bool +) -> object: + if int(nbytes) == 0: + if allow_none: + if obj is not None: + raise ValueError("worker-chip queue zero-byte payload requires buffer == None") + return None + if obj is None: + raise ValueError("worker-chip queue nonzero payload requires a host buffer") + if isinstance(obj, Buffer): + return _admit_registered_buffer(worker, obj, nbytes, writable=writable) + return _admit_bytes_like(obj, nbytes, writable=writable) + + +def _admit_worker_chip_payload_kind(worker: Any, obj: object, *, writable: bool) -> None: + if obj is None: + return + _admit_worker_chip_payload(worker, obj, 0, writable=writable, allow_none=False) + + +def _require_live_bound_queue(lane: Any) -> None: + lane._queue._ensure_usable() + + def make_worker_chip_queue_layout(depth: int, input_arena_bytes: int, output_arena_bytes: int) -> _SpscQueueLayout: return _SpscQueueLayout.create( _SpscQueueConfig( - depth=int(depth), - input_arena_bytes=int(input_arena_bytes), - output_arena_bytes=int(output_arena_bytes), + depth=depth, + input_arena_bytes=input_arena_bytes, + output_arena_bytes=output_arena_bytes, ) ) @@ -55,15 +121,16 @@ def make_worker_chip_queue_layout(depth: int, input_arena_bytes: int, output_are def create_worker_chip_queue( orch: Any, *, - worker_id: int, - depth: int, - input_arena_bytes: int, - output_arena_bytes: int, + worker_id: object, + depth: object, + input_arena_bytes: object, + output_arena_bytes: object, ) -> WorkerChipQueue: worker = orch._worker - worker._validate_worker_chip_id(int(worker_id)) + worker_id = _require_worker_chip_id(worker_id) + worker._validate_worker_chip_id(worker_id) root_path = _format_worker_path(int(worker.level)) - provider_path = _format_worker_path(2, parent_path=root_path, index=int(worker_id)) + provider_path = _format_worker_path(2, parent_path=root_path, index=worker_id) host = at(root_path, HOST_CPU) peer = at(provider_path, DEVICE_AICPU) placement = _RegionTemplatePlacementRequest( @@ -75,9 +142,9 @@ def create_worker_chip_queue( ), ) config = _SpscQueueConfig( - depth=int(depth), - input_arena_bytes=int(input_arena_bytes), - output_arena_bytes=int(output_arena_bytes), + depth=depth, + input_arena_bytes=input_arena_bytes, + output_arena_bytes=output_arena_bytes, ) return _RegionTemplateCoordinator(worker).create( template=_SpscQueueTemplate(), @@ -97,15 +164,82 @@ def _project_worker_chip_queue(worker: Any, bound: object) -> WorkerChipQueue: raise RuntimeError("worker-chip queue projector requires PAYLOAD and COUNTER local views") desc = worker_chip_orch_region_desc_from_local_views(instance.provider_resource_id, payload_view, counter_view) region = WorkerChipOrchRegion(worker, instance, desc) - return WorkerChipQueue(bound, region) + return WorkerChipQueue(worker, bound, region) + + +class _WorkerChipQueueInput: + def __init__(self, worker: Any, bound: _BoundSpscQueue) -> None: + self._worker = worker + self._lane = bound.input + + def try_enqueue(self, buffer_or_none: object, nbytes: int) -> bool: + _require_live_bound_queue(self._lane) + admitted = _admit_worker_chip_payload( + self._worker, buffer_or_none, nbytes, writable=False, allow_none=True + ) + return self._lane.try_enqueue(admitted, nbytes) + + def enqueue(self, buffer_or_none: object, nbytes: int, timeout: float) -> None: + _require_live_bound_queue(self._lane) + admitted = _admit_worker_chip_payload( + self._worker, buffer_or_none, nbytes, writable=False, allow_none=True + ) + self._lane.enqueue(admitted, nbytes, timeout) + + +class _WorkerChipQueueOutput: + def __init__(self, worker: Any, bound: _BoundSpscQueue) -> None: + self._worker = worker + self._lane = bound.output + + def try_peek(self) -> _SpscQueueMessage | None: + return self._lane.try_peek() + + def peek(self, timeout: float) -> _SpscQueueMessage: + return self._lane.peek(timeout) + + def read_into(self, handle: _SpscQueueMessage, buffer: object) -> None: + _require_live_bound_queue(self._lane) + self._lane._require_active_handle(handle, ownership_violation=True) + admitted = _admit_worker_chip_payload( + self._worker, buffer, handle.payload_nbytes, writable=True, allow_none=True + ) + self._lane.read_into(handle, admitted) + + def release(self, handle: _SpscQueueMessage) -> None: + self._lane.release(handle) + + def dequeue_into(self, buffer: object, timeout: float) -> _SpscQueueMessage: + _require_live_bound_queue(self._lane) + _admit_worker_chip_payload_kind(self._worker, buffer, writable=True) + handle = self._lane.peek(timeout) + admitted = _admit_worker_chip_payload( + self._worker, buffer, handle.payload_nbytes, writable=True, allow_none=True + ) + self._lane.read_into(handle, admitted) + self._lane.release(handle) + return handle + + def try_dequeue_into(self, buffer: object) -> _SpscQueueMessage | None: + _require_live_bound_queue(self._lane) + _admit_worker_chip_payload_kind(self._worker, buffer, writable=True) + handle = self._lane.try_peek() + if handle is None: + return None + admitted = _admit_worker_chip_payload( + self._worker, buffer, handle.payload_nbytes, writable=True, allow_none=True + ) + self._lane.read_into(handle, admitted) + self._lane.release(handle) + return handle class WorkerChipQueue: - def __init__(self, bound: _BoundSpscQueue, region: WorkerChipOrchRegion) -> None: + def __init__(self, worker: Any, bound: _BoundSpscQueue, region: WorkerChipOrchRegion) -> None: self._bound = bound self._region = region - self.input = bound.input - self.output = bound.output + self.input = _WorkerChipQueueInput(worker, bound) + self.output = _WorkerChipQueueOutput(worker, bound) @property def region(self) -> WorkerChipOrchRegion: diff --git a/src/common/platform/include/common/region_template.h b/src/common/platform/include/common/region_template.h index 4fdd3a9464..d488d9cd5a 100644 --- a/src/common/platform/include/common/region_template.h +++ b/src/common/platform/include/common/region_template.h @@ -232,7 +232,7 @@ inline bool decode_descriptor(const uint8_t *src, size_t nbytes, SpscQueueDescri inline bool encode_endpoint_binding(const SpscQueueEndpointBinding &binding, uint64_t *out, size_t count) { if (out == nullptr || count != kSpscQueueEndpointBindingScalarCount || - binding.magic_version != kSpscQueueMagicVersion) { + binding.magic_version != kSpscQueueMagicVersion || binding.transaction_id == 0) { return false; } out[0] = binding.magic_version; @@ -250,7 +250,7 @@ inline bool encode_endpoint_binding(const SpscQueueEndpointBinding &binding, uin inline bool decode_endpoint_binding(const uint64_t *scalars, size_t count, SpscQueueEndpointBinding *out) { if (scalars == nullptr || out == nullptr || count != kSpscQueueEndpointBindingScalarCount || - scalars[0] != kSpscQueueMagicVersion) { + scalars[0] != kSpscQueueMagicVersion || scalars[2] == 0) { return false; } SpscQueueEndpointBinding decoded{ @@ -429,10 +429,6 @@ class SpscQueueEndpoint { return false; } if (timeout_ns == 0) { - parent_->set_error( - SpscQueueErrorKind::BAD_ARGUMENT, SpscQueueOp::INPUT_TRY_PEEK, - "blocking operations require a positive timeout" - ); return false; } uint64_t now = parent_->clock_.now_ns(); @@ -702,14 +698,10 @@ class SpscQueueEndpoint { if (!parent_->ensure_live()) { return false; } - if (nbytes > parent_->layout_.output_arena_bytes) { + if (timeout_ns == 0) { return false; } - if (timeout_ns == 0) { - parent_->set_error( - SpscQueueErrorKind::BAD_ARGUMENT, SpscQueueOp::OUTPUT_TRY_RESERVE, - "blocking operations require a positive timeout" - ); + if (nbytes > parent_->layout_.output_arena_bytes) { return false; } uint64_t now = parent_->clock_.now_ns(); @@ -900,7 +892,7 @@ class SpscQueueEndpoint { uint64_t owner_cookie() const { return static_cast(reinterpret_cast(this)); } void construct(const SpscQueueEndpointBinding &binding) { - if (binding.magic_version != kSpscQueueMagicVersion) { + if (binding.magic_version != kSpscQueueMagicVersion || binding.transaction_id == 0) { identity_trusted_ = false; set_error(SpscQueueErrorKind::BAD_BINDING, SpscQueueOp::INIT, "invalid queue binding"); return; diff --git a/tests/ut/cpp/common/test_region_template.cpp b/tests/ut/cpp/common/test_region_template.cpp index 17f6dfa7b6..3d1029abc4 100644 --- a/tests/ut/cpp/common/test_region_template.cpp +++ b/tests/ut/cpp/common/test_region_template.cpp @@ -72,11 +72,9 @@ TEST(RegionTemplateTest, PackedMagicVersionIsSpsqAbi10) { TEST(RegionTemplateTest, LayoutGoldenVectors) { for (const auto &test_case : kLayoutGolden) { spsc_queue::SpscQueueLayout layout{}; - ASSERT_TRUE( - spsc_queue::SpscQueueLayout::create( - test_case.depth, test_case.input_arena_bytes, test_case.output_arena_bytes, &layout - ) - ); + ASSERT_TRUE(spsc_queue::SpscQueueLayout::create( + test_case.depth, test_case.input_arena_bytes, test_case.output_arena_bytes, &layout + )); EXPECT_EQ(layout.input_desc_offset, 0u); EXPECT_EQ(layout.output_desc_offset, test_case.output_desc_offset); EXPECT_EQ(layout.input_arena_offset, test_case.input_arena_offset); @@ -331,8 +329,7 @@ struct FakeRegionView { bool read(uint64_t offset, uint64_t nbytes, spsc_queue::SpscQueuePayloadView &out) { out = spsc_queue::SpscQueuePayloadView{0, 0}; - view_->state->log.push_back( - FakeAccess{FakeAccessKind::PayloadRead, offset, nbytes, 0, RegionWaitCmp::EQ, 0} + view_->state->log.push_back(FakeAccess{FakeAccessKind::PayloadRead, offset, nbytes, 0, RegionWaitCmp::EQ, 0} ); if (view_->state->sticky_failed) { return false; @@ -557,6 +554,16 @@ bool has_notify(const std::vector &log, uint64_t offset) { return false; } +size_t count_notify(const std::vector &log, uint64_t offset) { + size_t n = 0; + for (const auto &entry : log) { + if (entry.kind == FakeAccessKind::CounterNotify && entry.offset == offset) { + n += 1; + } + } + return n; +} + } // namespace TEST(RegionTemplateTest, ConstructionRejectsNullClockWithoutSharedAccess) { @@ -899,3 +906,251 @@ TEST(RegionTemplateTest, OutputWrapReplayDoesNotFlushBorrowedPayload) { } } } + +TEST(RegionTemplateTest, EncodeRejectsZeroTransactionWithoutWritingOutput) { + spsc_queue::SpscQueueEndpointBinding binding{}; + ASSERT_TRUE(spsc_queue::decode_endpoint_binding(kBindingGolden.data(), kBindingGolden.size(), &binding)); + binding.transaction_id = 0; + std::array encoded{}; + encoded.fill(0x1111111111111111ull); + auto before = encoded; + EXPECT_FALSE(spsc_queue::encode_endpoint_binding(binding, encoded.data(), encoded.size())); + EXPECT_EQ(encoded, before); +} + +TEST(RegionTemplateTest, DecodeRejectsZeroTransactionWithoutWritingTarget) { + spsc_queue::SpscQueueEndpointBinding original{ + 7, 11, 13, 17, 19, 23, 29, 31, 37, 41, + }; + spsc_queue::SpscQueueEndpointBinding decoded = original; + std::array scalars = kBindingGolden; + scalars[2] = 0; + EXPECT_FALSE(spsc_queue::decode_endpoint_binding(scalars.data(), scalars.size(), &decoded)); + EXPECT_EQ(decoded.magic_version, original.magic_version); + EXPECT_EQ(decoded.session_instance_id_bits, original.session_instance_id_bits); + EXPECT_EQ(decoded.transaction_id, original.transaction_id); + EXPECT_EQ(decoded.payload_base, original.payload_base); + EXPECT_EQ(decoded.depth, original.depth); +} + +TEST(RegionTemplateTest, DecodeAcceptsAllZeroSessionBits) { + std::array scalars = kBindingGolden; + scalars[1] = 0; + spsc_queue::SpscQueueEndpointBinding binding{}; + ASSERT_TRUE(spsc_queue::decode_endpoint_binding(scalars.data(), scalars.size(), &binding)); + EXPECT_EQ(binding.session_instance_id_bits, 0u); + EXPECT_EQ(binding.transaction_id, 42u); +} + +TEST(RegionTemplateTest, ConstructRejectsZeroTransactionWithoutViewClockOrIdentity) { + reset_clock(); + auto layout = make_layout(2, 64, 64); + FakeRegionView view(layout.payload_bytes); + auto state = view.state; + auto binding = make_binding(view, layout); + binding.transaction_id = 0; + uint64_t now_before = g_now_ns; + Queue1 queue(binding, std::move(view), test_clock()); + EXPECT_FALSE(queue.live()); + EXPECT_EQ(queue.error().kind, spsc_queue::SpscQueueErrorKind::BAD_BINDING); + EXPECT_FALSE(has_session_marker(queue.error().message)); + EXPECT_EQ(std::string(queue.error().message).find("transaction="), std::string::npos); + EXPECT_EQ(queue.error().transaction_id, 0u); + EXPECT_TRUE(state->log.empty()); + EXPECT_EQ(g_now_ns, now_before); +} + +TEST(RegionTemplateTest, CounterLow32PreservesSignedBitPattern) { + EXPECT_EQ(spsc_queue::counter_low32(0x7fffffffull), 2147483647); + EXPECT_EQ(spsc_queue::counter_low32(0x80000000ull), static_cast(0x80000000u)); + EXPECT_EQ(spsc_queue::counter_low32(0xffffffffull), -1); + EXPECT_EQ(spsc_queue::counter_low32(0x100000000ull), 0); + uint64_t local = 0x7fffffffull; + ASSERT_TRUE(spsc_queue::reconstruct_counter(spsc_queue::counter_low32(0x80000000ull), 4, &local)); + EXPECT_EQ(local, 0x80000000ull); + local = 0xffffffffull; + ASSERT_TRUE(spsc_queue::reconstruct_counter(0, 4, &local)); + EXPECT_EQ(local, 0x100000000ull); +} + +TEST(RegionTemplateTest, ZeroTimeoutPeekAndReserveAreNoAttempt) { + reset_clock(); + auto layout = make_layout(4, 64, 64); + FakeRegionView view(layout.payload_bytes); + auto state = view.state; + auto binding = make_binding(view, layout); + Queue1 queue(binding, std::move(view), test_clock()); + ASSERT_TRUE(queue.live()); + const char payload[] = "abcdefgh"; + plant_input(state.get(), layout, 1, spsc_queue::SpscQueueOpcode::DATA, 8, payload); + size_t log_before = state->log.size(); + uint64_t now_before = g_now_ns; + spsc_queue::SpscQueueInputHandle peek_out{}; + peek_out.seq = 7; + EXPECT_FALSE(queue.input().peek(0, peek_out)); + EXPECT_EQ(peek_out.seq, 0u); + EXPECT_EQ(queue.error().kind, spsc_queue::SpscQueueErrorKind::NONE); + EXPECT_TRUE(queue.live()); + EXPECT_EQ(state->log.size(), log_before); + EXPECT_EQ(g_now_ns, now_before); + + spsc_queue::SpscQueueOutputReservation reserve_out{}; + reserve_out.seq = 9; + EXPECT_FALSE(queue.output().reserve(8, 0, reserve_out)); + EXPECT_FALSE(reserve_out.valid); + EXPECT_EQ(reserve_out.seq, 0u); + EXPECT_EQ(queue.error().kind, spsc_queue::SpscQueueErrorKind::NONE); + EXPECT_TRUE(queue.live()); + EXPECT_EQ(state->log.size(), log_before); + EXPECT_EQ(g_now_ns, now_before); + + spsc_queue::SpscQueueInputHandle ready{}; + ASSERT_TRUE(queue.input().try_peek(ready)); + EXPECT_EQ(ready.seq, 1u); + ASSERT_TRUE(queue.input().release(ready)); +} + +TEST(RegionTemplateTest, ZeroTimeoutReserveLeavesActiveReservationAndSkipsOversize) { + reset_clock(); + auto layout = make_layout(4, 64, 64); + FakeRegionView view(layout.payload_bytes); + auto state = view.state; + auto binding = make_binding(view, layout); + Queue1 queue(binding, std::move(view), test_clock()); + ASSERT_TRUE(queue.live()); + spsc_queue::SpscQueueOutputReservation first{}; + ASSERT_TRUE(queue.output().try_reserve(8, first)); + size_t log_before = state->log.size(); + spsc_queue::SpscQueueOutputReservation ignored{}; + EXPECT_FALSE(queue.output().reserve(8, 0, ignored)); + EXPECT_FALSE(ignored.valid); + EXPECT_EQ(queue.error().kind, spsc_queue::SpscQueueErrorKind::NONE); + EXPECT_TRUE(queue.live()); + EXPECT_EQ(state->log.size(), log_before); + EXPECT_TRUE(first.valid); + ASSERT_TRUE(queue.output().publish(first, spsc_queue::SpscQueueOpcode::DATA)); + + FakeRegionView view2(layout.payload_bytes); + auto state2 = view2.state; + auto binding2 = make_binding(view2, layout); + Queue1 queue2(binding2, std::move(view2), test_clock()); + spsc_queue::SpscQueueOutputReservation oversize{}; + EXPECT_FALSE(queue2.output().reserve(layout.output_arena_bytes + 1, 0, oversize)); + EXPECT_FALSE(oversize.valid); + EXPECT_EQ(queue2.error().kind, spsc_queue::SpscQueueErrorKind::NONE); + EXPECT_TRUE(queue2.live()); + EXPECT_TRUE(state2->log.empty()); + spsc_queue::SpscQueueOutputReservation ok{}; + ASSERT_TRUE(queue2.output().try_reserve(8, ok)); +} + +TEST(RegionTemplateTest, ZeroTimeoutPreservesPreexistingTerminalError) { + reset_clock(); + auto layout = make_layout(4, 64, 64); + FakeRegionView view(layout.payload_bytes); + auto state = view.state; + auto binding = make_binding(view, layout); + Queue1 queue(binding, std::move(view), test_clock()); + plant_input(state.get(), layout, 1, spsc_queue::SpscQueueOpcode::ERROR, 0, nullptr); + spsc_queue::SpscQueueInputHandle handle{}; + EXPECT_FALSE(queue.input().try_peek(handle)); + EXPECT_EQ(queue.error().kind, spsc_queue::SpscQueueErrorKind::INVALID_DESCRIPTOR); + size_t log_before = state->log.size(); + EXPECT_FALSE(queue.input().peek(0, handle)); + spsc_queue::SpscQueueOutputReservation reserved{}; + EXPECT_FALSE(queue.output().reserve(8, 0, reserved)); + EXPECT_EQ(queue.error().kind, spsc_queue::SpscQueueErrorKind::INVALID_DESCRIPTOR); + EXPECT_NE(std::string(queue.error().message).find("invalid input opcode"), std::string::npos); + EXPECT_EQ(state->log.size(), log_before); +} + +TEST(RegionTemplateTest, PublishStopPoisonsWithoutTailAdvance) { + reset_clock(); + auto layout = make_layout(4, 64, 64); + FakeRegionView view(layout.payload_bytes); + auto state = view.state; + auto binding = make_binding(view, layout); + Queue1 queue(binding, std::move(view), test_clock()); + spsc_queue::SpscQueueOutputReservation reserved{}; + ASSERT_TRUE(queue.output().try_reserve(8, reserved)); + EXPECT_FALSE(queue.output().publish(reserved, spsc_queue::SpscQueueOpcode::STOP)); + EXPECT_EQ(queue.error().kind, spsc_queue::SpscQueueErrorKind::INVALID_DESCRIPTOR); + EXPECT_NE(std::string(queue.error().message).find("invalid output opcode"), std::string::npos); + EXPECT_EQ(count_notify(state->log, spsc_queue::kPeerAbortOffset), 1u); + EXPECT_FALSE(has_notify(state->log, layout.output_desc_tail_offset)); + EXPECT_EQ(load_counter(state.get(), layout.output_desc_tail_offset), 0); + EXPECT_FALSE(queue.output().publish(reserved, spsc_queue::SpscQueueOpcode::DATA)); + EXPECT_EQ(count_notify(state->log, spsc_queue::kPeerAbortOffset), 1u); + EXPECT_NE(std::string(queue.error().message).find("invalid output opcode"), std::string::npos); +} + +TEST(RegionTemplateTest, InputPayloadOutsideArenaPoisonsWithoutPayloadRead) { + reset_clock(); + auto layout = make_layout(4, 64, 64); + FakeRegionView view(layout.payload_bytes); + auto state = view.state; + auto binding = make_binding(view, layout); + Queue1 queue(binding, std::move(view), test_clock()); + uint64_t outside = layout.input_arena_offset + layout.input_arena_bytes; + plant_descriptor(state.get(), layout.input_desc_offset, 1, spsc_queue::SpscQueueOpcode::DATA, outside, 8); + store_counter(state.get(), layout.input_desc_tail_offset, 1); + spsc_queue::SpscQueueInputHandle handle{}; + handle.seq = 11; + EXPECT_FALSE(queue.input().try_peek(handle)); + EXPECT_EQ(handle.seq, 0u); + EXPECT_EQ(queue.error().kind, spsc_queue::SpscQueueErrorKind::INVALID_DESCRIPTOR); + EXPECT_NE(std::string(queue.error().message).find("input payload out of arena"), std::string::npos); + for (const auto &entry : state->log) { + if (entry.kind == FakeAccessKind::PayloadRead) { + EXPECT_NE(entry.offset, outside); + } + } + EXPECT_EQ(count_notify(state->log, spsc_queue::kPeerAbortOffset), 1u); + EXPECT_FALSE(has_notify(state->log, layout.input_desc_head_offset)); + EXPECT_FALSE(queue.input().try_peek(handle)); + EXPECT_EQ(count_notify(state->log, spsc_queue::kPeerAbortOffset), 1u); +} + +TEST(RegionTemplateTest, SecondNoncanonicalInputReplayPoisonsAndAbortNotifyFailureIsSecondary) { + reset_clock(); + auto layout = make_layout(4, 64, 64); + FakeRegionView view(layout.payload_bytes); + auto state = view.state; + state->fail_peer_abort_notify = true; + auto binding = make_binding(view, layout); + Queue2 queue(binding, std::move(view), test_clock()); + const char first[] = "12345678"; + plant_input(state.get(), layout, 1, spsc_queue::SpscQueueOpcode::DATA, 8, first); + spsc_queue::SpscQueueInputHandle acquired{}; + ASSERT_TRUE(queue.input().try_peek(acquired)); + EXPECT_EQ(acquired.payload_offset, layout.input_arena_offset); + plant_descriptor( + state.get(), layout.input_desc_offset + spsc_queue::kDescriptorBytes, 2, spsc_queue::SpscQueueOpcode::DATA, + layout.input_arena_offset, 8 + ); + store_counter(state.get(), layout.input_desc_tail_offset, 2); + size_t arena_reads_before = 0; + for (const auto &entry : state->log) { + if (entry.kind == FakeAccessKind::PayloadRead && entry.offset == layout.input_arena_offset) { + arena_reads_before += 1; + } + } + spsc_queue::SpscQueueInputHandle second{}; + EXPECT_FALSE(queue.input().try_peek(second)); + EXPECT_EQ(second.seq, 0u); + EXPECT_EQ(queue.error().kind, spsc_queue::SpscQueueErrorKind::INVALID_DESCRIPTOR); + EXPECT_NE(std::string(queue.error().message).find("payload replay offset mismatch"), std::string::npos); + EXPECT_NE(std::string(queue.error().message).find("abort notify failed"), std::string::npos); + size_t arena_reads_after = 0; + for (const auto &entry : state->log) { + if (entry.kind == FakeAccessKind::PayloadRead && entry.offset == layout.input_arena_offset) { + arena_reads_after += 1; + } + } + EXPECT_EQ(arena_reads_after, arena_reads_before); + EXPECT_EQ(count_notify(state->log, spsc_queue::kPeerAbortOffset), 1u); + EXPECT_FALSE(has_notify(state->log, layout.input_desc_head_offset)); + EXPECT_FALSE(queue.input().try_peek(second)); + EXPECT_EQ(count_notify(state->log, spsc_queue::kPeerAbortOffset), 1u); + EXPECT_NE(std::string(queue.error().message).find("payload replay offset mismatch"), std::string::npos); +} diff --git a/tests/ut/py/test_worker/test_comm_region_template.py b/tests/ut/py/test_worker/test_comm_region_template.py index 6d3b5c7a49..91c6363b60 100644 --- a/tests/ut/py/test_worker/test_comm_region_template.py +++ b/tests/ut/py/test_worker/test_comm_region_template.py @@ -11,6 +11,7 @@ from __future__ import annotations import copy +import ctypes import inspect import pickle import struct @@ -61,6 +62,8 @@ _BoundSpscQueue, _checked_add_u64, _checked_mul_u64, + _counter_low32, + _payload_expected_offset, _RegionTemplateCoordinator, _RegionTemplatePlacementRequest, _require_distinct_initiator_peer, @@ -344,10 +347,31 @@ def test_binding_rejects_bool_range_and_version_mismatch(): SpscQueueEndpointBinding.from_scalars(wrong_minor) +def test_binding_rejects_zero_transaction_id_and_keeps_zero_session_bits(): + with pytest.raises(ValueError, match="nonzero"): + SpscQueueEndpointBinding(*(_BINDING_GOLDEN[:2] + (0,) + _BINDING_GOLDEN[3:])) + zero = list(_BINDING_GOLDEN) + zero[2] = 0 + with pytest.raises(ValueError, match="nonzero"): + SpscQueueEndpointBinding.from_scalars(zero) + session_zero = list(_BINDING_GOLDEN) + session_zero[1] = 0 + binding = SpscQueueEndpointBinding.from_scalars(session_zero) + assert binding.session_instance_id_bits == 0 + assert binding.transaction_id == 42 + assert binding.to_scalars()[2] == 42 + + class _UnknownTemplateSlot(Enum): EXTRA = "extra" +def _require_signed_int32(value: object) -> int: + if type(value) is not int or value < -2147483648 or value > 2147483647: + raise OverflowError("signed int32 counter operand out of range") + return value + + def _compare_counter(observed: int, operand: int, cmp: WaitCmp) -> bool: if cmp is WaitCmp.EQ: return observed == operand @@ -370,26 +394,30 @@ def __init__(self, region: _MemoryRegion, offset: int) -> None: self._offset = int(offset) def test(self, cmp_value: int, cmp: WaitCmp) -> SignalTestResult: - observed = int(self._region.counters.get(self._offset, 0)) - return SignalTestResult(matched=_compare_counter(observed, int(cmp_value), WaitCmp(cmp)), observed=observed) + operand = _require_signed_int32(cmp_value) + observed = ctypes.c_int32(int(self._region.counters.get(self._offset, 0)) & 0xFFFFFFFF).value + return SignalTestResult(matched=_compare_counter(observed, operand, WaitCmp(cmp)), observed=observed) def wait(self, cmp_value: int, cmp: WaitCmp, timeout: float) -> int: + operand = _require_signed_int32(cmp_value) if timeout is None or float(timeout) <= 0: raise ValueError("region counter wait requires a positive timeout") if self._region.on_wait is not None: self._region.on_wait(self._offset) - result = self.test(cmp_value, cmp) + result = self.test(operand, cmp) if result.matched: return int(result.observed) raise TimeoutError(f"queue counter wait timed out; observed={result.observed}") def notify(self, value: int, op: NotifyOp = NotifyOp.Set) -> None: + operand = _require_signed_int32(value) if self._region.fail_notify is not None: raise self._region.fail_notify if op is NotifyOp.Add: - self._region.counters[self._offset] = int(self._region.counters.get(self._offset, 0)) + int(value) + current = ctypes.c_int32(int(self._region.counters.get(self._offset, 0)) & 0xFFFFFFFF).value + self._region.counters[self._offset] = ctypes.c_int32((current + operand) & 0xFFFFFFFF).value else: - self._region.counters[self._offset] = int(value) & 0xFFFF_FFFF + self._region.counters[self._offset] = operand @dataclass @@ -621,6 +649,146 @@ def test_bind_rejects_layout_mismatch_and_incomplete_identity(): plan._bind(region, slots) +def test_spsc_plan_rejects_role_mismatch_after_consume(): + session, host, peer, _extra, registry = _endpoint_bundle() + plan = _SpscQueueTemplate().plan(_SpscQueueConfig(4, 128, 128)) + members = registry.resolve_members((at("L3", HOST_CPU), at("L3/L2[1]", DEVICE_AICPU))) + slots = _resolve_template_slots( + registry, members, _queue_placement(host, peer).slot_bindings, _SpscQueueTemplate.required_slots + ) + swapped = _MemoryRegion(plan.region_layout, consumer=peer, provider=host, session=session, transaction_id=7) + with pytest.raises(ValueError, match="INITIATOR slot does not match"): + plan._bind(swapped, slots) + assert plan._state is _SpscQueuePlanState.CONSUMED + with pytest.raises(RuntimeError, match="already consumed"): + plan._bind(swapped, slots) + + +class _GenericTemplateSlot(Enum): + ALPHA = "alpha" + BETA = "beta" + GAMMA = "gamma" + + +class _GenericTemplatePlan: + def __init__(self, region_layout: RegionLayoutSpec, *, fail_bind: bool = False) -> None: + self._region_layout = region_layout + self._fail_bind = fail_bind + self._state = _SpscQueuePlanState.AVAILABLE + self._bind_lock = threading.Lock() + self.bound: dict[str, object] | None = None + + @property + def region_layout(self) -> RegionLayoutSpec: + return self._region_layout + + def _consume_for_bind(self) -> None: + with self._bind_lock: + if self._state is not _SpscQueuePlanState.AVAILABLE: + raise RuntimeError("queue plan bind token is already consumed") + self._state = _SpscQueuePlanState.CONSUMED + + def _bind(self, instance: object, slots: object) -> dict[str, object]: + self._consume_for_bind() + alpha = slots.endpoint(_GenericTemplateSlot.ALPHA) + beta = slots.endpoint(_GenericTemplateSlot.BETA) + gamma = slots.endpoint(_GenericTemplateSlot.GAMMA) + if alpha.identity == beta.identity: + raise ValueError("ALPHA and BETA must bind different endpoint identities") + if self._fail_bind: + raise RuntimeError("injected generic bind failure") + self.bound = {"instance": instance, "alpha": alpha, "beta": beta, "gamma": gamma} + return self.bound + + +class _GenericTemplate: + required_slots = ( + _GenericTemplateSlot.ALPHA, + _GenericTemplateSlot.BETA, + _GenericTemplateSlot.GAMMA, + ) + + def __init__(self, *, fail_bind: bool = False) -> None: + self._fail_bind = fail_bind + self.last_plan: _GenericTemplatePlan | None = None + + def plan(self, config: object) -> _GenericTemplatePlan: + if not isinstance(config, RegionLayoutSpec): + raise TypeError("generic template.plan requires RegionLayoutSpec") + self.last_plan = _GenericTemplatePlan(config, fail_bind=self._fail_bind) + return self.last_plan + + +def _generic_placement(host: EndpointRecord, peer: EndpointRecord) -> _RegionTemplatePlacementRequest: + return _RegionTemplatePlacementRequest( + members=(at(host.path, host.deployment), at(peer.path, peer.deployment)), + topology=SingleOwner(provider=at(peer.path, peer.deployment)), + slot_bindings=( + _TemplateSlotBindingRequest(_GenericTemplateSlot.ALPHA, at(host.path, host.deployment)), + _TemplateSlotBindingRequest(_GenericTemplateSlot.BETA, at(peer.path, peer.deployment)), + _TemplateSlotBindingRequest(_GenericTemplateSlot.GAMMA, at(host.path, host.deployment)), + ), + ) + + +def test_generic_template_slot_resolve_enforces_required_slots_and_same_endpoint_policy(): + _session, host, peer, _extra, registry = _endpoint_bundle() + members = registry.resolve_members((at("L3", HOST_CPU), at("L3/L2[1]", DEVICE_AICPU))) + required = _GenericTemplate.required_slots + with pytest.raises(ValueError, match="missing"): + _resolve_template_slots( + registry, + members, + ( + _TemplateSlotBindingRequest(_GenericTemplateSlot.ALPHA, at("L3", HOST_CPU)), + _TemplateSlotBindingRequest(_GenericTemplateSlot.BETA, at("L3/L2[1]", DEVICE_AICPU)), + ), + required, + ) + with pytest.raises(ValueError, match="duplicate"): + _resolve_template_slots( + registry, + members, + ( + _TemplateSlotBindingRequest(_GenericTemplateSlot.ALPHA, at("L3", HOST_CPU)), + _TemplateSlotBindingRequest(_GenericTemplateSlot.ALPHA, at("L3/L2[1]", DEVICE_AICPU)), + _TemplateSlotBindingRequest(_GenericTemplateSlot.BETA, at("L3/L2[1]", DEVICE_AICPU)), + _TemplateSlotBindingRequest(_GenericTemplateSlot.GAMMA, at("L3", HOST_CPU)), + ), + required, + ) + with pytest.raises(ValueError, match="unknown"): + _resolve_template_slots( + registry, + members, + ( + *_generic_placement(host, peer).slot_bindings, + _TemplateSlotBindingRequest(_UnknownTemplateSlot.EXTRA, at("L3/L2[0]", DEVICE_AICPU)), + ), + required, + ) + slots = _resolve_template_slots(registry, members, _generic_placement(host, peer).slot_bindings, required) + plan = _GenericTemplate().plan(RegionLayoutSpec(64, 128)) + bound = plan._bind(_MemoryRegion(plan.region_layout, host, peer, _session, 9), slots) + assert bound["alpha"] is host + assert bound["beta"] is peer + assert bound["gamma"] is host + same = _resolve_template_slots( + registry, + registry.resolve_members((at("L3", HOST_CPU),)), + ( + _TemplateSlotBindingRequest(_GenericTemplateSlot.ALPHA, at("L3", HOST_CPU)), + _TemplateSlotBindingRequest(_GenericTemplateSlot.BETA, at("L3", HOST_CPU)), + _TemplateSlotBindingRequest(_GenericTemplateSlot.GAMMA, at("L3", HOST_CPU)), + ), + required, + ) + rejected = _GenericTemplate().plan(RegionLayoutSpec(64, 128)) + with pytest.raises(ValueError, match="ALPHA and BETA"): + rejected._bind(_MemoryRegion(rejected.region_layout, host, host, _session, 9), same) + assert rejected._state is _SpscQueuePlanState.CONSUMED + + def test_bound_queue_refuses_copy_and_does_not_own_physical_cleanup(): _plan, region, _slots, queue = _bind_memory_queue() with pytest.raises(TypeError, match="cannot be copied"): @@ -676,8 +844,9 @@ def test_duplex_data_zero_byte_wrap_replay_stop_and_error(): queue.output.release(handle) data2 = b"more" - desc2 = encode_spsc_queue_descriptor(2, SpscQueueOpcode.DATA, out_offset, 4) - region.payload[out_offset : out_offset + 4] = data2 + desc2_offset = out_offset + 8 + region.payload[desc2_offset : desc2_offset + 4] = data2 + desc2 = encode_spsc_queue_descriptor(2, SpscQueueOpcode.DATA, desc2_offset, 4) region.payload[queue._layout.output_desc_offset + 32 : queue._layout.output_desc_offset + 64] = desc2 region.counters[queue._layout.output_desc_tail_offset] = 2 handle = queue.output.try_peek() @@ -730,6 +899,21 @@ def set_abort(_offset: int) -> None: assert any("abort notify failed" in note for note in notes) +def test_poison_attaches_abort_notify_note_without_add_note(): + class _LegacyError(RuntimeError): + add_note = None + + _plan, region, _slots, queue = _bind_memory_queue() + region.fail_payload = _LegacyError("payload write failed") + region.fail_notify = RuntimeError("abort notify failed") + with pytest.raises(_LegacyError, match="payload write failed") as excinfo: + queue.input.try_enqueue(b"hi", 2) + assert queue._state is _SpscQueueState.POISONED_LOCAL + assert queue._first_error is excinfo.value + notes = getattr(excinfo.value, "__notes__", []) + assert any("abort notify failed" in note for note in notes) + + def test_zero_byte_enqueue_rejects_non_none_and_stop_is_zero_byte(): _plan, _region, _slots, queue = _bind_memory_queue() with pytest.raises(ValueError, match="buffer_or_none == None"): @@ -1015,6 +1199,115 @@ def test_coordinator_cleanup_failure_records_unreclaimable_and_keeps_first_error assert worker._ordered_cleanup_error is not None +def test_coordinator_creates_generic_three_slot_template(region_worker): + worker, calls, _leases = region_worker() + _session, host, peer, _extra, _registry = _endpoint_bundle() + template = _GenericTemplate() + result = _RegionTemplateCoordinator(worker).create( + template=template, + config=RegionLayoutSpec(64, 128), + placement=_generic_placement(host, peer), + result_projector=lambda bound: bound, + ) + assert result["alpha"].path == "L3" + assert result["beta"].path == "L3/L2[1]" + assert result["gamma"].path == "L3" + assert result["alpha"].identity == result["gamma"].identity + assert result["instance"] is not None + assert template.last_plan is not None + assert template.last_plan._state is _SpscQueuePlanState.CONSUMED + assert ("allocate", 64, 128) in calls + assert not any(item[0] == "release" for item in calls) + + +def test_coordinator_generic_missing_slot_does_not_materialize(region_worker): + worker, calls, _leases = region_worker() + host_sel = at("L3", HOST_CPU) + peer_sel = at("L3/L2[1]", DEVICE_AICPU) + placement = _RegionTemplatePlacementRequest( + members=(host_sel, peer_sel), + topology=SingleOwner(provider=peer_sel), + slot_bindings=( + _TemplateSlotBindingRequest(_GenericTemplateSlot.ALPHA, host_sel), + _TemplateSlotBindingRequest(_GenericTemplateSlot.BETA, peer_sel), + ), + ) + with pytest.raises(ValueError, match="missing"): + _RegionTemplateCoordinator(worker).create( + template=_GenericTemplate(), + config=RegionLayoutSpec(64, 128), + placement=placement, + result_projector=lambda bound: bound, + ) + assert not any(item[0] == "allocate" for item in calls) + + +def test_coordinator_generic_bind_failure_rolls_back_and_consumes_plan(region_worker): + worker, calls, _leases = region_worker() + _session, host, peer, _extra, _registry = _endpoint_bundle() + template = _GenericTemplate(fail_bind=True) + with pytest.raises(RuntimeError, match="injected generic bind failure"): + _RegionTemplateCoordinator(worker).create( + template=template, + config=RegionLayoutSpec(64, 128), + placement=_generic_placement(host, peer), + result_projector=lambda bound: bound, + ) + assert template.last_plan is not None + assert template.last_plan._state is _SpscQueuePlanState.CONSUMED + assert any(item[0] == "release" for item in calls) + assert worker._region_instance_registry._instances == {} + + +def test_coordinator_generic_cleanup_failure_keeps_first_error(region_worker): + worker, calls, _leases = region_worker(fail_mapping_close=True) + _session, host, peer, _extra, _registry = _endpoint_bundle() + with pytest.raises(RuntimeError, match="injected generic bind failure") as excinfo: + _RegionTemplateCoordinator(worker).create( + template=_GenericTemplate(fail_bind=True), + config=RegionLayoutSpec(64, 128), + placement=_generic_placement(host, peer), + result_projector=lambda bound: bound, + ) + assert str(excinfo.value) == "injected generic bind failure" + assert worker._ordered_cleanup_error is not None + assert any(item[0] == "release" for item in calls) + + +def test_coordinator_spsc_role_mismatch_consumes_plan_and_rolls_back(region_worker): + worker, calls, _leases = region_worker() + host_sel = at("L3", HOST_CPU) + peer_sel = at("L3/L2[1]", DEVICE_AICPU) + placement = _RegionTemplatePlacementRequest( + members=(host_sel, peer_sel), + topology=SingleOwner(provider=peer_sel), + slot_bindings=( + _TemplateSlotBindingRequest(_SpscQueueSlot.INITIATOR, peer_sel), + _TemplateSlotBindingRequest(_SpscQueueSlot.PEER, host_sel), + ), + ) + with pytest.raises(ValueError, match="INITIATOR slot does not match"): + _RegionTemplateCoordinator(worker).create( + template=_SpscQueueTemplate(), + config=_SpscQueueConfig(4, 64, 64), + placement=placement, + result_projector=lambda bound: bound, + ) + assert any(item[0] == "allocate" for item in calls) + assert any(item[0] == "release" for item in calls) + assert worker._region_instance_registry._instances == {} + + +def test_coordinator_source_has_no_spsc_roles(): + source = inspect.getsource(_RegionTemplateCoordinator) + assert "_SpscQueueSlot" not in source + assert "_prove_initiator_peer_access" not in source + assert "_require_distinct_initiator_peer" not in source + assert "INITIATOR" not in source + assert "PEER" not in source + assert "consumer/provider" not in source + + def test_production_facade_uses_coordinator(): from simpler import worker_chip_message_queue from simpler.worker_chip_message_queue import create_worker_chip_queue @@ -1025,3 +1318,131 @@ def test_production_facade_uses_coordinator(): module_source = inspect.getsource(worker_chip_message_queue) assert "time.sleep" not in module_source assert "orch.alloc" not in module_source + + +def _plant_output_descriptor(region, queue, seq, opcode, payload_offset, payload=b""): + if payload: + begin = int(payload_offset) + region.payload[begin : begin + len(payload)] = payload + slot_index = (int(seq) - 1) & (queue._layout.depth - 1) + slot = queue._layout.output_desc_offset + slot_index * _SPSC_QUEUE_DESCRIPTOR_BYTES + region.payload[slot : slot + 32] = encode_spsc_queue_descriptor(seq, opcode, payload_offset, len(payload)) + region.counters[queue._layout.output_desc_tail_offset] = int(seq) + + +def test_payload_expected_offset_matches_native_boundaries(): + arena_offset = 128 + arena_bytes = 64 + assert _payload_expected_offset(0, 8, arena_offset, arena_bytes) == 128 + assert _payload_expected_offset(8, 8, arena_offset, arena_bytes) == 136 + assert _payload_expected_offset(56, 8, arena_offset, arena_bytes) == 184 + assert _payload_expected_offset(56, 16, arena_offset, arena_bytes) == 128 + assert _payload_expected_offset(64, 8, arena_offset, arena_bytes) == 128 + assert _payload_expected_offset(0, 64, arena_offset, arena_bytes) == 128 + assert _payload_expected_offset(0, 0, arena_offset, arena_bytes) == 128 + assert _counter_low32(0x7FFFFFFF) == 2147483647 + assert _counter_low32(0x80000000) == -2147483648 + assert _counter_low32(0xFFFFFFFF) == -1 + assert _counter_low32(1 << 32) == 0 + + +def test_output_early_wrap_poisons_once_without_committing_head(): + _plan, region, _slots, queue = _bind_memory_queue(depth=4, input_arena_bytes=64, output_arena_bytes=64) + out = queue._layout.output_arena_offset + _plant_output_descriptor(region, queue, 1, SpscQueueOpcode.DATA, out, b"abcdefgh") + handle = queue.output.try_peek() + assert handle is not None + assert queue._layout.output_desc_head_offset not in region.counters + queue.output.release(handle) + assert region.counters[queue._layout.output_desc_head_offset] == 1 + assert queue._output_payload_head == 8 + _plant_output_descriptor(region, queue, 2, SpscQueueOpcode.DATA, out, b"xxxxxxxx") + with pytest.raises(RuntimeError, match="payload replay offset mismatch") as excinfo: + queue.output.try_peek() + assert queue._state is _SpscQueueState.POISONED_LOCAL + assert queue._first_error is excinfo.value + assert queue._output_payload_head == 8 + assert queue._output_head == 1 + assert region.counters[queue._layout.initiator_abort_offset] == 1 + with pytest.raises(RuntimeError, match="payload replay offset mismatch"): + queue.output.try_peek() + assert region.counters[queue._layout.initiator_abort_offset] == 1 + + +def test_output_exact_fit_true_wrap_and_zero_byte_replay(): + _plan, region, _slots, queue = _bind_memory_queue(depth=4, input_arena_bytes=64, output_arena_bytes=64) + out = queue._layout.output_arena_offset + _plant_output_descriptor(region, queue, 1, SpscQueueOpcode.DATA, out, b"x" * 56) + handle = queue.output.try_peek() + queue.output.release(handle) + _plant_output_descriptor(region, queue, 2, SpscQueueOpcode.DATA, out + 56, b"yz123456") + handle = queue.output.try_peek() + assert handle is not None + assert handle.payload_offset == out + 56 + queue.output.release(handle) + assert queue._output_payload_head == 64 + _plant_output_descriptor(region, queue, 3, SpscQueueOpcode.DATA, out, b"w" * 16) + handle = queue.output.try_peek() + assert handle is not None + assert handle.payload_offset == out + queue.output.release(handle) + _plant_output_descriptor(region, queue, 4, SpscQueueOpcode.DATA, 0, b"") + handle = queue.output.try_peek() + assert handle is not None + assert handle.payload_nbytes == 0 + assert handle.payload_offset == 0 + queue.output.release(handle) + assert queue._output_payload_head == 80 + + +def test_input_release_replay_rejects_early_wrap_and_preserves_abort_failure(): + _plan, region, _slots, queue = _bind_memory_queue(depth=4, input_arena_bytes=64, output_arena_bytes=64) + assert queue.input.try_enqueue(b"abcdefgh", 8) is True + slot = queue._layout.input_desc_offset + region.payload[slot : slot + 32] = encode_spsc_queue_descriptor( + 1, SpscQueueOpcode.DATA, queue._layout.input_arena_offset + 8, 8 + ) + region.counters[queue._layout.input_desc_head_offset] = 1 + region.fail_notify = RuntimeError("abort notify failed") + with pytest.raises(RuntimeError, match="payload replay offset mismatch") as excinfo: + queue.input.try_enqueue(b"xxxxxxxx", 8) + assert queue._state is _SpscQueueState.POISONED_LOCAL + assert queue._first_error is excinfo.value + notes = getattr(excinfo.value, "__notes__", []) + assert any("abort notify failed" in note for note in notes) + assert queue._input_tail == 1 + assert region.counters[queue._layout.input_desc_tail_offset] == 1 + + +def test_signed_int32_counter_operands_cross_bit31_and_wrap_to_zero(): + _plan, region, _slots, queue = _bind_memory_queue(depth=4, input_arena_bytes=64, output_arena_bytes=64) + queue._input_head = 0x7FFFFFFF + queue._input_tail = 0x7FFFFFFF + region.counters[queue._layout.input_desc_head_offset] = 2147483647 + assert queue.input.try_enqueue(None, 0) is True + assert region.counters[queue._layout.input_desc_tail_offset] == -2147483648 + + _plan, region, _slots, queue = _bind_memory_queue(depth=4, input_arena_bytes=64, output_arena_bytes=64) + queue._input_head = 0xFFFFFFFF + queue._input_tail = 0xFFFFFFFF + region.counters[queue._layout.input_desc_head_offset] = -1 + assert queue.input.try_enqueue(None, 0) is True + assert region.counters[queue._layout.input_desc_tail_offset] == 0 + + _plan, region, _slots, queue = _bind_memory_queue(depth=4, input_arena_bytes=64, output_arena_bytes=64) + queue._output_tail = 0x7FFFFFFF + region.counters[queue._layout.output_desc_tail_offset] = -2147483648 + _plant_output_descriptor(region, queue, 1, SpscQueueOpcode.DATA, 0, b"") + region.counters[queue._layout.output_desc_tail_offset] = -2147483648 + handle = queue.output.try_peek() + assert handle is not None + assert handle.seq == 1 + assert queue._output_tail == 0x80000000 + + _plan, region, _slots, queue = _bind_memory_queue(depth=4, input_arena_bytes=64, output_arena_bytes=64) + queue._output_head = 0x80000000 + queue._output_tail = 0x80000000 + region.counters[queue._layout.output_desc_tail_offset] = -2147483648 + with pytest.raises(TimeoutError, match="timed out"): + queue.output.peek(0.01) + assert queue._state is _SpscQueueState.LIVE diff --git a/tests/ut/py/test_worker/test_worker_chip_message_queue.py b/tests/ut/py/test_worker/test_worker_chip_message_queue.py index 0316d38b5d..c071e859bf 100644 --- a/tests/ut/py/test_worker/test_worker_chip_message_queue.py +++ b/tests/ut/py/test_worker/test_worker_chip_message_queue.py @@ -6,8 +6,10 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. # ----------------------------------------------------------------------------------------------------------- +# ruff: noqa: PLC0415 import ctypes +import inspect import math import struct from dataclasses import dataclass @@ -18,7 +20,7 @@ import simpler.worker_chip_message_queue as queue_mod from simpler import comm_region from simpler import worker as worker_module -from simpler.buffer import BackendKind +from simpler.buffer import AccessMode, AddressSpace, BackendKind from simpler.comm_provider import ( PosixShmImport, ProviderReleaseResult, @@ -995,3 +997,218 @@ def test_create_worker_chip_queue_publish_survives_later_caller_failure(): assert instance._close_attempted is False finally: _close(worker, shm) + + +class _IntOnly: + def __int__(self): + return 4 + + +class _IndexOnly: + def __init__(self, value: int) -> None: + self._value = value + + def __index__(self): + return self._value + + +class _DuckPayload: + def __init__(self, base: int, nbytes: int) -> None: + self.base = base + self.nbytes = nbytes + + +def _shared_snapshot(fake_client: _FakeClient): + return ( + list(fake_client.requests), + list(fake_client.payload_writes), + dict(fake_client.counters), + ) + + +def test_layout_and_factory_reject_non_exact_ints_before_materialization(monkeypatch): + layout = make_worker_chip_queue_layout(4, 128, 192) + assert layout.depth == 4 + assert layout.input_arena_bytes == 128 + assert layout.output_arena_bytes == 192 + invalid_values: list[object] = [True, 4.0, _IntOnly(), _IndexOnly(4)] + try: + import numpy + + invalid_values.append(numpy.int64(4)) + except ImportError: + pass + for value in invalid_values: + with pytest.raises(TypeError): + make_worker_chip_queue_layout(value, 128, 128) + with pytest.raises(TypeError): + make_worker_chip_queue_layout(4, value, 128) + with pytest.raises(TypeError): + make_worker_chip_queue_layout(4, 128, value) + + orch, worker, shm, fake_client = _make_orchestrator() + materializations: list[object] = [] + import simpler.comm_region_template as template_mod + + original = template_mod.materialize_region_instance + + def spy(*args, **kwargs): + materializations.append(True) + return original(*args, **kwargs) + + monkeypatch.setattr(template_mod, "materialize_region_instance", spy) + try: + with pytest.raises(TypeError): + orch.create_worker_chip_queue(worker_id=0, depth=True, input_arena_bytes=128, output_arena_bytes=128) + with pytest.raises(TypeError): + orch.create_worker_chip_queue(worker_id=0, depth=4.0, input_arena_bytes=128, output_arena_bytes=128) + with pytest.raises(TypeError): + orch.create_worker_chip_queue( + worker_id=0, depth=_IntOnly(), input_arena_bytes=128, output_arena_bytes=128 + ) + with pytest.raises(TypeError): + orch.create_worker_chip_queue( + worker_id=0, depth=_IndexOnly(4), input_arena_bytes=128, output_arena_bytes=128 + ) + with pytest.raises(TypeError): + orch.create_worker_chip_queue(worker_id=True, depth=4, input_arena_bytes=128, output_arena_bytes=128) + with pytest.raises(TypeError): + orch.create_worker_chip_queue(worker_id=0.0, depth=4, input_arena_bytes=128, output_arena_bytes=128) + with pytest.raises(TypeError): + orch.create_worker_chip_queue(worker_id=_IntOnly(), depth=4, input_arena_bytes=128, output_arena_bytes=128) + assert materializations == [] + queue = orch.create_worker_chip_queue( + worker_id=_IndexOnly(0), depth=4, input_arena_bytes=128, output_arena_bytes=128 + ) + assert len(queue.chip_task_arg_scalars()) == 10 + assert materializations == [True] + assert fake_client.requests[0][0].cmd == "alloc_region" + finally: + _close(worker, shm) + + +def test_unbound_orchestrator_rejects_before_admission(): + orch = Orchestrator(_FakeCOrch(), None) + with pytest.raises(RuntimeError, match="bound to a Worker"): + orch.create_worker_chip_queue(worker_id=True, depth=True, input_arena_bytes=True, output_arena_bytes=True) + + +def test_payload_admission_accepts_registered_buffer_and_bytes_like(): + orch, worker, shm, fake_client = _make_orchestrator() + try: + queue = orch.create_worker_chip_queue(worker_id=0, depth=4, input_arena_bytes=128, output_arena_bytes=128) + host = orch.alloc([16], DataType.UINT8) + host.access = AccessMode.READ + fake_client.requests.clear() + fake_client.payload_writes.clear() + queue.input.enqueue(host, nbytes=16, timeout=0.001) + assert queue.layout.input_arena_offset in [offset for offset, _data in fake_client.payload_writes] + + queue.input.enqueue(b"abcdefgh", nbytes=8, timeout=0.001) + queue.input.enqueue(bytearray(b"ijklmnop"), nbytes=8, timeout=0.001) + view = memoryview(b"readonly!") + assert view.readonly + queue.input.enqueue(view, nbytes=8, timeout=0.001) + + _publish_output(fake_client, queue, payload=b"abcdefghijklmnop") + dest = bytearray(16) + handle = queue.output.peek(timeout=0.001) + queue.output.read_into(handle, memoryview(dest)) + queue.output.release(handle) + assert bytes(dest) == b"abcdefghijklmnop" + finally: + _close(worker, shm) + + +def test_payload_admission_rejects_invalid_objects_without_shared_access(): + orch, worker, shm, fake_client = _make_orchestrator() + try: + queue = orch.create_worker_chip_queue(worker_id=0, depth=4, input_arena_bytes=128, output_arena_bytes=128) + host = orch.alloc([16], DataType.UINT8) + device = orch.alloc([16], DataType.UINT8) + device.address_space = AddressSpace.DEVICE + closed = orch.alloc([16], DataType.UINT8) + closed.close() + write_only = orch.alloc([16], DataType.UINT8) + write_only.access = AccessMode.WRITE + read_only = orch.alloc([16], DataType.UINT8) + read_only.access = AccessMode.READ + stale = orch.alloc([16], DataType.UINT8) + stale.base = int(stale.base) + 4096 + fake_client.requests.clear() + fake_client.payload_writes.clear() + before = _shared_snapshot(fake_client) + + def refuse(call): + with pytest.raises(ValueError): + call() + assert _shared_snapshot(fake_client) == before + assert queue._bound._state.name == "LIVE" + + refuse(lambda: queue.input.enqueue(device, nbytes=16, timeout=0.001)) + refuse(lambda: queue.input.enqueue(closed, nbytes=16, timeout=0.001)) + refuse(lambda: queue.input.enqueue(write_only, nbytes=16, timeout=0.001)) + refuse(lambda: queue.input.enqueue(stale, nbytes=16, timeout=0.001)) + refuse(lambda: queue.input.enqueue(_DuckPayload(host.base, 16), nbytes=16, timeout=0.001)) + refuse(lambda: queue.input.enqueue(memoryview(bytearray(b"abcdefgh"))[::2], nbytes=4, timeout=0.001)) + refuse(lambda: queue.input.try_enqueue(object(), nbytes=1)) + + _publish_output(fake_client, queue, payload=b"abcdefghijklmnop") + fake_client.requests.clear() + fake_client.payload_writes.clear() + before = _shared_snapshot(fake_client) + refuse(lambda: queue.output.try_dequeue_into(memoryview(b"xxxxxxxxxxxxxxxx"))) + refuse(lambda: queue.output.try_dequeue_into(read_only)) + refuse(lambda: queue.output.try_dequeue_into(device)) + dest = bytearray(4) + with pytest.raises(ValueError): + queue.output.try_dequeue_into(dest) + assert queue._bound._output_active is not None + retry = bytearray(16) + message = queue.output.try_dequeue_into(retry) + assert message is not None + assert bytes(retry) == b"abcdefghijklmnop" + assert queue._bound._output_active is None + finally: + _close(worker, shm) + + +def test_payload_admission_preserves_terminal_and_ownership_errors(): + orch, worker, shm, fake_client = _make_orchestrator() + try: + queue = orch.create_worker_chip_queue(worker_id=0, depth=4, input_arena_bytes=128, output_arena_bytes=128) + fake_client.fail_next_cmd = "payload_write" + with pytest.raises(RuntimeError, match="injected failure") as first: + queue.input.enqueue(b"abcdefgh", nbytes=8, timeout=0.001) + fake_client.requests.clear() + with pytest.raises(RuntimeError, match="injected failure") as later: + queue.input.enqueue(object(), nbytes=8, timeout=0.001) + assert later.value is first.value + assert fake_client.requests == [] + + orch2, worker2, shm2, fake_client2 = _make_orchestrator() + try: + queue2 = orch2.create_worker_chip_queue( + worker_id=0, depth=4, input_arena_bytes=128, output_arena_bytes=128 + ) + _publish_output(fake_client2, queue2, payload=b"abcdefghijklmnop") + handle = queue2.output.peek(timeout=0.001) + forged = WorkerChipQueueMessage(handle.seq, handle.opcode, handle.payload_offset, handle.payload_nbytes) + fake_client2.requests.clear() + with pytest.raises(RuntimeError, match="not active"): + queue2.output.read_into(forged, object()) + assert queue2._bound._state.name == "POISONED_LOCAL" + finally: + _close(worker2, shm2) + finally: + _close(worker, shm) + + +def test_facade_source_has_no_queue_algorithm(): + source = inspect.getsource(queue_mod) + assert "_advance_payload_head" not in source + assert "payload replay offset mismatch" not in source + assert "decode_spsc_queue_descriptor" not in source + assert "encode_spsc_queue_descriptor" not in source + assert "input_desc_tail_offset" not in source + assert "_RegionTemplateCoordinator" in source From 83d2bcf79f7e7f62c5088d3f392154dc3c732556 Mon Sep 17 00:00:00 2001 From: ccyywwen <75376396+ccyywwen@users.noreply.github.com> Date: Thu, 3 Sep 2026 16:39:10 +0800 Subject: [PATCH 3/7] Fix: satisfy pyright and formatters on SPSC queue APIs Make required_slots read-only so concrete templates are protocol- compatible, and type facade scalars as object so admission stays at runtime. Format the queue tests the hooks already rewrote. --- python/simpler/comm_region_template.py | 11 +++---- python/simpler/orchestrator.py | 4 ++- python/simpler/worker_chip_message_queue.py | 30 +++++++------------ tests/ut/cpp/common/test_region_template.cpp | 11 ++++--- .../test_worker/test_comm_region_template.py | 5 +++- .../test_worker_chip_message_queue.py | 8 ++--- 6 files changed, 33 insertions(+), 36 deletions(-) diff --git a/python/simpler/comm_region_template.py b/python/simpler/comm_region_template.py index aec70b3386..d3547a83e2 100644 --- a/python/simpler/comm_region_template.py +++ b/python/simpler/comm_region_template.py @@ -154,9 +154,9 @@ def decode_spsc_queue_descriptor(data: bytes) -> tuple[int, int, int, int]: @dataclass(frozen=True) class _SpscQueueConfig: - depth: int - input_arena_bytes: int - output_arena_bytes: int + depth: object + input_arena_bytes: object + output_arena_bytes: object @dataclass(frozen=True) @@ -350,7 +350,8 @@ def endpoint(self, slot: Enum) -> EndpointRecord: class _RegionTemplate(Protocol): - required_slots: Sequence[Enum] + @property + def required_slots(self) -> Sequence[Enum]: ... def plan(self, config: object) -> _RegionTemplatePlan: ... @@ -420,7 +421,7 @@ def _require_distinct_initiator_peer(slots: _ResolvedTemplateSlots) -> tuple[End class _SpscQueueTemplate: - required_slots = (_SpscQueueSlot.INITIATOR, _SpscQueueSlot.PEER) + required_slots: Sequence[Enum] = (_SpscQueueSlot.INITIATOR, _SpscQueueSlot.PEER) def plan(self, config: object) -> _SpscQueuePlan: if not isinstance(config, _SpscQueueConfig): diff --git a/python/simpler/orchestrator.py b/python/simpler/orchestrator.py index c3135521c2..f47bd41924 100644 --- a/python/simpler/orchestrator.py +++ b/python/simpler/orchestrator.py @@ -668,7 +668,9 @@ def create_worker_chip_region(self, *, worker_id: int, payload_bytes: int, count with self._control_admission("create_worker_chip_region"): return self._worker._create_worker_chip_region(int(worker_id), int(payload_bytes), int(counter_bytes)) - def create_worker_chip_queue(self, *, worker_id: int, depth: int, input_arena_bytes: int, output_arena_bytes: int): + def create_worker_chip_queue( + self, *, worker_id: object, depth: object, input_arena_bytes: object, output_arena_bytes: object + ): """Create an L3-L2 message queue on one NEXT_LEVEL chip worker.""" if self._worker is None: raise RuntimeError("create_worker_chip_queue requires an Orchestrator bound to a Worker") diff --git a/python/simpler/worker_chip_message_queue.py b/python/simpler/worker_chip_message_queue.py index 3a5ca32df1..f2e5893c82 100644 --- a/python/simpler/worker_chip_message_queue.py +++ b/python/simpler/worker_chip_message_queue.py @@ -11,7 +11,7 @@ from __future__ import annotations import operator -from typing import Any +from typing import Any, SupportsIndex, cast from .buffer import AccessMode, Buffer from .comm_endpoints import DEVICE_AICPU, HOST_CPU, SingleOwner, _format_worker_path, at @@ -48,12 +48,12 @@ def _require_worker_chip_id(value: object) -> int: if type(value) is bool: raise TypeError("worker_id must not be bool") try: - return operator.index(value) + return operator.index(cast(SupportsIndex, value)) except TypeError as exc: raise TypeError("worker_id must support operator.index()") from exc -def _admit_bytes_like(obj: object, nbytes: int, *, writable: bool) -> object: +def _admit_bytes_like(obj: Any, nbytes: int, *, writable: bool) -> object: try: view = memoryview(obj) except TypeError as exc: @@ -63,9 +63,7 @@ def _admit_bytes_like(obj: object, nbytes: int, *, writable: bool) -> object: if writable and view.readonly: raise ValueError("worker-chip queue output target must be writable") if int(view.nbytes) < int(nbytes): - raise ValueError( - f"worker-chip queue nbytes={int(nbytes)} exceeds ordinary host buffer size {int(view.nbytes)}" - ) + raise ValueError(f"worker-chip queue nbytes={int(nbytes)} exceeds ordinary host buffer size {int(view.nbytes)}") return obj @@ -75,17 +73,13 @@ def _admit_registered_buffer(worker: Any, obj: Buffer, nbytes: int, *, writable: worker._validate_worker_chip_orch_comm_host_buffer(obj) needed = AccessMode.WRITE if writable else AccessMode.READ if obj.access not in (needed, AccessMode.READWRITE): - raise ValueError( - f"worker-chip queue buffer grants {obj.access.name} but this direction needs {needed.name}" - ) + raise ValueError(f"worker-chip queue buffer grants {obj.access.name} but this direction needs {needed.name}") if int(obj.nbytes) < int(nbytes): raise ValueError(f"worker-chip queue nbytes={int(nbytes)} exceeds registered buffer size {int(obj.nbytes)}") return obj -def _admit_worker_chip_payload( - worker: Any, obj: object, nbytes: int, *, writable: bool, allow_none: bool -) -> object: +def _admit_worker_chip_payload(worker: Any, obj: object, nbytes: int, *, writable: bool, allow_none: bool) -> object: if int(nbytes) == 0: if allow_none: if obj is not None: @@ -108,7 +102,9 @@ def _require_live_bound_queue(lane: Any) -> None: lane._queue._ensure_usable() -def make_worker_chip_queue_layout(depth: int, input_arena_bytes: int, output_arena_bytes: int) -> _SpscQueueLayout: +def make_worker_chip_queue_layout( + depth: object, input_arena_bytes: object, output_arena_bytes: object +) -> _SpscQueueLayout: return _SpscQueueLayout.create( _SpscQueueConfig( depth=depth, @@ -174,16 +170,12 @@ def __init__(self, worker: Any, bound: _BoundSpscQueue) -> None: def try_enqueue(self, buffer_or_none: object, nbytes: int) -> bool: _require_live_bound_queue(self._lane) - admitted = _admit_worker_chip_payload( - self._worker, buffer_or_none, nbytes, writable=False, allow_none=True - ) + admitted = _admit_worker_chip_payload(self._worker, buffer_or_none, nbytes, writable=False, allow_none=True) return self._lane.try_enqueue(admitted, nbytes) def enqueue(self, buffer_or_none: object, nbytes: int, timeout: float) -> None: _require_live_bound_queue(self._lane) - admitted = _admit_worker_chip_payload( - self._worker, buffer_or_none, nbytes, writable=False, allow_none=True - ) + admitted = _admit_worker_chip_payload(self._worker, buffer_or_none, nbytes, writable=False, allow_none=True) self._lane.enqueue(admitted, nbytes, timeout) diff --git a/tests/ut/cpp/common/test_region_template.cpp b/tests/ut/cpp/common/test_region_template.cpp index 3d1029abc4..87b85ad3f7 100644 --- a/tests/ut/cpp/common/test_region_template.cpp +++ b/tests/ut/cpp/common/test_region_template.cpp @@ -72,9 +72,11 @@ TEST(RegionTemplateTest, PackedMagicVersionIsSpsqAbi10) { TEST(RegionTemplateTest, LayoutGoldenVectors) { for (const auto &test_case : kLayoutGolden) { spsc_queue::SpscQueueLayout layout{}; - ASSERT_TRUE(spsc_queue::SpscQueueLayout::create( - test_case.depth, test_case.input_arena_bytes, test_case.output_arena_bytes, &layout - )); + ASSERT_TRUE( + spsc_queue::SpscQueueLayout::create( + test_case.depth, test_case.input_arena_bytes, test_case.output_arena_bytes, &layout + ) + ); EXPECT_EQ(layout.input_desc_offset, 0u); EXPECT_EQ(layout.output_desc_offset, test_case.output_desc_offset); EXPECT_EQ(layout.input_arena_offset, test_case.input_arena_offset); @@ -329,7 +331,8 @@ struct FakeRegionView { bool read(uint64_t offset, uint64_t nbytes, spsc_queue::SpscQueuePayloadView &out) { out = spsc_queue::SpscQueuePayloadView{0, 0}; - view_->state->log.push_back(FakeAccess{FakeAccessKind::PayloadRead, offset, nbytes, 0, RegionWaitCmp::EQ, 0} + view_->state->log.push_back( + FakeAccess{FakeAccessKind::PayloadRead, offset, nbytes, 0, RegionWaitCmp::EQ, 0} ); if (view_->state->sticky_failed) { return false; diff --git a/tests/ut/py/test_worker/test_comm_region_template.py b/tests/ut/py/test_worker/test_comm_region_template.py index 91c6363b60..1abf85c6f4 100644 --- a/tests/ut/py/test_worker/test_comm_region_template.py +++ b/tests/ut/py/test_worker/test_comm_region_template.py @@ -16,6 +16,7 @@ import pickle import struct import threading +from collections.abc import Sequence from dataclasses import dataclass from enum import Enum from typing import Any @@ -702,7 +703,7 @@ def _bind(self, instance: object, slots: object) -> dict[str, object]: class _GenericTemplate: - required_slots = ( + required_slots: Sequence[Enum] = ( _GenericTemplateSlot.ALPHA, _GenericTemplateSlot.BETA, _GenericTemplateSlot.GAMMA, @@ -1209,6 +1210,7 @@ def test_coordinator_creates_generic_three_slot_template(region_worker): placement=_generic_placement(host, peer), result_projector=lambda bound: bound, ) + assert isinstance(result, dict) assert result["alpha"].path == "L3" assert result["beta"].path == "L3/L2[1]" assert result["gamma"].path == "L3" @@ -1374,6 +1376,7 @@ def test_output_exact_fit_true_wrap_and_zero_byte_replay(): out = queue._layout.output_arena_offset _plant_output_descriptor(region, queue, 1, SpscQueueOpcode.DATA, out, b"x" * 56) handle = queue.output.try_peek() + assert handle is not None queue.output.release(handle) _plant_output_descriptor(region, queue, 2, SpscQueueOpcode.DATA, out + 56, b"yz123456") handle = queue.output.try_peek() diff --git a/tests/ut/py/test_worker/test_worker_chip_message_queue.py b/tests/ut/py/test_worker/test_worker_chip_message_queue.py index c071e859bf..97e5f44e5c 100644 --- a/tests/ut/py/test_worker/test_worker_chip_message_queue.py +++ b/tests/ut/py/test_worker/test_worker_chip_message_queue.py @@ -1063,9 +1063,7 @@ def spy(*args, **kwargs): with pytest.raises(TypeError): orch.create_worker_chip_queue(worker_id=0, depth=4.0, input_arena_bytes=128, output_arena_bytes=128) with pytest.raises(TypeError): - orch.create_worker_chip_queue( - worker_id=0, depth=_IntOnly(), input_arena_bytes=128, output_arena_bytes=128 - ) + orch.create_worker_chip_queue(worker_id=0, depth=_IntOnly(), input_arena_bytes=128, output_arena_bytes=128) with pytest.raises(TypeError): orch.create_worker_chip_queue( worker_id=0, depth=_IndexOnly(4), input_arena_bytes=128, output_arena_bytes=128 @@ -1188,9 +1186,7 @@ def test_payload_admission_preserves_terminal_and_ownership_errors(): orch2, worker2, shm2, fake_client2 = _make_orchestrator() try: - queue2 = orch2.create_worker_chip_queue( - worker_id=0, depth=4, input_arena_bytes=128, output_arena_bytes=128 - ) + queue2 = orch2.create_worker_chip_queue(worker_id=0, depth=4, input_arena_bytes=128, output_arena_bytes=128) _publish_output(fake_client2, queue2, payload=b"abcdefghijklmnop") handle = queue2.output.peek(timeout=0.001) forged = WorkerChipQueueMessage(handle.seq, handle.opcode, handle.payload_offset, handle.payload_nbytes) From c87f0bad4aca39912dcbeae179faf4e8841ce345 Mon Sep 17 00:00:00 2001 From: ccyywwen <75376396+ccyywwen@users.noreply.github.com> Date: Thu, 3 Sep 2026 17:45:03 +0800 Subject: [PATCH 4/7] Fix: remove legacy L3Q2 native queue and its unit test The worker-chip facade already binds through the SPSC template, so the old AICPU header, 12-scalar decoder, and matching CMake target are dead production surface. --- .../include/aicpu/worker_chip_message_queue.h | 999 ------------------ tests/ut/cpp/CMakeLists.txt | 17 - .../common/test_worker_chip_message_queue.cpp | 842 --------------- 3 files changed, 1858 deletions(-) delete mode 100644 src/common/platform/include/aicpu/worker_chip_message_queue.h delete mode 100644 tests/ut/cpp/common/test_worker_chip_message_queue.cpp diff --git a/src/common/platform/include/aicpu/worker_chip_message_queue.h b/src/common/platform/include/aicpu/worker_chip_message_queue.h deleted file mode 100644 index 374119e9d9..0000000000 --- a/src/common/platform/include/aicpu/worker_chip_message_queue.h +++ /dev/null @@ -1,999 +0,0 @@ -/* - * Copyright (c) PyPTO Contributors. - * This program is free software, you can redistribute it and/or modify it under the terms and conditions of - * CANN Open Software License Agreement Version 2.0 (the "License"). - * Please refer to the License for details. You may not use this file except in compliance with the License. - * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, - * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. - * See LICENSE in the root of the software repository for the full text of the License. - * ----------------------------------------------------------------------------------------------------------- - */ - -#pragma once - -#include -#include -#include -#include - -#include "aicpu/worker_chip_orch_endpoint.h" - -static constexpr uint32_t WORKER_CHIP_QUEUE_MAGIC = 0x4C335132u; // "L3Q2" -static constexpr uint16_t WORKER_CHIP_QUEUE_ABI_MAJOR = 1; -static constexpr uint16_t WORKER_CHIP_QUEUE_ABI_MINOR = 1; -static constexpr uint64_t WORKER_CHIP_QUEUE_DESC_SLOT_BYTES = 32; -static constexpr uint64_t WORKER_CHIP_QUEUE_DESC_RING_ALIGNMENT = 8; -static constexpr uint64_t WORKER_CHIP_QUEUE_PAYLOAD_ARENA_ALIGNMENT = 64; -static constexpr uint64_t WORKER_CHIP_QUEUE_COUNTER_STRIDE = 64; -static constexpr uint64_t WORKER_CHIP_QUEUE_INPUT_DESC_TAIL_OFFSET = 0; -static constexpr uint64_t WORKER_CHIP_QUEUE_INPUT_DESC_HEAD_OFFSET = 64; -static constexpr uint64_t WORKER_CHIP_QUEUE_OUTPUT_DESC_TAIL_OFFSET = 128; -static constexpr uint64_t WORKER_CHIP_QUEUE_OUTPUT_DESC_HEAD_OFFSET = 192; -static constexpr uint64_t WORKER_CHIP_QUEUE_WORKER_ABORT_FLAG_OFFSET = 256; -static constexpr uint64_t WORKER_CHIP_QUEUE_CHIP_ABORT_FLAG_OFFSET = 320; -static constexpr uint64_t WORKER_CHIP_QUEUE_COUNTER_BYTES = 384; -static constexpr uint64_t WORKER_CHIP_QUEUE_MAX_DEPTH = 1ull << 30; -static constexpr uint64_t WORKER_CHIP_QUEUE_MAGIC_VERSION = worker_chip_orch_comm_pack_magic_version( - WORKER_CHIP_QUEUE_MAGIC, WORKER_CHIP_QUEUE_ABI_MAJOR, WORKER_CHIP_QUEUE_ABI_MINOR -); - -struct WorkerChipQueueDescSlot { - uint64_t seq; - uint64_t opcode; - uint64_t payload_offset; - uint64_t payload_nbytes; -}; - -static_assert( - sizeof(WorkerChipQueueDescSlot) == WORKER_CHIP_QUEUE_DESC_SLOT_BYTES, "WorkerChipQueueDescSlot ABI size changed" -); -static_assert(offsetof(WorkerChipQueueDescSlot, seq) == 0, "WorkerChipQueueDescSlot::seq offset changed"); -static_assert(offsetof(WorkerChipQueueDescSlot, opcode) == 8, "WorkerChipQueueDescSlot::opcode offset changed"); -static_assert( - offsetof(WorkerChipQueueDescSlot, payload_offset) == 16, "WorkerChipQueueDescSlot::payload_offset changed" -); -static_assert( - offsetof(WorkerChipQueueDescSlot, payload_nbytes) == 24, "WorkerChipQueueDescSlot::payload_nbytes changed" -); - -enum class WorkerChipQueueOpcode : uint64_t { - INVALID = 0, - DATA = 1, - STOP = 2, - ERROR = 3, -}; - -enum class WorkerChipQueueErrorKind : uint32_t { - NONE = 0, - BAD_ARGUMENT = 1, - BAD_DESCRIPTOR = 2, - INVALID_DESCRIPTOR = 3, - OUT_OF_SPACE = 4, - OWNERSHIP = 5, - REMOTE_ABORTED = 6, - ENDPOINT_ERROR = 7, -}; - -enum class WorkerChipQueueTimeoutStatus : uint32_t { - ORDINARY_TIMEOUT = 0, - REMOTE_ABORTED = 1, -}; - -enum class WorkerChipQueueOp : uint32_t { - INIT = 1, - TIMEOUT = 2, - INPUT_TRY_PEEK = 3, - INPUT_RELEASE = 4, - OUTPUT_TRY_RESERVE = 5, - OUTPUT_PUBLISH = 6, -}; - -inline const char *worker_chip_queue_op_to_string(WorkerChipQueueOp op) { - switch (op) { - case WorkerChipQueueOp::INIT: - return "init"; - case WorkerChipQueueOp::TIMEOUT: - return "timeout"; - case WorkerChipQueueOp::INPUT_TRY_PEEK: - return "input.try_peek"; - case WorkerChipQueueOp::INPUT_RELEASE: - return "input.release"; - case WorkerChipQueueOp::OUTPUT_TRY_RESERVE: - return "output.try_reserve"; - case WorkerChipQueueOp::OUTPUT_PUBLISH: - return "output.publish"; - default: - return "unknown"; - } -} - -struct WorkerChipQueueError { - WorkerChipQueueErrorKind kind; - WorkerChipQueueOp op; - uint64_t region_id; - char message[256]; -}; - -struct WorkerChipQueueLayout { - uint64_t depth; - uint64_t input_desc_offset; - uint64_t output_desc_offset; - uint64_t input_arena_offset; - uint64_t output_arena_offset; - uint64_t input_arena_bytes; - uint64_t output_arena_bytes; - uint64_t payload_bytes; - uint64_t input_desc_tail_offset; - uint64_t input_desc_head_offset; - uint64_t output_desc_tail_offset; - uint64_t output_desc_head_offset; - uint64_t worker_abort_flag_offset; - uint64_t chip_abort_flag_offset; - uint64_t counter_bytes; -}; - -struct WorkerChipQueueArgs { - uint64_t magic_version; - uint64_t depth; - uint64_t input_arena_bytes; - uint64_t output_arena_bytes; - uint64_t payload_bytes; - uint64_t counter_bytes; -}; - -struct WorkerChipQueueInputHandle { - uint64_t seq; - WorkerChipQueueOpcode opcode; - uint64_t payload_offset; - uint64_t payload_nbytes; - WorkerChipOrchPayloadView payload; -}; - -struct WorkerChipQueueOutputReservation { - uint64_t seq; - uint64_t payload_offset; - uint64_t payload_nbytes; - WorkerChipOrchPayloadView payload; - bool valid; -}; - -static inline uint64_t worker_chip_queue_magic_version() { return WORKER_CHIP_QUEUE_MAGIC_VERSION; } - -static inline bool worker_chip_queue_is_power_of_two(uint64_t value) { - return value != 0 && (value & (value - 1)) == 0; -} - -static inline uint64_t worker_chip_queue_align_up(uint64_t value, uint64_t align) { - if (align == 0) { - return value; - } - uint64_t remainder = value % align; - return remainder == 0 ? value : value + (align - remainder); -} - -static inline bool worker_chip_queue_align_up_checked(uint64_t value, uint64_t align, uint64_t *out) { - if (out == nullptr || align == 0) { - return false; - } - uint64_t remainder = value % align; - uint64_t bump = remainder == 0 ? 0 : align - remainder; - if (worker_chip_orch_comm_add_overflows(value, bump)) { - return false; - } - *out = value + bump; - return true; -} - -static inline bool worker_chip_queue_valid_opcode(WorkerChipQueueOpcode opcode) { - return opcode == WorkerChipQueueOpcode::DATA || opcode == WorkerChipQueueOpcode::STOP || - opcode == WorkerChipQueueOpcode::ERROR; -} - -static inline bool worker_chip_queue_make_layout( - uint64_t depth, uint64_t input_arena_bytes, uint64_t output_arena_bytes, WorkerChipQueueLayout &out -) { - if (!worker_chip_queue_is_power_of_two(depth) || depth > WORKER_CHIP_QUEUE_MAX_DEPTH || input_arena_bytes == 0 || - output_arena_bytes == 0 || input_arena_bytes % WORKER_CHIP_QUEUE_PAYLOAD_ARENA_ALIGNMENT != 0 || - output_arena_bytes % WORKER_CHIP_QUEUE_PAYLOAD_ARENA_ALIGNMENT != 0) { - return false; - } - - uint64_t desc_ring_bytes = depth * WORKER_CHIP_QUEUE_DESC_SLOT_BYTES; - uint64_t input_desc_offset = 0; - if (worker_chip_orch_comm_add_overflows(input_desc_offset, desc_ring_bytes)) { - return false; - } - uint64_t output_desc_offset = input_desc_offset + desc_ring_bytes; - if (worker_chip_orch_comm_add_overflows(output_desc_offset, desc_ring_bytes)) { - return false; - } - uint64_t desc_end = output_desc_offset + desc_ring_bytes; - uint64_t input_arena_offset = 0; - if (!worker_chip_queue_align_up_checked(desc_end, WORKER_CHIP_QUEUE_PAYLOAD_ARENA_ALIGNMENT, &input_arena_offset)) { - return false; - } - if (worker_chip_orch_comm_add_overflows(input_arena_offset, input_arena_bytes)) { - return false; - } - uint64_t input_arena_end = input_arena_offset + input_arena_bytes; - uint64_t output_arena_offset = 0; - if (!worker_chip_queue_align_up_checked( - input_arena_end, WORKER_CHIP_QUEUE_PAYLOAD_ARENA_ALIGNMENT, &output_arena_offset - )) { - return false; - } - if (worker_chip_orch_comm_add_overflows(output_arena_offset, output_arena_bytes)) { - return false; - } - uint64_t payload_bytes = output_arena_offset + output_arena_bytes; - - out = WorkerChipQueueLayout{ - depth, - input_desc_offset, - output_desc_offset, - input_arena_offset, - output_arena_offset, - input_arena_bytes, - output_arena_bytes, - payload_bytes, - WORKER_CHIP_QUEUE_INPUT_DESC_TAIL_OFFSET, - WORKER_CHIP_QUEUE_INPUT_DESC_HEAD_OFFSET, - WORKER_CHIP_QUEUE_OUTPUT_DESC_TAIL_OFFSET, - WORKER_CHIP_QUEUE_OUTPUT_DESC_HEAD_OFFSET, - WORKER_CHIP_QUEUE_WORKER_ABORT_FLAG_OFFSET, - WORKER_CHIP_QUEUE_CHIP_ABORT_FLAG_OFFSET, - WORKER_CHIP_QUEUE_COUNTER_BYTES, - }; - return output_desc_offset % WORKER_CHIP_QUEUE_DESC_RING_ALIGNMENT == 0 && - input_arena_offset % WORKER_CHIP_QUEUE_PAYLOAD_ARENA_ALIGNMENT == 0 && - output_arena_offset % WORKER_CHIP_QUEUE_PAYLOAD_ARENA_ALIGNMENT == 0; -} - -static inline bool worker_chip_queue_validate_region( - const WorkerChipOrchRegionDesc &desc, const WorkerChipQueueArgs &args, WorkerChipQueueLayout *out_layout -) { - WorkerChipQueueLayout layout{}; - if (args.magic_version != worker_chip_queue_magic_version() || - worker_chip_orch_comm_validate_desc(desc) != WorkerChipOrchCommValidationError::OK || - !worker_chip_queue_make_layout(args.depth, args.input_arena_bytes, args.output_arena_bytes, layout)) { - return false; - } - if (args.payload_bytes != layout.payload_bytes || args.counter_bytes != layout.counter_bytes || - desc.payload_bytes != layout.payload_bytes || desc.counter_bytes != layout.counter_bytes) { - return false; - } - if (out_layout != nullptr) { - *out_layout = layout; - } - return true; -} - -static inline void worker_chip_queue_encode_desc( - WorkerChipQueueDescSlot *slot, uint64_t seq, WorkerChipQueueOpcode opcode, uint64_t payload_offset, - uint64_t payload_nbytes -) { - if (slot == nullptr) { - return; - } - slot->seq = seq; - slot->opcode = static_cast(opcode); - slot->payload_offset = payload_offset; - slot->payload_nbytes = payload_nbytes; -} - -static inline bool -worker_chip_queue_reconstruct_counter(int32_t observed_low32, uint64_t depth, uint64_t &local_value) { - if (depth > WORKER_CHIP_QUEUE_MAX_DEPTH) { - return false; - } - uint32_t local_low32 = static_cast(local_value); - int32_t delta = static_cast(static_cast(observed_low32) - local_low32); - if (delta < 0 || static_cast(delta) > depth) { - return false; - } - local_value += static_cast(delta); - return true; -} - -namespace worker_chip_message_queue { - -static inline uint64_t magic_version() { return ::worker_chip_queue_magic_version(); } - -static inline bool is_power_of_two(uint64_t value) { return ::worker_chip_queue_is_power_of_two(value); } - -static inline uint64_t align_up(uint64_t value, uint64_t align) { return ::worker_chip_queue_align_up(value, align); } - -static inline bool align_up_checked(uint64_t value, uint64_t align, uint64_t *out) { - return ::worker_chip_queue_align_up_checked(value, align, out); -} - -static inline bool valid_opcode(WorkerChipQueueOpcode opcode) { return ::worker_chip_queue_valid_opcode(opcode); } - -static inline bool -make_layout(uint64_t depth, uint64_t input_arena_bytes, uint64_t output_arena_bytes, WorkerChipQueueLayout &out) { - return ::worker_chip_queue_make_layout(depth, input_arena_bytes, output_arena_bytes, out); -} - -static inline bool validate_region( - const WorkerChipOrchRegionDesc &desc, const WorkerChipQueueArgs &args, WorkerChipQueueLayout *out_layout -) { - return ::worker_chip_queue_validate_region(desc, args, out_layout); -} - -static inline void encode_desc( - WorkerChipQueueDescSlot *slot, uint64_t seq, WorkerChipQueueOpcode opcode, uint64_t payload_offset, - uint64_t payload_nbytes -) { - ::worker_chip_queue_encode_desc(slot, seq, opcode, payload_offset, payload_nbytes); -} - -static inline bool reconstruct_counter(int32_t observed_low32, uint64_t depth, uint64_t &local_value) { - return ::worker_chip_queue_reconstruct_counter(observed_low32, depth, local_value); -} - -} // namespace worker_chip_message_queue - -template -class WorkerChipQueueEndpoint { - static_assert(MaxInflight > 0, "MaxInflight must be positive"); - -public: - static constexpr uint64_t kStopEntrySlots = 1; - static constexpr uint64_t kEntryCapacity = MaxInflight + kStopEntrySlots; - - class InputQueue { - struct ActiveInputEntry { - uint64_t seq; - WorkerChipQueueOpcode opcode; - uint64_t payload_offset; - uint64_t payload_nbytes; - WorkerChipOrchPayloadView payload; - bool completed; - }; - - public: - explicit InputQueue(WorkerChipQueueEndpoint *parent) : - parent_(parent) {} - - InputQueue(const InputQueue &) = delete; - InputQueue &operator=(const InputQueue &) = delete; - InputQueue(InputQueue &&) = delete; - InputQueue &operator=(InputQueue &&) = delete; - - friend class WorkerChipQueueEndpoint; - - private: - bool initialize() { - for (uint64_t i = 0; i < kEntryCapacity; ++i) { - active_entries_[i] = ActiveInputEntry{}; - } - active_head_ = 0; - active_count_ = 0; - active_non_stop_count_ = 0; - input_head_ = 0; - input_tail_ = 0; - input_payload_head_ = 0; - input_payload_acquire_head_ = 0; - input_acquire_ = input_head_; - stop_observed_ = false; - drained_ = false; - return true; - } - - public: - bool peek(uint64_t timeout_ns, WorkerChipQueueInputHandle &out) { - uint64_t start = device_time_now_ticks(); - uint64_t frequency_hz = device_time_frequency_hz(); - uint64_t spins = 0; - while (true) { - if (try_peek(out)) { - return true; - } - if (parent_->error_.kind != WorkerChipQueueErrorKind::NONE) { - return false; - } - spins += 1; - if (timeout_ns == 0 || (spins & 1023ull) == 0) { - uint64_t now = device_time_now_ticks(); - if (timeout_ns == 0 || sys_cnt_elapsed_ns(start, now, frequency_hz) >= timeout_ns) { - parent_->disambiguate_timeout(); - return false; - } - } - } - } - - bool try_peek(WorkerChipQueueInputHandle &out) { - out = WorkerChipQueueInputHandle{0, WorkerChipQueueOpcode::INVALID, 0, 0, WorkerChipOrchPayloadView{0, 0}}; - if (!parent_->ensure_live()) { - return false; - } - const WorkerChipQueueLayout &layout = parent_->layout_; - WorkerChipOrchEndpoint &endpoint = parent_->endpoint_; - if constexpr (MaxInflight == 1) { - if (active_count_ != 0) { - parent_->poison( - WorkerChipQueueErrorKind::OWNERSHIP, WorkerChipQueueOp::INPUT_TRY_PEEK, - "input handle already active" - ); - return false; - } - } - if (!parent_->refresh_counter( - layout.input_desc_tail_offset, input_tail_, layout.depth, WorkerChipQueueOp::INPUT_TRY_PEEK - )) { - return false; - } - if (stop_observed_) { - if (input_tail_ != input_acquire_) { - parent_->poison( - WorkerChipQueueErrorKind::INVALID_DESCRIPTOR, WorkerChipQueueOp::INPUT_TRY_PEEK, - "input descriptor published after STOP" - ); - } - return false; - } - if (input_tail_ == input_acquire_) { - return false; - } - if (input_tail_ - input_head_ > layout.depth || input_acquire_ < input_head_ || - input_acquire_ > input_tail_) { - parent_->poison( - WorkerChipQueueErrorKind::INVALID_DESCRIPTOR, WorkerChipQueueOp::INPUT_TRY_PEEK, - "input descriptor state invalid" - ); - return false; - } - - WorkerChipQueueDescSlot slot{}; - uint64_t slot_index = input_acquire_ & (layout.depth - 1); - uint64_t slot_offset = layout.input_desc_offset + slot_index * sizeof(WorkerChipQueueDescSlot); - if (!parent_->read_desc_slot(slot_offset, &slot, WorkerChipQueueOp::INPUT_TRY_PEEK)) { - return false; - } - uint64_t expected_seq = input_acquire_ + 1; - if (slot.seq != expected_seq) { - parent_->poison( - WorkerChipQueueErrorKind::INVALID_DESCRIPTOR, WorkerChipQueueOp::INPUT_TRY_PEEK, - "input descriptor seq mismatch" - ); - return false; - } - WorkerChipQueueOpcode opcode = static_cast(slot.opcode); - if (!worker_chip_message_queue::valid_opcode(opcode)) { - parent_->poison( - WorkerChipQueueErrorKind::INVALID_DESCRIPTOR, WorkerChipQueueOp::INPUT_TRY_PEEK, - "invalid input opcode" - ); - return false; - } - if (opcode == WorkerChipQueueOpcode::STOP && (slot.payload_offset != 0 || slot.payload_nbytes != 0)) { - parent_->poison( - WorkerChipQueueErrorKind::INVALID_DESCRIPTOR, WorkerChipQueueOp::INPUT_TRY_PEEK, - "STOP descriptor must be zero-byte" - ); - return false; - } - bool counts_against_window = - opcode == WorkerChipQueueOpcode::DATA || opcode == WorkerChipQueueOpcode::ERROR; - if (counts_against_window && active_non_stop_count_ >= MaxInflight) { - return false; - } - if (active_count_ >= kEntryCapacity) { - parent_->poison( - WorkerChipQueueErrorKind::OWNERSHIP, WorkerChipQueueOp::INPUT_TRY_PEEK, "input window state full" - ); - return false; - } - - WorkerChipOrchPayloadView view{0, 0}; - if (slot.payload_nbytes == 0) { - if (slot.payload_offset != 0) { - parent_->poison( - WorkerChipQueueErrorKind::INVALID_DESCRIPTOR, WorkerChipQueueOp::INPUT_TRY_PEEK, - "zero-byte descriptor uses nonzero payload offset" - ); - return false; - } - } else if (!parent_->payload_in_arena( - slot.payload_offset, slot.payload_nbytes, layout.input_arena_offset, layout.input_arena_bytes - )) { - parent_->poison( - WorkerChipQueueErrorKind::INVALID_DESCRIPTOR, WorkerChipQueueOp::INPUT_TRY_PEEK, - "input payload out of arena" - ); - return false; - } else if (!parent_->payload_matches_head( - input_payload_acquire_head_, slot.payload_offset, slot.payload_nbytes, - layout.input_arena_offset, layout.input_arena_bytes, WorkerChipQueueOp::INPUT_TRY_PEEK - )) { - return false; - } else if (!endpoint.payload_read(slot.payload_offset, slot.payload_nbytes, view)) { - parent_->poison( - WorkerChipQueueErrorKind::ENDPOINT_ERROR, WorkerChipQueueOp::INPUT_TRY_PEEK, - endpoint.error().message - ); - return false; - } else { - parent_->advance_payload_head( - input_payload_acquire_head_, slot.payload_offset, slot.payload_nbytes, layout.input_arena_offset, - layout.input_arena_bytes, WorkerChipQueueOp::INPUT_TRY_PEEK - ); - if (parent_->error_.kind != WorkerChipQueueErrorKind::NONE) { - return false; - } - } - - out = WorkerChipQueueInputHandle{slot.seq, opcode, slot.payload_offset, slot.payload_nbytes, view}; - uint64_t insert_index = (active_head_ + active_count_) % kEntryCapacity; - active_entries_[insert_index] = - ActiveInputEntry{slot.seq, opcode, slot.payload_offset, slot.payload_nbytes, view, false}; - active_count_ += 1; - if (counts_against_window) { - active_non_stop_count_ += 1; - } - input_acquire_ += 1; - if (opcode == WorkerChipQueueOpcode::STOP) { - stop_observed_ = true; - if (input_tail_ != input_acquire_) { - parent_->poison( - WorkerChipQueueErrorKind::INVALID_DESCRIPTOR, WorkerChipQueueOp::INPUT_TRY_PEEK, - "input descriptor published after STOP" - ); - return false; - } - } - return true; - } - - bool release(const WorkerChipQueueInputHandle &handle) { - if (!parent_->ensure_live()) { - return false; - } - ActiveInputEntry *entry = entry_for_seq(handle.seq); - if (entry == nullptr || handle.opcode != entry->opcode || handle.payload_offset != entry->payload_offset || - handle.payload_nbytes != entry->payload_nbytes || handle.payload.gm_addr != entry->payload.gm_addr || - handle.payload.nbytes != entry->payload.nbytes) { - parent_->poison( - WorkerChipQueueErrorKind::OWNERSHIP, WorkerChipQueueOp::INPUT_RELEASE, "input handle is not active" - ); - return false; - } - if (entry->completed) { - parent_->poison( - WorkerChipQueueErrorKind::OWNERSHIP, WorkerChipQueueOp::INPUT_RELEASE, - "input handle already released" - ); - return false; - } - entry->completed = true; - return release_completed_prefix(); - } - - bool drained() const { return drained_; } - - private: - ActiveInputEntry *entry_for_seq(uint64_t seq) { - uint64_t first_seq = input_head_ + 1; - if (seq < first_seq) { - return nullptr; - } - uint64_t ordinal = seq - first_seq; - if (ordinal >= active_count_) { - return nullptr; - } - uint64_t index = (active_head_ + ordinal) % kEntryCapacity; - return active_entries_[index].seq == seq ? &active_entries_[index] : nullptr; - } - - bool release_completed_prefix() { - while (active_count_ != 0 && active_entries_[active_head_].completed) { - ActiveInputEntry entry = active_entries_[active_head_]; - if (entry.payload_nbytes != 0) { - parent_->advance_payload_head( - input_payload_head_, entry.payload_offset, entry.payload_nbytes, - parent_->layout_.input_arena_offset, parent_->layout_.input_arena_bytes, - WorkerChipQueueOp::INPUT_RELEASE - ); - if (parent_->error_.kind != WorkerChipQueueErrorKind::NONE) { - return false; - } - } - input_head_ += 1; - if (entry.opcode == WorkerChipQueueOpcode::DATA || entry.opcode == WorkerChipQueueOpcode::ERROR) { - active_non_stop_count_ -= 1; - } - if (entry.opcode == WorkerChipQueueOpcode::STOP) { - drained_ = true; - } - active_entries_[active_head_] = ActiveInputEntry{}; - active_head_ = (active_head_ + 1) % kEntryCapacity; - active_count_ -= 1; - if (!parent_->notify_counter( - parent_->layout_.input_desc_head_offset, static_cast(input_head_), - WorkerChipQueueOp::INPUT_RELEASE - )) { - return false; - } - } - return true; - } - - WorkerChipQueueEndpoint *parent_; - ActiveInputEntry active_entries_[kEntryCapacity]{}; - uint64_t active_head_{0}; - uint64_t active_count_{0}; - uint64_t active_non_stop_count_{0}; - uint64_t input_head_{0}; - uint64_t input_tail_{0}; - uint64_t input_payload_head_{0}; - uint64_t input_payload_acquire_head_{0}; - uint64_t input_acquire_{0}; - bool stop_observed_{false}; - bool drained_{false}; - }; - - class OutputQueue { - public: - explicit OutputQueue(WorkerChipQueueEndpoint *parent) : - parent_(parent) {} - - bool initialize() { - output_head_ = 0; - output_tail_ = 0; - output_payload_head_ = 0; - output_payload_tail_ = 0; - reservation_active_ = false; - reservation_seq_ = 0; - reservation_offset_ = 0; - reservation_nbytes_ = 0; - return true; - } - - bool reserve(uint64_t nbytes, uint64_t timeout_ns, WorkerChipQueueOutputReservation &out) { - uint64_t start = device_time_now_ticks(); - uint64_t frequency_hz = device_time_frequency_hz(); - uint64_t spins = 0; - while (true) { - if (try_reserve(nbytes, out)) { - return true; - } - if (parent_->error_.kind != WorkerChipQueueErrorKind::NONE) { - return false; - } - spins += 1; - if (timeout_ns == 0 || (spins & 1023ull) == 0) { - uint64_t now = device_time_now_ticks(); - if (timeout_ns == 0 || sys_cnt_elapsed_ns(start, now, frequency_hz) >= timeout_ns) { - parent_->disambiguate_timeout(); - return false; - } - } - } - } - - bool try_reserve(uint64_t nbytes, WorkerChipQueueOutputReservation &out) { - out = WorkerChipQueueOutputReservation{0, 0, 0, WorkerChipOrchPayloadView{0, 0}, false}; - if (!parent_->ensure_live()) { - return false; - } - const WorkerChipQueueLayout &layout = parent_->layout_; - if (reservation_active_) { - parent_->poison( - WorkerChipQueueErrorKind::OWNERSHIP, WorkerChipQueueOp::OUTPUT_TRY_RESERVE, - "output reservation already active" - ); - return false; - } - if (nbytes > layout.output_arena_bytes) { - return false; - } - uint64_t old_head = output_head_; - if (!parent_->refresh_counter( - layout.output_desc_head_offset, output_head_, layout.depth, WorkerChipQueueOp::OUTPUT_TRY_RESERVE - )) { - return false; - } - if (output_head_ != old_head && - !replay_output_releases(old_head, output_head_, WorkerChipQueueOp::OUTPUT_TRY_RESERVE)) { - return false; - } - if (output_tail_ - output_head_ >= layout.depth) { - return false; - } - - uint64_t payload_offset = 0; - WorkerChipOrchPayloadView view{0, 0}; - if (nbytes != 0) { - uint64_t arena_base = layout.output_arena_offset; - uint64_t arena_bytes = layout.output_arena_bytes; - uint64_t arena_pos = output_payload_tail_ % arena_bytes; - if (arena_pos + nbytes > arena_bytes) { - // Payloads are never split across arena wrap. The skipped tail bytes are retired in the - // monotonic virtual cursor even if this reservation later finds the arena full. - output_payload_tail_ += arena_bytes - arena_pos; - arena_pos = 0; - } - if (output_payload_tail_ + nbytes - output_payload_head_ > arena_bytes) { - return false; - } - payload_offset = arena_base + arena_pos; - view = WorkerChipOrchPayloadView{parent_->endpoint_.descriptor().payload_base + payload_offset, nbytes}; - output_payload_tail_ += nbytes; - } - - reservation_active_ = true; - reservation_seq_ = output_tail_ + 1; - reservation_offset_ = payload_offset; - reservation_nbytes_ = nbytes; - out = WorkerChipQueueOutputReservation{reservation_seq_, payload_offset, nbytes, view, true}; - return true; - } - - bool publish(const WorkerChipQueueOutputReservation &reservation, WorkerChipQueueOpcode opcode) { - if (!parent_->ensure_live()) { - return false; - } - if (!reservation_active_ || !reservation.valid || reservation.seq != reservation_seq_ || - reservation.payload_offset != reservation_offset_ || - reservation.payload_nbytes != reservation_nbytes_) { - parent_->poison( - WorkerChipQueueErrorKind::OWNERSHIP, WorkerChipQueueOp::OUTPUT_PUBLISH, "unknown output reservation" - ); - return false; - } - if (opcode == WorkerChipQueueOpcode::STOP || !worker_chip_message_queue::valid_opcode(opcode)) { - parent_->poison( - WorkerChipQueueErrorKind::INVALID_DESCRIPTOR, WorkerChipQueueOp::OUTPUT_PUBLISH, - "invalid output opcode" - ); - return false; - } - WorkerChipQueueDescSlot slot{}; - worker_chip_message_queue::encode_desc( - &slot, 0, opcode, reservation.payload_offset, reservation.payload_nbytes - ); - uint64_t slot_index = output_tail_ & (parent_->layout_.depth - 1); - uint64_t slot_offset = parent_->layout_.output_desc_offset + slot_index * sizeof(WorkerChipQueueDescSlot); - if (!parent_->write_desc_slot(slot_offset, slot, reservation.seq, WorkerChipQueueOp::OUTPUT_PUBLISH)) { - return false; - } - output_tail_ += 1; - reservation_active_ = false; - reservation_seq_ = 0; - reservation_offset_ = 0; - reservation_nbytes_ = 0; - return parent_->notify_counter( - parent_->layout_.output_desc_tail_offset, static_cast(output_tail_), - WorkerChipQueueOp::OUTPUT_PUBLISH - ); - } - - private: - bool replay_output_releases(uint64_t old_head, uint64_t new_head, WorkerChipQueueOp op) { - uint64_t cursor = old_head; - while (cursor < new_head) { - WorkerChipQueueDescSlot slot{}; - uint64_t slot_index = cursor & (parent_->layout_.depth - 1); - uint64_t slot_offset = - parent_->layout_.output_desc_offset + slot_index * sizeof(WorkerChipQueueDescSlot); - if (!parent_->read_desc_slot(slot_offset, &slot, op)) { - return false; - } - if (slot.seq != cursor + 1) { - parent_->poison( - WorkerChipQueueErrorKind::INVALID_DESCRIPTOR, op, "output release replay seq mismatch" - ); - return false; - } - if (slot.payload_nbytes != 0) { - parent_->advance_payload_head( - output_payload_head_, slot.payload_offset, slot.payload_nbytes, - parent_->layout_.output_arena_offset, parent_->layout_.output_arena_bytes, op - ); - if (parent_->error_.kind != WorkerChipQueueErrorKind::NONE) { - return false; - } - } - cursor += 1; - } - return true; - } - - WorkerChipQueueEndpoint *parent_; - uint64_t output_head_{0}; - uint64_t output_tail_{0}; - uint64_t output_payload_head_{0}; - uint64_t output_payload_tail_{0}; - bool reservation_active_{false}; - uint64_t reservation_seq_{0}; - uint64_t reservation_offset_{0}; - uint64_t reservation_nbytes_{0}; - }; - - WorkerChipQueueEndpoint(const WorkerChipOrchRegionDesc &desc, const WorkerChipQueueArgs &args) : - endpoint_(desc), - input_queue_(this), - output_queue_(this) { - if (endpoint_.error().kind != WorkerChipEndpointErrorKind::NONE || - !worker_chip_message_queue::validate_region(desc, args, &layout_)) { - set_error( - WorkerChipQueueErrorKind::BAD_DESCRIPTOR, WorkerChipQueueOp::INIT, desc.region_id, - "invalid queue descriptor" - ); - return; - } - if (MaxInflight > layout_.depth) { - set_error( - WorkerChipQueueErrorKind::BAD_ARGUMENT, WorkerChipQueueOp::INIT, desc.region_id, "invalid input window" - ); - return; - } - input_queue_.initialize(); - output_queue_.initialize(); - } - - WorkerChipQueueEndpoint(const WorkerChipQueueEndpoint &) = delete; - WorkerChipQueueEndpoint &operator=(const WorkerChipQueueEndpoint &) = delete; - WorkerChipQueueEndpoint(WorkerChipQueueEndpoint &&) = delete; - WorkerChipQueueEndpoint &operator=(WorkerChipQueueEndpoint &&) = delete; - - const WorkerChipQueueError &error() const { return error_; } - const WorkerChipQueueLayout &layout() const { return layout_; } - InputQueue &input() { return input_queue_; } - OutputQueue &output() { return output_queue_; } - - WorkerChipQueueTimeoutStatus disambiguate_timeout() { - if (error_.kind != WorkerChipQueueErrorKind::NONE) { - return error_.kind == WorkerChipQueueErrorKind::REMOTE_ABORTED ? - WorkerChipQueueTimeoutStatus::REMOTE_ABORTED : - WorkerChipQueueTimeoutStatus::ORDINARY_TIMEOUT; - } - WorkerChipOrchSignalTestResult result{}; - uint64_t addr = 0; - if (!endpoint_.counter_addr(layout_.worker_abort_flag_offset, addr) || - !endpoint_.signal_test(addr, 1, WorkerChipOrchWaitCmp::GE, result)) { - poison(WorkerChipQueueErrorKind::ENDPOINT_ERROR, WorkerChipQueueOp::TIMEOUT, endpoint_.error().message); - return WorkerChipQueueTimeoutStatus::ORDINARY_TIMEOUT; - } - if (result.matched) { - set_error( - WorkerChipQueueErrorKind::REMOTE_ABORTED, WorkerChipQueueOp::TIMEOUT, endpoint_.descriptor().region_id, - "remote abort" - ); - return WorkerChipQueueTimeoutStatus::REMOTE_ABORTED; - } - return WorkerChipQueueTimeoutStatus::ORDINARY_TIMEOUT; - } - -private: - bool ensure_live() { - if (error_.kind == WorkerChipQueueErrorKind::NONE) { - return true; - } - return false; - } - - void set_error(WorkerChipQueueErrorKind kind, WorkerChipQueueOp op, uint64_t region_id, const char *message) { - if (error_.kind != WorkerChipQueueErrorKind::NONE) { - return; - } - error_ = WorkerChipQueueError{kind, op, region_id, ""}; - worker_chip_orch_comm::copy_error_message(error_.message, sizeof(error_.message), message); - } - - void poison(WorkerChipQueueErrorKind kind, WorkerChipQueueOp op, const char *message) { - set_error(kind, op, endpoint_.descriptor().region_id, message); - if (kind != WorkerChipQueueErrorKind::REMOTE_ABORTED) { - uint64_t addr = 0; - if (endpoint_.counter_addr(layout_.chip_abort_flag_offset, addr)) { - endpoint_.signal_notify(addr, 1, WorkerChipOrchNotifyOp::Set); - } - } - } - - bool notify_counter(uint64_t offset, int32_t value, WorkerChipQueueOp op) { - uint64_t addr = 0; - if (!endpoint_.counter_addr(offset, addr) || - !endpoint_.signal_notify(addr, value, WorkerChipOrchNotifyOp::Set)) { - poison(WorkerChipQueueErrorKind::ENDPOINT_ERROR, op, endpoint_.error().message); - return false; - } - return true; - } - - bool refresh_counter(uint64_t offset, uint64_t &local, uint64_t depth, WorkerChipQueueOp op) { - uint64_t addr = 0; - WorkerChipOrchSignalTestResult result{}; - if (!endpoint_.counter_addr(offset, addr) || - !endpoint_.signal_test(addr, static_cast(local), WorkerChipOrchWaitCmp::NE, result)) { - poison(WorkerChipQueueErrorKind::ENDPOINT_ERROR, op, endpoint_.error().message); - return false; - } - if (!result.matched) { - return true; - } - if (!worker_chip_message_queue::reconstruct_counter(result.observed, depth, local)) { - poison(WorkerChipQueueErrorKind::INVALID_DESCRIPTOR, op, "counter reconstruction failed"); - return false; - } - return true; - } - - bool read_desc_slot(uint64_t slot_offset, WorkerChipQueueDescSlot *slot, WorkerChipQueueOp op) { - WorkerChipOrchPayloadView view{}; - if (!endpoint_.payload_read(slot_offset, sizeof(WorkerChipQueueDescSlot), view)) { - poison(WorkerChipQueueErrorKind::ENDPOINT_ERROR, op, endpoint_.error().message); - return false; - } - memcpy( - slot, reinterpret_cast(static_cast(view.gm_addr)), sizeof(WorkerChipQueueDescSlot) - ); - return true; - } - - bool - write_desc_slot(uint64_t slot_offset, const WorkerChipQueueDescSlot &slot, uint64_t seq, WorkerChipQueueOp op) { - WorkerChipQueueDescSlot fields = slot; - fields.seq = 0; - if (!endpoint_.payload_write(slot_offset + offsetof(WorkerChipQueueDescSlot, opcode), &fields.opcode, 24)) { - poison(WorkerChipQueueErrorKind::ENDPOINT_ERROR, op, endpoint_.error().message); - return false; - } - if (!endpoint_.payload_write(slot_offset + offsetof(WorkerChipQueueDescSlot, seq), &seq, sizeof(seq))) { - poison(WorkerChipQueueErrorKind::ENDPOINT_ERROR, op, endpoint_.error().message); - return false; - } - return true; - } - - static bool payload_in_arena(uint64_t offset, uint64_t nbytes, uint64_t arena_offset, uint64_t arena_bytes) { - if (nbytes == 0 || worker_chip_orch_comm::add_overflows(offset, nbytes)) { - return false; - } - return offset >= arena_offset && offset + nbytes <= arena_offset + arena_bytes; - } - - static uint64_t - payload_expected_offset(uint64_t cursor, uint64_t nbytes, uint64_t arena_offset, uint64_t arena_bytes) { - uint64_t arena_pos = cursor % arena_bytes; - return arena_pos + nbytes > arena_bytes ? arena_offset : arena_offset + arena_pos; - } - - bool payload_matches_head( - uint64_t cursor, uint64_t payload_offset, uint64_t nbytes, uint64_t arena_offset, uint64_t arena_bytes, - WorkerChipQueueOp op - ) { - if (nbytes == 0) { - return true; - } - uint64_t expected_offset = payload_expected_offset(cursor, nbytes, arena_offset, arena_bytes); - if (payload_offset != expected_offset) { - poison(WorkerChipQueueErrorKind::INVALID_DESCRIPTOR, op, "payload replay offset mismatch"); - return false; - } - return true; - } - - void advance_payload_head( - uint64_t &cursor, uint64_t payload_offset, uint64_t nbytes, uint64_t arena_offset, uint64_t arena_bytes, - WorkerChipQueueOp op - ) { - uint64_t arena_pos = cursor % arena_bytes; - uint64_t expected_offset = payload_expected_offset(cursor, nbytes, arena_offset, arena_bytes); - if (expected_offset != payload_offset) { - poison(WorkerChipQueueErrorKind::INVALID_DESCRIPTOR, op, "payload replay offset mismatch"); - return; - } - if (arena_pos + nbytes > arena_bytes) { - cursor += arena_bytes - (cursor % arena_bytes); - } - cursor += nbytes; - } - - WorkerChipOrchEndpoint endpoint_; - WorkerChipQueueLayout layout_{}; - WorkerChipQueueError error_{WorkerChipQueueErrorKind::NONE, WorkerChipQueueOp::INIT, 0, ""}; - InputQueue input_queue_; - OutputQueue output_queue_; -}; diff --git a/tests/ut/cpp/CMakeLists.txt b/tests/ut/cpp/CMakeLists.txt index 33b018fc7b..b54ab7432b 100644 --- a/tests/ut/cpp/CMakeLists.txt +++ b/tests/ut/cpp/CMakeLists.txt @@ -627,23 +627,6 @@ target_link_libraries(test_a5_trb_runtime_temp_buffer PRIVATE add_test(NAME test_a5_trb_runtime_temp_buffer COMMAND test_a5_trb_runtime_temp_buffer) set_tests_properties(test_a5_trb_runtime_temp_buffer PROPERTIES LABELS "no_hardware") -add_executable(test_worker_chip_message_queue - common/test_worker_chip_message_queue.cpp - stubs/test_stubs.cpp -) -target_include_directories(test_worker_chip_message_queue PRIVATE - ${GTEST_INCLUDE_DIRS} - ${CMAKE_SOURCE_DIR}/../../../src/a2a3/platform/include - ${CMAKE_SOURCE_DIR}/../../../src/common/platform/include -) -target_link_libraries(test_worker_chip_message_queue PRIVATE - ${GTEST_MAIN_LIB} - ${GTEST_LIB} - pthread -) -add_test(NAME test_worker_chip_message_queue COMMAND test_worker_chip_message_queue) -set_tests_properties(test_worker_chip_message_queue PROPERTIES LABELS "no_hardware") - add_executable(test_region_template common/test_region_template.cpp ) diff --git a/tests/ut/cpp/common/test_worker_chip_message_queue.cpp b/tests/ut/cpp/common/test_worker_chip_message_queue.cpp deleted file mode 100644 index ac48baf1a4..0000000000 --- a/tests/ut/cpp/common/test_worker_chip_message_queue.cpp +++ /dev/null @@ -1,842 +0,0 @@ -/* - * Copyright (c) PyPTO Contributors. - * This program is free software, you can redistribute it and/or modify it under the terms and conditions of - * CANN Open Software License Agreement Version 2.0 (the "License"). - * Please refer to the License for details. You may not use this file except in compliance with the License. - * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, - * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. - * See LICENSE in the root of the software repository for the full text of the License. - * ----------------------------------------------------------------------------------------------------------- - */ - -#include -#include -#include -#include -#include -#include - -#include - -#include "aicpu/worker_chip_message_queue.h" - -namespace { - -struct RegionStorage { - alignas(64) std::array payload{}; - alignas(64) std::array counters{}; -}; - -WorkerChipOrchRegionDesc make_desc(RegionStorage *storage, uint64_t payload_bytes = 512, uint64_t counter_bytes = 512) { - return WorkerChipOrchRegionDesc{ - worker_chip_orch_comm_magic_version(), - 19, - reinterpret_cast(storage->payload.data()), - payload_bytes, - reinterpret_cast(storage->counters.data()), - counter_bytes, - }; -} - -size_t counter_index(uint64_t offset) { return static_cast(offset / sizeof(int32_t)); } - -WorkerChipQueueArgs make_args(uint64_t depth, uint64_t input_arena_bytes, uint64_t output_arena_bytes) { - WorkerChipQueueLayout layout{}; - EXPECT_TRUE(worker_chip_queue_make_layout(depth, input_arena_bytes, output_arena_bytes, layout)); - return WorkerChipQueueArgs{ - WORKER_CHIP_QUEUE_MAGIC_VERSION, - depth, - input_arena_bytes, - output_arena_bytes, - layout.payload_bytes, - layout.counter_bytes, - }; -} - -WorkerChipOrchRegionDesc make_desc(RegionStorage *storage, const WorkerChipQueueArgs &args) { - return make_desc(storage, args.payload_bytes, args.counter_bytes); -} - -void publish_input_desc( - RegionStorage *storage, const WorkerChipQueueLayout &layout, uint64_t seq, WorkerChipQueueOpcode opcode, - uint64_t payload_offset = 0, uint64_t payload_nbytes = 0 -) { - WorkerChipQueueDescSlot slot{}; - worker_chip_queue_encode_desc(&slot, seq, opcode, payload_offset, payload_nbytes); - uint64_t desc_offset = - layout.input_desc_offset + ((seq - 1) & (layout.depth - 1)) * sizeof(WorkerChipQueueDescSlot); - std::memcpy(storage->payload.data() + desc_offset, &slot, sizeof(slot)); - storage->counters[counter_index(layout.input_desc_tail_offset)] = static_cast(seq); -} - -TEST(WorkerChipMessageQueueTest, MagicVersionConstantMatchesCompatibilityWrapper) { - EXPECT_EQ( - WORKER_CHIP_QUEUE_MAGIC_VERSION, - worker_chip_orch_comm_pack_magic_version( - WORKER_CHIP_QUEUE_MAGIC, WORKER_CHIP_QUEUE_ABI_MAJOR, WORKER_CHIP_QUEUE_ABI_MINOR - ) - ); - EXPECT_EQ(worker_chip_queue_magic_version(), WORKER_CHIP_QUEUE_MAGIC_VERSION); - EXPECT_EQ(worker_chip_message_queue::magic_version(), WORKER_CHIP_QUEUE_MAGIC_VERSION); -} - -TEST(WorkerChipMessageQueueTest, LayoutAssignsPayloadAndAbortCounterOffsets) { - WorkerChipQueueLayout layout{}; - - ASSERT_TRUE(worker_chip_queue_make_layout(4, 128, 192, layout)); - - EXPECT_EQ(layout.input_desc_offset, 0u); - EXPECT_EQ(layout.output_desc_offset, 4u * sizeof(WorkerChipQueueDescSlot)); - EXPECT_EQ(layout.input_arena_offset % 64u, 0u); - EXPECT_EQ(layout.output_arena_offset % 64u, 0u); - EXPECT_EQ(layout.input_desc_tail_offset, 0u); - EXPECT_EQ(layout.input_desc_head_offset, 64u); - EXPECT_EQ(layout.output_desc_tail_offset, 128u); - EXPECT_EQ(layout.output_desc_head_offset, 192u); - EXPECT_EQ(layout.worker_abort_flag_offset, 256u); - EXPECT_EQ(layout.chip_abort_flag_offset, 320u); - EXPECT_EQ(layout.counter_bytes, 384u); - EXPECT_GE(layout.payload_bytes, layout.output_arena_offset + 192u); -} - -TEST(WorkerChipMessageQueueTest, LayoutLockstepCasesMatchPythonMirrorExpectations) { - struct LayoutCase { - uint64_t depth; - uint64_t input_arena_bytes; - uint64_t output_arena_bytes; - uint64_t output_desc_offset; - uint64_t input_arena_offset; - uint64_t output_arena_offset; - uint64_t payload_bytes; - }; - - const std::array cases{{ - {1, 64, 64, 32, 64, 128, 192}, - {4, 128, 192, 128, 256, 384, 576}, - {8, 192, 64, 256, 512, 704, 768}, - }}; - - for (const auto &test_case : cases) { - WorkerChipQueueLayout layout{}; - ASSERT_TRUE(worker_chip_queue_make_layout( - test_case.depth, test_case.input_arena_bytes, test_case.output_arena_bytes, layout - )); - - EXPECT_EQ(layout.input_desc_offset, 0u); - EXPECT_EQ(layout.output_desc_offset, test_case.output_desc_offset); - EXPECT_EQ(layout.output_desc_offset, test_case.depth * sizeof(WorkerChipQueueDescSlot)); - EXPECT_EQ(layout.input_arena_offset, test_case.input_arena_offset); - EXPECT_EQ(layout.output_arena_offset, test_case.output_arena_offset); - EXPECT_EQ(layout.payload_bytes, test_case.payload_bytes); - EXPECT_EQ(layout.input_desc_tail_offset, 0u); - EXPECT_EQ(layout.input_desc_head_offset, 64u); - EXPECT_EQ(layout.output_desc_tail_offset, 128u); - EXPECT_EQ(layout.output_desc_head_offset, 192u); - EXPECT_EQ(layout.worker_abort_flag_offset, 256u); - EXPECT_EQ(layout.chip_abort_flag_offset, 320u); - EXPECT_EQ(layout.counter_bytes, 384u); - } -} - -TEST(WorkerChipMessageQueueTest, LayoutRejectsInvalidDepthArenaAndCounterBytes) { - WorkerChipQueueLayout layout{}; - - EXPECT_FALSE(worker_chip_queue_make_layout(3, 64, 64, layout)); - EXPECT_FALSE(worker_chip_queue_make_layout((1ull << 30) + 1, 64, 64, layout)); - EXPECT_FALSE(worker_chip_queue_make_layout(2, 0, 64, layout)); - EXPECT_FALSE(worker_chip_queue_make_layout(2, 65, 64, layout)); - - RegionStorage storage{}; - WorkerChipQueueArgs args = make_args(2, 64, 64); - EXPECT_FALSE(worker_chip_queue_validate_region(make_desc(&storage, 256, 320), args, &layout)); - EXPECT_FALSE(worker_chip_queue_validate_region(make_desc(&storage, 512, 384), args, &layout)); - EXPECT_TRUE(worker_chip_queue_validate_region(make_desc(&storage, args), args, &layout)); -} - -TEST(WorkerChipMessageQueueTest, LayoutOverflowFailsClosedWithoutModifyingOutput) { - WorkerChipQueueLayout layout{ - 7, 11, 13, 17, 19, 23, 29, 31, 37, 41, 43, 47, 53, 59, 61, - }; - const WorkerChipQueueLayout original = layout; - - EXPECT_FALSE(worker_chip_queue_make_layout(2, std::numeric_limits::max() - 63, 64, layout)); - - EXPECT_EQ(layout.depth, original.depth); - EXPECT_EQ(layout.input_desc_offset, original.input_desc_offset); - EXPECT_EQ(layout.output_desc_offset, original.output_desc_offset); - EXPECT_EQ(layout.input_arena_offset, original.input_arena_offset); - EXPECT_EQ(layout.output_arena_offset, original.output_arena_offset); - EXPECT_EQ(layout.input_arena_bytes, original.input_arena_bytes); - EXPECT_EQ(layout.output_arena_bytes, original.output_arena_bytes); - EXPECT_EQ(layout.payload_bytes, original.payload_bytes); - EXPECT_EQ(layout.counter_bytes, original.counter_bytes); -} - -TEST(WorkerChipMessageQueueTest, DescriptorSlotEncodingIsStable) { - static_assert(std::is_standard_layout::value, "descriptor must be POD-like"); - static_assert(std::is_trivially_copyable::value, "descriptor must be fixed-size"); - static_assert(std::is_standard_layout::value, "error must be POD-like"); - static_assert(std::is_trivially_copyable::value, "error must be fixed-size"); - - EXPECT_EQ(sizeof(WorkerChipQueueDescSlot), 32u); - EXPECT_EQ(offsetof(WorkerChipQueueDescSlot, seq), 0u); - EXPECT_EQ(offsetof(WorkerChipQueueDescSlot, opcode), 8u); - EXPECT_EQ(offsetof(WorkerChipQueueDescSlot, payload_offset), 16u); - EXPECT_EQ(offsetof(WorkerChipQueueDescSlot, payload_nbytes), 24u); - EXPECT_EQ(sizeof(WorkerChipQueueError::message), 256u); - - WorkerChipQueueDescSlot slot{}; - worker_chip_queue_encode_desc(&slot, 7, WorkerChipQueueOpcode::ERROR, 128, 16); - EXPECT_EQ(slot.seq, 7u); - EXPECT_EQ(slot.opcode, 3u); - EXPECT_EQ(slot.payload_offset, 128u); - EXPECT_EQ(slot.payload_nbytes, 16u); -} - -TEST(WorkerChipMessageQueueTest, ErrorOperationStringsAndMessageCopyAreStable) { - EXPECT_STREQ(worker_chip_queue_op_to_string(WorkerChipQueueOp::INIT), "init"); - EXPECT_STREQ(worker_chip_queue_op_to_string(WorkerChipQueueOp::INPUT_TRY_PEEK), "input.try_peek"); - EXPECT_STREQ(worker_chip_queue_op_to_string(WorkerChipQueueOp::INPUT_RELEASE), "input.release"); - EXPECT_STREQ(worker_chip_queue_op_to_string(WorkerChipQueueOp::OUTPUT_TRY_RESERVE), "output.try_reserve"); - EXPECT_STREQ(worker_chip_queue_op_to_string(WorkerChipQueueOp::OUTPUT_PUBLISH), "output.publish"); - EXPECT_STREQ(worker_chip_queue_op_to_string(WorkerChipQueueOp::TIMEOUT), "timeout"); - EXPECT_STREQ(worker_chip_queue_op_to_string(static_cast(99)), "unknown"); - - char message[256]; - worker_chip_orch_comm::copy_error_message(message, sizeof(message), nullptr); - EXPECT_STREQ(message, ""); - - std::array long_message{}; - long_message.fill('x'); - long_message.back() = '\0'; - worker_chip_orch_comm::copy_error_message(message, sizeof(message), long_message.data()); - EXPECT_EQ(std::strlen(message), sizeof(message) - 1); - EXPECT_EQ(message[254], 'x'); - EXPECT_EQ(message[255], '\0'); -} - -TEST(WorkerChipMessageQueueTest, Low32ReconstructionAcceptsWrapAndRejectsImpossibleDeltas) { - uint64_t value = 0xFFFF'FFFFull; - - EXPECT_TRUE(worker_chip_queue_reconstruct_counter(0, 4, value)); - EXPECT_EQ(value, 0x1'0000'0000ull); - - value = (1ull << 31) - 2; - EXPECT_TRUE(worker_chip_queue_reconstruct_counter(static_cast(0x8000'0001u), 4, value)); - EXPECT_EQ(value, (1ull << 31) + 1); - - value = 100; - EXPECT_TRUE(worker_chip_queue_reconstruct_counter(104, 4, value)); - EXPECT_EQ(value, 104u); - - value = 100; - EXPECT_FALSE(worker_chip_queue_reconstruct_counter(99, 4, value)); - - value = 100; - EXPECT_FALSE(worker_chip_queue_reconstruct_counter(105, 4, value)); -} - -TEST(WorkerChipMessageQueueTest, L2InputPeekHandlesZeroByteDescriptorBeforeArenaValidation) { - RegionStorage storage{}; - WorkerChipQueueArgs args = make_args(2, 64, 64); - WorkerChipQueueEndpoint<> queue(make_desc(&storage, args), args); - ASSERT_EQ(queue.error().kind, WorkerChipQueueErrorKind::NONE) << queue.error().message; - - WorkerChipQueueDescSlot slot{}; - worker_chip_queue_encode_desc(&slot, 1, WorkerChipQueueOpcode::DATA, 0, 0); - std::memcpy(storage.payload.data() + queue.layout().input_desc_offset, &slot, sizeof(slot)); - storage.counters[0] = 1; - - WorkerChipQueueInputHandle handle{}; - ASSERT_TRUE(queue.input().try_peek(handle)) << queue.error().message; - - EXPECT_EQ(handle.seq, 1u); - EXPECT_EQ(handle.opcode, WorkerChipQueueOpcode::DATA); - EXPECT_EQ(handle.payload_nbytes, 0u); - EXPECT_EQ(handle.payload.gm_addr, 0u); - EXPECT_TRUE(queue.input().release(handle)) << queue.error().message; - EXPECT_EQ(storage.counters[16], 1); -} - -TEST(WorkerChipMessageQueueTest, L2InputPeekPoisonsZeroByteDescriptorWithNonzeroOffset) { - RegionStorage storage{}; - WorkerChipQueueArgs args = make_args(2, 64, 64); - WorkerChipQueueEndpoint<> queue(make_desc(&storage, args), args); - ASSERT_EQ(queue.error().kind, WorkerChipQueueErrorKind::NONE) << queue.error().message; - - WorkerChipQueueDescSlot slot{}; - worker_chip_queue_encode_desc(&slot, 1, WorkerChipQueueOpcode::DATA, 8, 0); - std::memcpy(storage.payload.data() + queue.layout().input_desc_offset, &slot, sizeof(slot)); - storage.counters[0] = 1; - - WorkerChipQueueInputHandle handle{}; - EXPECT_FALSE(queue.input().try_peek(handle)); - - EXPECT_EQ(queue.error().kind, WorkerChipQueueErrorKind::INVALID_DESCRIPTOR); - EXPECT_EQ(storage.counters[80], 1); -} - -TEST(WorkerChipMessageQueueTest, L2InputPeekExposesNonzeroPayloadBytes) { - RegionStorage storage{}; - WorkerChipQueueArgs args = make_args(2, 64, 64); - WorkerChipQueueEndpoint<> queue(make_desc(&storage, args), args); - ASSERT_EQ(queue.error().kind, WorkerChipQueueErrorKind::NONE) << queue.error().message; - const std::array payload{{0x11, 0x22, 0x33, 0x44}}; - std::memcpy(storage.payload.data() + queue.layout().input_arena_offset, payload.data(), payload.size()); - publish_input_desc( - &storage, queue.layout(), 1, WorkerChipQueueOpcode::DATA, queue.layout().input_arena_offset, payload.size() - ); - - WorkerChipQueueInputHandle handle{}; - ASSERT_TRUE(queue.input().try_peek(handle)) << queue.error().message; - - ASSERT_EQ(handle.payload_nbytes, payload.size()); - const auto *observed = reinterpret_cast(static_cast(handle.payload.gm_addr)); - EXPECT_EQ(std::memcmp(observed, payload.data(), payload.size()), 0); - ASSERT_TRUE(queue.input().release(handle)) << queue.error().message; - EXPECT_EQ(queue.error().kind, WorkerChipQueueErrorKind::NONE); -} - -TEST(WorkerChipMessageQueueTest, L2InputPeekAllowsArenaWrapAtExpectedPayloadHead) { - RegionStorage storage{}; - WorkerChipQueueArgs args = make_args(2, 128, 64); - WorkerChipQueueEndpoint<> queue(make_desc(&storage, args), args); - ASSERT_EQ(queue.error().kind, WorkerChipQueueErrorKind::NONE) << queue.error().message; - - publish_input_desc(&storage, queue.layout(), 1, WorkerChipQueueOpcode::DATA, queue.layout().input_arena_offset, 80); - WorkerChipQueueInputHandle first{}; - ASSERT_TRUE(queue.input().try_peek(first)) << queue.error().message; - ASSERT_TRUE(queue.input().release(first)) << queue.error().message; - - publish_input_desc(&storage, queue.layout(), 2, WorkerChipQueueOpcode::DATA, queue.layout().input_arena_offset, 64); - WorkerChipQueueInputHandle second{}; - ASSERT_TRUE(queue.input().try_peek(second)) << queue.error().message; - - EXPECT_EQ(second.payload_offset, queue.layout().input_arena_offset); - EXPECT_EQ(second.payload_nbytes, 64u); - ASSERT_TRUE(queue.input().release(second)) << queue.error().message; - EXPECT_EQ(queue.error().kind, WorkerChipQueueErrorKind::NONE); -} - -TEST(WorkerChipMessageQueueTest, L2InputPeekRejectsPayloadOffsetMismatchBeforeRelease) { - RegionStorage storage{}; - WorkerChipQueueArgs args = make_args(2, 128, 64); - WorkerChipQueueEndpoint<> queue(make_desc(&storage, args), args); - ASSERT_EQ(queue.error().kind, WorkerChipQueueErrorKind::NONE) << queue.error().message; - publish_input_desc( - &storage, queue.layout(), 1, WorkerChipQueueOpcode::DATA, queue.layout().input_arena_offset + 64, 16 - ); - - WorkerChipQueueInputHandle handle{}; - EXPECT_FALSE(queue.input().try_peek(handle)); - - EXPECT_EQ(queue.error().kind, WorkerChipQueueErrorKind::INVALID_DESCRIPTOR); - EXPECT_EQ(storage.counters[counter_index(WORKER_CHIP_QUEUE_INPUT_DESC_HEAD_OFFSET)], 0); - EXPECT_EQ(storage.counters[counter_index(WORKER_CHIP_QUEUE_CHIP_ABORT_FLAG_OFFSET)], 1); -} - -TEST(WorkerChipMessageQueueTest, L2OutputReservePublishWritesDescriptorAndTail) { - RegionStorage storage{}; - WorkerChipQueueArgs args = make_args(2, 64, 64); - WorkerChipQueueEndpoint<> queue(make_desc(&storage, args), args); - ASSERT_EQ(queue.error().kind, WorkerChipQueueErrorKind::NONE) << queue.error().message; - - WorkerChipQueueOutputReservation reservation{}; - ASSERT_TRUE(queue.output().try_reserve(16, reservation)) << queue.error().message; - EXPECT_EQ(reservation.payload_nbytes, 16u); - EXPECT_NE(reservation.payload.gm_addr, 0u); - - ASSERT_TRUE(queue.output().publish(reservation, WorkerChipQueueOpcode::DATA)) << queue.error().message; - - WorkerChipQueueDescSlot slot{}; - std::memcpy(&slot, storage.payload.data() + queue.layout().output_desc_offset, sizeof(slot)); - EXPECT_EQ(slot.seq, 1u); - EXPECT_EQ(slot.opcode, 1u); - EXPECT_EQ(slot.payload_nbytes, 16u); - EXPECT_EQ(storage.counters[32], 1); -} - -TEST(WorkerChipMessageQueueTest, L2OutputReserveReplaysReleasedDescriptorsBeforeReusingArena) { - RegionStorage storage{}; - WorkerChipQueueArgs args = make_args(4, 64, 128); - WorkerChipQueueEndpoint<> queue(make_desc(&storage, args), args); - ASSERT_EQ(queue.error().kind, WorkerChipQueueErrorKind::NONE) << queue.error().message; - - WorkerChipQueueOutputReservation first{}; - ASSERT_TRUE(queue.output().try_reserve(80, first)) << queue.error().message; - ASSERT_EQ(first.payload_offset, queue.layout().output_arena_offset); - ASSERT_TRUE(queue.output().publish(first, WorkerChipQueueOpcode::DATA)) << queue.error().message; - - storage.counters[48] = 1; - WorkerChipQueueOutputReservation second{}; - ASSERT_TRUE(queue.output().try_reserve(80, second)) << queue.error().message; - - EXPECT_EQ(second.payload_offset, queue.layout().output_arena_offset); -} - -TEST(WorkerChipMessageQueueTest, RemoteAbortObservationDoesNotSetOwnAbortFlag) { - RegionStorage storage{}; - WorkerChipQueueArgs args = make_args(2, 64, 64); - WorkerChipQueueEndpoint<> queue(make_desc(&storage, args), args); - ASSERT_EQ(queue.error().kind, WorkerChipQueueErrorKind::NONE) << queue.error().message; - storage.counters[64] = 1; - - EXPECT_EQ(queue.disambiguate_timeout(), WorkerChipQueueTimeoutStatus::REMOTE_ABORTED); - - EXPECT_EQ(queue.error().kind, WorkerChipQueueErrorKind::REMOTE_ABORTED); - EXPECT_EQ(storage.counters[80], 0); -} - -TEST(WorkerChipMessageQueueTest, OrdinaryTimeoutDoesNotSetOwnAbortFlag) { - RegionStorage storage{}; - WorkerChipQueueArgs args = make_args(2, 64, 64); - WorkerChipQueueEndpoint<> queue(make_desc(&storage, args), args); - ASSERT_EQ(queue.error().kind, WorkerChipQueueErrorKind::NONE) << queue.error().message; - - EXPECT_EQ(queue.disambiguate_timeout(), WorkerChipQueueTimeoutStatus::ORDINARY_TIMEOUT); - - EXPECT_EQ(queue.error().kind, WorkerChipQueueErrorKind::NONE); - EXPECT_EQ(storage.counters[counter_index(WORKER_CHIP_QUEUE_CHIP_ABORT_FLAG_OFFSET)], 0); -} - -TEST(WorkerChipMessageQueueTest, OutputCapacityEqualsDepthAndFullIsNoProgressWithoutAbort) { - RegionStorage storage{}; - WorkerChipQueueArgs args = make_args(2, 64, 64); - WorkerChipQueueEndpoint<> queue(make_desc(&storage, args), args); - ASSERT_EQ(queue.error().kind, WorkerChipQueueErrorKind::NONE) << queue.error().message; - - for (int i = 0; i < 2; ++i) { - WorkerChipQueueOutputReservation reservation{}; - ASSERT_TRUE(queue.output().try_reserve(0, reservation)) << queue.error().message; - ASSERT_TRUE(queue.output().publish(reservation, WorkerChipQueueOpcode::DATA)) << queue.error().message; - } - WorkerChipQueueOutputReservation third{}; - EXPECT_FALSE(queue.output().try_reserve(0, third)); - - EXPECT_EQ(queue.error().kind, WorkerChipQueueErrorKind::NONE); - EXPECT_EQ(storage.counters[counter_index(WORKER_CHIP_QUEUE_OUTPUT_DESC_TAIL_OFFSET)], 2); - EXPECT_EQ(storage.counters[counter_index(WORKER_CHIP_QUEUE_CHIP_ABORT_FLAG_OFFSET)], 0); -} - -TEST(WorkerChipMessageQueueTest, FullAndEmptyUseMonotonicCountersNotMaskedIndices) { - RegionStorage storage{}; - WorkerChipQueueArgs args = make_args(2, 64, 64); - WorkerChipQueueEndpoint<> queue(make_desc(&storage, args), args); - ASSERT_EQ(queue.error().kind, WorkerChipQueueErrorKind::NONE) << queue.error().message; - - for (int i = 0; i < 2; ++i) { - WorkerChipQueueOutputReservation reservation{}; - ASSERT_TRUE(queue.output().try_reserve(0, reservation)) << queue.error().message; - ASSERT_TRUE(queue.output().publish(reservation, WorkerChipQueueOpcode::DATA)) << queue.error().message; - } - storage.counters[counter_index(WORKER_CHIP_QUEUE_OUTPUT_DESC_HEAD_OFFSET)] = 1; - - WorkerChipQueueOutputReservation third{}; - ASSERT_TRUE(queue.output().try_reserve(0, third)) << queue.error().message; - ASSERT_TRUE(queue.output().publish(third, WorkerChipQueueOpcode::DATA)) << queue.error().message; - - EXPECT_EQ(third.seq, 3u); - EXPECT_EQ(storage.counters[counter_index(WORKER_CHIP_QUEUE_OUTPUT_DESC_TAIL_OFFSET)], 3); - EXPECT_EQ(queue.error().kind, WorkerChipQueueErrorKind::NONE); - EXPECT_EQ(storage.counters[counter_index(WORKER_CHIP_QUEUE_CHIP_ABORT_FLAG_OFFSET)], 0); -} - -TEST(WorkerChipMessageQueueTest, OutputReserveTooLargeIsPreMutationNoProgressWithoutAbort) { - RegionStorage storage{}; - WorkerChipQueueArgs args = make_args(2, 64, 64); - WorkerChipQueueEndpoint<> queue(make_desc(&storage, args), args); - ASSERT_EQ(queue.error().kind, WorkerChipQueueErrorKind::NONE) << queue.error().message; - - WorkerChipQueueOutputReservation reservation{}; - EXPECT_FALSE(queue.output().try_reserve(65, reservation)); - - EXPECT_EQ(queue.error().kind, WorkerChipQueueErrorKind::NONE); - EXPECT_EQ(storage.counters[counter_index(WORKER_CHIP_QUEUE_OUTPUT_DESC_TAIL_OFFSET)], 0); - EXPECT_EQ(storage.counters[counter_index(WORKER_CHIP_QUEUE_CHIP_ABORT_FLAG_OFFSET)], 0); -} - -TEST(WorkerChipMessageQueueTest, OutputPublishApplicationErrorDoesNotSetAbortFlag) { - RegionStorage storage{}; - WorkerChipQueueArgs args = make_args(2, 64, 64); - WorkerChipQueueEndpoint<> queue(make_desc(&storage, args), args); - ASSERT_EQ(queue.error().kind, WorkerChipQueueErrorKind::NONE) << queue.error().message; - - WorkerChipQueueOutputReservation reservation{}; - ASSERT_TRUE(queue.output().try_reserve(0, reservation)) << queue.error().message; - ASSERT_TRUE(queue.output().publish(reservation, WorkerChipQueueOpcode::ERROR)) << queue.error().message; - - WorkerChipQueueDescSlot slot{}; - std::memcpy(&slot, storage.payload.data() + queue.layout().output_desc_offset, sizeof(slot)); - EXPECT_EQ(slot.opcode, static_cast(WorkerChipQueueOpcode::ERROR)); - EXPECT_EQ(queue.error().kind, WorkerChipQueueErrorKind::NONE); - EXPECT_EQ(storage.counters[counter_index(WORKER_CHIP_QUEUE_CHIP_ABORT_FLAG_OFFSET)], 0); -} - -TEST(WorkerChipMessageQueueTest, OutputPublishStaleReservationPoisonsAndSetsOwnAbortFlag) { - RegionStorage storage{}; - WorkerChipQueueArgs args = make_args(2, 64, 64); - WorkerChipQueueEndpoint<> queue(make_desc(&storage, args), args); - ASSERT_EQ(queue.error().kind, WorkerChipQueueErrorKind::NONE) << queue.error().message; - - WorkerChipQueueOutputReservation reservation{}; - ASSERT_TRUE(queue.output().try_reserve(0, reservation)) << queue.error().message; - ASSERT_TRUE(queue.output().publish(reservation, WorkerChipQueueOpcode::DATA)) << queue.error().message; - EXPECT_FALSE(queue.output().publish(reservation, WorkerChipQueueOpcode::DATA)); - - EXPECT_EQ(queue.error().kind, WorkerChipQueueErrorKind::OWNERSHIP); - EXPECT_EQ(storage.counters[counter_index(WORKER_CHIP_QUEUE_CHIP_ABORT_FLAG_OFFSET)], 1); -} - -TEST(WorkerChipMessageQueueTest, InputApplicationErrorIsNormalMessageAndDoesNotSetAbortFlag) { - RegionStorage storage{}; - WorkerChipQueueArgs args = make_args(2, 64, 64); - WorkerChipQueueEndpoint<> queue(make_desc(&storage, args), args); - ASSERT_EQ(queue.error().kind, WorkerChipQueueErrorKind::NONE) << queue.error().message; - publish_input_desc(&storage, queue.layout(), 1, WorkerChipQueueOpcode::ERROR); - - WorkerChipQueueInputHandle handle{}; - ASSERT_TRUE(queue.input().try_peek(handle)) << queue.error().message; - EXPECT_EQ(handle.opcode, WorkerChipQueueOpcode::ERROR); - ASSERT_TRUE(queue.input().release(handle)) << queue.error().message; - - EXPECT_EQ(queue.error().kind, WorkerChipQueueErrorKind::NONE); - EXPECT_EQ(storage.counters[counter_index(WORKER_CHIP_QUEUE_CHIP_ABORT_FLAG_OFFSET)], 0); -} - -TEST(WorkerChipMessageQueueTest, InputReleaseRejectsCallerMutatedHandleMetadata) { - RegionStorage storage{}; - WorkerChipQueueArgs args = make_args(2, 64, 64); - WorkerChipQueueEndpoint<> queue(make_desc(&storage, args), args); - ASSERT_EQ(queue.error().kind, WorkerChipQueueErrorKind::NONE) << queue.error().message; - publish_input_desc(&storage, queue.layout(), 1, WorkerChipQueueOpcode::DATA, queue.layout().input_arena_offset, 16); - - WorkerChipQueueInputHandle handle{}; - ASSERT_TRUE(queue.input().try_peek(handle)) << queue.error().message; - handle.payload_nbytes = 0; - - EXPECT_FALSE(queue.input().release(handle)); - - EXPECT_EQ(queue.error().kind, WorkerChipQueueErrorKind::OWNERSHIP); - EXPECT_EQ(storage.counters[counter_index(WORKER_CHIP_QUEUE_INPUT_DESC_HEAD_OFFSET)], 0); - EXPECT_EQ(storage.counters[counter_index(WORKER_CHIP_QUEUE_CHIP_ABORT_FLAG_OFFSET)], 1); -} - -TEST(WorkerChipMessageQueueTest, InputStopReleaseRejectsLaterPublishedInputAsInvalidState) { - RegionStorage storage{}; - WorkerChipQueueArgs args = make_args(2, 64, 64); - WorkerChipQueueEndpoint<> queue(make_desc(&storage, args), args); - ASSERT_EQ(queue.error().kind, WorkerChipQueueErrorKind::NONE) << queue.error().message; - publish_input_desc(&storage, queue.layout(), 1, WorkerChipQueueOpcode::STOP); - - WorkerChipQueueInputHandle stop{}; - ASSERT_TRUE(queue.input().try_peek(stop)) << queue.error().message; - ASSERT_TRUE(queue.input().release(stop)) << queue.error().message; - - publish_input_desc(&storage, queue.layout(), 2, WorkerChipQueueOpcode::DATA); - WorkerChipQueueInputHandle later{}; - EXPECT_FALSE(queue.input().try_peek(later)); - - EXPECT_EQ(queue.error().kind, WorkerChipQueueErrorKind::INVALID_DESCRIPTOR); - EXPECT_EQ(storage.counters[counter_index(WORKER_CHIP_QUEUE_CHIP_ABORT_FLAG_OFFSET)], 1); -} - -TEST(WorkerChipMessageQueueTest, InputStopWithPayloadMetadataPoisonsAndSetsOwnAbortFlag) { - RegionStorage storage{}; - WorkerChipQueueArgs args = make_args(2, 64, 64); - WorkerChipQueueEndpoint<> queue(make_desc(&storage, args), args); - ASSERT_EQ(queue.error().kind, WorkerChipQueueErrorKind::NONE) << queue.error().message; - publish_input_desc(&storage, queue.layout(), 1, WorkerChipQueueOpcode::STOP, queue.layout().input_arena_offset, 8); - - WorkerChipQueueInputHandle handle{}; - EXPECT_FALSE(queue.input().try_peek(handle)); - - EXPECT_EQ(queue.error().kind, WorkerChipQueueErrorKind::INVALID_DESCRIPTOR); - EXPECT_EQ(storage.counters[counter_index(WORKER_CHIP_QUEUE_CHIP_ABORT_FLAG_OFFSET)], 1); -} - -TEST(WorkerChipMessageQueueTest, InputSecondPeekBeforeReleasePoisonsOwnershipAndSetsOwnAbortFlag) { - RegionStorage storage{}; - WorkerChipQueueArgs args = make_args(2, 64, 64); - WorkerChipQueueEndpoint<> queue(make_desc(&storage, args), args); - ASSERT_EQ(queue.error().kind, WorkerChipQueueErrorKind::NONE) << queue.error().message; - publish_input_desc(&storage, queue.layout(), 1, WorkerChipQueueOpcode::DATA); - - WorkerChipQueueInputHandle handle{}; - ASSERT_TRUE(queue.input().try_peek(handle)) << queue.error().message; - WorkerChipQueueInputHandle second{}; - EXPECT_FALSE(queue.input().try_peek(second)); - - EXPECT_EQ(queue.error().kind, WorkerChipQueueErrorKind::OWNERSHIP); - EXPECT_EQ(queue.error().op, WorkerChipQueueOp::INPUT_TRY_PEEK); - EXPECT_STREQ(worker_chip_queue_op_to_string(queue.error().op), "input.try_peek"); - EXPECT_EQ(storage.counters[counter_index(WORKER_CHIP_QUEUE_CHIP_ABORT_FLAG_OFFSET)], 1); -} - -TEST(WorkerChipMessageQueueTest, MaxInflightGreaterThanDepthSetsBadArgumentWithoutAbortFlag) { - RegionStorage too_large_storage{}; - WorkerChipQueueArgs args = make_args(2, 64, 64); - - WorkerChipQueueEndpoint<3> too_large(make_desc(&too_large_storage, args), args); - EXPECT_EQ(too_large.error().kind, WorkerChipQueueErrorKind::BAD_ARGUMENT); - EXPECT_EQ(too_large.error().op, WorkerChipQueueOp::INIT); - EXPECT_EQ(too_large_storage.counters[counter_index(WORKER_CHIP_QUEUE_CHIP_ABORT_FLAG_OFFSET)], 0); -} - -TEST(WorkerChipMessageQueueTest, MultiInflightAcquireAllowsSeveralDataInputs) { - RegionStorage storage{}; - WorkerChipQueueArgs args = make_args(4, 128, 128); - WorkerChipQueueEndpoint<3> queue(make_desc(&storage, args), args); - ASSERT_EQ(queue.error().kind, WorkerChipQueueErrorKind::NONE) << queue.error().message; - publish_input_desc(&storage, queue.layout(), 1, WorkerChipQueueOpcode::DATA); - publish_input_desc(&storage, queue.layout(), 2, WorkerChipQueueOpcode::DATA); - publish_input_desc(&storage, queue.layout(), 3, WorkerChipQueueOpcode::DATA); - - WorkerChipQueueInputHandle first{}; - WorkerChipQueueInputHandle second{}; - WorkerChipQueueInputHandle third{}; - EXPECT_TRUE(queue.input().try_peek(first)) << queue.error().message; - EXPECT_TRUE(queue.input().try_peek(second)) << queue.error().message; - EXPECT_TRUE(queue.input().try_peek(third)) << queue.error().message; - - EXPECT_EQ(first.seq, 1u); - EXPECT_EQ(second.seq, 2u); - EXPECT_EQ(third.seq, 3u); - EXPECT_EQ(queue.error().kind, WorkerChipQueueErrorKind::NONE); - EXPECT_EQ(storage.counters[counter_index(WORKER_CHIP_QUEUE_INPUT_DESC_HEAD_OFFSET)], 0); -} - -TEST(WorkerChipMessageQueueTest, MultiInflightAcquireAllowsNonZeroPayloadOffsetsBeforeRelease) { - RegionStorage storage{}; - WorkerChipQueueArgs args = make_args(4, 128, 128); - WorkerChipQueueEndpoint<3> queue(make_desc(&storage, args), args); - ASSERT_EQ(queue.error().kind, WorkerChipQueueErrorKind::NONE) << queue.error().message; - const uint64_t first_offset = queue.layout().input_arena_offset; - const uint64_t second_offset = first_offset + 16; - publish_input_desc(&storage, queue.layout(), 1, WorkerChipQueueOpcode::DATA, first_offset, 16); - publish_input_desc(&storage, queue.layout(), 2, WorkerChipQueueOpcode::DATA, second_offset, 16); - - WorkerChipQueueInputHandle first{}; - WorkerChipQueueInputHandle second{}; - ASSERT_TRUE(queue.input().try_peek(first)) << queue.error().message; - ASSERT_TRUE(queue.input().try_peek(second)) << queue.error().message; - - EXPECT_EQ(first.payload_offset, first_offset); - EXPECT_EQ(first.payload_nbytes, 16u); - EXPECT_EQ(second.payload_offset, second_offset); - EXPECT_EQ(second.payload_nbytes, 16u); - EXPECT_EQ(queue.error().kind, WorkerChipQueueErrorKind::NONE); - EXPECT_EQ(storage.counters[counter_index(WORKER_CHIP_QUEUE_INPUT_DESC_HEAD_OFFSET)], 0); -} - -TEST(WorkerChipMessageQueueTest, ErrorCountsAgainstInputWindowAndFullDoesNotPoison) { - RegionStorage storage{}; - WorkerChipQueueArgs args = make_args(4, 128, 128); - WorkerChipQueueEndpoint<2> queue(make_desc(&storage, args), args); - ASSERT_EQ(queue.error().kind, WorkerChipQueueErrorKind::NONE) << queue.error().message; - publish_input_desc(&storage, queue.layout(), 1, WorkerChipQueueOpcode::DATA); - publish_input_desc(&storage, queue.layout(), 2, WorkerChipQueueOpcode::ERROR); - publish_input_desc(&storage, queue.layout(), 3, WorkerChipQueueOpcode::DATA); - - WorkerChipQueueInputHandle first{}; - WorkerChipQueueInputHandle second{}; - WorkerChipQueueInputHandle third{}; - ASSERT_TRUE(queue.input().try_peek(first)) << queue.error().message; - ASSERT_TRUE(queue.input().try_peek(second)) << queue.error().message; - EXPECT_FALSE(queue.input().try_peek(third)); - - EXPECT_EQ(queue.error().kind, WorkerChipQueueErrorKind::NONE); - EXPECT_EQ(storage.counters[counter_index(WORKER_CHIP_QUEUE_INPUT_DESC_HEAD_OFFSET)], 0); - EXPECT_EQ(storage.counters[counter_index(WORKER_CHIP_QUEUE_CHIP_ABORT_FLAG_OFFSET)], 0); -} - -TEST(WorkerChipMessageQueueTest, OutOfOrderInputReleaseOnlyAdvancesCompletedFifoPrefix) { - RegionStorage storage{}; - WorkerChipQueueArgs args = make_args(4, 128, 128); - WorkerChipQueueEndpoint<3> queue(make_desc(&storage, args), args); - ASSERT_EQ(queue.error().kind, WorkerChipQueueErrorKind::NONE) << queue.error().message; - publish_input_desc(&storage, queue.layout(), 1, WorkerChipQueueOpcode::DATA); - publish_input_desc(&storage, queue.layout(), 2, WorkerChipQueueOpcode::DATA); - publish_input_desc(&storage, queue.layout(), 3, WorkerChipQueueOpcode::DATA); - - WorkerChipQueueInputHandle first{}; - WorkerChipQueueInputHandle second{}; - WorkerChipQueueInputHandle third{}; - ASSERT_TRUE(queue.input().try_peek(first)) << queue.error().message; - ASSERT_TRUE(queue.input().try_peek(second)) << queue.error().message; - ASSERT_TRUE(queue.input().try_peek(third)) << queue.error().message; - - ASSERT_TRUE(queue.input().release(second)) << queue.error().message; - EXPECT_EQ(storage.counters[counter_index(WORKER_CHIP_QUEUE_INPUT_DESC_HEAD_OFFSET)], 0); - - ASSERT_TRUE(queue.input().release(first)) << queue.error().message; - EXPECT_EQ(storage.counters[counter_index(WORKER_CHIP_QUEUE_INPUT_DESC_HEAD_OFFSET)], 2); - - ASSERT_TRUE(queue.input().release(third)) << queue.error().message; - EXPECT_EQ(storage.counters[counter_index(WORKER_CHIP_QUEUE_INPUT_DESC_HEAD_OFFSET)], 3); -} - -TEST(WorkerChipMessageQueueTest, RingWrapReleaseKeepsLogicalFifoOrder) { - RegionStorage storage{}; - WorkerChipQueueArgs args = make_args(4, 128, 128); - WorkerChipQueueEndpoint<3> queue(make_desc(&storage, args), args); - ASSERT_EQ(queue.error().kind, WorkerChipQueueErrorKind::NONE) << queue.error().message; - - for (uint64_t seq = 1; seq <= 3; ++seq) { - publish_input_desc(&storage, queue.layout(), seq, WorkerChipQueueOpcode::DATA); - } - - WorkerChipQueueInputHandle first{}; - WorkerChipQueueInputHandle second{}; - WorkerChipQueueInputHandle third{}; - ASSERT_TRUE(queue.input().try_peek(first)) << queue.error().message; - ASSERT_TRUE(queue.input().try_peek(second)) << queue.error().message; - ASSERT_TRUE(queue.input().try_peek(third)) << queue.error().message; - ASSERT_TRUE(queue.input().release(first)) << queue.error().message; - ASSERT_TRUE(queue.input().release(second)) << queue.error().message; - ASSERT_TRUE(queue.input().release(third)) << queue.error().message; - EXPECT_EQ(storage.counters[counter_index(WORKER_CHIP_QUEUE_INPUT_DESC_HEAD_OFFSET)], 3); - - for (uint64_t seq = 4; seq <= 6; ++seq) { - publish_input_desc(&storage, queue.layout(), seq, WorkerChipQueueOpcode::DATA); - } - - WorkerChipQueueInputHandle fourth{}; - WorkerChipQueueInputHandle fifth{}; - WorkerChipQueueInputHandle sixth{}; - ASSERT_TRUE(queue.input().try_peek(fourth)) << queue.error().message; - ASSERT_TRUE(queue.input().try_peek(fifth)) << queue.error().message; - ASSERT_TRUE(queue.input().try_peek(sixth)) << queue.error().message; - - ASSERT_TRUE(queue.input().release(fifth)) << queue.error().message; - EXPECT_EQ(storage.counters[counter_index(WORKER_CHIP_QUEUE_INPUT_DESC_HEAD_OFFSET)], 3); - ASSERT_TRUE(queue.input().release(fourth)) << queue.error().message; - EXPECT_EQ(storage.counters[counter_index(WORKER_CHIP_QUEUE_INPUT_DESC_HEAD_OFFSET)], 5); - ASSERT_TRUE(queue.input().release(sixth)) << queue.error().message; - EXPECT_EQ(storage.counters[counter_index(WORKER_CHIP_QUEUE_INPUT_DESC_HEAD_OFFSET)], 6); - EXPECT_EQ(queue.error().kind, WorkerChipQueueErrorKind::NONE); -} - -TEST(WorkerChipMessageQueueTest, StopDoesNotCountAgainstWindowAndDrainsAfterEarlierInputs) { - RegionStorage storage{}; - WorkerChipQueueArgs args = make_args(4, 128, 128); - WorkerChipQueueEndpoint<2> queue(make_desc(&storage, args), args); - ASSERT_EQ(queue.error().kind, WorkerChipQueueErrorKind::NONE) << queue.error().message; - publish_input_desc(&storage, queue.layout(), 1, WorkerChipQueueOpcode::DATA); - publish_input_desc(&storage, queue.layout(), 2, WorkerChipQueueOpcode::DATA); - publish_input_desc(&storage, queue.layout(), 3, WorkerChipQueueOpcode::STOP); - - WorkerChipQueueInputHandle first{}; - WorkerChipQueueInputHandle second{}; - WorkerChipQueueInputHandle stop{}; - ASSERT_TRUE(queue.input().try_peek(first)) << queue.error().message; - ASSERT_TRUE(queue.input().try_peek(second)) << queue.error().message; - ASSERT_TRUE(queue.input().try_peek(stop)) << queue.error().message; - EXPECT_EQ(stop.opcode, WorkerChipQueueOpcode::STOP); - - ASSERT_TRUE(queue.input().release(stop)) << queue.error().message; - EXPECT_FALSE(queue.input().drained()); - EXPECT_EQ(storage.counters[counter_index(WORKER_CHIP_QUEUE_INPUT_DESC_HEAD_OFFSET)], 0); - - ASSERT_TRUE(queue.input().release(first)) << queue.error().message; - EXPECT_FALSE(queue.input().drained()); - EXPECT_EQ(storage.counters[counter_index(WORKER_CHIP_QUEUE_INPUT_DESC_HEAD_OFFSET)], 1); - - ASSERT_TRUE(queue.input().release(second)) << queue.error().message; - EXPECT_TRUE(queue.input().drained()); - EXPECT_EQ(storage.counters[counter_index(WORKER_CHIP_QUEUE_INPUT_DESC_HEAD_OFFSET)], 3); -} - -TEST(WorkerChipMessageQueueTest, TryPeekAfterStopAcquireIsNoProgressWithoutPoison) { - RegionStorage storage{}; - WorkerChipQueueArgs args = make_args(4, 128, 128); - WorkerChipQueueEndpoint<2> queue(make_desc(&storage, args), args); - ASSERT_EQ(queue.error().kind, WorkerChipQueueErrorKind::NONE) << queue.error().message; - publish_input_desc(&storage, queue.layout(), 1, WorkerChipQueueOpcode::DATA); - publish_input_desc(&storage, queue.layout(), 2, WorkerChipQueueOpcode::STOP); - - WorkerChipQueueInputHandle first{}; - WorkerChipQueueInputHandle stop{}; - WorkerChipQueueInputHandle later{}; - ASSERT_TRUE(queue.input().try_peek(first)) << queue.error().message; - ASSERT_TRUE(queue.input().try_peek(stop)) << queue.error().message; - EXPECT_FALSE(queue.input().try_peek(later)); - - EXPECT_EQ(queue.error().kind, WorkerChipQueueErrorKind::NONE); - EXPECT_EQ(storage.counters[counter_index(WORKER_CHIP_QUEUE_CHIP_ABORT_FLAG_OFFSET)], 0); -} - -TEST(WorkerChipMessageQueueTest, StopAcquirePoisonsIfLaterInputAlreadyObserved) { - RegionStorage storage{}; - WorkerChipQueueArgs args = make_args(4, 128, 128); - WorkerChipQueueEndpoint<2> queue(make_desc(&storage, args), args); - ASSERT_EQ(queue.error().kind, WorkerChipQueueErrorKind::NONE) << queue.error().message; - publish_input_desc(&storage, queue.layout(), 1, WorkerChipQueueOpcode::DATA); - publish_input_desc(&storage, queue.layout(), 2, WorkerChipQueueOpcode::STOP); - publish_input_desc(&storage, queue.layout(), 3, WorkerChipQueueOpcode::DATA); - - WorkerChipQueueInputHandle input{}; - WorkerChipQueueInputHandle stop{}; - ASSERT_TRUE(queue.input().try_peek(input)) << queue.error().message; - EXPECT_FALSE(queue.input().try_peek(stop)); - - EXPECT_EQ(queue.error().kind, WorkerChipQueueErrorKind::INVALID_DESCRIPTOR); - EXPECT_EQ(storage.counters[counter_index(WORKER_CHIP_QUEUE_CHIP_ABORT_FLAG_OFFSET)], 1); -} - -TEST(WorkerChipMessageQueueTest, InputReleaseRejectsStaleAndDoubleRelease) { - RegionStorage stale_storage{}; - RegionStorage double_storage{}; - WorkerChipQueueArgs args = make_args(4, 128, 128); - - WorkerChipQueueEndpoint<2> stale_queue(make_desc(&stale_storage, args), args); - ASSERT_EQ(stale_queue.error().kind, WorkerChipQueueErrorKind::NONE) << stale_queue.error().message; - publish_input_desc(&stale_storage, stale_queue.layout(), 1, WorkerChipQueueOpcode::DATA); - publish_input_desc(&stale_storage, stale_queue.layout(), 2, WorkerChipQueueOpcode::DATA); - WorkerChipQueueInputHandle stale_first{}; - WorkerChipQueueInputHandle stale_second{}; - ASSERT_TRUE(stale_queue.input().try_peek(stale_first)) << stale_queue.error().message; - ASSERT_TRUE(stale_queue.input().try_peek(stale_second)) << stale_queue.error().message; - ASSERT_TRUE(stale_queue.input().release(stale_first)) << stale_queue.error().message; - ASSERT_TRUE(stale_queue.input().release(stale_second)) << stale_queue.error().message; - EXPECT_FALSE(stale_queue.input().release(stale_first)); - EXPECT_EQ(stale_queue.error().kind, WorkerChipQueueErrorKind::OWNERSHIP); - EXPECT_EQ(stale_storage.counters[counter_index(WORKER_CHIP_QUEUE_CHIP_ABORT_FLAG_OFFSET)], 1); - - WorkerChipQueueEndpoint<2> double_queue(make_desc(&double_storage, args), args); - ASSERT_EQ(double_queue.error().kind, WorkerChipQueueErrorKind::NONE) << double_queue.error().message; - publish_input_desc(&double_storage, double_queue.layout(), 1, WorkerChipQueueOpcode::DATA); - publish_input_desc(&double_storage, double_queue.layout(), 2, WorkerChipQueueOpcode::DATA); - WorkerChipQueueInputHandle first{}; - WorkerChipQueueInputHandle second{}; - ASSERT_TRUE(double_queue.input().try_peek(first)) << double_queue.error().message; - ASSERT_TRUE(double_queue.input().try_peek(second)) << double_queue.error().message; - ASSERT_TRUE(double_queue.input().release(second)) << double_queue.error().message; - EXPECT_FALSE(double_queue.input().release(second)); - EXPECT_EQ(double_queue.error().kind, WorkerChipQueueErrorKind::OWNERSHIP); - EXPECT_EQ(double_storage.counters[counter_index(WORKER_CHIP_QUEUE_CHIP_ABORT_FLAG_OFFSET)], 1); -} - -TEST(WorkerChipMessageQueueTest, OutputReservePublishWorksDuringStopDrain) { - RegionStorage storage{}; - WorkerChipQueueArgs args = make_args(4, 128, 128); - WorkerChipQueueEndpoint<2> queue(make_desc(&storage, args), args); - ASSERT_EQ(queue.error().kind, WorkerChipQueueErrorKind::NONE) << queue.error().message; - publish_input_desc(&storage, queue.layout(), 1, WorkerChipQueueOpcode::DATA); - publish_input_desc(&storage, queue.layout(), 2, WorkerChipQueueOpcode::STOP); - - WorkerChipQueueInputHandle input{}; - WorkerChipQueueInputHandle stop{}; - ASSERT_TRUE(queue.input().try_peek(input)) << queue.error().message; - ASSERT_TRUE(queue.input().try_peek(stop)) << queue.error().message; - ASSERT_TRUE(queue.input().release(stop)) << queue.error().message; - - WorkerChipQueueOutputReservation reservation{}; - ASSERT_TRUE(queue.output().try_reserve(16, reservation)) << queue.error().message; - ASSERT_TRUE(queue.output().publish(reservation, WorkerChipQueueOpcode::DATA)) << queue.error().message; - - EXPECT_FALSE(queue.input().drained()); - EXPECT_EQ(storage.counters[counter_index(WORKER_CHIP_QUEUE_OUTPUT_DESC_TAIL_OFFSET)], 1); - EXPECT_EQ(queue.error().kind, WorkerChipQueueErrorKind::NONE); -} - -} // namespace From 2d0f9744e8472f7efddab0b4cc8197138a89a3cb Mon Sep 17 00:00:00 2001 From: ccyywwen <75376396+ccyywwen@users.noreply.github.com> Date: Thu, 3 Sep 2026 19:02:03 +0800 Subject: [PATCH 5/7] Update: document the current SPSQ binding without version labels Rewrite the L3-L2 message queue page around the 10-scalar binding and recorded CI evidence, and report unsupported SPSQ versions without advertising ABI 1.0. --- docs/README.md | 2 +- docs/l3-l2-message-queue.md | 519 +++++++----------- docs/l3-l2-orch-comm.md | 2 +- .../l3/worker_chip_message_queue/README.md | 41 +- python/simpler/comm_region_template.py | 6 +- .../test_worker/test_comm_region_template.py | 6 +- 6 files changed, 229 insertions(+), 347 deletions(-) diff --git a/docs/README.md b/docs/README.md index 4913a48784..4e7bd33103 100644 --- a/docs/README.md +++ b/docs/README.md @@ -73,7 +73,7 @@ changing simpler's own internals. | -------- | -------------- | | [Communication Domains](comm-domain.md) | Dynamic `CommDomain` allocation and the symmetric window | | [L3-L2 Orchestrator Communication](l3-l2-orch-comm.md) | Host-side L3 talking directly to the L2 AICPU orchestrator | -| [L3-L2 Message Queue](l3-l2-message-queue.md) | The queue channel between an L3 host and L2 | +| [L3-L2 Message Queue](l3-l2-message-queue.md) | The current SPSQ binding between an L3 host and one L2 task | | [Directed NEXT_LEVEL Scheduling](directed-next-level-scheduling.md) | Targeting a specific next-level child instead of any free one | | [Remote L3 Worker Design](remote-l3-worker-design.md) | L4 host-to-host workers — protocol, transports, status | | [remote-l3-worker-design/](remote-l3-worker-design/README.md) | Full design set: protocol, buffers and transports, implementation plan and record | diff --git a/docs/l3-l2-message-queue.md b/docs/l3-l2-message-queue.md index dbaadc3a65..23b16fef59 100644 --- a/docs/l3-l2-message-queue.md +++ b/docs/l3-l2-message-queue.md @@ -5,13 +5,16 @@ with one persistent L2 AICPU Orchestrator task. The intended use case is repeated in-flight work: L3 enqueues input messages, L2 consumes them while the L2 task stays alive, L2 publishes output messages, -and L3 dequeues those outputs. The queue is built on top of the lower-level -L3-L2 orchestration communication primitives described in -[l3-l2-orch-comm.md](l3-l2-orch-comm.md). For where L3 and L2 sit in -the runtime stack, see +and L3 dequeues those outputs. The queue is a Region Template bound onto a +payload-and-counter `RegionInstance`. The lower-level orchestration primitives +are documented in [l3-l2-orch-comm.md](l3-l2-orch-comm.md). For where L3 and +L2 sit in the runtime stack, see [hierarchical-level-runtime.md](hierarchical-level-runtime.md). -## 1. API +There is one current binding: exactly ten little-endian `uint64` scalars. There +is no 12-scalar TaskArgs path and no decoder fallback. + +## 1. Create And Bind L3 creates one queue for one chip worker: @@ -24,11 +27,13 @@ queue = orch.create_worker_chip_queue( ) ``` -The queue owns one underlying `WorkerChipOrchRegion`. Its payload range is split into -input/output descriptor rings and input/output payload arenas. Its counter -range stores descriptor head/tail signals and abort flags. +`create_worker_chip_queue` is a compatibility factory. It does not call +`create_worker_chip_region()`. A Region Template coordinator materializes one +`RegionInstance`, binds the SPSC template onto that instance, and projects a +`WorkerChipQueue` for the L3 initiator. -L3 passes the primitive region descriptor and queue layout arguments to L2: +L3 hands the injected peer binding to L2 as TaskArgs scalars starting at +offset 0: ```python l2_args = TaskArgs() @@ -38,150 +43,105 @@ for value in queue.chip_task_arg_scalars(): orch.submit_next_level(l2_handle, l2_args, cfg, worker=0) ``` -`chip_task_arg_scalars()` returns: +`chip_task_arg_scalars()` returns the ten-field `SpscQueueEndpointBinding` in +this order: ```text -primitive region descriptor scalars[0..5] -queue_magic_version +magic_version +session_instance_id_bits +transaction_id +payload_base +payload_bytes +counter_base +counter_bytes depth input_arena_bytes output_arena_bytes -payload_bytes -counter_bytes ``` -L3 sends input messages through `queue.input`: - -```python -host_input = orch.alloc([nbytes], DataType.UINT8) -fill_input(host_input) - -queue.input.enqueue(host_input, nbytes=nbytes, timeout=timeout_s) -``` +`magic_version` packs the `SPSQ` magic (`0x53505351`) with the compiled major +and minor wire fields. A decoder that sees any other packed value fails closed +as an unsupported SPSQ version. `transaction_id` is part of the allocation +identity and must be nonzero. `session_instance_id_bits` may be zero. Native +poison diagnostics carry `(session_instance_id_bits, transaction_id)` so a +failure can be correlated to that `RegionInstance`. -`try_enqueue(buffer, nbytes)` is the non-blocking form. It returns `False` -when the input descriptor ring or payload arena has no space. That result is -ordinary backpressure and does not poison the queue. +`queue.region` is a compatibility escape hatch: a `WorkerChipOrchRegion` +projector over the same instance's PAYLOAD and COUNTER local views. Queue +layout, publication, failure, and lifecycle authority stay on the bound queue. +Independent `create_worker_chip_region()` remains available for primitive +orchestrator communication that is not a queue. -L3 receives output messages through `queue.output`: +L3 sends input through `queue.input` and receives output through +`queue.output`: ```python -host_output = orch.alloc([max_output_nbytes], DataType.UINT8) - +queue.input.enqueue(host_input, nbytes=nbytes, timeout=timeout_s) message = queue.output.peek(timeout=timeout_s) queue.output.read_into(message, host_output) queue.output.release(message) ``` -The convenience form reads and releases in one operation: +`try_enqueue(buffer, nbytes)` returns `False` for ordinary descriptor-ring or +arena backpressure and does not poison the queue. `try_peek()` and +`try_dequeue_into(buffer)` return `None` when no output is available. +`dequeue_into(buffer, timeout)` peeks, copies, and releases in one call. -```python -message = queue.output.dequeue_into(host_output, timeout=timeout_s) -``` +Admitted payloads are a registered HOST `Buffer`, or a contiguous host +`bytes` / `bytearray` / `memoryview`. Zero-byte messages use +`buffer_or_none=None` and `nbytes=0`. -`try_peek()` and `try_dequeue_into(buffer)` are the non-blocking forms. They -return `None` when no output message is available. +L3 requests graceful shutdown with `queue.request_stop(timeout)` or +`try_request_stop()`. `queue.free()` is logical: it releases the L3 queue +handle and marks the projected region handle released. It does not +synchronously free device memory. Physical cleanup follows the underlying +`RegionInstance` lifetime after submitted L2 work has drained. -The L3 buffer arguments may be runtime-managed tensors returned by -`orch.alloc(...)` or ordinary contiguous Python byte buffers such as `bytes` -and `bytearray`. The queue delegates payload movement to the underlying -primitive region backend: simulation uses the parent mapping, and onboard uses -VMM shareable-handle imports plus ACL copy operations from the L3 Host process. -Zero-byte messages use `buffer_or_none=None` and `nbytes=0`. - -L3 requests graceful shutdown by publishing an input-side `STOP` descriptor: - -```python -queue.request_stop(timeout=timeout_s) -queue.free() -``` +On L2, orchestration code decodes the ten scalars and constructs an injected +endpoint view: -`try_request_stop()` is the non-blocking form. `queue.free()` releases the L3 -queue handle and marks the underlying `WorkerChipOrchRegion` handle released. It does -not synchronously free device memory; physical cleanup follows the underlying -region lifetime model after submitted L2 work has drained. Small Python wrapper -scratch tensors used for descriptor packing are owned by the queue object and -follow normal Python object lifetime. Payload-transfer staging is delegated to -the underlying primitive region backend. +```cpp +#include "aicpu/region_instance_view.h" +#include "common/region_template.h" -On L2, orchestration code receives the primitive descriptor and queue args, -then constructs an endpoint: +uint64_t scalars[spsc_queue::kSpscQueueEndpointBindingScalarCount]; +for (size_t i = 0; i < spsc_queue::kSpscQueueEndpointBindingScalarCount; ++i) { + scalars[i] = orch_args.scalar(static_cast(i)); +} -```cpp -WorkerChipOrchRegionDesc desc{/* scalars from TaskArgs */}; -WorkerChipQueueArgs queue_args{ - magic_version, - depth, - input_arena_bytes, - output_arena_bytes, - payload_bytes, - counter_bytes, -}; +spsc_queue::SpscQueueEndpointBinding binding{}; +if (!spsc_queue::decode_endpoint_binding( + scalars, spsc_queue::kSpscQueueEndpointBindingScalarCount, &binding)) { + return; +} -WorkerChipQueueEndpoint<> queue(desc, queue_args); -if (queue.error().kind != WorkerChipQueueErrorKind::NONE) { +RegionInstanceView view( + RegionPartLocalSpan{binding.payload_base, binding.payload_bytes}, + RegionPartLocalSpan{binding.counter_base, binding.counter_bytes} +); +spsc_queue::SpscQueueEndpoint queue(binding, std::move(view), clock); +if (!queue.live()) { return; } ``` The default endpoint allows one active L2 DATA/ERROR input handle at a time. -L2 can opt into a larger input window with a compile-time endpoint structure -parameter: +L2 can opt into a larger input window with a compile-time parameter: ```cpp -WorkerChipQueueEndpoint<4> queue(desc, queue_args); +spsc_queue::SpscQueueEndpoint queue(binding, std::move(view), clock); ``` -The template argument is not part of L3 queue creation and does not change the -queue layout or the shared ABI. The valid range is `1 <= MaxInflight <= depth`. -Invalid template/layout combinations report `BAD_ARGUMENT` without setting the -L2 abort flag. STOP does not count against `MaxInflight`; the endpoint keeps -one extra active-entry slot so a STOP handle can remain pending behind earlier -DATA/ERROR handles. +`MaxInflight` is a local construction parameter. It is not part of L3 queue +creation and does not change the shared layout or the ten-scalar binding. The +valid range is `1 <= MaxInflight <= depth`. Invalid combinations report +`BAD_ARGUMENT` without setting the L2 abort flag. STOP does not count against +`MaxInflight`; the endpoint keeps one extra slot so a STOP handle can remain +pending behind earlier DATA/ERROR handles. -L2 consumes input messages from `queue.input()` and publishes outputs through -`queue.output()`: +## 2. Layout And Descriptor -```cpp -while (true) { - WorkerChipQueueInputHandle input{}; - if (!queue.input().peek(timeout_ns, input)) { - return; - } - - if (input.opcode == WorkerChipQueueOpcode::STOP) { - queue.input().release(input); - return; - } - - WorkerChipQueueOutputReservation output{}; - if (!queue.output().reserve(input.payload_nbytes, timeout_ns, output)) { - return; - } - - launch_aicore(input.payload, output.payload); - wait_aicore_done(); - - queue.output().publish(output, WorkerChipQueueOpcode::DATA); - queue.input().release(input); -} -``` - -`queue.input().try_peek(input)` and -`queue.output().try_reserve(nbytes, reservation)` are non-blocking. A `false` -return can mean ordinary no-progress, validation failure, or poison; check -`queue.error().kind` to distinguish ordinary no-progress from terminal error. - -With `WorkerChipQueueEndpoint` where `N > 1`, L2 may acquire several DATA or -ERROR inputs before releasing earlier ones. `release(handle)` then marks the -input logically complete; the queue physically advances the shared input head -only for the completed FIFO prefix. This lets L2 publish outputs in an -application-defined order while keeping the input descriptor and payload -release protocol FIFO. - -## 2. Layout - -The physical region has one payload range: +The physical payload range is split as: ```text payload region @@ -191,45 +151,24 @@ payload region `-- output payload arena ``` -The two payload arenas are separate: - -```text -input arena: producer = L3, consumer = L2 -output arena: producer = L2, consumer = L3 -``` +Input arena: producer = L3, consumer = L2. Output arena: producer = L2, +consumer = L3. `depth` is the descriptor-ring capacity in each direction. It must be a power of two and at most `2^30`. Queue capacity is exactly `depth` messages, not -`depth - 1`. - -`input_arena_bytes` and `output_arena_bytes` must be positive 64-byte -multiples. They do not need to be powers of two. A single message payload must -fit as one contiguous span inside its direction's arena. Payloads are not split -across arena wrap. +`depth - 1`. `input_arena_bytes` and `output_arena_bytes` must be positive +64-byte multiples. A single message payload must fit as one contiguous span +inside its direction's arena. Payloads are not split across arena wrap. -Python and C++ mirror the same deterministic queue layout calculation: - -```text -input_desc_offset -output_desc_offset -input_arena_offset -output_arena_offset -payload_bytes -counter_bytes -``` - -Python exposes this as `queue.layout`; L2 exposes it as `queue.layout()`. -L3 passes the derived `payload_bytes` and `counter_bytes` to L2. L2 rejects -initialization unless those values match both its local layout calculation and -the primitive region descriptor sizes. Lockstep tests cover representative -layout cases for the mirrored Python and C++ calculations. - -## 3. Descriptor ABI +Python `queue.layout` and C++ `queue.layout()` expose the same mirrored +offsets: descriptor rings, arenas, `payload_bytes`, and `counter_bytes`. L2 +rejects construction unless the binding sizes match both the local layout +calculation and the injected view spans. Each descriptor slot is 32 bytes: ```cpp -struct WorkerChipQueueDescSlot { +struct SpscQueueDescriptor { uint64_t seq; uint64_t opcode; uint64_t payload_offset; @@ -237,218 +176,154 @@ struct WorkerChipQueueDescSlot { }; ``` -`seq` is the transport sequence number for ring validation, wrap detection, and -diagnostics. It is not a user request ID. Applications that need request IDs, -batch IDs, final markers, or correlation fields should put them in their own -payload header. - -`payload_offset` is relative to the primitive region payload base. The payload -must be wholly inside the matching direction's arena. Zero-byte messages use -`payload_offset == 0` and `payload_nbytes == 0`. - -The queue currently defines these opcodes: +`seq` is the transport sequence number. It is not a user request ID. +Applications that need request IDs or correlation fields put them in a payload +header. `payload_offset` is relative to the payload base. Zero-byte messages +use `payload_offset == 0` and `payload_nbytes == 0`. | Opcode | Meaning | | ------ | ------- | | `DATA` | Ordinary application payload message. | | `STOP` | Graceful input-side shutdown request. | -| `ERROR` | Ordinary application-level error payload message. | +| `ERROR` | Ordinary application-level error payload. | -`STOP` is valid only on the input queue. The output queue has no `STOP` -message; L2 exit is observed through normal `Worker.run` drain. +`STOP` is valid only on the input queue. L2 exit is observed through normal +`Worker.run` drain. `ERROR` is a normal message: the queue does not interpret +its payload and does not poison on receipt. Infrastructure failures use poison +state instead. -`ERROR` is a normal queue message. The queue layer does not interpret its -payload and does not poison the queue when an `ERROR` message is received. -Infrastructure failures use poison state instead. +## 3. Publication And Timeouts -## 4. Signals And Ordering - -The queue uses the primitive signal counters as descriptor head/tail values. -Each shared signal is placed on a 64-byte stride: +Shared signals sit on a 64-byte stride: ```text -offset 0: input_desc_tail writer=L3 -offset 64: input_desc_head writer=L2 -offset 128: output_desc_tail writer=L2 -offset 192: output_desc_head writer=L3 -offset 256: worker_abort_flag writer=L3 -offset 320: chip_abort_flag writer=L2 +offset 0: input_desc_tail writer=L3 +offset 64: input_desc_head writer=L2 +offset 128: output_desc_tail writer=L2 +offset 192: output_desc_head writer=L3 +offset 256: initiator_abort_flag writer=L3 +offset 320: peer_abort_flag writer=L2 ``` -Descriptor counters store the low 32 bits of monotonic logical head/tail -values. Each endpoint reconstructs its local 64-bit value from observed -progress. The unobserved progress must be between zero and `depth`; anything -else is inconsistent shared state and poisons the queue. +Descriptor counters store the signed low 32 bits of monotonic logical +head/tail values. Each endpoint reconstructs its local 64-bit value from +observed progress. Unobserved progress must be between zero and `depth`; +anything else poisons the queue. The producer sequence is: ```text reserve payload space write payload bytes +make those bytes visible to the peer write descriptor fields write descriptor seq publish descriptor tail counter ``` -The consumer sequence is: - -```text -observe descriptor tail progress -read and validate descriptor -use payload bytes or payload view -release descriptor and payload -publish descriptor head counter -``` +Payload bytes must be visible to the peer before the producer writes `seq` and +publishes the tail. L2 `RegionInstanceView` payload writes flush the span. +AIV-produced output must be flushed before `publish`. The consumer observes +tail progress, validates the descriptor, uses the payload, then releases and +publishes the head. -All Python blocking queue operations require finite positive timeouts; passing -`timeout <= 0` is a caller error and raises `ValueError`. Python `try_*` APIs -are non-blocking and return `False` or `None` for ordinary no-progress. +Python blocking queue operations require a finite positive timeout; +`timeout <= 0` raises `ValueError`. Python `try_*` APIs are the non-blocking +path and return `False` or `None` for ordinary no-progress. -C++ blocking queue operations take `timeout_ns`; `timeout_ns == 0` is an -immediate timeout probe. They return `false` on no-progress, timeout, -validation failure, or poison. C++ `try_*` APIs are non-blocking and also -return `false` for ordinary no-progress, validation failure, or poison. +C++ blocking `peek(timeout_ns, ...)` and `reserve(nbytes, timeout_ns, ...)` +treat `timeout_ns == 0` as a live no-attempt: on a live endpoint they return +`false` without waiting and without attempting the operation. C++ `try_*` +APIs are the non-blocking progress path. A `false` return can mean +ordinary no-progress, validation failure, or poison; check `queue.error().kind` +to distinguish ordinary no-progress from terminal error. -Timeout under ordinary backpressure is not poison. After timeout, an endpoint -samples the peer abort flag; if the peer flag is set, the local endpoint -reports remote abort. +Timeout under ordinary backpressure is not poison. After a positive-timeout +wait expires, an endpoint samples the peer abort flag; if that flag is set, +the local endpoint reports remote abort. -## 5. Ownership +## 4. Ownership, STOP, And Errors Queue ownership is per message. On L3 output, `peek()` returns a handle that remains active until `release(handle)`. While a handle is active, repeated `try_peek()` returns the -same handle. The caller may read the payload with `read_into(handle, buffer)` -before releasing it. Releasing the wrong handle is an ownership error and -poisons the queue. - -On L2 input, `WorkerChipQueueEndpoint<>` keeps one active DATA/ERROR input handle. -L2 must not call `peek()` again before releasing that handle, except that STOP -may also be acquired into the endpoint's extra STOP slot. - -When L2 constructs `WorkerChipQueueEndpoint`, it may hold up to `N` active DATA -or ERROR input handles. DATA and ERROR both count against the window because -either may carry payload bytes that remain owned by L2 application code. STOP -does not count against the DATA/ERROR window, but it is still normal FIFO -content and is released only by the completed prefix. - -In window mode, `release(handle)` is logical completion: the application is -declaring that no future L2 code or in-flight AICore task will read that input -payload. The queue then physically releases only the completed FIFO prefix. If -input 2 is released before input 1, input 2 remains physically owned until -input 1 is also released. - -On L2 output, `reserve()` returns one active output reservation. L2 fills the -reserved payload span, then calls `publish(reservation, opcode)`. Publishing an -unknown, stale, already-published, or cross-queue reservation is an ownership -error and poisons the queue. - -The queue supports at most one active L2 output reservation. The input window -does not introduce multiple concurrent output reservations; output ordering and -output cardinality remain application-defined. - -## 6. STOP Semantics - -`STOP` is an input descriptor with no payload. It is acquired through -`queue.input().peek()` / `try_peek()` like DATA and ERROR, and the user must -release the STOP handle. - -With the default single-input endpoint, L2 observes and releases messages -before STOP, then releases STOP and returns from the persistent run. +same handle. Releasing the wrong handle poisons the queue. + +On L2 input, the default endpoint keeps one active DATA/ERROR handle. L2 must +not call `peek()` again before releasing that handle, except that STOP may also +be acquired into the extra STOP slot. With `SpscQueueEndpoint`, L2 +may hold up to `N` active DATA or ERROR inputs. `release(handle)` is logical +completion; the queue physically advances the shared input head only for the +completed FIFO prefix. If input 2 is released before input 1, input 2 remains +physically owned until input 1 is also released. + +On L2 output, `reserve()` returns one active reservation. L2 fills that span, +then calls `publish(reservation, opcode)`. Publishing an unknown, stale, +already-published, or cross-queue reservation poisons the queue. The input +window does not introduce multiple concurrent output reservations. + +`STOP` is an input descriptor with no payload. After L3 publishes `STOP`, +further input messages are rejected locally without poisoning. L3 may still +dequeue outputs that L2 publishes before returning. `request_stop(timeout)` +waits only until the `STOP` descriptor is published; it does not wait for L2 +exit and does not drain outputs. With an input window, STOP may be acquired while earlier DATA or ERROR inputs are still active. After STOP is acquired, the input queue enters draining mode -and does not acquire later DATA or ERROR descriptors. Earlier active inputs may -still produce outputs, and L2 may still use `queue.output().reserve()` and -`queue.output().publish()` while draining. STOP is physically released only -after all earlier active inputs are physically released. - -`queue.input().drained()` returns true only after STOP has been physically -released. Persistent L2 code should return only after `drained()` is true and -after it has published all outputs required by its own payload protocol. - -After L3 successfully publishes `STOP`, the input queue rejects further input -messages locally without poisoning. L3 may still dequeue output messages that -L2 publishes before returning. - -`request_stop(timeout)` waits only until the `STOP` descriptor is published. -It does not wait for L2 exit and does not drain outputs. Applications that need -all outputs must keep dequeuing until their own protocol-level final condition -is satisfied before returning from the L3 orchestration function. - -If L2 observes a published input descriptor after STOP, that descriptor is -invalid shared state and poisons the queue with `INVALID_DESCRIPTOR`. - -## 7. Error Handling - -The queue distinguishes no-progress, application errors, and infrastructure -poison. - -No-progress is non-terminal: - -- descriptor ring full; -- payload arena full; -- empty output queue; -- blocking operation timeout with no peer abort flag. - -Application-level error is represented by `opcode=ERROR`. It is delivered to -the peer as a normal message and does not set an abort flag. - -Infrastructure poison is terminal for the local queue handle: - -- descriptor sequence mismatch; -- invalid opcode in a published descriptor; -- output-side `STOP`; -- descriptor payload outside its direction's arena; -- impossible counter reconstruction or payload replay; -- payload command failure after shared mutation begins; -- counter notify failure; -- stale or invalid handle/reservation ownership. - -When an endpoint enters local infrastructure poison, it sets its own abort flag -for the peer. Observing the peer abort flag reports remote abort but does not -set the local abort flag. - +and does not acquire later DATA or ERROR descriptors. Earlier active inputs +may still produce outputs. STOP is physically released only after all earlier +active inputs are physically released. `queue.input().drained()` returns true +only after STOP has been physically released. If L2 observes a published input +descriptor after STOP, that descriptor poisons the queue with +`INVALID_DESCRIPTOR`. + +No-progress is non-terminal: descriptor ring full, payload arena full, empty +output queue, or a blocking timeout with no peer abort flag. Application-level +`ERROR` is a normal message and does not set an abort flag. + +Infrastructure poison is terminal for the local queue handle: descriptor +sequence mismatch, invalid opcode, output-side `STOP`, payload outside its +arena, impossible counter reconstruction or payload replay, payload command +failure after shared mutation begins, counter notify failure, or stale handle +ownership. A local poison sets the local abort flag for the peer. Observing +the peer abort flag reports remote abort but does not set the local flag. After poison, normal queue operations reject. Cleanup remains valid. -## 8. Example +## 5. Example And Platform Evidence -The example lives at: +The shipped smoke example lives at: ```text examples/workers/l3/worker_chip_message_queue/ ``` -It uses `WorkerChipQueueEndpoint<4>` and a PTO-ISA AIV kernel. L3 sends an initial -pair of DATA inputs, drains the outputs that the persistent L2 run publishes -for them, then sends another pair of DATA inputs followed by STOP. L2 acquires -multiple inputs before releasing the earlier ones, publishes outputs in a -different order from input acquisition, emits multiple outputs for one input, -combines two inputs into one output during STOP drain, and returns only after -`queue.input().drained()`. -Application request IDs and output kinds are carried in the payload headers; -the transport sequence number is not used as a request ID. - -Data-plane routing between L3 Python and an L2 host service is intentionally -deferred. That needs a separate design for L2 host-service routing, registered -host tensors, and possible IPC virtual-address mapping. - -## 9. Platform Support - -The message queue uses the existing L3-L2 orchestration communication region, -payload, and counter primitives. - -- `a2a3sim`: supported. -- `a5sim`: supported. -- `a2a3` onboard: supported where the underlying L3-L2 communication - primitives are supported. -- `a5` onboard: supported where the underlying L3-L2 communication - primitive is available. - -Simulation backends preserve the same API, ordering, timeout, and error -semantics as onboard backends. - -The runnable example lives in -`examples/workers/l3/worker_chip_message_queue` and is marked for `a2a3sim`, -`a2a3`, `a5sim`, and `a5`. +It uses `spsc_queue::SpscQueueEndpoint` and a PTO-ISA +AIV kernel. L3 sends an initial pair of DATA inputs, drains the outputs that +the persistent L2 run publishes for them, then sends another pair of DATA +inputs followed by STOP. L2 acquires multiple inputs before releasing earlier +ones, publishes outputs in a different order from input acquisition, emits +multiple outputs for one input, combines two inputs into one output during +STOP drain, and returns only after `queue.input().drained()`. Application +request IDs in that example are payload-header fields `0, 0, 0, 7`; the +transport `seq` is not used as a request ID. + +That example is a smoke path. It is not a formal Queue Acceptance record, and +it does not demonstrate wrap, `ERROR` opcode delivery, or dedicated +HostVmmCopyAccess/cache instrumentation. + +Recorded evidence at commit `c87f0bad`, CI run +[33740884796](https://github.com/hw-native-sys/simpler/actions/runs/33740884796): + +| Surface | Result | +| ------- | ------ | +| `a2a3sim` example ST | PASS (CI `st-sim-a2a3`; also reproduced locally) | +| `a5sim` example ST | PASS (CI `st-sim-a5`) | +| `a2a3` onboard example ST | PASS (`test_worker_chip_message_queue`, 8.1s, device 0, CI `st-onboard-a2a3`) | +| `a5` onboard example ST | PASS (`test_worker_chip_message_queue`, 7.0s, device 4, CI `st-onboard-a5`) | +| Python / C++ UT | PASS (CI `ut`, `ut-a2a3`, `ut-a5`) | +| Template-level ST under `tests/st/worker/comm_region/templates/queue/` | pending; directory is not in the tree | +| Dedicated wrap / `ERROR` / HostVmmCopyAccess instrumentation | not recorded | + +Simulation evidence does not stand in for hardware cache or HostVMM copy +behavior. Hardware rows above are example ST pass/fail only. diff --git a/docs/l3-l2-orch-comm.md b/docs/l3-l2-orch-comm.md index a4cfa9e4ec..910216ed8f 100644 --- a/docs/l3-l2-orch-comm.md +++ b/docs/l3-l2-orch-comm.md @@ -4,7 +4,7 @@ L3-L2 Orchestrator Communication lets an L3 Host Orchestrator exchange payload bytes and signal counters with a running L2 AICPU Orchestrator task. This page documents the low-level region, payload, and counter primitives. For -the ordered SPSC message queue wrapper built on these primitives, see +the ordered SPSC message queue Region Template bound onto those primitives, see [l3-l2-message-queue.md](l3-l2-message-queue.md). The intended use case is in-flight interaction: L3 can write input payload, diff --git a/examples/workers/l3/worker_chip_message_queue/README.md b/examples/workers/l3/worker_chip_message_queue/README.md index 2d04de758e..92f1ee9f0c 100644 --- a/examples/workers/l3/worker_chip_message_queue/README.md +++ b/examples/workers/l3/worker_chip_message_queue/README.md @@ -5,46 +5,53 @@ submits it, and waits. Here the host submits **one long-lived L2 task** and then feeds it a stream of requests while it runs, reading results back as they appear — a serving loop, not a batch. -The transport is the L3-L2 SPSC queue (`SPSQ` ABI 1.0): two arenas (input and +The transport is the L3-L2 SPSC queue (`SPSQ` binding): two arenas (input and output) plus a 10-scalar endpoint binding the L2 orchestration receives as -plain `TaskArgs` scalars starting at offset 0. Binaries built against the -previous `L3Q2` 12-scalar layout must be recompiled. This example is a smoke -path, not the formal Queue Acceptance record. See +plain `TaskArgs` scalars starting at offset 0. This example is a smoke path, +not the formal Queue Acceptance record. See [`docs/l3-l2-message-queue.md`](../../../../docs/l3-l2-message-queue.md) for -the channel's design. +the channel's current design. ## What this exercises | Concept | How | | ------- | --- | | **Creating the channel** | `orch.create_worker_chip_queue(worker_id=0, depth=8, input_arena_bytes=..., output_arena_bytes=...)`. | -| **Handing it to L2** | `queue.chip_task_arg_scalars()` packs the `SPSQ` 1.0 binding into `TaskArgs` as exactly 10 scalars — the L2 side needs no other setup. | +| **Handing it to L2** | `queue.chip_task_arg_scalars()` packs the `SPSQ` binding into `TaskArgs` as exactly 10 scalars — the L2 side needs no other setup. | | **Full-duplex, decoupled** | The host enqueues two requests, drains **three** responses, then enqueues two more. Requests and responses are not paired one-to-one. | | **Zero-copy reads** | `queue.output.peek(timeout)` → `read_into(message, buf)` → `release(message)`. `release` is what returns arena space; skipping it stalls the producer once the queue fills. | | **Cooperative shutdown** | `queue.request_stop(timeout)` lets the L2 side finish work already accepted. Responses queued before the stop are still drained afterwards. | | **Application-level correlation** | The queue's `seq` is transport ordering only. This example carries its own 64-byte headers — `request_id` in, `(request_id, kind, aux)` out — because responses may arrive out of request order and one request may produce several. | -The mode field in each request drives the L2 kernel: +The mode field in each request drives the L2 kernel. Payload-header +`request_id` values are `0, 0, 0, 7`: -| Request | Mode | Produces | -| ------- | ---- | -------- | -| 101 | 1 | two responses (kinds 10 and 11) | -| 102 | 2 | one response (kind 20) | -| 103 + 104 | 3 | one response combining both tiles (kind 30, `aux = 104`) | +| Request | request_id | Mode | Produces | +| ------- | ---------- | ---- | -------- | +| 1 | 0 | 1 | two responses (kinds 10 and 11) | +| 2 | 0 | 2 | one response (kind 20) | +| 3 | 0 | 3 | combined with request 4 | +| 4 | 7 | 3 | one response combining both tiles (kind 30, `aux = 7`) | -Note the expected output order: `102`'s response comes back before `101`'s two. -That is the point — a queue, not a call stack. +Note the expected output order: request 2's kind-20 response comes back before +request 1's two. That is the point — a queue, not a call stack. ## Run -Single device, and unlike the collective examples it runs on all four -platforms: +Single device. Local simulation: ```bash pytest examples/workers/l3/worker_chip_message_queue --platform a2a3sim -pytest examples/workers/l3/worker_chip_message_queue --platform a2a3 --device 0 ``` +Recorded platform evidence at commit `c87f0bad`, CI run +[33740884796](https://github.com/hw-native-sys/simpler/actions/runs/33740884796): +example ST PASS on `a2a3sim`, `a5sim`, `a2a3` onboard (device 0), and `a5` +onboard (device 4). Simulation does not stand in for hardware cache or +HostVMM copy behavior. The evidence table, including pending template-level +ST, is in +[`docs/l3-l2-message-queue.md`](../../../../docs/l3-l2-message-queue.md). + The test file is also the example — `run_worker_chip_message_queue_example(platform, device_id)` is importable directly. diff --git a/python/simpler/comm_region_template.py b/python/simpler/comm_region_template.py index d3547a83e2..4084d5b90d 100644 --- a/python/simpler/comm_region_template.py +++ b/python/simpler/comm_region_template.py @@ -6,7 +6,7 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. # ----------------------------------------------------------------------------------------------------------- -"""Internal Region Template types and the duplex SPSC queue ABI.""" +"""Internal Region Template types and the duplex SPSC queue binding.""" from __future__ import annotations @@ -253,7 +253,7 @@ def __post_init__(self) -> None: self, "output_arena_bytes", _require_exact_u64("output_arena_bytes", self.output_arena_bytes) ) if self.magic_version != _SPSC_QUEUE_MAGIC_VERSION: - raise ValueError("binding magic_version is not SPSQ ABI 1.0") + raise ValueError("unsupported SPSQ version") if self.transaction_id == 0: raise ValueError("transaction_id must be nonzero") @@ -281,7 +281,7 @@ def from_scalars(cls, scalars: Sequence[object]) -> SpscQueueEndpointBinding: raise ValueError("binding requires exactly 10 uint64 scalars") values = tuple(_require_exact_u64(f"binding[{index}]", scalars[index]) for index in range(count)) if values[0] != _SPSC_QUEUE_MAGIC_VERSION: - raise ValueError("binding magic_version is not SPSQ ABI 1.0") + raise ValueError("unsupported SPSQ version") if values[2] == 0: raise ValueError("transaction_id must be nonzero") return cls( diff --git a/tests/ut/py/test_worker/test_comm_region_template.py b/tests/ut/py/test_worker/test_comm_region_template.py index 1abf85c6f4..08c34b167b 100644 --- a/tests/ut/py/test_worker/test_comm_region_template.py +++ b/tests/ut/py/test_worker/test_comm_region_template.py @@ -336,15 +336,15 @@ def test_binding_rejects_bool_range_and_version_mismatch(): SpscQueueEndpointBinding.from_scalars(over) wrong_magic = list(_BINDING_GOLDEN) wrong_magic[0] = 0x4C33513200010001 - with pytest.raises(ValueError, match="SPSQ ABI 1.0"): + with pytest.raises(ValueError, match="unsupported SPSQ version"): SpscQueueEndpointBinding.from_scalars(wrong_magic) wrong_major = list(_BINDING_GOLDEN) wrong_major[0] = (_SPSC_QUEUE_MAGIC << 32) | (2 << 16) | 0 - with pytest.raises(ValueError, match="SPSQ ABI 1.0"): + with pytest.raises(ValueError, match="unsupported SPSQ version"): SpscQueueEndpointBinding.from_scalars(wrong_major) wrong_minor = list(_BINDING_GOLDEN) wrong_minor[0] = (_SPSC_QUEUE_MAGIC << 32) | (1 << 16) | 1 - with pytest.raises(ValueError, match="SPSQ ABI 1.0"): + with pytest.raises(ValueError, match="unsupported SPSQ version"): SpscQueueEndpointBinding.from_scalars(wrong_minor) From 33042a54bc9f37026d9c01075c98e89c4de2fcbf Mon Sep 17 00:00:00 2001 From: ccyywwen <75376396+ccyywwen@users.noreply.github.com> Date: Fri, 4 Sep 2026 09:50:26 +0800 Subject: [PATCH 6/7] Add: L3 SPSC queue template scene tests Give the bound queue a formal L3 single-hop record covering wrap, ERROR, STOP, and sim-only fail-closed paths, and point the channel docs at that directory instead of claiming it is missing. --- docs/l3-l2-message-queue.md | 22 +- .../l3/worker_chip_message_queue/README.md | 5 +- .../templates/spsc_queue/README.md | 40 ++++ .../templates/spsc_queue/__init__.py | 8 + .../templates/spsc_queue/_helpers.py | 201 ++++++++++++++++ .../kernels/aiv/kernel_queue_transform.cpp | 62 +++++ .../kernels/orchestration/spsc_queue_orch.cpp | 225 ++++++++++++++++++ .../templates/spsc_queue/test_failure.py | 181 ++++++++++++++ .../spsc_queue/test_l3_single_hop.py | 117 +++++++++ 9 files changed, 853 insertions(+), 8 deletions(-) create mode 100644 tests/st/worker/comm_region/templates/spsc_queue/README.md create mode 100644 tests/st/worker/comm_region/templates/spsc_queue/__init__.py create mode 100644 tests/st/worker/comm_region/templates/spsc_queue/_helpers.py create mode 100644 tests/st/worker/comm_region/templates/spsc_queue/kernels/aiv/kernel_queue_transform.cpp create mode 100644 tests/st/worker/comm_region/templates/spsc_queue/kernels/orchestration/spsc_queue_orch.cpp create mode 100644 tests/st/worker/comm_region/templates/spsc_queue/test_failure.py create mode 100644 tests/st/worker/comm_region/templates/spsc_queue/test_l3_single_hop.py diff --git a/docs/l3-l2-message-queue.md b/docs/l3-l2-message-queue.md index 23b16fef59..9beea9a7a6 100644 --- a/docs/l3-l2-message-queue.md +++ b/docs/l3-l2-message-queue.md @@ -308,9 +308,16 @@ STOP drain, and returns only after `queue.input().drained()`. Application request IDs in that example are payload-header fields `0, 0, 0, 7`; the transport `seq` is not used as a request ID. -That example is a smoke path. It is not a formal Queue Acceptance record, and -it does not demonstrate wrap, `ERROR` opcode delivery, or dedicated -HostVmmCopyAccess/cache instrumentation. +That example is a smoke path. The formal Queue Acceptance record is: + +```text +tests/st/worker/comm_region/templates/spsc_queue/ +``` + +The example does not replace that record. Template ST covers wrap, `ERROR` +delivery, descriptor-full backpressure, STOP-then-output, and logical free. +HostVmmCopyAccess is exercised by queue payload copies on `a2a3` onboard; there +is no extra cache-probe instrumentation. Recorded evidence at commit `c87f0bad`, CI run [33740884796](https://github.com/hw-native-sys/simpler/actions/runs/33740884796): @@ -322,8 +329,11 @@ Recorded evidence at commit `c87f0bad`, CI run | `a2a3` onboard example ST | PASS (`test_worker_chip_message_queue`, 8.1s, device 0, CI `st-onboard-a2a3`) | | `a5` onboard example ST | PASS (`test_worker_chip_message_queue`, 7.0s, device 4, CI `st-onboard-a5`) | | Python / C++ UT | PASS (CI `ut`, `ut-a2a3`, `ut-a5`) | -| Template-level ST under `tests/st/worker/comm_region/templates/queue/` | pending; directory is not in the tree | -| Dedicated wrap / `ERROR` / HostVmmCopyAccess instrumentation | not recorded | +| Template-level ST under `tests/st/worker/comm_region/templates/spsc_queue/` | directory is in the tree | +| `a2a3sim` template ST | PASS (local; 6 cases; `path_has_hdy: False`) | +| `a5sim` template ST | pending | +| `a2a3` onboard template ST | pending | +| `a5` onboard template ST | pending | Simulation evidence does not stand in for hardware cache or HostVMM copy -behavior. Hardware rows above are example ST pass/fail only. +behavior. Example hardware rows above are example ST pass/fail only. diff --git a/examples/workers/l3/worker_chip_message_queue/README.md b/examples/workers/l3/worker_chip_message_queue/README.md index 92f1ee9f0c..84e64802c4 100644 --- a/examples/workers/l3/worker_chip_message_queue/README.md +++ b/examples/workers/l3/worker_chip_message_queue/README.md @@ -48,8 +48,9 @@ Recorded platform evidence at commit `c87f0bad`, CI run [33740884796](https://github.com/hw-native-sys/simpler/actions/runs/33740884796): example ST PASS on `a2a3sim`, `a5sim`, `a2a3` onboard (device 0), and `a5` onboard (device 4). Simulation does not stand in for hardware cache or -HostVMM copy behavior. The evidence table, including pending template-level -ST, is in +HostVMM copy behavior. The formal Queue Acceptance record is +[`tests/st/worker/comm_region/templates/spsc_queue/`](../../../../tests/st/worker/comm_region/templates/spsc_queue/). +Platform evidence for that record is in [`docs/l3-l2-message-queue.md`](../../../../docs/l3-l2-message-queue.md). The test file is also the example — `run_worker_chip_message_queue_example(platform, diff --git a/tests/st/worker/comm_region/templates/spsc_queue/README.md b/tests/st/worker/comm_region/templates/spsc_queue/README.md new file mode 100644 index 0000000000..ccda66ead4 --- /dev/null +++ b/tests/st/worker/comm_region/templates/spsc_queue/README.md @@ -0,0 +1,40 @@ +# SPSC queue Region Template scene tests + +Private L3→L2 ST for the bound SPSC queue. This directory is the formal Queue +Acceptance record. The example under +`examples/workers/l3/worker_chip_message_queue/` remains a smoke path. + +| Test | Topologies | Platforms | +| ---- | ---------- | --------- | +| [`test_l3_single_hop.py`](test_l3_single_hop.py) | L3 → L2 | `a2a3sim`, `a2a3`, `a5sim`, `a5` | +| [`test_failure.py`](test_failure.py) | L3 → L2 construction and poison | `a2a3sim`, `a5sim` | + +Each success case uses `Orchestrator.create_worker_chip_queue`. The factory +binds the SPSC template; it does not call `create_worker_chip_region()`. + +## Contract under test + +- Exact ten-scalar `SPSQ` binding, nonzero `transaction_id`, asymmetric arenas. +- Descriptor-ring exact-depth full before the peer starts, then empty after drain. +- Bidirectional variable-length DATA, zero-byte DATA, arena wrap, and padding + retirement onto the output arena base. +- `ERROR` is a normal message: ERROR, DATA, ERROR stay live. +- One AIV-produced DATA output flushes before `publish`. +- After STOP, further input is rejected locally; one post-STOP DATA still drains. +- `queue.free()` is logical and rejects later enqueue. +- Sim-only faults: positive-timeout empty peek, zero-transaction and wrong-magic + peer construction (run fails without a trusted `transaction=` marker on the + host exception), smashed descriptor (host observes remote abort), and stale + output release (local poison). Peer fatal strings live on the AICPU log and + are not Python exception text. Those cases do not stand in for hardware cache + or HostVmmCopyAccess. + +A2/A3 onboard covers HostVmmCopyAccess through queue payload copies. A5 is +CI-only. Fault cases stay on sim. + +## Run + +```bash +source .venv/bin/activate +python -m pytest tests/st/worker/comm_region/templates/spsc_queue --platform a2a3sim +``` diff --git a/tests/st/worker/comm_region/templates/spsc_queue/__init__.py b/tests/st/worker/comm_region/templates/spsc_queue/__init__.py new file mode 100644 index 0000000000..ad03ca31bf --- /dev/null +++ b/tests/st/worker/comm_region/templates/spsc_queue/__init__.py @@ -0,0 +1,8 @@ +# Copyright (c) PyPTO Contributors. +# This program is free software, you can redistribute it and/or modify it under the terms and conditions of +# CANN Open Software License Agreement Version 2.0 (the "License"). +# Please refer to the License for details. You may not use this file except in compliance with the License. +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, +# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. +# See LICENSE in the root of the software repository for the full text of the License. +# ----------------------------------------------------------------------------------------------------------- diff --git a/tests/st/worker/comm_region/templates/spsc_queue/_helpers.py b/tests/st/worker/comm_region/templates/spsc_queue/_helpers.py new file mode 100644 index 0000000000..4a2e855e5f --- /dev/null +++ b/tests/st/worker/comm_region/templates/spsc_queue/_helpers.py @@ -0,0 +1,201 @@ +# Copyright (c) PyPTO Contributors. +# This program is free software, you can redistribute it and/or modify it under the terms and conditions of +# CANN Open Software License Agreement Version 2.0 (the "License"). +# Please refer to the License for details. You may not use this file except in compliance with the License. +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, +# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. +# See LICENSE in the root of the software repository for the full text of the License. +# ----------------------------------------------------------------------------------------------------------- +"""Private L3 SPSC queue scene-test fixtures.""" + +from __future__ import annotations + +import os +import struct +from typing import Any + +from simpler.comm_region_template import SpscQueueEndpointBinding, _BoundSpscQueue +from simpler.task_interface import ArgDirection as D +from simpler.task_interface import CallConfig, ChipCallable, CoreCallable, TaskArgs +from simpler.worker import Worker, attach_exception_note +from simpler.worker_chip_message_queue import WorkerChipQueue, WorkerChipQueueOpcode + +from simpler_setup.elf_parser import extract_text_section +from simpler_setup.kernel_compiler import KernelCompiler +from simpler_setup.pto_isa import ensure_pto_isa_root + +_RUNTIME = "tensormap_and_ringbuffer" +_HERE = os.path.dirname(os.path.abspath(__file__)) +_ORCH_SRC = os.path.join(_HERE, "kernels", "orchestration", "spsc_queue_orch.cpp") +_AIV_SRC = os.path.join(_HERE, "kernels", "aiv", "kernel_queue_transform.cpp") +_TIMEOUT_S = 5.0 +_QUEUE_DEPTH = 4 +_INPUT_ARENA_BYTES = 256 * 1024 +_OUTPUT_ARENA_BYTES = 512 * 1024 +_CMD_ECHO = 1 +_CMD_ERROR = 2 +_CMD_COMPUTE = 3 +_HEADER = struct.Struct(" ChipCallable: + kc = KernelCompiler(platform=platform) + pto_isa_root = ensure_pto_isa_root() + inc_dirs = kc.get_orchestration_include_dirs(_RUNTIME) + aiv = kc.compile_incore( + _AIV_SRC, + core_type="aiv", + pto_isa_root=pto_isa_root, + extra_include_dirs=inc_dirs, + ) + if not platform.endswith("sim"): + aiv = extract_text_section(aiv) + orch = kc.compile_orchestration( + runtime_name=_RUNTIME, + source_path=_ORCH_SRC, + extra_include_dirs=[str(kc.project_root / "src" / "common")], + ) + try: + child = CoreCallable.build(signature=[D.IN, D.OUT], binary=aiv) + except ValueError as exc: + if "arg_index" not in str(exc): + raise + child = CoreCallable.build(signature=[D.IN, D.OUT], arg_index=[0, 1], binary=aiv) + return ChipCallable.build( + signature=[], + func_name="spsc_queue_orchestration", + binary=orch, + children=[(0, child)], + ) + + +def stream_config() -> CallConfig: + config = CallConfig() + config.aicpu_thread_num = 2 + return config + + +def close_owned_workers(primary: BaseException | None, *workers: Any) -> None: + first_cleanup: BaseException | None = None + for worker in workers: + if worker is None: + continue + try: + worker.close() + except BaseException as cleanup: + if primary is not None: + try: + attach_exception_note(primary, f"{type(cleanup).__name__}: {cleanup}") + except BaseException: + pass + elif first_cleanup is None: + first_cleanup = cleanup + else: + try: + attach_exception_note(first_cleanup, f"{type(cleanup).__name__}: {cleanup}") + except BaseException: + pass + if primary is None and first_cleanup is not None: + raise first_cleanup + + +def make_l3_worker(platform: str, device_id: int) -> tuple[Worker, Any]: + worker = Worker( + level=3, + device_ids=[int(device_id)], + num_sub_workers=0, + platform=platform, + runtime=_RUNTIME, + ) + try: + handle = worker.register(build_chip_callable(platform)) + worker.init() + return worker, handle + except BaseException as primary: + close_owned_workers(primary, worker) + raise + + +def create_queue(orch_handle) -> WorkerChipQueue: + queue = orch_handle.create_worker_chip_queue( + worker_id=0, + depth=_QUEUE_DEPTH, + input_arena_bytes=_INPUT_ARENA_BYTES, + output_arena_bytes=_OUTPUT_ARENA_BYTES, + ) + assert isinstance(queue, WorkerChipQueue) + assert isinstance(queue._bound, _BoundSpscQueue) + assert queue.layout.depth == _QUEUE_DEPTH + assert queue.layout.input_arena_bytes == _INPUT_ARENA_BYTES + assert queue.layout.output_arena_bytes == _OUTPUT_ARENA_BYTES + assert queue.layout.input_arena_bytes != queue.layout.output_arena_bytes + scalars = queue.chip_task_arg_scalars() + assert len(scalars) == 10 + binding = SpscQueueEndpointBinding.from_scalars(scalars) + assert list(binding.to_scalars()) == scalars + assert binding.transaction_id != 0 + return queue + + +def submit_queue(orch_handle, chip_handle, queue: WorkerChipQueue, cfg) -> None: + task_args = TaskArgs() + for scalar in queue.chip_task_arg_scalars(): + task_args.add_scalar(int(scalar)) + orch_handle.submit_next_level(chip_handle, task_args, cfg, worker=0) + + +def enqueue_bytes(queue: WorkerChipQueue, payload: bytes) -> None: + queue.input.enqueue(payload, nbytes=len(payload), timeout=_TIMEOUT_S) + + +def drain_message(queue: WorkerChipQueue) -> tuple[WorkerChipQueueOpcode, bytes, int]: + message = queue.output.peek(timeout=_TIMEOUT_S) + payload = _read_payload(queue, message) + offset = int(message.payload_offset) + opcode = message.opcode + queue.output.release(message) + return opcode, payload, offset + + +def _read_payload(queue: WorkerChipQueue, message) -> bytes: + if message.payload_nbytes == 0: + queue.output.read_into(message, None) + return b"" + buf = bytearray(message.payload_nbytes) + queue.output.read_into(message, buf) + return bytes(buf) + + +def echo_payload(body: bytes) -> bytes: + return _HEADER.pack(_CMD_ECHO) + body + + +def error_payload(body: bytes) -> bytes: + return _HEADER.pack(_CMD_ERROR) + body + + +def compute_payload(base: float) -> bytes: + tile = struct.pack(f"<{_TILE_ELEMS}f", *[(base + float(i % _TILE_COLS)) for i in range(_TILE_ELEMS)]) + return _HEADER.pack(_CMD_COMPUTE) + tile + + +def expected_compute_payload(base: float) -> bytes: + tile = struct.pack( + f"<{_TILE_ELEMS}f", *[(base + float(i % _TILE_COLS) + _COMPUTE_SCALAR) for i in range(_TILE_ELEMS)] + ) + return _HEADER.pack(_CMD_COMPUTE) + tile + + +def wrap_payload(seed: int) -> bytes: + body = bytes((seed + i) & 0xFF for i in range(_WRAP_NBYTES - _HEADER.size)) + return echo_payload(body) diff --git a/tests/st/worker/comm_region/templates/spsc_queue/kernels/aiv/kernel_queue_transform.cpp b/tests/st/worker/comm_region/templates/spsc_queue/kernels/aiv/kernel_queue_transform.cpp new file mode 100644 index 0000000000..9794fa2781 --- /dev/null +++ b/tests/st/worker/comm_region/templates/spsc_queue/kernels/aiv/kernel_queue_transform.cpp @@ -0,0 +1,62 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + * ----------------------------------------------------------------------------------------------------------- + */ + +#include + +#include + +#include "pipe_sync.h" +#include "tensor.h" // NOLINT(build/include_subdir) + +// NOLINTNEXTLINE(build/namespaces) +using namespace pto; + +#ifndef __gm__ +#define __gm__ +#endif + +#ifndef __aicore__ +#define __aicore__ [aicore] // NOLINT(whitespace/braces) +#endif + +extern "C" __aicore__ __attribute__((always_inline)) void kernel_entry(__gm__ int64_t *args) { + __gm__ Tensor *in_tensor = reinterpret_cast<__gm__ Tensor *>(args[0]); + __gm__ Tensor *out_tensor = reinterpret_cast<__gm__ Tensor *>(args[1]); + float scalar = from_u64(static_cast(args[2])); + + __gm__ float *in = reinterpret_cast<__gm__ float *>(in_tensor->buffer.addr) + in_tensor->start_offset; + __gm__ float *out = reinterpret_cast<__gm__ float *>(out_tensor->buffer.addr) + out_tensor->start_offset; + + constexpr int kRows = 128; + constexpr int kCols = 128; + using DynShapeDim5 = pto::Shape<1, 1, 1, kRows, kCols>; + using DynStrideDim5 = pto::Stride<1, 1, 1, kCols, 1>; + using GlobalData = GlobalTensor; + using TileData = Tile; + + TileData in_tile(kRows, kCols); + TileData out_tile(kRows, kCols); + TASSIGN(in_tile, 0x0); + TASSIGN(out_tile, 0x10000); + + GlobalData in_global(in); + GlobalData out_global(out); + + TLOAD(in_tile, in_global); + set_flag(PIPE_MTE2, PIPE_V, EVENT_ID0); + wait_flag(PIPE_MTE2, PIPE_V, EVENT_ID0); + TADDS(out_tile, in_tile, scalar); + set_flag(PIPE_V, PIPE_MTE3, EVENT_ID0); + wait_flag(PIPE_V, PIPE_MTE3, EVENT_ID0); + TSTORE(out_global, out_tile); + + pipe_sync(); +} diff --git a/tests/st/worker/comm_region/templates/spsc_queue/kernels/orchestration/spsc_queue_orch.cpp b/tests/st/worker/comm_region/templates/spsc_queue/kernels/orchestration/spsc_queue_orch.cpp new file mode 100644 index 0000000000..58ad123b33 --- /dev/null +++ b/tests/st/worker/comm_region/templates/spsc_queue/kernels/orchestration/spsc_queue_orch.cpp @@ -0,0 +1,225 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + * ----------------------------------------------------------------------------------------------------------- + */ + +#include +#include +#include + +#include "aicpu/cache_maintenance.h" +#include "aicpu/device_time.h" +#include "aicpu/region_instance_view.h" +#include "common/region_template.h" +#include "orchestration_api.h" // NOLINT(build/include_subdir) + +namespace { + +constexpr int kExpectedArgCount = static_cast(spsc_queue::kSpscQueueEndpointBindingScalarCount); +constexpr uint32_t kComputeFuncId = 0; +constexpr uint64_t kQueueTimeoutNs = 5000000000ULL; +constexpr uint64_t kCmdEcho = 1; +constexpr uint64_t kCmdError = 2; +constexpr uint64_t kCmdCompute = 3; +constexpr uint64_t kHeaderBytes = 8; +constexpr uint32_t kTileRows = 128; +constexpr uint32_t kTileCols = 128; +constexpr uint64_t kTileBytes = static_cast(kTileRows) * kTileCols * sizeof(float); +constexpr uint64_t kComputeBytes = kHeaderBytes + kTileBytes; +constexpr uint64_t kPostStopMarker = 0x11; +constexpr float kComputeScalar = 1.0F; + +using QueueEndpoint = spsc_queue::SpscQueueEndpoint; + +uint64_t spsc_queue_now_ns() { return sys_cnt_ticks_to_ns(device_time_now_ticks(), device_time_frequency_hz()); } + +void report_queue_error(const QueueEndpoint &queue) { + rt_report_fatal(SIMPLER_ERROR_EXPLICIT_ORCH_FATAL, "%s", queue.error().message); +} + +bool has_queue_error(const QueueEndpoint &queue) { return queue.error().kind != spsc_queue::SpscQueueErrorKind::NONE; } + +bool copy_and_publish( + QueueEndpoint &queue, const spsc_queue::SpscQueueInputHandle &input, spsc_queue::SpscQueueOpcode opcode +) { + spsc_queue::SpscQueueOutputReservation output{}; + if (!queue.output().reserve(input.payload_nbytes, kQueueTimeoutNs, output)) { + report_queue_error(queue); + return false; + } + if (input.payload_nbytes != 0) { + memcpy( + reinterpret_cast(static_cast(output.payload.local_addr)), + reinterpret_cast(static_cast(input.payload.local_addr)), input.payload_nbytes + ); + cache_flush_range( + reinterpret_cast(static_cast(output.payload.local_addr)), input.payload_nbytes + ); + } + if (!queue.output().publish(output, opcode)) { + report_queue_error(queue); + return false; + } + return true; +} + +bool publish_compute(QueueEndpoint &queue, const spsc_queue::SpscQueueInputHandle &input) { + if (input.payload_nbytes != kComputeBytes) { + rt_report_fatal(SIMPLER_ERROR_EXPLICIT_ORCH_FATAL, "spsc queue ST compute payload size mismatch"); + return false; + } + spsc_queue::SpscQueueOutputReservation output{}; + if (!queue.output().reserve(kComputeBytes, kQueueTimeoutNs, output)) { + report_queue_error(queue); + return false; + } + + uint8_t *dst = reinterpret_cast(static_cast(output.payload.local_addr)); + memcpy(dst, reinterpret_cast(static_cast(input.payload.local_addr)), kHeaderBytes); + + uint32_t shape[2] = {kTileRows, kTileCols}; + void *src_tile = reinterpret_cast(static_cast(input.payload.local_addr + kHeaderBytes)); + void *dst_tile = dst + kHeaderBytes; + simpler::tmr::Tensor in_tensor = simpler::tmr::make_tensor_external(src_tile, shape, 2, DataType::FLOAT32); + simpler::tmr::Tensor out_tensor = simpler::tmr::make_tensor_external(dst_tile, shape, 2, DataType::FLOAT32); + + CoreTaskArgs params; + params.add_input(in_tensor); + params.add_output(out_tensor); + params.add_scalar(to_u64(kComputeScalar)); + rt_submit_aiv_task(kComputeFuncId, params); + + uint32_t first_index[2] = {0, 0}; + (void)get_tensor_data(out_tensor, 2, first_index); + cache_flush_range(dst, kComputeBytes); + + if (!queue.output().publish(output, spsc_queue::SpscQueueOpcode::DATA)) { + report_queue_error(queue); + return false; + } + return true; +} + +bool publish_post_stop(QueueEndpoint &queue) { + spsc_queue::SpscQueueOutputReservation output{}; + if (!queue.output().reserve(kHeaderBytes, kQueueTimeoutNs, output)) { + report_queue_error(queue); + return false; + } + uint64_t marker = kPostStopMarker; + memcpy(reinterpret_cast(static_cast(output.payload.local_addr)), &marker, kHeaderBytes); + cache_flush_range(reinterpret_cast(static_cast(output.payload.local_addr)), kHeaderBytes); + if (!queue.output().publish(output, spsc_queue::SpscQueueOpcode::DATA)) { + report_queue_error(queue); + return false; + } + return true; +} + +bool handle_data(QueueEndpoint &queue, const spsc_queue::SpscQueueInputHandle &input) { + if (input.payload_nbytes == 0) { + return copy_and_publish(queue, input, spsc_queue::SpscQueueOpcode::DATA); + } + if (input.payload_nbytes < kHeaderBytes) { + rt_report_fatal(SIMPLER_ERROR_EXPLICIT_ORCH_FATAL, "spsc queue ST data payload is shorter than the command"); + return false; + } + uint64_t command = 0; + memcpy(&command, reinterpret_cast(static_cast(input.payload.local_addr)), kHeaderBytes); + if (command == kCmdError) { + return copy_and_publish(queue, input, spsc_queue::SpscQueueOpcode::ERROR); + } + if (command == kCmdCompute) { + return publish_compute(queue, input); + } + if (command != kCmdEcho) { + rt_report_fatal( + SIMPLER_ERROR_EXPLICIT_ORCH_FATAL, "spsc queue ST unexpected command=%llu", + static_cast(command) + ); + return false; + } + return copy_and_publish(queue, input, spsc_queue::SpscQueueOpcode::DATA); +} + +} // namespace + +extern "C" { + +__attribute__((visibility("default"))) OrchestrationConfig aicpu_orchestration_config(const ChipTaskArgs &orch_args) { + (void)orch_args; // NOLINT(readability/casting) + return OrchestrationConfig{.expected_arg_count = kExpectedArgCount}; +} + +__attribute__((visibility("default"))) void spsc_queue_orchestration(const ChipTaskArgs &orch_args) { + uint64_t scalars[spsc_queue::kSpscQueueEndpointBindingScalarCount]; + for (size_t i = 0; i < spsc_queue::kSpscQueueEndpointBindingScalarCount; ++i) { + scalars[i] = orch_args.scalar(static_cast(i)); + } + spsc_queue::SpscQueueEndpointBinding binding{}; + if (!spsc_queue::decode_endpoint_binding(scalars, spsc_queue::kSpscQueueEndpointBindingScalarCount, &binding)) { + rt_report_fatal(SIMPLER_ERROR_EXPLICIT_ORCH_FATAL, "invalid queue binding"); + return; + } + RegionInstanceView view( + RegionPartLocalSpan{binding.payload_base, binding.payload_bytes}, + RegionPartLocalSpan{binding.counter_base, binding.counter_bytes} + ); + spsc_queue::MonotonicClock clock{&spsc_queue_now_ns}; + QueueEndpoint queue(binding, std::move(view), clock); + if (!queue.live()) { + report_queue_error(queue); + return; + } + + for (;;) { + spsc_queue::SpscQueueInputHandle input{}; + if (!queue.input().peek(kQueueTimeoutNs, input)) { + if (has_queue_error(queue)) { + report_queue_error(queue); + return; + } + continue; + } + if (input.opcode == spsc_queue::SpscQueueOpcode::STOP) { + if (!publish_post_stop(queue)) { + return; + } + if (!queue.input().release(input)) { + report_queue_error(queue); + return; + } + if (!queue.input().drained()) { + rt_report_fatal(SIMPLER_ERROR_EXPLICIT_ORCH_FATAL, "spsc queue ST returned before input drain"); + } + return; + } + if (input.opcode == spsc_queue::SpscQueueOpcode::ERROR) { + if (!copy_and_publish(queue, input, spsc_queue::SpscQueueOpcode::ERROR)) { + return; + } + } else if (input.opcode == spsc_queue::SpscQueueOpcode::DATA) { + if (!handle_data(queue, input)) { + return; + } + } else { + rt_report_fatal( + SIMPLER_ERROR_EXPLICIT_ORCH_FATAL, "spsc queue ST unexpected input opcode=%llu", + static_cast(input.opcode) + ); + return; + } + if (!queue.input().release(input)) { + report_queue_error(queue); + return; + } + } +} + +} // extern "C" diff --git a/tests/st/worker/comm_region/templates/spsc_queue/test_failure.py b/tests/st/worker/comm_region/templates/spsc_queue/test_failure.py new file mode 100644 index 0000000000..e7cd0adb48 --- /dev/null +++ b/tests/st/worker/comm_region/templates/spsc_queue/test_failure.py @@ -0,0 +1,181 @@ +#!/usr/bin/env python3 +# Copyright (c) PyPTO Contributors. +# This program is free software, you can redistribute it and/or modify it under the terms and conditions of +# CANN Open Software License Agreement Version 2.0 (the "License"). +# Please refer to the License for details. You may not use this file except in compliance with the License. +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, +# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. +# See LICENSE in the root of the software repository for the full text of the License. +# ----------------------------------------------------------------------------------------------------------- +"""Scene-layer SPSC queue construction and poison failures. Sim-only.""" + +from __future__ import annotations + +import struct + +import pytest +from simpler.task_interface import TaskArgs + +from ._helpers import ( + _SMALL_ECHO, + _TIMEOUT_S, + FAULT_PLATFORMS, + close_owned_workers, + create_queue, + echo_payload, + enqueue_bytes, + make_l3_worker, + stream_config, + submit_queue, +) + + +def _exception_text(exc: BaseException) -> str: + parts = [str(exc)] + current: BaseException | None = exc + seen: set[int] = set() + while current is not None and id(current) not in seen: + seen.add(id(current)) + parts.append(str(current)) + current = current.__cause__ if current.__cause__ is not None else current.__context__ + return "\n".join(parts) + + +@pytest.mark.platforms(FAULT_PLATFORMS) +@pytest.mark.device_count(1) +@pytest.mark.runtime("tensormap_and_ringbuffer") +def test_timeout_is_nonterminal(st_platform, st_device_ids): + worker, _chip_handle = make_l3_worker(st_platform, int(st_device_ids[0])) + primary = None + try: + + def orch(orch_handle, _args, cfg): + queue = create_queue(orch_handle) + with pytest.raises(TimeoutError, match="timed out"): + queue.output.peek(timeout=0.2) + assert queue.input.try_enqueue(_SMALL_ECHO, len(_SMALL_ECHO)) is True + queue.free() + + worker.run(orch, args=None, config=stream_config()) + except BaseException as exc: + primary = exc + raise + finally: + close_owned_workers(primary, worker) + + +@pytest.mark.platforms(FAULT_PLATFORMS) +@pytest.mark.device_count(1) +@pytest.mark.runtime("tensormap_and_ringbuffer") +def test_peer_rejects_zero_transaction(st_platform, st_device_ids): + worker, chip_handle = make_l3_worker(st_platform, int(st_device_ids[0])) + primary = None + try: + + def orch(orch_handle, _args, cfg): + queue = create_queue(orch_handle) + scalars = list(queue.chip_task_arg_scalars()) + scalars[2] = 0 + task_args = TaskArgs() + for value in scalars: + task_args.add_scalar(int(value)) + orch_handle.submit_next_level(chip_handle, task_args, cfg, worker=0) + + with pytest.raises(BaseException) as excinfo: # noqa: PT011 + worker.run(orch, args=None, config=stream_config()) + text = _exception_text(excinfo.value) + assert "transaction=" not in text + except BaseException as exc: + primary = exc + raise + finally: + close_owned_workers(primary, worker) + + +@pytest.mark.platforms(FAULT_PLATFORMS) +@pytest.mark.device_count(1) +@pytest.mark.runtime("tensormap_and_ringbuffer") +def test_peer_rejects_wrong_magic(st_platform, st_device_ids): + worker, chip_handle = make_l3_worker(st_platform, int(st_device_ids[0])) + primary = None + try: + + def orch(orch_handle, _args, cfg): + queue = create_queue(orch_handle) + scalars = list(queue.chip_task_arg_scalars()) + scalars[0] = 0x4C33513200010001 + task_args = TaskArgs() + for value in scalars: + task_args.add_scalar(int(value)) + orch_handle.submit_next_level(chip_handle, task_args, cfg, worker=0) + + with pytest.raises(BaseException) as excinfo: # noqa: PT011 + worker.run(orch, args=None, config=stream_config()) + text = _exception_text(excinfo.value) + assert "transaction=" not in text + except BaseException as exc: + primary = exc + raise + finally: + close_owned_workers(primary, worker) + + +@pytest.mark.platforms(FAULT_PLATFORMS) +@pytest.mark.device_count(1) +@pytest.mark.runtime("tensormap_and_ringbuffer") +def test_malformed_descriptor_poisons_with_identity(st_platform, st_device_ids): + worker, chip_handle = make_l3_worker(st_platform, int(st_device_ids[0])) + primary = None + try: + + def orch(orch_handle, _args, cfg): + queue = create_queue(orch_handle) + enqueue_bytes(queue, echo_payload(b"bad-seq")) + smashed = bytearray(8) + struct.pack_into(" wrap_offsets[0] + assert wrap_offsets[2] == layout.output_arena_offset + + queue.input.enqueue(None, nbytes=0, timeout=_TIMEOUT_S) + opcode, payload, offset = drain_message(queue) + assert opcode is WorkerChipQueueOpcode.DATA + assert payload == b"" + assert offset == 0 + + err1 = error_payload(b"e1") + echo = echo_payload(b"ok") + err2 = error_payload(b"e2") + enqueue_bytes(queue, err1) + opcode, payload, _offset = drain_message(queue) + assert opcode is WorkerChipQueueOpcode.ERROR + assert payload == err1 + enqueue_bytes(queue, echo) + opcode, payload, _offset = drain_message(queue) + assert opcode is WorkerChipQueueOpcode.DATA + assert payload == echo + enqueue_bytes(queue, err2) + opcode, payload, _offset = drain_message(queue) + assert opcode is WorkerChipQueueOpcode.ERROR + assert payload == err2 + + compute = compute_payload(3.0) + enqueue_bytes(queue, compute) + opcode, payload, _offset = drain_message(queue) + assert opcode is WorkerChipQueueOpcode.DATA + assert payload == expected_compute_payload(3.0) + + queue.request_stop(timeout=_TIMEOUT_S) + assert queue.input.try_enqueue(_SMALL_ECHO, len(_SMALL_ECHO)) is False + opcode, payload, _offset = drain_message(queue) + assert opcode is WorkerChipQueueOpcode.DATA + assert payload == _POST_STOP_MARKER + assert queue.output.try_peek() is None + + queue.free() + with pytest.raises(RuntimeError): + queue.input.try_enqueue(_SMALL_ECHO, len(_SMALL_ECHO)) + + worker.run(orch, args=None, config=stream_config()) + except BaseException as exc: + primary = exc + raise + finally: + close_owned_workers(primary, worker) From 2a7697df21ba9b433096b1f24b1aa2f82cca89fa Mon Sep 17 00:00:00 2001 From: ccyywwen <75376396+ccyywwen@users.noreply.github.com> Date: Fri, 4 Sep 2026 11:59:23 +0800 Subject: [PATCH 7/7] Fix: keep SPSC queue docs user-focused - Correct the L2 input lane to accept DATA and STOP only - Remove development evidence and milestone wording from user docs - Keep scene-test documentation focused on test scope --- docs/l3-l2-message-queue.md | 46 ++++--------------- .../l3/worker_chip_message_queue/README.md | 17 ++----- .../templates/spsc_queue/README.md | 8 ++-- 3 files changed, 16 insertions(+), 55 deletions(-) diff --git a/docs/l3-l2-message-queue.md b/docs/l3-l2-message-queue.md index 9beea9a7a6..3727d6c5fd 100644 --- a/docs/l3-l2-message-queue.md +++ b/docs/l3-l2-message-queue.md @@ -125,7 +125,7 @@ if (!queue.live()) { } ``` -The default endpoint allows one active L2 DATA/ERROR input handle at a time. +The default endpoint allows one active L2 DATA input handle at a time. L2 can opt into a larger input window with a compile-time parameter: ```cpp @@ -137,7 +137,7 @@ creation and does not change the shared layout or the ten-scalar binding. The valid range is `1 <= MaxInflight <= depth`. Invalid combinations report `BAD_ARGUMENT` without setting the L2 abort flag. STOP does not count against `MaxInflight`; the endpoint keeps one extra slot so a STOP handle can remain -pending behind earlier DATA/ERROR handles. +pending behind earlier DATA handles. ## 2. Layout And Descriptor @@ -250,10 +250,10 @@ On L3 output, `peek()` returns a handle that remains active until `release(handle)`. While a handle is active, repeated `try_peek()` returns the same handle. Releasing the wrong handle poisons the queue. -On L2 input, the default endpoint keeps one active DATA/ERROR handle. L2 must +On L2 input, the default endpoint keeps one active DATA handle. L2 must not call `peek()` again before releasing that handle, except that STOP may also be acquired into the extra STOP slot. With `SpscQueueEndpoint`, L2 -may hold up to `N` active DATA or ERROR inputs. `release(handle)` is logical +may hold up to `N` active DATA inputs. `release(handle)` is logical completion; the queue physically advances the shared input head only for the completed FIFO prefix. If input 2 is released before input 1, input 2 remains physically owned until input 1 is also released. @@ -269,9 +269,9 @@ dequeue outputs that L2 publishes before returning. `request_stop(timeout)` waits only until the `STOP` descriptor is published; it does not wait for L2 exit and does not drain outputs. -With an input window, STOP may be acquired while earlier DATA or ERROR inputs +With an input window, STOP may be acquired while earlier DATA inputs are still active. After STOP is acquired, the input queue enters draining mode -and does not acquire later DATA or ERROR descriptors. Earlier active inputs +and does not acquire later DATA descriptors. Earlier active inputs may still produce outputs. STOP is physically released only after all earlier active inputs are physically released. `queue.input().drained()` returns true only after STOP has been physically released. If L2 observes a published input @@ -290,9 +290,9 @@ ownership. A local poison sets the local abort flag for the peer. Observing the peer abort flag reports remote abort but does not set the local flag. After poison, normal queue operations reject. Cleanup remains valid. -## 5. Example And Platform Evidence +## 5. Example -The shipped smoke example lives at: +The shipped example lives at: ```text examples/workers/l3/worker_chip_message_queue/ @@ -307,33 +307,3 @@ multiple outputs for one input, combines two inputs into one output during STOP drain, and returns only after `queue.input().drained()`. Application request IDs in that example are payload-header fields `0, 0, 0, 7`; the transport `seq` is not used as a request ID. - -That example is a smoke path. The formal Queue Acceptance record is: - -```text -tests/st/worker/comm_region/templates/spsc_queue/ -``` - -The example does not replace that record. Template ST covers wrap, `ERROR` -delivery, descriptor-full backpressure, STOP-then-output, and logical free. -HostVmmCopyAccess is exercised by queue payload copies on `a2a3` onboard; there -is no extra cache-probe instrumentation. - -Recorded evidence at commit `c87f0bad`, CI run -[33740884796](https://github.com/hw-native-sys/simpler/actions/runs/33740884796): - -| Surface | Result | -| ------- | ------ | -| `a2a3sim` example ST | PASS (CI `st-sim-a2a3`; also reproduced locally) | -| `a5sim` example ST | PASS (CI `st-sim-a5`) | -| `a2a3` onboard example ST | PASS (`test_worker_chip_message_queue`, 8.1s, device 0, CI `st-onboard-a2a3`) | -| `a5` onboard example ST | PASS (`test_worker_chip_message_queue`, 7.0s, device 4, CI `st-onboard-a5`) | -| Python / C++ UT | PASS (CI `ut`, `ut-a2a3`, `ut-a5`) | -| Template-level ST under `tests/st/worker/comm_region/templates/spsc_queue/` | directory is in the tree | -| `a2a3sim` template ST | PASS (local; 6 cases; `path_has_hdy: False`) | -| `a5sim` template ST | pending | -| `a2a3` onboard template ST | pending | -| `a5` onboard template ST | pending | - -Simulation evidence does not stand in for hardware cache or HostVMM copy -behavior. Example hardware rows above are example ST pass/fail only. diff --git a/examples/workers/l3/worker_chip_message_queue/README.md b/examples/workers/l3/worker_chip_message_queue/README.md index 84e64802c4..b4726a5e13 100644 --- a/examples/workers/l3/worker_chip_message_queue/README.md +++ b/examples/workers/l3/worker_chip_message_queue/README.md @@ -7,8 +7,7 @@ appear — a serving loop, not a batch. The transport is the L3-L2 SPSC queue (`SPSQ` binding): two arenas (input and output) plus a 10-scalar endpoint binding the L2 orchestration receives as -plain `TaskArgs` scalars starting at offset 0. This example is a smoke path, -not the formal Queue Acceptance record. See +plain `TaskArgs` scalars starting at offset 0. See [`docs/l3-l2-message-queue.md`](../../../../docs/l3-l2-message-queue.md) for the channel's current design. @@ -38,21 +37,15 @@ request 1's two. That is the point — a queue, not a call stack. ## Run -Single device. Local simulation: +Single device: ```bash pytest examples/workers/l3/worker_chip_message_queue --platform a2a3sim +pytest examples/workers/l3/worker_chip_message_queue --platform a5sim +pytest examples/workers/l3/worker_chip_message_queue --platform a2a3 --device +pytest examples/workers/l3/worker_chip_message_queue --platform a5 --device ``` -Recorded platform evidence at commit `c87f0bad`, CI run -[33740884796](https://github.com/hw-native-sys/simpler/actions/runs/33740884796): -example ST PASS on `a2a3sim`, `a5sim`, `a2a3` onboard (device 0), and `a5` -onboard (device 4). Simulation does not stand in for hardware cache or -HostVMM copy behavior. The formal Queue Acceptance record is -[`tests/st/worker/comm_region/templates/spsc_queue/`](../../../../tests/st/worker/comm_region/templates/spsc_queue/). -Platform evidence for that record is in -[`docs/l3-l2-message-queue.md`](../../../../docs/l3-l2-message-queue.md). - The test file is also the example — `run_worker_chip_message_queue_example(platform, device_id)` is importable directly. diff --git a/tests/st/worker/comm_region/templates/spsc_queue/README.md b/tests/st/worker/comm_region/templates/spsc_queue/README.md index ccda66ead4..443b09ac00 100644 --- a/tests/st/worker/comm_region/templates/spsc_queue/README.md +++ b/tests/st/worker/comm_region/templates/spsc_queue/README.md @@ -1,8 +1,6 @@ # SPSC queue Region Template scene tests -Private L3→L2 ST for the bound SPSC queue. This directory is the formal Queue -Acceptance record. The example under -`examples/workers/l3/worker_chip_message_queue/` remains a smoke path. +L3→L2 scene tests for the bound SPSC queue. | Test | Topologies | Platforms | | ---- | ---------- | --------- | @@ -29,8 +27,8 @@ binds the SPSC template; it does not call `create_worker_chip_region()`. are not Python exception text. Those cases do not stand in for hardware cache or HostVmmCopyAccess. -A2/A3 onboard covers HostVmmCopyAccess through queue payload copies. A5 is -CI-only. Fault cases stay on sim. +Success cases run on simulation and onboard platforms. Fault cases stay on +simulation. ## Run