From 3569bf7b067982fbccb64b67011fa7ec774bae51 Mon Sep 17 00:00:00 2001 From: ThanKarab Date: Fri, 13 Nov 2026 13:50:24 +0200 Subject: [PATCH 1/6] feat(aggregation): isolate execution state by context id --- .../base_aggregation_client.py | 3 + .../controller_aggregation_client.py | 10 +- .../exareme3_udf_aggregation_client.py | 4 +- exaflow/aggregation_server/server.py | 159 +++++++++++------- .../algorithm_flow_engine_interface.py | 1 + .../services/exareme3/strategies.py | 1 + .../controller/services/strategy_interface.py | 4 +- exaflow/protos/README.md | 10 ++ .../aggregation_server.proto | 4 + .../aggregation_server_pb2.py | 44 ++--- exaflow/worker/exareme3/udf/udf_service.py | 3 + exaflow/worker_communication.py | 1 + .../test_aggregation_clients.py | 26 +-- .../test_aggregation_ownership.py | 12 +- .../test_aggregation_server_grpc.py | 115 ++++++++++++- .../test_udf_aggregation_client_deadline.py | 7 +- .../test_aggregation_cleanup_on_cancel.py | 8 +- .../test_exareme3_aggregation_strategy.py | 25 ++- .../controller/services/test_run_udf_call.py | 1 + .../services/test_shutdown_budget.py | 8 +- .../test_blocking_calls_off_loop.py | 4 +- .../exareme3/test_udf_min_rows_enforcement.py | 2 + .../exareme3/test_udf_service.py | 13 +- .../test_lazy_aggregation_algorithms.py | 3 +- .../test_worker_grpc_message_size.py | 8 +- 25 files changed, 350 insertions(+), 126 deletions(-) diff --git a/exaflow/aggregation_clients/base_aggregation_client.py b/exaflow/aggregation_clients/base_aggregation_client.py index ccff4a9e4..a5f661840 100644 --- a/exaflow/aggregation_clients/base_aggregation_client.py +++ b/exaflow/aggregation_clients/base_aggregation_client.py @@ -28,11 +28,13 @@ class BaseAggregationClient: def __init__( self, request_id: str, + context_id: str, aggregator_dns: str | None = None, max_message_mb: int | None = None, timeout: float | None = None, ): self._request_id = request_id + self._context_id = context_id self._step_counter = 1 self._timeout = timeout @@ -82,6 +84,7 @@ def _aggregate_batch_request( ) req = pb2.AggregateRequest( request_id=self._request_id, + context_id=self._context_id, step=step or self._next_step(), operations=operations, ) diff --git a/exaflow/aggregation_clients/controller_aggregation_client.py b/exaflow/aggregation_clients/controller_aggregation_client.py index 2c5672036..602ede3e0 100644 --- a/exaflow/aggregation_clients/controller_aggregation_client.py +++ b/exaflow/aggregation_clients/controller_aggregation_client.py @@ -21,11 +21,13 @@ class ControllerAggregationClient(BaseAggregationClient): def __init__( self, request_id: str, + context_id: str, aggregator_dns: str | None = None, max_message_mb: int | None = None, ): super().__init__( request_id, + context_id, aggregator_dns=aggregator_dns, max_message_mb=max_message_mb, ) @@ -37,7 +39,9 @@ def _ownership_metadata(self): def configure(self, num_workers: int, timeout: float) -> str: future = self._stub.Configure.future( pb2.ConfigureRequest( - request_id=self._request_id, num_of_workers=num_workers + request_id=self._request_id, + context_id=self._context_id, + num_of_workers=num_workers, ), timeout=timeout, metadata=self._ownership_metadata(), @@ -52,7 +56,9 @@ def configure(self, num_workers: int, timeout: float) -> str: def cleanup(self, timeout: float) -> str: try: response = self._stub.Cleanup( - pb2.CleanupRequest(request_id=self._request_id), + pb2.CleanupRequest( + request_id=self._request_id, context_id=self._context_id + ), timeout=timeout, metadata=self._ownership_metadata(), ) diff --git a/exaflow/aggregation_clients/exareme3_udf_aggregation_client.py b/exaflow/aggregation_clients/exareme3_udf_aggregation_client.py index 0fe4c5ca7..2e1e821c9 100644 --- a/exaflow/aggregation_clients/exareme3_udf_aggregation_client.py +++ b/exaflow/aggregation_clients/exareme3_udf_aggregation_client.py @@ -45,7 +45,9 @@ def _aggregate_batch( def unregister(self) -> tuple[str, int]: response = self._stub.Unregister( - pb2.UnregisterRequest(request_id=self._request_id), + pb2.UnregisterRequest( + request_id=self._request_id, context_id=self._context_id + ), timeout=self._timeout, ) return response.status, response.remaining_workers diff --git a/exaflow/aggregation_server/server.py b/exaflow/aggregation_server/server.py index 2c20e4c4a..b2a0d47a2 100644 --- a/exaflow/aggregation_server/server.py +++ b/exaflow/aggregation_server/server.py @@ -2,12 +2,12 @@ Lifecycle of a request ---------------------- -- The controller calls `Configure(request_id, num_of_workers)` before running - the algorithm and `Cleanup(request_id)` when the request ends, in a `finally` +- The controller calls `Configure(request_id, context_id, num_of_workers)` before running + the algorithm and `Cleanup(request_id, context_id)` when the request ends, in a `finally` block. Both calls carry an ownership token in gRPC metadata. Cleanup deletes the context only when the token matches the one stored at Configure, so a duplicate request that lost its response cannot remove someone else's context. -- Each worker calls `Aggregate(request_id, step, operations)` from its UDF. +- Each worker calls `Aggregate(request_id, context_id, step, operations)` from its UDF. `step` is numbered by the worker's client (1, 2, 3, ...). `operations` holds one or more `(aggregation_type, tensor)` pairs. Numeric types (`SUM`, `MIN`, `MAX`) send Arrow tensors; `UNION` sends JSON-encoded values. @@ -16,7 +16,7 @@ Synchronization --------------- -Each request has an `AggregationContext` guarded by one `Condition`. It keeps +Each execution attempt has an `AggregationContext` guarded by one `Condition`. It keeps one slot per step number that is still in progress: - A worker's `Aggregate` call adds its inputs to the slot for its step, creating the slot if it is the first to arrive. @@ -103,7 +103,7 @@ # Cleanup is remembered even when it arrives before Configure. The set is # capped so a stream of unknown ids cannot grow it without bound; the TTL # still drops entries that age out first. -CANCELLED_REQUEST_ID_LIMIT = 1024 +CANCELLED_CONTEXT_ID_LIMIT = 1024 def _result_dtype( @@ -124,7 +124,9 @@ def _result_dtype( def _decode_operations(request) -> Tuple[List[AggregationType], List[np.ndarray]]: if not request.operations: - raise ValueError(f"request_id='{request.request_id}' sent no operations") + raise ValueError( + f"request_id='{request.request_id}' context_id='{request.context_id}' sent no operations" + ) aggregation_types = [] payloads = [] for op in request.operations: @@ -136,7 +138,7 @@ def _decode_operations(request) -> Tuple[List[AggregationType], List[np.ndarray] ) from None if not op.tensor: raise ValueError( - f"request_id='{request.request_id}' missing tensor payload" + f"request_id='{request.request_id}' context_id='{request.context_id}' missing tensor payload" ) if aggregation_type == AggregationType.UNION: payload = bytes_to_values(op.tensor) @@ -199,8 +201,10 @@ def __init__( request_id: str, expected_workers: int, ownership_token: str = "", + context_id: str = "", ): self.request_id = request_id + self.context_id = context_id self.expected_workers = expected_workers self.ownership_token = ownership_token self.steps: Dict[int, _Step] = {} @@ -216,7 +220,7 @@ def fail(self, exc: Exception, *, log: bool = True) -> None: self.error = exc if log: logger.error( - f"[AGGREGATE] request_id='{self.request_id}' failed: {exc}", + f"[AGGREGATE] request_id='{self.request_id}' context_id='{self.context_id}' failed: {exc}", exc_info=exc, ) self.condition.notify_all() @@ -240,7 +244,7 @@ def aggregate( self._raise_error() if step_no <= self.last_completed_step: raise ValueError( - f"Step {step_no} already completed for request_id='{self.request_id}'" + f"Step {step_no} already completed for request_id='{self.request_id}' context_id='{self.context_id}'" ) step = self.steps.setdefault( step_no, _Step(aggregation_types, [[] for _ in aggregation_types]) @@ -263,7 +267,7 @@ def aggregate( self._raise_error() if not done: raise TimeoutError( - f"Timeout waiting for aggregation result for request_id='{self.request_id}' " + f"Timeout waiting for aggregation result for request_id='{self.request_id}' context_id='{self.context_id}' " f"step {step_no} (received {len(step.inputs[0])}/{self.expected_workers})" ) step.reads += 1 @@ -300,7 +304,7 @@ def _add_inputs( ) -> None: if aggregation_types != step.aggregation_types: raise ValueError( - f"Batch aggregation_type mismatch for request_id='{self.request_id}' " + f"Batch aggregation_type mismatch for request_id='{self.request_id}' context_id='{self.context_id}' " f"step {step_no} (expected {step.aggregation_types}, " f"got {aggregation_types})" ) @@ -323,27 +327,41 @@ def _add_inputs( class AggregationServer(AggregationServerServicer): def __init__(self): self.aggregation_contexts: Dict[str, AggregationContext] = {} - # request_id -> monotonic time of Cleanup (or stale eviction). Lets late + # context_id -> monotonic time of Cleanup (or stale eviction). Lets late # Aggregate/Unregister calls get ABORTED instead of looking unconfigured. - self.cleaned_up_request_ids: Dict[str, float] = {} + self.cleaned_up_context_ids: Dict[str, float] = {} self.global_lock = threading.Lock() - def _get_aggregation_context(self, request_id, context) -> AggregationContext: - agg_ctx = self.aggregation_contexts.get(request_id) + @staticmethod + def _validate_context_request(agg_ctx, request_id, context): + if agg_ctx.request_id != request_id: + context.abort( + grpc.StatusCode.INVALID_ARGUMENT, + "request_id does not match the configured context_id", + ) + return agg_ctx + + def _get_aggregation_context( + self, context_id, context, request_id + ) -> AggregationContext: + if not context_id or not request_id: + context.abort( + grpc.StatusCode.INVALID_ARGUMENT, + "request_id and context_id are required", + ) + agg_ctx = self.aggregation_contexts.get(context_id) if agg_ctx: - return agg_ctx + return self._validate_context_request(agg_ctx, request_id, context) with self.global_lock: - agg_ctx = self.aggregation_contexts.get(request_id) + agg_ctx = self.aggregation_contexts.get(context_id) if agg_ctx: - return agg_ctx - if request_id in self.cleaned_up_request_ids: - msg = ( - f"Aggregation for request_id='{request_id}' was cancelled " - "because the controller cleaned up the request." - ) - logger.error(f"[AGGREGATE] {msg}") + return self._validate_context_request(agg_ctx, request_id, context) + label = f"request_id='{request_id}' context_id='{context_id}'" + if context_id in self.cleaned_up_context_ids: + msg = f"Aggregation for {label} was cancelled because the controller cleaned up the request." + logger.error("[AGGREGATE] %s", msg) context.abort(grpc.StatusCode.ABORTED, msg) - msg = f"request_id='{request_id}' is not configured." + msg = f"{label} is not configured." logger.error(msg) context.abort(grpc.StatusCode.INVALID_ARGUMENT, msg) return agg_ctx @@ -363,87 +381,97 @@ def _reduce( def _evict_stale_contexts(self) -> None: cutoff = time.monotonic() - STALE_CONTEXT_TTL_SECONDS - stale_request_ids = [ - request_id - for request_id, agg_ctx in self.aggregation_contexts.items() + stale_context_ids = [ + context_id + for context_id, agg_ctx in self.aggregation_contexts.items() if agg_ctx.last_activity < cutoff ] - for request_id in stale_request_ids: - agg_ctx = self.aggregation_contexts.pop(request_id) - self.cleaned_up_request_ids[request_id] = time.monotonic() + for context_id in stale_context_ids: + agg_ctx = self.aggregation_contexts.pop(context_id) + self.cleaned_up_context_ids[context_id] = time.monotonic() agg_ctx.fail( AggregationCancelledError( - f"Aggregation for request_id='{request_id}' was cancelled " + f"Aggregation for request_id='{agg_ctx.request_id}' context_id='{context_id}' was cancelled " "because its context was evicted as stale." ), log=False, ) logger.warning( - f"[CONFIGURE] Evicted stale AggregationContext for request_id='{request_id}' " + f"[CONFIGURE] Evicted stale AggregationContext for request_id='{agg_ctx.request_id}' context_id='{context_id}' " f"(no activity for {STALE_CONTEXT_TTL_SECONDS}s and no Cleanup received)" ) stale_cleaned = [ - request_id - for request_id, cleaned_at in self.cleaned_up_request_ids.items() + context_id + for context_id, cleaned_at in self.cleaned_up_context_ids.items() if cleaned_at < cutoff ] - for request_id in stale_cleaned: - del self.cleaned_up_request_ids[request_id] + for context_id in stale_cleaned: + del self.cleaned_up_context_ids[context_id] - def _remember_cancelled(self, request_id: str) -> None: + def _remember_cancelled(self, context_id: str) -> None: """Record a Cleanup, including one that arrived before Configure.""" - self.cleaned_up_request_ids[request_id] = time.monotonic() + self.cleaned_up_context_ids[context_id] = time.monotonic() self._evict_stale_contexts() - overflow = len(self.cleaned_up_request_ids) - CANCELLED_REQUEST_ID_LIMIT + overflow = len(self.cleaned_up_context_ids) - CANCELLED_CONTEXT_ID_LIMIT if overflow <= 0: return oldest = sorted( - self.cleaned_up_request_ids, - key=self.cleaned_up_request_ids.__getitem__, + self.cleaned_up_context_ids, + key=self.cleaned_up_context_ids.__getitem__, ) for stale_id in oldest[:overflow]: - del self.cleaned_up_request_ids[stale_id] + del self.cleaned_up_context_ids[stale_id] def Configure(self, request, context): + if not request.context_id or not request.request_id: + context.abort( + grpc.StatusCode.INVALID_ARGUMENT, + "request_id and context_id are required", + ) with self.global_lock: self._evict_stale_contexts() - if request.request_id in self.aggregation_contexts: + current = self.aggregation_contexts.get(request.context_id) + if current is not None: + self._validate_context_request(current, request.request_id, context) logger.warning( - f"[CONFIGURE] Request context already exists for request_id='{request.request_id}'" + f"[CONFIGURE] Request context already exists for request_id='{request.request_id}' context_id='{request.context_id}'" ) return ConfigureResponse( status=Status.ALREADY_CONFIGURED, - status_message="Already configured for this request_id", + status_message="Already configured for this context_id", ) - if request.request_id in self.cleaned_up_request_ids: + if request.context_id in self.cleaned_up_context_ids: logger.warning( "[CONFIGURE] Refusing request_id='%s': cleanup already cancelled it", - request.request_id, + f"{request.request_id} context_id={request.context_id}", ) return ConfigureResponse( status=Status.ERROR, status_message="Request was already cancelled", ) - self.aggregation_contexts[request.request_id] = AggregationContext( + self.aggregation_contexts[request.context_id] = AggregationContext( request.request_id, request.num_of_workers, _ownership_token(context), + context_id=request.context_id, ) logger.info( - f"[CONFIGURE] Created AggregationContext for request_id='{request.request_id}' " + f"[CONFIGURE] Created AggregationContext for request_id='{request.request_id}' context_id='{request.context_id}' " f"with expected workers: {request.num_of_workers}" ) return ConfigureResponse(status=Status.OK, status_message="Configured") def Unregister(self, request, context): - agg_ctx = self._get_aggregation_context(request.request_id, context) + agg_ctx = self._get_aggregation_context( + request.context_id, context, request.request_id + ) agg_ctx.last_activity = time.monotonic() with agg_ctx.condition: if agg_ctx.expected_workers <= 1: msg = ( - f"[UNREGISTER] request_id='{request.request_id}' cannot go below 1 worker " + f"[UNREGISTER] request_id='{request.request_id}' context_id='{request.context_id}' cannot go below 1 worker " f"(current: {agg_ctx.expected_workers})" ) logger.warning(msg) @@ -455,7 +483,7 @@ def Unregister(self, request, context): agg_ctx.expected_workers -= 1 logger.info( - f"[UNREGISTER] request_id='{request.request_id}' decreased expected workers " + f"[UNREGISTER] request_id='{request.request_id}' context_id='{request.context_id}' decreased expected workers " f"to {agg_ctx.expected_workers}" ) @@ -472,7 +500,9 @@ def Unregister(self, request, context): ) def Aggregate(self, request, context): - agg_ctx = self._get_aggregation_context(request.request_id, context) + agg_ctx = self._get_aggregation_context( + request.context_id, context, request.request_id + ) agg_ctx.last_activity = time.monotonic() try: aggregation_types, payloads = _decode_operations(request) @@ -495,24 +525,31 @@ def Aggregate(self, request, context): ) def Cleanup(self, request, context): + if not request.context_id or not request.request_id: + context.abort( + grpc.StatusCode.INVALID_ARGUMENT, + "request_id and context_id are required", + ) token = _ownership_token(context) with self.global_lock: - current = self.aggregation_contexts.get(request.request_id) + current = self.aggregation_contexts.get(request.context_id) + if current is not None: + self._validate_context_request(current, request.request_id, context) if current is not None and current.ownership_token != token: logger.warning( "[CLEANUP] Refused to remove request_id='%s': the ownership " "token does not match the caller that configured it", - request.request_id, + f"{request.request_id} context_id={request.context_id}", ) return CleanupResponse( status=Status.ERROR, status_message="Aggregation context is owned by another caller", ) - agg_ctx = self.aggregation_contexts.pop(request.request_id, None) - self._remember_cancelled(request.request_id) + agg_ctx = self.aggregation_contexts.pop(request.context_id, None) + self._remember_cancelled(request.context_id) if not agg_ctx: - msg = f"[CLEANUP] No aggregation context found for request_id='{request.request_id}'" + msg = f"[CLEANUP] No aggregation context found for request_id='{request.request_id}' context_id='{request.context_id}'" logger.warning(msg) return CleanupResponse( status=Status.NOT_FOUND, status_message="No such request_id" @@ -524,14 +561,14 @@ def Cleanup(self, request, context): with agg_ctx.condition: agg_ctx.fail( AggregationCancelledError( - f"Aggregation for request_id='{request.request_id}' was " + f"Aggregation for request_id='{request.request_id}' context_id='{request.context_id}' was " "cancelled because the controller cleaned up the request." ), log=bool(agg_ctx.steps), ) logger.info( - f"[CLEANUP] Removed AggregationContext for request_id='{request.request_id}'" + f"[CLEANUP] Removed AggregationContext for request_id='{request.request_id}' context_id='{request.context_id}'" ) return CleanupResponse(status=Status.OK, status_message="Cleaned up") diff --git a/exaflow/controller/services/exareme3/algorithm_flow_engine_interface.py b/exaflow/controller/services/exareme3/algorithm_flow_engine_interface.py index 29668f920..716c25c81 100644 --- a/exaflow/controller/services/exareme3/algorithm_flow_engine_interface.py +++ b/exaflow/controller/services/exareme3/algorithm_flow_engine_interface.py @@ -55,6 +55,7 @@ def run_udf( udf_registry_key = get_udf_registry_key(func) system_args = RunUdfSystemArgs( + context_id=self._context_id, inputdata=self._inputdata, metadata=add_ordered_enums(self._metadata), add_dataset_variable=add_dataset_variable, diff --git a/exaflow/controller/services/exareme3/strategies.py b/exaflow/controller/services/exareme3/strategies.py index ddfcff104..d076ffa1a 100644 --- a/exaflow/controller/services/exareme3/strategies.py +++ b/exaflow/controller/services/exareme3/strategies.py @@ -129,6 +129,7 @@ async def execute(self) -> str: ) with ControllerAggregationClient( self._request_id, + context_id=self._context_id, aggregator_dns=agg_dns, max_message_mb=controller_config.grpc.max_message_mb, ) as agg_client: diff --git a/exaflow/controller/services/strategy_interface.py b/exaflow/controller/services/strategy_interface.py index 3c1dc4b83..b21f814ad 100644 --- a/exaflow/controller/services/strategy_interface.py +++ b/exaflow/controller/services/strategy_interface.py @@ -3,12 +3,12 @@ from logging import Logger from typing import List from typing import Optional +from uuid import uuid4 from exaflow.controller import logger as ctrl_logger from exaflow.controller.services.api.analysis_request_dtos import AnalysisRequestDTO from exaflow.controller.services.controller_interface import ControllerI from exaflow.controller.services.tasks_handler_interface import TasksHandlerI -from exaflow.controller.uid_generator import UIDGenerator class AlgorithmExecutionStrategyI(ABC): @@ -37,7 +37,7 @@ def __init__( self._analysis_request_dto = analysis_request_dto self._algorithm_name = analysis_request_dto.algorithm.name self._request_id = self._analysis_request_dto.request_id - self._context_id = UIDGenerator().get_a_uid() + self._context_id = uuid4().hex self._logger = ctrl_logger.get_request_logger(self._request_id) self._local_worker_tasks_handlers = ( self._controller.get_local_worker_tasks_handlers( diff --git a/exaflow/protos/README.md b/exaflow/protos/README.md index ce05d759a..b29879783 100644 --- a/exaflow/protos/README.md +++ b/exaflow/protos/README.md @@ -21,3 +21,13 @@ uv run python -m grpc_tools.protoc \ --grpc_python_out=. \ exaflow/protos/worker/worker.proto ``` + +## Execution context compatibility + +Aggregation RPCs carry `request_id` for logical request tracing and a required +`context_id` for the execution attempt. Configure, Aggregate, Unregister, Cleanup, +and cancellation tombstones use `context_id` as their state key. UDF system +arguments carry the same context to every worker; logs retain the request ID. +Controller, workers, and aggregation server must be upgraded together. Older +clients that omit the context are rejected; there is no request-ID fallback. +Ownership-token checks on Cleanup remain mandatory. diff --git a/exaflow/protos/aggregation_server/aggregation_server.proto b/exaflow/protos/aggregation_server/aggregation_server.proto index 47d028a26..c653fa4c7 100644 --- a/exaflow/protos/aggregation_server/aggregation_server.proto +++ b/exaflow/protos/aggregation_server/aggregation_server.proto @@ -22,6 +22,7 @@ service AggregationServer { message ConfigureRequest { string request_id = 1; int32 num_of_workers = 2; + string context_id = 3; } message ConfigureResponse { @@ -38,6 +39,7 @@ message AggregateRequest { string request_id = 1; int32 step = 2; repeated Operation operations = 3; + string context_id = 4; } message AggregateResponse { @@ -46,6 +48,7 @@ message AggregateResponse { message CleanupRequest { string request_id = 1; + string context_id = 2; } message CleanupResponse { @@ -55,6 +58,7 @@ message CleanupResponse { message UnregisterRequest { string request_id = 1; + string context_id = 2; } message UnregisterResponse { diff --git a/exaflow/protos/aggregation_server/aggregation_server_pb2.py b/exaflow/protos/aggregation_server/aggregation_server_pb2.py index fe590c8f7..d579adf23 100644 --- a/exaflow/protos/aggregation_server/aggregation_server_pb2.py +++ b/exaflow/protos/aggregation_server/aggregation_server_pb2.py @@ -25,7 +25,7 @@ DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile( - b'\n:exaflow/protos/aggregation_server/aggregation_server.proto\x12\x12\x61ggregation_server">\n\x10\x43onfigureRequest\x12\x12\n\nrequest_id\x18\x01 \x01(\t\x12\x16\n\x0enum_of_workers\x18\x02 \x01(\x05"W\n\x11\x43onfigureResponse\x12*\n\x06status\x18\x01 \x01(\x0e\x32\x1a.aggregation_server.Status\x12\x16\n\x0estatus_message\x18\x02 \x01(\t"5\n\tOperation\x12\x18\n\x10\x61ggregation_type\x18\x01 \x01(\t\x12\x0e\n\x06tensor\x18\x03 \x01(\x0c"g\n\x10\x41ggregateRequest\x12\x12\n\nrequest_id\x18\x01 \x01(\t\x12\x0c\n\x04step\x18\x02 \x01(\x05\x12\x31\n\noperations\x18\x03 \x03(\x0b\x32\x1d.aggregation_server.Operation"$\n\x11\x41ggregateResponse\x12\x0f\n\x07tensors\x18\x03 \x03(\x0c"$\n\x0e\x43leanupRequest\x12\x12\n\nrequest_id\x18\x01 \x01(\t"U\n\x0f\x43leanupResponse\x12*\n\x06status\x18\x01 \x01(\x0e\x32\x1a.aggregation_server.Status\x12\x16\n\x0estatus_message\x18\x02 \x01(\t"\'\n\x11UnregisterRequest\x12\x12\n\nrequest_id\x18\x01 \x01(\t"s\n\x12UnregisterResponse\x12*\n\x06status\x18\x01 \x01(\x0e\x32\x1a.aggregation_server.Status\x12\x16\n\x0estatus_message\x18\x02 \x01(\t\x12\x19\n\x11remaining_workers\x18\x03 \x01(\x05*x\n\x06Status\x12\x16\n\x12STATUS_UNSPECIFIED\x10\x00\x12\x06\n\x02OK\x10\x01\x12\x16\n\x12\x41LREADY_CONFIGURED\x10\x02\x12\r\n\tNOT_FOUND\x10\x03\x12\x0f\n\x0bMIN_WORKERS\x10\x04\x12\x0b\n\x07TIMEOUT\x10\x05\x12\t\n\x05\x45RROR\x10\x06\x32\xf8\x02\n\x11\x41ggregationServer\x12X\n\tConfigure\x12$.aggregation_server.ConfigureRequest\x1a%.aggregation_server.ConfigureResponse\x12[\n\nUnregister\x12%.aggregation_server.UnregisterRequest\x1a&.aggregation_server.UnregisterResponse\x12X\n\tAggregate\x12$.aggregation_server.AggregateRequest\x1a%.aggregation_server.AggregateResponse\x12R\n\x07\x43leanup\x12".aggregation_server.CleanupRequest\x1a#.aggregation_server.CleanupResponseb\x06proto3' + b'\n:exaflow/protos/aggregation_server/aggregation_server.proto\x12\x12\x61ggregation_server"R\n\x10\x43onfigureRequest\x12\x12\n\nrequest_id\x18\x01 \x01(\t\x12\x16\n\x0enum_of_workers\x18\x02 \x01(\x05\x12\x12\n\ncontext_id\x18\x03 \x01(\t"W\n\x11\x43onfigureResponse\x12*\n\x06status\x18\x01 \x01(\x0e\x32\x1a.aggregation_server.Status\x12\x16\n\x0estatus_message\x18\x02 \x01(\t"5\n\tOperation\x12\x18\n\x10\x61ggregation_type\x18\x01 \x01(\t\x12\x0e\n\x06tensor\x18\x03 \x01(\x0c"{\n\x10\x41ggregateRequest\x12\x12\n\nrequest_id\x18\x01 \x01(\t\x12\x0c\n\x04step\x18\x02 \x01(\x05\x12\x31\n\noperations\x18\x03 \x03(\x0b\x32\x1d.aggregation_server.Operation\x12\x12\n\ncontext_id\x18\x04 \x01(\t"$\n\x11\x41ggregateResponse\x12\x0f\n\x07tensors\x18\x03 \x03(\x0c"8\n\x0e\x43leanupRequest\x12\x12\n\nrequest_id\x18\x01 \x01(\t\x12\x12\n\ncontext_id\x18\x02 \x01(\t"U\n\x0f\x43leanupResponse\x12*\n\x06status\x18\x01 \x01(\x0e\x32\x1a.aggregation_server.Status\x12\x16\n\x0estatus_message\x18\x02 \x01(\t";\n\x11UnregisterRequest\x12\x12\n\nrequest_id\x18\x01 \x01(\t\x12\x12\n\ncontext_id\x18\x02 \x01(\t"s\n\x12UnregisterResponse\x12*\n\x06status\x18\x01 \x01(\x0e\x32\x1a.aggregation_server.Status\x12\x16\n\x0estatus_message\x18\x02 \x01(\t\x12\x19\n\x11remaining_workers\x18\x03 \x01(\x05*x\n\x06Status\x12\x16\n\x12STATUS_UNSPECIFIED\x10\x00\x12\x06\n\x02OK\x10\x01\x12\x16\n\x12\x41LREADY_CONFIGURED\x10\x02\x12\r\n\tNOT_FOUND\x10\x03\x12\x0f\n\x0bMIN_WORKERS\x10\x04\x12\x0b\n\x07TIMEOUT\x10\x05\x12\t\n\x05\x45RROR\x10\x06\x32\xf8\x02\n\x11\x41ggregationServer\x12X\n\tConfigure\x12$.aggregation_server.ConfigureRequest\x1a%.aggregation_server.ConfigureResponse\x12[\n\nUnregister\x12%.aggregation_server.UnregisterRequest\x1a&.aggregation_server.UnregisterResponse\x12X\n\tAggregate\x12$.aggregation_server.AggregateRequest\x1a%.aggregation_server.AggregateResponse\x12R\n\x07\x43leanup\x12".aggregation_server.CleanupRequest\x1a#.aggregation_server.CleanupResponseb\x06proto3' ) _globals = globals() @@ -35,26 +35,26 @@ ) if not _descriptor._USE_C_DESCRIPTORS: DESCRIPTOR._loaded_options = None - _globals["_STATUS"]._serialized_start = 716 - _globals["_STATUS"]._serialized_end = 836 + _globals["_STATUS"]._serialized_start = 796 + _globals["_STATUS"]._serialized_end = 916 _globals["_CONFIGUREREQUEST"]._serialized_start = 82 - _globals["_CONFIGUREREQUEST"]._serialized_end = 144 - _globals["_CONFIGURERESPONSE"]._serialized_start = 146 - _globals["_CONFIGURERESPONSE"]._serialized_end = 233 - _globals["_OPERATION"]._serialized_start = 235 - _globals["_OPERATION"]._serialized_end = 288 - _globals["_AGGREGATEREQUEST"]._serialized_start = 290 - _globals["_AGGREGATEREQUEST"]._serialized_end = 393 - _globals["_AGGREGATERESPONSE"]._serialized_start = 395 - _globals["_AGGREGATERESPONSE"]._serialized_end = 431 - _globals["_CLEANUPREQUEST"]._serialized_start = 433 - _globals["_CLEANUPREQUEST"]._serialized_end = 469 - _globals["_CLEANUPRESPONSE"]._serialized_start = 471 - _globals["_CLEANUPRESPONSE"]._serialized_end = 556 - _globals["_UNREGISTERREQUEST"]._serialized_start = 558 - _globals["_UNREGISTERREQUEST"]._serialized_end = 597 - _globals["_UNREGISTERRESPONSE"]._serialized_start = 599 - _globals["_UNREGISTERRESPONSE"]._serialized_end = 714 - _globals["_AGGREGATIONSERVER"]._serialized_start = 839 - _globals["_AGGREGATIONSERVER"]._serialized_end = 1215 + _globals["_CONFIGUREREQUEST"]._serialized_end = 164 + _globals["_CONFIGURERESPONSE"]._serialized_start = 166 + _globals["_CONFIGURERESPONSE"]._serialized_end = 253 + _globals["_OPERATION"]._serialized_start = 255 + _globals["_OPERATION"]._serialized_end = 308 + _globals["_AGGREGATEREQUEST"]._serialized_start = 310 + _globals["_AGGREGATEREQUEST"]._serialized_end = 433 + _globals["_AGGREGATERESPONSE"]._serialized_start = 435 + _globals["_AGGREGATERESPONSE"]._serialized_end = 471 + _globals["_CLEANUPREQUEST"]._serialized_start = 473 + _globals["_CLEANUPREQUEST"]._serialized_end = 529 + _globals["_CLEANUPRESPONSE"]._serialized_start = 531 + _globals["_CLEANUPRESPONSE"]._serialized_end = 616 + _globals["_UNREGISTERREQUEST"]._serialized_start = 618 + _globals["_UNREGISTERREQUEST"]._serialized_end = 677 + _globals["_UNREGISTERRESPONSE"]._serialized_start = 679 + _globals["_UNREGISTERRESPONSE"]._serialized_end = 794 + _globals["_AGGREGATIONSERVER"]._serialized_start = 919 + _globals["_AGGREGATIONSERVER"]._serialized_end = 1295 # @@protoc_insertion_point(module_scope) diff --git a/exaflow/worker/exareme3/udf/udf_service.py b/exaflow/worker/exareme3/udf/udf_service.py index 6638f1816..13e0d19ce 100644 --- a/exaflow/worker/exareme3/udf/udf_service.py +++ b/exaflow/worker/exareme3/udf/udf_service.py @@ -44,6 +44,7 @@ def run_udf( ) agg_client = _create_aggregation_client_if_required( request_id=request_id, + context_id=system_args.context_id, udf_registry_key=udf_registry_key, preprocessing=preprocessing, ) @@ -102,6 +103,7 @@ def _wrap_udf_with_lazy_aggregation_if_enabled(udf_registry_key: str, udf): def _create_aggregation_client_if_required( request_id, + context_id, udf_registry_key: str, preprocessing: Optional[List[Dict[str, Any]]], ) -> Optional[AggregationClient]: @@ -113,6 +115,7 @@ def _create_aggregation_client_if_required( agg_dns = worker_config.aggregation_server.dns return AggregationClient( request_id, + context_id=context_id, aggregator_dns=agg_dns, max_message_mb=worker_config.grpc.max_message_mb, timeout=worker_config.worker_tasks.tasks_timeout, diff --git a/exaflow/worker_communication.py b/exaflow/worker_communication.py index 405350a8e..bb41d60df 100644 --- a/exaflow/worker_communication.py +++ b/exaflow/worker_communication.py @@ -199,6 +199,7 @@ def _are_equal_dataset_cdes( class RunUdfSystemArgs(BaseModel): + context_id: str inputdata: Inputdata metadata: Dict[str, Any] add_dataset_variable: bool diff --git a/tests/standalone_tests/aggregation_server/test_aggregation_clients.py b/tests/standalone_tests/aggregation_server/test_aggregation_clients.py index d630e5740..cc8b5a6db 100644 --- a/tests/standalone_tests/aggregation_server/test_aggregation_clients.py +++ b/tests/standalone_tests/aggregation_server/test_aggregation_clients.py @@ -137,7 +137,7 @@ def controller_factory(): created: list[ControllerAggregationClient] = [] def factory(request_id: str) -> ControllerAggregationClient: - controller = ControllerAggregationClient(request_id) + controller = ControllerAggregationClient(request_id, context_id=request_id) created.append(controller) return controller @@ -152,7 +152,7 @@ def worker_factory(): created: list[Exareme3UDFAggregationClient] = [] def factory(request_id: str) -> Exareme3UDFAggregationClient: - client = Exareme3UDFAggregationClient(request_id) + client = Exareme3UDFAggregationClient(request_id, context_id=request_id) created.append(client) return client @@ -190,7 +190,7 @@ def test_cleanup_before_configure_rejects_the_later_configure( controller = controller_factory(request_id) assert controller.cleanup(timeout=300) == server_pb2.Status.NOT_FOUND - assert request_id in inline_grpc_stub.cleaned_up_request_ids + assert request_id in inline_grpc_stub.cleaned_up_context_ids assert controller.configure(1, timeout=300) == server_pb2.Status.ERROR assert request_id not in inline_grpc_stub.aggregation_contexts @@ -199,7 +199,7 @@ def test_cancelled_request_ids_are_bounded( inline_grpc_stub, controller_factory, monkeypatch ): monkeypatch.setattr( - "exaflow.aggregation_server.server.CANCELLED_REQUEST_ID_LIMIT", + "exaflow.aggregation_server.server.CANCELLED_CONTEXT_ID_LIMIT", 2, raising=False, ) @@ -210,7 +210,7 @@ def test_cancelled_request_ids_are_bounded( == server_pb2.Status.NOT_FOUND ) - assert set(inline_grpc_stub.cleaned_up_request_ids) == {"bound-1", "bound-2"} + assert set(inline_grpc_stub.cleaned_up_context_ids) == {"bound-1", "bound-2"} assert ( controller_factory("bound-0").configure(1, timeout=300) == server_pb2.Status.OK ) @@ -224,9 +224,9 @@ def test_expired_cancellation_can_be_configured(inline_grpc_stub, controller_fac request_id = "cleanup-expired" controller = controller_factory(request_id) assert controller.cleanup(timeout=300) == server_pb2.Status.NOT_FOUND - assert request_id in inline_grpc_stub.cleaned_up_request_ids + assert request_id in inline_grpc_stub.cleaned_up_context_ids - inline_grpc_stub.cleaned_up_request_ids[request_id] = ( + inline_grpc_stub.cleaned_up_context_ids[request_id] = ( time.monotonic() - STALE_CONTEXT_TTL_SECONDS - 1 ) @@ -813,7 +813,7 @@ def test_unregister_reduces_expected_workers(controller_factory): request_id = "udf-unregister" controller = controller_factory(request_id) controller.configure(2, timeout=300) - client = Exareme3UDFAggregationClient(request_id) + client = Exareme3UDFAggregationClient(request_id, context_id=request_id) status, remaining = client.unregister() assert status == server_pb2.Status.OK @@ -881,6 +881,7 @@ def test_unsupported_aggregation_type_is_rejected(inline_grpc_stub, controller_f aggregation_type="MEDIAN", tensor=ndarray_to_bytes(np.array([1.0])) ) ], + context_id="bad-type", ) with pytest.raises(InlineRpcError) as exc: @@ -898,6 +899,7 @@ def _sum_request(request_id: str, step: int = 1) -> server_pb2.AggregateRequest: aggregation_type="SUM", tensor=ndarray_to_bytes(np.array([1.0])) ) ], + context_id=request_id, ) @@ -938,7 +940,9 @@ def test_step_result_reaches_every_worker_after_an_unregister( monkeypatch.setattr(aggregation_config, "max_wait_for_aggregation_inputs", 5) request_id = "unregister-after-read" inline_grpc_stub.Configure( - server_pb2.ConfigureRequest(request_id=request_id, num_of_workers=3), + server_pb2.ConfigureRequest( + request_id=request_id, num_of_workers=3, context_id=request_id + ), InlineGrpcContext(), ) agg_ctx = inline_grpc_stub.aggregation_contexts[request_id] @@ -962,7 +966,9 @@ def test_step_result_reaches_every_worker_after_an_unregister( with agg_ctx.condition: inline_grpc_stub.Aggregate(_sum_request(request_id), InlineGrpcContext()) inline_grpc_stub.Unregister( - server_pb2.UnregisterRequest(request_id=request_id), + server_pb2.UnregisterRequest( + request_id=request_id, context_id=request_id + ), InlineGrpcContext(), ) diff --git a/tests/standalone_tests/aggregation_server/test_aggregation_ownership.py b/tests/standalone_tests/aggregation_server/test_aggregation_ownership.py index 21d59d1e6..ee92bebb5 100644 --- a/tests/standalone_tests/aggregation_server/test_aggregation_ownership.py +++ b/tests/standalone_tests/aggregation_server/test_aggregation_ownership.py @@ -91,7 +91,7 @@ def cleanup(request, timeout=None, metadata=None): return servicer.Cleanup(request, _MetadataContext(metadata)) owner = _wire( - ControllerAggregationClient(request_id), + ControllerAggregationClient(request_id, context_id=request_id), owner_future, cleanup, ) @@ -103,12 +103,13 @@ def lost_future(request, timeout=None, metadata=None): servicer.Configure(request, _MetadataContext(metadata)) return _LostResponse() - def factory(request_id, aggregator_dns=None, max_message_mb=None): + def factory(request_id, context_id, aggregator_dns=None, max_message_mb=None): return _wire( ControllerAggregationClient( request_id, aggregator_dns=aggregator_dns, max_message_mb=max_message_mb, + context_id=request_id, ), lost_future, cleanup, @@ -117,6 +118,7 @@ def factory(request_id, aggregator_dns=None, max_message_mb=None): monkeypatch.setattr(strategies, "ControllerAggregationClient", factory) strategy = build_strategy(strategies.Exareme3WithAggregationServerStrategy) strategy._request_id = request_id + strategy._context_id = request_id strategy._logger = logging.getLogger("aggregation-ownership") with pytest.raises(ControllerShuttingDownError): @@ -140,7 +142,11 @@ def lost_future(request, timeout=None, metadata=None): def cleanup(request, timeout=None, metadata=None): return servicer.Cleanup(request, _MetadataContext(metadata)) - client = _wire(ControllerAggregationClient(request_id), lost_future, cleanup) + client = _wire( + ControllerAggregationClient(request_id, context_id=request_id), + lost_future, + cleanup, + ) with pytest.raises(ControllerShuttingDownError): client.configure(2, timeout=5) assert request_id in servicer.aggregation_contexts diff --git a/tests/standalone_tests/aggregation_server/test_aggregation_server_grpc.py b/tests/standalone_tests/aggregation_server/test_aggregation_server_grpc.py index 39f682c1f..280599b2a 100644 --- a/tests/standalone_tests/aggregation_server/test_aggregation_server_grpc.py +++ b/tests/standalone_tests/aggregation_server/test_aggregation_server_grpc.py @@ -24,6 +24,9 @@ ) from exaflow.aggregation_server import config as aggregation_config from exaflow.aggregation_server.server import AggregationServer +from exaflow.protos.aggregation_server.aggregation_server_pb2_grpc import ( + AggregationServerStub, +) from exaflow.protos.aggregation_server.aggregation_server_pb2_grpc import ( add_AggregationServerServicer_to_server, ) @@ -42,15 +45,57 @@ def target(monkeypatch) -> Iterator[Tuple[str, AggregationServer]]: def _configure(target: str, request_id: str, num_workers: int): - controller = ControllerAggregationClient(request_id, aggregator_dns=target) + controller = ControllerAggregationClient( + request_id, aggregator_dns=target, context_id=request_id + ) controller.configure(num_workers, timeout=300) return controller +@pytest.mark.parametrize("rpc", ["Configure", "Aggregate", "Unregister", "Cleanup"]) +@pytest.mark.parametrize("missing", ["request_id", "context_id"]) +def test_empty_identifiers_are_rejected_over_grpc(target, rpc, missing): + endpoint, servicer = target + fields = {"request_id": "logical-request", "context_id": "attempt"} + fields[missing] = "" + request = getattr(pb2, f"{rpc}Request")(**fields) + with grpc.insecure_channel(endpoint) as channel: + with pytest.raises(grpc.RpcError) as exc: + getattr(AggregationServerStub(channel), rpc)(request, timeout=5) + assert exc.value.code() == grpc.StatusCode.INVALID_ARGUMENT + assert not servicer.aggregation_contexts + assert not servicer.cleaned_up_context_ids + + +@pytest.mark.parametrize("rpc", ["Configure", "Aggregate", "Unregister", "Cleanup"]) +def test_mismatched_request_cannot_modify_an_existing_context(target, rpc): + endpoint, servicer = target + owner = _configure(endpoint, "original", 1) + worker = Exareme3UDFAggregationClient( + "original", "original", aggregator_dns=endpoint + ) + request = getattr(pb2, f"{rpc}Request")(request_id="wrong", context_id="original") + try: + with grpc.insecure_channel(endpoint) as channel: + with pytest.raises(grpc.RpcError) as exc: + getattr(AggregationServerStub(channel), rpc)( + request, timeout=5, metadata=owner._ownership_metadata() + ) + assert exc.value.code() == grpc.StatusCode.INVALID_ARGUMENT + assert servicer.aggregation_contexts["original"].request_id == "original" + assert worker.sum([7.0]).tolist() == [7.0] + assert owner.cleanup(timeout=5) == pb2.Status.OK + finally: + worker.close() + owner.close() + + def test_replayed_step_keeps_invalid_argument_status(target): endpoint, _ = target _configure(endpoint, "replayed-step", 1) - worker = Exareme3UDFAggregationClient("replayed-step", aggregator_dns=endpoint) + worker = Exareme3UDFAggregationClient( + "replayed-step", aggregator_dns=endpoint, context_id="replayed-step" + ) worker._aggregate_batch_request([(AggregationType.SUM, [1.0])], step=1) with pytest.raises(grpc.RpcError) as exc: @@ -63,8 +108,12 @@ def test_replayed_step_keeps_invalid_argument_status(target): def test_aggregation_type_mismatch_reaches_all_workers_with_status(target): endpoint, _ = target _configure(endpoint, "type-mismatch", 2) - worker_a = Exareme3UDFAggregationClient("type-mismatch", aggregator_dns=endpoint) - worker_b = Exareme3UDFAggregationClient("type-mismatch", aggregator_dns=endpoint) + worker_a = Exareme3UDFAggregationClient( + "type-mismatch", aggregator_dns=endpoint, context_id="type-mismatch" + ) + worker_b = Exareme3UDFAggregationClient( + "type-mismatch", aggregator_dns=endpoint, context_id="type-mismatch" + ) with ThreadPoolExecutor(max_workers=2) as pool: future_a = pool.submit(worker_a.sum, [1.0]) @@ -92,8 +141,12 @@ def test_cleanup_aborts_workers_still_waiting(target): endpoint, servicer = target controller = _configure(endpoint, "cleanup-waiters", 3) # worker_a waits for the step 1 result, worker_b waits to start step 2. - worker_a = Exareme3UDFAggregationClient("cleanup-waiters", aggregator_dns=endpoint) - worker_b = Exareme3UDFAggregationClient("cleanup-waiters", aggregator_dns=endpoint) + worker_a = Exareme3UDFAggregationClient( + "cleanup-waiters", aggregator_dns=endpoint, context_id="cleanup-waiters" + ) + worker_b = Exareme3UDFAggregationClient( + "cleanup-waiters", aggregator_dns=endpoint, context_id="cleanup-waiters" + ) with ThreadPoolExecutor(max_workers=2) as pool: future_a = pool.submit( @@ -117,7 +170,9 @@ def test_aggregate_after_cleanup_is_aborted_not_invalid_argument(target): endpoint, _ = target controller = _configure(endpoint, "late-after-cleanup", 1) controller.cleanup(timeout=300) - worker = Exareme3UDFAggregationClient("late-after-cleanup", aggregator_dns=endpoint) + worker = Exareme3UDFAggregationClient( + "late-after-cleanup", aggregator_dns=endpoint, context_id="late-after-cleanup" + ) with pytest.raises(grpc.RpcError) as exc: worker.sum([1.0]) @@ -128,7 +183,9 @@ def test_aggregate_after_cleanup_is_aborted_not_invalid_argument(target): def test_aggregate_without_configure_keeps_invalid_argument(target): endpoint, _ = target - worker = Exareme3UDFAggregationClient("never-configured", aggregator_dns=endpoint) + worker = Exareme3UDFAggregationClient( + "never-configured", aggregator_dns=endpoint, context_id="never-configured" + ) with pytest.raises(grpc.RpcError) as exc: worker.sum([1.0]) @@ -141,11 +198,14 @@ def test_aggregate_without_configure_keeps_invalid_argument(target): def test_unknown_aggregation_type_is_rejected_with_its_name(target): endpoint, servicer = target _configure(endpoint, "unknown-type", 2) - worker = Exareme3UDFAggregationClient("unknown-type", aggregator_dns=endpoint) + worker = Exareme3UDFAggregationClient( + "unknown-type", aggregator_dns=endpoint, context_id="unknown-type" + ) request = pb2.AggregateRequest( request_id="unknown-type", step=1, operations=[pb2.Operation(aggregation_type="AVG", tensor=b"x")], + context_id="unknown-type", ) # Rejected on the first worker's call, without waiting for the second worker. @@ -154,3 +214,40 @@ def test_unknown_aggregation_type_is_rejected_with_its_name(target): assert exc.value.code() == grpc.StatusCode.INVALID_ARGUMENT assert "Unsupported aggregation type 'AVG'" in exc.value.details() + + +def test_attempts_share_request_logging_id_but_isolate_aggregation_state( + target, caplog +): + endpoint, servicer = target + first = ControllerAggregationClient( + "logical-request", "attempt-one", aggregator_dns=endpoint + ) + second = ControllerAggregationClient( + "logical-request", "attempt-two", aggregator_dns=endpoint + ) + worker = Exareme3UDFAggregationClient( + "logical-request", "attempt-two", aggregator_dns=endpoint + ) + try: + with caplog.at_level("INFO"): + assert first.configure(1, timeout=5) == pb2.Status.OK + assert second.configure(1, timeout=5) == pb2.Status.OK + assert set(servicer.aggregation_contexts) == {"attempt-one", "attempt-two"} + assert "logical-request" in caplog.text and "attempt-two" in caplog.text + assert first.cleanup(timeout=5) == pb2.Status.OK + assert worker.sum([7.0]).tolist() == [7.0] + late = Exareme3UDFAggregationClient( + "logical-request", "attempt-one", aggregator_dns=endpoint + ) + try: + with pytest.raises(grpc.RpcError) as exc: + late.sum([1.0]) + assert exc.value.code() == grpc.StatusCode.ABORTED + finally: + late.close() + assert second.cleanup(timeout=5) == pb2.Status.OK + finally: + worker.close() + first.close() + second.close() diff --git a/tests/standalone_tests/aggregation_server/test_udf_aggregation_client_deadline.py b/tests/standalone_tests/aggregation_server/test_udf_aggregation_client_deadline.py index 84b7749d8..b6190d7d8 100644 --- a/tests/standalone_tests/aggregation_server/test_udf_aggregation_client_deadline.py +++ b/tests/standalone_tests/aggregation_server/test_udf_aggregation_client_deadline.py @@ -38,7 +38,10 @@ def Unregister(self, request, timeout=None): def _client(**kwargs): client = Exareme3UDFAggregationClient( - "req-deadline", aggregator_dns="127.0.0.1:1", **kwargs + "req-deadline", + context_id="ctx-deadline", + aggregator_dns="127.0.0.1:1", + **kwargs, ) client._stub = _RecordingStub() # noqa: SLF001 return client @@ -93,9 +96,11 @@ def __init__(self, request_id, **kwargs): udf_service._create_aggregation_client_if_required( request_id="req-deadline", + context_id="ctx-deadline", udf_registry_key="any_udf", preprocessing=[], ) assert created["request_id"] == "req-deadline" + assert created["context_id"] == "ctx-deadline" assert created.get("timeout") == 42 diff --git a/tests/standalone_tests/controller/services/test_aggregation_cleanup_on_cancel.py b/tests/standalone_tests/controller/services/test_aggregation_cleanup_on_cancel.py index 30b1638b8..10462a2d3 100644 --- a/tests/standalone_tests/controller/services/test_aggregation_cleanup_on_cancel.py +++ b/tests/standalone_tests/controller/services/test_aggregation_cleanup_on_cancel.py @@ -24,7 +24,9 @@ def test_cancel_during_analysis_cleans_up_only_after_run_finishes( release_run = threading.Event() class FakeAggregationClient: - def __init__(self, request_id, aggregator_dns=None, max_message_mb=None): + def __init__( + self, request_id, context_id, aggregator_dns=None, max_message_mb=None + ): order.append("init") def configure(self, num_workers, timeout=None): @@ -91,7 +93,9 @@ async def release(): class _RecordingAggregationClient: configure_status = agg_pb2.Status.OK - def __init__(self, request_id, aggregator_dns=None, max_message_mb=None): + def __init__( + self, request_id, context_id, aggregator_dns=None, max_message_mb=None + ): self.calls = [] _RecordingAggregationClient.last = self diff --git a/tests/standalone_tests/controller/services/test_exareme3_aggregation_strategy.py b/tests/standalone_tests/controller/services/test_exareme3_aggregation_strategy.py index 1994e880f..6f859d249 100644 --- a/tests/standalone_tests/controller/services/test_exareme3_aggregation_strategy.py +++ b/tests/standalone_tests/controller/services/test_exareme3_aggregation_strategy.py @@ -16,7 +16,9 @@ class _RecordingAggregationClient: - def __init__(self, request_id, aggregator_dns=None, max_message_mb=None): + def __init__( + self, request_id, context_id, aggregator_dns=None, max_message_mb=None + ): self.calls = [] def configure(self, num_workers, timeout=None): @@ -39,9 +41,10 @@ def test_aggregation_strategy_cleans_up_and_closes_client_when_algorithm_fails( ): created = [] - def factory(request_id, aggregator_dns=None, max_message_mb=None): + def factory(request_id, context_id, aggregator_dns=None, max_message_mb=None): client = _RecordingAggregationClient( request_id, + context_id, aggregator_dns=aggregator_dns, max_message_mb=max_message_mb, ) @@ -55,6 +58,7 @@ async def failing_execute(self): monkeypatch.setattr(strategies.Exareme3Strategy, "execute", failing_execute) strategy = object.__new__(strategies.Exareme3WithAggregationServerStrategy) strategy._request_id = "req" + strategy._context_id = "ctx" strategy._local_worker_tasks_handlers = [object(), object()] strategy._logger = logging.getLogger(__name__) strategy._controller = types.SimpleNamespace(task_timeout=300) @@ -85,6 +89,7 @@ def details(self): def _make_strategy(task_timeout): strategy = object.__new__(strategies.Exareme3WithAggregationServerStrategy) strategy._request_id = "req" + strategy._context_id = "ctx" strategy._local_worker_tasks_handlers = [object(), object()] strategy._logger = logging.getLogger(__name__) strategy._controller = types.SimpleNamespace(task_timeout=task_timeout) @@ -99,7 +104,9 @@ def test_configure_and_cleanup_send_the_controller_task_timeout_as_deadline( seen = {} class DeadlineClient: - def __init__(self, request_id, aggregator_dns=None, max_message_mb=None): + def __init__( + self, request_id, context_id, aggregator_dns=None, max_message_mb=None + ): pass def configure(self, num_workers, timeout=None): @@ -141,7 +148,9 @@ def Cleanup(self, request, timeout=None, metadata=None): grpc.StatusCode.DEADLINE_EXCEEDED, "deadline exceeded" ) - client = ControllerAggregationClient("deadline-cleanup") + client = ControllerAggregationClient( + "deadline-cleanup", context_id="deadline-cleanup" + ) client._stub = DeadlineStub() with caplog.at_level(logging.WARNING): @@ -167,7 +176,9 @@ def Cleanup(self, request, timeout=None, metadata=None): seen["cleanup"] = timeout return agg_pb2.CleanupResponse(status=agg_pb2.Status.OK) - client = ControllerAggregationClient("deadline-pass-through") + client = ControllerAggregationClient( + "deadline-pass-through", context_id="deadline-pass-through" + ) client._stub = RecordingStub() assert client.configure(2, timeout=7) == agg_pb2.Status.OK @@ -182,7 +193,9 @@ class _ConfigureFailsClient: error = None calls = [] - def __init__(self, request_id, aggregator_dns=None, max_message_mb=None): + def __init__( + self, request_id, context_id, aggregator_dns=None, max_message_mb=None + ): pass def configure(self, num_workers, timeout=None): diff --git a/tests/standalone_tests/controller/services/test_run_udf_call.py b/tests/standalone_tests/controller/services/test_run_udf_call.py index 784bfd060..e8ac72b29 100644 --- a/tests/standalone_tests/controller/services/test_run_udf_call.py +++ b/tests/standalone_tests/controller/services/test_run_udf_call.py @@ -86,6 +86,7 @@ def run_udf(**_kwargs): inputdata={"data_model": "dm", "datasets": ["ds"], "variables": ["x"]}, metadata={}, add_dataset_variable=False, + context_id="test-context", ) assert ( diff --git a/tests/standalone_tests/controller/services/test_shutdown_budget.py b/tests/standalone_tests/controller/services/test_shutdown_budget.py index 65076670f..ac8b8fa3d 100644 --- a/tests/standalone_tests/controller/services/test_shutdown_budget.py +++ b/tests/standalone_tests/controller/services/test_shutdown_budget.py @@ -179,7 +179,9 @@ def test_shutdown_cleanup_uses_the_short_deadline(monkeypatch): timeouts = [] class FakeAggregationClient: - def __init__(self, request_id, aggregator_dns=None, max_message_mb=None): + def __init__( + self, request_id, context_id, aggregator_dns=None, max_message_mb=None + ): pass def configure(self, num_workers, timeout=None): @@ -361,7 +363,7 @@ def test_cleanup_and_metadata_rpcs_are_not_cancelled_by_shutdown(): def test_aggregation_cleanup_rpc_is_not_cancelled_by_shutdown(): - client = ControllerAggregationClient("req-cleanup") + client = ControllerAggregationClient("req-cleanup", context_id="req-cleanup") seen = {} def cleanup(request, timeout=None, metadata=None): @@ -389,7 +391,7 @@ def test_cancelled_worker_rpc_raises_the_shutting_down_error(): def test_aggregation_configure_unblocks_when_its_rpc_is_cancelled(): - client = ControllerAggregationClient("req-shutdown") + client = ControllerAggregationClient("req-shutdown", context_id="req-shutdown") call = _BlockingCall() def future(request, timeout=None, metadata=None): diff --git a/tests/standalone_tests/controller/test_blocking_calls_off_loop.py b/tests/standalone_tests/controller/test_blocking_calls_off_loop.py index 06579c2e4..8cb04783f 100644 --- a/tests/standalone_tests/controller/test_blocking_calls_off_loop.py +++ b/tests/standalone_tests/controller/test_blocking_calls_off_loop.py @@ -47,7 +47,9 @@ def run(self): def _aggregation_strategy(monkeypatch, configure, cleanup): class FakeAggregationClient: - def __init__(self, request_id, aggregator_dns=None, max_message_mb=None): + def __init__( + self, request_id, context_id, aggregator_dns=None, max_message_mb=None + ): pass def configure(self, num_workers, timeout=None): diff --git a/tests/standalone_tests/exareme3/test_udf_min_rows_enforcement.py b/tests/standalone_tests/exareme3/test_udf_min_rows_enforcement.py index b378603c2..30c85052e 100644 --- a/tests/standalone_tests/exareme3/test_udf_min_rows_enforcement.py +++ b/tests/standalone_tests/exareme3/test_udf_min_rows_enforcement.py @@ -42,6 +42,7 @@ def two_row_worker(monkeypatch): def _system_args(**extra): return SimpleNamespace( + context_id="ctx-min-rows", preprocessing=[], metadata={"x": {"is_categorical": False}}, inputdata=None, @@ -101,6 +102,7 @@ def test_run_udf_system_args_carry_no_minimum_row_flag(): def test_a_minimum_row_flag_in_the_payload_is_ignored(): parsed = RunUdfSystemArgs.model_validate( { + "context_id": "ctx-min-rows", "inputdata": {"data_model": "dm:1", "datasets": ["ds"], "variables": ["x"]}, "metadata": {}, "add_dataset_variable": False, diff --git a/tests/standalone_tests/exareme3/test_udf_service.py b/tests/standalone_tests/exareme3/test_udf_service.py index 585b2d716..821504398 100644 --- a/tests/standalone_tests/exareme3/test_udf_service.py +++ b/tests/standalone_tests/exareme3/test_udf_service.py @@ -246,10 +246,16 @@ def test_preprocessing_requirement_can_create_client_for_non_aggregation_udf( class _CreatedAggClient(_DummyAggClient): def __init__( - self, request_id, aggregator_dns=None, max_message_mb=None, timeout=None + self, + request_id, + context_id, + aggregator_dns=None, + max_message_mb=None, + timeout=None, ): super().__init__() self.request_id = request_id + self.context_id = context_id self.aggregator_dns = aggregator_dns created_clients.append(self) @@ -261,6 +267,7 @@ def __init__( ) client = udf_service._create_aggregation_client_if_required( + context_id="test-context", request_id="req", udf_registry_key="non_agg_udf", preprocessing=[{"name": "agg_aware", "parameters": {}}], @@ -268,6 +275,7 @@ def __init__( assert client is created_clients[0] assert client.request_id == "req" + assert client.context_id == "test-context" def test_execute_udf_does_not_pass_preprocessing_only_agg_client_to_udf(): @@ -333,6 +341,7 @@ def udf(data): {"bad_input": _BadInputStep}, ) system_args = SimpleNamespace( + context_id="test-context", preprocessing=[{"name": "bad_input", "parameters": {}}], metadata={"x": {"is_categorical": False}}, inputdata=None, @@ -383,6 +392,7 @@ def test_run_udf_closes_agg_client_without_unregistering(monkeypatch, udf, expec lambda **_: pd.DataFrame({"x": [1, 2]}), ) system_args = SimpleNamespace( + context_id="test-context", preprocessing=[], metadata={"x": {"is_categorical": False}}, inputdata=None, @@ -438,6 +448,7 @@ def test_run_udf_keeps_error_when_unregister_fails( {"bad_input": _BadInputStep}, ) system_args = SimpleNamespace( + context_id="test-context", preprocessing=preprocessing, metadata={"x": {"is_categorical": False}}, inputdata=None, diff --git a/tests/standalone_tests/lazy_aggregation/test_lazy_aggregation_algorithms.py b/tests/standalone_tests/lazy_aggregation/test_lazy_aggregation_algorithms.py index 102b2ad82..577aac6d6 100644 --- a/tests/standalone_tests/lazy_aggregation/test_lazy_aggregation_algorithms.py +++ b/tests/standalone_tests/lazy_aggregation/test_lazy_aggregation_algorithms.py @@ -365,7 +365,7 @@ def __init__(self, inputdata: Inputdata, metadata: dict, preprocessing: list): def run_udf(self, func, add_dataset_variable, kw_args): request_id = f"lazy-agg-{uuid.uuid4().hex}" - controller = ControllerAggregationClient(request_id) + controller = ControllerAggregationClient(request_id, context_id=request_id) controller.configure( num_workers=len(self._worker_inputdata), timeout=AGG_CLIENT_TIMEOUT ) @@ -377,6 +377,7 @@ def run_worker(inputdata): metadata=add_ordered_enums(self._metadata), add_dataset_variable=add_dataset_variable, preprocessing=self._preprocessing, + context_id=request_id, ) wire_system_args = RunUdfSystemArgs.model_validate( _struct_to_dict(_dict_to_struct(system_args.model_dump())) diff --git a/tests/standalone_tests/test_worker_grpc_message_size.py b/tests/standalone_tests/test_worker_grpc_message_size.py index e4ec6d395..32af2d9de 100644 --- a/tests/standalone_tests/test_worker_grpc_message_size.py +++ b/tests/standalone_tests/test_worker_grpc_message_size.py @@ -158,6 +158,8 @@ def test_large_kw_args_and_result_round_trip_through_run_udf(monkeypatch): ) def fake_run_udf(**kwargs): + assert kwargs["request_id"] == "req-large" + assert kwargs["system_args"].context_id == "size-test-context" assert kwargs["kw_args"]["blob"] == blob assert kwargs["system_args"].inputdata.data_model == "dementia" return {"blob": kwargs["kw_args"]["blob"], "result": result_blob} @@ -174,6 +176,7 @@ def fake_run_udf(**kwargs): udf_registry_key="size_probe", kw_args={"blob": blob}, system_args={ + "context_id": "size-test-context", "inputdata": { "data_model": "dementia", "datasets": ["ppmi0"], @@ -228,7 +231,10 @@ def capture_server(*args, **kwargs): udf_service.exareme3_registry, "aggregation_server_required", lambda _: True ) udf_service._create_aggregation_client_if_required( - request_id="req-size", udf_registry_key="udf", preprocessing=None + request_id="req-size", + context_id="size-test-context", + udf_registry_key="udf", + preprocessing=None, ) serve() From 1ca76fe4b1e86017b5c23bb30eb20eacd7b8fdce Mon Sep 17 00:00:00 2001 From: ThanKarab Date: Fri, 13 Nov 2026 13:51:27 +0200 Subject: [PATCH 2/6] feat(controller): support waiting for analysis capacity --- .../controller/services/analysis_execution.py | 10 ++--- .../controller/services/analysis_service.py | 14 ++++++- .../services/test_analysis_service.py | 31 ++++++++++++++ .../controller/test_analysis_concurrency.py | 41 +++++++++++++++++++ 4 files changed, 90 insertions(+), 6 deletions(-) diff --git a/exaflow/controller/services/analysis_execution.py b/exaflow/controller/services/analysis_execution.py index f60b2910e..2479cb79e 100644 --- a/exaflow/controller/services/analysis_execution.py +++ b/exaflow/controller/services/analysis_execution.py @@ -1,8 +1,8 @@ """Bounded off-loop execution for analyses. Analyses share one thread pool sized to ``max_concurrent_experiments``. -A semaphore of the same size rejects extra requests instead of queueing -them on that pool. Health checks and worker-landscape updates stay on +A semaphore of the same size rejects excess HTTP requests; queue consumers +can explicitly wait for a slot instead of queueing work on that pool. Health checks and worker-landscape updates stay on the default executor. """ @@ -117,10 +117,10 @@ def _ensure_semaphore() -> asyncio.Semaphore: @asynccontextmanager -async def analysis_slot(): - """Hold one analysis slot, or reject when every slot is taken.""" +async def analysis_slot(*, wait_for_capacity: bool = False): + """Hold one analysis slot; reject excess work unless waiting is requested.""" semaphore = _ensure_semaphore() - if semaphore.locked(): + if not wait_for_capacity and semaphore.locked(): raise AnalysisCapacityExceeded() await semaphore.acquire() try: diff --git a/exaflow/controller/services/analysis_service.py b/exaflow/controller/services/analysis_service.py index f3ec156e2..b2b26ab0d 100644 --- a/exaflow/controller/services/analysis_service.py +++ b/exaflow/controller/services/analysis_service.py @@ -13,7 +13,7 @@ ) -async def run_analysis(analysis_request_dto: AnalysisRequestDTO) -> str: +def _validate(analysis_request_dto: AnalysisRequestDTO) -> None: validate_analysis_request( analysis_request_dto=analysis_request_dto, algorithms_specs=specifications.enabled_algorithms, @@ -23,6 +23,18 @@ async def run_analysis(analysis_request_dto: AnalysisRequestDTO) -> str: smpc_optional=ctrl_config.smpc.optional, ) + +async def run_analysis( + analysis_request_dto: AnalysisRequestDTO, *, wait_for_capacity: bool = False +) -> str: + if wait_for_capacity: + async with analysis_slot(wait_for_capacity=True): + _validate(analysis_request_dto) + return await get_algorithm_execution_strategy( + analysis_request_dto + ).execute() + # Preserve HTTP validation and strategy creation before capacity rejection. + _validate(analysis_request_dto) strategy = get_algorithm_execution_strategy(analysis_request_dto) async with analysis_slot(): return await strategy.execute() diff --git a/tests/standalone_tests/controller/services/test_analysis_service.py b/tests/standalone_tests/controller/services/test_analysis_service.py index e5b88b02b..63625225e 100644 --- a/tests/standalone_tests/controller/services/test_analysis_service.py +++ b/tests/standalone_tests/controller/services/test_analysis_service.py @@ -43,3 +43,34 @@ def validate(**kwargs): assert entered["value"] assert seen["dto"] is request assert seen["aggregator"] is aggregator + + +def test_queued_analysis_validates_and_creates_strategy_after_admission(monkeypatch): + events = [] + + @asynccontextmanager + async def slot(*, wait_for_capacity): + assert wait_for_capacity + events.append("admit") + yield + + class Strategy: + async def execute(self): + events.append("execute") + return "{}" + + def validate(**kwargs): + events.append("validate") + + def strategy(dto): + events.append("strategy") + return Strategy() + + monkeypatch.setattr(analysis_service, "analysis_slot", slot) + monkeypatch.setattr( + analysis_service, "get_worker_landscape_aggregator", lambda: object() + ) + monkeypatch.setattr(analysis_service, "validate_analysis_request", validate) + monkeypatch.setattr(analysis_service, "get_algorithm_execution_strategy", strategy) + assert asyncio.run(run_analysis(analysis_request(), wait_for_capacity=True)) == "{}" + assert events == ["admit", "validate", "strategy", "execute"] diff --git a/tests/standalone_tests/controller/test_analysis_concurrency.py b/tests/standalone_tests/controller/test_analysis_concurrency.py index 952eaf22d..5a8d9f8ca 100644 --- a/tests/standalone_tests/controller/test_analysis_concurrency.py +++ b/tests/standalone_tests/controller/test_analysis_concurrency.py @@ -221,3 +221,44 @@ async def scenario(): release.set() assert healthy, "healthcheck did not return while analyses held the executor" + + +def test_queued_analysis_waits_for_http_capacity_without_blocking_health(monkeypatch): + set_max_concurrent_experiments(1) + + async def scenario(): + entered = asyncio.Event() + release = asyncio.Event() + calls = [] + + class Strategy: + async def execute(self): + calls.append("execute") + entered.set() + await release.wait() + return "{}" + + _patch_entrypoint(monkeypatch, lambda dto: Strategy()) + from tests.standalone_tests.controller.analysis_support import analysis_request + + async with controller_client(app) as client: + http = asyncio.create_task(client.post("/analysis", json=ANALYSIS_BODY)) + await entered.wait() + queued = asyncio.create_task( + analysis_service.run_analysis( + analysis_request(), wait_for_capacity=True + ) + ) + await asyncio.sleep(0) + assert not queued.done() + assert len(calls) == 1 + assert (await client.get("/livez")).status_code == 200 + assert ( + await client.post("/analysis", json=ANALYSIS_BODY) + ).status_code == 503 + release.set() + assert (await http).status_code == 200 + assert await queued == "{}" + assert len(calls) == 2 + + asyncio.run(scenario()) From 71fea70102fc3106960b77fd3d8e1f90c89a41ac Mon Sep 17 00:00:00 2001 From: ThanKarab Date: Fri, 13 Nov 2026 14:00:16 +0200 Subject: [PATCH 3/6] feat(rabbitmq): add analysis message consumer --- exaflow/controller/__init__.py | 9 + exaflow/controller/http/request_body_limit.py | 3 +- exaflow/controller/logger.py | 27 +- .../services/api/analysis_request_dtos.py | 6 +- exaflow/controller/services/api/limits.py | 4 + .../controller/services/rabbitmq/__init__.py | 1 + .../controller/services/rabbitmq/consumer.py | 516 ++++++++++++ .../controller/services/rabbitmq/settings.py | 38 + exaflow/logging_utils.py | 24 + exaflow/worker/utils/logger.py | 23 +- examples/rabbitmq_analysis.py | 76 ++ pyproject.toml | 1 + .../controller/analysis_support.py | 16 + .../controller/rabbitmq/__init__.py | 0 .../controller/rabbitmq/test_consumer.py | 796 ++++++++++++++++++ .../api/test_analysis_request_dtos.py | 30 + .../controller/test_http_contract.py | 33 + .../standalone_tests/test_request_logging.py | 124 +++ uv.lock | 125 ++- 19 files changed, 1812 insertions(+), 40 deletions(-) create mode 100644 exaflow/controller/services/api/limits.py create mode 100644 exaflow/controller/services/rabbitmq/__init__.py create mode 100644 exaflow/controller/services/rabbitmq/consumer.py create mode 100644 exaflow/controller/services/rabbitmq/settings.py create mode 100644 exaflow/logging_utils.py create mode 100644 examples/rabbitmq_analysis.py create mode 100644 tests/standalone_tests/controller/rabbitmq/__init__.py create mode 100644 tests/standalone_tests/controller/rabbitmq/test_consumer.py create mode 100644 tests/standalone_tests/controller/services/api/test_analysis_request_dtos.py create mode 100644 tests/standalone_tests/test_request_logging.py diff --git a/exaflow/controller/__init__.py b/exaflow/controller/__init__.py index 32f806f48..f8fae74ce 100644 --- a/exaflow/controller/__init__.py +++ b/exaflow/controller/__init__.py @@ -26,6 +26,14 @@ def _ensure_grpc_message_config(cfg: AttrDict) -> None: cfg.grpc = grpc_cfg +def _ensure_log_levels(cfg: AttrDict) -> None: + for name in ("log_level", "framework_log_level"): + value = cfg.get(name) + if isinstance(value, str): + value = value.strip() + cfg[name] = "INFO" if value is None or value == "" else value + + # Initializing the configurations from the config file config = None if config_file := os.getenv("EXAFLOW_CONTROLLER_CONFIG_FILE"): @@ -36,5 +44,6 @@ def _ensure_grpc_message_config(cfg: AttrDict) -> None: config = AttrDict(envtoml.load(fp)) _ensure_grpc_message_config(config) +_ensure_log_levels(config) worker_landscape_aggregator = None diff --git a/exaflow/controller/http/request_body_limit.py b/exaflow/controller/http/request_body_limit.py index b9209382d..cacef287e 100644 --- a/exaflow/controller/http/request_body_limit.py +++ b/exaflow/controller/http/request_body_limit.py @@ -2,8 +2,7 @@ from fastapi import HTTPException -# Uvicorn does not limit the request body. 16 MiB is the limit Quart applied. -MAX_REQUEST_BODY_BYTES = 16 * 1024 * 1024 +from exaflow.controller.services.api.limits import MAX_REQUEST_BODY_BYTES class RequestBodyTooLargeError(HTTPException): diff --git a/exaflow/controller/logger.py b/exaflow/controller/logger.py index 17df9c975..5f61bba5f 100644 --- a/exaflow/controller/logger.py +++ b/exaflow/controller/logger.py @@ -2,37 +2,24 @@ from exaflow.controller import BACKGROUND_LOGGER_NAME from exaflow.controller import config as ctrl_config +from exaflow.logging_utils import request_logger def get_request_logger(request_id): """ Used for logging information produced after an endpoint request. """ - logger = ( - init_logger(request_id) - if not logging.getLogger(request_id).hasHandlers() - else logging.getLogger(request_id) - ) - return logger + return init_logger(request_id) def init_logger(request_id, log_level=None): - logger = logging.getLogger(request_id) - formatter = logging.Formatter( - f"%(asctime)s - %(levelname)s - %(module)s.%(funcName)s(%(lineno)d) - [{ctrl_config.federation}] - [exaflow-controller] - [{ctrl_config.node_identifier}] - [{request_id}] - %(message)s" + return request_logger( + "exaflow.controller.requests", + request_id, + f"%(asctime)s - %(levelname)s - %(module)s.%(funcName)s(%(lineno)d) - [{ctrl_config.federation}] - [exaflow-controller] - [{ctrl_config.node_identifier}] - [%(request_id)s] - %(message)s", + log_level or ctrl_config.log_level, ) - # StreamHandler - sh = logging.StreamHandler() - sh.setFormatter(formatter) - logger.addHandler(sh) - if log_level: - logger.setLevel(log_level) - else: - logger.setLevel(ctrl_config.log_level) - - return logger - def get_background_service_logger() -> logging.Logger: """ diff --git a/exaflow/controller/services/api/analysis_request_dtos.py b/exaflow/controller/services/api/analysis_request_dtos.py index d9d58ea3f..fb7579c69 100644 --- a/exaflow/controller/services/api/analysis_request_dtos.py +++ b/exaflow/controller/services/api/analysis_request_dtos.py @@ -1,6 +1,7 @@ from abc import ABC from enum import Enum from enum import unique +from typing import Annotated from typing import Any from typing import Dict from typing import List @@ -9,6 +10,9 @@ from pydantic import BaseModel from pydantic import ConfigDict from pydantic import Field +from pydantic import StringConstraints + +RequestID = Annotated[str, StringConstraints(pattern=r"^[A-Za-z0-9_-]{1,128}$")] @unique @@ -46,7 +50,7 @@ class AnalysisAlgorithmDTO(ImmutableBaseModel): class AnalysisRequestDTO(BaseModel): model_config = ConfigDict(extra="forbid") - request_id: Optional[str] = None + request_id: Optional[RequestID] = None inputdata: AnalysisInputDataDTO preprocessing: Optional[List[AnalysisPreprocessingStepDTO]] = None algorithm: AnalysisAlgorithmDTO diff --git a/exaflow/controller/services/api/limits.py b/exaflow/controller/services/api/limits.py new file mode 100644 index 000000000..e81eafeb5 --- /dev/null +++ b/exaflow/controller/services/api/limits.py @@ -0,0 +1,4 @@ +"""Request limits shared by the HTTP and AMQP transports.""" + +# Uvicorn does not limit request bodies. Retain the former HTTP limit for both. +MAX_REQUEST_BODY_BYTES = 16 * 1024 * 1024 diff --git a/exaflow/controller/services/rabbitmq/__init__.py b/exaflow/controller/services/rabbitmq/__init__.py new file mode 100644 index 000000000..06649ba25 --- /dev/null +++ b/exaflow/controller/services/rabbitmq/__init__.py @@ -0,0 +1 @@ +"""Optional AMQP transport for the shared analysis service.""" diff --git a/exaflow/controller/services/rabbitmq/consumer.py b/exaflow/controller/services/rabbitmq/consumer.py new file mode 100644 index 000000000..2253e00cf --- /dev/null +++ b/exaflow/controller/services/rabbitmq/consumer.py @@ -0,0 +1,516 @@ +"""Bounded AMQP delivery handling; the broker owns unfinished requests.""" + +import asyncio +import json +import logging +import re +import time +import traceback +from collections import OrderedDict +from collections.abc import Callable + +import aio_pika +from pydantic import BaseModel +from pydantic import TypeAdapter +from pydantic import ValidationError + +from exaflow.controller import BACKGROUND_LOGGER_NAME +from exaflow.controller.services import analysis_service +from exaflow.controller.services.analysis_execution import max_concurrent_experiments +from exaflow.controller.services.api.analysis_request_dtos import AnalysisRequestDTO +from exaflow.controller.services.api.analysis_request_dtos import RequestID +from exaflow.controller.services.api.analysis_request_validator import BadRequest +from exaflow.controller.services.api.limits import ( + MAX_REQUEST_BODY_BYTES as MAX_REQUEST_BYTES, +) +from exaflow.controller.services.errors import ControllerShuttingDownError +from exaflow.controller.services.errors import GlobalWorkerUnavailableError +from exaflow.controller.services.errors import WorkerTaskTimeoutError +from exaflow.controller.services.errors import WorkerUnresponsiveError +from exaflow.controller.services.rabbitmq.settings import RabbitMQSettings +from exaflow.controller.worker_client.app import WorkerMessageTooLargeError +from exaflow.data_filters import FilterError +from exaflow.logging_utils import safe_log_value +from exaflow.smpc_cluster_communication import SMPCUsageError +from exaflow.worker_communication import BadUserInput +from exaflow.worker_communication import DataModelUnavailable +from exaflow.worker_communication import DatasetUnavailable +from exaflow.worker_communication import InsufficientDataError + +LOGGER = logging.getLogger(BACKGROUND_LOGGER_NAME) +REQUEST_ID_ADAPTER = TypeAdapter(RequestID) +REPLY_CACHE_TTL = 300 +REPLY_CACHE_ENTRIES = 128 +REPLY_CACHE_BYTES = 64 * 1024 * 1024 + + +def error_summary(exc: Exception) -> str: + if isinstance(exc, BaseExceptionGroup): + return error_summary(exc.exceptions[0]) + summary = type(exc).__name__ + if isinstance(exc, aio_pika.exceptions.ChannelClosed): + # Only broker channel errors carry diagnostic text. DeliveryError includes + # the published body, and connection errors can contain the AMQP URL. + details = " ".join(str(arg) for arg in exc.args) + details = re.sub(r"amqps?://\S+", "", details) + summary += ": " + safe_log_value(details)[:512] + return summary + + +class ReplyCache: + """Best-effort retention of bounded replies for unsettled deliveries.""" + + def __init__(self): + self.entries = OrderedDict() + self.size = 0 + + def discard(self, request_id): + entry = self.entries.pop(request_id, None) + if entry is not None: + self.size -= len(entry[1]) + + def prune(self): + now = time.monotonic() + for request_id, (expires, _) in list(self.entries.items()): + if expires <= now: + self.discard(request_id) + + def get(self, request_id): + self.prune() + entry = self.entries.get(request_id) + return None if entry is None else entry[1] + + def put(self, request_id, body): + self.prune() + self.discard(request_id) + if len(body) > REPLY_CACHE_BYTES: + return + self.entries[request_id] = (time.monotonic() + REPLY_CACHE_TTL, body) + self.size += len(body) + while len(self.entries) > REPLY_CACHE_ENTRIES or self.size > REPLY_CACHE_BYTES: + self.discard(next(iter(self.entries))) + + def clear(self): + self.entries.clear() + self.size = 0 + + +class FailureDetail(BaseModel): + code: str + message: str + + +class AnalysisReply(BaseModel): + status: str + result: object | None = None + error: FailureDetail | None = None + + +def failure(code: str, message: str) -> AnalysisReply: + return AnalysisReply( + status="failed", error=FailureDetail(code=code, message=message) + ) + + +class AnalysisConsumer: + def __init__(self, settings: RabbitMQSettings, landscape_ready: Callable[[], bool]): + self.settings = settings + self.landscape_ready = landscape_ready + self.connection = None + self.runner = None + self.iterator = None + self.finish = asyncio.Event() + self.retry_stop = asyncio.Event() + self.consuming = False + self.stopping = False + self.last_error = None + self.deliveries: set[asyncio.Task] = set() + self.executions: dict[str, asyncio.Task] = {} + self.completed = ReplyCache() + self.limit = max_concurrent_experiments() + self.handlers = asyncio.Semaphore(self.limit) + + def start(self): + if self.settings.enabled: + self.runner = asyncio.create_task(self._run(), name="rabbitmq-consumer") + + def health(self) -> dict: + return { + "enabled": self.settings.enabled, + "connected": self.connection is not None and not self.connection.is_closed, + "consuming": self.consuming, + "active_jobs": sum(not task.done() for task in self.executions.values()), + "last_error": self.last_error, + } + + async def _run(self): + delay = 1 + while not self.stopping: + try: + while not self.stopping and not self.landscape_ready(): + await asyncio.sleep(0.1) + if self.stopping: + return + self.connection = await aio_pika.connect( + self.settings.url, + timeout=self.settings.operation_timeout, + heartbeat=30, + client_properties={"connection_name": "exaflow-analysis-consumer"}, + ) + async with self.connection: + async with asyncio.timeout(self.settings.operation_timeout): + channel = await self.connection.channel( + publisher_confirms=True, + on_return_raises=True, + ) + await channel.set_qos(prefetch_count=self.limit) + await channel.declare_queue( + self.settings.dead_letter_queue, + durable=True, + arguments={ + "x-queue-type": "quorum", + "x-message-ttl": self.settings.dead_letter_ttl_ms, + "x-max-length-bytes": self.settings.queue_max_bytes, + "x-overflow": "reject-publish", + }, + ) + queue = await channel.declare_queue( + self.settings.request_queue, + durable=True, + arguments={ + "x-queue-type": "quorum", + "x-dead-letter-exchange": "", + "x-dead-letter-routing-key": self.settings.dead_letter_queue, + "x-dead-letter-strategy": "at-least-once", + "x-max-length-bytes": self.settings.queue_max_bytes, + "x-overflow": "reject-publish", + }, + ) + self.iterator = queue.iterator() + async with self.iterator as messages: + self.consuming = True + self.last_error = None + delay = 1 + LOGGER.info("RabbitMQ consumer connected") + async with asyncio.TaskGroup() as tasks: + monitor = tasks.create_task( + self._monitor_subscription(queue) + ) + try: + async for message in messages: + if self.stopping: + break + await self.handlers.acquire() + if self.stopping: + self.handlers.release() + break + task = asyncio.create_task( + self.handle(message, self.connection) + ) + self.deliveries.add(task) + task.add_done_callback(self._delivery_done) + finally: + monitor.cancel() + if self.stopping: + await self.finish.wait() + except asyncio.CancelledError: + raise + except Exception as exc: + # Exception text can include credentials or broker message bodies. + self.last_error = type(exc).__name__ + LOGGER.warning( + "RabbitMQ connection unavailable (%s)", error_summary(exc) + ) + finally: + self.consuming = False + self.iterator = None + self.completed.prune() + if not self.stopping: + await asyncio.sleep(delay) + delay = min(delay * 2, 30) + + async def _monitor_subscription(self, queue): + # aiormq consumes Basic.Cancel without waking QueueIterator. This process + # owns the sole subscription, so zero consumers means it must resubscribe. + while not self.stopping: + await asyncio.sleep(1) + self.completed.prune() + if self.stopping: + return + try: + declaration = await queue.declare( + timeout=self.settings.operation_timeout + ) + if declaration.consumer_count == 0: + raise ConnectionError("RabbitMQ subscription cancelled") + except Exception: + self.consuming = False + raise + + def _delivery_done(self, task): + self.deliveries.discard(task) + self.handlers.release() + if not task.cancelled() and task.exception() is not None: + LOGGER.error( + "RabbitMQ delivery task failed (%s)", type(task.exception()).__name__ + ) + + async def _execute(self, request: AnalysisRequestDTO) -> AnalysisReply: + try: + result = await analysis_service.run_analysis( + request, wait_for_capacity=True + ) + return AnalysisReply(status="succeeded", result=json.loads(result)) + except ControllerShuttingDownError: + raise + except InsufficientDataError: + return failure("insufficient_data", "Insufficient data for this analysis.") + except ( + BadRequest, + BadUserInput, + DataModelUnavailable, + DatasetUnavailable, + FilterError, + SMPCUsageError, + GlobalWorkerUnavailableError, + WorkerTaskTimeoutError, + WorkerUnresponsiveError, + WorkerMessageTooLargeError, + ) as exc: + return failure(type(exc).__name__, getattr(exc, "message", str(exc))) + except Exception as exc: + LOGGER.error( + "Analysis failed request_id=%s (%s)\n%s", + safe_log_value(request.request_id), + type(exc).__name__, + "".join(traceback.format_tb(exc.__traceback__)), + ) + return failure("internal_error", "Analysis execution failed.") + + def _encode_reply(self, reply): + body = reply.model_dump_json(exclude_none=True).encode() + if len(body) > self.settings.max_result_bytes: + body = ( + failure( + "result_too_large", "Result exceeds the configured message limit." + ) + .model_dump_json(exclude_none=True) + .encode() + ) + return body + + async def _execute_reply(self, request): + body = self._encode_reply(await self._execute(request)) + if not self.finish.is_set(): + self.completed.put(request.request_id, body) + return body + + def _request_reply(self, message): + if len(message.body) > MAX_REQUEST_BYTES: + return None, self._encode_reply( + failure("request_too_large", "Request exceeds 16 MiB.") + ) + try: + request_id = REQUEST_ID_ADAPTER.validate_python(message.correlation_id) + request = AnalysisRequestDTO.model_validate_json(message.body) + if request.request_id is not None and request.request_id != request_id: + raise ValueError("Conflicting request identifiers") + request.request_id = request_id + except (ValidationError, ValueError): + return None, self._encode_reply( + failure( + "invalid_request", + "Request does not match the analysis schema or identifiers.", + ) + ) + execution = self.executions.get(request_id) + if execution is None and message.redelivered: + body = self.completed.get(request_id) + if body is not None: + return None, body + if execution is None: + execution = asyncio.create_task( + self._execute_reply(request), name="queued-analysis" + ) + self.executions[request_id] = execution + return execution, None + + def _delivery_available(self, message, connection): + try: + return not connection.is_closed and not message.channel.is_closed + except aio_pika.exceptions.ChannelInvalidStateError: + return False + + async def _close_reply_channel(self, channel): + if channel is not None: + try: + await asyncio.wait_for(channel.close(), self.settings.operation_timeout) + except Exception as exc: + LOGGER.warning( + "RabbitMQ reply channel close failed (%s)", error_summary(exc) + ) + + async def _retry(self, delay): + try: + await asyncio.wait_for(self.retry_stop.wait(), delay) + except TimeoutError: + pass + + async def _settle(self, message, request_id, *, reject=False): + try: + await asyncio.wait_for( + message.reject(requeue=False) if reject else message.ack(), + self.settings.operation_timeout, + ) + except Exception as exc: + LOGGER.warning("RabbitMQ settlement interrupted (%s)", error_summary(exc)) + # A failed consumer-channel write is an actual transport failure. + # Retain completed replies while recovery requeues its deliveries. + try: + await asyncio.wait_for( + message.channel.close(), self.settings.operation_timeout + ) + except Exception as close_error: + LOGGER.warning( + "RabbitMQ delivery channel close failed (%s)", + error_summary(close_error), + ) + else: + self.completed.discard(request_id) + + async def handle(self, message, connection): + request_id = message.correlation_id + execution = None + channel = None + body = None + delay = 1 + try: + if not request_id or not message.reply_to: + await self._settle(message, request_id, reject=True) + return + while self._delivery_available(message, connection): + try: + channel = await asyncio.wait_for( + connection.channel( + publisher_confirms=True, on_return_raises=True + ), + self.settings.operation_timeout, + ) + # A missing queue closes this isolated reply channel only. + await channel.declare_queue( + message.reply_to, + passive=True, + timeout=self.settings.operation_timeout, + ) + if body is None: + execution, body = self._request_reply(message) + if execution is not None: + body = await asyncio.shield(execution) + if not self._delivery_available(message, connection): + return + await channel.default_exchange.publish( + aio_pika.Message( + body=body, + content_type="application/json", + correlation_id=request_id, + delivery_mode=aio_pika.DeliveryMode.PERSISTENT, + expiration=self.settings.reply_ttl_ms / 1000, + ), + routing_key=message.reply_to, + mandatory=True, + timeout=self.settings.publish_timeout, + ) + except aio_pika.exceptions.ChannelNotFoundEntity as exc: + LOGGER.warning( + "RabbitMQ reply destination unavailable request_id=%s (%s)", + safe_log_value(request_id), + error_summary(exc), + ) + await self._settle(message, request_id, reject=True) + return + except aio_pika.exceptions.DeliveryError as exc: + # Both mandatory returns and broker nacks are terminal for + # this request; neither invalidates the consuming channel. + LOGGER.warning( + "RabbitMQ reply rejected request_id=%s (%s)", + safe_log_value(request_id), + type(exc).__name__, + ) + await self._settle(message, request_id, reject=True) + return + except ControllerShuttingDownError: + return + except Exception as exc: + self.last_error = type(exc).__name__ + LOGGER.warning( + "RabbitMQ delivery interrupted request_id=%s (%s)", + safe_log_value(request_id), + error_summary(exc), + ) + await self._close_reply_channel(channel) + channel = None + if self.stopping or not self._delivery_available( + message, connection + ): + return + await self._retry(delay) + if self.stopping: + return + delay = min(delay * 2, 30) + else: + await self._settle(message, request_id) + return + except asyncio.CancelledError: + raise + finally: + if execution is not None: + + def release(task): + if self.executions.get(request_id) is task: + self.executions.pop(request_id) + if not task.cancelled(): + task.exception() + + if execution.done(): + release(execution) + else: + execution.add_done_callback(release) + await self._close_reply_channel(channel) + + async def drain(self): + self.stopping = True + self.retry_stop.set() + self.consuming = False + if self.iterator is not None: + try: + await asyncio.wait_for( + self.iterator.close(), self.settings.operation_timeout + ) + except Exception as exc: + LOGGER.warning("RabbitMQ intake stop failed (%s)", type(exc).__name__) + if self.deliveries: + await asyncio.wait( + self.deliveries, timeout=self.settings.shutdown_drain_timeout + ) + + async def close(self): + self.stopping = True + self.retry_stop.set() + self.finish.set() + tasks = set(self.deliveries) | set(self.executions.values()) + if self.runner is not None: + tasks.add(self.runner) + for task in tasks: + task.cancel() + if tasks: + await asyncio.wait(tasks, timeout=self.settings.operation_timeout) + if self.connection is not None and not self.connection.is_closed: + try: + await asyncio.wait_for( + self.connection.close(), self.settings.operation_timeout + ) + except Exception as exc: + LOGGER.warning( + "RabbitMQ connection close failed (%s)", type(exc).__name__ + ) + self.completed.clear() diff --git a/exaflow/controller/services/rabbitmq/settings.py b/exaflow/controller/services/rabbitmq/settings.py new file mode 100644 index 000000000..9dfed8043 --- /dev/null +++ b/exaflow/controller/services/rabbitmq/settings.py @@ -0,0 +1,38 @@ +from pydantic import BaseModel +from pydantic import ConfigDict +from pydantic import Field +from pydantic import TypeAdapter +from pydantic import model_validator + + +class RabbitMQSettings(BaseModel): + model_config = ConfigDict(extra="forbid", frozen=True, hide_input_in_errors=True) + + enabled: bool = False + url: str = Field(default="", repr=False) + request_queue: str = "exaflow.analysis.requests" + dead_letter_queue: str = "exaflow.analysis.dead" + max_result_bytes: int = Field(default=16 * 1024 * 1024, ge=1024) + queue_max_bytes: int = Field(default=256 * 1024 * 1024, ge=1024) + reply_ttl_ms: int = Field(default=86400000, ge=1) + dead_letter_ttl_ms: int = Field(default=604800000, ge=1) + operation_timeout: float = Field(default=5, gt=0, le=5) + publish_timeout: float = Field(default=30, gt=0) + shutdown_drain_timeout: float = Field(default=20, ge=0, le=20) + + @classmethod + def from_config(cls, values): + enabled = TypeAdapter( + bool, config=ConfigDict(hide_input_in_errors=True) + ).validate_python(values.get("enabled", False)) + return cls.model_validate(values) if enabled else cls() + + @model_validator(mode="after") + def validate_connection(self): + if self.enabled and not self.url.startswith(("amqp://", "amqps://")): + raise ValueError("Enabled RabbitMQ requires an AMQP URL") + if not self.request_queue or not self.dead_letter_queue: + raise ValueError("RabbitMQ queue names must be nonempty") + if self.request_queue == self.dead_letter_queue: + raise ValueError("Request and dead-letter queues must differ") + return self diff --git a/exaflow/logging_utils.py b/exaflow/logging_utils.py new file mode 100644 index 000000000..f8f7035c5 --- /dev/null +++ b/exaflow/logging_utils.py @@ -0,0 +1,24 @@ +"""Request context without permanently registering one logger per request.""" + +import logging +import threading + +_configuration_lock = threading.Lock() + + +def safe_log_value(value: object) -> str: + return "".join( + char if char.isprintable() else ascii(char)[1:-1] for char in str(value) + ) + + +def request_logger(name, request_id, format_string, level): + logger = logging.getLogger(name) + with _configuration_lock: + if not logger.handlers: + handler = logging.StreamHandler() + handler.setFormatter(logging.Formatter(format_string)) + logger.addHandler(handler) + logger.propagate = False + logger.setLevel(level) + return logging.LoggerAdapter(logger, {"request_id": safe_log_value(request_id)}) diff --git a/exaflow/worker/utils/logger.py b/exaflow/worker/utils/logger.py index 9770ac967..cdf754056 100644 --- a/exaflow/worker/utils/logger.py +++ b/exaflow/worker/utils/logger.py @@ -4,34 +4,25 @@ from contextvars import ContextVar from functools import wraps +from exaflow.logging_utils import request_logger from exaflow.worker import config as worker_config from exaflow.worker_communication import RequestIDNotFound LOGGING_ID_TASK_PARAM = "request_id" -_request_logger: ContextVar[logging.Logger | None] = ContextVar( +_request_logger: ContextVar[logging.LoggerAdapter | None] = ContextVar( "_request_logger", default=None ) def init_logger(request_id): - logger = logging.getLogger(request_id) - - formatter = logging.Formatter( - f"%(asctime)s - %(levelname)s - %(module)s.%(funcName)s(%(lineno)d) - [{worker_config.federation}] - [exaflow-{worker_config.role.lower()}] - [{worker_config.identifier}] - [{request_id}] - %(message)s" + return request_logger( + "exaflow.worker.requests", + request_id, + f"%(asctime)s - %(levelname)s - %(module)s.%(funcName)s(%(lineno)d) - [{worker_config.federation}] - [exaflow-{worker_config.role.lower()}] - [{worker_config.identifier}] - [%(request_id)s] - %(message)s", + worker_config.log_level, ) - # StreamHandler - sh = logging.StreamHandler() - sh.setFormatter(formatter) - for hdlr in logger.handlers[:]: # remove all old handlers - logger.removeHandler(hdlr) - logger.addHandler(sh) - logger.setLevel(worker_config.log_level) - logger.propagate = False - - return logger - def get_logger(): logger = _request_logger.get() diff --git a/examples/rabbitmq_analysis.py b/examples/rabbitmq_analysis.py new file mode 100644 index 000000000..b132fc2a0 --- /dev/null +++ b/examples/rabbitmq_analysis.py @@ -0,0 +1,76 @@ +"""Submit one existing analysis JSON payload; print the corresponding AMQP reply.""" + +import argparse +import asyncio +import json +import os +import sys +from pathlib import Path +from uuid import uuid4 + +import aio_pika + + +async def submit(args): + payload = json.loads(Path(args.request).read_text()) + request_id = payload.get("request_id") or uuid4().hex + payload["request_id"] = request_id + connection = await aio_pika.connect(os.environ["RABBITMQ_URL"], timeout=10) + async with connection: + channel = await connection.channel( + publisher_confirms=True, on_return_raises=True + ) + # One durable reply queue per logical request allows retrying this CLI with + # the same request_id after a client disconnect. The broker expires idle queues. + replies = await channel.declare_queue( + f"exaflow.analysis.replies.{request_id}", + durable=True, + arguments={ + "x-message-ttl": 86400000, + "x-expires": 172800000, + "x-max-length-bytes": 268435456, + "x-overflow": "reject-publish", + }, + ) + await channel.default_exchange.publish( + aio_pika.Message( + json.dumps(payload).encode(), + content_type="application/json", + correlation_id=request_id, + reply_to=replies.name, + delivery_mode=aio_pika.DeliveryMode.PERSISTENT, + ), + routing_key=args.queue, + mandatory=True, + timeout=10, + ) + async with asyncio.timeout(args.timeout): + async with replies.iterator() as messages: + async for message in messages: + if message.correlation_id != request_id: + raise ValueError("Unexpected reply correlation ID") + result = json.loads(message.body) + print(json.dumps(result, indent=2)) + await message.ack() + return 0 if result["status"] == "succeeded" else 1 + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("request", help="Path to an existing /analysis JSON payload") + parser.add_argument("--queue", default="exaflow.analysis.requests") + parser.add_argument("--timeout", type=float, default=600) + args = parser.parse_args() + try: + return asyncio.run(submit(args)) + except Exception as exc: + # Connection exception messages can contain URLs or authentication details. + print( + f"Analysis submission failed ({type(exc).__name__}). Check configuration and controller health.", + file=sys.stderr, + ) + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/pyproject.toml b/pyproject.toml index 0a8f9c139..752b30df1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,6 +25,7 @@ dependencies = [ [dependency-groups] controller = [ + "aio-pika>=9.5,<10", "dnspython>=2.8.0,<2.9.0", "fastapi>=0.115,<0.116", "starlette>=0.46,<0.47", diff --git a/tests/standalone_tests/controller/analysis_support.py b/tests/standalone_tests/controller/analysis_support.py index 75f5f21ed..9e27d8652 100644 --- a/tests/standalone_tests/controller/analysis_support.py +++ b/tests/standalone_tests/controller/analysis_support.py @@ -24,6 +24,22 @@ "algorithm": {"name": "sample_algo", "y": ["x"]}, } +VALID_REQUEST_IDS = ("a", "X" * 128, "Analysis_123-abC") +INVALID_REQUEST_IDS = ( + "", + "a" * 129, + "request id", + "request.id", + "request:id", + "requést", + "request\tid", + "request\rid", + "request\nid", + "request\n", + "request\x00id", + "request\u2028id", +) + class ResultModel(BaseModel): thread_name: str diff --git a/tests/standalone_tests/controller/rabbitmq/__init__.py b/tests/standalone_tests/controller/rabbitmq/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/standalone_tests/controller/rabbitmq/test_consumer.py b/tests/standalone_tests/controller/rabbitmq/test_consumer.py new file mode 100644 index 000000000..d603f4759 --- /dev/null +++ b/tests/standalone_tests/controller/rabbitmq/test_consumer.py @@ -0,0 +1,796 @@ +import asyncio +import json +from types import SimpleNamespace +from unittest.mock import AsyncMock +from unittest.mock import Mock + +import pytest +from aio_pika.exceptions import ChannelNotFoundEntity +from aio_pika.exceptions import ChannelPreconditionFailed +from aio_pika.exceptions import DeliveryError +from aio_pika.exceptions import PublishError +from pamqp.commands import Basic + +from exaflow.controller.services.rabbitmq import consumer as module +from exaflow.controller.services.rabbitmq.consumer import AnalysisConsumer +from exaflow.controller.services.rabbitmq.settings import RabbitMQSettings +from tests.standalone_tests.controller.analysis_support import ANALYSIS_BODY +from tests.standalone_tests.controller.analysis_support import INVALID_REQUEST_IDS +from tests.standalone_tests.controller.analysis_support import VALID_REQUEST_IDS + + +def resources(body=None, **metadata): + events = [] + message = SimpleNamespace( + body=json.dumps(ANALYSIS_BODY if body is None else body).encode(), + correlation_id="request-1", + reply_to="replies", + redelivered=False, + channel=SimpleNamespace(is_closed=False, close=AsyncMock()), + ack=AsyncMock(side_effect=lambda: events.append("ack")), + reject=AsyncMock(side_effect=lambda **kw: events.append("reject")), + ) + for key, value in metadata.items(): + setattr(message, key, value) + reply_channel = SimpleNamespace( + declare_queue=AsyncMock(), + default_exchange=SimpleNamespace( + publish=AsyncMock(side_effect=lambda *a, **kw: events.append("publish")) + ), + close=AsyncMock(), + ) + channel = SimpleNamespace( + is_closed=False, + channel=AsyncMock(return_value=reply_channel), + default_exchange=reply_channel.default_exchange, + reply_channel=reply_channel, + close=AsyncMock(), + ) + return message, channel, events + + +def consumer(**settings): + return AnalysisConsumer(RabbitMQSettings(**settings), lambda: True) + + +def test_success_confirms_reply_before_acknowledging(monkeypatch): + execute = AsyncMock(return_value='{"value": 7}') + monkeypatch.setattr(module.analysis_service, "run_analysis", execute) + message, channel, events = resources() + runtime = consumer(publish_timeout=45) + asyncio.run(runtime.handle(message, channel)) + assert events == ["publish", "ack"] + request = execute.call_args.args[0] + assert request.request_id == "request-1" + assert execute.call_args.kwargs == {"wait_for_capacity": True} + outgoing = channel.default_exchange.publish.call_args.args[0] + assert json.loads(outgoing.body) == {"status": "succeeded", "result": {"value": 7}} + assert outgoing.correlation_id == "request-1" + assert outgoing.delivery_mode == 2 + assert channel.default_exchange.publish.call_args.kwargs["timeout"] == 45 + assert not runtime.executions + + +@pytest.mark.parametrize("body", [{}, {**ANALYSIS_BODY, "request_id": "conflict"}]) +def test_invalid_body_replies_without_execution(monkeypatch, body): + execute = AsyncMock() + monkeypatch.setattr(module.analysis_service, "run_analysis", execute) + message, channel, events = resources(body) + asyncio.run(consumer().handle(message, channel)) + execute.assert_not_called() + reply = json.loads(channel.default_exchange.publish.call_args.args[0].body) + assert reply["error"]["code"] == "invalid_request" + assert events == ["publish", "ack"] + + +@pytest.mark.parametrize("request_id", INVALID_REQUEST_IDS[1:]) +@pytest.mark.parametrize("body_id", ["omitted", "null", "same"]) +def test_invalid_correlation_id_cannot_bypass_dto_validation( + monkeypatch, request_id, body_id +): + execute = AsyncMock() + monkeypatch.setattr(module.analysis_service, "run_analysis", execute) + body = dict(ANALYSIS_BODY) + if body_id != "omitted": + body["request_id"] = None if body_id == "null" else request_id + message, connection, events = resources(body, correlation_id=request_id) + runtime = consumer() + asyncio.run(runtime.handle(message, connection)) + reply = connection.default_exchange.publish.call_args.args[0] + assert json.loads(reply.body)["error"]["code"] == "invalid_request" + assert reply.correlation_id == request_id + execute.assert_not_called() + assert not runtime.executions + assert not runtime.completed.entries + assert events == ["publish", "ack"] + + +@pytest.mark.parametrize("request_id", INVALID_REQUEST_IDS) +def test_invalid_body_request_id_is_rejected_with_valid_correlation_id( + monkeypatch, request_id +): + execute = AsyncMock() + monkeypatch.setattr(module.analysis_service, "run_analysis", execute) + message, connection, _ = resources({**ANALYSIS_BODY, "request_id": request_id}) + runtime = consumer() + asyncio.run(runtime.handle(message, connection)) + reply = connection.default_exchange.publish.call_args.args[0] + assert json.loads(reply.body)["error"]["code"] == "invalid_request" + execute.assert_not_called() + assert not runtime.executions + assert not runtime.completed.entries + + +@pytest.mark.parametrize("request_id", VALID_REQUEST_IDS) +@pytest.mark.parametrize("body_id", ["omitted", "null", "same"]) +def test_valid_correlation_id_reaches_execution_and_reply_unchanged( + monkeypatch, request_id, body_id +): + execute = AsyncMock(return_value="{}") + monkeypatch.setattr(module.analysis_service, "run_analysis", execute) + body = dict(ANALYSIS_BODY) + if body_id != "omitted": + body["request_id"] = None if body_id == "null" else request_id + message, connection, _ = resources(body, correlation_id=request_id) + asyncio.run(consumer().handle(message, connection)) + assert execute.await_args.args[0].request_id == request_id + assert ( + connection.default_exchange.publish.call_args.args[0].correlation_id + == request_id + ) + + +def test_invalid_correlation_id_is_rejected_before_retained_reply_lookup(monkeypatch): + execute = AsyncMock() + monkeypatch.setattr(module.analysis_service, "run_analysis", execute) + message, connection, _ = resources(correlation_id="request\n", redelivered=True) + runtime = consumer() + runtime.completed.get = Mock( + side_effect=AssertionError("Invalid ID reached reply cache") + ) + asyncio.run(runtime.handle(message, connection)) + runtime.completed.get.assert_not_called() + execute.assert_not_called() + reply = connection.default_exchange.publish.call_args.args[0] + assert json.loads(reply.body)["error"]["code"] == "invalid_request" + + +@pytest.mark.parametrize("field", ["correlation_id", "reply_to"]) +@pytest.mark.parametrize("value", [None, ""]) +def test_missing_routing_metadata_dead_letters(field, value): + message, channel, events = resources(**{field: value}) + asyncio.run(consumer().handle(message, channel)) + assert events == ["reject"] + message.reject.assert_awaited_once_with(requeue=False) + channel.default_exchange.publish.assert_not_called() + + +def test_transient_publish_failure_retries_reply_without_reexecution(monkeypatch): + monkeypatch.setattr( + module.analysis_service, "run_analysis", AsyncMock(return_value="{}") + ) + message, channel, _ = resources() + channel.default_exchange.publish.side_effect = [TimeoutError(), None] + runtime = consumer() + runtime._retry = AsyncMock() + asyncio.run(runtime.handle(message, channel)) + message.ack.assert_awaited_once() + message.reject.assert_not_called() + message.channel.close.assert_not_called() + channel.close.assert_not_called() + assert channel.default_exchange.publish.await_count == 2 + module.analysis_service.run_analysis.assert_awaited_once() + + +def test_unroutable_reply_dead_letters_request(monkeypatch): + monkeypatch.setattr( + module.analysis_service, "run_analysis", AsyncMock(return_value="{}") + ) + message, channel, _ = resources() + channel.default_exchange.publish.side_effect = PublishError( + SimpleNamespace( + delivery=Basic.Return( + reply_code=312, reply_text="NO_ROUTE", routing_key="replies" + ) + ), + Basic.Return(), + ) + asyncio.run(consumer().handle(message, channel)) + message.reject.assert_awaited_once_with(requeue=False) + message.ack.assert_not_called() + + +def test_missing_destination_is_rejected_before_execution(monkeypatch): + execute = AsyncMock() + monkeypatch.setattr(module.analysis_service, "run_analysis", execute) + message, connection, _ = resources() + connection.reply_channel.declare_queue.side_effect = ChannelNotFoundEntity( + 404, "NOT_FOUND" + ) + asyncio.run(consumer().handle(message, connection)) + execute.assert_not_called() + connection.reply_channel.declare_queue.assert_awaited_once_with( + "replies", passive=True, timeout=5 + ) + message.reject.assert_awaited_once_with(requeue=False) + message.channel.close.assert_not_called() + + +def test_broker_nack_does_not_interrupt_another_delivery(monkeypatch): + async def scenario(): + running, release = asyncio.Event(), asyncio.Event() + + async def execute(request, **kwargs): + if request.request_id == "other-request": + running.set() + await release.wait() + return "{}" + + mock = AsyncMock(side_effect=execute) + monkeypatch.setattr(module.analysis_service, "run_analysis", mock) + runtime = consumer() + bad, connection, _ = resources() + good, other, _ = resources(correlation_id="other-request") + # Both source deliveries share the same consuming channel/connection. + good.channel = bad.channel + connection.channel.side_effect = [other.reply_channel, connection.reply_channel] + connection.default_exchange.publish.side_effect = DeliveryError( + None, Basic.Nack(delivery_tag=1) + ) + first = asyncio.create_task(runtime.handle(good, connection)) + await running.wait() + await runtime.handle(bad, connection) + bad.reject.assert_awaited_once_with(requeue=False) + assert not first.done() + bad.channel.close.assert_not_called() + connection.close.assert_not_called() + release.set() + await first + good.ack.assert_awaited_once() + assert sorted(call.args[0].request_id for call in mock.await_args_list) == [ + "other-request", + "request-1", + ] + + asyncio.run(scenario()) + + +@pytest.mark.parametrize("failure_stage", ["publish", "ack"]) +def test_redelivery_reuses_completed_result_after_handler_exits( + monkeypatch, failure_stage +): + async def scenario(): + execute = AsyncMock(return_value='{"value": 42}') + monkeypatch.setattr(module.analysis_service, "run_analysis", execute) + runtime = consumer() + first, connection, _ = resources() + + async def disconnected(*args, **kwargs): + connection.is_closed = True + raise ConnectionError("disconnected") + + if failure_stage == "publish": + connection.default_exchange.publish.side_effect = disconnected + else: + first.ack.side_effect = disconnected + await runtime.handle(first, connection) + assert not runtime.executions + assert runtime.completed.get("request-1") is not None + second, replacement, _ = resources(redelivered=True) + await runtime.handle(second, replacement) + execute.assert_awaited_once() + assert json.loads(replacement.default_exchange.publish.call_args.args[0].body)[ + "result" + ] == {"value": 42} + second.ack.assert_awaited_once() + assert runtime.completed.get("request-1") is None + + asyncio.run(scenario()) + + +def test_settled_request_id_can_start_a_fresh_execution(monkeypatch): + async def scenario(): + execute = AsyncMock(return_value="{}") + monkeypatch.setattr(module.analysis_service, "run_analysis", execute) + runtime = consumer() + for _ in range(2): + message, connection, _ = resources() + await runtime.handle(message, connection) + assert execute.await_count == 2 + + asyncio.run(scenario()) + + +@pytest.mark.parametrize( + "bound", ["REPLY_CACHE_TTL", "REPLY_CACHE_ENTRIES", "REPLY_CACHE_BYTES"] +) +def test_expired_or_evicted_reply_allows_recomputation(monkeypatch, bound): + async def scenario(): + monkeypatch.setattr(module, bound, 0) + execute = AsyncMock(return_value="{}") + monkeypatch.setattr(module.analysis_service, "run_analysis", execute) + runtime = consumer() + first, connection, _ = resources() + + async def disconnected(*args, **kwargs): + connection.is_closed = True + raise ConnectionError() + + connection.default_exchange.publish.side_effect = disconnected + await runtime.handle(first, connection) + second, replacement, _ = resources(redelivered=True) + await runtime.handle(second, replacement) + assert execute.await_count == 2 + second.ack.assert_awaited_once() + + asyncio.run(scenario()) + + +def test_cancelled_handler_retains_its_finishing_execution_for_redelivery(monkeypatch): + async def scenario(): + entered, release = asyncio.Event(), asyncio.Event() + + async def execute(*args, **kwargs): + entered.set() + await release.wait() + return "{}" + + mock = AsyncMock(side_effect=execute) + monkeypatch.setattr(module.analysis_service, "run_analysis", mock) + runtime = consumer() + first, connection, _ = resources() + handler = asyncio.create_task(runtime.handle(first, connection)) + await asyncio.wait_for(entered.wait(), 1) + execution = runtime.executions["request-1"] + handler.cancel() + with pytest.raises(asyncio.CancelledError): + await handler + assert not execution.cancelled() + release.set() + await execution + assert not runtime.executions + second, replacement, _ = resources(redelivered=True) + await runtime.handle(second, replacement) + mock.assert_awaited_once() + second.ack.assert_awaited_once() + + asyncio.run(scenario()) + + +def test_reply_cache_expires_and_evicts_by_bytes_and_count(monkeypatch): + now = [0] + monkeypatch.setattr(module.time, "monotonic", lambda: now[0]) + monkeypatch.setattr(module, "REPLY_CACHE_ENTRIES", 2) + monkeypatch.setattr(module, "REPLY_CACHE_BYTES", 5) + cache = module.ReplyCache() + cache.put("first", b"12") + cache.put("second", b"34") + cache.put("third", b"56") + assert cache.get("first") is None + assert cache.get("second") == b"34" + cache.put("fourth", b"7890") + assert cache.size == 4 + assert cache.get("second") is None + cache.put("oversize", b"123456") + assert cache.get("oversize") is None + now[0] = module.REPLY_CACHE_TTL + assert cache.get("fourth") is None + assert cache.size == 0 + + +def test_diagnostics_include_channel_reason_but_never_bodies_or_urls(): + reason = module.error_summary( + ExceptionGroup( + "connection", + [ + ChannelPreconditionFailed( + 406, + "PRECONDITION_FAILED inequivalent arg x-max-length\namqp://user:secret@host", + ) + ], + ) + ) + assert "406" in reason and "inequivalent arg" in reason + assert "secret" not in reason and "\n" not in reason + assert "secret" not in module.error_summary( + ConnectionError("amqp://user:secret@host") + ) + assert "secret" not in module.error_summary( + DeliveryError(SimpleNamespace(body=b"secret"), Basic.Nack()) + ) + + +def test_active_redelivery_shares_execution(monkeypatch): + async def scenario(): + entered, release = asyncio.Event(), asyncio.Event() + + async def execute(*args, **kw): + entered.set() + await release.wait() + return "{}" + + mock = AsyncMock(side_effect=execute) + monkeypatch.setattr(module.analysis_service, "run_analysis", mock) + runtime = consumer() + first, ch1, _ = resources() + second, ch2, _ = resources() + task1 = asyncio.create_task(runtime.handle(first, ch1)) + await entered.wait() + task2 = asyncio.create_task(runtime.handle(second, ch2)) + await asyncio.sleep(0) + release.set() + await asyncio.gather(task1, task2) + assert mock.await_count == 1 + first.ack.assert_awaited_once() + second.ack.assert_awaited_once() + + asyncio.run(scenario()) + + +def test_request_and_result_size_limits(monkeypatch): + message, channel, _ = resources() + monkeypatch.setattr(module, "MAX_REQUEST_BYTES", 1) + asyncio.run(consumer().handle(message, channel)) + assert ( + json.loads(channel.default_exchange.publish.call_args.args[0].body)["error"][ + "code" + ] + == "request_too_large" + ) + monkeypatch.setattr(module, "MAX_REQUEST_BYTES", 16 * 1024 * 1024) + monkeypatch.setattr( + module.analysis_service, + "run_analysis", + AsyncMock(return_value=json.dumps({"x": "x" * 2000})), + ) + message, channel, _ = resources() + asyncio.run(consumer(max_result_bytes=1024).handle(message, channel)) + assert ( + json.loads(channel.default_exchange.publish.call_args.args[0].body)["error"][ + "code" + ] + == "result_too_large" + ) + + +def test_unexpected_errors_do_not_expose_exception_details(monkeypatch): + monkeypatch.setattr( + module.analysis_service, + "run_analysis", + AsyncMock(side_effect=RuntimeError("secret payload")), + ) + message, channel, _ = resources() + asyncio.run(consumer().handle(message, channel)) + body = channel.default_exchange.publish.call_args.args[0].body + assert b"internal_error" in body and b"secret payload" not in body + + +def test_shutdown_error_is_redelivered_not_terminal(monkeypatch): + monkeypatch.setattr( + module.analysis_service, + "run_analysis", + AsyncMock(side_effect=module.ControllerShuttingDownError()), + ) + message, channel, _ = resources() + asyncio.run(consumer().handle(message, channel)) + message.ack.assert_not_called() + channel.default_exchange.publish.assert_not_called() + + +def test_drain_stops_deliveries_before_waiting_for_work(): + async def scenario(): + runtime = consumer(shutdown_drain_timeout=0) + runtime.iterator = SimpleNamespace(close=AsyncMock()) + job = asyncio.create_task(asyncio.Event().wait()) + runtime.deliveries.add(job) + await runtime.drain() + runtime.iterator.close.assert_awaited_once() + assert not job.cancelled() + await runtime.close() + assert job.cancelled() + + asyncio.run(scenario()) + + +def test_drain_interrupts_reply_retry_without_acknowledging(monkeypatch): + async def scenario(): + monkeypatch.setattr( + module.analysis_service, "run_analysis", AsyncMock(return_value="{}") + ) + message, connection, _ = resources() + connection.default_exchange.publish.side_effect = TimeoutError() + runtime = consumer(shutdown_drain_timeout=1) + retrying = asyncio.Event() + original_retry = runtime._retry + + async def retry(delay): + retrying.set() + await original_retry(delay) + + runtime._retry = retry + delivery = asyncio.create_task(runtime.handle(message, connection)) + runtime.deliveries.add(delivery) + await asyncio.wait_for(retrying.wait(), 1) + await asyncio.wait_for(runtime.drain(), 1) + await delivery + message.ack.assert_not_called() + message.reject.assert_not_called() + message.channel.close.assert_not_called() + await runtime.close() + + asyncio.run(scenario()) + + +def test_close_cleans_up_reply_channels_concurrently(monkeypatch): + async def scenario(): + runtime = consumer(operation_timeout=1) + entered, closing, release = asyncio.Event(), asyncio.Event(), asyncio.Event() + started_count, closing_count = 0, 0 + + async def execute(*args, **kwargs): + nonlocal started_count + started_count += 1 + if started_count == 3: + entered.set() + await asyncio.Event().wait() + + async def close(): + nonlocal closing_count + closing_count += 1 + if closing_count == 3: + closing.set() + await release.wait() + + monkeypatch.setattr(module.analysis_service, "run_analysis", execute) + for index in range(3): + message, connection, _ = resources(correlation_id=f"job-{index}") + connection.reply_channel.close.side_effect = close + runtime.deliveries.add( + asyncio.create_task(runtime.handle(message, connection)) + ) + await asyncio.wait_for(entered.wait(), 1) + shutdown = asyncio.create_task(runtime.close()) + try: + await asyncio.wait_for(closing.wait(), 1) + finally: + release.set() + await asyncio.wait_for(shutdown, 1) + assert all(task.done() for task in runtime.deliveries) + + asyncio.run(scenario()) + + +def test_broker_outage_retries_and_stays_independent_of_http(monkeypatch): + async def scenario(): + disconnected = asyncio.Event() + + async def connect(*args, **kwargs): + disconnected.set() + raise ConnectionError("amqp://private-credentials") + + monkeypatch.setattr(module.aio_pika, "connect", connect) + runtime = consumer(enabled=True, url="amqp://localhost") + runtime.start() + await disconnected.wait() + await asyncio.sleep(0) + assert runtime.health()["last_error"] == "ConnectionError" + assert not runtime.health()["consuming"] + await runtime.drain() + await runtime.close() + + asyncio.run(scenario()) + + +def test_configuration_validation_redacts_credentials(): + from pydantic import ValidationError + + with pytest.raises(ValidationError) as exc: + RabbitMQSettings(enabled=True, url="bad://secret-password") + assert "secret-password" not in str(exc.value) + + +def test_supervisor_bounds_delivery_handlers_and_drains_without_closing_active_replies( + monkeypatch, +): + async def scenario(): + monkeypatch.setattr(module, "max_concurrent_experiments", lambda: 2) + messages = [resources()[0] for _ in range(3)] + for index, message in enumerate(messages): + message.correlation_id = f"job-{index}" + released = asyncio.Event() + started = asyncio.Event() + completed = asyncio.Event() + calls = 0 + + async def execute(*args, **kwargs): + nonlocal calls + calls += 1 + if calls == 2: + started.set() + await released.wait() + if calls == 3: + completed.set() + return "{}" + + monkeypatch.setattr( + module.analysis_service, "run_analysis", AsyncMock(side_effect=execute) + ) + + class Iterator: + def __init__(self): + self.items = iter(messages) + self.closed = asyncio.Event() + + async def __aenter__(self): + return self + + async def __aexit__(self, *args): + await self.close() + + def __aiter__(self): + return self + + async def __anext__(self): + if self.closed.is_set(): + raise StopAsyncIteration + item = next(self.items, None) + if item is not None: + return item + await self.closed.wait() + raise StopAsyncIteration + + async def close(self): + self.closed.set() + + iterator = Iterator() + queue = SimpleNamespace( + iterator=lambda: iterator, + declare=AsyncMock(return_value=SimpleNamespace(consumer_count=1)), + ) + channel = SimpleNamespace( + set_qos=AsyncMock(), + declare_queue=AsyncMock(return_value=queue), + default_exchange=SimpleNamespace(publish=AsyncMock()), + close=AsyncMock(), + ) + + class Connection: + is_closed = False + + def __init__(self): + self.opened = 0 + + async def __aenter__(self): + return self + + async def __aexit__(self, *args): + await self.close() + + async def channel(self, **kwargs): + self.opened += 1 + return channel if self.opened == 1 else resources()[1].reply_channel + + async def close(self): + self.is_closed = True + + connection = Connection() + monkeypatch.setattr( + module.aio_pika, "connect", AsyncMock(return_value=connection) + ) + runtime = consumer(enabled=True, url="amqp://localhost") + runtime.start() + await asyncio.wait_for(started.wait(), 2) + channel.set_qos.assert_awaited_once_with(prefetch_count=2) + assert runtime.health()["active_jobs"] == 2 + assert len(runtime.deliveries) == 2 + released.set() + await asyncio.wait_for(completed.wait(), 2) + await runtime.drain() + assert not connection.is_closed + for message in messages: + message.ack.assert_awaited_once() + await runtime.close() + assert connection.is_closed + + asyncio.run(scenario()) + + +def test_cancelled_subscription_interrupts_waiting_iterator(monkeypatch): + async def scenario(): + runtime = consumer(enabled=True, url="amqp://localhost") + runtime.consuming = True + queue = SimpleNamespace( + declare=AsyncMock(return_value=SimpleNamespace(consumer_count=0)) + ) + interrupted = asyncio.Event() + + async def wait_for_message(): + try: + await asyncio.Event().wait() + finally: + interrupted.set() + + with pytest.raises(ExceptionGroup) as exc: + async with asyncio.TaskGroup() as tasks: + tasks.create_task(wait_for_message()) + tasks.create_task(runtime._monitor_subscription(queue)) + assert isinstance(exc.value.exceptions[0], ConnectionError) + assert interrupted.is_set() + assert not runtime.health()["consuming"] + + asyncio.run(scenario()) + + +def test_supervisor_reconnects_after_subscription_disappears(monkeypatch): + async def scenario(): + restored = asyncio.Event() + connections = [] + + class Iterator: + def __init__(self, second): + self.closed = asyncio.Event() + self.second = second + + async def __aenter__(self): + if self.second: + restored.set() + return self + + async def __aexit__(self, *args): + await self.close() + + def __aiter__(self): + return self + + async def __anext__(self): + await self.closed.wait() + raise StopAsyncIteration + + async def close(self): + self.closed.set() + + class Connection: + is_closed = False + + def __init__(self, second): + iterator = Iterator(second) + queue = SimpleNamespace( + iterator=lambda: iterator, + declare=AsyncMock( + return_value=SimpleNamespace(consumer_count=int(second)) + ), + ) + self.amqp_channel = SimpleNamespace( + set_qos=AsyncMock(), declare_queue=AsyncMock(return_value=queue) + ) + + async def __aenter__(self): + return self + + async def __aexit__(self, *args): + self.is_closed = True + + async def channel(self, **kwargs): + return self.amqp_channel + + async def close(self): + self.is_closed = True + + async def connect(*args, **kwargs): + connection = Connection(bool(connections)) + connections.append(connection) + return connection + + monkeypatch.setattr(module.aio_pika, "connect", connect) + runtime = consumer(enabled=True, url="amqp://localhost") + runtime.start() + try: + await asyncio.wait_for(restored.wait(), 4) + assert len(connections) == 2 + assert connections[0].is_closed + assert runtime.health()["consuming"] + finally: + await runtime.drain() + await runtime.close() + + asyncio.run(scenario()) diff --git a/tests/standalone_tests/controller/services/api/test_analysis_request_dtos.py b/tests/standalone_tests/controller/services/api/test_analysis_request_dtos.py new file mode 100644 index 000000000..0d2eb5952 --- /dev/null +++ b/tests/standalone_tests/controller/services/api/test_analysis_request_dtos.py @@ -0,0 +1,30 @@ +import pytest +from pydantic import ValidationError + +from exaflow.controller.services.api.analysis_request_dtos import AnalysisRequestDTO +from tests.standalone_tests.controller.analysis_support import ANALYSIS_BODY +from tests.standalone_tests.controller.analysis_support import INVALID_REQUEST_IDS +from tests.standalone_tests.controller.analysis_support import VALID_REQUEST_IDS + + +@pytest.mark.parametrize("request_id", VALID_REQUEST_IDS) +def test_request_ids_are_preserved(request_id): + request = AnalysisRequestDTO.model_validate( + {**ANALYSIS_BODY, "request_id": request_id} + ) + assert request.request_id == request_id + + +@pytest.mark.parametrize("request_id", INVALID_REQUEST_IDS) +def test_invalid_request_ids_are_rejected(request_id): + with pytest.raises(ValidationError) as exc: + AnalysisRequestDTO.model_validate({**ANALYSIS_BODY, "request_id": request_id}) + assert exc.value.errors()[0]["loc"] == ("request_id",) + + +@pytest.mark.parametrize("fields", [{}, {"request_id": None}]) +def test_request_id_can_be_omitted_or_null(fields): + assert ( + AnalysisRequestDTO.model_validate({**ANALYSIS_BODY, **fields}).request_id + is None + ) diff --git a/tests/standalone_tests/controller/test_http_contract.py b/tests/standalone_tests/controller/test_http_contract.py index 1a71f5867..d3a54d70f 100644 --- a/tests/standalone_tests/controller/test_http_contract.py +++ b/tests/standalone_tests/controller/test_http_contract.py @@ -11,6 +11,7 @@ import asyncio import json from types import SimpleNamespace +from unittest.mock import AsyncMock import pytest @@ -41,6 +42,8 @@ from exaflow.worker_communication import DataModelUnavailable from exaflow.worker_communication import DatasetUnavailable from exaflow.worker_communication import InsufficientDataError +from tests.standalone_tests.controller.analysis_support import INVALID_REQUEST_IDS +from tests.standalone_tests.controller.analysis_support import VALID_REQUEST_IDS from tests.standalone_tests.controller.asgi_client import controller_client TEXT_CONTENT_TYPE = "text/plain; charset=utf-8" @@ -287,6 +290,36 @@ def test_analysis_success_route_contract(monkeypatch): assert body == ANALYSIS_SUCCESS_BODY +@pytest.mark.parametrize("request_id", INVALID_REQUEST_IDS) +def test_invalid_request_id_returns_422_before_orchestration(monkeypatch, request_id): + execute = AsyncMock() + monkeypatch.setattr(analysis_service, "run_analysis", execute) + status, content_type, body = asyncio.run( + _post("/analysis", json_body={**ANALYSIS_REQUEST, "request_id": request_id}) + ) + assert status == 422 + assert content_type == TEXT_CONTENT_TYPE + assert "body.request_id:" in body + execute.assert_not_called() + + +@pytest.mark.parametrize( + "fields", + [{}, {"request_id": None}, *({"request_id": value} for value in VALID_REQUEST_IDS)], +) +def test_valid_and_optional_request_ids_reach_orchestration_unchanged( + monkeypatch, fields +): + execute = AsyncMock(return_value=ANALYSIS_SUCCESS_BODY) + monkeypatch.setattr(analysis_service, "run_analysis", execute) + status, _, body = asyncio.run( + _post("/analysis", json_body={**ANALYSIS_REQUEST, **fields}) + ) + assert status == 200 + assert body == ANALYSIS_SUCCESS_BODY + assert execute.await_args.args[0].request_id == fields.get("request_id") + + def test_analysis_body_that_does_not_match_the_model_returns_422(monkeypatch): _stub_analysis(monkeypatch) diff --git a/tests/standalone_tests/test_request_logging.py b/tests/standalone_tests/test_request_logging.py new file mode 100644 index 000000000..3adcba7e7 --- /dev/null +++ b/tests/standalone_tests/test_request_logging.py @@ -0,0 +1,124 @@ +import logging +import os +import subprocess +import sys +import threading +from concurrent.futures import ThreadPoolExecutor + +import pytest + +from exaflow import AttrDict +from exaflow.controller import _ensure_log_levels +from exaflow.controller.logger import get_request_logger +from exaflow.worker.utils import logger as worker_logging +from exaflow.worker_communication import RequestIDNotFound + + +@pytest.mark.parametrize("value", [None, "", " "]) +def test_empty_controller_log_levels_default_to_info(value): + cfg = AttrDict({"log_level": value, "framework_log_level": value}) + _ensure_log_levels(cfg) + assert cfg.log_level == "INFO" + assert cfg.framework_log_level == "INFO" + + +@pytest.mark.parametrize("level", ["DEBUG", "WARNING", 0]) +def test_explicit_controller_log_levels_are_preserved(level): + cfg = AttrDict({"log_level": level, "framework_log_level": level}) + _ensure_log_levels(cfg) + assert cfg.log_level == level + assert cfg.framework_log_level == level + + +def test_controller_logging_works_without_log_level_environment_variables(): + env = dict(os.environ) + for name in ("LOG_LEVEL", "FRAMEWORK_LOG_LEVEL", "EXAFLOW_CONTROLLER_CONFIG_FILE"): + env.pop(name, None) + result = subprocess.run( + [ + sys.executable, + "-c", + """ +import logging +from exaflow.controller import config +from exaflow.controller.http.loggers import loggers +from exaflow.controller.logger import get_request_logger + +assert config.log_level == config.framework_log_level == "INFO" +assert loggers["handlers"]["framework"]["level"] == "INFO" +assert get_request_logger("unset-level").logger.level == logging.INFO +""", + ], + env=env, + capture_output=True, + text=True, + timeout=30, + ) + assert result.returncode == 0, result.stderr + + +@pytest.mark.parametrize("factory", [get_request_logger, worker_logging.init_logger]) +def test_unique_requests_reuse_logger_and_handler(factory, monkeypatch): + # Other suites can attach pytest capture handlers while importing workers. + # Exercise our handler initialization independently of that instrumentation. + base = factory("setup").logger + monkeypatch.setattr(base, "handlers", []) + first = factory("first") + registry_size = len(logging.Logger.manager.loggerDict) + handlers = list(first.logger.handlers) + for index in range(1000): + adapter = factory(f"unique-analysis-{index}") + assert adapter.logger is first.logger + assert adapter.extra["request_id"] == f"unique-analysis-{index}" + assert len(logging.Logger.manager.loggerDict) == registry_size + assert first.logger.handlers == handlers + assert len(handlers) == 1 + + +@pytest.mark.parametrize("factory", [get_request_logger, worker_logging.init_logger]) +def test_concurrent_requests_keep_their_logging_context(factory, monkeypatch): + records = [] + barrier = threading.Barrier(2) + base = factory("setup").logger + + class Capture(logging.Handler): + def emit(self, record): + records.append((record.request_id, record.getMessage())) + + monkeypatch.setattr(base, "handlers", [Capture()]) + + @worker_logging.initialise_logger + def emit(request_id): + adapter = factory(request_id) + barrier.wait(timeout=5) + adapter.info("message for %s", request_id) + assert worker_logging.get_logger().extra["request_id"] == request_id + + with ThreadPoolExecutor(max_workers=2) as pool: + list(pool.map(emit, ["alpha", "beta"])) + assert sorted(records) == [ + ("alpha", "message for alpha"), + ("beta", "message for beta"), + ] + + +def test_worker_context_is_restored_after_nested_failure(): + @worker_logging.initialise_logger + def inner(request_id): + assert worker_logging.get_logger().extra["request_id"] == "inner" + raise ValueError("failed") + + @worker_logging.initialise_logger + def outer(request_id): + with pytest.raises(ValueError): + inner("inner") + assert worker_logging.get_logger().extra["request_id"] == "outer" + + outer("outer") + with pytest.raises(RequestIDNotFound): + worker_logging.get_logger() + + +@pytest.mark.parametrize("factory", [get_request_logger, worker_logging.init_logger]) +def test_control_characters_are_escaped_in_request_context(factory): + assert factory("id\n\r\t\u2028").extra["request_id"] == r"id\n\r\t\u2028" diff --git a/uv.lock b/uv.lock index 7e504bc00..47c5cde65 100644 --- a/uv.lock +++ b/uv.lock @@ -1,10 +1,36 @@ version = 1 -revision = 5 +revision = 3 requires-python = "==3.11.*" [manifest] constraints = [{ name = "python-dateutil", specifier = "<3" }] +[[package]] +name = "aio-pika" +version = "9.6.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "aiormq" }, + { name = "yarl" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/96/63/56354526f2e6e915c93bee6e4dedb35888fe82d6bc1a19f35f5a77e795ff/aio_pika-9.6.2.tar.gz", hash = "sha256:c49e9246080dc8ffa1bb0e4aca407bf3d8ad78c3ee3a93df88b68fe65d7a49b9", size = 70851, upload-time = "2026-03-22T19:03:20.878Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/25/05/256fa313f48bed075056d13593b92ce804be05d75f4f312be24edb82860a/aio_pika-9.6.2-py3-none-any.whl", hash = "sha256:2a5478af920d169795071c9c09c7542cd8cdece60438cf7804533dcbcce93b7f", size = 56269, upload-time = "2026-03-22T19:03:19.558Z" }, +] + +[[package]] +name = "aiormq" +version = "6.9.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pamqp" }, + { name = "yarl" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/6c/0e/db90154d52d399108903fe603e5110a533c42065180265dd003788264080/aiormq-6.9.4.tar.gz", hash = "sha256:0e7c01b662804e1cc7ace9a17794e8c1192a27fc2afa96162362a6e61ae8e8ef", size = 49232, upload-time = "2026-03-23T09:18:19.493Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6c/48/1ce3773f392f02ceda37aee168fade9d725483a9592c202d06044cd093ff/aiormq-6.9.4-py3-none-any.whl", hash = "sha256:726a8586695e863fba68cf88842065ab12348c9438dcebdfc9d0bddaf6083277", size = 32166, upload-time = "2026-03-23T09:18:17.523Z" }, +] + [[package]] name = "annotated-types" version = "0.8.0" @@ -226,6 +252,7 @@ aggregation-server = [ { name = "pyarrow" }, ] controller = [ + { name = "aio-pika" }, { name = "dnspython" }, { name = "fastapi" }, { name = "starlette" }, @@ -275,6 +302,7 @@ aggregation-server = [ { name = "pyarrow", specifier = ">=25.0.1,<26.0.0" }, ] controller = [ + { name = "aio-pika", specifier = ">=9.5,<10" }, { name = "dnspython", specifier = ">=2.8.0,<2.9.0" }, { name = "fastapi", specifier = ">=0.115,<0.116" }, { name = "starlette", specifier = ">=0.46,<0.47" }, @@ -475,6 +503,35 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/18/53/84099323c2ec4be98d935f63c033ac4151ee83836ca1050ede3b3aadf155/joblib-1.6.0-py3-none-any.whl", hash = "sha256:3dbbf9f6e4b592a2357b854608e980fe6390d131d7a82f011a377ef2ebef7aba", size = 306115, upload-time = "2026-08-31T09:39:02.298Z" }, ] +[[package]] +name = "multidict" +version = "7.2.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/7b/6b/29ef8e752f5854dcc267ded0652c475bec99b86f255957a0d4fcad657b18/multidict-7.2.1.tar.gz", hash = "sha256:e8af3e6f1bdbc31fd762b094133b478b89988eaebc63cbe82b1d9e3bbd15cede", size = 369994, upload-time = "2026-10-24T20:41:35.457Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ef/24/4347a7502ad47cb05a1b8d4e219514b4f0b1e38d4e4734c053c20328949f/multidict-7.2.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:fd24c4425fdb8e822601335602375c6d0cb020f7a368e1d5842eb7925a9c8748", size = 173141, upload-time = "2026-10-24T20:40:02.481Z" }, + { url = "https://files.pythonhosted.org/packages/8d/f2/2a8fcfe4c2c435101f9227b7f7ea81d786ffef993cadd2394e6926020ca9/multidict-7.2.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:cecb2c778f00a65c6ca01410dd194d0e797a9eebb7c6dea0db87718719f293c7", size = 97197, upload-time = "2026-10-24T20:40:06.534Z" }, + { url = "https://files.pythonhosted.org/packages/7a/57/39be4ac4fa9c20e3af7dd7e89adf7dd404b4547fb4453c5c96082e197b9c/multidict-7.2.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:0d852d3d9638f852158d7de7200dd77f7ff8d77fc8f3d396032f3d847883f631", size = 104843, upload-time = "2026-10-24T20:40:05.829Z" }, + { url = "https://files.pythonhosted.org/packages/63/da/6a5c78daa34e458ced17573b7527395c4c39ebb253220274382d361a1493/multidict-7.2.1-cp311-cp311-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:76a4f5132b8dc29eb3c768ee9b47e132a1d24fbc2e9259854c01af9beaff1d49", size = 490713, upload-time = "2026-10-24T20:40:41.715Z" }, + { url = "https://files.pythonhosted.org/packages/8a/75/b8252bc6411f7f4b9649b83e143789b7bc1dda44f656b09ce74f894ca027/multidict-7.2.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:295a15ccb8cfde9351dd6062b80af4e839460636b3c46813b4aa443905ded86a", size = 504189, upload-time = "2026-10-24T20:40:04.042Z" }, + { url = "https://files.pythonhosted.org/packages/c0/45/38e8191d0a7c7b56015dfc4c2910b731aa719c5ad3f369b2357567f03434/multidict-7.2.1-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:987f65f6c9321402bc7012f6c6d09ba501b88ddb1640f5b0b4a19f2caafe24a5", size = 465834, upload-time = "2026-10-24T20:40:08.058Z" }, + { url = "https://files.pythonhosted.org/packages/9f/55/378869573886f7efcac24d2c698938dadf973e0da40b812c8b0df847fc46/multidict-7.2.1-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:36b868758fb11cfa748c788b793994e73660abec90f83c780a678539c2c468bf", size = 522423, upload-time = "2026-10-24T20:40:07.427Z" }, + { url = "https://files.pythonhosted.org/packages/af/c5/8936fe02c11196994ac05c4876ad62336a66ef353ea2d1a2ca3adce71678/multidict-7.2.1-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5b944091cfa7da4f7a9bd2587569cd90c94b89ec4721f8e87fe62504f2432b67", size = 544512, upload-time = "2026-10-24T20:40:43.362Z" }, + { url = "https://files.pythonhosted.org/packages/d0/38/b8403d97ae5273002fa664c3c440c5cc186082fa0ba20fcc93885d13a686/multidict-7.2.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3b7542fad2706595c1b1853b46f2c3c4ae5cc5a6252b7225f294d607a29e1bdf", size = 510054, upload-time = "2026-10-24T20:40:05.525Z" }, + { url = "https://files.pythonhosted.org/packages/18/5d/473e1436ea8577cdfb42371c0507a244d51f8d8ef08bea0f9e6e63906478/multidict-7.2.1-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:363e5db93fc3f65f3218a79fe108d217b9d264181a4e993c77552bfe275d0393", size = 452816, upload-time = "2026-10-24T20:40:09.49Z" }, + { url = "https://files.pythonhosted.org/packages/1e/b7/67376e6821fb28c21de1fb37731efd2e3a6a523bdefc4af6997bd043da7a/multidict-7.2.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:5deadf03fb4a7842696d551f9f9a06dd4445bb43ca06b5f9736e4e1885561333", size = 490712, upload-time = "2026-10-24T20:40:09.097Z" }, + { url = "https://files.pythonhosted.org/packages/1a/89/5144d8b36f133109ff36408089a475b9a3a3c1840176a39a9a8fb67a0000/multidict-7.2.1-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:0ec95a02f9f08246cc7f1d85ac425e4d5c233fd3f098f5be1b2cca16aa690186", size = 462856, upload-time = "2026-10-24T20:40:44.771Z" }, + { url = "https://files.pythonhosted.org/packages/ff/7f/38310388c4c5e351dfc419a04e4c7a26cde14b0e1019bc72bbd1b0d95613/multidict-7.2.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:3d6fce7cc4ef63d9a7633cbae43e737798412805a4c3ba876cdf472dc6ff9c51", size = 490108, upload-time = "2026-10-24T20:40:06.965Z" }, + { url = "https://files.pythonhosted.org/packages/ce/0d/94162600e5bbf24bf33398f3b33ae2f78211375313dbdaaf1513b2554a66/multidict-7.2.1-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:37b161535e92776788957a165fcf38a80b33c1efde7b18cab287db4e4574df78", size = 508935, upload-time = "2026-10-24T20:40:10.882Z" }, + { url = "https://files.pythonhosted.org/packages/1d/40/a0cae932493346cb0d442ffa380e65491fdae8df338d8cdf2f39695c0947/multidict-7.2.1-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:e47d058c2db95010f5d2e1f6037acebdfb0e2fa4507650e1041437985c91db38", size = 447620, upload-time = "2026-10-24T20:40:10.548Z" }, + { url = "https://files.pythonhosted.org/packages/5e/a5/6ec5cd1c15574aca277ab08920f7e78a6a1e1c15912bfaf59b48683230f7/multidict-7.2.1-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:7238f7135481926d52a94c18a8cca633a9e72c1cbfccd5796548298a479b0656", size = 523896, upload-time = "2026-10-24T20:40:46.258Z" }, + { url = "https://files.pythonhosted.org/packages/ae/60/d164a2583548561f7e596f13bbd1c3492ea33a77d499d3ceaa2acf76d850/multidict-7.2.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:c10ac5e3e76837180656079008f71add8339b00c3c26314092bcd1e8e449049f", size = 502961, upload-time = "2026-10-24T20:40:08.732Z" }, + { url = "https://files.pythonhosted.org/packages/30/6b/474aee537e0c87210ce8f4fd21558b461e8a8e74c79440de036022a9eeda/multidict-7.2.1-cp311-cp311-win32.whl", hash = "sha256:2b4d0c4e13d329ffee73d46128e32dd3b4aeb2f9c6ac9c8a89eb28d3b82dcea8", size = 84159, upload-time = "2026-10-24T20:40:12.222Z" }, + { url = "https://files.pythonhosted.org/packages/1f/02/41d769148f32c9bd535fd0ddd946c4e42f38acaddf7899f3bf49d9e67703/multidict-7.2.1-cp311-cp311-win_amd64.whl", hash = "sha256:a789e3d27fc5665cc0477f10a35a89383313e4c9cf8b5e6af7ea71f039cdcc0e", size = 94895, upload-time = "2026-10-24T20:40:12.036Z" }, + { url = "https://files.pythonhosted.org/packages/23/be/e04fcf29542d0db25cdec79acbe5662481c3ba6b760353c393f9c46156f2/multidict-7.2.1-cp311-cp311-win_arm64.whl", hash = "sha256:2022bd3194968a027f64ed16f9779e62e71a823a9df3971a7328c1d45dfeb9ab", size = 92384, upload-time = "2026-10-24T20:40:47.827Z" }, + { url = "https://files.pythonhosted.org/packages/9d/96/1aaad0e5df33f1f5af21775c411349c0234a6edd581ea403bc81919a6eab/multidict-7.2.1-py3-none-any.whl", hash = "sha256:08ea94663d9e7c25d14d78222f7aee906c472d4837766bec62477c38f6e419dc", size = 28145, upload-time = "2026-10-24T20:40:59.159Z" }, +] + [[package]] name = "nodeenv" version = "1.11.0" @@ -511,6 +568,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/63/34/ba1c580383c9eada3711951fef0795c80b829a078d72188184bcab9dd527/packaging-26.3-py3-none-any.whl", hash = "sha256:d7193f7c8e4e93f444fde0262bf90af30e16fa0ad0ad44cb553c87339b23cd1c", size = 129956, upload-time = "2026-08-04T18:15:27.159Z" }, ] +[[package]] +name = "pamqp" +version = "3.3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/fb/62/35bbd3d3021e008606cd0a9532db7850c65741bbf69ac8a3a0d8cfeb7934/pamqp-3.3.0.tar.gz", hash = "sha256:40b8795bd4efcf2b0f8821c1de83d12ca16d5760f4507836267fd7a02b06763b", size = 30993, upload-time = "2024-01-12T20:37:25.085Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ac/8d/c1e93296e109a320e508e38118cf7d1fc2a4d1c2ec64de78565b3c445eb5/pamqp-3.3.0-py2.py3-none-any.whl", hash = "sha256:c901a684794157ae39b52cbf700db8c9aae7a470f13528b9d7b4e5f7202f8eb0", size = 33848, upload-time = "2024-01-12T20:37:21.359Z" }, +] + [[package]] name = "pandas" version = "2.3.3" @@ -579,6 +645,32 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/45/e2/bbb7129c9e7999a6b8ee9cca3b66486c25c423ab5a75f34071798b74ce94/pre_commit-4.6.2-py2.py3-none-any.whl", hash = "sha256:e2dde9a75d3bce11bd3831c26d134df00a2803c1d818be6a0383c3dcda25dc4e", size = 226202, upload-time = "2026-08-10T22:07:16.942Z" }, ] +[[package]] +name = "propcache" +version = "0.5.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/b3/9a/9fbf4e4ec0c2d7f1c32519fff782ef467859b8faa9fbc5331a96f6395d43/propcache-0.5.4.tar.gz", hash = "sha256:ff6b113f50bc066a698db5d944d2c6dc7507168dd3341e255a8892fd0715a558", size = 61545, upload-time = "2026-09-16T00:17:14.386Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1e/40/14b21e505b7921617466576423f188a5c9caddfdaa1cf4b2b8a83d8fe216/propcache-0.5.4-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:897d1ddf6716e8f47200f7aad9a0efa6cc7586df66c6defa572f9eab379c078e", size = 86393, upload-time = "2026-09-16T00:14:06.9Z" }, + { url = "https://files.pythonhosted.org/packages/e7/4b/5a52e1a7b43563f7d408814194bb23cc8bf214eb6b86639b667a33a8d0d0/propcache-0.5.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:9cbfff4423eef4cc6cafc021469641a2b835f610b2647a6c5281903e21b8670d", size = 50431, upload-time = "2026-09-16T00:14:08.025Z" }, + { url = "https://files.pythonhosted.org/packages/05/cf/b5248180bf056cc76acc60c9c6e8c0ebbfdbd1c6cffd31fd14996927b7c8/propcache-0.5.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:3fc24f209c1b7f7f688b66b98293954f5504279760999b58920ee12dd8471c1d", size = 52116, upload-time = "2026-09-16T00:14:09.114Z" }, + { url = "https://files.pythonhosted.org/packages/86/a8/7c6cd6bfead1a11f2e411e688640e6d26574cb0bde7dcaa7423b0b65ed7a/propcache-0.5.4-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:62530ca89187827e4a4fe733f971abe81a7542eeea48ff61995f19b64d7199c8", size = 238729, upload-time = "2026-09-16T00:14:10.357Z" }, + { url = "https://files.pythonhosted.org/packages/5c/b4/442715b2e980df51be52d203549279e027728f24c80b00b5e525e31cd5ea/propcache-0.5.4-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:56fc3f7599528db40b1efa0889a620116e2704144495273d66066e8164e45838", size = 246121, upload-time = "2026-09-16T00:14:11.735Z" }, + { url = "https://files.pythonhosted.org/packages/bc/5d/df0684fc2b1732a01a7bec26d7897369022712422d25b09c37ce7dbc88a2/propcache-0.5.4-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:4f2d880ff60f45898f4acfa152aac8d04e3ee627d90ff4003491bf92239d5757", size = 251735, upload-time = "2026-09-16T00:14:13.053Z" }, + { url = "https://files.pythonhosted.org/packages/c7/06/519a5ebb48b6f94beb48396e55c905f12246a25c3a3608a7ec7bceabf50e/propcache-0.5.4-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6e9368e87a3efc285e559131092c5db643eb8e56de4ee42064d5baec22ef2bb5", size = 235381, upload-time = "2026-09-16T00:14:14.398Z" }, + { url = "https://files.pythonhosted.org/packages/3c/07/1e0a9bb310830f2245edbd5cd3c6d24a783c053c4efd8e08e386e513c940/propcache-0.5.4-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:004e685b315646c410771836e72a44f143bbe624f29653a42687815069a303d5", size = 208973, upload-time = "2026-09-16T00:14:15.715Z" }, + { url = "https://files.pythonhosted.org/packages/62/5c/9324fab27d6088eecc47fe4332bf7aaf8c1ded93c36f558391e8a06d41a7/propcache-0.5.4-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:594eb4c6ec35e7179b058481f4e9f02521b56de16fa577c4b85c76fb1bf8a9f8", size = 233897, upload-time = "2026-09-16T00:14:17.25Z" }, + { url = "https://files.pythonhosted.org/packages/9c/a3/570d92fc952eae93b676f3a1568f4b89264102abd3c982ab6a9ebec58dcf/propcache-0.5.4-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:2dba2f02d2d5c09ef8a0e6c1a42aeaa451f4be9898cb00b04fe98717da2eb23b", size = 223512, upload-time = "2026-09-16T00:14:18.87Z" }, + { url = "https://files.pythonhosted.org/packages/65/47/26810d889d89bba31db397e6a88f8984af775f5ed6bad0a29dce84324cff/propcache-0.5.4-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:c3ef2818d63bc86071e9d2989ae75a1bc32b8f7059cfd9f5abbbee70c32e2ed6", size = 239043, upload-time = "2026-09-16T00:14:20.366Z" }, + { url = "https://files.pythonhosted.org/packages/89/2d/f9c47691aa024c8299a3afacd78d22a01ab57eb627b481b6089708e71017/propcache-0.5.4-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:dd2ac8f5b643454c2cc6b6118b13da16e88f4a6434fc3ba61aca384029f04f36", size = 208218, upload-time = "2026-09-16T00:14:21.801Z" }, + { url = "https://files.pythonhosted.org/packages/74/6b/d510c0c378cabbf9d0ac7b663af6d00f2e9074073d93b20c85f24aa5c071/propcache-0.5.4-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:4054acf80d40456a0537f2913b349718649d8d6458a14ab7f48d0ce28c30869d", size = 240301, upload-time = "2026-09-16T00:14:23.121Z" }, + { url = "https://files.pythonhosted.org/packages/3d/80/c80f6adaaa1e51f0db2dce8c9b3714d94ec45e358a21f9a1910b10b40a80/propcache-0.5.4-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:40e94adb1e7d39ff28a8bd8d8b8fbd1df6b9f40976dbe379134f1ce058e532dd", size = 230785, upload-time = "2026-09-16T00:14:24.458Z" }, + { url = "https://files.pythonhosted.org/packages/d9/e2/c32a7df3f39caa7f11b2eb37ea5b6960a6946f2bc7c4b8ff97bbdf6d6b6e/propcache-0.5.4-cp311-cp311-win32.whl", hash = "sha256:9f86f7259efe2c951f43e57d471c9b41daa5bfc7db9f67189059cf1ae6d77fd9", size = 42747, upload-time = "2026-09-16T00:14:25.715Z" }, + { url = "https://files.pythonhosted.org/packages/0a/8a/3db6a3543d8101263b4c52978b6276a04ead2caff2c5ab880d934f47bd89/propcache-0.5.4-cp311-cp311-win_amd64.whl", hash = "sha256:e904d4d01f36bd6e197590be1533c44e06058771e0746dd073a8ebb3ef880858", size = 46268, upload-time = "2026-09-16T00:14:26.996Z" }, + { url = "https://files.pythonhosted.org/packages/41/07/5222e2665bbf6e45847492ecbf3b9f3e4975a0ae300e5fd465df7d48ce55/propcache-0.5.4-cp311-cp311-win_arm64.whl", hash = "sha256:d42a9a856a4a6e2f6c10f1318c07e7daa498d6593abe745c71dae4521a26ca39", size = 43547, upload-time = "2026-09-16T00:14:28.143Z" }, + { url = "https://files.pythonhosted.org/packages/f5/cd/785c64ed382f3f04201870267b02783f63b4678c2acfddc177a3ebcc2727/propcache-0.5.4-py3-none-any.whl", hash = "sha256:62c60aec739ed00124573cce1178138fd690c7676352d67a37328c1cf51d7468", size = 16338, upload-time = "2026-09-16T00:17:13.106Z" }, +] + [[package]] name = "protobuf" version = "7.36.2" @@ -1047,3 +1139,34 @@ sdist = { url = "https://files.pythonhosted.org/packages/70/aa/bff50daee0488e16d wheels = [ { url = "https://files.pythonhosted.org/packages/bb/a9/c10d8a44a29fb94a18b43fdf11f38caca53ae7c9406f11a72b1cb8f624f1/virtualenv-21.14.13-py3-none-any.whl", hash = "sha256:5fbb04138a4861580b78ba9110d529a26bc48fcc93c8ec36bc0202eeed96a788", size = 5557094, upload-time = "2026-10-22T20:10:15.096Z" }, ] + +[[package]] +name = "yarl" +version = "1.25.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "idna" }, + { name = "multidict" }, + { name = "propcache" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/75/16/e8be8e2fb175bbf41a0680381a319f1199fae256588241a2ac8677eafb49/yarl-1.25.1.tar.gz", hash = "sha256:03dd38de09bc213e9a8b29761eec33ee1d5318dac0e49d8af36e4d27830e23a7", size = 246245, upload-time = "2026-09-15T19:35:02.264Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/83/b3/2cea721d495ca57f8f414aa4867ee263f486b274e07463158cf52f02ba7f/yarl-1.25.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:9d693bf4bf534e9ba3ae2780cfd577f5135629f7b5ac653490859d0b77864865", size = 143794, upload-time = "2026-09-15T19:30:26.946Z" }, + { url = "https://files.pythonhosted.org/packages/55/e6/cd145cff8e5cf60b8b3c41fbecfa2a45028a8dec3fbc52bec03595ff3d3b/yarl-1.25.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:ab2054c5531af2a9ba7b69b8ec91e4f884420e83a8c5e579b013084cb57e5e5d", size = 103993, upload-time = "2026-09-15T19:30:28.661Z" }, + { url = "https://files.pythonhosted.org/packages/ce/7c/fbb40fe2d53747c40aa36a9e2bd2178a202f942bda0b670f3306d4aefbce/yarl-1.25.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:564fdc7085d2245ab84f88882fdb1d6ac0723124bff6ded35bfb1c00f812630d", size = 104010, upload-time = "2026-09-15T19:30:31.069Z" }, + { url = "https://files.pythonhosted.org/packages/f0/0b/5a516f70641092283f57cf3670bdb75e7327bcc0dcb5038697e4dfbfd569/yarl-1.25.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:acae6b45d1ace09b6ba3876da43b88366ef368f73b988c7f57e14231753d4420", size = 116444, upload-time = "2026-09-15T19:30:32.894Z" }, + { url = "https://files.pythonhosted.org/packages/b6/8a/d1a627f827b0a404ae0f5647cab0534959081cde13b0756a783469fb3b5e/yarl-1.25.1-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:1fb2a01ba8cd9c5d2c5dc1ec35e0fc951d04b4f037541d4ac090c993ce58b3d7", size = 107565, upload-time = "2026-09-15T19:30:35.188Z" }, + { url = "https://files.pythonhosted.org/packages/aa/cf/c8e0aaec886840a6c4480fe44eaec7cd4319f79a563b79321473be79c56f/yarl-1.25.1-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e92b6bcc741b86d67606c40d3cb9c7cc8e6c737f81e31f4a94efc204456c92e3", size = 125006, upload-time = "2026-09-15T19:30:37.1Z" }, + { url = "https://files.pythonhosted.org/packages/50/26/0cce366d54a93cdc8342965dc7663385e4db161625cc1e8b18e786753d24/yarl-1.25.1-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:72c34ac7ad4314c19362d5ce27626dcc8429bd30bbf8c179f4234078851f9492", size = 128717, upload-time = "2026-09-15T19:30:38.904Z" }, + { url = "https://files.pythonhosted.org/packages/95/0c/a71501bbc1a674ff72c4d6c2b75f4d9a5af819f5244c3a7558080a8802c5/yarl-1.25.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d5add7b4ca7afeea91d52e4d4e4db3b1fe9885b71f07054560d8c4296b7441a2", size = 117728, upload-time = "2026-09-15T19:30:40.809Z" }, + { url = "https://files.pythonhosted.org/packages/e7/6f/c3267ca01defeed9ed9c4ff9b17bd54915432c405945233265b707475d1c/yarl-1.25.1-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:def538065f9e4d4cf1ae164bd59aba00dfa84f03923e0de4c3788f252d6bcd17", size = 116224, upload-time = "2026-09-15T19:30:42.818Z" }, + { url = "https://files.pythonhosted.org/packages/8f/69/fad57ee52d648431718ee0f1f68966a99c1352c3924688ffbdcc9d3fe51a/yarl-1.25.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:0a191bfdb30a79b98e5d175d75285f9fcb78bf0e46ba5efda042e1c72071a0de", size = 116299, upload-time = "2026-09-15T19:30:44.966Z" }, + { url = "https://files.pythonhosted.org/packages/2a/d4/6a8c1e29f33338687ca278cba0a8fbf6525a322c2c02a9a500ccbe041152/yarl-1.25.1-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:71f42c5b9a948c113bbdebfa544598321431d064ff959d32e99b1feb61d68345", size = 108625, upload-time = "2026-09-15T19:30:46.874Z" }, + { url = "https://files.pythonhosted.org/packages/e8/d2/3a35ae791c9cb6522c106923ff25c3230d999091e5e65511bca23bbd9914/yarl-1.25.1-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:72849d892954be4d09e569b8b831ac39ce58417fedc767d4308a0fe542018a40", size = 124515, upload-time = "2026-09-15T19:30:49.082Z" }, + { url = "https://files.pythonhosted.org/packages/be/fd/2b022109a6b4af0f7dc371cf7500af380b0d4f034010243e1b0ce218dc93/yarl-1.25.1-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:efb01a106f971cb3752856bca2318bbdf7f01bd8823779c461586cbe5ffd5258", size = 115711, upload-time = "2026-09-15T19:30:51.208Z" }, + { url = "https://files.pythonhosted.org/packages/18/59/f7586271136c3ddb0126bbfe661844699369b76b555fe38c4efe86870b2e/yarl-1.25.1-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:a1daf47cd95a7c3a63456336bc5aaa8c86dd3a47d07ed3d0e76132ae4666a5a1", size = 122751, upload-time = "2026-09-15T19:30:53.535Z" }, + { url = "https://files.pythonhosted.org/packages/a0/0b/07f7a2d881f7e16c385b47fc1753382600847cad305dcc7fa0c25828acf8/yarl-1.25.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:9489e6abf47ba37f332075a91444c7cfedb03e6ce99fbb2f116bfe1ce810da3b", size = 117983, upload-time = "2026-09-15T19:30:55.277Z" }, + { url = "https://files.pythonhosted.org/packages/aa/9d/8cdceec66a9b940700cb45931741403f045b162afd79bcb93c41cadd0972/yarl-1.25.1-cp311-cp311-win_amd64.whl", hash = "sha256:d7306dee25b8a0e737363f347362b875094b4dc4e367311470656ae420fdbf8e", size = 102894, upload-time = "2026-09-15T19:30:57.606Z" }, + { url = "https://files.pythonhosted.org/packages/17/f1/7ec357db1d3ad2863542d71e8fe64a126bbec17b134cd7d30951196809a5/yarl-1.25.1-cp311-cp311-win_arm64.whl", hash = "sha256:abb1384477f5901d436b5d2e5465954de46ea6098f59163d243660b5c4461d35", size = 98609, upload-time = "2026-09-15T19:30:59.705Z" }, + { url = "https://files.pythonhosted.org/packages/54/22/318c7980066769c6bcd9221ed2248294f5698811da099013098c670565ed/yarl-1.25.1-py3-none-any.whl", hash = "sha256:681c758b0490f9e96b78e5fa8e8dc6e648e9185bb6eaebe73183c33ea0c445f3", size = 63617, upload-time = "2026-09-15T19:34:59.616Z" }, +] From 57610e9fa88642fb931ba1ec0a5c8ce68281e974 Mon Sep 17 00:00:00 2001 From: ThanKarab Date: Fri, 13 Nov 2026 14:02:35 +0200 Subject: [PATCH 4/6] feat(controller): manage rabbitmq lifecycle and health --- exaflow/controller/http/app.py | 20 ++++- exaflow/controller/http/endpoints.py | 20 +++++ .../worker_landscape_aggregator.py | 2 + .../controller/rabbitmq/test_lifecycle.py | 81 +++++++++++++++++++ .../controller/shutdown_probe_app.py | 72 ++++++++++++++++- .../controller/test_uvicorn_shutdown.py | 19 ++++- 6 files changed, 209 insertions(+), 5 deletions(-) create mode 100644 tests/standalone_tests/controller/rabbitmq/test_lifecycle.py diff --git a/exaflow/controller/http/app.py b/exaflow/controller/http/app.py index 039cff067..801e3f5ba 100644 --- a/exaflow/controller/http/app.py +++ b/exaflow/controller/http/app.py @@ -2,18 +2,36 @@ from fastapi import FastAPI +from exaflow.controller import config as ctrl_config from exaflow.controller.http import endpoints from exaflow.controller.http.error_handlers import register_error_handlers from exaflow.controller.http.request_body_limit import RequestBodyLimitMiddleware +from exaflow.controller.services import get_worker_landscape_aggregator +from exaflow.controller.services.rabbitmq.consumer import AnalysisConsumer +from exaflow.controller.services.rabbitmq.settings import RabbitMQSettings @asynccontextmanager async def lifespan(_app: FastAPI): + # Validate configuration before starting resources, without connecting to AMQP. + settings = RabbitMQSettings.from_config(ctrl_config.get("rabbitmq", {})) await endpoints.startup() + consumer = AnalysisConsumer( + settings, + lambda: get_worker_landscape_aggregator().initial_refresh_complete.is_set(), + ) + _app.state.rabbitmq = consumer try: + consumer.start() yield finally: - await endpoints.shutdown() + try: + await consumer.drain() + finally: + try: + await endpoints.shutdown() + finally: + await consumer.close() app = FastAPI( diff --git a/exaflow/controller/http/endpoints.py b/exaflow/controller/http/endpoints.py index d9adba253..0250d4eae 100644 --- a/exaflow/controller/http/endpoints.py +++ b/exaflow/controller/http/endpoints.py @@ -5,7 +5,9 @@ from fastapi import APIRouter from fastapi import Body +from fastapi import Request from fastapi import Response +from fastapi.responses import JSONResponse from exaflow.controller.http.loggers import loggers from exaflow.controller.services import analysis_service @@ -142,6 +144,24 @@ async def livez(): return Response() +@router.get("/healthcheck/rabbitmq") +async def rabbitmq_health(request: Request): + consumer = getattr(request.app.state, "rabbitmq", None) + state = ( + consumer.health() + if consumer + else { + "enabled": False, + "connected": False, + "consuming": False, + "active_jobs": 0, + "last_error": None, + } + ) + available = not state["enabled"] or (state["connected"] and state["consuming"]) + return JSONResponse(state, status_code=200 if available else 503) + + @router.post("/analysis") async def run_analysis(analysis_request: AnalysisRequestDTO): # Each algorithm has its own result model, and the strategy returns it diff --git a/exaflow/controller/services/worker_landscape_aggregator/worker_landscape_aggregator.py b/exaflow/controller/services/worker_landscape_aggregator/worker_landscape_aggregator.py index e684cba7d..6c51a21f2 100644 --- a/exaflow/controller/services/worker_landscape_aggregator/worker_landscape_aggregator.py +++ b/exaflow/controller/services/worker_landscape_aggregator/worker_landscape_aggregator.py @@ -287,6 +287,7 @@ def __init__( self._stop_event = threading.Event() self._update_loop_thread = None self._loaded_workers = set() + self.initial_refresh_complete = threading.Event() def update(self): """ @@ -317,6 +318,7 @@ def update(self): ) self._set_new_registries(worker_registry, dmr) + self.initial_refresh_complete.set() self._logger.debug( f"Workers:{[worker_info.id for worker_info in self.get_workers()]}" diff --git a/tests/standalone_tests/controller/rabbitmq/test_lifecycle.py b/tests/standalone_tests/controller/rabbitmq/test_lifecycle.py new file mode 100644 index 000000000..aa57fd4a4 --- /dev/null +++ b/tests/standalone_tests/controller/rabbitmq/test_lifecycle.py @@ -0,0 +1,81 @@ +import asyncio +from types import SimpleNamespace + +from exaflow import AttrDict +from exaflow.controller.http import app as app_module +from exaflow.controller.services.rabbitmq.consumer import AnalysisConsumer +from exaflow.controller.services.rabbitmq.settings import RabbitMQSettings +from tests.standalone_tests.controller.asgi_client import controller_client + + +def test_lifespan_stops_consumer_before_controller_shutdown_and_always_closes( + monkeypatch, +): + events = [] + + class Consumer: + def __init__(self, settings, readiness): + assert not settings.enabled + + def start(self): + events.append("start-consumer") + + async def drain(self): + events.append("drain-consumer") + + async def close(self): + events.append("close-consumer") + + async def startup(): + events.append("start-controller") + + async def shutdown(): + events.append("stop-controller") + raise RuntimeError("shutdown failed") + + monkeypatch.setattr(app_module, "ctrl_config", AttrDict({})) + monkeypatch.setattr(app_module, "AnalysisConsumer", Consumer) + monkeypatch.setattr(app_module.endpoints, "startup", startup) + monkeypatch.setattr(app_module.endpoints, "shutdown", shutdown) + + async def scenario(): + import pytest + + with pytest.raises(RuntimeError, match="shutdown failed"): + async with app_module.lifespan(app_module.app): + pass + + previous = getattr(app_module.app.state, "rabbitmq", None) + try: + asyncio.run(scenario()) + finally: + app_module.app.state.rabbitmq = previous + assert events == [ + "start-controller", + "start-consumer", + "drain-consumer", + "stop-controller", + "close-consumer", + ] + + +def test_broker_health_does_not_fail_http_liveness(): + async def scenario(): + runtime = AnalysisConsumer( + RabbitMQSettings(enabled=True, url="amqp://localhost"), lambda: False + ) + previous = getattr(app_module.app.state, "rabbitmq", None) + app_module.app.state.rabbitmq = runtime + try: + async with controller_client(app_module.app) as client: + assert (await client.get("/livez")).status_code == 200 + response = await client.get("/healthcheck/rabbitmq") + assert response.status_code == 503 + assert response.json()["active_jobs"] == 0 + runtime.connection = SimpleNamespace(is_closed=False) + runtime.consuming = True + assert (await client.get("/healthcheck/rabbitmq")).status_code == 200 + finally: + app_module.app.state.rabbitmq = previous + + asyncio.run(scenario()) diff --git a/tests/standalone_tests/controller/shutdown_probe_app.py b/tests/standalone_tests/controller/shutdown_probe_app.py index 3045a4ba7..820c4dd51 100644 --- a/tests/standalone_tests/controller/shutdown_probe_app.py +++ b/tests/standalone_tests/controller/shutdown_probe_app.py @@ -3,14 +3,17 @@ It wires the controller's own shutdown pieces (``run_blocking``, the in-flight RPC registry and the ``endpoints.shutdown`` lifespan hook) around a gRPC call that never finishes by itself, and appends what happens to the file named by -``SHUTDOWN_PROBE_EVENTS``, one `` `` line each. +``SHUTDOWN_PROBE_EVENTS``, one `` `` line each. """ +import asyncio +import json import os import threading import time from concurrent import futures from contextlib import asynccontextmanager +from types import SimpleNamespace import grpc from fastapi import FastAPI @@ -21,11 +24,15 @@ from exaflow.controller.services.analysis_execution import run_blocking from exaflow.controller.services.analysis_execution import start_analysis_execution from exaflow.controller.services.controller_shutdown import register_inflight_rpc +from exaflow.controller.services.rabbitmq.consumer import AnalysisConsumer +from exaflow.controller.services.rabbitmq.consumer import AnalysisReply +from exaflow.controller.services.rabbitmq.settings import RabbitMQSettings +from tests.standalone_tests.controller.analysis_support import ANALYSIS_BODY def _event(name: str) -> None: with open(os.environ["SHUTDOWN_PROBE_EVENTS"], "a") as events: - events.write(f"{time.time():.3f} {name}\n") + events.write(f"{time.monotonic():.3f} {name}\n") class _Landscape: @@ -58,10 +65,19 @@ def _never_finishes(request, context): async def _lifespan(_app: FastAPI): set_worker_landscape_aggregator(_Landscape()) start_analysis_execution() + _app.state.rabbitmq = _QueueProbeConsumer( + RabbitMQSettings(shutdown_drain_timeout=0.1, operation_timeout=1), + lambda: True, + ) try: yield finally: - await endpoints.shutdown() + await _app.state.rabbitmq.drain() + _event("queue-drained") + try: + await endpoints.shutdown() + finally: + await _app.state.rabbitmq.close() app = FastAPI(lifespan=_lifespan) @@ -92,3 +108,53 @@ async def analysis(name: str): # Like the aggregation cleanup in the strategy: it runs after the # analysis thread ended, also when shutdown ended it. await run_blocking(_event, f"{name}:cleanup-ran") + + +class _QueueProbeConsumer(AnalysisConsumer): + async def _execute(self, request): + try: + await run_blocking(_analysis, request.request_id) + return AnalysisReply(status="succeeded", result={}) + finally: + await run_blocking(_event, f"{request.request_id}:cleanup-ran") + + +@app.post("/queue/{name}") +async def queue_analysis(name: str): + async def ack(): + _event(f"{name}:acked") + + async def close(): + _event(f"{name}:channel-closed") + + async def publish(*args, **kwargs): + _event(f"{name}:reply-published") + + async def declare(*args, **kwargs): + pass + + channel = SimpleNamespace( + default_exchange=SimpleNamespace(publish=publish), + declare_queue=declare, + close=close, + is_closed=False, + ) + + async def open_channel(**kwargs): + return channel + + message = SimpleNamespace( + body=json.dumps(ANALYSIS_BODY).encode(), + correlation_id=name, + reply_to="replies", + redelivered=False, + channel=SimpleNamespace(is_closed=False), + ack=ack, + ) + connection = SimpleNamespace(is_closed=False, channel=open_channel) + runtime = app.state.rabbitmq + await runtime.handlers.acquire() + task = asyncio.create_task(runtime.handle(message, connection)) + runtime.deliveries.add(task) + task.add_done_callback(runtime._delivery_done) + return {"submitted": name} diff --git a/tests/standalone_tests/controller/test_uvicorn_shutdown.py b/tests/standalone_tests/controller/test_uvicorn_shutdown.py index 3901db345..a03a02b77 100644 --- a/tests/standalone_tests/controller/test_uvicorn_shutdown.py +++ b/tests/standalone_tests/controller/test_uvicorn_shutdown.py @@ -99,7 +99,7 @@ def output(self): def terminate(self): """Send SIGTERM; return when it was sent.""" - sent_at = time.time() + sent_at = time.monotonic() self.process.send_signal(signal.SIGTERM) return sent_at @@ -162,3 +162,20 @@ def test_client_disconnect_during_the_drain_does_not_cancel_other_analyses(probe events = probe.events() assert returncode in CLEAN_EXIT, probe.output() assert events["a:rpc-cancelled"] - sent_at >= GRACEFUL_TIMEOUT - 1 + + +def test_queued_analysis_is_drained_cancelled_and_left_unacked(probe): + request = urllib.request.Request( + f"http://127.0.0.1:{probe.port}/queue/queued", data=b"", method="POST" + ) + urllib.request.urlopen(request, timeout=5).read() + deadline = time.monotonic() + 10 + while "queued:rpc-started" not in probe.events(): + assert time.monotonic() < deadline, probe.output() + time.sleep(0.05) + probe.terminate() + assert probe.exit_code() in CLEAN_EXIT, probe.output() + events = probe.events() + assert events["queue-drained"] <= events["queued:rpc-cancelled"] + assert events["queued:cleanup-ran"] >= events["queued:rpc-cancelled"] + assert "queued:acked" not in events From 1d6d3123e449c4ff4ca209cbfc8e839f43e31131 Mon Sep 17 00:00:00 2001 From: ThanKarab Date: Fri, 13 Nov 2026 14:04:22 +0200 Subject: [PATCH 5/6] feat(deployment): configure rabbitmq and provision test broker --- .deployment.sample.toml | 7 ++ .github/workflows/prod_env_tests.yml | 20 ++++ exaflow/controller/config.toml | 13 +++ kubernetes/templates/exaflow-controller.yaml | 19 +++ kubernetes/values.yaml | 15 +++ tasks.py | 4 + .../kind_configuration/kind_cluster.yaml | 3 + .../deployment_configs/kubernetes_values.yaml | 20 ++++ .../deployment_configs/rabbitmq.yaml | 87 ++++++++++++++ .../controller/rabbitmq/test_settings.py | 108 ++++++++++++++++++ .../standalone_tests/test_kill_controller.py | 35 ++++++ 11 files changed, 331 insertions(+) create mode 100644 tests/prod_env_tests/deployment_configs/rabbitmq.yaml create mode 100644 tests/standalone_tests/controller/rabbitmq/test_settings.py diff --git a/.deployment.sample.toml b/.deployment.sample.toml index f008d4d84..b54e1cbfc 100644 --- a/.deployment.sample.toml +++ b/.deployment.sample.toml @@ -18,6 +18,13 @@ server_port = 8080 [controller] port = 5100 +# Connect to an externally provisioned broker. Keep credentials out of version control. +[rabbitmq] +enabled = false +url = "${RABBITMQ_URL:-}" +request_queue = "exaflow.analysis.requests" +dead_letter_queue = "exaflow.analysis.dead" + [privacy] minimum_row_count = 10 protect_local_data = false diff --git a/.github/workflows/prod_env_tests.yml b/.github/workflows/prod_env_tests.yml index 223d7ad13..d80692a1b 100644 --- a/.github/workflows/prod_env_tests.yml +++ b/.github/workflows/prod_env_tests.yml @@ -147,6 +147,21 @@ jobs: - name: Print Helm Templates run: helm template kubernetes/ + - name: Deploy test RabbitMQ + run: | + set +x + password=$(openssl rand -hex 24) + echo "::add-mask::$password" + # Pipe the Secret directly to kubectl; never persist credentials in artifacts. + kubectl create secret generic exaflow-rabbitmq \ + --from-literal=username=exaflow_tests \ + --from-literal=password="$password" \ + --from-literal=url="amqp://exaflow_tests:$password@exaflow-rabbitmq:5672/" \ + --dry-run=client -o yaml | kubectl apply -f - + echo "RABBITMQ_TEST_URL=amqp://exaflow_tests:$password@127.0.0.1:5679/" >> "$GITHUB_ENV" + kubectl apply -f tests/prod_env_tests/deployment_configs/rabbitmq.yaml + kubectl rollout status statefulset/exaflow-rabbitmq --timeout=180s + - name: Deploy Helm run: helm install exaflow kubernetes/ --debug @@ -257,6 +272,11 @@ jobs: - name: Run production env tests run: uv run --no-sync pytest tests/prod_env_tests --verbosity=4 + - name: Run isolated RabbitMQ recovery test + env: + RABBITMQ_TEST_RECOVERY: "1" + run: uv run --no-sync pytest -q tests/prod_env_tests/test_rabbitmq_analysis.py::test_redelivery_survives_broker_restart_and_controller_replacement + - name: Pod restarts and probe failures if: always() run: | diff --git a/exaflow/controller/config.toml b/exaflow/controller/config.toml index 4216171fb..0e03e68eb 100644 --- a/exaflow/controller/config.toml +++ b/exaflow/controller/config.toml @@ -38,3 +38,16 @@ privacy_budget = "$DP_PRIVACY_BUDGET" [grpc] max_message_mb = "${GRPC_MAX_MESSAGE_MB:-100}" + +[rabbitmq] +enabled = "${RABBITMQ_ENABLED:-false}" +url = "${RABBITMQ_URL:-}" +request_queue = "${RABBITMQ_REQUEST_QUEUE:-exaflow.analysis.requests}" +dead_letter_queue = "${RABBITMQ_DEAD_LETTER_QUEUE:-exaflow.analysis.dead}" +max_result_bytes = "${RABBITMQ_MAX_RESULT_BYTES:-16777216}" +queue_max_bytes = "${RABBITMQ_QUEUE_MAX_BYTES:-268435456}" +reply_ttl_ms = "${RABBITMQ_REPLY_TTL_MS:-86400000}" +dead_letter_ttl_ms = "${RABBITMQ_DEAD_LETTER_TTL_MS:-604800000}" +operation_timeout = "${RABBITMQ_OPERATION_TIMEOUT:-5}" +publish_timeout = "${RABBITMQ_PUBLISH_TIMEOUT:-30}" +shutdown_drain_timeout = "${RABBITMQ_SHUTDOWN_DRAIN_TIMEOUT:-20}" diff --git a/kubernetes/templates/exaflow-controller.yaml b/kubernetes/templates/exaflow-controller.yaml index 8b754ef3f..71996745c 100644 --- a/kubernetes/templates/exaflow-controller.yaml +++ b/kubernetes/templates/exaflow-controller.yaml @@ -42,6 +42,25 @@ spec: command: ["sleep", {{ quote .Values.controller.shutdown_prestop_delay }}] env: + - name: RABBITMQ_ENABLED + value: {{ .Values.rabbitmq.enabled | quote }} + {{- if .Values.rabbitmq.enabled }} + - name: RABBITMQ_URL + valueFrom: + secretKeyRef: + name: {{ .Values.rabbitmq.existing_secret | quote }} + key: {{ .Values.rabbitmq.url_secret_key | quote }} + {{- range $key, $value := .Values.rabbitmq }} + {{- if has $key (list "request_queue" "dead_letter_queue" "max_result_bytes" "queue_max_bytes" "reply_ttl_ms" "dead_letter_ttl_ms" "operation_timeout" "publish_timeout" "shutdown_drain_timeout") }} + - name: RABBITMQ_{{ $key | upper }} + {{- if has $key (list "max_result_bytes" "queue_max_bytes" "reply_ttl_ms" "dead_letter_ttl_ms") }} + value: {{ $value | int64 | quote }} + {{- else }} + value: {{ $value | quote }} + {{- end }} + {{- end }} + {{- end }} + {{- end }} - name: NODE_IDENTIFIER valueFrom: fieldRef: diff --git a/kubernetes/values.yaml b/kubernetes/values.yaml index a25754db8..4fbb73c5d 100644 --- a/kubernetes/values.yaml +++ b/kubernetes/values.yaml @@ -44,6 +44,21 @@ controller: shutdown_timeout: 60 shutdown_grace_margin: 20 +# External broker; the chart does not provision RabbitMQ. +rabbitmq: + enabled: false + existing_secret: "exaflow-rabbitmq" + url_secret_key: "url" + request_queue: "exaflow.analysis.requests" + dead_letter_queue: "exaflow.analysis.dead" + max_result_bytes: 16777216 + queue_max_bytes: 268435456 + reply_ttl_ms: 86400000 + dead_letter_ttl_ms: 604800000 + operation_timeout: 5 + publish_timeout: 30 + shutdown_drain_timeout: 20 + globalworker_identifier: globalworker aggregation_server: diff --git a/tasks.py b/tasks.py index b30b7e136..252cdb930 100644 --- a/tasks.py +++ b/tasks.py @@ -304,6 +304,10 @@ def create_configs(c): ] controller_config["max_concurrent_experiments"] = max_concurrent_experiments + # Preserve the template's defaults and defer URL interpolation to the + # controller loader. Invoke tasks must not import the Exaflow runtime. + controller_config["rabbitmq"].update(deployment_config.get("rabbitmq", {})) + controller_config["deployment_type"] = "LOCAL" controller_config["localworkers"]["config_file"] = str( diff --git a/tests/prod_env_tests/deployment_configs/kind_configuration/kind_cluster.yaml b/tests/prod_env_tests/deployment_configs/kind_configuration/kind_cluster.yaml index f40df8b7b..d383fda34 100644 --- a/tests/prod_env_tests/deployment_configs/kind_configuration/kind_cluster.yaml +++ b/tests/prod_env_tests/deployment_configs/kind_configuration/kind_cluster.yaml @@ -9,6 +9,9 @@ nodes: extraPortMappings: - containerPort: 30000 hostPort: 5100 + - containerPort: 30001 + hostPort: 5679 + listenAddress: "127.0.0.1" extraMounts: - hostPath: ./tests/prod_env_tests/deployment_configs/kind_configuration/master/hostname containerPath: /etc/hostname diff --git a/tests/prod_env_tests/deployment_configs/kubernetes_values.yaml b/tests/prod_env_tests/deployment_configs/kubernetes_values.yaml index d7044df5b..c508e44f8 100644 --- a/tests/prod_env_tests/deployment_configs/kubernetes_values.yaml +++ b/tests/prod_env_tests/deployment_configs/kubernetes_values.yaml @@ -28,6 +28,12 @@ controller: workers_dns: "exaflow-workers-service" worker_landscape_aggregator_update_interval: 20 worker_tasks_timeout: 240 + shutdown_prestop_delay: 5 + graceful_timeout: 120 + shutdown_rpc_cancel_timeout: 5 + shutdown_cleanup_timeout: 10 + shutdown_timeout: 60 + shutdown_grace_margin: 20 smpc: enabled: false @@ -39,3 +45,17 @@ storage: db: localworker: "/opt/exaflow/localworker" globalworker: "/opt/exaflow/globalworker" + +rabbitmq: + enabled: true + existing_secret: "exaflow-rabbitmq" + url_secret_key: "url" + request_queue: "exaflow.analysis.requests" + dead_letter_queue: "exaflow.analysis.dead" + max_result_bytes: 16777216 + queue_max_bytes: 268435456 + reply_ttl_ms: 86400000 + dead_letter_ttl_ms: 604800000 + operation_timeout: 5 + publish_timeout: 30 + shutdown_drain_timeout: 20 diff --git a/tests/prod_env_tests/deployment_configs/rabbitmq.yaml b/tests/prod_env_tests/deployment_configs/rabbitmq.yaml new file mode 100644 index 000000000..ebf87737e --- /dev/null +++ b/tests/prod_env_tests/deployment_configs/rabbitmq.yaml @@ -0,0 +1,87 @@ +# Test infrastructure only. Create exaflow-rabbitmq Secret before applying. +apiVersion: v1 +kind: ConfigMap +metadata: + name: exaflow-rabbitmq-config +data: + rabbitmq.conf: | + # Longer than legitimate analysis execution plus admission waits. + consumer_timeout = 7200000 + max_message_size = 33554432 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: exaflow-rabbitmq +spec: + serviceName: exaflow-rabbitmq + replicas: 1 + selector: + matchLabels: + app: exaflow-rabbitmq + template: + metadata: + labels: + app: exaflow-rabbitmq + spec: + containers: + - name: rabbitmq + image: rabbitmq:4.1.0 + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + cpu: "1" + memory: 1Gi + ports: + - containerPort: 5672 + env: + - name: RABBITMQ_DEFAULT_USER + valueFrom: + secretKeyRef: + name: exaflow-rabbitmq + key: username + - name: RABBITMQ_DEFAULT_PASS + valueFrom: + secretKeyRef: + name: exaflow-rabbitmq + key: password + readinessProbe: + exec: + command: ["rabbitmq-diagnostics", "-q", "check_running"] + initialDelaySeconds: 10 + periodSeconds: 5 + timeoutSeconds: 10 + volumeMounts: + - name: config + mountPath: /etc/rabbitmq/rabbitmq.conf + subPath: rabbitmq.conf + - name: data + mountPath: /var/lib/rabbitmq + volumes: + - name: config + configMap: + name: exaflow-rabbitmq-config + volumeClaimTemplates: + - metadata: + name: data + spec: + accessModes: ["ReadWriteOnce"] + resources: + requests: + storage: 1Gi +--- +apiVersion: v1 +kind: Service +metadata: + name: exaflow-rabbitmq +spec: + type: NodePort + selector: + app: exaflow-rabbitmq + ports: + - name: amqp + port: 5672 + targetPort: 5672 + nodePort: 30001 diff --git a/tests/standalone_tests/controller/rabbitmq/test_settings.py b/tests/standalone_tests/controller/rabbitmq/test_settings.py new file mode 100644 index 000000000..9587c87b0 --- /dev/null +++ b/tests/standalone_tests/controller/rabbitmq/test_settings.py @@ -0,0 +1,108 @@ +import asyncio +import re +import shutil +import subprocess +from pathlib import Path +from unittest.mock import AsyncMock + +import pytest +from pydantic import ValidationError + +from exaflow import AttrDict +from exaflow.controller.http import app as app_module +from exaflow.controller.services import controller_shutdown as budget +from exaflow.controller.services.rabbitmq.settings import RabbitMQSettings + + +def test_disabled_broker_settings_do_not_block_controller_startup(monkeypatch): + monkeypatch.setattr( + app_module, + "ctrl_config", + AttrDict( + { + "rabbitmq": { + "enabled": "false", + "url": "invalid", + "operation_timeout": 10, + "request_queue": "", + "publish_timeout": -1, + } + } + ), + ) + startup, shutdown = AsyncMock(), AsyncMock() + monkeypatch.setattr(app_module.endpoints, "startup", startup) + monkeypatch.setattr(app_module.endpoints, "shutdown", shutdown) + + async def scenario(): + async with app_module.lifespan(app_module.app): + assert not app_module.app.state.rabbitmq.settings.enabled + + previous = getattr(app_module.app.state, "rabbitmq", None) + try: + asyncio.run(scenario()) + finally: + app_module.app.state.rabbitmq = previous + startup.assert_awaited_once() + shutdown.assert_awaited_once() + + +@pytest.mark.parametrize( + "overrides", + [ + {"url": "invalid"}, + {"operation_timeout": 10}, + {"publish_timeout": 0}, + {"shutdown_drain_timeout": 30}, + ], +) +def test_enabled_configuration_is_validated(overrides): + with pytest.raises(ValidationError): + RabbitMQSettings.from_config( + {"enabled": True, "url": "amqp://localhost", **overrides} + ) + + +def test_publish_timeout_is_independent_and_shutdown_has_headroom(): + settings = RabbitMQSettings(publish_timeout=60) + assert settings.operation_timeout == 5 + assert settings.publish_timeout == 60 + worst_case = ( + 3 * settings.operation_timeout + + settings.shutdown_drain_timeout + + budget.SHUTDOWN_CLEANUP_TIMEOUT + + budget.SHUTDOWN_RPC_CANCEL_TIMEOUT + ) + assert worst_case <= budget.SHUTDOWN_TIMEOUT - 10 + + +@pytest.mark.skipif(shutil.which("helm") is None, reason="helm is not installed") +def test_helm_rabbitmq_values_keep_decimal_integers_and_fractional_timeouts(): + root = Path(__file__).resolve().parents[4] + rendered = subprocess.check_output( + [ + "helm", + "template", + str(root / "kubernetes"), + "--show-only", + "templates/exaflow-controller.yaml", + "--set", + "rabbitmq.enabled=true", + "--set", + "rabbitmq.operation_timeout=2.5", + "--set", + "rabbitmq.publish_timeout=45.5", + ], + text=True, + ) + expected = { + "MAX_RESULT_BYTES": "16777216", + "QUEUE_MAX_BYTES": "268435456", + "REPLY_TTL_MS": "86400000", + "DEAD_LETTER_TTL_MS": "604800000", + "OPERATION_TIMEOUT": "2.5", + "PUBLISH_TIMEOUT": "45.5", + "SHUTDOWN_DRAIN_TIMEOUT": "20", + } + for key, value in expected.items(): + assert re.search(rf'name: RABBITMQ_{key}\s+value: "{value}"', rendered) diff --git a/tests/standalone_tests/test_kill_controller.py b/tests/standalone_tests/test_kill_controller.py index 8507936b1..8ed5f4dbf 100644 --- a/tests/standalone_tests/test_kill_controller.py +++ b/tests/standalone_tests/test_kill_controller.py @@ -469,3 +469,38 @@ def sleep(seconds): interval = tasks.CONTROLLER_TERMINATE_POLL_INTERVAL_SECONDS polls = int(tasks.CONTROLLER_TERMINATE_TIMEOUT_SECONDS / interval) assert sleeps == [interval] * polls + + +@pytest.mark.parametrize("enabled", [False, True]) +def test_config_generation_preserves_rabbitmq_template_defaults_without_runtime_import( + tmp_path, monkeypatch, enabled +): + import envtoml + + from exaflow.controller.services.rabbitmq.settings import RabbitMQSettings + + deployment = tasks.toml.load(tasks.PROJECT_ROOT / ".deployment.sample.toml") + if enabled: + deployment["rabbitmq"] = {"enabled": True, "url": "${RABBITMQ_URL:-}"} + else: + deployment.pop("rabbitmq", None) + config_file = tmp_path / "deployment.toml" + config_file.write_text(tasks.toml.dumps(deployment)) + controller_dir = tmp_path / "controller" + monkeypatch.setattr(tasks, "DEPLOYMENT_CONFIG_FILE", config_file) + monkeypatch.setattr(tasks, "WORKERS_CONFIG_DIR", tmp_path / "workers") + monkeypatch.setattr(tasks, "CONTROLLER_CONFIG_DIR", controller_dir) + monkeypatch.setattr( + tasks, "CONTROLLER_LOCALWORKERS_CONFIG_FILE", controller_dir / "workers.json" + ) + monkeypatch.setattr(tasks, "AGG_SERVER_CONFIG_DIR", tmp_path / "aggregation") + monkeypatch.setenv("RABBITMQ_URL", "amqp://localhost") + monkeypatch.delenv("RABBITMQ_ENABLED", raising=False) + + tasks.create_configs(_CTX) + + with (controller_dir / "controller.toml").open("rb") as generated: + settings = RabbitMQSettings.model_validate(envtoml.load(generated)["rabbitmq"]) + assert settings.enabled is enabled + assert settings.url == "amqp://localhost" + assert settings.max_result_bytes == 16 * 1024 * 1024 From 712ec62402f366913c25e3096fd6ff6ed7591600 Mon Sep 17 00:00:00 2001 From: ThanKarab Date: Fri, 13 Nov 2026 14:12:48 +0200 Subject: [PATCH 6/6] test(prod): verify rabbitmq analysis flow end to end --- documentation/api-specification.md | 20 +- documentation/context/architecture.md | 25 ++ documentation/context/commands.md | 12 + documentation/context/decision-log.md | 24 ++ documentation/context/module-index.md | 10 + documentation/context/risk-register.md | 26 ++ documentation/context/testing.md | 41 +++ documentation/rabbitmq-analysis.md | 244 +++++++++++++ .../exareme3/helpers.py | 9 +- .../prod_env_tests/test_rabbitmq_analysis.py | 321 ++++++++++++++++++ .../controller/rabbitmq/test_prod_helpers.py | 100 ++++++ 11 files changed, 830 insertions(+), 2 deletions(-) create mode 100644 documentation/rabbitmq-analysis.md create mode 100644 tests/prod_env_tests/test_rabbitmq_analysis.py create mode 100644 tests/standalone_tests/controller/rabbitmq/test_prod_helpers.py diff --git a/documentation/api-specification.md b/documentation/api-specification.md index 66219653e..5a16e8502 100644 --- a/documentation/api-specification.md +++ b/documentation/api-specification.md @@ -414,12 +414,18 @@ Example: | Field | Type | Meaning | Validation / UI Notes | | --- | --- | --- | --- | -| `request_id` | `string \| null` | Optional client request id. | If omitted, the server generates one. | +| `request_id` | `string \| null` | Optional client request id. | Supplied strings must match `^[A-Za-z0-9_-]{1,128}$`. If omitted or null, the server generates one. | | `inputdata` | `AnalysisInputDataDTO` | Data model, datasets, filter, and source variables loaded before preprocessing. | Required. | | `preprocessing` | `AnalysisPreprocessingStepDTO[] \| null` | Ordered preprocessing step configuration. | Step names must exist in enabled preprocessing specs, and required preprocessing steps must be included. | | `algorithm` | `AnalysisAlgorithmDTO` | Selected algorithm name, final algorithm `x`, `y`, and parameter values. | Required. | | `flags` | `object \| null` | System flags. | Optional. | +**Request ID compatibility:** caller-provided IDs are restricted to 1–128 ASCII +letters, digits, underscores, or hyphens. Empty strings, whitespace, other +punctuation, Unicode, and control characters return HTTP `422` before analysis +orchestration. Accepted IDs are preserved exactly; they are not trimmed or +rewritten. Clients previously using IDs outside this format must update them. + ### `AnalysisInputDataDTO` | Field | Type | Meaning | Validation / UI Notes | @@ -570,3 +576,15 @@ ______________________________________________________________________ - `exaflow/controller/http/error_handlers.py` - Algorithm/preprocessing names: - `exaflow/algorithms/specifications.py` + +## Optional RabbitMQ transport + +`POST /analysis` keeps its current body, response, and error behavior. An optional +RabbitMQ consumer accepts the same analysis JSON and publishes a correlated +success/failure envelope; see [RabbitMQ analysis submission](rabbitmq-analysis.md). + +`GET /healthcheck/rabbitmq` reports `enabled`, `connected`, `consuming`, +`active_jobs`, and `last_error` (a safe error type or null). Its status is 503 when +enabled but unavailable, otherwise 200. It does not change `/livez` or worker +health checks. The aggregate HTTP/AMQP analysis limit is shared; only analysis +submissions acquire a slot. diff --git a/documentation/context/architecture.md b/documentation/context/architecture.md index 512c7aea3..5985fe72f 100644 --- a/documentation/context/architecture.md +++ b/documentation/context/architecture.md @@ -209,3 +209,28 @@ that caller configured it. need verification. - Some Flower and SMPC paths appear optional or partly disabled in CI; verify before broad changes. + +## Optional RabbitMQ analysis transport + +FastAPI lifespan can own a bounded async AMQP consumer alongside HTTP. Both call +`analysis_service.run_analysis`; HTTP rejects excess analyses while AMQP waits +for the same capacity and validates after admission. Metadata and health requests +remain independent of analysis slots. The worker landscape refresher keeps its +background thread and analyses keep their dedicated executor. + +A queue backlog can make HTTP analysis submissions return 503 until queued +waiters are admitted. Each delivery passively checks its reply destination and +publishes through its own channel on the existing broker connection. A rejected +reply dead-letters only that request. Transient publication retries reuse the +result; completed replies for unsettled deliveries have bounded in-memory +retention for same-controller redelivery. + +Controller and worker request logging uses fixed service loggers with per-record +request IDs, rather than permanently registering a logger and handler for every +logical request. Worker request adapters retain their ContextVar isolation. + +Logical `request_id` values survive retries; UUID `context_id` values identify +attempts and key all aggregation state. Both are carried to aggregation services +and logged. This changes internal RPC contracts and requires coordinated rollout. +See `documentation/rabbitmq-analysis.md` for delivery, lifecycle, configuration, +and rollback behavior. diff --git a/documentation/context/commands.md b/documentation/context/commands.md index 40b1f56fc..487d37fd5 100644 --- a/documentation/context/commands.md +++ b/documentation/context/commands.md @@ -288,3 +288,15 @@ uv run python .agents/skills/exaflow-algorithm-validate/scripts/validate_algorit EBRAINS integration path. - `docker-bake.hcl` defines the multi-image build targets used by the publish workflows. + +## RabbitMQ + +Enable the optional external-broker transport through the existing deployment +configuration and privately supplied `RABBITMQ_URL`. Configuration, Helm Secret +wiring, producer examples, and test-broker provisioning are documented in +`documentation/rabbitmq-analysis.md`. + +```bash +uv run python examples/rabbitmq_analysis.py analysis-request.json +uv run pytest -q tests/standalone_tests/controller/rabbitmq +``` diff --git a/documentation/context/decision-log.md b/documentation/context/decision-log.md index 12e5d4570..db6089d02 100644 --- a/documentation/context/decision-log.md +++ b/documentation/context/decision-log.md @@ -213,3 +213,27 @@ Files affected: `.agents/skills`, `documentation/new-algorithm-setup.md`, algorithm wrapper/core/test/docs paths. Date: 2026-05-22 + +## Separate logical requests from aggregation attempts + +RabbitMQ redelivery may repeat a logical request. Preserve `request_id` for +tracing and use a fresh UUID `context_id` for each attempt, including all worker +aggregation operations and cleanup. Propagate both explicitly; aggregation logs +include both. This changes the aggregation RPC contract and requires coordinated +controller, worker, and aggregation-server upgrades. Regenerate protobuf bindings +using `exaflow/protos/README.md`. + +## Isolate RabbitMQ replies and retain unsettled results briefly + +Each delivery checks that its producer-created reply queue exists and publishes +through an isolated channel on the existing connection. Missing destinations and +broker nacks dead-letter only their request; transient failures retry the +serialized result. Completed replies can survive redelivery within one +controller for five minutes, bounded to 128 entries and 64 MiB. This remains +at-least-once execution without a durable result store. A sustained backlog can +reject HTTP analysis submissions under the intentionally shared capacity limit. + +The broker is dedicated to Exaflow; no application queue-prefix allowlist or +request-body collision checks are added. RabbitMQ remains opt-in and externally +provisioned. The independent 30-second publication timeout is interruptible; the +20-second drain cap leaves ten seconds of lifespan shutdown headroom. diff --git a/documentation/context/module-index.md b/documentation/context/module-index.md index 990420429..d850a418d 100644 --- a/documentation/context/module-index.md +++ b/documentation/context/module-index.md @@ -225,3 +225,13 @@ Rules: Preserve dry-run behavior and JSON merge semantics. Tests: No dedicated test found during inspection; validate manually if changed. Notes: Can accept piped JSON and override fields with CLI args. + +## RabbitMQ analysis submission + +`exaflow/controller/services/rabbitmq` owns typed broker settings, bounded async +consumption, reply envelopes, manual acknowledgement, and reconnect/drain behavior. +FastAPI lifespan owns its runtime instance; the HTTP layer exposes independent +consumer health. Both transports call the shared analysis service, with AMQP +waiting for capacity before semantic validation. Production broker provisioning +remains external; test-only kind manifests live with production test deployment +configuration. See `documentation/rabbitmq-analysis.md`. diff --git a/documentation/context/risk-register.md b/documentation/context/risk-register.md index 56a4c7986..f29b8aec0 100644 --- a/documentation/context/risk-register.md +++ b/documentation/context/risk-register.md @@ -30,3 +30,29 @@ persistence. Publishing workflows build without restoring or saving CI caches; test caches remain enabled. Controller, worker, and staging aggregation images retain separate publishing jobs so one failure does not skip other images. Publishing credentials and triggers are unchanged. Rollback is a revert of these changes, restoring the corresponding exposures. + +## RabbitMQ analysis transport + +Broker persistence adds storage for analysis results and dead letters. Bound +message size, queue growth, and retention; provide credentials through existing +configuration/Secrets and keep URLs out of logs. AMQP is disabled by default and +normal deployments provide an external broker. HTTP and AMQP share one analysis +capacity limit; other HTTP routes remain independent. + +A sustained queue backlog can reject HTTP analyses with 503. Reply failures +must remain isolated from the consuming channel. Finished replies retained for +redelivery are bounded to five minutes, 128 entries, and 64 MiB; eviction or a +controller restart still permits recomputation. Acknowledge only after confirmed +publication, and keep reply bodies and connection URLs out of diagnostics. + +Delivery is at least once, with recomputation and duplicate replies allowed. +Preserve logical request IDs and isolate aggregation state using attempt context +IDs, including late calls, cancellation tombstones, and cleanup ownership checks. +The internal protocol change requires coordinated controller/worker/aggregation +rollout. Consumer tasks require explicit draining before existing shutdown cancels +RPCs; cancelling asyncio tasks alone does not stop executor threads. + +Real-broker recovery tests may restart only their dedicated kind test broker and +replace the test controller. They require the explicit test recovery environment +flag and must never target shared deployments. See +`documentation/rabbitmq-analysis.md` for configuration, tests, and rollback. diff --git a/documentation/context/testing.md b/documentation/context/testing.md index 741c48d39..0d770fc13 100644 --- a/documentation/context/testing.md +++ b/documentation/context/testing.md @@ -162,3 +162,44 @@ CI also builds the package and smoke-tests the wheel. | Dependency update | Lockfile update, focused tests, lint/format, and CI-equivalent suite where feasible. | | Refactor without behavior change | Focused tests for touched modules and broader standalone tests if shared code moved. | | `exadata-validator` change | `cd exadata-validator && uv run pytest -q`; build/smoke test if packaging or CLI changes. | + +## RabbitMQ transport + +Focused fake-AMQP tests live in `tests/standalone_tests/controller/rabbitmq`. +Controller concurrency and Uvicorn shutdown tests cover shared admission and queue +cancellation; aggregation gRPC tests cover independent execution contexts. +Algorithm validation stays on HTTP and requires no broker. + +RabbitMQ configuration tests cover disabled-feature validation, independent +publication deadlines, and ten seconds of headroom in the controller's lifespan +shutdown budget. Helm rendering checks decimal integer environment values and +fractional timeout values; it requires Helm on `PATH`. + +Consumer regressions cover missing destinations before execution, unroutable +replies, broker nacks, transient publish retries, failed settlement, completed +result retention and eviction, and concurrent reply-channel cleanup. Real broker +tests also exercise deleted destinations and a full reply queue while another +analysis succeeds. Shared logging tests check bounded logger/handler counts and +concurrent request context isolation in the controller and worker. + +Shared DTO and intake tests enforce `^[A-Za-z0-9_-]{1,128}$` for caller-provided +analysis request IDs. HTTP rejects invalid body IDs with 422 before orchestration; +AMQP validates its correlation property before assigning it to the DTO or +consulting execution/result caches. Cover lengths 1/128/129, control characters, +spaces, punctuation, Unicode, nullable/omitted body IDs, and body/header conflicts. + +Run controller/logging regressions without supplying `LOG_LEVEL` or +`FRAMEWORK_LOG_LEVEL`: the config loader defaults missing/empty values to `INFO`. +The full-reply-queue production regression checks the source queue's `x-death` +queue, reason, and count; a dead-letter delivery's `x-delivery-count` is not a +reliable measure of analysis executions. + +Production tests provision `tests/prod_env_tests/deployment_configs/rabbitmq.yaml` +in kind. `RABBITMQ_TEST_URL` enables real-broker cases; the CI-only +`RABBITMQ_TEST_RECOVERY=1` flag permits restarting the dedicated test broker and +replacing its test controller. CI sets that flag only in a separate recovery +step after the normal production suite, and recovery cleanup waits for controller +rollout and transport readiness, including on failures. Broker access uses kind +host port 5679 mapped to NodePort 30001 instead of a restart-sensitive port-forward. +Queue and HTTP parity cases share the existing request builder, including required +missing-value preprocessing. See `documentation/rabbitmq-analysis.md`. diff --git a/documentation/rabbitmq-analysis.md b/documentation/rabbitmq-analysis.md new file mode 100644 index 000000000..4bd86550d --- /dev/null +++ b/documentation/rabbitmq-analysis.md @@ -0,0 +1,244 @@ +# Analysis submission through RabbitMQ + +RabbitMQ is an optional second transport for the existing analysis service. +HTTP routes, response formats, and algorithm result shapes remain unchanged. +Both transports validate request IDs using the shared rule below. There is no +job database or HTTP result polling endpoint. + +## Execution and identifiers + +The FastAPI lifespan starts one asynchronous consumer after the first worker +landscape refresh. HTTP and AMQP share an asyncio loop; blocking native analysis +work stays in the dedicated analysis executor. The landscape refresher retains +its existing background thread. Broker connection failures do not prevent HTTP +startup or operation. + +`request_id` identifies the logical analysis and is used for tracing across the +controller and workers. The AMQP `correlation_id` must equal it. An omitted body +`request_id` is populated from the correlation ID; conflicting values are rejected. +Use the same ID only for the same logical request. Additional collision detection +is deliberately outside this transport's contract. + +Caller-provided request IDs must match `^[A-Za-z0-9_-]{1,128}$`: 1–128 ASCII +letters, digits, underscores, or hyphens. This rule applies to HTTP body IDs, +AMQP `correlation_id`, and any body `request_id`. Accepted values are preserved +exactly, without trimming or rewriting. These are Exaflow restrictions on the +standard AMQP correlation property, not RabbitMQ format requirements. + +Invalid supplied HTTP IDs return `422`; omitted/null HTTP IDs retain server-side +generation. Nonempty but invalid AMQP correlation IDs, invalid body IDs, and +conflicting IDs receive `invalid_request` replies without execution or cache +lookup. Missing/empty correlation IDs retain existing dead-letter handling. +Clients previously using IDs outside this format must update them. No validation +is repeated at individual analysis log statements. + +Controller and worker request logs use fixed service loggers with per-request +adapters. Unique IDs do not register permanent loggers or handlers. Worker +context remains isolated between concurrent UDFs; control characters in logging +identifiers are escaped without changing the protocol IDs. + +Each strategy generates a UUID `context_id` for its execution attempt. UDF system +arguments carry that context to workers. Aggregation Configure, Aggregate, +Unregister, Cleanup, and cancellation tombstones use the context ID as their key; +aggregation logs retain both IDs and Cleanup still enforces its ownership token. +Before this change context IDs appeared only in experiment-start logs and were +stored, unused, in the flow engine. + +RPCs for an existing context must pair its ID with the request ID that configured it; mismatches +return `INVALID_ARGUMENT` without changing the context. Fresh context IDs make +`ALREADY_CONFIGURED` a defensive check for accidental context reuse. + +**Compatibility:** the aggregation RPC and UDF system-argument contracts change. +Upgrade controller, workers, and aggregation server together, even when AMQP is +disabled. There is no fallback for old clients omitting the execution context. +Regenerate bindings using `exaflow/protos/README.md`. + +## Message contract + +Publish persistent JSON messages with publisher confirms and mandatory routing to +`exaflow.analysis.requests` (configurable). The body is the existing +`AnalysisRequestDTO`, including `inputdata`, `preprocessing`, `algorithm`, and any +existing flags. Required AMQP properties are: + +| Property | Meaning | +| --- | --- | +| `correlation_id` | Logical request ID, preserved on replies. | +| `reply_to` | Name of a durable queue provisioned by the producer. | +| `content_type` | Producers should set `application/json`. | +| `delivery_mode` | Producers must use persistent delivery for restart recovery. | + +Provision the durable reply queue before submitting an analysis. Every delivery +checks its destination with a passive declaration on a separate reply channel, +before analysis execution. This uses the existing connection, adding channel +setup and one broker round trip for the check. Missing queues dead-letter the +request without running the analysis. There are no queue-prefix restrictions or +additional application permission checks; the broker is dedicated to Exaflow. + +The consumer parses the DTO, waits for shared analysis capacity, then validates +against current worker metadata and constructs the execution strategy. HTTP +keeps its original validation/admission order. Both transports share +`max_concurrent_experiments`; there are no reserved slots. Other HTTP endpoints +do not acquire analysis slots. Queue prefetch and active delivery handlers are +bounded by that same limit, also across reconnects. + +A sustained queue backlog can keep HTTP `POST /analysis` returning 503 until +waiting queue analyses have been admitted. This shared-capacity policy is +intentional; metadata and health routes remain available. + +Replies contain parsed result JSON, rather than a JSON string inside JSON: + +```json +{"status": "succeeded", "result": {"example": "algorithm-specific result"}} +``` + +```json +{"status": "failed", "error": {"code": "invalid_request", "message": "Request does not match the analysis schema or identifiers."}} +``` + +Protocol error codes include `invalid_request`, `request_too_large`, +`result_too_large`, `insufficient_data`, and `internal_error`. Recognized user-input +and method errors use their existing exception class name, for example +`BadRequest`. Unexpected errors return a generic message without exception details. +Validation and algorithm errors are terminal replies, not automatic retry jobs. + +## Delivery, recovery, and shutdown + +A request is acknowledged only after a persistent reply has been routed and +confirmed by the broker. Missing routing metadata, missing destinations, +unroutable replies, and negative broker confirmations dead-letter only the +affected request. Each delivery publishes through an isolated reply channel; +these failures do not close the consuming channel or interrupt other analyses. +The destination can disappear or fill up after the initial check, so replies +retain mandatory routing and publisher confirms. Consumer declarations create bounded, +durable quorum request and dead-letter queues, with at-least-once dead-lettering. +A full queue rejects publication; producers must handle negative confirms. + +Processing is **at least once**. A controller crash after reply publication but +before request acknowledgement can repeat the analysis and its reply. Clients +must deduplicate by correlation ID; recomputation can yield different results for +randomized algorithms or changing datasets. No completed-result database exists. +Active duplicate deliveries in a surviving controller await the existing attempt. +Fresh execution contexts prevent stale aggregation calls from affecting retries. + +Completed serialized replies for unsettled deliveries are retained in memory for +up to five minutes, bounded to 128 entries and 64 MiB total. Expired entries are +removed first, then the oldest entries; replies larger than the cache budget are +not retained. Redelivery in the same controller can reuse a retained reply after +its execution has finished. Successful acknowledgement or terminal rejection +removes it; a subsequent submission starts a fresh attempt. Cache expiry, +eviction, and controller replacement can still lead to recomputation. + +The single-controller consumer checks subscription registration once per second. +If the broker cancels it without closing the connection, zero registered consumers +interrupts the iterator, marks consumption unavailable, and restarts the connection +and subscription. + +Transient reply publication failures retry the existing result on a fresh reply +channel, with exponential backoff from 1 to 30 seconds. A confirmation timeout +can still produce duplicate replies. Connection or consumer-channel failures +restart consumption with the same backoff, retaining completed replies while the +controller survives. Deliveries belong to their original channel and are never +acknowledged through a replacement channel. Only redelivered messages consult +the retained-reply cache. + +At lifespan shutdown, intake stops, active deliveries drain for up to 20 seconds, +and existing shutdown logic cancels RPCs and performs aggregation cleanup. Then +remaining AMQP tasks and the connection close. Unfinished requests remain +unacknowledged for redelivery. The shutdown phases total at most 50 seconds with +default settings, leaving ten seconds of headroom in the existing 60-second +lifespan budget. Cancellation does not forcibly terminate Python threads. +Uvicorn's existing HTTP drain precedes the lifespan hook. + +## Configuration and provisioning + +AMQP defaults to disabled. Provision an external RabbitMQ 4.x broker; the normal +Helm chart does not install it. Use one controller replica and one Uvicorn process. +Disabled consumption ignores broker-specific settings. When enabled, invalid +configuration fails startup; an unreachable broker retries independently of HTTP. +Provide broker credentials and queue permissions through the existing settings +layer. Restrict producer access to the appropriate request and reply queues; +use a dedicated vhost and `amqps://` when TLS is required by the deployment. + +For local configuration, add `[rabbitmq]` to the deployment TOML, enable it, and +set `url = "${RABBITMQ_URL:-}"`. Run the existing config-generation task. +Credentials are interpolated by the controller loader, not written into the +sample file. Never commit generated private configuration. + +For Helm, enable `rabbitmq.enabled` and supply `rabbitmq.existing_secret` containing +an AMQP URL at `rabbitmq.url_secret_key` (default `url`). Other settings are wired +through the controller's TOML template: + +| Setting | Default | +| --- | --- | +| `request_queue` | `exaflow.analysis.requests` | +| `dead_letter_queue` | `exaflow.analysis.dead` | +| `max_result_bytes` | 16 MiB, including the result envelope | +| `queue_max_bytes` | 256 MiB per request/dead-letter queue | +| `reply_ttl_ms` | 24 hours, applied to each result message | +| `dead_letter_ttl_ms` | 7 days | +| `operation_timeout` | 5 seconds, maximum 5 | +| `publish_timeout` | 30 seconds, positive; independent of operation timeout | +| `shutdown_drain_timeout` | 20 seconds, maximum 20 | + +Existing configurations setting `shutdown_drain_timeout` above 20 seconds must +lower it before upgrading. Reply publication is interruptible during shutdown, +including when `publish_timeout` exceeds the drain budget. + +Request bodies have the same 16 MiB limit as HTTP. Configure the broker's +`max_message_size` high enough to let the application report oversized-request +errors; the production test broker uses 32 MiB. Provision bounded reply queues +with appropriate retention; reply messages also carry the configured expiration. +There is no default request expiry. Changing queue declaration arguments requires +an explicit queue migration; RabbitMQ rejects conflicting declarations. +Configure the broker consumer acknowledgement timeout above legitimate capacity +waits plus analysis and reply publication. The test broker uses two hours. +Persistent broker storage and high availability remain deployment responsibilities. + +`GET /healthcheck/rabbitmq` returns enabled/connected/consuming state, active job +count, and the last safe error type. It returns 503 when enabled but unavailable, +and 200 when disabled or consuming. Existing `/livez` and `/healthcheck` semantics +remain unchanged. Background logs record connection transitions and delivery +failures without credentials or message bodies. +Broker channel failures include bounded, sanitized diagnostic text so queue +declaration mismatches can be diagnosed. Connection errors and publication errors +that can contain URLs or message bodies log only their error type. + +## Producer example + +With `RABBITMQ_URL` supplied privately and controller consumption enabled: + +```bash +uv run python examples/rabbitmq_analysis.py analysis-request.json +``` + +The example declares a durable reply queue per logical request, confirms request +publication, prints the matching reply, and acknowledges it. Supply a stable body +`request_id` when retrying after a client interruption. Idle example reply queues +expire after two days; result messages expire after one day. + +## Testing and rollback + +- Standalone tests use fake AMQP boundaries for parsing, admission, replies, + acknowledgements, reconnect decisions, health, limits, and shutdown. Real gRPC + aggregation tests check attempt isolation and cleanup ownership. Uvicorn process + tests cover cancellation of queue analyses that run in executor threads. +- Production broker regressions cover missing/deleted reply destinations and a + full reply queue rejecting one reply while another analysis succeeds. Unit + tests cover a destination disappearing between validation and publication, + transient publish retries, completed-result reuse, and cache bounds. +- Algorithm validation tests continue submitting through HTTP, with no RabbitMQ + dependency or transport mocks. +- Production environment tests deploy pinned RabbitMQ in kind with persistent + storage and an isolated generated Secret. Set `RABBITMQ_TEST_URL` to enable the + AMQP cases. `RABBITMQ_TEST_RECOVERY=1` additionally permits replacing the test + controller and restarting this dedicated test broker; never use that flag + against a shared or production deployment. CI enables it only in a separate + recovery step after the normal production test suite. Recovery cleanup waits + for the replacement controller rollout and HTTP/AMQP readiness before returning. + The test broker uses NodePort 30001, mapped to localhost port 5679 by kind; + this endpoint survives broker restarts without a port-forward process. + +Disable consumption to roll back the transport; pending durable requests stay in +the broker and HTTP remains usable. Revert protocol changes across controller, +workers, and aggregation server together. Retention and dead-letter inspection +must be considered before removing broker resources. diff --git a/tests/algorithm_validation_tests/exareme3/helpers.py b/tests/algorithm_validation_tests/exareme3/helpers.py index e25b88cbb..32b3ca1db 100644 --- a/tests/algorithm_validation_tests/exareme3/helpers.py +++ b/tests/algorithm_validation_tests/exareme3/helpers.py @@ -9,7 +9,9 @@ MISSING_VALUE_DROP_STRATEGY = "drop" -def analysis_request(algorithm: str, input: dict, drop_na: bool = True): +def build_analysis_request_payload( + algorithm: str, input: dict, drop_na: bool = True +) -> dict: request_payload = dict(input) request_payload.pop("test_case_num", None) request_payload.pop("type", None) @@ -28,6 +30,11 @@ def analysis_request(algorithm: str, input: dict, drop_na: bool = True): ) request_payload["preprocessing"] = preprocessing + return request_payload + + +def analysis_request(algorithm: str, input: dict, drop_na: bool = True): + request_payload = build_analysis_request_payload(algorithm, input, drop_na) url = "http://127.0.0.1:5100/analysis" headers = {"Content-type": "application/json", "Accept": "text/plain"} response = requests.post(url, data=json.dumps(request_payload), headers=headers) diff --git a/tests/prod_env_tests/test_rabbitmq_analysis.py b/tests/prod_env_tests/test_rabbitmq_analysis.py new file mode 100644 index 000000000..95d30ac36 --- /dev/null +++ b/tests/prod_env_tests/test_rabbitmq_analysis.py @@ -0,0 +1,321 @@ +"""Real AMQP transport against the kind deployment, never a mocked broker.""" + +import asyncio +import json +import os +import subprocess +import time +from contextlib import asynccontextmanager +from pathlib import Path +from uuid import uuid4 + +import aio_pika +import numpy as np +import pytest +import requests + +from tests.algorithm_validation_tests.exareme3.helpers import analysis_request +from tests.algorithm_validation_tests.exareme3.helpers import ( + build_analysis_request_payload, +) +from tests.algorithm_validation_tests.exareme3.helpers import parse_response + +pytestmark = pytest.mark.skipif( + not os.environ.get("RABBITMQ_TEST_URL"), + reason="Requires the production test RabbitMQ deployment and RABBITMQ_TEST_URL", +) + + +def payload(): + cases = json.loads( + (Path(__file__).parent / "expected/pca_expected.json").read_text() + ) + body = build_analysis_request_payload("pca", cases["test_cases"][0]["input"]) + body["request_id"] = uuid4().hex + return body + + +async def ready(): + deadline = time.monotonic() + 120 + while time.monotonic() < deadline: + try: + response = await asyncio.to_thread( + requests.get, + "http://127.0.0.1:5100/healthcheck/rabbitmq", + timeout=5, + ) + if response.status_code == 200 and response.json()["consuming"]: + return + except requests.RequestException: + pass + await asyncio.sleep(0.5) + raise AssertionError("Controller AMQP consumer did not become ready") + + +@asynccontextmanager +async def broker(reply_arguments=None): + connection = await aio_pika.connect_robust( + os.environ["RABBITMQ_TEST_URL"], timeout=10 + ) + channel = await connection.channel(publisher_confirms=True, on_return_raises=True) + replies = await channel.declare_queue( + f"exaflow.tests.replies.{uuid4().hex}", + durable=True, + arguments={ + "x-message-ttl": 86400000, + "x-expires": 3600000, + "x-max-length-bytes": 268435456, + "x-overflow": "reject-publish", + **(reply_arguments or {}), + }, + ) + try: + yield connection, channel, replies + finally: + try: + await asyncio.wait_for(replies.delete(if_unused=False, if_empty=False), 10) + finally: + await asyncio.wait_for(connection.close(), 10) + + +async def submit(channel, replies, body, **properties): + request_id = body.get("request_id", uuid4().hex) + message = aio_pika.Message( + json.dumps(body).encode(), + content_type="application/json", + delivery_mode=aio_pika.DeliveryMode.PERSISTENT, + correlation_id=request_id, + reply_to=replies.name, + message_id=uuid4().hex, + ) + for key, value in properties.items(): + setattr(message, key, value) + await channel.default_exchange.publish( + message, + routing_key="exaflow.analysis.requests", + mandatory=True, + timeout=10, + ) + return message + + +async def receive(queue, predicate=lambda message: True, timeout=180): + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + message = await queue.get(fail=False, timeout=5) + if message is not None: + if predicate(message): + await message.ack() + return message + # Leave unrelated dead letters unacknowledged; channel closure restores them. + await asyncio.sleep(0.1) + raise AssertionError("Expected broker message did not arrive") + + +def test_queue_pca_matches_http_and_repeat_request_gets_fresh_aggregation_context(): + async def scenario(): + await ready() + body = payload() + case = {k: v for k, v in body.items() if k != "request_id"} + expected = parse_response( + await asyncio.to_thread(analysis_request, "pca", case) + ) + async with broker() as (_, channel, replies): + for _ in range(2): + await submit(channel, replies, body) + message = await receive(replies) + assert message.correlation_id == body["request_id"] + result = json.loads(message.body) + assert result["status"] == "succeeded", result + actual = result["result"] + assert actual["n_obs"] == expected["n_obs"] + assert actual["variables"] == expected["variables"] + np.testing.assert_allclose( + [c["explained_variance"] for c in actual["components"]], + [c["explained_variance"] for c in expected["components"]], + rtol=1e-6, + atol=1e-9, + ) + assert ( + await asyncio.to_thread( + requests.get, "http://127.0.0.1:5100/livez", timeout=5 + ) + ).status_code == 200 + + asyncio.run(scenario()) + + +@pytest.mark.parametrize("malformed", [False, True]) +def test_queue_validation_failures_are_terminal_replies(malformed): + async def scenario(): + await ready() + body = payload() + if malformed: + body = {"request_id": body["request_id"]} + else: + body["algorithm"]["name"] = "does_not_exist" + async with broker() as (_, channel, replies): + await submit(channel, replies, body) + reply = json.loads((await receive(replies)).body) + assert reply["status"] == "failed" + assert reply["error"]["code"] == ( + "invalid_request" if malformed else "BadRequest" + ) + + asyncio.run(scenario()) + + +@pytest.mark.parametrize( + "routing_error", ["missing_id", "absent_queue", "deleted_queue"] +) +def test_bad_routing_goes_to_real_dead_letter_queue(routing_error): + async def scenario(): + await ready() + async with broker() as (_, channel, replies): + properties = {"correlation_id": None} + if routing_error != "missing_id": + destination = f"exaflow.tests.deleted.{uuid4().hex}" + if routing_error == "deleted_queue": + deleted = await channel.declare_queue(destination, durable=True) + await deleted.delete() + properties = {"reply_to": destination} + sent = await submit(channel, replies, payload(), **properties) + dead = await channel.get_queue("exaflow.analysis.dead", ensure=True) + received = await receive( + dead, lambda m: m.message_id == sent.message_id, timeout=30 + ) + assert received.body == sent.body + assert received.headers["x-death"] + + asyncio.run(scenario()) + + +def test_full_reply_queue_dead_letters_only_its_request(): + async def scenario(): + await ready() + async with broker({"x-max-length": 1}) as (_, channel, full): + await channel.default_exchange.publish( + aio_pika.Message( + b"occupied", delivery_mode=aio_pika.DeliveryMode.PERSISTENT + ), + routing_key=full.name, + mandatory=True, + timeout=10, + ) + async with broker() as (_, other_channel, replies): + failed = await submit(channel, full, payload()) + successful = await submit(other_channel, replies, payload()) + dead = await channel.get_queue("exaflow.analysis.dead", ensure=True) + rejected, response = await asyncio.gather( + receive( + dead, lambda message: message.message_id == failed.message_id + ), + receive( + replies, + lambda message: ( + message.correlation_id == successful.correlation_id + ), + ), + ) + assert json.loads(response.body)["status"] == "succeeded" + assert rejected.body == failed.body + # Delivery counters on a DLQ message are not execution counters. + # Check the source queue's explicit rejection, not a broker- + # dependent x-delivery-count value or delivery-limit exhaustion. + death = rejected.headers["x-death"][0] + assert death["queue"] == "exaflow.analysis.requests" + assert death["reason"] == "rejected" + assert death["count"] == 1 + occupied = await full.get(timeout=5) + assert occupied.body == b"occupied" + await occupied.ack() + await ready() + + asyncio.run(scenario()) + + +async def kubectl(*args): + return await asyncio.to_thread( + subprocess.run, + ["kubectl", *args], + check=True, + capture_output=True, + text=True, + timeout=180, + ) + + +async def restore_controller(): + await kubectl("scale", "deployment/exaflow-controller-deployment", "--replicas=1") + await kubectl( + "rollout", + "status", + "deployment/exaflow-controller-deployment", + "--timeout=180s", + ) + await ready() + + +@pytest.mark.skipif( + os.environ.get("RABBITMQ_TEST_RECOVERY") != "1", + reason="Requires exclusive ownership of the production test deployment", +) +def test_redelivery_survives_broker_restart_and_controller_replacement(): + async def scenario(): + await ready() + async with broker() as (connection, channel, replies): + broker_stopped = False + try: + await kubectl( + "scale", "deployment/exaflow-controller-deployment", "--replicas=0" + ) + deadline = time.monotonic() + 180 + while True: + pods = await kubectl( + "get", "pods", "-l", "app=exaflow-controller", "-o", "json" + ) + if not json.loads(pods.stdout)["items"]: + break + assert time.monotonic() < deadline, "Controller did not stop" + await asyncio.sleep(0.5) + sent = await submit(channel, replies, payload()) + delivery_channel = await connection.channel() + requests_queue = await delivery_channel.get_queue( + "exaflow.analysis.requests", ensure=True + ) + # Simulate loss of an unacknowledged delivery before restarting the broker. + held = await requests_queue.get(timeout=5) + assert held.correlation_id == sent.correlation_id + await delivery_channel.close() + broker_stopped = True + await kubectl( + "exec", "exaflow-rabbitmq-0", "--", "rabbitmqctl", "stop_app" + ) + await kubectl( + "exec", "exaflow-rabbitmq-0", "--", "rabbitmqctl", "start_app" + ) + broker_stopped = False + await kubectl( + "rollout", + "status", + "statefulset/exaflow-rabbitmq", + "--timeout=120s", + ) + await asyncio.wait_for(connection.ready(), 30) + finally: + try: + if broker_stopped: + await kubectl( + "exec", + "exaflow-rabbitmq-0", + "--", + "rabbitmqctl", + "start_app", + ) + finally: + await restore_controller() + result = await receive(replies) + assert result.correlation_id == sent.correlation_id + assert json.loads(result.body)["status"] == "succeeded" + + asyncio.run(scenario()) diff --git a/tests/standalone_tests/controller/rabbitmq/test_prod_helpers.py b/tests/standalone_tests/controller/rabbitmq/test_prod_helpers.py new file mode 100644 index 000000000..d77fad7ae --- /dev/null +++ b/tests/standalone_tests/controller/rabbitmq/test_prod_helpers.py @@ -0,0 +1,100 @@ +"""Catch production request and recovery-harness regressions without a cluster.""" + +import asyncio +import json +import subprocess +from contextlib import asynccontextmanager +from types import SimpleNamespace +from unittest.mock import AsyncMock +from unittest.mock import Mock + +import pytest + +from exaflow import exareme3_algorithm_classes +from exaflow.controller.services.api.analysis_request_dtos import AnalysisRequestDTO +from exaflow.controller.services.api.analysis_request_validator import ( + _validate_required_preprocessing, +) +from tests.algorithm_validation_tests.exareme3 import helpers +from tests.prod_env_tests import test_rabbitmq_analysis as prod + + +def test_production_pca_payload_satisfies_preprocessing_contract_and_matches_http( + monkeypatch, +): + body = prod.payload() + request = AnalysisRequestDTO.model_validate(body) + _validate_required_preprocessing( + exareme3_algorithm_classes["pca"].get_specification(), + request.preprocessing, + ) + post = Mock(return_value=SimpleNamespace(status_code=200, content=b"{}")) + monkeypatch.setattr(helpers.requests, "post", post) + helpers.analysis_request("pca", body) + assert json.loads(post.call_args.kwargs["data"]) == body + steps = [step.name for step in request.preprocessing] + assert steps.count("missing_values_handler") == 1 + + +@pytest.mark.parametrize("fault", ["connection", "broker_start", "scale_down"]) +def test_recovery_waits_for_controller_restoration_even_when_it_fails( + monkeypatch, fault +): + commands = [] + starts = 0 + readiness = AsyncMock() + monkeypatch.setattr(prod, "ready", readiness) + + async def kubectl(*args): + nonlocal starts + commands.append(args) + if fault == "scale_down" and args[0] == "scale" and args[-1] == "--replicas=0": + raise RuntimeError("scale down failed") + if args[-1] == "start_app": + starts += 1 + if fault == "broker_start" and starts == 1: + raise subprocess.CalledProcessError(1, "kubectl") + return SimpleNamespace(stdout='{"items": []}') + + monkeypatch.setattr(prod, "kubectl", kubectl) + sent = SimpleNamespace(correlation_id="recovery-request") + monkeypatch.setattr(prod, "submit", AsyncMock(return_value=sent)) + queue = SimpleNamespace(get=AsyncMock(return_value=sent)) + delivery_channel = SimpleNamespace( + get_queue=AsyncMock(return_value=queue), close=AsyncMock() + ) + connection = SimpleNamespace( + channel=AsyncMock(return_value=delivery_channel), + ready=AsyncMock(side_effect=TimeoutError("broker reconnect timed out")), + ) + broker_exited = [] + + @asynccontextmanager + async def broker(): + try: + yield connection, object(), object() + finally: + broker_exited.append(readiness.await_count) + + monkeypatch.setattr(prod, "broker", broker) + + expected = { + "connection": TimeoutError, + "broker_start": subprocess.CalledProcessError, + "scale_down": RuntimeError, + }[fault] + with pytest.raises(expected): + prod.test_redelivery_survives_broker_restart_and_controller_replacement() + + scale_up = ("scale", "deployment/exaflow-controller-deployment", "--replicas=1") + rollout = ( + "rollout", + "status", + "deployment/exaflow-controller-deployment", + "--timeout=180s", + ) + assert commands[-2:] == [scale_up, rollout] + assert readiness.await_count == 2 + assert broker_exited == [2] + if fault == "broker_start": + assert starts == 2