From a0d99de63fad114bd3e221523e696173cbcd7fd0 Mon Sep 17 00:00:00 2001 From: Connor Carpenter Date: Thu, 23 Jul 2026 07:37:09 -0700 Subject: [PATCH 1/2] refactor(api): remove the Drain RPC Remove Control.Drain and its DrainRequest/DrainResponse/DrainState messages. Graceful shutdown does not need an engine-facing drain RPC: the client stops admitting (discovery unregister), lets in-flight Generate streams complete, polls GetLoad for quiescence, and cancels stragglers via gRPC stream cancellation into the engine's existing Abort path. Drop lifecycle.proto's now-unused error.proto import (only DrainResponse used EngineError; Health/Abort do not). Keep the drain-adjacent state signals that stand on their own -- HEALTH_STATE_DRAINING, supports_drain on KvConnectorInfo (validated by the sidecar for disagg connectors), and ERROR_CODE_DRAINING -- since an engine still drains on its own SIGTERM. Sync docs/api.md and the README tables. Validated with protoc and markdownlint-cli2. Signed-off-by: Connor Carpenter --- README.md | 4 +-- docs/api.md | 41 +++------------------------- proto/openengine/v1/README.md | 2 +- proto/openengine/v1/lifecycle.proto | 26 +----------------- proto/openengine/v1/openengine.proto | 1 - 5 files changed, 8 insertions(+), 66 deletions(-) diff --git a/README.md b/README.md index 0adc338..49d49c3 100644 --- a/README.md +++ b/README.md @@ -109,7 +109,7 @@ The canonical schema is organized by domain under | Structured output | JSON Schema, JSON object, regex, EBNF grammar, structural tags, and fixed choices | | Token information | Prompt and output logprobs, ranks, candidate-token selection, per-token records, and streamed text deltas | | Discovery | Server identity, deployment capacity, model limits, topology, parsers, and inference capabilities | -| Lifecycle | Health checks, targeted or global abort, graceful drain, progress, and terminal failures | +| Lifecycle | Health checks and targeted or global abort | | Disaggregated serving | Prefill/decode roles, decode-context parallel topology, KV handoff, connector discovery, and cache controls | | KV-aware routing | Typed KV event streams plus discovery of engine-native event sources | | Model extensions | Multimodal inputs and LoRA adapter lifecycle | @@ -169,7 +169,7 @@ depend on it. Expect direct schema refinement during this phase. The intended adoption path is incremental: -1. Aggregated generation and non-generative tasks, discovery, health, abort, and drain. +1. Aggregated generation, discovery, health, and abort. 2. Prefill/decode roles, KV handoff, rank affinity, and KV event integration. 3. Logprobs, guided decoding, LoRA, and multimodal input as needed. diff --git a/docs/api.md b/docs/api.md index 2226237..6d8d2b0 100644 --- a/docs/api.md +++ b/docs/api.md @@ -34,7 +34,6 @@ service Control { // Health and lifecycle. rpc Health(HealthRequest) returns (HealthResponse); rpc Abort(AbortRequest) returns (AbortResponse); - rpc Drain(DrainRequest) returns (stream DrainResponse); // LoRA lifecycle. rpc LoadLora(LoadLoraRequest) returns (LoadLoraResponse); @@ -744,7 +743,7 @@ server closes the stream with gRPC `OK`. --- -## Health, abort, and drain +## Health and abort ```protobuf message HealthRequest { @@ -800,29 +799,6 @@ enum AbortStatus { ABORT_STATUS_ABORTED = 1; ABORT_STATUS_ALREADY_FINISHED = 2; } - -message DrainRequest { - bool stop_accepting_new_requests = 1; - optional uint32 deadline_ms = 2; - bool abort_after_deadline = 3; -} - -message DrainResponse { - oneof event { - DrainState state = 1; - EngineError error = 5; - } - optional uint32 in_flight_requests = 2; - optional uint32 open_kv_sessions = 3; - string message = 4; -} - -enum DrainState { - DRAIN_STATE_UNSPECIFIED = 0; - DRAIN_STATE_STARTED = 1; - DRAIN_STATE_IN_PROGRESS = 2; - DRAIN_STATE_COMPLETE = 3; -} ``` Exactly one abort target must be set. Use `all_requests {}` to abort every @@ -831,13 +807,6 @@ or KV session target returns gRPC `NOT_FOUND`; an engine that does not support abort returns gRPC `UNIMPLEMENTED`. `ALREADY_FINISHED` remains a successful idempotent outcome rather than an error. -`STARTED` and `IN_PROGRESS` are progress events; `COMPLETE` is terminal. A -failure after the drain is accepted is represented by one terminal -`EngineError`, not by a failed drain state. An absent `deadline_ms` means no -deadline; an explicit zero means the deadline is immediate. Absent progress -counts are unknown, while present zero values report that no requests or KV -sessions remain. - --- ## Standard errors @@ -883,11 +852,9 @@ accepted stream phase and report failures with non-OK gRPC status. `GenerationFinished` is terminal for its `output_index`; other output indexes may continue. The last `GenerationFinished` ends a successful aggregated or decode stream, `PrefillReady` ends a successful prefill stream, and -`EngineError` ends any failed generation stream. `DrainState.COMPLETE` and -`EngineError` terminate a drain stream. An `EngineError` also terminates a -KV-event or runtime-event subscription. No response may follow a terminal -`EngineError`. Application failure is neither a `GenerationFinished` reason nor -a failed drain state. +`EngineError` ends any failed generation stream. An `EngineError` also +terminates a KV-event subscription. No response may follow a terminal +`EngineError`. Application failure is not a `GenerationFinished` reason. `retryable` states whether the unchanged operation can succeed on retry. `retry_after_ms` is present only for retryable errors and is the recommended diff --git a/proto/openengine/v1/README.md b/proto/openengine/v1/README.md index fcc090f..83da80e 100644 --- a/proto/openengine/v1/README.md +++ b/proto/openengine/v1/README.md @@ -17,7 +17,7 @@ share the same package and together define the API. | [`generation.proto`](generation.proto) | Generation requests, parameters, streamed events, and usage | | [`lora.proto`](lora.proto) | LoRA adapter lifecycle | | [`kv.proto`](kv.proto) | KV sessions, connector discovery, and cache events | -| [`lifecycle.proto`](lifecycle.proto) | Health, abort, and drain operations | +| [`lifecycle.proto`](lifecycle.proto) | Health and abort operations | | [`observability.proto`](observability.proto) | Load snapshots | | [`error.proto`](error.proto) | Terminal errors for accepted streaming requests | diff --git a/proto/openengine/v1/lifecycle.proto b/proto/openengine/v1/lifecycle.proto index 25bf42f..741cb20 100644 --- a/proto/openengine/v1/lifecycle.proto +++ b/proto/openengine/v1/lifecycle.proto @@ -1,13 +1,12 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -// Health, abort, and drain operations. +// Health and abort operations. syntax = "proto3"; package openengine.v1; -import "openengine/v1/error.proto"; import "openengine/v1/kv.proto"; import "openengine/v1/server.proto"; @@ -64,26 +63,3 @@ enum AbortStatus { ABORT_STATUS_ABORTED = 1; ABORT_STATUS_ALREADY_FINISHED = 2; } - -message DrainRequest { - bool stop_accepting_new_requests = 1; - optional uint32 deadline_ms = 2; // Absent means no deadline; zero is immediate. - bool abort_after_deadline = 3; -} - -message DrainResponse { - oneof event { - DrainState state = 1; // Progress or successful completion. - EngineError error = 5; // Terminal failure. - } - optional uint32 in_flight_requests = 2; - optional uint32 open_kv_sessions = 3; - string message = 4; -} - -enum DrainState { - DRAIN_STATE_UNSPECIFIED = 0; - DRAIN_STATE_STARTED = 1; - DRAIN_STATE_IN_PROGRESS = 2; - DRAIN_STATE_COMPLETE = 3; -} diff --git a/proto/openengine/v1/openengine.proto b/proto/openengine/v1/openengine.proto index 101099d..50a784b 100644 --- a/proto/openengine/v1/openengine.proto +++ b/proto/openengine/v1/openengine.proto @@ -50,7 +50,6 @@ service Control { // Health and lifecycle. rpc Health(HealthRequest) returns (HealthResponse); rpc Abort(AbortRequest) returns (AbortResponse); - rpc Drain(DrainRequest) returns (stream DrainResponse); // LoRA lifecycle. rpc LoadLora(LoadLoraRequest) returns (LoadLoraResponse); From b059cc86daf52bf7fd0e73ee6f49417025b4c02f Mon Sep 17 00:00:00 2001 From: Connor Carpenter Date: Thu, 23 Jul 2026 07:58:25 -0700 Subject: [PATCH 2/2] refactor(api): consolidate proto files and trim EngineError Reduce the openengine/v1 file set from 10 to 8 by folding two small files into the domain file that owns their concern, and remove dead message/field surface. Package, message names, and field numbers are unchanged -- wire- and (whole-package) codegen-compatible. - Fold observability.proto into server.proto. Post-cleanup it held only load reporting (GetLoadRequest/LoadInfo/RankLoadInfo); "observability" was a misnomer. server.proto now covers static identity/capacity plus dynamic load -- everything the server reports about itself. - Fold input.proto into generation.proto, its sole remaining consumer, and delete the dead task-input types (TaskInput, MultimodalTaskInput, TaskInputType) orphaned when the non-generative task RPCs were removed. TokenIds/Modality/MediaItem move as-is. - Trim EngineError: drop `details` (never set or read by any engine/sidecar) and `retry_after_ms` (only ever absent), and the now-unused struct import. Keep `retryable` -- servers populate it (SGLang computes it from a 503) even though the sidecar does not yet read it. Sync docs/api.md and the proto README table. Validated with protoc and markdownlint-cli2. Signed-off-by: Connor Carpenter --- docs/api.md | 6 --- proto/openengine/v1/README.md | 6 +-- proto/openengine/v1/error.proto | 4 -- proto/openengine/v1/generation.proto | 29 +++++++++++- proto/openengine/v1/input.proto | 60 ------------------------- proto/openengine/v1/observability.proto | 40 ----------------- proto/openengine/v1/openengine.proto | 1 - proto/openengine/v1/server.proto | 32 ++++++++++++- 8 files changed, 60 insertions(+), 118 deletions(-) delete mode 100644 proto/openengine/v1/input.proto delete mode 100644 proto/openengine/v1/observability.proto diff --git a/docs/api.md b/docs/api.md index 6d8d2b0..3ed9cba 100644 --- a/docs/api.md +++ b/docs/api.md @@ -816,8 +816,6 @@ message EngineError { ErrorCode code = 1; string message = 2; bool retryable = 3; - optional uint64 retry_after_ms = 4; - google.protobuf.Struct details = 5; } enum ErrorCode { @@ -857,7 +855,3 @@ terminates a KV-event subscription. No response may follow a terminal `EngineError`. Application failure is not a `GenerationFinished` reason. `retryable` states whether the unchanged operation can succeed on retry. -`retry_after_ms` is present only for retryable errors and is the recommended -minimum delay; an explicit zero permits immediate retry. `details` contains -machine-readable error context. Stable detail keys are part of this API; -engine-specific keys should be namespaced to avoid collisions. diff --git a/proto/openengine/v1/README.md b/proto/openengine/v1/README.md index 83da80e..6feccc1 100644 --- a/proto/openengine/v1/README.md +++ b/proto/openengine/v1/README.md @@ -11,14 +11,12 @@ share the same package and together define the API. | File | Area | | --- | --- | | [`openengine.proto`](openengine.proto) | `Inference` and `Control` service declarations | -| [`input.proto`](input.proto) | Shared text, token, and multimodal inputs | -| [`server.proto`](server.proto) | Server identity, deployment capacity, engine roles, and parallelism | +| [`server.proto`](server.proto) | Server identity, deployment capacity, engine roles, parallelism, and load | | [`model.proto`](model.proto) | Model metadata and inference capabilities | -| [`generation.proto`](generation.proto) | Generation requests, parameters, streamed events, and usage | +| [`generation.proto`](generation.proto) | Generation inputs, requests, parameters, streamed events, and usage | | [`lora.proto`](lora.proto) | LoRA adapter lifecycle | | [`kv.proto`](kv.proto) | KV sessions, connector discovery, and cache events | | [`lifecycle.proto`](lifecycle.proto) | Health and abort operations | -| [`observability.proto`](observability.proto) | Load snapshots | | [`error.proto`](error.proto) | Terminal errors for accepted streaming requests | Generate bindings from every `.proto` file in this directory. Compiling only diff --git a/proto/openengine/v1/error.proto b/proto/openengine/v1/error.proto index dfba86f..2ca097b 100644 --- a/proto/openengine/v1/error.proto +++ b/proto/openengine/v1/error.proto @@ -7,16 +7,12 @@ syntax = "proto3"; package openengine.v1; -import "google/protobuf/struct.proto"; - // Accepted failures emit one terminal EngineError and close with OK. // Validation and transport failures use non-OK gRPC status instead. message EngineError { ErrorCode code = 1; string message = 2; bool retryable = 3; // Retry may succeed without changing the request. - optional uint64 retry_after_ms = 4; // Zero permits immediate retry. - google.protobuf.Struct details = 5; // Machine-readable context. } enum ErrorCode { diff --git a/proto/openengine/v1/generation.proto b/proto/openengine/v1/generation.proto index 119cdac..c1f81dd 100644 --- a/proto/openengine/v1/generation.proto +++ b/proto/openengine/v1/generation.proto @@ -1,7 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -// Generation requests, streamed events, and usage. +// Generation inputs, requests, streamed events, and usage. syntax = "proto3"; @@ -9,9 +9,34 @@ package openengine.v1; import "google/protobuf/struct.proto"; import "openengine/v1/error.proto"; -import "openengine/v1/input.proto"; import "openengine/v1/kv.proto"; +message TokenIds { + repeated uint32 ids = 1; +} + +// Multimodal modality discriminator. UNSPECIFIED means the sender left it unset. +enum Modality { + MODALITY_UNSPECIFIED = 0; + MODALITY_IMAGE = 1; + MODALITY_VIDEO = 2; + MODALITY_AUDIO = 3; +} + +// A single multimodal input. Exactly one `source` should be set. The engine +// owns fetch, decode, and preprocessing, so pre-decoded or RDMA media +// descriptors are not represented here. +message MediaItem { + Modality modality = 1; + oneof source { + string url = 2; // http(s):// -- engine fetches + string data_uri = 3; // data:;base64,<...> -- engine decodes + bytes raw_bytes = 4; // pre-fetched bytes -- engine still preprocesses + } + string mime_type = 5; // optional, hints raw_bytes decode + string uuid = 6; // optional caller id / mm_hash +} + message GenerateRequest { string request_id = 1; string model = 2; diff --git a/proto/openengine/v1/input.proto b/proto/openengine/v1/input.proto deleted file mode 100644 index da64f9e..0000000 --- a/proto/openengine/v1/input.proto +++ /dev/null @@ -1,60 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -// Portable text, token, and multimodal inference inputs. - -syntax = "proto3"; - -package openengine.v1; - -message TokenIds { - repeated uint32 ids = 1; -} - -// Multimodal modality discriminator. UNSPECIFIED means the sender left it unset. -enum Modality { - MODALITY_UNSPECIFIED = 0; - MODALITY_IMAGE = 1; - MODALITY_VIDEO = 2; - MODALITY_AUDIO = 3; -} - -// A single multimodal input. Exactly one `source` should be set. The engine -// owns fetch, decode, and preprocessing, so pre-decoded or RDMA media -// descriptors are not represented here. -message MediaItem { - Modality modality = 1; - oneof source { - string url = 2; // http(s):// -- engine fetches - string data_uri = 3; // data:;base64,<...> -- engine decodes - bytes raw_bytes = 4; // pre-fetched bytes -- engine still preprocesses - } - string mime_type = 5; // optional, hints raw_bytes decode - string uuid = 6; // optional caller id / mm_hash -} - -// Text and token prompts may accompany media in a multimodal task input. -message MultimodalTaskInput { - oneof prompt { - string text = 1; - TokenIds token_ids = 2; - } - repeated MediaItem media = 3; -} - -// One independently correlated non-generative task input. -message TaskInput { - string item_id = 1; - oneof input { - string text = 2; - TokenIds token_ids = 3; - MultimodalTaskInput multimodal = 4; - } -} - -enum TaskInputType { - TASK_INPUT_TYPE_UNSPECIFIED = 0; - TASK_INPUT_TYPE_TEXT = 1; - TASK_INPUT_TYPE_TOKEN_IDS = 2; - TASK_INPUT_TYPE_MULTIMODAL = 3; -} diff --git a/proto/openengine/v1/observability.proto b/proto/openengine/v1/observability.proto deleted file mode 100644 index d254abd..0000000 --- a/proto/openengine/v1/observability.proto +++ /dev/null @@ -1,40 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -// Load snapshots. - -syntax = "proto3"; - -package openengine.v1; - -import "google/protobuf/struct.proto"; - -message GetLoadRequest { - bool include_per_rank = 1; -} - -message LoadInfo { - string instance_id = 1; - optional uint64 timestamp_unix_nanos = 2; - optional uint32 running_requests = 3; - optional uint32 queued_requests = 4; - optional uint32 active_kv_sessions = 5; - optional uint64 used_kv_blocks = 6; - optional uint64 total_kv_blocks = 7; - optional uint64 running_tokens = 8; - optional uint64 waiting_tokens = 9; - optional uint32 prefill_batch_size = 10; - optional uint32 decode_batch_size = 11; - repeated RankLoadInfo ranks = 20; - google.protobuf.Struct attributes = 30; -} - -message RankLoadInfo { - optional uint32 data_parallel_rank = 1; - optional uint32 running_requests = 2; - optional uint32 queued_requests = 3; - optional uint64 used_kv_blocks = 4; - optional uint64 total_kv_blocks = 5; - optional uint32 prefill_batch_size = 6; - optional uint32 decode_batch_size = 7; -} diff --git a/proto/openengine/v1/openengine.proto b/proto/openengine/v1/openengine.proto index 50a784b..bd55318 100644 --- a/proto/openengine/v1/openengine.proto +++ b/proto/openengine/v1/openengine.proto @@ -21,7 +21,6 @@ import "openengine/v1/kv.proto"; import "openengine/v1/lifecycle.proto"; import "openengine/v1/lora.proto"; import "openengine/v1/model.proto"; -import "openengine/v1/observability.proto"; import "openengine/v1/server.proto"; // Standard ASCII gRPC request metadata: diff --git a/proto/openengine/v1/server.proto b/proto/openengine/v1/server.proto index 1839caa..df04f3b 100644 --- a/proto/openengine/v1/server.proto +++ b/proto/openengine/v1/server.proto @@ -1,7 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -// Server identity, deployment capacity, engine role, and parallelism metadata. +// Server identity, deployment capacity, roles, parallelism, and load. syntax = "proto3"; @@ -55,3 +55,33 @@ message ParallelismInfo { optional uint32 data_parallel_start_rank = 5; optional uint32 decode_context_parallel_size = 6; // Ranks per decode-context group; at least 1. } + +message GetLoadRequest { + bool include_per_rank = 1; +} + +message LoadInfo { + string instance_id = 1; + optional uint64 timestamp_unix_nanos = 2; + optional uint32 running_requests = 3; + optional uint32 queued_requests = 4; + optional uint32 active_kv_sessions = 5; + optional uint64 used_kv_blocks = 6; + optional uint64 total_kv_blocks = 7; + optional uint64 running_tokens = 8; + optional uint64 waiting_tokens = 9; + optional uint32 prefill_batch_size = 10; + optional uint32 decode_batch_size = 11; + repeated RankLoadInfo ranks = 20; + google.protobuf.Struct attributes = 30; +} + +message RankLoadInfo { + optional uint32 data_parallel_rank = 1; + optional uint32 running_requests = 2; + optional uint32 queued_requests = 3; + optional uint64 used_kv_blocks = 4; + optional uint64 total_kv_blocks = 5; + optional uint32 prefill_batch_size = 6; + optional uint32 decode_batch_size = 7; +}