From a56a754fdc63139eccf07ac4d4bb4ebc5dfe48f4 Mon Sep 17 00:00:00 2001 From: Connor Carpenter Date: Wed, 22 Jul 2026 21:33:47 -0700 Subject: [PATCH 1/3] refactor(api): remove unused RPCs Remove five RPCs that no OpenEngine client exercises today. The Dynamo sidecar -- the single client for the vLLM, SGLang, and TensorRT-LLM OpenEngine servers -- never issues any of them: - Inference.Embed / Classify / Score: the sidecar only calls Generate; the non-generative inference plane has no consumer. - Control.SubscribeKvEvents: KV routing is driven entirely by GetKvEventSources plus the advertised out-of-band (ZMQ) event stream, never the in-band subscription. - Control.SubscribeRuntimeEvents: no client or server implementation. Drop the now-unused classification/embedding/scoring imports from the top-level service file and sync the docs service overview. The backing message schemas (embedding/scoring/classification, KV/runtime event envelopes) remain for now; pruning those and their model.proto capability references is a follow-up. Signed-off-by: Connor Carpenter --- docs/api.md | 9 --------- proto/openengine/v1/openengine.proto | 12 ------------ 2 files changed, 21 deletions(-) diff --git a/docs/api.md b/docs/api.md index 807c068..25f60be 100644 --- a/docs/api.md +++ b/docs/api.md @@ -23,11 +23,6 @@ import "google/protobuf/struct.proto"; service Inference { // Core inference path. rpc Generate(GenerateRequest) returns (stream GenerateResponse); - - // Non-generative inference paths. - rpc Embed(EmbedRequest) returns (EmbedResponse); - rpc Classify(ClassifyRequest) returns (ClassifyResponse); - rpc Score(ScoreRequest) returns (ScoreResponse); } service Control { @@ -49,10 +44,6 @@ service Control { // Disaggregated serving / KV transfer. rpc GetKvConnectorInfo(GetKvConnectorInfoRequest) returns (KvConnectorInfo); rpc GetKvEventSources(GetKvEventSourcesRequest) returns (GetKvEventSourcesResponse); - rpc SubscribeKvEvents(SubscribeKvEventsRequest) returns (stream SubscribeKvEventsResponse); - - // Structured runtime events for planners/controllers. - rpc SubscribeRuntimeEvents(SubscribeRuntimeEventsRequest) returns (stream SubscribeRuntimeEventsResponse); } ``` diff --git a/proto/openengine/v1/openengine.proto b/proto/openengine/v1/openengine.proto index 8a50f7c..1d985e8 100644 --- a/proto/openengine/v1/openengine.proto +++ b/proto/openengine/v1/openengine.proto @@ -16,15 +16,12 @@ syntax = "proto3"; package openengine.v1; -import "openengine/v1/classification.proto"; -import "openengine/v1/embedding.proto"; import "openengine/v1/generation.proto"; import "openengine/v1/kv.proto"; import "openengine/v1/lifecycle.proto"; import "openengine/v1/lora.proto"; import "openengine/v1/model.proto"; import "openengine/v1/observability.proto"; -import "openengine/v1/scoring.proto"; import "openengine/v1/server.proto"; // Standard ASCII gRPC request metadata: @@ -42,11 +39,6 @@ import "openengine/v1/server.proto"; service Inference { // Core inference path. rpc Generate(GenerateRequest) returns (stream GenerateResponse); - - // Non-generative inference paths. - rpc Embed(EmbedRequest) returns (EmbedResponse); - rpc Classify(ClassifyRequest) returns (ClassifyResponse); - rpc Score(ScoreRequest) returns (ScoreResponse); } service Control { @@ -68,8 +60,4 @@ service Control { // Disaggregated serving / KV transfer. rpc GetKvConnectorInfo(GetKvConnectorInfoRequest) returns (KvConnectorInfo); rpc GetKvEventSources(GetKvEventSourcesRequest) returns (GetKvEventSourcesResponse); - rpc SubscribeKvEvents(SubscribeKvEventsRequest) returns (stream SubscribeKvEventsResponse); - - // Structured runtime events for planners/controllers. - rpc SubscribeRuntimeEvents(SubscribeRuntimeEventsRequest) returns (stream SubscribeRuntimeEventsResponse); } From 2c45d264c35ec446e53cd6c96c5084c0003cb01f Mon Sep 17 00:00:00 2001 From: Connor Carpenter Date: Wed, 22 Jul 2026 21:40:43 -0700 Subject: [PATCH 2/3] refactor(api): drop dead task/runtime schemas, restore SubscribeKvEvents Correct the previous commit and remove the message schemas behind the genuinely-unused RPCs. SubscribeKvEvents is NOT unused: the Dynamo sidecar consumes it on the grpc/protobuf KV-event transport advertised by GetKvEventSources (kv.rs subscribe_loop -> Control::subscribe_kv_events), decoding the KvEventBatch/KvEvent/BlockStored payloads. The prior commit removed it based on a grep that only matched the bare method name and missed the snake_case call site. Restore the RPC; its messages in kv.proto stay. Remove the schemas for the four RPCs that are actually dead (no client call site anywhere in the sidecar): - Delete embedding.proto, classification.proto, scoring.proto and their shared tasks.proto vocabulary (Embed/Classify/Score). - Drop ModelInfo.tasks (field 27) and the TaskCapabilities message from model.proto. - Remove SubscribeRuntimeEvents{Request,Response}, RuntimeEvent, and RuntimeEventType from observability.proto (GetLoad's LoadInfo stays); drop its now-unused error.proto import. - Sync docs/api.md (service overview, Model identity block, delete the Non-generative task API and Runtime observability sections) and the proto/root README tables. Validated with protoc (descriptors build) and markdownlint-cli2 (clean). Signed-off-by: Connor Carpenter --- README.md | 1 - docs/api.md | 427 +---------------------- proto/openengine/v1/README.md | 6 +- proto/openengine/v1/classification.proto | 50 --- proto/openengine/v1/embedding.proto | 69 ---- proto/openengine/v1/model.proto | 10 - proto/openengine/v1/observability.proto | 29 +- proto/openengine/v1/openengine.proto | 1 + proto/openengine/v1/scoring.proto | 80 ----- proto/openengine/v1/tasks.proto | 52 --- 10 files changed, 4 insertions(+), 721 deletions(-) delete mode 100644 proto/openengine/v1/classification.proto delete mode 100644 proto/openengine/v1/embedding.proto delete mode 100644 proto/openengine/v1/scoring.proto delete mode 100644 proto/openengine/v1/tasks.proto diff --git a/README.md b/README.md index 4dcd880..0adc338 100644 --- a/README.md +++ b/README.md @@ -106,7 +106,6 @@ The canonical schema is organized by domain under | Area | What the contract provides | | --------------------- | ----------------------------------------------------------------------------------------------------------------- | | Portable generation | Text or token input, sampling, stopping, transport priorities, multiple sequences, and deterministic seeds | -| Non-generative tasks | Typed embedding, classification, and grouped query/candidate scoring with stable correlation | | Structured output | JSON Schema, JSON object, regex, EBNF grammar, structural tags, and fixed choices | | Token information | Prompt and output logprobs, ranks, candidate-token selection, per-token records, and streamed text deltas | | Discovery | Server identity, deployment capacity, model limits, topology, parsers, and inference capabilities | diff --git a/docs/api.md b/docs/api.md index 25f60be..3989794 100644 --- a/docs/api.md +++ b/docs/api.md @@ -44,6 +44,7 @@ service Control { // Disaggregated serving / KV transfer. rpc GetKvConnectorInfo(GetKvConnectorInfoRequest) returns (KvConnectorInfo); rpc GetKvEventSources(GetKvEventSourcesRequest) returns (GetKvEventSourcesResponse); + rpc SubscribeKvEvents(SubscribeKvEventsRequest) returns (stream SubscribeKvEventsResponse); } ``` @@ -203,7 +204,6 @@ message ModelInfo { string reasoning_parser = 25; string tool_call_parser = 26; - TaskCapabilities tasks = 27; // Optional non-generative task support for this model. google.protobuf.Struct extra = 28; // Engine-specific, non-portable; read opportunistically. } @@ -264,351 +264,6 @@ the LoRA lifecycle RPCs on `Control`. --- -## Non-generative task API - -`Embed`, `Classify`, and `Score` are unary inference operations. They share a -request context and typed inputs, but retain task-specific options and outputs. -Support is optional per model and advertised through `ModelInfo.tasks`. - -```protobuf -message TaskRequestContext { - string request_id = 1; - string model = 2; - string lora_name = 3; - google.protobuf.Struct extra = 4; // Engine-specific, non-portable; may be ignored. -} - -message TaskInput { - string item_id = 1; - oneof input { - string text = 2; - TokenIds token_ids = 3; - MultimodalTaskInput multimodal = 4; - } -} - -message MultimodalTaskInput { - oneof prompt { - string text = 1; - TokenIds token_ids = 2; - } - repeated MediaItem media = 3; -} - -message DenseFloatTensor { - repeated uint64 shape = 1; - repeated float values = 2; -} - -message SparseFloatTensor { - repeated uint64 shape = 1; - repeated uint64 indices = 2; - repeated float values = 3; -} - -message TaskUsage { - uint64 input_tokens = 1; - optional uint64 cached_input_tokens = 2; -} - -enum TaskInputType { - TASK_INPUT_TYPE_UNSPECIFIED = 0; - TASK_INPUT_TYPE_TEXT = 1; - TASK_INPUT_TYPE_TOKEN_IDS = 2; - TASK_INPUT_TYPE_MULTIMODAL = 3; -} - -enum TaskOutputGranularity { - TASK_OUTPUT_GRANULARITY_UNSPECIFIED = 0; - TASK_OUTPUT_GRANULARITY_SEQUENCE = 1; - TASK_OUTPUT_GRANULARITY_TOKEN = 2; -} - -enum EmbeddingEncoding { - EMBEDDING_ENCODING_UNSPECIFIED = 0; - EMBEDDING_ENCODING_DENSE = 1; - EMBEDDING_ENCODING_SPARSE = 2; -} - -enum TaskValueSemantics { - TASK_VALUE_SEMANTICS_UNSPECIFIED = 0; - TASK_VALUE_SEMANTICS_LOGITS = 1; - TASK_VALUE_SEMANTICS_PROBABILITIES = 2; - TASK_VALUE_SEMANTICS_LOG_PROBABILITIES = 3; - TASK_VALUE_SEMANTICS_SIMILARITY = 4; - TASK_VALUE_SEMANTICS_RELEVANCE = 5; - TASK_VALUE_SEMANTICS_REWARD = 6; - TASK_VALUE_SEMANTICS_MODEL_DEFINED = 7; -} - -enum ScoreNormalization { - SCORE_NORMALIZATION_UNSPECIFIED = 0; - SCORE_NORMALIZATION_NONE = 1; - SCORE_NORMALIZATION_SOFTMAX = 2; -} -``` - -`TaskRequestContext.request_id` and `model` are required and non-empty. Request -IDs share the same namespace and abort semantics as generation request IDs. -A non-empty `lora_name` selects an already loaded adapter. Clients use -`openengine-priority` only when the corresponding task capability advertises -priority support, and use `lora_name` only when it advertises LoRA support. - -Each request batch must be non-empty. `item_id` is required and unique within -an embed/classify batch, and every query/candidate item ID is unique within one -score group. Exactly one `TaskInput.input` variant is set. A multimodal input must -contain at least one `MediaItem`; its optional prompt is either text or token -IDs, never both. Media ordering and validation follow `GenerateRequest.media`. - -`DenseFloatTensor` is row-major FP32 data. Every dimension is greater than zero, -the product of `shape` equals `values.size()`, and every value is finite. -`SparseFloatTensor` uses flattened row-major indices; `indices` and `values` -have equal length, indices are unique and strictly increasing, and every index -is smaller than the product of `shape`. A scalar is encoded with shape `[1]`, -not an empty shape. `cached_input_tokens`, when present, does not exceed -`input_tokens`. - -### Embedding - -```protobuf -message EmbedRequest { - TaskRequestContext context = 1; - repeated TaskInput inputs = 2; - EmbedOptions options = 3; -} - -message EmbedOptions { - optional TaskOutputGranularity granularity = 1; - optional bool normalize = 2; - optional uint32 dimensions = 3; - optional EmbeddingEncoding encoding = 4; -} - -message EmbedResponse { - string request_id = 1; - repeated EmbeddingOutput outputs = 2; - TaskUsage usage = 3; -} - -message EmbeddingOutput { - string item_id = 1; - optional uint32 input_index = 2; - TaskOutputGranularity granularity = 3; - oneof embedding { - DenseFloatTensor dense = 4; - SparseFloatTensor sparse = 5; - } - repeated uint32 token_ids = 6; -} -``` - -Absent embedding options select model defaults. An explicit granularity, -normalization, or encoding request must be implemented exactly or rejected. -`dimensions` must be greater than zero and requests dimensionality reduction; -it does not permit padding a smaller model output. - -Sequence embeddings have shape `[dimension]`. Token embeddings have shape -`[token_count, dimension]`; `token_ids`, when returned, has `token_count` -entries aligned to the first tensor dimension. Outputs preserve request order, -and `input_index` is present even for index zero and identifies the matching -input. A successful response contains exactly one output for every request -input. - -### Classification - -```protobuf -message ClassifyRequest { - TaskRequestContext context = 1; - repeated TaskInput inputs = 2; - ClassifyOptions options = 3; -} - -message ClassifyOptions { - optional TaskOutputGranularity granularity = 1; - optional TaskValueSemantics output_semantics = 2; -} - -message ClassifyResponse { - string request_id = 1; - repeated ClassificationOutput outputs = 2; - TaskUsage usage = 3; -} - -message ClassificationOutput { - string item_id = 1; - optional uint32 input_index = 2; - TaskOutputGranularity granularity = 3; - TaskValueSemantics semantics = 4; - DenseFloatTensor scores = 5; - repeated string labels = 6; - repeated uint32 token_ids = 7; -} -``` - -Classification normally returns logits, probabilities, log probabilities, or -model-defined values. The engine reports the actual non-`UNSPECIFIED` -semantics. Sequence classification has shape `[class_count]`; token -classification has shape `[token_count, class_count]`. When labels are -returned, their count equals `class_count` and their order matches the final -tensor dimension. Token IDs follow the same alignment rule as token -embeddings. Outputs preserve request order and correlation. A successful -response contains exactly one output for every request input, and `input_index` -is present even for index zero. - -### Scoring - -```protobuf -message ScoreRequest { - TaskRequestContext context = 1; - repeated ScoreGroup groups = 2; - ScoreOptions options = 3; -} - -message ScoreGroup { - string group_id = 1; - TaskInput query = 2; - repeated TaskInput candidates = 3; -} - -message ScoreOptions { - optional TaskOutputGranularity granularity = 1; - optional TaskValueSemantics output_semantics = 2; - ScoreNormalization normalization = 3; - repeated uint32 label_token_ids = 4; - string instruction = 5; -} - -message ScoreResponse { - string request_id = 1; - repeated ScoreGroupOutput groups = 2; - TaskOutputGranularity granularity = 3; - TaskValueSemantics semantics = 4; - optional bool higher_is_better = 5; - ScoreNormalization normalization = 6; - repeated uint32 label_token_ids = 7; - TaskUsage usage = 8; -} - -message ScoreGroupOutput { - string group_id = 1; - optional uint32 group_index = 2; - repeated ScoreCandidateOutput candidates = 3; -} - -message ScoreCandidateOutput { - string candidate_id = 1; - optional uint32 candidate_index = 2; - DenseFloatTensor scores = 3; - repeated uint32 token_ids = 4; -} -``` - -Every score request has at least one group. Group IDs are unique, every group -has one query and at least one candidate, and candidate IDs are unique within -the group. Repeating groups represents N:N paired scoring; one group with many -candidates represents the optimized 1:N path used by rerankers and multi-item -scoring engines. - -Absent granularity and output semantics select model defaults. An explicit -value must be supported. `SCORE_NORMALIZATION_UNSPECIFIED` selects the model -default, `NONE` requests native unnormalized values, and `SOFTMAX` requests -normalization across each returned label vector. Duplicate label token IDs are -invalid. When `label_token_ids` is non-empty, the engine performs causal-model -label-token scoring and the response repeats those IDs in the same order. -When it is empty, the engine performs its advertised model-native scoring -operation. A non-empty `instruction` is valid only when instruction support is -advertised; template selection and rendering remain engine-owned. - -`ScoreResponse` reports the actual granularity, semantics, normalization, and -label-token order. Group and candidate results preserve request order and carry -their present zero-based original indexes, including index zero. A successful -response contains every group and candidate exactly once; partial success is -not represented. Token-granularity candidate tensors may return aligned token -IDs. The engine never sorts candidates or echoes source documents. - -A gateway may derive reranking only when every candidate has exactly one score -and `higher_is_better` is present. It stable-sorts using that direction, breaks -ties by `candidate_index`, applies its external `top_n`, and attaches documents -from gateway-owned request state. Reranking is response shaping, not a separate -OpenEngine inference capability. - -### Task capability discovery - -```protobuf -message TaskCapabilities { - EmbedCapabilities embed = 1; - ClassifyCapabilities classify = 2; - ScoreCapabilities score = 3; -} - -message EmbedCapabilities { - optional bool supported = 1; - repeated TaskInputType input_types = 2; - repeated TaskOutputGranularity granularities = 3; - repeated EmbeddingEncoding encodings = 4; - optional uint32 dimension = 5; - optional uint32 max_batch_size = 6; - optional uint64 max_output_values_per_item = 7; - optional bool supports_priority = 8; - optional bool supports_lora = 9; - optional bool supports_normalization = 10; - optional bool supports_dimension_override = 11; - repeated Modality modalities = 12; -} - -message ClassifyCapabilities { - optional bool supported = 1; - repeated TaskInputType input_types = 2; - repeated TaskOutputGranularity granularities = 3; - repeated TaskValueSemantics semantics = 4; - optional uint32 max_batch_size = 5; - optional uint64 max_output_values_per_item = 6; - optional bool supports_priority = 7; - optional bool supports_lora = 8; - repeated Modality modalities = 9; -} - -message ScoreCapabilities { - optional bool supported = 1; - repeated TaskInputType input_types = 2; - repeated TaskOutputGranularity granularities = 3; - repeated TaskValueSemantics semantics = 4; - repeated ScoreNormalization normalizations = 5; - optional bool supports_label_token_scoring = 6; - optional bool supports_instruction = 7; - optional uint32 max_groups = 8; - optional uint32 max_candidates_per_group = 9; - optional uint64 max_output_values_per_candidate = 10; - optional bool supports_priority = 11; - optional bool supports_lora = 12; - optional bool higher_is_better = 13; - repeated Modality modalities = 14; - optional uint32 max_label_token_ids = 15; -} -``` - -An absent `ModelInfo.tasks` means the model does not advertise non-generative -task support. Within it, an absent task capability is unreported; a present -capability uses `supported` presence to distinguish unreported support from an -explicit `true` or `false`. Capability lists never contain `UNSPECIFIED`. -Reported dimensions and limits are greater than zero. `dimension` is present -only when a model has one fixed native embedding dimension. A rankable native -score advertises its default direction through `higher_is_better`; the response -still reports the actual direction for each request. `modalities` is meaningful -only when the task includes `TASK_INPUT_TYPE_MULTIMODAL`, and never contains -`MODALITY_UNSPECIFIED`. -For dense embeddings, `max_output_values_per_item` limits the flattened element -count; for sparse embeddings, it limits the number of returned nonzero values. - -The server rejects an unsupported task with `FAILED_PRECONDITION`, an unknown -model with `NOT_FOUND`, malformed input or unsupported explicit options with -`INVALID_ARGUMENT`, and admission/capacity exhaustion with -`RESOURCE_EXHAUSTED`. A successful unary response is terminal. Client -cancellation or deadline expiration stops queued or running work, and -`Abort(request_id)` follows the same semantics as generation. - ---- - ## LoRA lifecycle ```protobuf @@ -1184,86 +839,6 @@ sessions remain. --- -## Runtime observability - -`GetLoad` returns a structured point-in-time load snapshot for schedulers and admission controllers. It is not a replacement for Prometheus metrics; it is the engine-facing control-plane signal for request routing and overload decisions. - -```protobuf -message GetLoadRequest { - bool include_per_rank = 1; -} - -message LoadInfo { - string instance_id = 1; - optional uint64 timestamp_unix_nanos = 2; - optional uint32 running_requests = 3; - optional uint32 queued_requests = 4; - optional uint32 active_kv_sessions = 5; - optional uint64 used_kv_blocks = 6; - optional uint64 total_kv_blocks = 7; - optional uint64 running_tokens = 8; - optional uint64 waiting_tokens = 9; - optional uint32 prefill_batch_size = 10; - optional uint32 decode_batch_size = 11; - repeated RankLoadInfo ranks = 20; - google.protobuf.Struct attributes = 30; -} - -message RankLoadInfo { - optional uint32 data_parallel_rank = 1; - optional uint32 running_requests = 2; - optional uint32 queued_requests = 3; - optional uint64 used_kv_blocks = 4; - optional uint64 total_kv_blocks = 5; - optional uint32 prefill_batch_size = 6; - optional uint32 decode_batch_size = 7; -} -``` - -Every load scalar has explicit presence. Absent means unavailable in that -engine or snapshot; present zero means the measured load is zero. - -`LoadInfo.attributes` and `RuntimeEvent.attributes` carry engine-specific metrics as a -`google.protobuf.Struct`, so numeric, boolean, and list values keep their JSON -type on the wire instead of being flattened to strings. `Struct` numbers are -IEEE-754 doubles (exact only to 2^53); carry larger integers as strings. - -Runtime event stream: - -```protobuf -message SubscribeRuntimeEventsRequest { - repeated RuntimeEventType types = 1; -} - -message SubscribeRuntimeEventsResponse { - oneof event { - RuntimeEvent runtime_event = 1; - EngineError error = 2; - } -} - -enum RuntimeEventType { - RUNTIME_EVENT_TYPE_UNSPECIFIED = 0; - RUNTIME_EVENT_TYPE_FORWARD_PASS = 1; - RUNTIME_EVENT_TYPE_BATCH = 2; - RUNTIME_EVENT_TYPE_QUEUE = 3; - RUNTIME_EVENT_TYPE_TRANSFER = 4; -} - -message RuntimeEvent { - string event_id = 1; - uint64 timestamp_unix_nanos = 2; - RuntimeEventType type = 3; - google.protobuf.Struct attributes = 4; -} -``` - -After subscription acceptance, an application failure is the final -`SubscribeRuntimeEventsResponse` with `error` set. No runtime event may follow -it, and the server closes the stream with gRPC `OK`. - ---- - ## Standard errors ```protobuf diff --git a/proto/openengine/v1/README.md b/proto/openengine/v1/README.md index f9f8a0c..fcc090f 100644 --- a/proto/openengine/v1/README.md +++ b/proto/openengine/v1/README.md @@ -15,14 +15,10 @@ share the same package and together define the API. | [`server.proto`](server.proto) | Server identity, deployment capacity, engine roles, and parallelism | | [`model.proto`](model.proto) | Model metadata and inference capabilities | | [`generation.proto`](generation.proto) | Generation requests, parameters, streamed events, and usage | -| [`tasks.proto`](tasks.proto) | Shared non-generative task request and output vocabulary | -| [`embedding.proto`](embedding.proto) | Dense and sparse embedding inference | -| [`classification.proto`](classification.proto) | Sequence and token classification inference | -| [`scoring.proto`](scoring.proto) | Grouped query and candidate scoring inference | | [`lora.proto`](lora.proto) | LoRA adapter lifecycle | | [`kv.proto`](kv.proto) | KV sessions, connector discovery, and cache events | | [`lifecycle.proto`](lifecycle.proto) | Health, abort, and drain operations | -| [`observability.proto`](observability.proto) | Load snapshots and runtime events | +| [`observability.proto`](observability.proto) | Load snapshots | | [`error.proto`](error.proto) | Terminal errors for accepted streaming requests | Generate bindings from every `.proto` file in this directory. Compiling only diff --git a/proto/openengine/v1/classification.proto b/proto/openengine/v1/classification.proto deleted file mode 100644 index 20b65af..0000000 --- a/proto/openengine/v1/classification.proto +++ /dev/null @@ -1,50 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -// Portable sequence and token classification inference. - -syntax = "proto3"; - -package openengine.v1; - -import "openengine/v1/input.proto"; -import "openengine/v1/tasks.proto"; - -message ClassifyCapabilities { - optional bool supported = 1; - repeated TaskInputType input_types = 2; - repeated TaskOutputGranularity granularities = 3; - repeated TaskValueSemantics semantics = 4; - optional uint32 max_batch_size = 5; - optional uint64 max_output_values_per_item = 6; - optional bool supports_priority = 7; - optional bool supports_lora = 8; - repeated Modality modalities = 9; -} - -message ClassifyRequest { - TaskRequestContext context = 1; - repeated TaskInput inputs = 2; - ClassifyOptions options = 3; -} - -message ClassifyOptions { - optional TaskOutputGranularity granularity = 1; - optional TaskValueSemantics output_semantics = 2; -} - -message ClassifyResponse { - string request_id = 1; - repeated ClassificationOutput outputs = 2; - TaskUsage usage = 3; -} - -message ClassificationOutput { - string item_id = 1; - optional uint32 input_index = 2; - TaskOutputGranularity granularity = 3; - TaskValueSemantics semantics = 4; - DenseFloatTensor scores = 5; - repeated string labels = 6; - repeated uint32 token_ids = 7; -} diff --git a/proto/openengine/v1/embedding.proto b/proto/openengine/v1/embedding.proto deleted file mode 100644 index 70b7fe6..0000000 --- a/proto/openengine/v1/embedding.proto +++ /dev/null @@ -1,69 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -// Portable dense and sparse embedding inference. - -syntax = "proto3"; - -package openengine.v1; - -import "openengine/v1/input.proto"; -import "openengine/v1/tasks.proto"; - -enum EmbeddingEncoding { - EMBEDDING_ENCODING_UNSPECIFIED = 0; - EMBEDDING_ENCODING_DENSE = 1; - EMBEDDING_ENCODING_SPARSE = 2; -} - -// Row-major sparse FP32 tensor with flattened, strictly increasing indices. -message SparseFloatTensor { - repeated uint64 shape = 1; - repeated uint64 indices = 2; - repeated float values = 3; -} - -message EmbedCapabilities { - optional bool supported = 1; - repeated TaskInputType input_types = 2; - repeated TaskOutputGranularity granularities = 3; - repeated EmbeddingEncoding encodings = 4; - optional uint32 dimension = 5; - optional uint32 max_batch_size = 6; - optional uint64 max_output_values_per_item = 7; - optional bool supports_priority = 8; - optional bool supports_lora = 9; - optional bool supports_normalization = 10; - optional bool supports_dimension_override = 11; - repeated Modality modalities = 12; -} - -message EmbedRequest { - TaskRequestContext context = 1; - repeated TaskInput inputs = 2; - EmbedOptions options = 3; -} - -message EmbedOptions { - optional TaskOutputGranularity granularity = 1; - optional bool normalize = 2; - optional uint32 dimensions = 3; - optional EmbeddingEncoding encoding = 4; -} - -message EmbedResponse { - string request_id = 1; - repeated EmbeddingOutput outputs = 2; - TaskUsage usage = 3; -} - -message EmbeddingOutput { - string item_id = 1; - optional uint32 input_index = 2; - TaskOutputGranularity granularity = 3; - oneof embedding { - DenseFloatTensor dense = 4; - SparseFloatTensor sparse = 5; - } - repeated uint32 token_ids = 6; -} diff --git a/proto/openengine/v1/model.proto b/proto/openengine/v1/model.proto index de92175..5995a20 100644 --- a/proto/openengine/v1/model.proto +++ b/proto/openengine/v1/model.proto @@ -8,9 +8,6 @@ syntax = "proto3"; package openengine.v1; import "google/protobuf/struct.proto"; -import "openengine/v1/classification.proto"; -import "openengine/v1/embedding.proto"; -import "openengine/v1/scoring.proto"; message GetModelInfoRequest { string model = 1; @@ -36,8 +33,6 @@ message ModelInfo { string reasoning_parser = 25; string tool_call_parser = 26; - TaskCapabilities tasks = 27; // Optional non-generative task support for this model. - // Engine-specific model metadata with no portable field (e.g. kv-cache dtype, // quantization, capabilities not yet standardized). NOT part of the portable // contract: clients read it opportunistically and never depend on it for @@ -45,11 +40,6 @@ message ModelInfo { google.protobuf.Struct extra = 28; } -message TaskCapabilities { - EmbedCapabilities embed = 1; - ClassifyCapabilities classify = 2; - ScoreCapabilities score = 3; -} message GenerationCapabilities { LogprobCapabilities prompt_logprobs = 1; diff --git a/proto/openengine/v1/observability.proto b/proto/openengine/v1/observability.proto index ed18f11..d254abd 100644 --- a/proto/openengine/v1/observability.proto +++ b/proto/openengine/v1/observability.proto @@ -1,14 +1,13 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -// Load snapshots and structured runtime events. +// Load snapshots. syntax = "proto3"; package openengine.v1; import "google/protobuf/struct.proto"; -import "openengine/v1/error.proto"; message GetLoadRequest { bool include_per_rank = 1; @@ -39,29 +38,3 @@ message RankLoadInfo { optional uint32 prefill_batch_size = 6; optional uint32 decode_batch_size = 7; } - -message SubscribeRuntimeEventsRequest { - repeated RuntimeEventType types = 1; -} - -message SubscribeRuntimeEventsResponse { - oneof event { - RuntimeEvent runtime_event = 1; - EngineError error = 2; // Terminal. - } -} - -enum RuntimeEventType { - RUNTIME_EVENT_TYPE_UNSPECIFIED = 0; - RUNTIME_EVENT_TYPE_FORWARD_PASS = 1; - RUNTIME_EVENT_TYPE_BATCH = 2; - RUNTIME_EVENT_TYPE_QUEUE = 3; - RUNTIME_EVENT_TYPE_TRANSFER = 4; -} - -message RuntimeEvent { - string event_id = 1; - uint64 timestamp_unix_nanos = 2; - RuntimeEventType type = 3; - google.protobuf.Struct attributes = 4; -} diff --git a/proto/openengine/v1/openengine.proto b/proto/openengine/v1/openengine.proto index 1d985e8..077ccd3 100644 --- a/proto/openengine/v1/openengine.proto +++ b/proto/openengine/v1/openengine.proto @@ -60,4 +60,5 @@ service Control { // Disaggregated serving / KV transfer. rpc GetKvConnectorInfo(GetKvConnectorInfoRequest) returns (KvConnectorInfo); rpc GetKvEventSources(GetKvEventSourcesRequest) returns (GetKvEventSourcesResponse); + rpc SubscribeKvEvents(SubscribeKvEventsRequest) returns (stream SubscribeKvEventsResponse); } diff --git a/proto/openengine/v1/scoring.proto b/proto/openengine/v1/scoring.proto deleted file mode 100644 index 4887913..0000000 --- a/proto/openengine/v1/scoring.proto +++ /dev/null @@ -1,80 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -// Portable grouped query and candidate scoring inference. - -syntax = "proto3"; - -package openengine.v1; - -import "openengine/v1/input.proto"; -import "openengine/v1/tasks.proto"; - -enum ScoreNormalization { - SCORE_NORMALIZATION_UNSPECIFIED = 0; - SCORE_NORMALIZATION_NONE = 1; - SCORE_NORMALIZATION_SOFTMAX = 2; -} - -message ScoreCapabilities { - optional bool supported = 1; - repeated TaskInputType input_types = 2; - repeated TaskOutputGranularity granularities = 3; - repeated TaskValueSemantics semantics = 4; - repeated ScoreNormalization normalizations = 5; - optional bool supports_label_token_scoring = 6; - optional bool supports_instruction = 7; - optional uint32 max_groups = 8; - optional uint32 max_candidates_per_group = 9; - optional uint64 max_output_values_per_candidate = 10; - optional bool supports_priority = 11; - optional bool supports_lora = 12; - optional bool higher_is_better = 13; - repeated Modality modalities = 14; - optional uint32 max_label_token_ids = 15; -} - -message ScoreRequest { - TaskRequestContext context = 1; - repeated ScoreGroup groups = 2; - ScoreOptions options = 3; -} - -// A query and one or more candidates scored as one correlation group. -message ScoreGroup { - string group_id = 1; - TaskInput query = 2; - repeated TaskInput candidates = 3; -} - -message ScoreOptions { - optional TaskOutputGranularity granularity = 1; - optional TaskValueSemantics output_semantics = 2; - ScoreNormalization normalization = 3; - repeated uint32 label_token_ids = 4; - string instruction = 5; -} - -message ScoreResponse { - string request_id = 1; - repeated ScoreGroupOutput groups = 2; - TaskOutputGranularity granularity = 3; - TaskValueSemantics semantics = 4; - optional bool higher_is_better = 5; - ScoreNormalization normalization = 6; - repeated uint32 label_token_ids = 7; - TaskUsage usage = 8; -} - -message ScoreGroupOutput { - string group_id = 1; - optional uint32 group_index = 2; - repeated ScoreCandidateOutput candidates = 3; -} - -message ScoreCandidateOutput { - string candidate_id = 1; - optional uint32 candidate_index = 2; - DenseFloatTensor scores = 3; - repeated uint32 token_ids = 4; -} diff --git a/proto/openengine/v1/tasks.proto b/proto/openengine/v1/tasks.proto deleted file mode 100644 index b5fcadc..0000000 --- a/proto/openengine/v1/tasks.proto +++ /dev/null @@ -1,52 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -// Shared request and output vocabulary for non-generative inference tasks. - -syntax = "proto3"; - -package openengine.v1; - -import "google/protobuf/struct.proto"; - -// Request identity and model selection shared by every non-generative task. -message TaskRequestContext { - string request_id = 1; - string model = 2; - string lora_name = 3; - - // Engine-specific task parameters that have no portable field. NOT part of - // the portable contract: an engine MAY ignore keys it does not recognize, and - // a request MUST remain valid with this field empty. Clients treat it as - // best-effort. Carried as a Struct so JSON types survive the wire. - google.protobuf.Struct extra = 4; -} - -// Row-major FP32 tensor. The product of shape must equal values.size(). -message DenseFloatTensor { - repeated uint64 shape = 1; - repeated float values = 2; -} - -// Token accounting for one complete unary task request. -message TaskUsage { - uint64 input_tokens = 1; - optional uint64 cached_input_tokens = 2; -} - -enum TaskOutputGranularity { - TASK_OUTPUT_GRANULARITY_UNSPECIFIED = 0; - TASK_OUTPUT_GRANULARITY_SEQUENCE = 1; - TASK_OUTPUT_GRANULARITY_TOKEN = 2; -} - -enum TaskValueSemantics { - TASK_VALUE_SEMANTICS_UNSPECIFIED = 0; - TASK_VALUE_SEMANTICS_LOGITS = 1; - TASK_VALUE_SEMANTICS_PROBABILITIES = 2; - TASK_VALUE_SEMANTICS_LOG_PROBABILITIES = 3; - TASK_VALUE_SEMANTICS_SIMILARITY = 4; - TASK_VALUE_SEMANTICS_RELEVANCE = 5; - TASK_VALUE_SEMANTICS_REWARD = 6; - TASK_VALUE_SEMANTICS_MODEL_DEFINED = 7; -} From 4b98f856bca608a3408cb151bb55f39583c16f61 Mon Sep 17 00:00:00 2001 From: Connor Carpenter Date: Wed, 22 Jul 2026 22:21:53 -0700 Subject: [PATCH 3/3] refactor(api): fold GetKvConnectorInfo into ServerInfo Remove the GetKvConnectorInfo RPC and its GetKvConnectorInfoRequest message. The connector capabilities it returned are static per deployment and are already carried on ServerInfo.kv_connector (GetServerInfo) -- all engines populate both surfaces from the same value, and the sidecar fetched GetServerInfo and then re-fetched the identical KvConnectorInfo, discarding the first. Follow the contract's static-vs-dynamic split: static discovery data (parallelism, capacity, connector) rides on ServerInfo; only dynamic state (LoadInfo via GetLoad) keeps a dedicated poll RPC. Keep the KvConnectorInfo message -- it remains ServerInfo.kv_connector's type, now its sole source. Sync docs/api.md (service overview + KV connector section). Validated with protoc and markdownlint-cli2. Signed-off-by: Connor Carpenter --- docs/api.md | 9 +++++---- proto/openengine/v1/kv.proto | 2 -- proto/openengine/v1/openengine.proto | 3 +-- 3 files changed, 6 insertions(+), 8 deletions(-) diff --git a/docs/api.md b/docs/api.md index 3989794..2226237 100644 --- a/docs/api.md +++ b/docs/api.md @@ -41,8 +41,7 @@ service Control { rpc UnloadLora(UnloadLoraRequest) returns (UnloadLoraResponse); rpc ListLoras(ListLorasRequest) returns (ListLorasResponse); - // Disaggregated serving / KV transfer. - rpc GetKvConnectorInfo(GetKvConnectorInfoRequest) returns (KvConnectorInfo); + // Disaggregated serving / KV transfer. Connector info: ServerInfo.kv_connector. rpc GetKvEventSources(GetKvEventSourcesRequest) returns (GetKvEventSourcesResponse); rpc SubscribeKvEvents(SubscribeKvEventsRequest) returns (stream SubscribeKvEventsResponse); } @@ -618,9 +617,11 @@ OpenEngine should support two KV-event modes: 2. **Compatibility source discovery:** `GetKvEventSources` advertises existing engine-native sources such as SGLang/vLLM ZMQ publishers. -```protobuf -message GetKvConnectorInfoRequest {} +`KvConnectorInfo` describes the disaggregation transfer connector. It is static +per deployment and is reported once through `ServerInfo.kv_connector` +(`GetServerInfo`), not a dedicated RPC. +```protobuf message KvConnectorInfo { optional bool enabled = 1; string transfer_backend = 2; diff --git a/proto/openengine/v1/kv.proto b/proto/openengine/v1/kv.proto index 95a2295..af180dd 100644 --- a/proto/openengine/v1/kv.proto +++ b/proto/openengine/v1/kv.proto @@ -29,8 +29,6 @@ message KvEndpoint { string protocol = 3; // grpc, nixl, ucx, tcp, shm, etc. } -message GetKvConnectorInfoRequest {} - message KvConnectorInfo { optional bool enabled = 1; string transfer_backend = 2; diff --git a/proto/openengine/v1/openengine.proto b/proto/openengine/v1/openengine.proto index 077ccd3..101099d 100644 --- a/proto/openengine/v1/openengine.proto +++ b/proto/openengine/v1/openengine.proto @@ -57,8 +57,7 @@ service Control { rpc UnloadLora(UnloadLoraRequest) returns (UnloadLoraResponse); rpc ListLoras(ListLorasRequest) returns (ListLorasResponse); - // Disaggregated serving / KV transfer. - rpc GetKvConnectorInfo(GetKvConnectorInfoRequest) returns (KvConnectorInfo); + // Disaggregated serving / KV transfer. Connector info: ServerInfo.kv_connector. rpc GetKvEventSources(GetKvEventSourcesRequest) returns (GetKvEventSourcesResponse); rpc SubscribeKvEvents(SubscribeKvEventsRequest) returns (stream SubscribeKvEventsResponse); }