diff --git a/README.md b/README.md index 4dcd880..0adc338 100644 --- a/README.md +++ b/README.md @@ -106,7 +106,6 @@ The canonical schema is organized by domain under | Area | What the contract provides | | --------------------- | ----------------------------------------------------------------------------------------------------------------- | | Portable generation | Text or token input, sampling, stopping, transport priorities, multiple sequences, and deterministic seeds | -| Non-generative tasks | Typed embedding, classification, and grouped query/candidate scoring with stable correlation | | Structured output | JSON Schema, JSON object, regex, EBNF grammar, structural tags, and fixed choices | | Token information | Prompt and output logprobs, ranks, candidate-token selection, per-token records, and streamed text deltas | | Discovery | Server identity, deployment capacity, model limits, topology, parsers, and inference capabilities | diff --git a/docs/api.md b/docs/api.md index 807c068..2226237 100644 --- a/docs/api.md +++ b/docs/api.md @@ -23,11 +23,6 @@ import "google/protobuf/struct.proto"; service Inference { // Core inference path. rpc Generate(GenerateRequest) returns (stream GenerateResponse); - - // Non-generative inference paths. - rpc Embed(EmbedRequest) returns (EmbedResponse); - rpc Classify(ClassifyRequest) returns (ClassifyResponse); - rpc Score(ScoreRequest) returns (ScoreResponse); } service Control { @@ -46,13 +41,9 @@ service Control { rpc UnloadLora(UnloadLoraRequest) returns (UnloadLoraResponse); rpc ListLoras(ListLorasRequest) returns (ListLorasResponse); - // Disaggregated serving / KV transfer. - rpc GetKvConnectorInfo(GetKvConnectorInfoRequest) returns (KvConnectorInfo); + // Disaggregated serving / KV transfer. Connector info: ServerInfo.kv_connector. rpc GetKvEventSources(GetKvEventSourcesRequest) returns (GetKvEventSourcesResponse); rpc SubscribeKvEvents(SubscribeKvEventsRequest) returns (stream SubscribeKvEventsResponse); - - // Structured runtime events for planners/controllers. - rpc SubscribeRuntimeEvents(SubscribeRuntimeEventsRequest) returns (stream SubscribeRuntimeEventsResponse); } ``` @@ -212,7 +203,6 @@ message ModelInfo { string reasoning_parser = 25; string tool_call_parser = 26; - TaskCapabilities tasks = 27; // Optional non-generative task support for this model. google.protobuf.Struct extra = 28; // Engine-specific, non-portable; read opportunistically. } @@ -273,351 +263,6 @@ the LoRA lifecycle RPCs on `Control`. --- -## Non-generative task API - -`Embed`, `Classify`, and `Score` are unary inference operations. They share a -request context and typed inputs, but retain task-specific options and outputs. -Support is optional per model and advertised through `ModelInfo.tasks`. - -```protobuf -message TaskRequestContext { - string request_id = 1; - string model = 2; - string lora_name = 3; - google.protobuf.Struct extra = 4; // Engine-specific, non-portable; may be ignored. -} - -message TaskInput { - string item_id = 1; - oneof input { - string text = 2; - TokenIds token_ids = 3; - MultimodalTaskInput multimodal = 4; - } -} - -message MultimodalTaskInput { - oneof prompt { - string text = 1; - TokenIds token_ids = 2; - } - repeated MediaItem media = 3; -} - -message DenseFloatTensor { - repeated uint64 shape = 1; - repeated float values = 2; -} - -message SparseFloatTensor { - repeated uint64 shape = 1; - repeated uint64 indices = 2; - repeated float values = 3; -} - -message TaskUsage { - uint64 input_tokens = 1; - optional uint64 cached_input_tokens = 2; -} - -enum TaskInputType { - TASK_INPUT_TYPE_UNSPECIFIED = 0; - TASK_INPUT_TYPE_TEXT = 1; - TASK_INPUT_TYPE_TOKEN_IDS = 2; - TASK_INPUT_TYPE_MULTIMODAL = 3; -} - -enum TaskOutputGranularity { - TASK_OUTPUT_GRANULARITY_UNSPECIFIED = 0; - TASK_OUTPUT_GRANULARITY_SEQUENCE = 1; - TASK_OUTPUT_GRANULARITY_TOKEN = 2; -} - -enum EmbeddingEncoding { - EMBEDDING_ENCODING_UNSPECIFIED = 0; - EMBEDDING_ENCODING_DENSE = 1; - EMBEDDING_ENCODING_SPARSE = 2; -} - -enum TaskValueSemantics { - TASK_VALUE_SEMANTICS_UNSPECIFIED = 0; - TASK_VALUE_SEMANTICS_LOGITS = 1; - TASK_VALUE_SEMANTICS_PROBABILITIES = 2; - TASK_VALUE_SEMANTICS_LOG_PROBABILITIES = 3; - TASK_VALUE_SEMANTICS_SIMILARITY = 4; - TASK_VALUE_SEMANTICS_RELEVANCE = 5; - TASK_VALUE_SEMANTICS_REWARD = 6; - TASK_VALUE_SEMANTICS_MODEL_DEFINED = 7; -} - -enum ScoreNormalization { - SCORE_NORMALIZATION_UNSPECIFIED = 0; - SCORE_NORMALIZATION_NONE = 1; - SCORE_NORMALIZATION_SOFTMAX = 2; -} -``` - -`TaskRequestContext.request_id` and `model` are required and non-empty. Request -IDs share the same namespace and abort semantics as generation request IDs. -A non-empty `lora_name` selects an already loaded adapter. Clients use -`openengine-priority` only when the corresponding task capability advertises -priority support, and use `lora_name` only when it advertises LoRA support. - -Each request batch must be non-empty. `item_id` is required and unique within -an embed/classify batch, and every query/candidate item ID is unique within one -score group. Exactly one `TaskInput.input` variant is set. A multimodal input must -contain at least one `MediaItem`; its optional prompt is either text or token -IDs, never both. Media ordering and validation follow `GenerateRequest.media`. - -`DenseFloatTensor` is row-major FP32 data. Every dimension is greater than zero, -the product of `shape` equals `values.size()`, and every value is finite. -`SparseFloatTensor` uses flattened row-major indices; `indices` and `values` -have equal length, indices are unique and strictly increasing, and every index -is smaller than the product of `shape`. A scalar is encoded with shape `[1]`, -not an empty shape. `cached_input_tokens`, when present, does not exceed -`input_tokens`. - -### Embedding - -```protobuf -message EmbedRequest { - TaskRequestContext context = 1; - repeated TaskInput inputs = 2; - EmbedOptions options = 3; -} - -message EmbedOptions { - optional TaskOutputGranularity granularity = 1; - optional bool normalize = 2; - optional uint32 dimensions = 3; - optional EmbeddingEncoding encoding = 4; -} - -message EmbedResponse { - string request_id = 1; - repeated EmbeddingOutput outputs = 2; - TaskUsage usage = 3; -} - -message EmbeddingOutput { - string item_id = 1; - optional uint32 input_index = 2; - TaskOutputGranularity granularity = 3; - oneof embedding { - DenseFloatTensor dense = 4; - SparseFloatTensor sparse = 5; - } - repeated uint32 token_ids = 6; -} -``` - -Absent embedding options select model defaults. An explicit granularity, -normalization, or encoding request must be implemented exactly or rejected. -`dimensions` must be greater than zero and requests dimensionality reduction; -it does not permit padding a smaller model output. - -Sequence embeddings have shape `[dimension]`. Token embeddings have shape -`[token_count, dimension]`; `token_ids`, when returned, has `token_count` -entries aligned to the first tensor dimension. Outputs preserve request order, -and `input_index` is present even for index zero and identifies the matching -input. A successful response contains exactly one output for every request -input. - -### Classification - -```protobuf -message ClassifyRequest { - TaskRequestContext context = 1; - repeated TaskInput inputs = 2; - ClassifyOptions options = 3; -} - -message ClassifyOptions { - optional TaskOutputGranularity granularity = 1; - optional TaskValueSemantics output_semantics = 2; -} - -message ClassifyResponse { - string request_id = 1; - repeated ClassificationOutput outputs = 2; - TaskUsage usage = 3; -} - -message ClassificationOutput { - string item_id = 1; - optional uint32 input_index = 2; - TaskOutputGranularity granularity = 3; - TaskValueSemantics semantics = 4; - DenseFloatTensor scores = 5; - repeated string labels = 6; - repeated uint32 token_ids = 7; -} -``` - -Classification normally returns logits, probabilities, log probabilities, or -model-defined values. The engine reports the actual non-`UNSPECIFIED` -semantics. Sequence classification has shape `[class_count]`; token -classification has shape `[token_count, class_count]`. When labels are -returned, their count equals `class_count` and their order matches the final -tensor dimension. Token IDs follow the same alignment rule as token -embeddings. Outputs preserve request order and correlation. A successful -response contains exactly one output for every request input, and `input_index` -is present even for index zero. - -### Scoring - -```protobuf -message ScoreRequest { - TaskRequestContext context = 1; - repeated ScoreGroup groups = 2; - ScoreOptions options = 3; -} - -message ScoreGroup { - string group_id = 1; - TaskInput query = 2; - repeated TaskInput candidates = 3; -} - -message ScoreOptions { - optional TaskOutputGranularity granularity = 1; - optional TaskValueSemantics output_semantics = 2; - ScoreNormalization normalization = 3; - repeated uint32 label_token_ids = 4; - string instruction = 5; -} - -message ScoreResponse { - string request_id = 1; - repeated ScoreGroupOutput groups = 2; - TaskOutputGranularity granularity = 3; - TaskValueSemantics semantics = 4; - optional bool higher_is_better = 5; - ScoreNormalization normalization = 6; - repeated uint32 label_token_ids = 7; - TaskUsage usage = 8; -} - -message ScoreGroupOutput { - string group_id = 1; - optional uint32 group_index = 2; - repeated ScoreCandidateOutput candidates = 3; -} - -message ScoreCandidateOutput { - string candidate_id = 1; - optional uint32 candidate_index = 2; - DenseFloatTensor scores = 3; - repeated uint32 token_ids = 4; -} -``` - -Every score request has at least one group. Group IDs are unique, every group -has one query and at least one candidate, and candidate IDs are unique within -the group. Repeating groups represents N:N paired scoring; one group with many -candidates represents the optimized 1:N path used by rerankers and multi-item -scoring engines. - -Absent granularity and output semantics select model defaults. An explicit -value must be supported. `SCORE_NORMALIZATION_UNSPECIFIED` selects the model -default, `NONE` requests native unnormalized values, and `SOFTMAX` requests -normalization across each returned label vector. Duplicate label token IDs are -invalid. When `label_token_ids` is non-empty, the engine performs causal-model -label-token scoring and the response repeats those IDs in the same order. -When it is empty, the engine performs its advertised model-native scoring -operation. A non-empty `instruction` is valid only when instruction support is -advertised; template selection and rendering remain engine-owned. - -`ScoreResponse` reports the actual granularity, semantics, normalization, and -label-token order. Group and candidate results preserve request order and carry -their present zero-based original indexes, including index zero. A successful -response contains every group and candidate exactly once; partial success is -not represented. Token-granularity candidate tensors may return aligned token -IDs. The engine never sorts candidates or echoes source documents. - -A gateway may derive reranking only when every candidate has exactly one score -and `higher_is_better` is present. It stable-sorts using that direction, breaks -ties by `candidate_index`, applies its external `top_n`, and attaches documents -from gateway-owned request state. Reranking is response shaping, not a separate -OpenEngine inference capability. - -### Task capability discovery - -```protobuf -message TaskCapabilities { - EmbedCapabilities embed = 1; - ClassifyCapabilities classify = 2; - ScoreCapabilities score = 3; -} - -message EmbedCapabilities { - optional bool supported = 1; - repeated TaskInputType input_types = 2; - repeated TaskOutputGranularity granularities = 3; - repeated EmbeddingEncoding encodings = 4; - optional uint32 dimension = 5; - optional uint32 max_batch_size = 6; - optional uint64 max_output_values_per_item = 7; - optional bool supports_priority = 8; - optional bool supports_lora = 9; - optional bool supports_normalization = 10; - optional bool supports_dimension_override = 11; - repeated Modality modalities = 12; -} - -message ClassifyCapabilities { - optional bool supported = 1; - repeated TaskInputType input_types = 2; - repeated TaskOutputGranularity granularities = 3; - repeated TaskValueSemantics semantics = 4; - optional uint32 max_batch_size = 5; - optional uint64 max_output_values_per_item = 6; - optional bool supports_priority = 7; - optional bool supports_lora = 8; - repeated Modality modalities = 9; -} - -message ScoreCapabilities { - optional bool supported = 1; - repeated TaskInputType input_types = 2; - repeated TaskOutputGranularity granularities = 3; - repeated TaskValueSemantics semantics = 4; - repeated ScoreNormalization normalizations = 5; - optional bool supports_label_token_scoring = 6; - optional bool supports_instruction = 7; - optional uint32 max_groups = 8; - optional uint32 max_candidates_per_group = 9; - optional uint64 max_output_values_per_candidate = 10; - optional bool supports_priority = 11; - optional bool supports_lora = 12; - optional bool higher_is_better = 13; - repeated Modality modalities = 14; - optional uint32 max_label_token_ids = 15; -} -``` - -An absent `ModelInfo.tasks` means the model does not advertise non-generative -task support. Within it, an absent task capability is unreported; a present -capability uses `supported` presence to distinguish unreported support from an -explicit `true` or `false`. Capability lists never contain `UNSPECIFIED`. -Reported dimensions and limits are greater than zero. `dimension` is present -only when a model has one fixed native embedding dimension. A rankable native -score advertises its default direction through `higher_is_better`; the response -still reports the actual direction for each request. `modalities` is meaningful -only when the task includes `TASK_INPUT_TYPE_MULTIMODAL`, and never contains -`MODALITY_UNSPECIFIED`. -For dense embeddings, `max_output_values_per_item` limits the flattened element -count; for sparse embeddings, it limits the number of returned nonzero values. - -The server rejects an unsupported task with `FAILED_PRECONDITION`, an unknown -model with `NOT_FOUND`, malformed input or unsupported explicit options with -`INVALID_ARGUMENT`, and admission/capacity exhaustion with -`RESOURCE_EXHAUSTED`. A successful unary response is terminal. Client -cancellation or deadline expiration stops queued or running work, and -`Abort(request_id)` follows the same semantics as generation. - ---- - ## LoRA lifecycle ```protobuf @@ -972,9 +617,11 @@ OpenEngine should support two KV-event modes: 2. **Compatibility source discovery:** `GetKvEventSources` advertises existing engine-native sources such as SGLang/vLLM ZMQ publishers. -```protobuf -message GetKvConnectorInfoRequest {} +`KvConnectorInfo` describes the disaggregation transfer connector. It is static +per deployment and is reported once through `ServerInfo.kv_connector` +(`GetServerInfo`), not a dedicated RPC. +```protobuf message KvConnectorInfo { optional bool enabled = 1; string transfer_backend = 2; @@ -1193,86 +840,6 @@ sessions remain. --- -## Runtime observability - -`GetLoad` returns a structured point-in-time load snapshot for schedulers and admission controllers. It is not a replacement for Prometheus metrics; it is the engine-facing control-plane signal for request routing and overload decisions. - -```protobuf -message GetLoadRequest { - bool include_per_rank = 1; -} - -message LoadInfo { - string instance_id = 1; - optional uint64 timestamp_unix_nanos = 2; - optional uint32 running_requests = 3; - optional uint32 queued_requests = 4; - optional uint32 active_kv_sessions = 5; - optional uint64 used_kv_blocks = 6; - optional uint64 total_kv_blocks = 7; - optional uint64 running_tokens = 8; - optional uint64 waiting_tokens = 9; - optional uint32 prefill_batch_size = 10; - optional uint32 decode_batch_size = 11; - repeated RankLoadInfo ranks = 20; - google.protobuf.Struct attributes = 30; -} - -message RankLoadInfo { - optional uint32 data_parallel_rank = 1; - optional uint32 running_requests = 2; - optional uint32 queued_requests = 3; - optional uint64 used_kv_blocks = 4; - optional uint64 total_kv_blocks = 5; - optional uint32 prefill_batch_size = 6; - optional uint32 decode_batch_size = 7; -} -``` - -Every load scalar has explicit presence. Absent means unavailable in that -engine or snapshot; present zero means the measured load is zero. - -`LoadInfo.attributes` and `RuntimeEvent.attributes` carry engine-specific metrics as a -`google.protobuf.Struct`, so numeric, boolean, and list values keep their JSON -type on the wire instead of being flattened to strings. `Struct` numbers are -IEEE-754 doubles (exact only to 2^53); carry larger integers as strings. - -Runtime event stream: - -```protobuf -message SubscribeRuntimeEventsRequest { - repeated RuntimeEventType types = 1; -} - -message SubscribeRuntimeEventsResponse { - oneof event { - RuntimeEvent runtime_event = 1; - EngineError error = 2; - } -} - -enum RuntimeEventType { - RUNTIME_EVENT_TYPE_UNSPECIFIED = 0; - RUNTIME_EVENT_TYPE_FORWARD_PASS = 1; - RUNTIME_EVENT_TYPE_BATCH = 2; - RUNTIME_EVENT_TYPE_QUEUE = 3; - RUNTIME_EVENT_TYPE_TRANSFER = 4; -} - -message RuntimeEvent { - string event_id = 1; - uint64 timestamp_unix_nanos = 2; - RuntimeEventType type = 3; - google.protobuf.Struct attributes = 4; -} -``` - -After subscription acceptance, an application failure is the final -`SubscribeRuntimeEventsResponse` with `error` set. No runtime event may follow -it, and the server closes the stream with gRPC `OK`. - ---- - ## Standard errors ```protobuf diff --git a/proto/openengine/v1/README.md b/proto/openengine/v1/README.md index f9f8a0c..fcc090f 100644 --- a/proto/openengine/v1/README.md +++ b/proto/openengine/v1/README.md @@ -15,14 +15,10 @@ share the same package and together define the API. | [`server.proto`](server.proto) | Server identity, deployment capacity, engine roles, and parallelism | | [`model.proto`](model.proto) | Model metadata and inference capabilities | | [`generation.proto`](generation.proto) | Generation requests, parameters, streamed events, and usage | -| [`tasks.proto`](tasks.proto) | Shared non-generative task request and output vocabulary | -| [`embedding.proto`](embedding.proto) | Dense and sparse embedding inference | -| [`classification.proto`](classification.proto) | Sequence and token classification inference | -| [`scoring.proto`](scoring.proto) | Grouped query and candidate scoring inference | | [`lora.proto`](lora.proto) | LoRA adapter lifecycle | | [`kv.proto`](kv.proto) | KV sessions, connector discovery, and cache events | | [`lifecycle.proto`](lifecycle.proto) | Health, abort, and drain operations | -| [`observability.proto`](observability.proto) | Load snapshots and runtime events | +| [`observability.proto`](observability.proto) | Load snapshots | | [`error.proto`](error.proto) | Terminal errors for accepted streaming requests | Generate bindings from every `.proto` file in this directory. Compiling only diff --git a/proto/openengine/v1/classification.proto b/proto/openengine/v1/classification.proto deleted file mode 100644 index 20b65af..0000000 --- a/proto/openengine/v1/classification.proto +++ /dev/null @@ -1,50 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -// Portable sequence and token classification inference. - -syntax = "proto3"; - -package openengine.v1; - -import "openengine/v1/input.proto"; -import "openengine/v1/tasks.proto"; - -message ClassifyCapabilities { - optional bool supported = 1; - repeated TaskInputType input_types = 2; - repeated TaskOutputGranularity granularities = 3; - repeated TaskValueSemantics semantics = 4; - optional uint32 max_batch_size = 5; - optional uint64 max_output_values_per_item = 6; - optional bool supports_priority = 7; - optional bool supports_lora = 8; - repeated Modality modalities = 9; -} - -message ClassifyRequest { - TaskRequestContext context = 1; - repeated TaskInput inputs = 2; - ClassifyOptions options = 3; -} - -message ClassifyOptions { - optional TaskOutputGranularity granularity = 1; - optional TaskValueSemantics output_semantics = 2; -} - -message ClassifyResponse { - string request_id = 1; - repeated ClassificationOutput outputs = 2; - TaskUsage usage = 3; -} - -message ClassificationOutput { - string item_id = 1; - optional uint32 input_index = 2; - TaskOutputGranularity granularity = 3; - TaskValueSemantics semantics = 4; - DenseFloatTensor scores = 5; - repeated string labels = 6; - repeated uint32 token_ids = 7; -} diff --git a/proto/openengine/v1/embedding.proto b/proto/openengine/v1/embedding.proto deleted file mode 100644 index 70b7fe6..0000000 --- a/proto/openengine/v1/embedding.proto +++ /dev/null @@ -1,69 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -// Portable dense and sparse embedding inference. - -syntax = "proto3"; - -package openengine.v1; - -import "openengine/v1/input.proto"; -import "openengine/v1/tasks.proto"; - -enum EmbeddingEncoding { - EMBEDDING_ENCODING_UNSPECIFIED = 0; - EMBEDDING_ENCODING_DENSE = 1; - EMBEDDING_ENCODING_SPARSE = 2; -} - -// Row-major sparse FP32 tensor with flattened, strictly increasing indices. -message SparseFloatTensor { - repeated uint64 shape = 1; - repeated uint64 indices = 2; - repeated float values = 3; -} - -message EmbedCapabilities { - optional bool supported = 1; - repeated TaskInputType input_types = 2; - repeated TaskOutputGranularity granularities = 3; - repeated EmbeddingEncoding encodings = 4; - optional uint32 dimension = 5; - optional uint32 max_batch_size = 6; - optional uint64 max_output_values_per_item = 7; - optional bool supports_priority = 8; - optional bool supports_lora = 9; - optional bool supports_normalization = 10; - optional bool supports_dimension_override = 11; - repeated Modality modalities = 12; -} - -message EmbedRequest { - TaskRequestContext context = 1; - repeated TaskInput inputs = 2; - EmbedOptions options = 3; -} - -message EmbedOptions { - optional TaskOutputGranularity granularity = 1; - optional bool normalize = 2; - optional uint32 dimensions = 3; - optional EmbeddingEncoding encoding = 4; -} - -message EmbedResponse { - string request_id = 1; - repeated EmbeddingOutput outputs = 2; - TaskUsage usage = 3; -} - -message EmbeddingOutput { - string item_id = 1; - optional uint32 input_index = 2; - TaskOutputGranularity granularity = 3; - oneof embedding { - DenseFloatTensor dense = 4; - SparseFloatTensor sparse = 5; - } - repeated uint32 token_ids = 6; -} diff --git a/proto/openengine/v1/kv.proto b/proto/openengine/v1/kv.proto index 95a2295..af180dd 100644 --- a/proto/openengine/v1/kv.proto +++ b/proto/openengine/v1/kv.proto @@ -29,8 +29,6 @@ message KvEndpoint { string protocol = 3; // grpc, nixl, ucx, tcp, shm, etc. } -message GetKvConnectorInfoRequest {} - message KvConnectorInfo { optional bool enabled = 1; string transfer_backend = 2; diff --git a/proto/openengine/v1/model.proto b/proto/openengine/v1/model.proto index de92175..5995a20 100644 --- a/proto/openengine/v1/model.proto +++ b/proto/openengine/v1/model.proto @@ -8,9 +8,6 @@ syntax = "proto3"; package openengine.v1; import "google/protobuf/struct.proto"; -import "openengine/v1/classification.proto"; -import "openengine/v1/embedding.proto"; -import "openengine/v1/scoring.proto"; message GetModelInfoRequest { string model = 1; @@ -36,8 +33,6 @@ message ModelInfo { string reasoning_parser = 25; string tool_call_parser = 26; - TaskCapabilities tasks = 27; // Optional non-generative task support for this model. - // Engine-specific model metadata with no portable field (e.g. kv-cache dtype, // quantization, capabilities not yet standardized). NOT part of the portable // contract: clients read it opportunistically and never depend on it for @@ -45,11 +40,6 @@ message ModelInfo { google.protobuf.Struct extra = 28; } -message TaskCapabilities { - EmbedCapabilities embed = 1; - ClassifyCapabilities classify = 2; - ScoreCapabilities score = 3; -} message GenerationCapabilities { LogprobCapabilities prompt_logprobs = 1; diff --git a/proto/openengine/v1/observability.proto b/proto/openengine/v1/observability.proto index ed18f11..d254abd 100644 --- a/proto/openengine/v1/observability.proto +++ b/proto/openengine/v1/observability.proto @@ -1,14 +1,13 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -// Load snapshots and structured runtime events. +// Load snapshots. syntax = "proto3"; package openengine.v1; import "google/protobuf/struct.proto"; -import "openengine/v1/error.proto"; message GetLoadRequest { bool include_per_rank = 1; @@ -39,29 +38,3 @@ message RankLoadInfo { optional uint32 prefill_batch_size = 6; optional uint32 decode_batch_size = 7; } - -message SubscribeRuntimeEventsRequest { - repeated RuntimeEventType types = 1; -} - -message SubscribeRuntimeEventsResponse { - oneof event { - RuntimeEvent runtime_event = 1; - EngineError error = 2; // Terminal. - } -} - -enum RuntimeEventType { - RUNTIME_EVENT_TYPE_UNSPECIFIED = 0; - RUNTIME_EVENT_TYPE_FORWARD_PASS = 1; - RUNTIME_EVENT_TYPE_BATCH = 2; - RUNTIME_EVENT_TYPE_QUEUE = 3; - RUNTIME_EVENT_TYPE_TRANSFER = 4; -} - -message RuntimeEvent { - string event_id = 1; - uint64 timestamp_unix_nanos = 2; - RuntimeEventType type = 3; - google.protobuf.Struct attributes = 4; -} diff --git a/proto/openengine/v1/openengine.proto b/proto/openengine/v1/openengine.proto index 8a50f7c..101099d 100644 --- a/proto/openengine/v1/openengine.proto +++ b/proto/openengine/v1/openengine.proto @@ -16,15 +16,12 @@ syntax = "proto3"; package openengine.v1; -import "openengine/v1/classification.proto"; -import "openengine/v1/embedding.proto"; import "openengine/v1/generation.proto"; import "openengine/v1/kv.proto"; import "openengine/v1/lifecycle.proto"; import "openengine/v1/lora.proto"; import "openengine/v1/model.proto"; import "openengine/v1/observability.proto"; -import "openengine/v1/scoring.proto"; import "openengine/v1/server.proto"; // Standard ASCII gRPC request metadata: @@ -42,11 +39,6 @@ import "openengine/v1/server.proto"; service Inference { // Core inference path. rpc Generate(GenerateRequest) returns (stream GenerateResponse); - - // Non-generative inference paths. - rpc Embed(EmbedRequest) returns (EmbedResponse); - rpc Classify(ClassifyRequest) returns (ClassifyResponse); - rpc Score(ScoreRequest) returns (ScoreResponse); } service Control { @@ -65,11 +57,7 @@ service Control { rpc UnloadLora(UnloadLoraRequest) returns (UnloadLoraResponse); rpc ListLoras(ListLorasRequest) returns (ListLorasResponse); - // Disaggregated serving / KV transfer. - rpc GetKvConnectorInfo(GetKvConnectorInfoRequest) returns (KvConnectorInfo); + // Disaggregated serving / KV transfer. Connector info: ServerInfo.kv_connector. rpc GetKvEventSources(GetKvEventSourcesRequest) returns (GetKvEventSourcesResponse); rpc SubscribeKvEvents(SubscribeKvEventsRequest) returns (stream SubscribeKvEventsResponse); - - // Structured runtime events for planners/controllers. - rpc SubscribeRuntimeEvents(SubscribeRuntimeEventsRequest) returns (stream SubscribeRuntimeEventsResponse); } diff --git a/proto/openengine/v1/scoring.proto b/proto/openengine/v1/scoring.proto deleted file mode 100644 index 4887913..0000000 --- a/proto/openengine/v1/scoring.proto +++ /dev/null @@ -1,80 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -// Portable grouped query and candidate scoring inference. - -syntax = "proto3"; - -package openengine.v1; - -import "openengine/v1/input.proto"; -import "openengine/v1/tasks.proto"; - -enum ScoreNormalization { - SCORE_NORMALIZATION_UNSPECIFIED = 0; - SCORE_NORMALIZATION_NONE = 1; - SCORE_NORMALIZATION_SOFTMAX = 2; -} - -message ScoreCapabilities { - optional bool supported = 1; - repeated TaskInputType input_types = 2; - repeated TaskOutputGranularity granularities = 3; - repeated TaskValueSemantics semantics = 4; - repeated ScoreNormalization normalizations = 5; - optional bool supports_label_token_scoring = 6; - optional bool supports_instruction = 7; - optional uint32 max_groups = 8; - optional uint32 max_candidates_per_group = 9; - optional uint64 max_output_values_per_candidate = 10; - optional bool supports_priority = 11; - optional bool supports_lora = 12; - optional bool higher_is_better = 13; - repeated Modality modalities = 14; - optional uint32 max_label_token_ids = 15; -} - -message ScoreRequest { - TaskRequestContext context = 1; - repeated ScoreGroup groups = 2; - ScoreOptions options = 3; -} - -// A query and one or more candidates scored as one correlation group. -message ScoreGroup { - string group_id = 1; - TaskInput query = 2; - repeated TaskInput candidates = 3; -} - -message ScoreOptions { - optional TaskOutputGranularity granularity = 1; - optional TaskValueSemantics output_semantics = 2; - ScoreNormalization normalization = 3; - repeated uint32 label_token_ids = 4; - string instruction = 5; -} - -message ScoreResponse { - string request_id = 1; - repeated ScoreGroupOutput groups = 2; - TaskOutputGranularity granularity = 3; - TaskValueSemantics semantics = 4; - optional bool higher_is_better = 5; - ScoreNormalization normalization = 6; - repeated uint32 label_token_ids = 7; - TaskUsage usage = 8; -} - -message ScoreGroupOutput { - string group_id = 1; - optional uint32 group_index = 2; - repeated ScoreCandidateOutput candidates = 3; -} - -message ScoreCandidateOutput { - string candidate_id = 1; - optional uint32 candidate_index = 2; - DenseFloatTensor scores = 3; - repeated uint32 token_ids = 4; -} diff --git a/proto/openengine/v1/tasks.proto b/proto/openengine/v1/tasks.proto deleted file mode 100644 index b5fcadc..0000000 --- a/proto/openengine/v1/tasks.proto +++ /dev/null @@ -1,52 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -// Shared request and output vocabulary for non-generative inference tasks. - -syntax = "proto3"; - -package openengine.v1; - -import "google/protobuf/struct.proto"; - -// Request identity and model selection shared by every non-generative task. -message TaskRequestContext { - string request_id = 1; - string model = 2; - string lora_name = 3; - - // Engine-specific task parameters that have no portable field. NOT part of - // the portable contract: an engine MAY ignore keys it does not recognize, and - // a request MUST remain valid with this field empty. Clients treat it as - // best-effort. Carried as a Struct so JSON types survive the wire. - google.protobuf.Struct extra = 4; -} - -// Row-major FP32 tensor. The product of shape must equal values.size(). -message DenseFloatTensor { - repeated uint64 shape = 1; - repeated float values = 2; -} - -// Token accounting for one complete unary task request. -message TaskUsage { - uint64 input_tokens = 1; - optional uint64 cached_input_tokens = 2; -} - -enum TaskOutputGranularity { - TASK_OUTPUT_GRANULARITY_UNSPECIFIED = 0; - TASK_OUTPUT_GRANULARITY_SEQUENCE = 1; - TASK_OUTPUT_GRANULARITY_TOKEN = 2; -} - -enum TaskValueSemantics { - TASK_VALUE_SEMANTICS_UNSPECIFIED = 0; - TASK_VALUE_SEMANTICS_LOGITS = 1; - TASK_VALUE_SEMANTICS_PROBABILITIES = 2; - TASK_VALUE_SEMANTICS_LOG_PROBABILITIES = 3; - TASK_VALUE_SEMANTICS_SIMILARITY = 4; - TASK_VALUE_SEMANTICS_RELEVANCE = 5; - TASK_VALUE_SEMANTICS_REWARD = 6; - TASK_VALUE_SEMANTICS_MODEL_DEFINED = 7; -}