From 577e39355e59c6fed64c49ee4e87b992966ec310 Mon Sep 17 00:00:00 2001 From: Randy <857271370@qq.com> Date: Sun, 20 Sep 2026 16:11:59 +0800 Subject: [PATCH 1/6] =?UTF-8?q?=F0=9F=92=A5=20refactor!:=20stream=20record?= =?UTF-8?q?=20and=20native=20page=20responses?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/ci.yml | 4 +- .github/workflows/release-image.yml | 4 +- .github/workflows/reliability.yml | 4 +- docs/design/store-isolated-architecture.md | 2 +- docs/native-access.md | 2 +- docs/rolling-upgrades.md | 27 ++- docs/store-isolation.md | 2 +- docs/streaming.md | 52 +++++ examples/quickstart/client/main.go | 45 +++- gen/forward/forward.pb.go | 157 +++++--------- gen/forward/forward_grpc.pb.go | 75 ++----- gen/forward/forward_vtproto.pb.go | 199 +++-------------- gen/sink/sink.pb.go | 58 +++-- gen/sink/sink_grpc.pb.go | 227 +++++++++++--------- gen/sink/sink_vtproto.pb.go | 66 ++++++ go.mod | 4 +- go.sum | 4 +- internal/app/logging.go | 2 +- internal/app/logging_test.go | 8 +- internal/app/role_contract_test.go | 31 ++- internal/app/service.go | 4 +- internal/app/transport.go | 1 + internal/engine/memory_test.go | 21 +- internal/engine/server.go | 62 ++++-- internal/engine/stream.go | 56 +++-- internal/forwarding/budget.go | 2 +- internal/forwarding/frames.go | 6 - internal/gateway/batch.go | 25 +-- internal/gateway/methods.go | 115 ++++++---- internal/gateway/metrics.go | 14 ++ internal/gateway/record_stream.go | 195 +++++++++++++++++ internal/gateway/response_budget_test.go | 24 --- internal/gateway/server.go | 40 ++-- internal/gateway/stream.go | 77 ++++--- internal/metrics/forwarding.go | 32 ++- internal/metrics/metrics.go | 9 +- internal/metrics/store_test.go | 6 +- internal/service/batching_server.go | 8 +- internal/service/native.go | 53 ++--- internal/service/native_boundary_test.go | 16 ++ internal/service/page_collect_test.go | 42 ++++ internal/service/query.go | 48 +++-- internal/service/request_budgets.go | 9 +- internal/service/server.go | 1 - internal/service/stream.go | 183 ++++++++++++++++ internal/service/stream_test.go | 108 ++++++++++ internal/service/write_return_retry_test.go | 4 +- internal/storage/mongodb/query.go | 15 +- internal/storage/mongodb/scan.go | 16 +- internal/storage/native.go | 2 + internal/storage/search/client.go | 17 ++ internal/storage/search/query.go | 33 ++- internal/storage/search/read.go | 39 +++- internal/storage/search/scan.go | 78 +++++-- internal/storage/search/stream.go | 129 +++++++++++ internal/storage/search/stream_test.go | 107 +++++++++ proto/forward/forward.proto | 13 +- proto/sink/sink.proto | 26 ++- 58 files changed, 1814 insertions(+), 795 deletions(-) create mode 100644 docs/streaming.md delete mode 100644 internal/forwarding/frames.go create mode 100644 internal/gateway/record_stream.go create mode 100644 internal/service/page_collect_test.go create mode 100644 internal/service/stream.go create mode 100644 internal/service/stream_test.go create mode 100644 internal/storage/search/stream.go create mode 100644 internal/storage/search/stream_test.go diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a8b68cf..ec94ace 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -147,8 +147,8 @@ jobs: qualification: name: External test suite - uses: batchstream/sink-production-suite/.github/workflows/server-qualification.yml@924aa80684708a06685625b8161fc3529c218529 + uses: batchstream/sink-production-suite/.github/workflows/server-qualification.yml@da993409f9a6fd3b99012a285b4308253158af10 with: - suite_ref: 924aa80684708a06685625b8161fc3529c218529 + suite_ref: da993409f9a6fd3b99012a285b4308253158af10 sink_ref: ${{ github.event.pull_request.head.sha || github.sha }} sdk_ref: ${{ github.head_ref || 'main' }} diff --git a/.github/workflows/release-image.yml b/.github/workflows/release-image.yml index 0c0a7a8..156de3a 100644 --- a/.github/workflows/release-image.yml +++ b/.github/workflows/release-image.yml @@ -20,9 +20,9 @@ env: jobs: qualify: name: Public production qualification - uses: batchstream/sink-production-suite/.github/workflows/release-qualification.yml@e084c611c2ce6efb2e2175cbcfc931dd497c9811 + uses: batchstream/sink-production-suite/.github/workflows/release-qualification.yml@da993409f9a6fd3b99012a285b4308253158af10 with: - suite_ref: e084c611c2ce6efb2e2175cbcfc931dd497c9811 + suite_ref: da993409f9a6fd3b99012a285b4308253158af10 sink_ref: ${{ github.event.release.tag_name }} binaries: diff --git a/.github/workflows/reliability.yml b/.github/workflows/reliability.yml index ae41700..f63040c 100644 --- a/.github/workflows/reliability.yml +++ b/.github/workflows/reliability.yml @@ -8,7 +8,7 @@ permissions: jobs: soak: - uses: batchstream/sink-production-suite/.github/workflows/reliability.yml@e084c611c2ce6efb2e2175cbcfc931dd497c9811 + uses: batchstream/sink-production-suite/.github/workflows/reliability.yml@da993409f9a6fd3b99012a285b4308253158af10 with: - suite_ref: e084c611c2ce6efb2e2175cbcfc931dd497c9811 + suite_ref: da993409f9a6fd3b99012a285b4308253158af10 sink_ref: ${{ github.sha }} diff --git a/docs/design/store-isolated-architecture.md b/docs/design/store-isolated-architecture.md index 9e035e6..050e6d5 100644 --- a/docs/design/store-isolated-architecture.md +++ b/docs/design/store-isolated-architecture.md @@ -241,7 +241,7 @@ Preserve explicit temporary Scan admission rejections and their retry details. K Internal forwarding results can distinguish `not_started`, complete per-operation results, and unknown outcomes. Only verifiable `not_started` evidence permits mapping the entire group to unexecuted operations. A gRPC status code alone, particularly downstream ResourceExhausted, is not such evidence. If complete results are received, their successes must not be overwritten because another Store failed. -If the entire public RPC has already been cancelled or its transport has failed, delivery of known successes cannot be guaranteed. This is the existing unary transport boundary. +If the entire public RPC has already been cancelled or its transport has failed, delivery of known successes cannot be guaranteed. Delivered stream results remain known; undelivered mutation outcomes remain unknown. Propagate the original context's remaining deadline and cancellation. Do not restart a full timeout at each hop. When the client supplies no deadline, Gateway applies the public request timeout. Downstream processes may impose shorter local limits but must not extend the original deadline. diff --git a/docs/native-access.md b/docs/native-access.md index bbe36f7..75d0c0d 100644 --- a/docs/native-access.md +++ b/docs/native-access.md @@ -287,7 +287,7 @@ byte limits as Execute. ## Scan -Scan is one unary RPC per page. `ScanRequest` contains the shared `Command`, +Scan is one server-streaming RPC per page. Documents are followed by a completion frame; the cursor is usable only after successful EOF. `ScanRequest` contains the shared `Command`, `batch_size` (default 100, maximum 1000), an opaque bytes `cursor`, and optional `projection` using the same `fields` and `exclude` controls as Query. `ScanResponse` contains native `documents` and `next_cursor`. A byte-limited diff --git a/docs/rolling-upgrades.md b/docs/rolling-upgrades.md index 2cf9714..8667034 100644 --- a/docs/rolling-upgrades.md +++ b/docs/rolling-upgrades.md @@ -10,17 +10,22 @@ snapshot for each Store throughout an accepted public batch. Stopping an Engine as soon as one DNS lookup removes it can still interrupt a later group of an already accepted request. -## Memory-admission protocol upgrade - -The new private protocol is version 6. It retains only response allowances and -removes snapshot/input/output grants. Engines reject other protocol versions; -retaining both `Forward` and `ForwardStream` methods does not provide compatibility -with older versions. Use matching Gateway/Engine deployments and an isolated -cutover for incompatible versions. Public SDK messages remain unchanged. - -Replace legacy admission/reservation dashboards and alerts with the new -[memory watermark metrics](observability.md#memory-capacity-and-keda). Old metric -families are removed, not kept as empty compatibility gauges. +## Streaming protocol upgrade + +The private forwarding protocol is version 7. `Forward` is one server-streaming +RPC carrying typed results and a final usage/status settlement. The old unary +endpoint and byte-chunk protocol have been removed. + +Public `Read`, `Write`, `Query` and `Scan` are server-streaming RPCs. Upgrade the +Gateway, Engine and SDK together; there is no fallback to an older protocol. +`Delete`, `Execute` and `Count` retain their scalar response contracts. Use a +separate matching cluster for cutover across this incompatible boundary. + +The SDK collects results by default. Supplying a callback consumes results as +they arrive without retaining document results. Completed record results survive +an interrupted stream; missing write outcomes are unknown and are never replayed +automatically. Query/Scan continuation metadata is valid only after the final +completion frame **and** successful EOF. See [streaming responses](streaming.md). ## Budget both intervals diff --git a/docs/store-isolation.md b/docs/store-isolation.md index 1ca0e90..c4a5cc9 100644 --- a/docs/store-isolation.md +++ b/docs/store-isolation.md @@ -108,7 +108,7 @@ from healthy Stores are retained when another Store fails. A lost Engine mutatio is represented as a failed operation with `retryable=false`, since the effect may already exist. A missing budget settlement consumes the entire grant; it cannot be reused by another Store. A proven local rejection before dispatch consumes -nothing. The public unary RPC itself may be cancelled before any partial response +nothing. The public result stream itself may be cancelled before a pending result can reach the caller; completed writes are not rolled back. Public gRPC status details survive forwarding, including the Scan admission retry diff --git a/docs/streaming.md b/docs/streaming.md new file mode 100644 index 0000000..8fca96a --- /dev/null +++ b/docs/streaming.md @@ -0,0 +1,52 @@ +# Streaming responses + +`Read`, `Write`, `Query` and `Scan` use server-streaming RPCs. Requests remain +batch-native. Each document/result frame contains one item. `Read` and `Write` +use `operation_index` to identify final results; consumers validate uniqueness, +coverage and successful EOF. Different records may complete out of order. +Writes for the same record retain request order and conditional revision checks. + +`Query` and `Scan` send documents in query order followed by a document-free +`complete` frame containing `has_more` or `next_cursor`. EOF must succeed before +those fields can be used. A late backend error can follow document frames. A +callback may therefore have processed part of a failed page: do not checkpoint +it; use idempotent processing when resuming from the previous cursor. + +Gateway forwards typed Engine results directly. It never serializes or +reassembles a complete response. Forward protocol version 7 has one `Forward` +stream with result frames followed by exactly one final usage/status settlement. +No unary or binary-chunk compatibility endpoint remains. + +Record execution pulls one existing synchronous microbatch at a time, using the +configured operation/byte batch boundaries. Sending happens outside the shared +batch executor; a slow receiver cannot hold its record scheduling locks. The next +microbatch starts only after sends finish. Merge snapshots/candidates and returned +documents are consequently bounded by the active microbatch rather than the +entire RPC. MongoDB native pages iterate cursors; search pages decode HTTP JSON +hits incrementally and validate trailing shard/timeout metadata before completion. +Search multi-get decodes directly from the HTTP body without a second whole-body +buffer. Existing message, backend response, scan-page and Lua bounds remain; +this change introduces no additional memory-limit setting. Concurrent requests, +transport/driver buffers and one large document still contribute to process RSS. + +The Go SDK accepts slices for record collections and an optional callback: + +```go +results, err := client.Read(ctx, addresses) // Collected in request order. +results, err = client.Read(ctx, addresses, onRead) // results is nil. +written, err := client.Write(ctx, mode, operations, onWrite) +queryPage, err := client.Query(ctx, query, onDocument) // queryPage.Documents is nil. +scanPage, err := client.Scan(ctx, scan, onDocument) // scanPage.Documents is nil. +``` + +Callbacks execute serially, provide backpressure, and cancel the RPC when they +return an error. They receive owned immutable documents. Callback errors are not +retried. Reads retry only unfinished/retryable operations without redelivering +completed results. Writes never retry transport errors. In collection mode, +record methods return acknowledged partial results together with a stream error. +In callback mode, the already invoked callbacks are the acknowledgement record; +the SDK does not retain their documents. Dataset methods still aggregate failure +metadata into `BatchError` without collecting callback documents. + +Deploy matching server and SDK versions. Worker consumption, prefetch pausing and +queue processing behavior are unchanged by this refactor. diff --git a/examples/quickstart/client/main.go b/examples/quickstart/client/main.go index d86848d..5a549d8 100644 --- a/examples/quickstart/client/main.go +++ b/examples/quickstart/client/main.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "io" "log" "os" "reflect" @@ -182,7 +183,7 @@ func writeRecords( CompletionMode: completionMode, Operations: operations, } - response, err := client.Write(ctx, request) + response, err := collectWrites(ctx, client, request) if err != nil { return fmt.Errorf("write records: %w", err) } @@ -203,7 +204,7 @@ func writeRecords( func readAndVerify(ctx context.Context, client sink.SinkClient, records []exampleRecord) error { request := readRequest(records) - response, err := client.Read(ctx, request) + response, err := readRecords(ctx, client, request) if err != nil { return fmt.Errorf("read records: %w", err) } @@ -229,7 +230,7 @@ func waitForRecord(ctx context.Context, client sink.SinkClient, record exampleRe records := []exampleRecord{record} request := readRequest(records) for { - response, err := client.Read(pollContext, request) + response, err := readRecords(pollContext, client, request) if err != nil { return fmt.Errorf("poll asynchronous record: %w", err) } @@ -278,7 +279,7 @@ func deleteRecords(ctx context.Context, client sink.SinkClient, records []exampl func verifyMissing(ctx context.Context, client sink.SinkClient, records []exampleRecord) error { request := readRequest(records) - response, err := client.Read(ctx, request) + response, err := readRecords(ctx, client, request) if err != nil { return fmt.Errorf("read deleted records: %w", err) } @@ -352,3 +353,39 @@ func failureMessage(failure *sink.Failure) string { } return failure.GetMessage() } + +func readRecords(ctx context.Context, client sink.SinkClient, req *sink.ReadRequest) (*sink.ReadResponse, error) { + stream, err := client.Read(ctx, req) + if err != nil { + return nil, err + } + response := &sink.ReadResponse{} + for { + frame, err := stream.Recv() + if err == io.EOF { + return response, nil + } + if err != nil { + return response, err + } + response.Results = append(response.Results, frame.Results...) + } +} + +func collectWrites(ctx context.Context, client sink.SinkClient, req *sink.WriteRequest) (*sink.WriteResponse, error) { + stream, err := client.Write(ctx, req) + if err != nil { + return nil, err + } + response := &sink.WriteResponse{} + for { + frame, err := stream.Recv() + if err == io.EOF { + return response, nil + } + if err != nil { + return response, err + } + response.Results = append(response.Results, frame.Results...) + } +} diff --git a/gen/forward/forward.pb.go b/gen/forward/forward.pb.go index 499b9b8..580aafd 100644 --- a/gen/forward/forward.pb.go +++ b/gen/forward/forward.pb.go @@ -256,10 +256,12 @@ func (*ForwardRequest_Count) isForwardRequest_Request() {} func (*ForwardRequest_Scan) isForwardRequest_Request() {} type ForwardResponse struct { - state protoimpl.MessageState `protogen:"open.v1"` - Version uint32 `protobuf:"varint,1,opt,name=version,proto3" json:"version,omitempty"` - Store string `protobuf:"bytes,2,opt,name=store,proto3" json:"store,omitempty"` - Used *Budget `protobuf:"bytes,4,opt,name=used,proto3" json:"used,omitempty"` + state protoimpl.MessageState `protogen:"open.v1"` + // The final frame carries usage and application status, with no response body. + Complete bool `protobuf:"varint,9,opt,name=complete,proto3" json:"complete,omitempty"` + Version uint32 `protobuf:"varint,1,opt,name=version,proto3" json:"version,omitempty"` + Store string `protobuf:"bytes,2,opt,name=store,proto3" json:"store,omitempty"` + Used *Budget `protobuf:"bytes,4,opt,name=used,proto3" json:"used,omitempty"` // gRPC application status is carried inside the envelope so usage survives errors. Code uint32 `protobuf:"varint,5,opt,name=code,proto3" json:"code,omitempty"` Message string `protobuf:"bytes,6,opt,name=message,proto3" json:"message,omitempty"` @@ -310,6 +312,13 @@ func (*ForwardResponse) Descriptor() ([]byte, []int) { return file_forward_forward_proto_rawDescGZIP(), []int{2} } +func (x *ForwardResponse) GetComplete() bool { + if x != nil { + return x.Complete + } + return false +} + func (x *ForwardResponse) GetVersion() uint32 { if x != nil { return x.Version @@ -475,58 +484,6 @@ func (*ForwardResponse_Count) isForwardResponse_Response() {} func (*ForwardResponse_Scan) isForwardResponse_Response() {} -type ResponseFrame struct { - state protoimpl.MessageState `protogen:"open.v1"` - Size uint64 `protobuf:"varint,1,opt,name=size,proto3" json:"size,omitempty"` - Data []byte `protobuf:"bytes,2,opt,name=data,proto3" json:"data,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache -} - -func (x *ResponseFrame) Reset() { - *x = ResponseFrame{} - mi := &file_forward_forward_proto_msgTypes[3] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) -} - -func (x *ResponseFrame) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*ResponseFrame) ProtoMessage() {} - -func (x *ResponseFrame) ProtoReflect() protoreflect.Message { - mi := &file_forward_forward_proto_msgTypes[3] - if x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use ResponseFrame.ProtoReflect.Descriptor instead. -func (*ResponseFrame) Descriptor() ([]byte, []int) { - return file_forward_forward_proto_rawDescGZIP(), []int{3} -} - -func (x *ResponseFrame) GetSize() uint64 { - if x != nil { - return x.Size - } - return 0 -} - -func (x *ResponseFrame) GetData() []byte { - if x != nil { - return x.Data - } - return nil -} - var File_forward_forward_proto protoreflect.FileDescriptor const file_forward_forward_proto_rawDesc = "" + @@ -546,8 +503,9 @@ const file_forward_forward_proto_rawDesc = "" + "\x05query\x18\x0e \x01(\v2\x15.sink.v1.QueryRequestH\x00R\x05query\x12-\n" + "\x05count\x18\x0f \x01(\v2\x15.sink.v1.CountRequestH\x00R\x05count\x12*\n" + "\x04scan\x18\x10 \x01(\v2\x14.sink.v1.ScanRequestH\x00R\x04scanB\t\n" + - "\arequest\"\xd9\x04\n" + - "\x0fForwardResponse\x12\x18\n" + + "\arequest\"\xf5\x04\n" + + "\x0fForwardResponse\x12\x1a\n" + + "\bcomplete\x18\t \x01(\bR\bcomplete\x12\x18\n" + "\aversion\x18\x01 \x01(\rR\aversion\x12\x14\n" + "\x05store\x18\x02 \x01(\tR\x05store\x12+\n" + "\x04used\x18\x04 \x01(\v2\x17.sink.forward.v1.BudgetR\x04used\x12\x12\n" + @@ -565,13 +523,9 @@ const file_forward_forward_proto_rawDesc = "" + "\x05count\x18\x0f \x01(\v2\x16.sink.v1.CountResponseH\x00R\x05count\x12+\n" + "\x04scan\x18\x10 \x01(\v2\x15.sink.v1.ScanResponseH\x00R\x04scanB\n" + "\n" + - "\bresponse\"7\n" + - "\rResponseFrame\x12\x12\n" + - "\x04size\x18\x01 \x01(\x04R\x04size\x12\x12\n" + - "\x04data\x18\x02 \x01(\fR\x04data2\xaa\x01\n" + - "\x06Engine\x12L\n" + - "\aForward\x12\x1f.sink.forward.v1.ForwardRequest\x1a .sink.forward.v1.ForwardResponse\x12R\n" + - "\rForwardStream\x12\x1f.sink.forward.v1.ForwardRequest\x1a\x1e.sink.forward.v1.ResponseFrame0\x01B+Z)github.com/liran/sink/gen/forward;forwardb\x06proto3" + "\bresponse2X\n" + + "\x06Engine\x12N\n" + + "\aForward\x12\x1f.sink.forward.v1.ForwardRequest\x1a .sink.forward.v1.ForwardResponse0\x01B+Z)github.com/liran/sink/gen/forward;forwardb\x06proto3" var ( file_forward_forward_proto_rawDescOnce sync.Once @@ -585,52 +539,49 @@ func file_forward_forward_proto_rawDescGZIP() []byte { return file_forward_forward_proto_rawDescData } -var file_forward_forward_proto_msgTypes = make([]protoimpl.MessageInfo, 4) +var file_forward_forward_proto_msgTypes = make([]protoimpl.MessageInfo, 3) var file_forward_forward_proto_goTypes = []any{ (*Budget)(nil), // 0: sink.forward.v1.Budget (*ForwardRequest)(nil), // 1: sink.forward.v1.ForwardRequest (*ForwardResponse)(nil), // 2: sink.forward.v1.ForwardResponse - (*ResponseFrame)(nil), // 3: sink.forward.v1.ResponseFrame - (*sink.ReadRequest)(nil), // 4: sink.v1.ReadRequest - (*sink.WriteRequest)(nil), // 5: sink.v1.WriteRequest - (*sink.DeleteRequest)(nil), // 6: sink.v1.DeleteRequest - (*sink.ExecuteRequest)(nil), // 7: sink.v1.ExecuteRequest - (*sink.QueryRequest)(nil), // 8: sink.v1.QueryRequest - (*sink.CountRequest)(nil), // 9: sink.v1.CountRequest - (*sink.ScanRequest)(nil), // 10: sink.v1.ScanRequest - (*anypb.Any)(nil), // 11: google.protobuf.Any - (*sink.ReadResponse)(nil), // 12: sink.v1.ReadResponse - (*sink.WriteResponse)(nil), // 13: sink.v1.WriteResponse - (*sink.DeleteResponse)(nil), // 14: sink.v1.DeleteResponse - (*sink.ExecuteResponse)(nil), // 15: sink.v1.ExecuteResponse - (*sink.QueryResponse)(nil), // 16: sink.v1.QueryResponse - (*sink.CountResponse)(nil), // 17: sink.v1.CountResponse - (*sink.ScanResponse)(nil), // 18: sink.v1.ScanResponse + (*sink.ReadRequest)(nil), // 3: sink.v1.ReadRequest + (*sink.WriteRequest)(nil), // 4: sink.v1.WriteRequest + (*sink.DeleteRequest)(nil), // 5: sink.v1.DeleteRequest + (*sink.ExecuteRequest)(nil), // 6: sink.v1.ExecuteRequest + (*sink.QueryRequest)(nil), // 7: sink.v1.QueryRequest + (*sink.CountRequest)(nil), // 8: sink.v1.CountRequest + (*sink.ScanRequest)(nil), // 9: sink.v1.ScanRequest + (*anypb.Any)(nil), // 10: google.protobuf.Any + (*sink.ReadResponse)(nil), // 11: sink.v1.ReadResponse + (*sink.WriteResponse)(nil), // 12: sink.v1.WriteResponse + (*sink.DeleteResponse)(nil), // 13: sink.v1.DeleteResponse + (*sink.ExecuteResponse)(nil), // 14: sink.v1.ExecuteResponse + (*sink.QueryResponse)(nil), // 15: sink.v1.QueryResponse + (*sink.CountResponse)(nil), // 16: sink.v1.CountResponse + (*sink.ScanResponse)(nil), // 17: sink.v1.ScanResponse } var file_forward_forward_proto_depIdxs = []int32{ 0, // 0: sink.forward.v1.ForwardRequest.grant:type_name -> sink.forward.v1.Budget - 4, // 1: sink.forward.v1.ForwardRequest.read:type_name -> sink.v1.ReadRequest - 5, // 2: sink.forward.v1.ForwardRequest.write:type_name -> sink.v1.WriteRequest - 6, // 3: sink.forward.v1.ForwardRequest.delete:type_name -> sink.v1.DeleteRequest - 7, // 4: sink.forward.v1.ForwardRequest.execute:type_name -> sink.v1.ExecuteRequest - 8, // 5: sink.forward.v1.ForwardRequest.query:type_name -> sink.v1.QueryRequest - 9, // 6: sink.forward.v1.ForwardRequest.count:type_name -> sink.v1.CountRequest - 10, // 7: sink.forward.v1.ForwardRequest.scan:type_name -> sink.v1.ScanRequest + 3, // 1: sink.forward.v1.ForwardRequest.read:type_name -> sink.v1.ReadRequest + 4, // 2: sink.forward.v1.ForwardRequest.write:type_name -> sink.v1.WriteRequest + 5, // 3: sink.forward.v1.ForwardRequest.delete:type_name -> sink.v1.DeleteRequest + 6, // 4: sink.forward.v1.ForwardRequest.execute:type_name -> sink.v1.ExecuteRequest + 7, // 5: sink.forward.v1.ForwardRequest.query:type_name -> sink.v1.QueryRequest + 8, // 6: sink.forward.v1.ForwardRequest.count:type_name -> sink.v1.CountRequest + 9, // 7: sink.forward.v1.ForwardRequest.scan:type_name -> sink.v1.ScanRequest 0, // 8: sink.forward.v1.ForwardResponse.used:type_name -> sink.forward.v1.Budget - 11, // 9: sink.forward.v1.ForwardResponse.status_details:type_name -> google.protobuf.Any - 12, // 10: sink.forward.v1.ForwardResponse.read:type_name -> sink.v1.ReadResponse - 13, // 11: sink.forward.v1.ForwardResponse.write:type_name -> sink.v1.WriteResponse - 14, // 12: sink.forward.v1.ForwardResponse.delete:type_name -> sink.v1.DeleteResponse - 15, // 13: sink.forward.v1.ForwardResponse.execute:type_name -> sink.v1.ExecuteResponse - 16, // 14: sink.forward.v1.ForwardResponse.query:type_name -> sink.v1.QueryResponse - 17, // 15: sink.forward.v1.ForwardResponse.count:type_name -> sink.v1.CountResponse - 18, // 16: sink.forward.v1.ForwardResponse.scan:type_name -> sink.v1.ScanResponse + 10, // 9: sink.forward.v1.ForwardResponse.status_details:type_name -> google.protobuf.Any + 11, // 10: sink.forward.v1.ForwardResponse.read:type_name -> sink.v1.ReadResponse + 12, // 11: sink.forward.v1.ForwardResponse.write:type_name -> sink.v1.WriteResponse + 13, // 12: sink.forward.v1.ForwardResponse.delete:type_name -> sink.v1.DeleteResponse + 14, // 13: sink.forward.v1.ForwardResponse.execute:type_name -> sink.v1.ExecuteResponse + 15, // 14: sink.forward.v1.ForwardResponse.query:type_name -> sink.v1.QueryResponse + 16, // 15: sink.forward.v1.ForwardResponse.count:type_name -> sink.v1.CountResponse + 17, // 16: sink.forward.v1.ForwardResponse.scan:type_name -> sink.v1.ScanResponse 1, // 17: sink.forward.v1.Engine.Forward:input_type -> sink.forward.v1.ForwardRequest - 1, // 18: sink.forward.v1.Engine.ForwardStream:input_type -> sink.forward.v1.ForwardRequest - 2, // 19: sink.forward.v1.Engine.Forward:output_type -> sink.forward.v1.ForwardResponse - 3, // 20: sink.forward.v1.Engine.ForwardStream:output_type -> sink.forward.v1.ResponseFrame - 19, // [19:21] is the sub-list for method output_type - 17, // [17:19] is the sub-list for method input_type + 2, // 18: sink.forward.v1.Engine.Forward:output_type -> sink.forward.v1.ForwardResponse + 18, // [18:19] is the sub-list for method output_type + 17, // [17:18] is the sub-list for method input_type 17, // [17:17] is the sub-list for extension type_name 17, // [17:17] is the sub-list for extension extendee 0, // [0:17] is the sub-list for field type_name @@ -665,7 +616,7 @@ func file_forward_forward_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_forward_forward_proto_rawDesc), len(file_forward_forward_proto_rawDesc)), NumEnums: 0, - NumMessages: 4, + NumMessages: 3, NumExtensions: 0, NumServices: 1, }, diff --git a/gen/forward/forward_grpc.pb.go b/gen/forward/forward_grpc.pb.go index 01ca471..0a0eeaf 100644 --- a/gen/forward/forward_grpc.pb.go +++ b/gen/forward/forward_grpc.pb.go @@ -19,8 +19,7 @@ import ( const _ = grpc.SupportPackageIsVersion9 const ( - Engine_Forward_FullMethodName = "/sink.forward.v1.Engine/Forward" - Engine_ForwardStream_FullMethodName = "/sink.forward.v1.Engine/ForwardStream" + Engine_Forward_FullMethodName = "/sink.forward.v1.Engine/Forward" ) // EngineClient is the client API for Engine service. @@ -29,10 +28,8 @@ const ( // // Private Gateway-to-Engine protocol. Never used by public clients. type EngineClient interface { - Forward(ctx context.Context, in *ForwardRequest, opts ...grpc.CallOption) (*ForwardResponse, error) - // The first frame announces the encoded response size; subsequent frames - // contain bounded chunks. Public clients continue using unary Sink RPCs. - ForwardStream(ctx context.Context, in *ForwardRequest, opts ...grpc.CallOption) (grpc.ServerStreamingClient[ResponseFrame], error) + // Typed result frames followed by exactly one complete settlement frame. + Forward(ctx context.Context, in *ForwardRequest, opts ...grpc.CallOption) (grpc.ServerStreamingClient[ForwardResponse], error) } type engineClient struct { @@ -43,23 +40,13 @@ func NewEngineClient(cc grpc.ClientConnInterface) EngineClient { return &engineClient{cc} } -func (c *engineClient) Forward(ctx context.Context, in *ForwardRequest, opts ...grpc.CallOption) (*ForwardResponse, error) { +func (c *engineClient) Forward(ctx context.Context, in *ForwardRequest, opts ...grpc.CallOption) (grpc.ServerStreamingClient[ForwardResponse], error) { cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) - out := new(ForwardResponse) - err := c.cc.Invoke(ctx, Engine_Forward_FullMethodName, in, out, cOpts...) + stream, err := c.cc.NewStream(ctx, &Engine_ServiceDesc.Streams[0], Engine_Forward_FullMethodName, cOpts...) if err != nil { return nil, err } - return out, nil -} - -func (c *engineClient) ForwardStream(ctx context.Context, in *ForwardRequest, opts ...grpc.CallOption) (grpc.ServerStreamingClient[ResponseFrame], error) { - cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) - stream, err := c.cc.NewStream(ctx, &Engine_ServiceDesc.Streams[0], Engine_ForwardStream_FullMethodName, cOpts...) - if err != nil { - return nil, err - } - x := &grpc.GenericClientStream[ForwardRequest, ResponseFrame]{ClientStream: stream} + x := &grpc.GenericClientStream[ForwardRequest, ForwardResponse]{ClientStream: stream} if err := x.ClientStream.SendMsg(in); err != nil { return nil, err } @@ -70,7 +57,7 @@ func (c *engineClient) ForwardStream(ctx context.Context, in *ForwardRequest, op } // This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. -type Engine_ForwardStreamClient = grpc.ServerStreamingClient[ResponseFrame] +type Engine_ForwardClient = grpc.ServerStreamingClient[ForwardResponse] // EngineServer is the server API for Engine service. // All implementations must embed UnimplementedEngineServer @@ -78,10 +65,8 @@ type Engine_ForwardStreamClient = grpc.ServerStreamingClient[ResponseFrame] // // Private Gateway-to-Engine protocol. Never used by public clients. type EngineServer interface { - Forward(context.Context, *ForwardRequest) (*ForwardResponse, error) - // The first frame announces the encoded response size; subsequent frames - // contain bounded chunks. Public clients continue using unary Sink RPCs. - ForwardStream(*ForwardRequest, grpc.ServerStreamingServer[ResponseFrame]) error + // Typed result frames followed by exactly one complete settlement frame. + Forward(*ForwardRequest, grpc.ServerStreamingServer[ForwardResponse]) error mustEmbedUnimplementedEngineServer() } @@ -92,11 +77,8 @@ type EngineServer interface { // pointer dereference when methods are called. type UnimplementedEngineServer struct{} -func (UnimplementedEngineServer) Forward(context.Context, *ForwardRequest) (*ForwardResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method Forward not implemented") -} -func (UnimplementedEngineServer) ForwardStream(*ForwardRequest, grpc.ServerStreamingServer[ResponseFrame]) error { - return status.Errorf(codes.Unimplemented, "method ForwardStream not implemented") +func (UnimplementedEngineServer) Forward(*ForwardRequest, grpc.ServerStreamingServer[ForwardResponse]) error { + return status.Errorf(codes.Unimplemented, "method Forward not implemented") } func (UnimplementedEngineServer) mustEmbedUnimplementedEngineServer() {} func (UnimplementedEngineServer) testEmbeddedByValue() {} @@ -119,34 +101,16 @@ func RegisterEngineServer(s grpc.ServiceRegistrar, srv EngineServer) { s.RegisterService(&Engine_ServiceDesc, srv) } -func _Engine_Forward_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(ForwardRequest) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return srv.(EngineServer).Forward(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: Engine_Forward_FullMethodName, - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(EngineServer).Forward(ctx, req.(*ForwardRequest)) - } - return interceptor(ctx, in, info, handler) -} - -func _Engine_ForwardStream_Handler(srv interface{}, stream grpc.ServerStream) error { +func _Engine_Forward_Handler(srv interface{}, stream grpc.ServerStream) error { m := new(ForwardRequest) if err := stream.RecvMsg(m); err != nil { return err } - return srv.(EngineServer).ForwardStream(m, &grpc.GenericServerStream[ForwardRequest, ResponseFrame]{ServerStream: stream}) + return srv.(EngineServer).Forward(m, &grpc.GenericServerStream[ForwardRequest, ForwardResponse]{ServerStream: stream}) } // This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. -type Engine_ForwardStreamServer = grpc.ServerStreamingServer[ResponseFrame] +type Engine_ForwardServer = grpc.ServerStreamingServer[ForwardResponse] // Engine_ServiceDesc is the grpc.ServiceDesc for Engine service. // It's only intended for direct use with grpc.RegisterService, @@ -154,16 +118,11 @@ type Engine_ForwardStreamServer = grpc.ServerStreamingServer[ResponseFrame] var Engine_ServiceDesc = grpc.ServiceDesc{ ServiceName: "sink.forward.v1.Engine", HandlerType: (*EngineServer)(nil), - Methods: []grpc.MethodDesc{ - { - MethodName: "Forward", - Handler: _Engine_Forward_Handler, - }, - }, + Methods: []grpc.MethodDesc{}, Streams: []grpc.StreamDesc{ { - StreamName: "ForwardStream", - Handler: _Engine_ForwardStream_Handler, + StreamName: "Forward", + Handler: _Engine_Forward_Handler, ServerStreams: true, }, }, diff --git a/gen/forward/forward_vtproto.pb.go b/gen/forward/forward_vtproto.pb.go index d5792de..9b85bf7 100644 --- a/gen/forward/forward_vtproto.pb.go +++ b/gen/forward/forward_vtproto.pb.go @@ -297,6 +297,16 @@ func (m *ForwardResponse) MarshalToSizedBufferVT(dAtA []byte) (int, error) { } i -= size } + if m.Complete { + i-- + if m.Complete { + dAtA[i] = 1 + } else { + dAtA[i] = 0 + } + i-- + dAtA[i] = 0x48 + } if len(m.StatusDetails) > 0 { for iNdEx := len(m.StatusDetails) - 1; iNdEx >= 0; iNdEx-- { size, err := (*anypb.Any)(m.StatusDetails[iNdEx]).MarshalToSizedBufferVT(dAtA[:i]) @@ -491,51 +501,6 @@ func (m *ForwardResponse_Scan) MarshalToSizedBufferVT(dAtA []byte) (int, error) } return len(dAtA) - i, nil } -func (m *ResponseFrame) MarshalVT() (dAtA []byte, err error) { - if m == nil { - return nil, nil - } - size := m.SizeVT() - dAtA = make([]byte, size) - n, err := m.MarshalToSizedBufferVT(dAtA[:size]) - if err != nil { - return nil, err - } - return dAtA[:n], nil -} - -func (m *ResponseFrame) MarshalToVT(dAtA []byte) (int, error) { - size := m.SizeVT() - return m.MarshalToSizedBufferVT(dAtA[:size]) -} - -func (m *ResponseFrame) MarshalToSizedBufferVT(dAtA []byte) (int, error) { - if m == nil { - return 0, nil - } - i := len(dAtA) - _ = i - var l int - _ = l - if m.unknownFields != nil { - i -= len(m.unknownFields) - copy(dAtA[i:], m.unknownFields) - } - if len(m.Data) > 0 { - i -= len(m.Data) - copy(dAtA[i:], m.Data) - i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.Data))) - i-- - dAtA[i] = 0x12 - } - if m.Size != 0 { - i = protohelpers.EncodeVarint(dAtA, i, uint64(m.Size)) - i-- - dAtA[i] = 0x8 - } - return len(dAtA) - i, nil -} - func (m *Budget) SizeVT() (n int) { if m == nil { return 0 @@ -690,6 +655,9 @@ func (m *ForwardResponse) SizeVT() (n int) { n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) } } + if m.Complete { + n += 2 + } if vtmsg, ok := m.Response.(interface{ SizeVT() int }); ok { n += vtmsg.SizeVT() } @@ -781,23 +749,6 @@ func (m *ForwardResponse_Scan) SizeVT() (n int) { } return n } -func (m *ResponseFrame) SizeVT() (n int) { - if m == nil { - return 0 - } - var l int - _ = l - if m.Size != 0 { - n += 1 + protohelpers.SizeOfVarint(uint64(m.Size)) - } - l = len(m.Data) - if l > 0 { - n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) - } - n += len(m.unknownFields) - return n -} - func (m *Budget) UnmarshalVT(dAtA []byte) error { l := len(dAtA) iNdEx := 0 @@ -1514,6 +1465,26 @@ func (m *ForwardResponse) UnmarshalVT(dAtA []byte) error { return err } iNdEx = postIndex + case 9: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Complete", wireType) + } + var v int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + m.Complete = bool(v != 0) case 10: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Read", wireType) @@ -1823,107 +1794,3 @@ func (m *ForwardResponse) UnmarshalVT(dAtA []byte) error { } return nil } -func (m *ResponseFrame) UnmarshalVT(dAtA []byte) error { - l := len(dAtA) - iNdEx := 0 - for iNdEx < l { - preIndex := iNdEx - var wire uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return protohelpers.ErrIntOverflow - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - wire |= uint64(b&0x7F) << shift - if b < 0x80 { - break - } - } - fieldNum := int32(wire >> 3) - wireType := int(wire & 0x7) - if wireType == 4 { - return fmt.Errorf("proto: ResponseFrame: wiretype end group for non-group") - } - if fieldNum <= 0 { - return fmt.Errorf("proto: ResponseFrame: illegal tag %d (wire type %d)", fieldNum, wire) - } - switch fieldNum { - case 1: - if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field Size", wireType) - } - m.Size = 0 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return protohelpers.ErrIntOverflow - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - m.Size |= uint64(b&0x7F) << shift - if b < 0x80 { - break - } - } - case 2: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Data", wireType) - } - var byteLen int - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return protohelpers.ErrIntOverflow - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - byteLen |= int(b&0x7F) << shift - if b < 0x80 { - break - } - } - if byteLen < 0 { - return protohelpers.ErrInvalidLength - } - postIndex := iNdEx + byteLen - if postIndex < 0 { - return protohelpers.ErrInvalidLength - } - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.Data = append(m.Data[:0], dAtA[iNdEx:postIndex]...) - if m.Data == nil { - m.Data = []byte{} - } - iNdEx = postIndex - default: - iNdEx = preIndex - skippy, err := protohelpers.Skip(dAtA[iNdEx:]) - if err != nil { - return err - } - if (skippy < 0) || (iNdEx+skippy) < 0 { - return protohelpers.ErrInvalidLength - } - if (iNdEx + skippy) > l { - return io.ErrUnexpectedEOF - } - m.unknownFields = append(m.unknownFields, dAtA[iNdEx:iNdEx+skippy]...) - iNdEx += skippy - } - } - - if iNdEx > l { - return io.ErrUnexpectedEOF - } - return nil -} diff --git a/gen/sink/sink.pb.go b/gen/sink/sink.pb.go index 0f94f7c..36a9919 100644 --- a/gen/sink/sink.pb.go +++ b/gen/sink/sink.pb.go @@ -645,8 +645,8 @@ func (x *ReadOperation) GetAddress() *RecordAddress { type ReadResponse struct { state protoimpl.MessageState `protogen:"open.v1"` - // Results are returned in request order. operation_index is included to make - // correlation explicit when an implementation executes records in parallel. + // Each frame contains one result. Results may arrive out of order; correlate + // by operation_index. Successful EOF requires exactly one result per operation. Results []*ReadResult `protobuf:"bytes,1,rep,name=results,proto3" json:"results,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache @@ -1096,6 +1096,8 @@ func (x *LuaProgram) GetSha256() []byte { return nil } +// Each frame contains one final result, identified by operation_index. +// A disconnected stream does not establish the outcome of missing operations. type WriteResponse struct { state protoimpl.MessageState `protogen:"open.v1"` Results []*WriteResult `protobuf:"bytes,1,rep,name=results,proto3" json:"results,omitempty"` @@ -1971,8 +1973,11 @@ func (x *Projection) GetExclude() bool { return false } +// Document frames contain one document. A final, document-free frame sets +// complete and has_more. Metadata is valid only after successful EOF. type QueryResponse struct { state protoimpl.MessageState `protogen:"open.v1"` + Complete bool `protobuf:"varint,3,opt,name=complete,proto3" json:"complete,omitempty"` Documents []*Document `protobuf:"bytes,1,rep,name=documents,proto3" json:"documents,omitempty"` // Determined by fetching one extra result, not by running Count. Result-window // limits apply to this extra result too. A byte limit fails the whole request; @@ -2012,6 +2017,13 @@ func (*QueryResponse) Descriptor() ([]byte, []int) { return file_sink_sink_proto_rawDescGZIP(), []int{26} } +func (x *QueryResponse) GetComplete() bool { + if x != nil { + return x.Complete + } + return false +} + func (x *QueryResponse) GetDocuments() []*Document { if x != nil { return x.Documents @@ -2212,13 +2224,16 @@ func (x *ScanRequest) GetProjection() *Projection { } // Documents retain native encodings: BSON documents or complete JSON hits. -// Failed requests return no page; callers may retry from their last saved -// cursor, with live-query semantics and idempotent business processing. +// A failed stream may have delivered documents but never a committed page. +// Resume from the last saved cursor with idempotent business processing. +// Document frames contain one document. A final, document-free frame sets +// complete and next_cursor. Checkpoint only after successful EOF. type ScanResponse struct { state protoimpl.MessageState `protogen:"open.v1"` + Complete bool `protobuf:"varint,3,opt,name=complete,proto3" json:"complete,omitempty"` Documents []*Document `protobuf:"bytes,1,rep,name=documents,proto3" json:"documents,omitempty"` // Empty means the scan reached the end observed by this request. Commit a - // checkpoint only after processing documents. No explicit close is needed. + // checkpoint only after processing documents and successful EOF. No close RPC is needed. NextCursor []byte `protobuf:"bytes,2,opt,name=next_cursor,json=nextCursor,proto3" json:"next_cursor,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache @@ -2254,6 +2269,13 @@ func (*ScanResponse) Descriptor() ([]byte, []int) { return file_sink_sink_proto_rawDescGZIP(), []int{30} } +func (x *ScanResponse) GetComplete() bool { + if x != nil { + return x.Complete + } + return false +} + func (x *ScanResponse) GetDocuments() []*Document { if x != nil { return x.Documents @@ -2379,8 +2401,9 @@ const file_sink_sink_proto_rawDesc = "" + "\n" + "Projection\x12\x16\n" + "\x06fields\x18\x01 \x03(\tR\x06fields\x12\x18\n" + - "\aexclude\x18\x02 \x01(\bR\aexclude\"[\n" + - "\rQueryResponse\x12/\n" + + "\aexclude\x18\x02 \x01(\bR\aexclude\"w\n" + + "\rQueryResponse\x12\x1a\n" + + "\bcomplete\x18\x03 \x01(\bR\bcomplete\x12/\n" + "\tdocuments\x18\x01 \x03(\v2\x11.sink.v1.DocumentR\tdocuments\x12\x19\n" + "\bhas_more\x18\x02 \x01(\bR\ahasMore\":\n" + "\fCountRequest\x12*\n" + @@ -2395,8 +2418,9 @@ const file_sink_sink_proto_rawDesc = "" + "\x06cursor\x18\x03 \x01(\fR\x06cursor\x123\n" + "\n" + "projection\x18\x04 \x01(\v2\x13.sink.v1.ProjectionR\n" + - "projection\"`\n" + - "\fScanResponse\x12/\n" + + "projection\"|\n" + + "\fScanResponse\x12\x1a\n" + + "\bcomplete\x18\x03 \x01(\bR\bcomplete\x12/\n" + "\tdocuments\x18\x01 \x03(\v2\x11.sink.v1.DocumentR\tdocuments\x12\x1f\n" + "\vnext_cursor\x18\x02 \x01(\fR\n" + "nextCursor*m\n" + @@ -2440,15 +2464,15 @@ const file_sink_sink_proto_rawDesc = "" + "\x1fFAILURE_CODE_RESOURCE_EXHAUSTED\x10\x05\x12\x1c\n" + "\x18FAILURE_CODE_UNAVAILABLE\x10\x06\x12\"\n" + "\x1eFAILURE_CODE_DEADLINE_EXCEEDED\x10\a\x12\x19\n" + - "\x15FAILURE_CODE_INTERNAL\x10\b2\x91\x03\n" + - "\x04Sink\x123\n" + - "\x04Read\x12\x14.sink.v1.ReadRequest\x1a\x15.sink.v1.ReadResponse\x126\n" + - "\x05Write\x12\x15.sink.v1.WriteRequest\x1a\x16.sink.v1.WriteResponse\x129\n" + + "\x15FAILURE_CODE_INTERNAL\x10\b2\x99\x03\n" + + "\x04Sink\x125\n" + + "\x04Read\x12\x14.sink.v1.ReadRequest\x1a\x15.sink.v1.ReadResponse0\x01\x128\n" + + "\x05Write\x12\x15.sink.v1.WriteRequest\x1a\x16.sink.v1.WriteResponse0\x01\x129\n" + "\x06Delete\x12\x16.sink.v1.DeleteRequest\x1a\x17.sink.v1.DeleteResponse\x12<\n" + - "\aExecute\x12\x17.sink.v1.ExecuteRequest\x1a\x18.sink.v1.ExecuteResponse\x126\n" + - "\x05Query\x12\x15.sink.v1.QueryRequest\x1a\x16.sink.v1.QueryResponse\x126\n" + - "\x05Count\x12\x15.sink.v1.CountRequest\x1a\x16.sink.v1.CountResponse\x123\n" + - "\x04Scan\x12\x14.sink.v1.ScanRequest\x1a\x15.sink.v1.ScanResponseB%Z#github.com/liran/sink/gen/sink;sinkb\x06proto3" + "\aExecute\x12\x17.sink.v1.ExecuteRequest\x1a\x18.sink.v1.ExecuteResponse\x128\n" + + "\x05Query\x12\x15.sink.v1.QueryRequest\x1a\x16.sink.v1.QueryResponse0\x01\x126\n" + + "\x05Count\x12\x15.sink.v1.CountRequest\x1a\x16.sink.v1.CountResponse\x125\n" + + "\x04Scan\x12\x14.sink.v1.ScanRequest\x1a\x15.sink.v1.ScanResponse0\x01B%Z#github.com/liran/sink/gen/sink;sinkb\x06proto3" var ( file_sink_sink_proto_rawDescOnce sync.Once diff --git a/gen/sink/sink_grpc.pb.go b/gen/sink/sink_grpc.pb.go index 5fd43cb..5e81d53 100644 --- a/gen/sink/sink_grpc.pb.go +++ b/gen/sink/sink_grpc.pb.go @@ -37,13 +37,13 @@ const ( // Record requests are batch-native. Atomicity is guaranteed per record, never // for the whole request. Native queries use Execute, Query, Count or paged Scan. type SinkClient interface { - Read(ctx context.Context, in *ReadRequest, opts ...grpc.CallOption) (*ReadResponse, error) - Write(ctx context.Context, in *WriteRequest, opts ...grpc.CallOption) (*WriteResponse, error) + Read(ctx context.Context, in *ReadRequest, opts ...grpc.CallOption) (grpc.ServerStreamingClient[ReadResponse], error) + Write(ctx context.Context, in *WriteRequest, opts ...grpc.CallOption) (grpc.ServerStreamingClient[WriteResponse], error) Delete(ctx context.Context, in *DeleteRequest, opts ...grpc.CallOption) (*DeleteResponse, error) Execute(ctx context.Context, in *ExecuteRequest, opts ...grpc.CallOption) (*ExecuteResponse, error) - Query(ctx context.Context, in *QueryRequest, opts ...grpc.CallOption) (*QueryResponse, error) + Query(ctx context.Context, in *QueryRequest, opts ...grpc.CallOption) (grpc.ServerStreamingClient[QueryResponse], error) Count(ctx context.Context, in *CountRequest, opts ...grpc.CallOption) (*CountResponse, error) - Scan(ctx context.Context, in *ScanRequest, opts ...grpc.CallOption) (*ScanResponse, error) + Scan(ctx context.Context, in *ScanRequest, opts ...grpc.CallOption) (grpc.ServerStreamingClient[ScanResponse], error) } type sinkClient struct { @@ -54,26 +54,44 @@ func NewSinkClient(cc grpc.ClientConnInterface) SinkClient { return &sinkClient{cc} } -func (c *sinkClient) Read(ctx context.Context, in *ReadRequest, opts ...grpc.CallOption) (*ReadResponse, error) { +func (c *sinkClient) Read(ctx context.Context, in *ReadRequest, opts ...grpc.CallOption) (grpc.ServerStreamingClient[ReadResponse], error) { cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) - out := new(ReadResponse) - err := c.cc.Invoke(ctx, Sink_Read_FullMethodName, in, out, cOpts...) + stream, err := c.cc.NewStream(ctx, &Sink_ServiceDesc.Streams[0], Sink_Read_FullMethodName, cOpts...) if err != nil { return nil, err } - return out, nil + x := &grpc.GenericClientStream[ReadRequest, ReadResponse]{ClientStream: stream} + if err := x.ClientStream.SendMsg(in); err != nil { + return nil, err + } + if err := x.ClientStream.CloseSend(); err != nil { + return nil, err + } + return x, nil } -func (c *sinkClient) Write(ctx context.Context, in *WriteRequest, opts ...grpc.CallOption) (*WriteResponse, error) { +// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. +type Sink_ReadClient = grpc.ServerStreamingClient[ReadResponse] + +func (c *sinkClient) Write(ctx context.Context, in *WriteRequest, opts ...grpc.CallOption) (grpc.ServerStreamingClient[WriteResponse], error) { cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) - out := new(WriteResponse) - err := c.cc.Invoke(ctx, Sink_Write_FullMethodName, in, out, cOpts...) + stream, err := c.cc.NewStream(ctx, &Sink_ServiceDesc.Streams[1], Sink_Write_FullMethodName, cOpts...) if err != nil { return nil, err } - return out, nil + x := &grpc.GenericClientStream[WriteRequest, WriteResponse]{ClientStream: stream} + if err := x.ClientStream.SendMsg(in); err != nil { + return nil, err + } + if err := x.ClientStream.CloseSend(); err != nil { + return nil, err + } + return x, nil } +// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. +type Sink_WriteClient = grpc.ServerStreamingClient[WriteResponse] + func (c *sinkClient) Delete(ctx context.Context, in *DeleteRequest, opts ...grpc.CallOption) (*DeleteResponse, error) { cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(DeleteResponse) @@ -94,16 +112,25 @@ func (c *sinkClient) Execute(ctx context.Context, in *ExecuteRequest, opts ...gr return out, nil } -func (c *sinkClient) Query(ctx context.Context, in *QueryRequest, opts ...grpc.CallOption) (*QueryResponse, error) { +func (c *sinkClient) Query(ctx context.Context, in *QueryRequest, opts ...grpc.CallOption) (grpc.ServerStreamingClient[QueryResponse], error) { cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) - out := new(QueryResponse) - err := c.cc.Invoke(ctx, Sink_Query_FullMethodName, in, out, cOpts...) + stream, err := c.cc.NewStream(ctx, &Sink_ServiceDesc.Streams[2], Sink_Query_FullMethodName, cOpts...) if err != nil { return nil, err } - return out, nil + x := &grpc.GenericClientStream[QueryRequest, QueryResponse]{ClientStream: stream} + if err := x.ClientStream.SendMsg(in); err != nil { + return nil, err + } + if err := x.ClientStream.CloseSend(); err != nil { + return nil, err + } + return x, nil } +// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. +type Sink_QueryClient = grpc.ServerStreamingClient[QueryResponse] + func (c *sinkClient) Count(ctx context.Context, in *CountRequest, opts ...grpc.CallOption) (*CountResponse, error) { cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(CountResponse) @@ -114,16 +141,25 @@ func (c *sinkClient) Count(ctx context.Context, in *CountRequest, opts ...grpc.C return out, nil } -func (c *sinkClient) Scan(ctx context.Context, in *ScanRequest, opts ...grpc.CallOption) (*ScanResponse, error) { +func (c *sinkClient) Scan(ctx context.Context, in *ScanRequest, opts ...grpc.CallOption) (grpc.ServerStreamingClient[ScanResponse], error) { cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) - out := new(ScanResponse) - err := c.cc.Invoke(ctx, Sink_Scan_FullMethodName, in, out, cOpts...) + stream, err := c.cc.NewStream(ctx, &Sink_ServiceDesc.Streams[3], Sink_Scan_FullMethodName, cOpts...) if err != nil { return nil, err } - return out, nil + x := &grpc.GenericClientStream[ScanRequest, ScanResponse]{ClientStream: stream} + if err := x.ClientStream.SendMsg(in); err != nil { + return nil, err + } + if err := x.ClientStream.CloseSend(); err != nil { + return nil, err + } + return x, nil } +// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. +type Sink_ScanClient = grpc.ServerStreamingClient[ScanResponse] + // SinkServer is the server API for Sink service. // All implementations must embed UnimplementedSinkServer // for forward compatibility. @@ -133,13 +169,13 @@ func (c *sinkClient) Scan(ctx context.Context, in *ScanRequest, opts ...grpc.Cal // Record requests are batch-native. Atomicity is guaranteed per record, never // for the whole request. Native queries use Execute, Query, Count or paged Scan. type SinkServer interface { - Read(context.Context, *ReadRequest) (*ReadResponse, error) - Write(context.Context, *WriteRequest) (*WriteResponse, error) + Read(*ReadRequest, grpc.ServerStreamingServer[ReadResponse]) error + Write(*WriteRequest, grpc.ServerStreamingServer[WriteResponse]) error Delete(context.Context, *DeleteRequest) (*DeleteResponse, error) Execute(context.Context, *ExecuteRequest) (*ExecuteResponse, error) - Query(context.Context, *QueryRequest) (*QueryResponse, error) + Query(*QueryRequest, grpc.ServerStreamingServer[QueryResponse]) error Count(context.Context, *CountRequest) (*CountResponse, error) - Scan(context.Context, *ScanRequest) (*ScanResponse, error) + Scan(*ScanRequest, grpc.ServerStreamingServer[ScanResponse]) error mustEmbedUnimplementedSinkServer() } @@ -150,11 +186,11 @@ type SinkServer interface { // pointer dereference when methods are called. type UnimplementedSinkServer struct{} -func (UnimplementedSinkServer) Read(context.Context, *ReadRequest) (*ReadResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method Read not implemented") +func (UnimplementedSinkServer) Read(*ReadRequest, grpc.ServerStreamingServer[ReadResponse]) error { + return status.Errorf(codes.Unimplemented, "method Read not implemented") } -func (UnimplementedSinkServer) Write(context.Context, *WriteRequest) (*WriteResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method Write not implemented") +func (UnimplementedSinkServer) Write(*WriteRequest, grpc.ServerStreamingServer[WriteResponse]) error { + return status.Errorf(codes.Unimplemented, "method Write not implemented") } func (UnimplementedSinkServer) Delete(context.Context, *DeleteRequest) (*DeleteResponse, error) { return nil, status.Errorf(codes.Unimplemented, "method Delete not implemented") @@ -162,14 +198,14 @@ func (UnimplementedSinkServer) Delete(context.Context, *DeleteRequest) (*DeleteR func (UnimplementedSinkServer) Execute(context.Context, *ExecuteRequest) (*ExecuteResponse, error) { return nil, status.Errorf(codes.Unimplemented, "method Execute not implemented") } -func (UnimplementedSinkServer) Query(context.Context, *QueryRequest) (*QueryResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method Query not implemented") +func (UnimplementedSinkServer) Query(*QueryRequest, grpc.ServerStreamingServer[QueryResponse]) error { + return status.Errorf(codes.Unimplemented, "method Query not implemented") } func (UnimplementedSinkServer) Count(context.Context, *CountRequest) (*CountResponse, error) { return nil, status.Errorf(codes.Unimplemented, "method Count not implemented") } -func (UnimplementedSinkServer) Scan(context.Context, *ScanRequest) (*ScanResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method Scan not implemented") +func (UnimplementedSinkServer) Scan(*ScanRequest, grpc.ServerStreamingServer[ScanResponse]) error { + return status.Errorf(codes.Unimplemented, "method Scan not implemented") } func (UnimplementedSinkServer) mustEmbedUnimplementedSinkServer() {} func (UnimplementedSinkServer) testEmbeddedByValue() {} @@ -192,42 +228,28 @@ func RegisterSinkServer(s grpc.ServiceRegistrar, srv SinkServer) { s.RegisterService(&Sink_ServiceDesc, srv) } -func _Sink_Read_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(ReadRequest) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return srv.(SinkServer).Read(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: Sink_Read_FullMethodName, +func _Sink_Read_Handler(srv interface{}, stream grpc.ServerStream) error { + m := new(ReadRequest) + if err := stream.RecvMsg(m); err != nil { + return err } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(SinkServer).Read(ctx, req.(*ReadRequest)) - } - return interceptor(ctx, in, info, handler) + return srv.(SinkServer).Read(m, &grpc.GenericServerStream[ReadRequest, ReadResponse]{ServerStream: stream}) } -func _Sink_Write_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(WriteRequest) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return srv.(SinkServer).Write(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: Sink_Write_FullMethodName, - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(SinkServer).Write(ctx, req.(*WriteRequest)) +// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. +type Sink_ReadServer = grpc.ServerStreamingServer[ReadResponse] + +func _Sink_Write_Handler(srv interface{}, stream grpc.ServerStream) error { + m := new(WriteRequest) + if err := stream.RecvMsg(m); err != nil { + return err } - return interceptor(ctx, in, info, handler) + return srv.(SinkServer).Write(m, &grpc.GenericServerStream[WriteRequest, WriteResponse]{ServerStream: stream}) } +// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. +type Sink_WriteServer = grpc.ServerStreamingServer[WriteResponse] + func _Sink_Delete_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(DeleteRequest) if err := dec(in); err != nil { @@ -264,24 +286,17 @@ func _Sink_Execute_Handler(srv interface{}, ctx context.Context, dec func(interf return interceptor(ctx, in, info, handler) } -func _Sink_Query_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(QueryRequest) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return srv.(SinkServer).Query(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: Sink_Query_FullMethodName, +func _Sink_Query_Handler(srv interface{}, stream grpc.ServerStream) error { + m := new(QueryRequest) + if err := stream.RecvMsg(m); err != nil { + return err } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(SinkServer).Query(ctx, req.(*QueryRequest)) - } - return interceptor(ctx, in, info, handler) + return srv.(SinkServer).Query(m, &grpc.GenericServerStream[QueryRequest, QueryResponse]{ServerStream: stream}) } +// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. +type Sink_QueryServer = grpc.ServerStreamingServer[QueryResponse] + func _Sink_Count_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(CountRequest) if err := dec(in); err != nil { @@ -300,24 +315,17 @@ func _Sink_Count_Handler(srv interface{}, ctx context.Context, dec func(interfac return interceptor(ctx, in, info, handler) } -func _Sink_Scan_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(ScanRequest) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return srv.(SinkServer).Scan(ctx, in) +func _Sink_Scan_Handler(srv interface{}, stream grpc.ServerStream) error { + m := new(ScanRequest) + if err := stream.RecvMsg(m); err != nil { + return err } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: Sink_Scan_FullMethodName, - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(SinkServer).Scan(ctx, req.(*ScanRequest)) - } - return interceptor(ctx, in, info, handler) + return srv.(SinkServer).Scan(m, &grpc.GenericServerStream[ScanRequest, ScanResponse]{ServerStream: stream}) } +// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. +type Sink_ScanServer = grpc.ServerStreamingServer[ScanResponse] + // Sink_ServiceDesc is the grpc.ServiceDesc for Sink service. // It's only intended for direct use with grpc.RegisterService, // and not to be introspected or modified (even as a copy) @@ -325,14 +333,6 @@ var Sink_ServiceDesc = grpc.ServiceDesc{ ServiceName: "sink.v1.Sink", HandlerType: (*SinkServer)(nil), Methods: []grpc.MethodDesc{ - { - MethodName: "Read", - Handler: _Sink_Read_Handler, - }, - { - MethodName: "Write", - Handler: _Sink_Write_Handler, - }, { MethodName: "Delete", Handler: _Sink_Delete_Handler, @@ -341,19 +341,32 @@ var Sink_ServiceDesc = grpc.ServiceDesc{ MethodName: "Execute", Handler: _Sink_Execute_Handler, }, - { - MethodName: "Query", - Handler: _Sink_Query_Handler, - }, { MethodName: "Count", Handler: _Sink_Count_Handler, }, + }, + Streams: []grpc.StreamDesc{ + { + StreamName: "Read", + Handler: _Sink_Read_Handler, + ServerStreams: true, + }, + { + StreamName: "Write", + Handler: _Sink_Write_Handler, + ServerStreams: true, + }, + { + StreamName: "Query", + Handler: _Sink_Query_Handler, + ServerStreams: true, + }, { - MethodName: "Scan", - Handler: _Sink_Scan_Handler, + StreamName: "Scan", + Handler: _Sink_Scan_Handler, + ServerStreams: true, }, }, - Streams: []grpc.StreamDesc{}, Metadata: "sink/sink.proto", } diff --git a/gen/sink/sink_vtproto.pb.go b/gen/sink/sink_vtproto.pb.go index a3cf8e6..fd3d0cb 100644 --- a/gen/sink/sink_vtproto.pb.go +++ b/gen/sink/sink_vtproto.pb.go @@ -1483,6 +1483,16 @@ func (m *QueryResponse) MarshalToSizedBufferVT(dAtA []byte) (int, error) { i -= len(m.unknownFields) copy(dAtA[i:], m.unknownFields) } + if m.Complete { + i-- + if m.Complete { + dAtA[i] = 1 + } else { + dAtA[i] = 0 + } + i-- + dAtA[i] = 0x18 + } if m.HasMore { i-- if m.HasMore { @@ -1694,6 +1704,16 @@ func (m *ScanResponse) MarshalToSizedBufferVT(dAtA []byte) (int, error) { i -= len(m.unknownFields) copy(dAtA[i:], m.unknownFields) } + if m.Complete { + i-- + if m.Complete { + dAtA[i] = 1 + } else { + dAtA[i] = 0 + } + i-- + dAtA[i] = 0x18 + } if len(m.NextCursor) > 0 { i -= len(m.NextCursor) copy(dAtA[i:], m.NextCursor) @@ -2275,6 +2295,9 @@ func (m *QueryResponse) SizeVT() (n int) { if m.HasMore { n += 2 } + if m.Complete { + n += 2 + } n += len(m.unknownFields) return n } @@ -2350,6 +2373,9 @@ func (m *ScanResponse) SizeVT() (n int) { if l > 0 { n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) } + if m.Complete { + n += 2 + } n += len(m.unknownFields) return n } @@ -5715,6 +5741,26 @@ func (m *QueryResponse) UnmarshalVT(dAtA []byte) error { } } m.HasMore = bool(v != 0) + case 3: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Complete", wireType) + } + var v int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + m.Complete = bool(v != 0) default: iNdEx = preIndex skippy, err := protohelpers.Skip(dAtA[iNdEx:]) @@ -6187,6 +6233,26 @@ func (m *ScanResponse) UnmarshalVT(dAtA []byte) error { m.NextCursor = []byte{} } iNdEx = postIndex + case 3: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Complete", wireType) + } + var v int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + m.Complete = bool(v != 0) default: iNdEx = preIndex skippy, err := protohelpers.Skip(dAtA[iNdEx:]) diff --git a/go.mod b/go.mod index 22b05cb..34077d0 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.27.0 require ( github.com/go-logr/logr v1.4.4 github.com/iceisfun/golua v1.1.1 - github.com/liran/sink-go v0.8.0 + github.com/liran/sink-go v0.9.1-0.20260920080920-9c80756d12cd github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 github.com/prometheus/client_golang v1.24.1 github.com/twmb/franz-go v1.21.7 @@ -20,6 +20,7 @@ require ( go.opentelemetry.io/otel/sdk v1.46.0 go.opentelemetry.io/otel/sdk/log v0.22.0 golang.org/x/net v0.59.0 + golang.org/x/sync v0.23.0 google.golang.org/genproto/googleapis/rpc v0.0.0-20260819154853-08b0e4226688 google.golang.org/grpc v1.83.2 google.golang.org/protobuf v1.36.12 @@ -50,7 +51,6 @@ require ( go.opentelemetry.io/otel/trace v1.46.0 // indirect go.opentelemetry.io/proto/otlp v1.11.0 // indirect golang.org/x/crypto v0.57.0 // indirect - golang.org/x/sync v0.23.0 // indirect golang.org/x/sys v0.48.0 // indirect golang.org/x/text v0.42.0 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20260819154853-08b0e4226688 // indirect diff --git a/go.sum b/go.sum index 08d5ef1..bc5fea7 100644 --- a/go.sum +++ b/go.sum @@ -30,8 +30,8 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= -github.com/liran/sink-go v0.8.0 h1:J03GLgfLHsuUoEtoOLRrOhCosh3QnMXyS+Bw5Tre808= -github.com/liran/sink-go v0.8.0/go.mod h1:yzlNhufcKx518eeC969hAidHTM2ISB+hBag+VDuiPE4= +github.com/liran/sink-go v0.9.1-0.20260920080920-9c80756d12cd h1:Gl4APsF7Fe/mG13Ipick/cR9E3v/CZpFyew+Lwb40MQ= +github.com/liran/sink-go v0.9.1-0.20260920080920-9c80756d12cd/go.mod h1:yzlNhufcKx518eeC969hAidHTM2ISB+hBag+VDuiPE4= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/pierrec/lz4/v4 v4.1.26 h1:GrpZw1gZttORinvzBdXPUXATeqlJjqUG/D87TKMnhjY= diff --git a/internal/app/logging.go b/internal/app/logging.go index e442d3f..ad54734 100644 --- a/internal/app/logging.go +++ b/internal/app/logging.go @@ -55,7 +55,7 @@ func diagnosticMethod(full string) string { } method := full[strings.LastIndexByte(full, '/')+1:] switch method { - case "Read", "Write", "Delete", "Execute", "Query", "Count", "Scan", "Forward", "ForwardStream": + case "Read", "Write", "Delete", "Execute", "Query", "Count", "Scan", "Forward": return method default: return "" diff --git a/internal/app/logging_test.go b/internal/app/logging_test.go index fdebde6..544b53c 100644 --- a/internal/app/logging_test.go +++ b/internal/app/logging_test.go @@ -104,9 +104,9 @@ func TestStreamDiagnosticsPreserveOutcomeAndExcludeHealth(t *testing.T) { err error level string }{ - {method: "/sink.forward.v1.Engine/ForwardStream", level: "debug"}, - {method: "/sink.forward.v1.Engine/ForwardStream", err: status.Error(codes.Canceled, "private request"), level: "warn"}, - {method: "/sink.forward.v1.Engine/ForwardStream", err: status.Error(codes.DataLoss, "private response"), level: "error"}, + {method: "/sink.forward.v1.Engine/Forward", level: "debug"}, + {method: "/sink.forward.v1.Engine/Forward", err: status.Error(codes.Canceled, "private request"), level: "warn"}, + {method: "/sink.forward.v1.Engine/Forward", err: status.Error(codes.DataLoss, "private response"), level: "error"}, {method: "/grpc.health.v1.Health/Watch"}, {method: "/sink.forward.v1.Engine/Unknown"}, } @@ -135,7 +135,7 @@ func TestStreamDiagnosticsPreserveOutcomeAndExcludeHealth(t *testing.T) { if err := json.Unmarshal(output.Bytes(), &record); err != nil { t.Fatal(err) } - if record["method"] != "ForwardStream" || record["status"] != status.Code(test.err).String() || record["level"] != test.level || strings.Contains(output.String(), "private") { + if record["method"] != "Forward" || record["status"] != status.Code(test.err).String() || record["level"] != test.level || strings.Contains(output.String(), "private") { t.Fatalf("wrong stream diagnostic: %s", output.String()) } } diff --git a/internal/app/role_contract_test.go b/internal/app/role_contract_test.go index f2e77c7..756f76f 100644 --- a/internal/app/role_contract_test.go +++ b/internal/app/role_contract_test.go @@ -2,6 +2,7 @@ package app import ( "context" + "io" "strings" "testing" "time" @@ -23,7 +24,10 @@ type deadlineReadStorage struct { } func (s *deadlineReadStorage) Read(ctx context.Context, req storage.ReadRequest) (storage.ReadResponse, error) { - s.seen <- ctx + select { + case s.seen <- ctx: + default: + } select { case <-s.release: return s.Storage.Read(ctx, req) @@ -61,7 +65,8 @@ func TestEngineOnlyAcceptsForwardingAndHonorsGatewayContext(t *testing.T) { defer engineConn.Close() direct := sink.NewSinkClient(engineConn) empty := &sink.ReadRequest{} - if _, err := direct.Read(t.Context(), empty); status.Code(err) != codes.Unimplemented { + _, err = readRPC(t.Context(), direct, empty) + if status.Code(err) != codes.Unimplemented { t.Fatalf("Engine exposes public RPC: %v", err) } component := "mode: gateway\ngrpc: {address: '127.0.0.1:0'}\nhealth: {address: '127.0.0.1:0'}\nrequest: {max_operations: 2000}\nforwarding:\n routes: [{store: primary, target: '" + engine.listener.Addr().String() + "', tls: {insecure: true}}]" @@ -91,7 +96,7 @@ func TestEngineOnlyAcceptsForwardingAndHonorsGatewayContext(t *testing.T) { caller, cancel := context.WithCancel(context.Background()) defer cancel() done := make(chan error, 1) - go func() { _, err := client.Read(caller, request); done <- err }() + go func() { _, err := readRPC(caller, client, request); done <- err }() var observed context.Context select { case observed = <-backend.seen: @@ -114,8 +119,26 @@ func TestEngineOnlyAcceptsForwardingAndHonorsGatewayContext(t *testing.T) { // More than 1000 operations in one Store is valid when Gateway permits it. ctx, stop := context.WithTimeout(context.Background(), 5*time.Second) defer stop() - response, err := client.Read(ctx, request) + response, err := readRPC(ctx, client, request) if err != nil || len(response.GetResults()) != 1500 { t.Fatalf("Engine reintroduced an operation cap: count=%d err=%v", len(response.GetResults()), err) } } + +func readRPC(ctx context.Context, client sink.SinkClient, req *sink.ReadRequest) (*sink.ReadResponse, error) { + stream, err := client.Read(ctx, req) + if err != nil { + return nil, err + } + response := &sink.ReadResponse{} + for { + frame, err := stream.Recv() + if err == io.EOF { + return response, nil + } + if err != nil { + return response, err + } + response.Results = append(response.Results, frame.Results...) + } +} diff --git a/internal/app/service.go b/internal/app/service.go index 6c1a1aa..49fd04d 100644 --- a/internal/app/service.go +++ b/internal/app/service.go @@ -51,10 +51,10 @@ func (app *Application) configureServer(sinkServer *service.Server, observed *si if err != nil { return err } - if err := app.configureGRPC(app.batchingServer, observed); err != nil { + if err := app.configureGRPC(app.batchingServer.RPC(), observed); err != nil { return err } - opts := engine.Options{Memory: app.memory, MaxRequestBytes: loaded.GRPC.MaxReceiveMessageBytes, Metrics: observed, Service: app.batchingServer, Store: loaded.Storage.Name, MaxReadBytes: loaded.GRPC.MaxSendMessageBytes} + opts := engine.Options{Memory: app.memory, MaxRequestBytes: loaded.GRPC.MaxReceiveMessageBytes, Metrics: observed, Service: app.batchingServer.RPC(), Store: loaded.Storage.Name, MaxReadBytes: loaded.GRPC.MaxSendMessageBytes} forwardingServer, err := engine.New(opts) if err != nil { return err diff --git a/internal/app/transport.go b/internal/app/transport.go index de8f943..eafa1da 100644 --- a/internal/app/transport.go +++ b/internal/app/transport.go @@ -39,6 +39,7 @@ func (app *Application) configureGRPC(server sink.SinkServer, observed *sinkmetr streamInterceptors := []grpc.StreamServerInterceptor{logStream} if app.gateway != nil { interceptors = append(interceptors, app.gateway.UnaryInterceptor()) + streamInterceptors = append(streamInterceptors, app.gateway.StreamInterceptor()) } if app.config.Mode == config.ModeEngine { store := app.config.Storage.Name diff --git a/internal/engine/memory_test.go b/internal/engine/memory_test.go index 4b30d27..5904c51 100644 --- a/internal/engine/memory_test.go +++ b/internal/engine/memory_test.go @@ -10,6 +10,7 @@ import ( sink "github.com/liran/sink/gen/sink" "github.com/liran/sink/internal/capacity" "github.com/liran/sink/internal/forwarding" + "google.golang.org/grpc" "google.golang.org/grpc/codes" ) @@ -18,10 +19,10 @@ type pressureService struct { calls int } -func (s *pressureService) Write(context.Context, *sink.WriteRequest) (*sink.WriteResponse, error) { +func (s *pressureService) Write(_ *sink.WriteRequest, stream grpc.ServerStreamingServer[sink.WriteResponse]) error { s.calls++ response := &sink.WriteResponse{} - return response, nil + return stream.Send(response) } func TestEnginePressureRejectsBeforeExecution(t *testing.T) { @@ -40,7 +41,9 @@ func TestEnginePressureRejectsBeforeExecution(t *testing.T) { write := &sink.WriteRequest{} body := &forward.ForwardRequest_Write{Write: write} request := &forward.ForwardRequest{Version: forwarding.Version, Store: "primary", Grant: forwarding.FullBudget(1 << 20), Request: body} - response, err := server.Forward(t.Context(), request) + stream := &forwardRecorder{ctx: t.Context()} + err = server.Forward(request, stream) + response := stream.response if err != nil || !response.GetNotStarted() || response.GetCode() != uint32(codes.ResourceExhausted) || backend.calls != 0 { t.Fatalf("pressure reached service: %v %v calls=%d", response, err, backend.calls) } @@ -48,3 +51,15 @@ func TestEnginePressureRejectsBeforeExecution(t *testing.T) { t.Fatal("rejection consumed response allowance") } } + +type forwardRecorder struct { + grpc.ServerStream + ctx context.Context + response *forward.ForwardResponse +} + +func (s *forwardRecorder) Context() context.Context { return s.ctx } +func (s *forwardRecorder) Send(response *forward.ForwardResponse) error { + s.response = response + return nil +} diff --git a/internal/engine/server.go b/internal/engine/server.go index 95bf1b8..433deca 100644 --- a/internal/engine/server.go +++ b/internal/engine/server.go @@ -2,8 +2,8 @@ package engine import ( - "context" "errors" + "google.golang.org/grpc" "time" forward "github.com/liran/sink/gen/forward" @@ -44,17 +44,22 @@ func New(opts Options) (*Server, error) { server := &Server{memory: opts.Memory, maxRequestBytes: opts.MaxRequestBytes, service: opts.Service, store: opts.Store, maximum: opts.MaxReadBytes, metrics: opts.Metrics} return server, nil } -func (s *Server) Forward(ctx context.Context, req *forward.ForwardRequest) (*forward.ForwardResponse, error) { - response := &forward.ForwardResponse{Version: forwarding.Version, Store: s.store, NotStarted: true} +func (s *Server) Forward(req *forward.ForwardRequest, stream grpc.ServerStreamingServer[forward.ForwardResponse]) error { + ctx := stream.Context() + response := &forward.ForwardResponse{Version: forwarding.Version, Store: s.store, NotStarted: true, Complete: true} started := time.Now() defer func() { s.metrics.ObserveForward(req, response, time.Since(started)) }() tracker := forwarding.NewTracker(req.GetGrant(), s.maximum) - reject := func(err error) (*forward.ForwardResponse, error) { + reject := func(err error) error { response.Code = uint32(status.Code(err)) response.Message = status.Convert(err).Message() response.StatusDetails = status.Convert(err).Proto().GetDetails() response.Used = tracker.Usage() - return response, nil + err = stream.Send(response) + if err != nil { + response.Code = uint32(status.Code(err)) + } + return err } if req.GetVersion() != forwarding.Version { return reject(status.Error(codes.FailedPrecondition, "unsupported forwarding protocol version")) @@ -102,33 +107,50 @@ func (s *Server) Forward(ctx context.Context, req *forward.ForwardRequest) (*for var err error switch body := req.GetRequest().(type) { case *forward.ForwardRequest_Read: - result, callErr := s.service.Read(ctx, body.Read) - err = callErr - response.Response = &forward.ForwardResponse_Read{Read: result} + output := &resultStream[sink.ReadResponse]{ServerStream: stream, metrics: s.metrics, request: req, ctx: ctx, store: s.store, maximum: s.maximum, send: stream.Send} + err = s.service.Read(body.Read, output) case *forward.ForwardRequest_Write: - result, callErr := s.service.Write(ctx, body.Write) - err = callErr - response.Response = &forward.ForwardResponse_Write{Write: result} + output := &resultStream[sink.WriteResponse]{ServerStream: stream, metrics: s.metrics, request: req, ctx: ctx, store: s.store, maximum: s.maximum, send: stream.Send} + err = s.service.Write(body.Write, output) case *forward.ForwardRequest_Delete: result, callErr := s.service.Delete(ctx, body.Delete) err = callErr - response.Response = &forward.ForwardResponse_Delete{Delete: result} + if callErr == nil { + frame := &forward.ForwardResponse{Version: forwarding.Version, Store: s.store} + frame.Response = &forward.ForwardResponse_Delete{Delete: result} + if sendErr := stream.Send(frame); sendErr != nil { + return sendErr + } + s.metrics.ObserveForwardResult(req, frame) + } case *forward.ForwardRequest_Execute: result, callErr := s.service.Execute(ctx, body.Execute) err = callErr - response.Response = &forward.ForwardResponse_Execute{Execute: result} + if callErr == nil { + frame := &forward.ForwardResponse{Version: forwarding.Version, Store: s.store} + frame.Response = &forward.ForwardResponse_Execute{Execute: result} + if sendErr := stream.Send(frame); sendErr != nil { + return sendErr + } + s.metrics.ObserveForwardResult(req, frame) + } case *forward.ForwardRequest_Query: - result, callErr := s.service.Query(ctx, body.Query) - err = callErr - response.Response = &forward.ForwardResponse_Query{Query: result} + output := &resultStream[sink.QueryResponse]{ServerStream: stream, metrics: s.metrics, request: req, ctx: ctx, store: s.store, maximum: s.maximum, send: stream.Send} + err = s.service.Query(body.Query, output) case *forward.ForwardRequest_Count: result, callErr := s.service.Count(ctx, body.Count) err = callErr - response.Response = &forward.ForwardResponse_Count{Count: result} + if callErr == nil { + frame := &forward.ForwardResponse{Version: forwarding.Version, Store: s.store} + frame.Response = &forward.ForwardResponse_Count{Count: result} + if sendErr := stream.Send(frame); sendErr != nil { + return sendErr + } + s.metrics.ObserveForwardResult(req, frame) + } case *forward.ForwardRequest_Scan: - result, callErr := s.service.Scan(ctx, body.Scan) - err = callErr - response.Response = &forward.ForwardResponse_Scan{Scan: result} + output := &resultStream[sink.ScanResponse]{ServerStream: stream, metrics: s.metrics, request: req, ctx: ctx, store: s.store, maximum: s.maximum, send: stream.Send} + err = s.service.Scan(body.Scan, output) } return reject(err) } diff --git a/internal/engine/stream.go b/internal/engine/stream.go index fa67970..28da83f 100644 --- a/internal/engine/stream.go +++ b/internal/engine/stream.go @@ -1,31 +1,49 @@ package engine import ( + "context" forward "github.com/liran/sink/gen/forward" + sink "github.com/liran/sink/gen/sink" "github.com/liran/sink/internal/forwarding" + "github.com/liran/sink/internal/metrics" "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" ) -func (s *Server) ForwardStream(req *forward.ForwardRequest, stream grpc.ServerStreamingServer[forward.ResponseFrame]) error { - ctx := stream.Context() - response, err := s.Forward(ctx, req) - if err != nil { - return err - } - size := response.SizeVT() - frame := &forward.ResponseFrame{Size: uint64(size)} - if err := stream.SendMsg(frame); err != nil { - return err +// resultStream adapts execution messages to the private typed stream without +// marshalling or retaining an aggregate response. +type resultStream[T any] struct { + metrics *metrics.Metrics + request *forward.ForwardRequest + grpc.ServerStream + ctx context.Context + store string + maximum int + send func(*forward.ForwardResponse) error +} + +func (s *resultStream[T]) Context() context.Context { return s.ctx } +func (s *resultStream[T]) Send(result *T) error { + if sized, ok := any(result).(interface{ SizeVT() int }); ok && sized.SizeVT() > s.maximum { + return status.Error(codes.ResourceExhausted, "result exceeds public message limit") } - encoded, err := response.MarshalVT() - if err != nil { - return err + frame := &forward.ForwardResponse{Version: forwarding.Version, Store: s.store} + switch result := any(result).(type) { + case *sink.ReadResponse: + frame.Response = &forward.ForwardResponse_Read{Read: result} + case *sink.WriteResponse: + frame.Response = &forward.ForwardResponse_Write{Write: result} + case *sink.QueryResponse: + frame.Response = &forward.ForwardResponse_Query{Query: result} + case *sink.ScanResponse: + frame.Response = &forward.ForwardResponse_Scan{Scan: result} + default: + return status.Error(codes.Internal, "invalid streaming result type") } - for offset := 0; offset < len(encoded); offset += forwarding.FrameBytes { - frame := &forward.ResponseFrame{Data: encoded[offset:min(offset+forwarding.FrameBytes, len(encoded))]} - if err := stream.SendMsg(frame); err != nil { - return err - } + err := s.send(frame) + if err == nil { + s.metrics.ObserveForwardResult(s.request, frame) } - return nil + return err } diff --git a/internal/forwarding/budget.go b/internal/forwarding/budget.go index 80e6da4..860fcd6 100644 --- a/internal/forwarding/budget.go +++ b/internal/forwarding/budget.go @@ -9,7 +9,7 @@ import ( "github.com/liran/sink/internal/storage" ) -const Version = 6 +const Version = 7 // EnvelopeBytes is reserved in the private transport in addition to the public message cap. const EnvelopeBytes = 4096 diff --git a/internal/forwarding/frames.go b/internal/forwarding/frames.go deleted file mode 100644 index c919329..0000000 --- a/internal/forwarding/frames.go +++ /dev/null @@ -1,6 +0,0 @@ -package forwarding - -// FrameBytes bounds allocations made before the receiver can inspect a private -// response. Keep the receive limit fixed even when a peer announces a tiny size. -const FrameBytes = 32 << 10 -const FrameMessageBytes = FrameBytes + 32 diff --git a/internal/gateway/batch.go b/internal/gateway/batch.go index 8084bc3..9ee0e01 100644 --- a/internal/gateway/batch.go +++ b/internal/gateway/batch.go @@ -19,7 +19,10 @@ type storeGroup struct { indices []int } -func (s *Server) records(ctx context.Context, req *forward.ForwardRequest) (*forward.ForwardResponse, error) { +func (s *Server) deleteRecords(ctx context.Context, req *forward.ForwardRequest) (*forward.ForwardResponse, error) { + if req.GetDelete() == nil { + return nil, status.Error(codes.Internal, "expected delete request") + } addresses, err := s.validateBatch(req) if err != nil { return nil, err @@ -106,7 +109,7 @@ func (s *Server) records(ctx context.Context, req *forward.ForwardRequest) (*for } return used } - if len(groups) > 1 && parallelBatch(req) { + if len(groups) > 1 { // These operations do not consume document budgets. Bound fanout independently // of the number of Stores; requests never create an unbounded goroutine fanout. var work sync.WaitGroup @@ -174,24 +177,6 @@ func (s *Server) validateBatch(req *forward.ForwardRequest) ([]*sink.RecordAddre func validCompletion(mode sink.CompletionMode) bool { return mode >= sink.CompletionMode_COMPLETION_MODE_WAIT_UNTIL_APPLIED && mode <= sink.CompletionMode_COMPLETION_MODE_WAIT_UNTIL_VISIBLE } -func parallelBatch(req *forward.ForwardRequest) bool { - if req.GetDelete() != nil { - return true - } - write := req.GetWrite() - if write == nil { - return false - } - if write.GetCompletionMode() == sink.CompletionMode_COMPLETION_MODE_RETURN_AFTER_ACCEPTED { - return true - } - for _, op := range write.GetOperations() { - if op.GetReturnDocument() { - return false - } - } - return true -} func splitRequest(req *forward.ForwardRequest, indices []int) *forward.ForwardRequest { split := &forward.ForwardRequest{} switch body := req.GetRequest().(type) { diff --git a/internal/gateway/methods.go b/internal/gateway/methods.go index 8780678..819fe1c 100644 --- a/internal/gateway/methods.go +++ b/internal/gateway/methods.go @@ -2,6 +2,7 @@ package gateway import ( "context" + "google.golang.org/grpc" forward "github.com/liran/sink/gen/forward" sink "github.com/liran/sink/gen/sink" @@ -10,30 +11,22 @@ import ( "google.golang.org/grpc/status" ) -func (s *Server) Read(ctx context.Context, req *sink.ReadRequest) (*sink.ReadResponse, error) { +func (s *Server) Read(req *sink.ReadRequest, stream grpc.ServerStreamingServer[sink.ReadResponse]) error { body := &forward.ForwardRequest_Read{Read: req} request := &forward.ForwardRequest{Request: body} - response, err := s.records(ctx, request) - if err != nil { - return nil, err - } - return response.GetRead(), nil + return s.streamRecords(stream.Context(), request, func(frame *forward.ForwardResponse) error { return stream.Send(frame.GetRead()) }) } -func (s *Server) Write(ctx context.Context, req *sink.WriteRequest) (*sink.WriteResponse, error) { +func (s *Server) Write(req *sink.WriteRequest, stream grpc.ServerStreamingServer[sink.WriteResponse]) error { body := &forward.ForwardRequest_Write{Write: req} request := &forward.ForwardRequest{Request: body} - response, err := s.records(ctx, request) - if err != nil { - return nil, err - } - return response.GetWrite(), nil + return s.streamRecords(stream.Context(), request, func(frame *forward.ForwardResponse) error { return stream.Send(frame.GetWrite()) }) } func (s *Server) Delete(ctx context.Context, req *sink.DeleteRequest) (*sink.DeleteResponse, error) { body := &forward.ForwardRequest_Delete{Delete: req} request := &forward.ForwardRequest{Request: body} - response, err := s.records(ctx, request) + response, err := s.deleteRecords(ctx, request) if err != nil { return nil, err } @@ -72,36 +65,54 @@ func (s *Server) Execute(ctx context.Context, req *sink.ExecuteRequest) (*sink.E return response.GetExecute(), nil } -func (s *Server) Query(ctx context.Context, req *sink.QueryRequest) (*sink.QueryResponse, error) { - if req == nil || req.GetCommand() == nil { - return nil, status.Error(codes.InvalidArgument, "command is required") +func (s *Server) Query(req *sink.QueryRequest, stream grpc.ServerStreamingServer[sink.QueryResponse]) error { + if req.GetCommand() == nil { + return status.Error(codes.InvalidArgument, "command is required") } body := &forward.ForwardRequest_Query{Query: req} + request := &forward.ForwardRequest{Request: body} grant, err := s.responseBudget(0) if err != nil { - return nil, err + return err } - request := &forward.ForwardRequest{Request: body, Grant: grant} - ctx, release, err := s.begin(ctx, request) + request.Grant = grant + ctx, release, err := s.begin(stream.Context(), request) if err != nil { - return nil, err + return err } defer release() route, err := routeFor(s.current, protocol.CommandStore(req.GetCommand())) if err != nil { - return nil, err + return err } - response, err := s.forward(ctx, route, request) + complete := false + call := forwardCall{route: route, request: request} + call.emit = func(frame *forward.ForwardResponse) error { + result := frame.GetQuery() + if result == nil || complete || len(result.GetDocuments()) > 1 { + return status.Error(codes.Internal, "invalid Query stream frame") + } + if result.GetComplete() { + if len(result.GetDocuments()) != 0 { + return status.Error(codes.Internal, "invalid Query completion frame") + } + complete = true + } else if len(result.GetDocuments()) != 1 { + return status.Error(codes.Internal, "empty Query document frame") + } + return stream.Send(result) + } + final, err := s.forwardEach(ctx, call) if err != nil { - return nil, err + return err } - if response.GetCode() != 0 { - return nil, forwardedError(response) + if final.GetCode() != 0 { + return forwardedError(final) } - if response.GetQuery() == nil { - return nil, status.Error(codes.Internal, "Engine returned an invalid response type") + if !complete { + return status.Error(codes.Internal, "Engine omitted Query completion") } - return response.GetQuery(), nil + return nil } func (s *Server) Count(ctx context.Context, req *sink.CountRequest) (*sink.CountResponse, error) { @@ -136,34 +147,52 @@ func (s *Server) Count(ctx context.Context, req *sink.CountRequest) (*sink.Count return response.GetCount(), nil } -func (s *Server) Scan(ctx context.Context, req *sink.ScanRequest) (*sink.ScanResponse, error) { - if req == nil || req.GetCommand() == nil { - return nil, status.Error(codes.InvalidArgument, "command is required") +func (s *Server) Scan(req *sink.ScanRequest, stream grpc.ServerStreamingServer[sink.ScanResponse]) error { + if req.GetCommand() == nil { + return status.Error(codes.InvalidArgument, "command is required") } body := &forward.ForwardRequest_Scan{Scan: req} + request := &forward.ForwardRequest{Request: body} grant, err := s.responseBudget(0) if err != nil { - return nil, err + return err } - request := &forward.ForwardRequest{Request: body, Grant: grant} - ctx, release, err := s.begin(ctx, request) + request.Grant = grant + ctx, release, err := s.begin(stream.Context(), request) if err != nil { - return nil, err + return err } defer release() route, err := routeFor(s.current, protocol.CommandStore(req.GetCommand())) if err != nil { - return nil, err + return err } - response, err := s.forward(ctx, route, request) + complete := false + call := forwardCall{route: route, request: request} + call.emit = func(frame *forward.ForwardResponse) error { + result := frame.GetScan() + if result == nil || complete || len(result.GetDocuments()) > 1 { + return status.Error(codes.Internal, "invalid Scan stream frame") + } + if result.GetComplete() { + if len(result.GetDocuments()) != 0 { + return status.Error(codes.Internal, "invalid Scan completion frame") + } + complete = true + } else if len(result.GetDocuments()) != 1 { + return status.Error(codes.Internal, "empty Scan document frame") + } + return stream.Send(result) + } + final, err := s.forwardEach(ctx, call) if err != nil { - return nil, err + return err } - if response.GetCode() != 0 { - return nil, forwardedError(response) + if final.GetCode() != 0 { + return forwardedError(final) } - if response.GetScan() == nil { - return nil, status.Error(codes.Internal, "Engine returned an invalid response type") + if !complete { + return status.Error(codes.Internal, "Engine omitted Scan completion") } - return response.GetScan(), nil + return nil } diff --git a/internal/gateway/metrics.go b/internal/gateway/metrics.go index 0a67d46..6c2cbec 100644 --- a/internal/gateway/metrics.go +++ b/internal/gateway/metrics.go @@ -57,3 +57,17 @@ func (s *Server) UnaryInterceptor() grpc.UnaryServerInterceptor { return response, err } } + +func (s *Server) StreamInterceptor() grpc.StreamServerInterceptor { + return func(server any, stream grpc.ServerStream, info *grpc.StreamServerInfo, handler grpc.StreamHandler) error { + started := time.Now() + err := handler(server, stream) + method := strings.TrimPrefix(info.FullMethod, "/sink.v1.Sink/") + switch method { + case "Read", "Write", "Query", "Scan": + s.metrics.requests.WithLabelValues(method, status.Code(err).String()).Inc() + s.metrics.duration.WithLabelValues(method).Observe(time.Since(started).Seconds()) + } + return err + } +} diff --git a/internal/gateway/record_stream.go b/internal/gateway/record_stream.go new file mode 100644 index 0000000..cb5b08e --- /dev/null +++ b/internal/gateway/record_stream.go @@ -0,0 +1,195 @@ +package gateway + +import ( + "context" + "sync" + + forward "github.com/liran/sink/gen/forward" + sink "github.com/liran/sink/gen/sink" + "github.com/liran/sink/internal/protocol" + "golang.org/x/sync/errgroup" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +func (s *Server) streamRecords(ctx context.Context, req *forward.ForwardRequest, emit func(*forward.ForwardResponse) error) error { + addresses, err := s.validateBatch(req) + if err != nil { + return err + } + ctx, release, err := s.begin(ctx, req) + if err != nil { + return err + } + defer release() + view := s.current + response := emptyResponse(req, len(addresses)) + groups := make([]storeGroup, 0) + positions := make(map[Route]int) + resolved := make(map[string][]Route) + resolutionErrors := make(map[string]error) + owners := make(map[string]Route) + for index, raw := range addresses { + address, parseErr := protocol.ParseAddress(raw) + if parseErr != nil { + failRecords(response, []int{index}, status.Error(codes.InvalidArgument, parseErr.Error()), true) + continue + } + store := address.Store() + route, routeErr := routeFor(view, store) + if routeErr != nil { + failRecords(response, []int{index}, routeErr, true) + continue + } + targets, found := resolved[store] + resolutionErr, failed := resolutionErrors[store] + if !found && !failed { + var release func() + targets, release, resolutionErr = s.pool.destinations(ctx, route) + if resolutionErr != nil { + resolutionErrors[store] = resolutionErr + } else { + resolved[store] = targets + defer release() + } + } + if resolutionErr != nil { + failRecords(response, []int{index}, resolutionErr, true) + continue + } + recordURI := raw.GetUri() + owner, found := owners[recordURI] + if !found { + owner = affinityRoute(recordURI, targets) + owners[recordURI] = owner + } + route = owner + position, exists := positions[route] + if !exists { + position = len(groups) + positions[route] = position + group := storeGroup{route: route} + groups = append(groups, group) + } + groups[position].indices = append(groups[position].indices, index) + } + remaining, err := s.responseBudget(1) + if err != nil { + return err + } + + // Routing failures contain metadata only and can be sent immediately. + for index := range addresses { + var frame *forward.ForwardResponse + switch body := response.GetResponse().(type) { + case *forward.ForwardResponse_Read: + if body.Read.Results[index] != nil { + read := &sink.ReadResponse{Results: []*sink.ReadResult{body.Read.Results[index]}} + frame = &forward.ForwardResponse{Response: &forward.ForwardResponse_Read{Read: read}} + } + case *forward.ForwardResponse_Write: + if body.Write.Results[index] != nil { + write := &sink.WriteResponse{Results: []*sink.WriteResult{body.Write.Results[index]}} + frame = &forward.ForwardResponse{Response: &forward.ForwardResponse_Write{Write: write}} + } + } + if frame != nil { + if err := emit(frame); err != nil { + return err + } + } + } + work, execution := errgroup.WithContext(ctx) + work.SetLimit(s.config.MaxFanout) + var sending sync.Mutex + for _, group := range groups { + work.Go(func() error { + sub := splitRequest(req, group.indices) + sub.Grant = remaining + seen := make([]bool, len(group.indices)) + delivered := 0 + var sendErr error + call := forwardCall{route: group.route, request: sub} + call.emit = func(frame *forward.ForwardResponse) error { + var index uint32 + switch req.GetRequest().(type) { + case *forward.ForwardRequest_Read: + results := frame.GetRead().GetResults() + if len(results) != 1 || results[0] == nil || !validResult(results[0]) { + return status.Error(codes.Internal, "invalid read result frame") + } + index = results[0].GetOperationIndex() + if int(index) >= len(seen) || seen[index] { + return status.Error(codes.Internal, "duplicate or invalid read result index") + } + results[0].OperationIndex = uint32(group.indices[index]) + case *forward.ForwardRequest_Write: + results := frame.GetWrite().GetResults() + if len(results) != 1 || results[0] == nil || !validResult(results[0]) { + return status.Error(codes.Internal, "invalid write result frame") + } + index = results[0].GetOperationIndex() + if int(index) >= len(seen) || seen[index] { + return status.Error(codes.Internal, "duplicate or invalid write result index") + } + results[0].OperationIndex = uint32(group.indices[index]) + } + sending.Lock() + err := emit(frame) + sendErr = err + sending.Unlock() + if err != nil { + return err + } + seen[index] = true + delivered++ + return nil + } + final, err := s.forwardEach(execution, call) + if sendErr != nil { + return sendErr + } + if execution.Err() != nil { + return status.FromContextError(execution.Err()).Err() + } + notStarted := false + if err == nil { + notStarted = final.GetNotStarted() + if final.GetCode() != 0 { + err = forwardedError(final) + } + } + if err == nil && delivered != len(group.indices) { + err = status.Error(codes.Internal, "Engine omitted record results") + } + if err != nil { + if delivered == len(group.indices) { + return err + } + // An interrupted Engine stream cannot revoke delivered results. + // Settle only missing slots and allow unrelated Stores to finish. + for index, known := range seen { + if known { + continue + } + failed := emptyResponse(sub, 1) + failRecords(failed, []int{0}, err, notStarted) + if result := failed.GetRead(); result != nil { + result.Results[0].OperationIndex = uint32(group.indices[index]) + } + if result := failed.GetWrite(); result != nil { + result.Results[0].OperationIndex = uint32(group.indices[index]) + } + sending.Lock() + err := emit(failed) + sending.Unlock() + if err != nil { + return err + } + } + } + return nil + }) + } + return work.Wait() +} diff --git a/internal/gateway/response_budget_test.go b/internal/gateway/response_budget_test.go index c155c96..55765b3 100644 --- a/internal/gateway/response_budget_test.go +++ b/internal/gateway/response_budget_test.go @@ -4,7 +4,6 @@ import ( "testing" forward "github.com/liran/sink/gen/forward" - sink "github.com/liran/sink/gen/sink" "github.com/liran/sink/internal/config" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" @@ -33,26 +32,3 @@ func TestResponseBudgetComesFromTransportAndSharesOnlyReturns(t *testing.T) { t.Fatalf("native encoded response: %v %v", native, err) } } - -func TestParallelWritesOnlySerializeReturnedDocuments(t *testing.T) { - for _, mode := range []sink.WriteMode{sink.WriteMode_WRITE_MODE_UPSERT, sink.WriteMode_WRITE_MODE_CREATE, sink.WriteMode_WRITE_MODE_REPLACE} { - put := &sink.PutOperation{Mode: mode} - action := &sink.WriteOperation_Put{Put: put} - operation := &sink.WriteOperation{Action: action} - write := &sink.WriteRequest{CompletionMode: sink.CompletionMode_COMPLETION_MODE_WAIT_UNTIL_APPLIED, Operations: []*sink.WriteOperation{operation}} - body := &forward.ForwardRequest_Write{Write: write} - request := &forward.ForwardRequest{Request: body} - if !parallelBatch(request) { - t.Fatal("write without returned documents unnecessarily serialized") - } - merge := &sink.MergeOperation{} - operation.Action = &sink.WriteOperation_Merge{Merge: merge} - if !parallelBatch(request) { - t.Fatal("Merge unnecessarily serialized by removed intermediate budgets") - } - operation.ReturnDocument = true - if parallelBatch(request) { - t.Fatal("returned documents lost their shared response allowance") - } - } -} diff --git a/internal/gateway/server.go b/internal/gateway/server.go index ca74e95..4ec5e73 100644 --- a/internal/gateway/server.go +++ b/internal/gateway/server.go @@ -87,39 +87,29 @@ func (s *Server) begin(ctx context.Context, req *forward.ForwardRequest) (contex release := func() { cancel(); s.metrics.inFlight.Dec() } return execution, release, nil } + +// Scalar RPCs carry one typed response plus the final settlement. func (s *Server) forward(ctx context.Context, route Route, req *forward.ForwardRequest) (*forward.ForwardResponse, error) { - if route.endpoint == "" { - targets, release, err := s.pool.destinations(ctx, route) - if err != nil { - return localRejection(route, status.Code(err), status.Convert(err).Message()), nil - } - defer release() - route = targets[(s.nativeSequence.Add(1)-1)%uint64(len(targets))] - } - req.Version = forwarding.Version - req.Store = route.Store - entry, err := s.pool.acquire(route) - if err != nil { - return localRejection(route, status.Code(err), status.Convert(err).Message()), nil - } - defer s.pool.release(entry) - started := time.Now() var response *forward.ForwardResponse - if s.memory != nil { - response, err = s.forwardStream(ctx, entry, req) - } else { - response, err = entry.client.Forward(ctx, req) + call := forwardCall{route: route, request: req} + call.emit = func(frame *forward.ForwardResponse) error { + if response != nil { + return status.Error(codes.Internal, "unexpected extra scalar response") + } + response = frame + return nil } - s.metrics.downstream.Observe(time.Since(started).Seconds()) + final, err := s.forwardEach(ctx, call) if err != nil { return nil, err } - if response.GetVersion() != forwarding.Version || response.GetStore() != route.Store || response.GetUsed() == nil || response.GetCode() > uint32(codes.Unauthenticated) { - return nil, status.Error(codes.Internal, "invalid Engine response identity or version") + if final.GetCode() != 0 { + return final, nil } - if !validUsage(req.GetGrant(), response.GetUsed()) { - return nil, status.Error(codes.Internal, "invalid Engine budget settlement") + if response == nil { + return nil, status.Error(codes.Internal, "Engine omitted scalar response") } + response.Used, response.NotStarted = final.Used, final.NotStarted return response, nil } func validUsage(grant, used *forward.Budget) bool { diff --git a/internal/gateway/stream.go b/internal/gateway/stream.go index 6689c2f..c8002f1 100644 --- a/internal/gateway/stream.go +++ b/internal/gateway/stream.go @@ -3,50 +3,75 @@ package gateway import ( "context" "io" + "time" forward "github.com/liran/sink/gen/forward" "github.com/liran/sink/internal/forwarding" - "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" ) -func (s *Server) forwardStream(ctx context.Context, entry *connection, req *forward.ForwardRequest) (*forward.ForwardResponse, error) { - ctx, cancel := context.WithCancel(ctx) - defer cancel() - stream, err := entry.client.ForwardStream(ctx, req, grpc.MaxCallRecvMsgSize(forwarding.FrameMessageBytes)) +type forwardCall struct { + route Route + request *forward.ForwardRequest + emit func(*forward.ForwardResponse) error +} + +// forwardEach reads the next result only after the public send completes. The +// transport therefore supplies backpressure without an intermediate queue. +func (s *Server) forwardEach(ctx context.Context, call forwardCall) (*forward.ForwardResponse, error) { + route, req := call.route, call.request + if route.endpoint == "" { + targets, release, err := s.pool.destinations(ctx, route) + if err != nil { + return localRejection(route, status.Code(err), status.Convert(err).Message()), nil + } + defer release() + route = targets[(s.nativeSequence.Add(1)-1)%uint64(len(targets))] + } + req.Version, req.Store = forwarding.Version, route.Store + entry, err := s.pool.acquire(route) if err != nil { - return nil, err + return localRejection(route, status.Code(err), status.Convert(err).Message()), nil } - header, err := stream.Recv() + defer s.pool.release(entry) + ctx, cancel := context.WithCancel(ctx) + defer cancel() + started := time.Now() + defer func() { s.metrics.downstream.Observe(time.Since(started).Seconds()) }() + stream, err := entry.client.Forward(ctx, req) if err != nil { return nil, err } - if header.GetSize() == 0 || header.GetSize() > uint64(s.pool.messageBytes) || len(header.GetData()) != 0 { - return nil, status.Error(codes.Internal, "invalid Engine response size frame") - } - size := int(header.GetSize()) - encoded := make([]byte, size) - offset := 0 - for offset < size { + var final *forward.ForwardResponse + received := false + for { frame, err := stream.Recv() + if err == io.EOF { + if final == nil { + return nil, status.Error(codes.Internal, "Engine omitted stream settlement") + } + return final, nil + } if err != nil { return nil, err } - if frame.GetSize() != 0 || len(frame.GetData()) == 0 || len(frame.GetData()) > min(forwarding.FrameBytes, size-offset) { - return nil, status.Error(codes.Internal, "invalid Engine response data frame") + if final != nil || frame.GetVersion() != forwarding.Version || frame.GetStore() != route.Store { + return nil, status.Error(codes.Internal, "invalid Engine stream identity or frame order") } - offset += copy(encoded[offset:], frame.GetData()) - } - if _, err := stream.Recv(); err != io.EOF { - if err != nil { + if frame.GetComplete() { + if frame.GetResponse() != nil || frame.GetUsed() == nil || !validUsage(req.GetGrant(), frame.GetUsed()) || frame.GetCode() > uint32(codes.Unauthenticated) || (received && frame.GetNotStarted()) { + return nil, status.Error(codes.Internal, "invalid Engine stream settlement") + } + final = frame + continue + } + if frame.GetResponse() == nil || frame.GetUsed() != nil || frame.GetCode() != 0 || frame.GetNotStarted() || frame.GetMessage() != "" || len(frame.GetStatusDetails()) != 0 { + return nil, status.Error(codes.Internal, "invalid Engine result frame") + } + received = true + if err := call.emit(frame); err != nil { return nil, err } - return nil, status.Error(codes.Internal, "unexpected Engine response frame") - } - response := &forward.ForwardResponse{} - if err := response.UnmarshalVT(encoded); err != nil { - return nil, status.Error(codes.Internal, "invalid encoded Engine response") } - return response, nil } diff --git a/internal/metrics/forwarding.go b/internal/metrics/forwarding.go index 36844d3..9984710 100644 --- a/internal/metrics/forwarding.go +++ b/internal/metrics/forwarding.go @@ -12,6 +12,29 @@ func (m *Metrics) ObserveForward(req *forward.ForwardRequest, resp *forward.Forw if m == nil { return } + method, request, response := forwardObservation(req, resp) + if method == "" { + return + } + store := m.RequestStore(request) + m.requests.WithLabelValues(store, method, codes.Code(resp.GetCode()).String()).Inc() + m.requestDuration.WithLabelValues(store, method).Observe(elapsed.Seconds()) + if resp.GetCode() == 0 { + m.observeOperationResults(method, request, response) + } +} + +// ObserveForwardResult records delivered outcomes without retaining documents. +func (m *Metrics) ObserveForwardResult(req *forward.ForwardRequest, resp *forward.ForwardResponse) { + if m == nil { + return + } + method, request, response := forwardObservation(req, resp) + if method != "" { + m.observeOperationResults(method, request, response) + } +} +func forwardObservation(req *forward.ForwardRequest, resp *forward.ForwardResponse) (string, any, any) { var method string var request, response any switch body := req.GetRequest().(type) { @@ -44,12 +67,7 @@ func (m *Metrics) ObserveForward(req *forward.ForwardRequest, resp *forward.Forw request = body.Scan response = resp.GetScan() default: - return - } - store := m.RequestStore(request) - m.requests.WithLabelValues(store, method, codes.Code(resp.GetCode()).String()).Inc() - m.requestDuration.WithLabelValues(store, method).Observe(elapsed.Seconds()) - if resp.GetCode() == 0 { - m.observeOperationResults(method, request, response) + return "", nil, nil } + return method, request, response } diff --git a/internal/metrics/metrics.go b/internal/metrics/metrics.go index d025708..e19624c 100644 --- a/internal/metrics/metrics.go +++ b/internal/metrics/metrics.go @@ -540,17 +540,20 @@ func (m *Metrics) observeOperationResults(method string, request any, response a switch typed := response.(type) { case *sink.ReadResponse: req, _ := request.(*sink.ReadRequest) - for index, result := range typed.GetResults() { + for _, result := range typed.GetResults() { + index := int(result.GetOperationIndex()) m.operationResults.WithLabelValues(operationStore(m, req.GetOperations(), index), method, readStatus(result.GetStatus())).Inc() } case *sink.WriteResponse: req, _ := request.(*sink.WriteRequest) - for index, result := range typed.GetResults() { + for _, result := range typed.GetResults() { + index := int(result.GetOperationIndex()) m.operationResults.WithLabelValues(operationStore(m, req.GetOperations(), index), method, writeStatus(result.GetStatus())).Inc() } case *sink.DeleteResponse: req, _ := request.(*sink.DeleteRequest) - for index, result := range typed.GetResults() { + for _, result := range typed.GetResults() { + index := int(result.GetOperationIndex()) m.operationResults.WithLabelValues(operationStore(m, req.GetOperations(), index), method, deleteStatus(result.GetStatus())).Inc() } case *sink.ExecuteResponse: diff --git a/internal/metrics/store_test.go b/internal/metrics/store_test.go index 96a757d..cdff8c2 100644 --- a/internal/metrics/store_test.go +++ b/internal/metrics/store_test.go @@ -33,9 +33,9 @@ func TestMixedStoreRPCsCountEachRequestOnceAndAttributeEveryResult(t *testing.T) read.Operations = append(read.Operations, readOp) write.Operations = append(write.Operations, writeOp) remove.Operations = append(remove.Operations, deleteOp) - readResult := &sink.ReadResult{Status: sink.ReadStatus_READ_STATUS_FOUND} - writeResult := &sink.WriteResult{Status: sink.WriteStatus_WRITE_STATUS_APPLIED} - deleteResult := &sink.DeleteResult{Status: sink.DeleteStatus_DELETE_STATUS_APPLIED} + readResult := &sink.ReadResult{OperationIndex: uint32(index), Status: sink.ReadStatus_READ_STATUS_FOUND} + writeResult := &sink.WriteResult{OperationIndex: uint32(index), Status: sink.WriteStatus_WRITE_STATUS_APPLIED} + deleteResult := &sink.DeleteResult{OperationIndex: uint32(index), Status: sink.DeleteStatus_DELETE_STATUS_APPLIED} if index >= 3 { readResult.Status = sink.ReadStatus_READ_STATUS_FAILED writeResult.Status = sink.WriteStatus_WRITE_STATUS_FAILED diff --git a/internal/service/batching_server.go b/internal/service/batching_server.go index a946954..58afac4 100644 --- a/internal/service/batching_server.go +++ b/internal/service/batching_server.go @@ -34,8 +34,6 @@ type BatchingOptions struct { } type BatchingServer struct { - sink.UnimplementedSinkServer - server *Server reads *requestBatcher[*sink.ReadRequest, *sink.ReadResponse] writes *requestBatcher[*sink.WriteRequest, *sink.WriteResponse] @@ -327,16 +325,16 @@ func (s *BatchingServer) executeWrites( continue } if parallel { - executions.Go(func() { s.executeWriteBatch(ctx, group) }) + executions.Go(func() { s.executeWrite(ctx, group) }) } else { - s.executeWriteBatch(ctx, group) + s.executeWrite(ctx, group) } } executions.Wait() } } -func (s *BatchingServer) executeWriteBatch(ctx context.Context, calls []*batchCall[*sink.WriteRequest, *sink.WriteResponse]) { +func (s *BatchingServer) executeWrite(ctx context.Context, calls []*batchCall[*sink.WriteRequest, *sink.WriteResponse]) { calls = liveMutationCalls(calls) if len(calls) == 0 { return diff --git a/internal/service/native.go b/internal/service/native.go index 80290d3..2480afd 100644 --- a/internal/service/native.go +++ b/internal/service/native.go @@ -139,18 +139,18 @@ func (s *Server) Execute(ctx context.Context, req *sink.ExecuteRequest) (*sink.E return response, nil } -func (s *Server) Scan(ctx context.Context, req *sink.ScanRequest) (*sink.ScanResponse, error) { +func (s *Server) scan(ctx context.Context, req *sink.ScanRequest, send func(*sink.ScanResponse) error) error { maximum := forwarding.FromContext(ctx).Limit(forwarding.Returns, s.maxReadBytes) if maximum <= 0 { - return nil, status.Error(codes.ResourceExhausted, "native response budget is exhausted") + return status.Error(codes.ResourceExhausted, "native response budget is exhausted") } if err := protocol.CheckStore(req, s.boundStore); err != nil { - return nil, err + return err } maximum = min(maximum, 4<<20) request, err := nativeRequest(req.GetCommand(), maximum) if err != nil { - return nil, err + return err } batchSize := int(req.GetBatchSize()) if batchSize == 0 { @@ -161,45 +161,50 @@ func (s *Server) Scan(ctx context.Context, req *sink.ScanRequest) (*sink.ScanRes scan.Projection = &storage.Projection{Fields: projection.GetFields(), Exclude: projection.GetExclude()} } if err := scan.Projection.Validate(); err != nil { - return nil, nativeStatus(err) + return nativeStatus(err) } if batchSize > 1000 || len(scan.Cursor) > storage.MaxScanCursorBytes { - return nil, status.Error(codes.InvalidArgument, "scan batch or cursor exceeds its limit") + return status.Error(codes.InvalidArgument, "scan batch or cursor exceeds its limit") } backend, ok := s.storage.(storage.NativeStorage) if !ok { - return nil, nativeStatus(storage.ErrNativeUnsupported) + return nativeStatus(storage.ErrNativeUnsupported) } if _, err := scan.Resume(); err != nil { - return nil, nativeStatus(err) + return nativeStatus(err) } // Leave room for cursor metadata and protobuf framing inside the page limit. scan.Request.MaxBytes = maximum - min(maximum/4, storage.MaxScanCursorBytes) - 128 if scan.Request.MaxBytes <= 0 { - return nil, status.Error(codes.ResourceExhausted, "scan page budget is too small") + return status.Error(codes.ResourceExhausted, "scan page budget is too small") + } + count := 0 + scan.Emit = func(document storage.Document) error { + count++ + if count > batchSize { + return status.Error(codes.Internal, "backend exceeded scan page size") + } + encoded := &sink.Document{Encoding: sink.DocumentEncoding(document.Encoding), Payload: document.Payload} + frame := &sink.ScanResponse{Documents: []*sink.Document{encoded}} + if frame.SizeVT() > maximum { + return status.Error(codes.ResourceExhausted, "scan document exceeds message limit") + } + return send(frame) } result, err := backend.Scan(ctx, scan) if err != nil { - return nil, nativeStatus(err) + return nativeStatus(err) } - if len(result.Documents) > batchSize || (len(result.NextCursor) != 0 && len(result.Documents) == 0) { - return nil, status.Error(codes.Internal, "backend returned an invalid scan page") - } - response := &sink.ScanResponse{NextCursor: result.NextCursor} - for _, document := range result.Documents { - encoded := &sink.Document{Encoding: sink.DocumentEncoding(document.Encoding), Payload: document.Payload} - response.Documents = append(response.Documents, encoded) + if len(result.Documents) != 0 || (len(result.NextCursor) != 0 && count == 0) || len(result.NextCursor) > storage.MaxScanCursorBytes { + return status.Error(codes.Internal, "backend returned an invalid streaming scan page") } - if response.SizeVT() > maximum { - return nil, status.Error(codes.ResourceExhausted, "scan response exceeds byte limit") + final := &sink.ScanResponse{Complete: true, NextCursor: result.NextCursor} + if final.SizeVT() > maximum { + return status.Error(codes.ResourceExhausted, "scan cursor exceeds message limit") } - return response, nil + return send(final) } func (s *BatchingServer) Execute(ctx context.Context, req *sink.ExecuteRequest) (*sink.ExecuteResponse, error) { return s.server.Execute(ctx, req) } - -func (s *BatchingServer) Scan(ctx context.Context, req *sink.ScanRequest) (*sink.ScanResponse, error) { - return s.server.Scan(ctx, req) -} diff --git a/internal/service/native_boundary_test.go b/internal/service/native_boundary_test.go index 177527a..63f34ca 100644 --- a/internal/service/native_boundary_test.go +++ b/internal/service/native_boundary_test.go @@ -35,6 +35,14 @@ func (s *nativeBoundaryStore) Query(_ context.Context, req storage.QueryRequest) s.query = req s.request = req.Request response := storage.QueryResponse{Documents: s.documents, HasMore: s.more} + if req.Emit != nil && s.failure == nil { + for _, doc := range response.Documents { + if err := req.Emit(doc); err != nil { + return response, err + } + } + response.Documents = nil + } return response, s.failure } func (s *nativeBoundaryStore) Count(_ context.Context, req storage.CountRequest) (storage.CountResponse, error) { @@ -48,6 +56,14 @@ func (s *nativeBoundaryStore) Scan(_ context.Context, req storage.ScanRequest) ( s.scan = req s.request = req.Request response := storage.ScanResponse{Documents: s.documents} + if req.Emit != nil && s.failure == nil { + for _, doc := range response.Documents { + if err := req.Emit(doc); err != nil { + return response, err + } + } + response.Documents = nil + } return response, s.failure } diff --git a/internal/service/page_collect_test.go b/internal/service/page_collect_test.go new file mode 100644 index 0000000..9dbd6d3 --- /dev/null +++ b/internal/service/page_collect_test.go @@ -0,0 +1,42 @@ +package service + +import ( + "context" + sink "github.com/liran/sink/gen/sink" +) + +func (s *Server) Query(ctx context.Context, req *sink.QueryRequest) (*sink.QueryResponse, error) { + response := &sink.QueryResponse{} + err := s.query(ctx, req, func(frame *sink.QueryResponse) error { + response.Documents = append(response.Documents, frame.Documents...) + response.HasMore = frame.HasMore + response.Complete = frame.Complete + return nil + }) + if err != nil { + return nil, err + } + return response, nil +} + +func (s *BatchingServer) Query(ctx context.Context, req *sink.QueryRequest) (*sink.QueryResponse, error) { + return s.server.Query(ctx, req) +} + +func (s *Server) Scan(ctx context.Context, req *sink.ScanRequest) (*sink.ScanResponse, error) { + response := &sink.ScanResponse{} + err := s.scan(ctx, req, func(frame *sink.ScanResponse) error { + response.Documents = append(response.Documents, frame.Documents...) + response.NextCursor = frame.NextCursor + response.Complete = frame.Complete + return nil + }) + if err != nil { + return nil, err + } + return response, nil +} + +func (s *BatchingServer) Scan(ctx context.Context, req *sink.ScanRequest) (*sink.ScanResponse, error) { + return s.server.Scan(ctx, req) +} diff --git a/internal/service/query.go b/internal/service/query.go index 366fd3f..b53b262 100644 --- a/internal/service/query.go +++ b/internal/service/query.go @@ -13,24 +13,24 @@ import ( const maxCountResponseBytes = 256 << 10 -func (s *Server) Query(ctx context.Context, req *sink.QueryRequest) (*sink.QueryResponse, error) { +func (s *Server) query(ctx context.Context, req *sink.QueryRequest, send func(*sink.QueryResponse) error) error { maximum := forwarding.FromContext(ctx).Limit(forwarding.Returns, s.maxReadBytes) if maximum <= 0 { - return nil, status.Error(codes.ResourceExhausted, "native response budget is exhausted") + return status.Error(codes.ResourceExhausted, "native response budget is exhausted") } if err := protocol.CheckStore(req, s.boundStore); err != nil { - return nil, err + return err } request, err := nativeRequest(req.GetCommand(), maximum) if err != nil { - return nil, err + return err } pageSize := int(req.GetPageSize()) if pageSize == 0 { pageSize = 100 } if pageSize > 1000 { - return nil, status.Error(codes.InvalidArgument, "query page size exceeds 1000") + return status.Error(codes.InvalidArgument, "query page size exceeds 1000") } page := max(req.GetPage(), 1) query := storage.QueryRequest{Request: request, Offset: int64(page-1) * int64(pageSize), PageSize: pageSize} @@ -42,28 +42,34 @@ func (s *Server) Query(ctx context.Context, req *sink.QueryRequest) (*sink.Query query.Projection = &storage.Projection{Fields: projection.GetFields(), Exclude: projection.GetExclude()} } if err := query.Validate(); err != nil { - return nil, nativeStatus(err) + return nativeStatus(err) } backend, ok := s.storage.(storage.NativeStorage) if !ok { - return nil, nativeStatus(storage.ErrNativeUnsupported) + return nativeStatus(storage.ErrNativeUnsupported) + } + count := 0 + query.Emit = func(document storage.Document) error { + count++ + if count > pageSize { + return status.Error(codes.Internal, "backend exceeded query page size") + } + encoded := &sink.Document{Encoding: sink.DocumentEncoding(document.Encoding), Payload: document.Payload} + frame := &sink.QueryResponse{Documents: []*sink.Document{encoded}} + if frame.SizeVT() > maximum { + return status.Error(codes.ResourceExhausted, "query document exceeds message limit") + } + return send(frame) } result, err := backend.Query(ctx, query) if err != nil { - return nil, nativeStatus(err) + return nativeStatus(err) } - if len(result.Documents) > pageSize || (result.HasMore && len(result.Documents) != pageSize) { - return nil, status.Error(codes.Internal, "backend returned an invalid query page") + if len(result.Documents) != 0 || (result.HasMore && count != pageSize) { + return status.Error(codes.Internal, "backend returned an invalid streaming query page") } - response := &sink.QueryResponse{HasMore: result.HasMore} - for _, document := range result.Documents { - encoded := &sink.Document{Encoding: sink.DocumentEncoding(document.Encoding), Payload: document.Payload} - response.Documents = append(response.Documents, encoded) - } - if response.SizeVT() > maximum { - return nil, status.Error(codes.ResourceExhausted, "query response exceeds byte limit") - } - return response, nil + final := &sink.QueryResponse{Complete: true, HasMore: result.HasMore} + return send(final) } func (s *Server) Count(ctx context.Context, req *sink.CountRequest) (*sink.CountResponse, error) { @@ -93,10 +99,6 @@ func (s *Server) Count(ctx context.Context, req *sink.CountRequest) (*sink.Count return response, nil } -func (s *BatchingServer) Query(ctx context.Context, req *sink.QueryRequest) (*sink.QueryResponse, error) { - return s.server.Query(ctx, req) -} - func (s *BatchingServer) Count(ctx context.Context, req *sink.CountRequest) (*sink.CountResponse, error) { return s.server.Count(ctx, req) } diff --git a/internal/service/request_budgets.go b/internal/service/request_budgets.go index 6fc2b4e..cc1df8c 100644 --- a/internal/service/request_budgets.go +++ b/internal/service/request_budgets.go @@ -51,11 +51,18 @@ func (b *requestBudgets) tracker(owner int) *forwarding.Tracker { return b.trackers[owner] } func (b *requestBudgets) addContext(ctx context.Context, count int) { + if streaming, _ := ctx.Value(streamingKey{}).(bool); streaming { + for range count { + b.add(1) + b.trackers = append(b.trackers, forwarding.FromContext(ctx)) + } + return + } b.add(count) b.trackers = append(b.trackers, forwarding.FromContext(ctx)) } func contextBudgets(ctx context.Context, count int) *requestBudgets { - if forwarding.FromContext(ctx) == nil { + if streaming, _ := ctx.Value(streamingKey{}).(bool); !streaming && forwarding.FromContext(ctx) == nil { return nil } budgets := &requestBudgets{} diff --git a/internal/service/server.go b/internal/service/server.go index 5155b0c..0b703cc 100644 --- a/internal/service/server.go +++ b/internal/service/server.go @@ -21,7 +21,6 @@ import ( type Server struct { boundStore string - sink.UnimplementedSinkServer storage storage.Storage lua *merge.LuaEngine diff --git a/internal/service/stream.go b/internal/service/stream.go new file mode 100644 index 0000000..8dcde7c --- /dev/null +++ b/internal/service/stream.go @@ -0,0 +1,183 @@ +package service + +import ( + "context" + + sink "github.com/liran/sink/gen/sink" + "github.com/liran/sink/internal/protocol" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +type streamingKey struct{} + +// RPCServer adapts batch execution to result streams. It executes one existing +// microbatch at a time and sends outside the shared executor. The execution +// Server retains its batch mutation API for queue processors. +type RPCServer struct { + sink.UnimplementedSinkServer + server *Server + batching *BatchingServer +} + +func (s *Server) RPC() *RPCServer { + rpc := &RPCServer{server: s} + return rpc +} +func (s *BatchingServer) RPC() *RPCServer { + rpc := &RPCServer{server: s.server, batching: s} + return rpc +} +func (s *RPCServer) Read(req *sink.ReadRequest, stream grpc.ServerStreamingServer[sink.ReadResponse]) error { + maximum := defaultBatchMaxOperations + if s.batching != nil { + maximum = s.batching.reads.maxOperations + } + opts := readStreamOptions{maximum: maximum, batching: s.batching, send: stream.Send} + return s.server.streamRead(stream.Context(), req, opts) +} + +type readStreamOptions struct { + maximum int + batching *BatchingServer + send func(*sink.ReadResponse) error +} + +func (s *Server) streamRead(ctx context.Context, req *sink.ReadRequest, opts readStreamOptions) error { + if err := protocol.CheckStore(req, s.boundStore); err != nil { + return err + } + if len(req.GetOperations()) == 0 { + return status.Error(codes.InvalidArgument, "read request must contain operations") + } + if err := s.validateOperationCount(len(req.Operations)); err != nil { + return err + } + ctx = context.WithValue(ctx, streamingKey{}, true) + for start := 0; start < len(req.Operations); start += opts.maximum { + if err := contextError(ctx); err != nil { + return err + } + end := min(start+opts.maximum, len(req.Operations)) + batch := &sink.ReadRequest{Operations: req.Operations[start:end]} + var response *sink.ReadResponse + var err error + if opts.batching != nil { + response, err = opts.batching.Read(ctx, batch) + } else { + response, err = s.Read(ctx, batch) + } + if err != nil { + return err + } + for i, result := range response.Results { + result.OperationIndex += uint32(start) + frame := &sink.ReadResponse{Results: []*sink.ReadResult{result}} + if frame.SizeVT() > s.maxReadBytes { + return status.Error(codes.ResourceExhausted, "read result exceeds message limit") + } + if err := opts.send(frame); err != nil { + return err + } + response.Results[i] = nil + } + } + return nil +} + +func (s *RPCServer) Write(req *sink.WriteRequest, stream grpc.ServerStreamingServer[sink.WriteResponse]) error { + maximum, bytes := defaultBatchMaxOperations, defaultBatchMaxBytes + if s.batching != nil { + maximum, bytes = s.batching.writes.maxOperations, s.batching.writes.maxBytes + } + opts := writeStreamOptions{maximum: maximum, bytes: bytes, batching: s.batching, send: stream.Send} + return s.server.streamWrite(stream.Context(), req, opts) +} + +type writeStreamOptions struct { + maximum int + bytes int + batching *BatchingServer + send func(*sink.WriteResponse) error +} + +func (s *Server) streamWrite(ctx context.Context, req *sink.WriteRequest, opts writeStreamOptions) error { + if err := protocol.CheckStore(req, s.boundStore); err != nil { + return err + } + if len(req.GetOperations()) == 0 { + return status.Error(codes.InvalidArgument, "write request must contain operations") + } + if err := s.validateOperationCount(len(req.Operations)); err != nil { + return err + } + if !validCompletionMode(req.GetCompletionMode()) { + return status.Error(codes.InvalidArgument, "invalid completion mode") + } + if hasWriteReturns(req) && req.GetCompletionMode() == sink.CompletionMode_COMPLETION_MODE_RETURN_AFTER_ACCEPTED { + return status.Error(codes.InvalidArgument, "returned write documents require synchronous completion") + } + // Validate declarations before any mutation can be submitted. + if err := protocol.ValidateLuaDeclarations(req.GetLuaPrograms()); err != nil { + return status.Error(codes.InvalidArgument, err.Error()) + } + ctx = context.WithValue(ctx, streamingKey{}, true) + for start := 0; start < len(req.Operations); { + if err := contextError(ctx); err != nil { + return err + } + end, size := start, 0 + for end < len(req.Operations) && end-start < opts.maximum { + next := req.Operations[end].SizeVT() + if end > start && next > opts.bytes-size { + break + } + size += next + end++ + } + batch := &sink.WriteRequest{Operations: req.Operations[start:end], CompletionMode: req.CompletionMode, LuaPrograms: req.LuaPrograms} + var response *sink.WriteResponse + var err error + if opts.batching != nil { + response, err = opts.batching.Write(ctx, batch) + } else { + response, err = s.Write(ctx, batch) + } + if err != nil { + return err + } + for i, result := range response.Results { + result.OperationIndex += uint32(start) + frame := &sink.WriteResponse{Results: []*sink.WriteResult{result}} + if frame.SizeVT() > s.maxReadBytes { + return status.Error(codes.ResourceExhausted, "write result exceeds message limit") + } + if err := opts.send(frame); err != nil { + return err + } + response.Results[i] = nil + } + start = end + } + return nil +} + +func (s *RPCServer) Query(req *sink.QueryRequest, stream grpc.ServerStreamingServer[sink.QueryResponse]) error { + return s.server.query(stream.Context(), req, stream.Send) +} +func (s *RPCServer) Scan(req *sink.ScanRequest, stream grpc.ServerStreamingServer[sink.ScanResponse]) error { + return s.server.scan(stream.Context(), req, stream.Send) +} +func (s *RPCServer) Delete(ctx context.Context, req *sink.DeleteRequest) (*sink.DeleteResponse, error) { + if s.batching != nil { + return s.batching.Delete(ctx, req) + } + return s.server.Delete(ctx, req) +} +func (s *RPCServer) Execute(ctx context.Context, req *sink.ExecuteRequest) (*sink.ExecuteResponse, error) { + return s.server.Execute(ctx, req) +} +func (s *RPCServer) Count(ctx context.Context, req *sink.CountRequest) (*sink.CountResponse, error) { + return s.server.Count(ctx, req) +} diff --git a/internal/service/stream_test.go b/internal/service/stream_test.go new file mode 100644 index 0000000..a496ffd --- /dev/null +++ b/internal/service/stream_test.go @@ -0,0 +1,108 @@ +package service + +import ( + "context" + "errors" + "fmt" + "strings" + "testing" + + sink "github.com/liran/sink/gen/sink" + "github.com/liran/sink/internal/storage/memory" + "google.golang.org/grpc" +) + +type readSender struct { + grpc.ServerStream + ctx context.Context + send func(*sink.ReadResponse) error +} + +func (s *readSender) Context() context.Context { return s.ctx } +func (s *readSender) Send(frame *sink.ReadResponse) error { return s.send(frame) } + +type writeSender struct { + grpc.ServerStream + ctx context.Context + send func(*sink.WriteResponse) error +} + +func (s *writeSender) Context() context.Context { return s.ctx } +func (s *writeSender) Send(frame *sink.WriteResponse) error { return s.send(frame) } + +func TestReadStreamBoundsWorkingBatchAndWaitsForReceiver(t *testing.T) { + backend := &readCapacityStorage{Storage: memory.New(), maximum: 32 * (33 << 10)} + batching := completionServer(t, backend) + batching.server.maxReadBytes = 256 << 10 + request := &sink.ReadRequest{} + writes := &sink.WriteRequest{CompletionMode: sink.CompletionMode_COMPLETION_MODE_WAIT_UNTIL_APPLIED} + for i := range 96 { + key := fmt.Sprint(i) + operation := completionPut(key, i) + operation.GetPut().Document.Payload = []byte(`{"padding":"` + strings.Repeat("x", 32<<10) + `"}`) + writes.Operations = append(writes.Operations, operation) + read := &sink.ReadOperation{Address: completionAddress(key)} + request.Operations = append(request.Operations, read) + } + if _, err := batching.server.Write(t.Context(), writes); err != nil { + t.Fatal(err) + } + count := 0 + stream := &readSender{ctx: t.Context()} + stream.send = func(frame *sink.ReadResponse) error { + if len(frame.Results) != 1 || frame.Results[0].GetStatus() != sink.ReadStatus_READ_STATUS_FOUND || frame.Results[0].OperationIndex != uint32(count) { + t.Fatalf("invalid frame %v", frame) + } + if backend.reads.Load() != int64(count/32+1) { + t.Fatal("executor read ahead of downstream consumption") + } + count++ + return nil + } + if err := batching.RPC().Read(request, stream); err != nil { + t.Fatal(err) + } + if count != 96 || backend.reads.Load() != 3 { + t.Fatalf("count=%d rounds=%d", count, backend.reads.Load()) + } + // A send failure ends execution before another batch is fetched. + backend.reads.Store(0) + stop := errors.New("receiver stopped") + stream.send = func(*sink.ReadResponse) error { return stop } + if err := batching.RPC().Read(request, stream); !errors.Is(err, stop) { + t.Fatal(err) + } + if backend.reads.Load() != 1 { + t.Fatal("read continued after failed send") + } +} + +func TestReturnedWriteStreamUsesPerResultBudgetAndPreservesSameRecordOrder(t *testing.T) { + backend := &syncCapacityStorage{Storage: memory.New()} + batching := completionServer(t, backend) + batching.server.maxReadBytes = 4096 + request := &sink.WriteRequest{CompletionMode: sink.CompletionMode_COMPLETION_MODE_WAIT_UNTIL_APPLIED} + for i := range 65 { + operation := completionPut("same", i) + operation.ReturnDocument = true + operation.GetPut().Document.Payload = []byte(fmt.Sprintf(`{"value":%d,"padding":"%s"}`, i, strings.Repeat("x", 2048))) + request.Operations = append(request.Operations, operation) + } + count := 0 + stream := &writeSender{ctx: t.Context()} + stream.send = func(frame *sink.WriteResponse) error { + result := frame.Results[0] + expected := fmt.Sprintf(`"value":%d,`, count) + if result.GetStatus() != sink.WriteStatus_WRITE_STATUS_APPLIED || result.OperationIndex != uint32(count) || !strings.Contains(string(result.GetDocument().GetPayload()), expected) { + t.Fatalf("order or document lost: %v", result) + } + count++ + return nil + } + if err := batching.RPC().Write(request, stream); err != nil { + t.Fatal(err) + } + if count != 65 || backend.writes.Load() != 65 { + t.Fatalf("count=%d writes=%d", count, backend.writes.Load()) + } +} diff --git a/internal/service/write_return_retry_test.go b/internal/service/write_return_retry_test.go index 2315e7e..7765750 100644 --- a/internal/service/write_return_retry_test.go +++ b/internal/service/write_return_retry_test.go @@ -70,7 +70,9 @@ func TestReturningQuotaSettlesFinalCASOutcome(t *testing.T) { if err != nil { t.Fatal(err) } - var server sink.SinkServer = core + var server interface { + Write(context.Context, *sink.WriteRequest) (*sink.WriteResponse, error) + } = core if batching { batchOpts := service.BatchingOptions{MaxWait: time.Millisecond} batch, err := service.NewBatchingServer(core, batchOpts) diff --git a/internal/storage/mongodb/query.go b/internal/storage/mongodb/query.go index 3dfda46..7fea084 100644 --- a/internal/storage/mongodb/query.go +++ b/internal/storage/mongodb/query.go @@ -104,19 +104,30 @@ func (s *Store) Query(ctx context.Context, req storage.QueryRequest) (storage.Qu _ = cursor.Close(cleanup) }() result := storage.QueryResponse{} + count := 0 budget := storage.NewReadBudget(req.Request.MaxBytes) for cursor.Next(ctx) { // Lookahead proves HasMore but is never retained or returned to the // caller, so a large following document must not reject this page. - if len(result.Documents) == req.PageSize { + if count == req.PageSize { result.HasMore = true break } + if req.Emit != nil { + budget = storage.NewReadBudget(req.Request.MaxBytes) + } if err := budget.Reserve(len(cursor.Current)); err != nil { return empty, err } document := storage.Document{Encoding: storage.DocumentEncodingBSON, Payload: bytes.Clone(cursor.Current)} - result.Documents = append(result.Documents, document) + if req.Emit != nil { + if err := req.Emit(document); err != nil { + return empty, err + } + } else { + result.Documents = append(result.Documents, document) + } + count++ } if err := cursor.Err(); err != nil { return empty, storage.BackendError(err) diff --git a/internal/storage/mongodb/scan.go b/internal/storage/mongodb/scan.go index adf463a..f3e1bc5 100644 --- a/internal/storage/mongodb/scan.go +++ b/internal/storage/mongodb/scan.go @@ -155,12 +155,13 @@ func (s *Store) Scan(ctx context.Context, req storage.ScanRequest) (storage.Scan defer cancel() _ = cursor.Close(cleanup) }() - documents := make([]storage.Document, 0, req.BatchSize) + var documents []storage.Document + count := 0 budget := storage.NewReadBudget(req.Request.MaxBytes) var position []byte more := false for cursor.Next(ctx) { - if len(documents) == req.BatchSize { + if count == req.BatchSize { more = true break } @@ -192,14 +193,21 @@ func (s *Store) Scan(ctx context.Context, req storage.ScanRequest) (storage.Scan } } if err := budget.Reserve(len(payload)); err != nil { - if len(documents) == 0 { + if count == 0 { return empty, err } more = true break } document := storage.Document{Encoding: storage.DocumentEncodingBSON, Payload: payload} - documents = append(documents, document) + if req.Emit != nil { + if err := req.Emit(document); err != nil { + return empty, err + } + } else { + documents = append(documents, document) + } + count++ position = next } if err := cursor.Err(); err != nil { diff --git a/internal/storage/native.go b/internal/storage/native.go index d949204..1746089 100644 --- a/internal/storage/native.go +++ b/internal/storage/native.go @@ -40,6 +40,7 @@ type NativeResponse struct { } type ScanRequest struct { + Emit func(Document) error `json:"-"` Request NativeRequest BatchSize int Cursor []byte @@ -52,6 +53,7 @@ type ScanResponse struct { } type QueryRequest struct { + Emit func(Document) error `json:"-"` Request NativeRequest Offset int64 PageSize int diff --git a/internal/storage/search/client.go b/internal/storage/search/client.go index 1f00002..6d60559 100644 --- a/internal/storage/search/client.go +++ b/internal/storage/search/client.go @@ -20,6 +20,8 @@ import ( var errResponseTooLarge = errors.New("search response exceeds configured byte limit") type requestOptions struct { + decode func(io.Reader) error + emitHit func(json.RawMessage) error method string path string rawPath string @@ -138,6 +140,21 @@ func (s *Store) performOnce(ctx context.Context, opts requestOptions, endpoint * if opts.maxBytes > 0 { maximum = min(maximum, opts.maxBytes) } + if httpResponse.ContentLength > maximum { + return empty, errResponseTooLarge + } + if opts.decode != nil && httpResponse.StatusCode >= 200 && httpResponse.StatusCode < 300 { + reader := &io.LimitedReader{R: httpResponse.Body, N: maximum + 1} + err := opts.decode(reader) + if reader.N == 0 { + return empty, errResponseTooLarge + } + if err != nil { + return empty, err + } + response := apiResponse{statusCode: httpResponse.StatusCode, headers: httpResponse.Header.Clone()} + return response, nil + } body, err := io.ReadAll(io.LimitReader(httpResponse.Body, maximum+1)) if err != nil { if status.Code(err) == codes.ResourceExhausted { diff --git a/internal/storage/search/query.go b/internal/storage/search/query.go index 78459c2..2f50498 100644 --- a/internal/storage/search/query.go +++ b/internal/storage/search/query.go @@ -5,6 +5,7 @@ import ( "encoding/json" "errors" "fmt" + "io" "math" "mime" "net/http" @@ -72,6 +73,15 @@ func (s *Store) pageOptions(req storage.NativeRequest) (requestOptions, map[stri func (s *Store) performQuery(ctx context.Context, opts requestOptions) (scanPage, error) { var page scanPage + if opts.emitHit != nil { + // Retrying after delivery would duplicate callback side effects. + opts.retrySafe = false + opts.decode = func(reader io.Reader) error { + decoded, err := decodeSearchPage(reader, opts.emitHit) + page = decoded + return err + } + } response, err := s.perform(ctx, opts) if err != nil { if errors.Is(err, errResponseTooLarge) { @@ -83,8 +93,10 @@ func (s *Store) performQuery(ctx context.Context, opts requestOptions) (scanPage if response.statusCode < 200 || response.statusCode >= 300 { return page, responseError(s.driver, response) } - if err := json.Unmarshal(response.body, &page); err != nil { - return page, fmt.Errorf("decode search query: %w", err) + if opts.decode == nil { + if err := json.Unmarshal(response.body, &page); err != nil { + return page, fmt.Errorf("decode search query: %w", err) + } } if page.ScrollID != "" || page.Hits == nil || page.Hits.Hits == nil || page.TimedOut == nil || *page.TimedOut || page.TerminatedEarly { return page, errors.New("search query returned incomplete results, failed shards or an unexpected cursor") @@ -155,6 +167,20 @@ func (s *Store) Query(ctx context.Context, req storage.QueryRequest) (storage.Qu if err != nil { return empty, storage.InvalidArgumentError(err) } + emitted := 0 + if req.Emit != nil { + opts.emitHit = func(hit json.RawMessage) error { + emitted++ + if emitted > req.PageSize { + return errors.New("search query exceeded its requested result count") + } + if len(hit) > req.Request.MaxBytes && req.Request.MaxBytes > 0 { + return storage.ResourceExhaustedError(errResponseTooLarge) + } + document := storage.Document{Encoding: storage.DocumentEncodingJSON, Payload: hit} + return req.Emit(document) + } + } page, err := s.performQuery(ctx, opts) if err != nil { return empty, err @@ -167,6 +193,9 @@ func (s *Store) Query(ctx context.Context, req storage.QueryRequest) (storage.Qu return empty, err } result := storage.QueryResponse{HasMore: hasMore} + if req.Emit != nil { + return result, nil + } budget := storage.NewReadBudget(req.Request.MaxBytes) for _, hit := range page.Hits.Hits { if err := budget.Reserve(len(hit)); err != nil { diff --git a/internal/storage/search/read.go b/internal/storage/search/read.go index 4ebc713..21ffe37 100644 --- a/internal/storage/search/read.go +++ b/internal/storage/search/read.go @@ -6,6 +6,7 @@ import ( "encoding/json" "errors" "fmt" + "io" "net/http" "github.com/liran/sink/internal/storage" @@ -112,6 +113,40 @@ func (s *Store) multiGet(ctx context.Context, works []readWork, source bool) (mu payload: payload, retrySafe: true, } + var decoded multiGetResponse + opts.decode = func(reader io.Reader) error { + decoded.Documents = nil + decoder := json.NewDecoder(reader) + decoder.UseNumber() + found := false + err := objectFields(decoder, func(field string) error { + if field != "docs" { + return discardJSON(decoder) + } + found = true + if err := delimiter(decoder, '['); err != nil { + return err + } + for decoder.More() { + if len(decoded.Documents) >= len(works) { + return errors.New("search returned too many multi-get results") + } + var document multiGetDocument + if err := decoder.Decode(&document); err != nil { + return err + } + decoded.Documents = append(decoded.Documents, document) + } + return delimiter(decoder, ']') + }) + if err != nil { + return err + } + if !found { + return errors.New("search response omitted multi-get results") + } + return jsonEnd(decoder) + } response, err := s.perform(ctx, opts) if err != nil { return empty, fmt.Errorf("read search documents: %w", err) @@ -120,10 +155,6 @@ func (s *Store) multiGet(ctx context.Context, works []readWork, source bool) (mu if response.statusCode < 200 || response.statusCode >= 300 { return empty, responseError(s.driver, response) } - var decoded multiGetResponse - if err := json.Unmarshal(response.body, &decoded); err != nil { - return empty, fmt.Errorf("decode search multi-get response: %w", err) - } if len(decoded.Documents) != len(works) { return empty, fmt.Errorf("search returned %d multi-get results for %d operations", len(decoded.Documents), len(works)) } diff --git a/internal/storage/search/scan.go b/internal/storage/search/scan.go index 8ed2f66..485f26d 100644 --- a/internal/storage/search/scan.go +++ b/internal/storage/search/scan.go @@ -109,55 +109,93 @@ func (s *Store) Scan(ctx context.Context, req storage.ScanRequest) (storage.Scan opts.query.Del("track_total_hits") opts.maxBytes = int64(storage.ScanBackendBytes(req.Request.MaxBytes)) pageSize := s.scanSizes.suggest(sizingKey, req.BatchSize) - page, pageSize, err := s.scanQuery(ctx, opts, body, pageSize) - if err != nil { - return empty, err - } - if len(page.Hits.Hits) > pageSize+1 { - return empty, errors.New("search Scan exceeded its requested result count") - } - documents := make([]storage.Document, 0, req.BatchSize) + var documents []storage.Document + emitted := 0 budget := storage.NewReadBudget(req.Request.MaxBytes) previous := seek.Position var position []byte more := false - for _, hit := range page.Hits.Hits { + consume := func(hit json.RawMessage) error { + if more { + return nil + } var metadata struct { Sort json.RawMessage `json:"sort"` } if err := json.Unmarshal(hit, &metadata); err != nil { - return empty, err + return err } next, err := scanSortValues(metadata.Sort, count) if err != nil { - return empty, err + return err } if bytes.Equal(previous, next) { - return empty, errors.New("Scan sort is not unique; add a unique immutable tie-breaker field") + return errors.New("Scan sort is not unique; add a unique immutable tie-breaker field") } previous = next - if len(documents) == pageSize { + if emitted == pageSize { more = true - break + return nil } if err := budget.Reserve(len(hit)); err != nil { - if len(documents) == 0 { - return empty, err + if emitted == 0 { + return err } more = true - break + return nil } document := storage.Document{Encoding: storage.DocumentEncodingJSON, Payload: hit} - documents = append(documents, document) + if req.Emit != nil { + if err := req.Emit(document); err != nil { + return err + } + } else { + documents = append(documents, document) + } + emitted++ position = next + return nil + } + var page scanPage + if req.Emit != nil { + hits := 0 + opts.emitHit = func(hit json.RawMessage) error { + hits++ + if hits > pageSize+1 { + return errors.New("search Scan exceeded its requested result count") + } + return consume(hit) + } + body["size"] = json.RawMessage(fmt.Sprint(pageSize + 1)) + opts.payload, err = json.Marshal(body) + if err != nil { + return empty, err + } + page, err = s.performQuery(ctx, opts) + } else { + page, pageSize, err = s.scanQuery(ctx, opts, body, pageSize) + } + if err != nil { + return empty, err + } + if len(page.Hits.Hits) > pageSize+1 { + return empty, errors.New("search Scan exceeded its requested result count") } + if req.Emit == nil { + for _, hit := range page.Hits.Hits { + if err := consume(hit); err != nil { + return empty, err + } + } + } + if err := ctx.Err(); err != nil { return empty, err } if !more { position = nil - } else if len(documents) < req.BatchSize { - s.scanSizes.remember(sizingKey, len(documents)) + } else if emitted < req.BatchSize { + s.scanSizes.remember(sizingKey, emitted) } return seek.Page(documents, position) } diff --git a/internal/storage/search/stream.go b/internal/storage/search/stream.go new file mode 100644 index 0000000..3747522 --- /dev/null +++ b/internal/storage/search/stream.go @@ -0,0 +1,129 @@ +package search + +import ( + "encoding/json" + "errors" + "io" +) + +// Decode one hit at a time. Metadata can appear after hits in a JSON response; +// callers must wait for validation and stream completion before checkpointing. +func decodeSearchPage(reader io.Reader, emit func(json.RawMessage) error) (scanPage, error) { + var page scanPage + decoder := json.NewDecoder(reader) + decoder.UseNumber() + err := objectFields(decoder, func(field string) error { + switch field { + case "_scroll_id": + return decoder.Decode(&page.ScrollID) + case "timed_out": + return decoder.Decode(&page.TimedOut) + case "terminated_early": + return decoder.Decode(&page.TerminatedEarly) + case "_shards": + return decoder.Decode(&page.Shards) + case "_clusters": + return decoder.Decode(&page.Clusters) + case "hits": + page.Hits = &scanHits{} + return objectFields(decoder, func(field string) error { + switch field { + case "total": + return decoder.Decode(&page.Hits.Total) + case "hits": + if err := delimiter(decoder, '['); err != nil { + return err + } + page.Hits.Hits = make([]json.RawMessage, 0) + for decoder.More() { + var hit json.RawMessage + if err := decoder.Decode(&hit); err != nil { + return err + } + if err := emit(hit); err != nil { + return err + } + // Keep only the count required by total/has_more validation. + page.Hits.Hits = append(page.Hits.Hits, nil) + } + return delimiter(decoder, ']') + default: + return discardJSON(decoder) + } + }) + default: + return discardJSON(decoder) + } + }) + if err == nil { + err = jsonEnd(decoder) + } + return page, err +} + +func objectFields(decoder *json.Decoder, visit func(string) error) error { + if err := delimiter(decoder, '{'); err != nil { + return err + } + seen := make(map[string]bool) + for decoder.More() { + token, err := decoder.Token() + if err != nil { + return err + } + field, ok := token.(string) + if !ok || seen[field] { + return errors.New("invalid or duplicate JSON field") + } + seen[field] = true + if err := visit(field); err != nil { + return err + } + } + return delimiter(decoder, '}') +} +func delimiter(decoder *json.Decoder, expected json.Delim) error { + token, err := decoder.Token() + if err != nil { + return err + } + if token != expected { + return errors.New("unexpected JSON delimiter") + } + return nil +} +func discardJSON(decoder *json.Decoder) error { + token, err := decoder.Token() + if err != nil { + return err + } + switch token { + case json.Delim('{'): + for decoder.More() { + if _, err := decoder.Token(); err != nil { + return err + } + if err := discardJSON(decoder); err != nil { + return err + } + } + return delimiter(decoder, '}') + case json.Delim('['): + for decoder.More() { + if err := discardJSON(decoder); err != nil { + return err + } + } + return delimiter(decoder, ']') + } + return nil +} +func jsonEnd(decoder *json.Decoder) error { + if _, err := decoder.Token(); err != io.EOF { + if err != nil { + return err + } + return errors.New("unexpected trailing JSON value") + } + return nil +} diff --git a/internal/storage/search/stream_test.go b/internal/storage/search/stream_test.go new file mode 100644 index 0000000..955d5e8 --- /dev/null +++ b/internal/storage/search/stream_test.go @@ -0,0 +1,107 @@ +package search + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/liran/sink/internal/storage" +) + +func TestQueryEmitsBeforeHTTPResponseCompletes(t *testing.T) { + first := make(chan struct{}) + release := make(chan struct{}) + backend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = io.WriteString(w, `{"hits":{"hits":[{"_source":{"value":1}}`) + w.(http.Flusher).Flush() + select { + case <-release: + case <-r.Context().Done(): + return + } + _, _ = io.WriteString(w, `,{"_source":{"value":2}}],"total":{"value":2,"relation":"eq"}},"timed_out":false,"_shards":{"total":1,"successful":1,"failed":0}}`) + })) + defer backend.Close() + opts := Options{Driver: DriverOpenSearch, Store: "primary", Endpoints: []string{backend.URL}} + store, err := New(opts) + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithTimeout(t.Context(), 3*time.Second) + defer cancel() + native := storage.NativeRequest{URI: "sink://primary/items", MaxBytes: 4096} + count := 0 + request := storage.QueryRequest{Request: native, PageSize: 2} + request.Emit = func(document storage.Document) error { + count++ + if count == 1 { + close(first) + } + return nil + } + done := make(chan error, 1) + go func() { + response, err := store.Query(ctx, request) + if len(response.Documents) != 0 || response.HasMore { + err = errors.New("streaming backend retained the page") + } + done <- err + }() + select { + case <-first: + case <-ctx.Done(): + t.Fatal("query buffered the entire HTTP response") + } + close(release) + if err := <-done; err != nil { + t.Fatal(err) + } + if count != 2 { + t.Fatalf("count=%d", count) + } +} + +func TestQueryLateMetadataFailureNeverCompletesOrReplays(t *testing.T) { + for _, suffix := range []string{ + `,"timed_out":true,"_shards":{"total":1,"successful":1,"failed":0}}`, + `,"timed_out":false,"_shards":{"total":2,"successful":1,"failed":1}}`, + `,"timed_out":false,"_shards":{"total":1,"successful":1,"failed":0}} {}`, + } { + t.Run(fmt.Sprint(len(suffix)), func(t *testing.T) { + calls := 0 + backend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + calls++ + _, _ = io.WriteString(w, `{"hits":{"hits":[{"_source":{"x":1}}],"total":{"value":1,"relation":"eq"}}`+suffix) + })) + defer backend.Close() + opts := Options{Driver: DriverOpenSearch, Store: "primary", Endpoints: []string{backend.URL, backend.URL}} + store, err := New(opts) + if err != nil { + t.Fatal(err) + } + request := storage.QueryRequest{Request: storage.NativeRequest{URI: "sink://primary/items", MaxBytes: 4096}, PageSize: 1} + emitted := 0 + request.Emit = func(storage.Document) error { emitted++; return nil } + _, err = store.Query(t.Context(), request) + if err == nil || emitted != 1 || calls != 1 { + t.Fatalf("late failure replay/commit: emitted=%d calls=%d err=%v", emitted, calls, err) + } + }) + } +} + +func TestSearchStreamRejectsMalformedAndDuplicateMetadata(t *testing.T) { + for _, body := range []string{`{"hits":{"hits":[],"hits":[]}}`, `{"hits":[]}`, `{"hits":{"hits":[]}},`, strings.Repeat("x", 3)} { + _, err := decodeSearchPage(strings.NewReader(body), func(json.RawMessage) error { return nil }) + if err == nil { + t.Fatalf("accepted malformed page %s", body) + } + } +} diff --git a/proto/forward/forward.proto b/proto/forward/forward.proto index 91c3083..c92abb3 100644 --- a/proto/forward/forward.proto +++ b/proto/forward/forward.proto @@ -7,10 +7,8 @@ import "google/protobuf/any.proto"; // Private Gateway-to-Engine protocol. Never used by public clients. service Engine { - rpc Forward(ForwardRequest) returns (ForwardResponse); - // The first frame announces the encoded response size; subsequent frames - // contain bounded chunks. Public clients continue using unary Sink RPCs. - rpc ForwardStream(ForwardRequest) returns (stream ResponseFrame); + // Typed result frames followed by exactly one complete settlement frame. + rpc Forward(ForwardRequest) returns (stream ForwardResponse); } // Public response allowance, including per-record envelope charges. @@ -37,6 +35,8 @@ message ForwardRequest { } message ForwardResponse { + // The final frame carries usage and application status, with no response body. + bool complete = 9; uint32 version = 1; string store = 2; Budget used = 4; @@ -56,8 +56,3 @@ message ForwardResponse { sink.v1.ScanResponse scan = 16; } } - -message ResponseFrame { - uint64 size = 1; - bytes data = 2; -} diff --git a/proto/sink/sink.proto b/proto/sink/sink.proto index 9399431..c1dd5c5 100644 --- a/proto/sink/sink.proto +++ b/proto/sink/sink.proto @@ -9,13 +9,13 @@ option go_package = "github.com/liran/sink/gen/sink;sink"; // Record requests are batch-native. Atomicity is guaranteed per record, never // for the whole request. Native queries use Execute, Query, Count or paged Scan. service Sink { - rpc Read(ReadRequest) returns (ReadResponse); - rpc Write(WriteRequest) returns (WriteResponse); + rpc Read(ReadRequest) returns (stream ReadResponse); + rpc Write(WriteRequest) returns (stream WriteResponse); rpc Delete(DeleteRequest) returns (DeleteResponse); rpc Execute(ExecuteRequest) returns (ExecuteResponse); - rpc Query(QueryRequest) returns (QueryResponse); + rpc Query(QueryRequest) returns (stream QueryResponse); rpc Count(CountRequest) returns (CountResponse); - rpc Scan(ScanRequest) returns (ScanResponse); + rpc Scan(ScanRequest) returns (stream ScanResponse); } // RecordAddress is a canonical sink:/// URI. The core treats the @@ -68,8 +68,8 @@ message ReadOperation { } message ReadResponse { - // Results are returned in request order. operation_index is included to make - // correlation explicit when an implementation executes records in parallel. + // Each frame contains one result. Results may arrive out of order; correlate + // by operation_index. Successful EOF requires exactly one result per operation. repeated ReadResult results = 1; } @@ -147,6 +147,8 @@ message LuaProgram { bytes sha256 = 2; } +// Each frame contains one final result, identified by operation_index. +// A disconnected stream does not establish the outcome of missing operations. message WriteResponse { repeated WriteResult results = 1; } @@ -326,7 +328,10 @@ message Projection { bool exclude = 2; // False includes listed fields; true excludes them. } +// Document frames contain one document. A final, document-free frame sets +// complete and has_more. Metadata is valid only after successful EOF. message QueryResponse { + bool complete = 3; repeated Document documents = 1; // Determined by fetching one extra result, not by running Count. Result-window @@ -376,12 +381,15 @@ message ScanRequest { } // Documents retain native encodings: BSON documents or complete JSON hits. -// Failed requests return no page; callers may retry from their last saved -// cursor, with live-query semantics and idempotent business processing. +// A failed stream may have delivered documents but never a committed page. +// Resume from the last saved cursor with idempotent business processing. +// Document frames contain one document. A final, document-free frame sets +// complete and next_cursor. Checkpoint only after successful EOF. message ScanResponse { + bool complete = 3; repeated Document documents = 1; // Empty means the scan reached the end observed by this request. Commit a - // checkpoint only after processing documents. No explicit close is needed. + // checkpoint only after processing documents and successful EOF. No close RPC is needed. bytes next_cursor = 2; } From 347f3b90fcecf7421c8d6d5d5ee22e65d15d850d Mon Sep 17 00:00:00 2001 From: Randy <857271370@qq.com> Date: Sun, 20 Sep 2026 16:35:37 +0800 Subject: [PATCH 2/6] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20refactor:=20remove=20f?= =?UTF-8?q?orwarding=20budget=20and=20settlement=20protocol?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/ci.yml | 4 +- .github/workflows/release-image.yml | 4 +- .github/workflows/reliability.yml | 4 +- docs/architecture.md | 2 +- docs/batching.md | 10 +- docs/configuration.md | 8 +- docs/design/demand-based-admission.md | 6 +- docs/design/store-isolated-architecture.md | 103 ++--- docs/design/store-isolated-validation.md | 20 +- docs/document-write-flow.md | 5 +- docs/merge-folding.md | 7 +- docs/native-access.md | 2 +- docs/production-sizing.md | 4 +- docs/record-addresses.md | 5 +- docs/reliability.md | 51 ++- docs/rolling-upgrades.md | 5 +- docs/store-isolation.md | 39 +- docs/streaming.md | 6 +- gen/forward/forward.pb.go | 229 +++-------- gen/forward/forward_grpc.pb.go | 4 +- gen/forward/forward_vtproto.pb.go | 411 -------------------- internal/app/logging.go | 17 +- internal/app/logging_test.go | 5 - internal/engine/memory_test.go | 15 +- internal/engine/server.go | 76 ++-- internal/engine/stream.go | 7 + internal/forwarding/budget.go | 84 ---- internal/forwarding/protocol.go | 11 + internal/gateway/batch.go | 76 +--- internal/gateway/gateway_test.go | 12 +- internal/gateway/methods.go | 43 +- internal/gateway/record_stream.go | 14 +- internal/gateway/response_budget_test.go | 34 -- internal/gateway/server.go | 53 +-- internal/gateway/stream.go | 78 ++-- internal/metrics/forwarding.go | 9 +- internal/metrics/forwarding_test.go | 4 +- internal/service/batching_server.go | 6 +- internal/service/failure_budget.go | 2 +- internal/service/native.go | 11 +- internal/service/query.go | 11 +- internal/service/read.go | 7 +- internal/service/request_budgets.go | 71 ---- internal/service/response_groups.go | 61 +++ internal/service/server.go | 6 +- internal/service/write.go | 2 +- internal/service/write_return.go | 11 +- internal/storage/read_budget.go | 15 +- internal/storage/search/read_budget_test.go | 15 +- internal/storage/search/stream_test.go | 114 ++++++ proto/forward/forward.proto | 21 +- 51 files changed, 539 insertions(+), 1281 deletions(-) delete mode 100644 internal/forwarding/budget.go create mode 100644 internal/forwarding/protocol.go delete mode 100644 internal/gateway/response_budget_test.go delete mode 100644 internal/service/request_budgets.go create mode 100644 internal/service/response_groups.go diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ec94ace..9f836b5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -147,8 +147,8 @@ jobs: qualification: name: External test suite - uses: batchstream/sink-production-suite/.github/workflows/server-qualification.yml@da993409f9a6fd3b99012a285b4308253158af10 + uses: batchstream/sink-production-suite/.github/workflows/server-qualification.yml@730b49c71d1a6344c2986eae7bc9baaf2e1f271b with: - suite_ref: da993409f9a6fd3b99012a285b4308253158af10 + suite_ref: 730b49c71d1a6344c2986eae7bc9baaf2e1f271b sink_ref: ${{ github.event.pull_request.head.sha || github.sha }} sdk_ref: ${{ github.head_ref || 'main' }} diff --git a/.github/workflows/release-image.yml b/.github/workflows/release-image.yml index 156de3a..5f51835 100644 --- a/.github/workflows/release-image.yml +++ b/.github/workflows/release-image.yml @@ -20,9 +20,9 @@ env: jobs: qualify: name: Public production qualification - uses: batchstream/sink-production-suite/.github/workflows/release-qualification.yml@da993409f9a6fd3b99012a285b4308253158af10 + uses: batchstream/sink-production-suite/.github/workflows/release-qualification.yml@730b49c71d1a6344c2986eae7bc9baaf2e1f271b with: - suite_ref: da993409f9a6fd3b99012a285b4308253158af10 + suite_ref: 730b49c71d1a6344c2986eae7bc9baaf2e1f271b sink_ref: ${{ github.event.release.tag_name }} binaries: diff --git a/.github/workflows/reliability.yml b/.github/workflows/reliability.yml index f63040c..6526688 100644 --- a/.github/workflows/reliability.yml +++ b/.github/workflows/reliability.yml @@ -8,7 +8,7 @@ permissions: jobs: soak: - uses: batchstream/sink-production-suite/.github/workflows/reliability.yml@da993409f9a6fd3b99012a285b4308253158af10 + uses: batchstream/sink-production-suite/.github/workflows/reliability.yml@730b49c71d1a6344c2986eae7bc9baaf2e1f271b with: - suite_ref: da993409f9a6fd3b99012a285b4308253158af10 + suite_ref: 730b49c71d1a6344c2986eae7bc9baaf2e1f271b sink_ref: ${{ github.sha }} diff --git a/docs/architecture.md b/docs/architecture.md index 118dcba..f300f8b 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -11,7 +11,7 @@ quickstart. Deployment settings and their validation rules live in the New deployments use Gateway for routing, one Engine process per Store, and one Worker role per Store. Engine and Worker replicas scale independently and share the execution core; Worker directly accesses its database. Gateway does not open -database or Kafka connections. The seven public RPCs remain unchanged. +database or Kafka connections. Read, Write, Query and Scan stream results; Delete, Execute and Count return scalar responses. See [isolated topology, budgets and migration](store-isolation.md) for the private forwarding contract and cross-Store request behavior. Execution details below diff --git a/docs/batching.md b/docs/batching.md index 103f21f..550c02b 100644 --- a/docs/batching.md +++ b/docs/batching.md @@ -47,8 +47,8 @@ alone does not release an executing document. Backend bulk calls still return together; Sink cannot acknowledge an item whose backend result is not yet known. Gateway splits cross-Store requests before forwarding them. Engine accepts only its -bound Store and never bypasses that check through the batching layer. Budget-sensitive -Store groups follow the scheduling policy in the [runtime guide](store-isolation.md). +bound Store and never bypasses that check through the batching layer. Independent +Store groups follow the bounded-fanout policy in the [runtime guide](store-isolation.md). Read, Write and Delete each have one bounded queue in an Engine process. Queue operation and byte limits apply per method; the process has three such queues. @@ -61,9 +61,9 @@ Execution follows the callers' deadlines and cancellation; there is no default whole-request timeout. Admitted batches continue through memory pressure. The entire collected batch executes without independent snapshot/output byte quotas or memory-based splitting. -Record dependencies and adapter/completion grouping still apply. Public response -allowances remain per original RPC; returned documents must fit the gRPC ceiling -before their own commit. Kafka producer buffers and Lua sandbox limits remain. +Record dependencies and adapter/completion grouping still apply. Streaming RPCs +submit one existing microbatch at a time and check each returned result against +the local gRPC ceiling before its own commit. Kafka producer buffers and Lua sandbox limits remain. Queue limits count waiting work only, excluding dispatched batches. Graceful shutdown drains gRPC calls before stopping batch dispatchers. See [memory admission](design/demand-based-admission.md). diff --git a/docs/configuration.md b/docs/configuration.md index aa63b1b..aa04b53 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -62,7 +62,7 @@ use duration strings such as `2ms`, `30s`, or `1m30s`. The `_bytes` field names describe what the limit measures; they do not require writing raw byte counts. This is a breaking YAML change. See the [migration guide](configuration-migration.md) -for the complete old-to-new mapping. The gRPC and SDK contracts are unchanged. +for the complete old-to-new mapping. The streaming gRPC and SDK contracts require coordinated upgrades. ## One Store per Engine or Worker @@ -186,9 +186,9 @@ bounds a Lua invocation, and `consumer.processing_timeout` bounds a Worker proce round; none of these is a whole-RPC deadline. There is no `request.max_response_bytes` or `request.max_read_bytes` knob. Gateway -uses `grpc.max_send_message_bytes` as the transport ceiling and reserves encoded -result overhead before forwarding. Returned documents share the remaining budget -across all Store groups of the original RPC. Oversized returned writes are rejected +uses `grpc.max_send_message_bytes` as the per-message transport ceiling. Each +streamed result is checked independently; no response allowance is transferred +between Gateway and Engine. Oversized returned writes are rejected before their own commit. Intermediate snapshots and candidate outputs use process memory protection without independent quotas. Collected batches execute together. diff --git a/docs/design/demand-based-admission.md b/docs/design/demand-based-admission.md index 7eddeab..0c4d97d 100644 --- a/docs/design/demand-based-admission.md +++ b/docs/design/demand-based-admission.md @@ -76,9 +76,9 @@ splitting. Record ordering, adapter grouping, completion modes and conflict retries still determine execution groups. The waiting queue retains its encoded byte and operation limits; dispatched work no longer counts toward those limits. -The gRPC message ceiling still limits public responses. Each original RPC keeps -its response allowance across Stores, batching and retries. A requested returned -document must fit before its own write commits. This is a wire-format limit, +The gRPC message ceiling limits each streamed result independently. Gateway +and Engine do not exchange response allowances. A requested returned document +must fit the Engine limit before its own write commits. This is a wire-format limit, separate from process memory protection. Lua sandbox and Kafka buffer limits also remain in effect. diff --git a/docs/design/store-isolated-architecture.md b/docs/design/store-isolated-architecture.md index 050e6d5..eac3c32 100644 --- a/docs/design/store-isolated-architecture.md +++ b/docs/design/store-isolated-architecture.md @@ -65,7 +65,7 @@ flowchart LR | Component | Responsibilities | Excluded responsibilities | Connections and configuration | | --- | --- | --- | --- | -| Gateway | Routing, public request boundary validation, cross-Store splitting and result merging, request-level budgets, forwarding backpressure, observability | Lua execution, database access, Kafka publishing or consumption, business-operation retries | Engine service addresses, downstream connections, secure transport settings; no database or Kafka credentials | +| Gateway | Routing, public request boundary validation, cross-Store splitting and result forwarding, message limits, stream backpressure, observability | Lua execution, database access, Kafka publishing or consumption, business-operation retries | Engine service addresses, downstream connections, secure transport settings; no database or Kafka credentials | | Engine | Read/Write/Delete/Execute/Query/Count/Scan for one Store, batching, Lua, conditional writes, asynchronous publishing | Multi-Store routing, Kafka consumption, cross-Store result merging | This Store's database connection pool; a producer when asynchronous publishing is enabled | | Worker | Consumption, execution, retries, DLQ handling, and offset commits for one Store | Public business RPCs, cross-Store routing, calling Engine RPCs to perform writes | This Store's database connection pool, consumer, and DLQ producer | @@ -136,7 +136,7 @@ check alone cannot detect a different database configured under the same name. ### 4.4 Meaning of Stateless -Route snapshots, connection caches, metrics, and in-flight request budgets are reconstructible process state, not persistent business state. +Route snapshots, connection caches, metrics, and in-flight request state are reconstructible process state, not persistent business state. Gateway replicas do not replicate request state or require sticky sessions. Scan cursors retain their existing semantics without database sessions. A Gateway crash can leave callers without results for in-flight RPCs. A new replica cannot recover those RPCs or automatically replay their writes. @@ -164,61 +164,42 @@ Cross-Store requests remain non-transactional. This design adds neither compensa ### 5.2 Internal Forwarding Contract -Copying the public RPC unchanged to multiple Engines is insufficient to preserve the original request's shared budgets. -Gateway and Engine use a versioned internal forwarding contract while the public client protobuf remains unchanged. -The internal contract adds only Store identity, request correlation, budget grants/settlements, and capability information. Business requests and responses reuse the public messages. -This is a new internal capability introduced by the design and requires implementation and validation. +Gateway and Engine use private protocol version 8. `Forward` carries typed public +results with Store identity and protocol version. Read, Write, Query and Scan +return streams; scalar methods carry one result. Standard gRPC status and EOF +finish every stream. There are no budget grants, usage reports, settlement frames +or custom status envelopes. -Internal protocol v2 has a fixed set of seven methods and four document-budget categories. Each call validates version and identity without a separate capability-negotiation round trip. Request limits are validated in each process's configuration. Version 2 uses only the Store name for identity; the removed database field number and name remain reserved. Gateway and Engine must run matching protocol versions; a version mismatch fails before execution. -An incompatible target makes only its route unavailable. Internal methods must not become unprotected entry points that bypass public limits. -Gateway does not trust internal control fields supplied by clients. Engine uses the smaller of each internal grant and its local limit. -When Engine combines multiple original calls, it retains their separate budget ownership and never pools one request's allowance with another's. +Engine validates version, identity and request size before invoking the service. +A private `sink-forward-not-started` trailer marks an execution-time rejection +only when the service was never invoked. Gateway trusts that marker only on a +failed stream with no delivered results. Status codes alone cannot prove that a +mutation did not run. Public status details pass through unchanged. -### 5.3 Request Budgets: Check Before Commit +### 5.3 Local Message Limits: Check Before Commit -There are three distinct kinds of budget; they must not be collapsed into one configuration value: +Gateway bounds fanout with `max_fanout` for every Store group. Engine processes +one existing microbatch at a time; sends occur after execution releases its +shared scheduling locks. A slow receiver delays the next microbatch through +stream backpressure. Same-record operations retain their request order. -1. **Public request contract:** total operations, input message size, read/returned-document limits, and deadline. Splitting a request must not turn a 32MiB allowance into 32MiB per Store. -2. **Gateway resources:** input, results, envelope overhead, in-flight subrequests, and connection caches. Both global and per-route usage are bounded. -3. **Local Engine/Worker resources:** working sets, driver buffers, Lua, queues, and connection pools, limited within each process. +Each process enforces its own configured message ceiling. Read and Write results +must fit individually; returned-write candidates are checked before their own +commit. Query emits individual documents and Scan retains its existing page +boundary. Scalar Delete reserves room for result envelopes before execution. +There is no cross-Store balance to allocate, transfer, settle or reuse. -**The first version uses two scheduling paths:** +### 5.4 Response and Memory Boundaries -| Request type | Scheduling | -| --- | --- | -| Delete, plain Put known not to consume shared document budgets, and asynchronous acceptance requests | Bounded parallelism by Store; reserve status/error envelope space for the entire public response | -| Cross-Store Read and synchronous Write containing Merge, conditional writes, or returned documents | Execute complete Store groups in order of first appearance, using the remaining budget; batching within each group remains available | -| Single-Store requests and the four native RPCs | No cross-Store scheduling; apply both public and Engine limits directly | - -Whether a plain Put qualifies for the fast path must follow actual execution semantics. Operations that coalesce into a conditional chain must not be misclassified. -If any part of a mixed synchronous Write needs shared document budgets, the entire request uses the budget-aware path. Do not discover a total-budget violation only after parallel commits. - -Rules for the budget-aware path: - -1. Gateway creates one document-budget ledger for the request. Account for budget categories separately according to existing semantics; do not arbitrarily add inputs, snapshots, and responses together. -2. Before scheduling a Store, grant Engine the remaining snapshot, conditional-output, and returned-document allowances, subject to its local limits. -3. Engine enforces limits when retaining read results, generating candidate documents, and **before committing writes**. It returns known results and the corresponding budget settlement. -4. Retained responses continue to consume Gateway capacity. Releasable temporary working sets are released only after their owner has finished. -5. Process the next Store only after Engine returns a complete settlement. CAS retries do not reset the public request's returned-document allowance. -6. A missing settlement is not evidence that a grant was unused. Conservatively retain the allowance delegated to that group. Continue other groups only with a balance that is provably available; do not wait indefinitely. -7. Operations with insufficient allowance receive the appropriate failure. Do not relabel known successful writes as unexecuted or silently discard returned documents promised by successful operations. - -For snapshot/conditional-document budgets that currently reset on each attempt, internal settlement reports the group's maximum charge across attempts for each category. Gateway accumulates those values across Stores. -This mapping is safe but conservative: combining different retry attempts from different Stores can exhaust a near-limit request earlier than the old single-process implementation. -That boundary difference must be explicitly accepted during review and validated against the old implementation. Scheduling and the exact set of near-limit requests that succeed are not claimed to be identical. - -**The cost is lower parallelism for budget-sensitive cross-Store requests.** The high-frequency single-Store path is unaffected by this serialization policy. -This avoids denying a large Store an otherwise available budget through equal allocation, and avoids distributed budget leases and wait cycles introduced solely for parallelism. -If review requires preserving parallelism on these cross-Store paths or every near-limit budget outcome, extend the coordination protocol before implementation. Never silently give every subrequest a separate full allowance. - -Do not let every Store execute with a full allowance and then truncate returned documents at Gateway. Existing returned-document limits are checked before commit; truncation after commit would break that safety boundary. - -### 5.4 Total Response and Memory Boundaries +Gateway relays typed frames without accumulating complete responses. It validates +each result's identity, type, size and operation index before delivery. Engine +also checks frame size before sending. Align Gateway and Engine message limits; +a smaller downstream limit can interrupt a stream after earlier writes commit. -Before scheduling, Gateway reserves space for result envelopes and the maximum permitted document occupancy. Final gRPC serialization must not be the first place that detects an oversized response. -Per-route and global forwarding budgets jointly bound active calls and retained responses, preventing a few slow Stores from consuming all Gateway capacity. -Capacity planning must also include input copies, downstream receive buffers, and protobuf encoding overhead. Public document allowances are not RSS limits. -If an Engine has a smaller request limit, Gateway must handle that mismatch before sending, based on declared capabilities. Configuration differences must not reject the rest of a request after an earlier part has already written. +Process admission, existing method queues, backend buffers and Lua limits still +bound their respective work. Input batches, active microbatches, transport buffers +and concurrent calls contribute to RSS; a message ceiling is not a process memory +limit. Worker consumes the existing internal batch execution API unchanged. ## 6. Partial Failures, Timeouts, and Unknown Outcomes @@ -229,12 +210,12 @@ If an Engine has a smaller request limit, Gateway must handle that mismatch befo | A Store route is absent, identity mismatches, or scheduling fails before sending | Fail the affected operations; identify a safe temporary rejection only when non-dispatch can be proven | Gateway still does not replay | | Engine returns complete, valid results | Preserve status, revision, document, and failure; restore original indexes | No | | A subrequest disconnects, times out, or returns malformed results after sending | Operations in that group without confirmed results have unknown outcomes; preserve known results from other groups | No | -| The original call is cancelled/times out, or Gateway crashes | Cancel downstream promptly; the complete aggregate response may be undeliverable, and existing side effects remain | No | +| The original call is cancelled/times out, or Gateway crashes | Cancel downstream promptly; pending stream results may be undeliverable, and existing side effects remain | No | Public Failure currently has only code/message/retryable, without a separate field indicating whether side effects occurred. -The first version leaves public messages unchanged. Unknown mutation outcomes use appropriate UNAVAILABLE/DEADLINE_EXCEEDED/INTERNAL failures, `retryable=false`, and an explicit explanation that the outcome is unknown. +Unknown mutation outcomes use appropriate UNAVAILABLE/DEADLINE_EXCEEDED/INTERNAL failures, `retryable=false`, and an explicit explanation that the outcome is unknown. This does not mean the error is permanently unrecoverable. It means safe automatic replay cannot be justified; applications must verify state or use their own idempotency mechanism. -Error text must not become a machine-readable protocol. If callers later need to distinguish unknown outcomes programmatically, review a backward-compatible field extension separately. +Error text must not become a machine-readable protocol. If callers later need to distinguish unknown outcomes programmatically, review an explicit field extension separately. Reads have no write side effects and may retain existing client read retries. The first Gateway version adds no application retry layer. Preserve explicit temporary Scan admission rejections and their retry details. Keep complete native Execute error responses distinct from transport errors. @@ -342,8 +323,8 @@ The following criteria define the required behavior. The [validation record](sto | Globally invalid declarations | No Store starts writing or publishing | | Partial downstream failure | Preserve complete known successes, never retry mutations, and never disguise unknown outcomes as unexecuted operations | | Request cancellation or Gateway crash | Downstream resources are eventually released; completed side effects are acknowledged as possible, and callers receive no fabricated success | -| Returned documents/conditional writes near the total allowance | Enforce limits before commit without Gateway truncation of successful returned documents; explicitly record differences from the old implementation | -| Missing budget settlement | No allowance reuse or deadlock; remaining Stores follow ledger and failure rules | +| Returned documents/conditional writes near the local message limit | Enforce limits before commit without Gateway truncation of successful returned documents; explicitly record differences from the old implementation | +| Stream disconnect or invalid rejection marker | Preserve delivered results; classify unacknowledged mutations as unknown and do not replay | | Route change and Gateway restart | Running instances retain startup routes; new instances validate changed routes; old connections drain during shutdown | | Downstream identity/version mismatch | Reject only the affected route without incorrect writes | | One Store's database slows down or disconnects | Effects on other Stores' queues and tail latency stay within agreed bounds; the same Store's asynchronous capability follows the failure matrix | @@ -354,7 +335,7 @@ The following criteria define the required behavior. The [validation record](sto | A hot Store adds Engine replicas | Other Stores' database connection-pool counts do not increase | Performance validation covers single-Store and cross-Store requests, large returned documents, slow dependencies, and different Store counts. Record throughput, P99, CPU, RSS, connections, and backlog. -Measure both the extra hop and serialization of budget-sensitive cross-Store requests. Do not promise zero performance loss during design. +Measure the extra hop, bounded fanout and backpressure from slow stream receivers. Do not promise zero performance loss during design. Set acceptance thresholds after defining workload and SLOs; arbitrary fixed percentages are not a substitute for measurement. ## 11. Implementation Decisions and Authorization @@ -367,10 +348,10 @@ The following engineering decisions follow that authorization. They do not imply | R01 | Gateway / Engine / Worker; one binary, three runtime modes, independent assembly | Implemented; Gateway branches early without constructing the execution core or storage/Kafka clients | | R02 | Use globally unique Store names; validate protocol version and Store on every internal forward | Implemented; deployment inventory guarantees names and actual database ownership across configurations | | R03 | Files + DNS; complete snapshots, one version per request, bounded on-demand connections, periodic DNS refresh | Implemented; reject duplicate names and invalid snapshots without retaining identity history | -| R04 | A private protobuf Forward contract carries budget grants/settlements; budget-sensitive groups run sequentially | Implemented; read snapshots, inputs, outputs, and returned documents are accounted for separately; CAS uses category peaks, conservatively relative to the old path | +| R04 | A private protobuf Forward stream carries typed results and standard gRPC status | Protocol v8 removes all grants, usage ledgers and settlement frames; local message limits, bounded fanout and backpressure remain | | R05 | Unknown write outcomes are neither replayed nor marked safe to retry; preserve other Stores' known results | Implemented; no public effect field is added, and cancellation may prevent delivery of partial responses | -| R06 | Separate process readiness from dependency capability health; each role exposes metrics | Implemented; Gateway does not proxy legacy named dependency health services; the seven public business RPCs remain unchanged | -| R07 | Only Gateway / Engine / Worker; remove server/all, plural storages, internal multi-Store routers, per-Store execution subquotas, and obsolete examples | The user explicitly authorized complete cleanup. Old configurations are rejected; migrate process configuration and topology before upgrading. The public client protocol remains unchanged. | +| R06 | Separate process readiness from dependency capability health; each role exposes metrics | Implemented; Gateway does not proxy legacy named dependency health services; public methods retain their operation semantics while result-bearing methods stream | +| R07 | Only Gateway / Engine / Worker; remove server/all, plural storages, internal multi-Store routers, per-Store execution subquotas, and obsolete examples | The user explicitly authorized complete cleanup. Old configurations are rejected; migrate process configuration and topology before upgrading. The streaming client protocol requires a coordinated SDK upgrade. | The [deployment and configuration guide](../store-isolation.md) is authoritative for runnable fields and migration steps. Sections 3–10 record semantic agreements; this table records implementation decisions; the runtime guide provides exact configuration fields. @@ -380,7 +361,7 @@ Sections 3–10 record semantic agreements; this table records implementation de - Preserve C02–C11. Do not reintroduce multi-Store Engines, make Workers call Engine for execution, or give Gateway database/Kafka connections. - Update this document and the runtime guide together when boundaries change, explicitly describing budgets, failure semantics, and migration costs. - Implementation and testing are authorized without waiting for individual review. Release and production deployment remain separate decisions. -- Keep the public protobuf unchanged. The internal protocol lives in `proto/forward/forward.proto`; reject version mismatches before side effects. +- Update the SDK alongside public protobuf changes. The internal protocol lives in `proto/forward/forward.proto`; reject version mismatches before side effects. - Run new configuration checks and route/budget/lost-response/snapshot/connection/DNS regression tests in the default offline suite. - Use isolated containers and the production suite's `make test-isolated` for real-backend validation. Successful compilation is not evidence of successful execution. - See the [validation record](store-isolated-validation.md) for completed work and limitations. @@ -394,7 +375,7 @@ Confirmed agreements come from this user discussion. Older documentation is used - [Existing storage initialization](../../internal/app/storage.go): driver instances and health checks for configured Stores. - [Existing Worker assembly](../../internal/app/kafka.go): direct reuse of the local execution service for consumed messages. - [Returned-document budgets](../../internal/service/write_return.go): reservation and settlement of returned-document space before commit. -- [Conditional-write retries](../../internal/service/write.go) and [original-request budget ownership](../../internal/service/request_budgets.go): boundaries that cross-Store splitting must preserve. +- [Conditional-write retries](../../internal/service/write.go) and [local response boundaries](../../internal/service/response_groups.go): boundaries that cross-Store splitting must preserve. - [Existing health checks](../../internal/app/health.go): aggregate dependency-readiness logic that the new architecture separates. - [Reliability contract](../reliability.md) and [metrics](../observability.md): compatibility baselines for failures, retries, and observability. - [gRPC deadlines](https://grpc.io/docs/guides/deadlines/) and [cancellation propagation](https://grpc.io/docs/guides/cancellation/): propagate remaining deadlines and cancellation across calls; business logic must stop derived work. diff --git a/docs/design/store-isolated-validation.md b/docs/design/store-isolated-validation.md index 0341581..9f1a93f 100644 --- a/docs/design/store-isolated-validation.md +++ b/docs/design/store-isolated-validation.md @@ -14,12 +14,12 @@ implementation are unchanged. No release or production rollout is included. rejected before effects. The removed `server`/`all` modes, plural `storages`, per-Store execution subquotas, and Store byte ceilings are rejected. - Cross-Store batches preserve indexes, duplicate keys, Lua references, global - validation, partial failures, and budgets for returned documents. + validation, partial failures, and per-result message limits. - Internal forwarding supports all seven public RPCs, gRPC status details, deadlines, and cancellation. Unknown mutation outcomes are never replayed. -- Budgets are checked before commit and remain scoped to each original RPC even - when requests are coalesced. Saturating one Store's forwarding allowance leaves - another Store's allowance available. +- Returned documents are checked against the local message limit before commit. + The private protocol has no grants or usage settlement; independent Store + groups use bounded fanout and per-result stream backpressure. - Gateway uses an immutable startup route snapshot. Configuration changes take effect after restart, and duplicate Store names fail validation. Removed route file, reload interval, and state settings are rejected. @@ -41,8 +41,8 @@ External backend tests remain explicit opt-ins. delivery, reads, deletes, and metrics. - Existing sink-go `TestSinkCompatibility` and `TestNativeCompatibility` pass through Gateway against that topology with the race detector enabled. -- Public protobuf and generated public bindings are unchanged. The public schema - matches sink-go after its Go package-name substitution. +- Public protobuf and generated bindings now stream result-bearing methods. The + schema matches sink-go after its Go package-name substitution; upgrades are coordinated. - The production suite now uses separate Engines and distinct database targets in cross-Store scenarios. Its seven-Store deployment has two Gateways, two Engines per Store, and one Worker per asynchronous Store. @@ -90,8 +90,8 @@ wait; measured before legacy cleanup: | Gateway → Engine | 1.71 ms | 31,964 B / 519 allocations | This measures the local extra hop. It does not establish production throughput, -P99, or a capacity recommendation for 2 CPU / 4 GiB. Budget-sensitive cross-Store -calls execute sequentially; production sizing still needs actual document sizes, +P99, or a capacity recommendation for 2 CPU / 4 GiB. Independent cross-Store +groups use bounded fanout; production sizing still needs actual document sizes, request mixes, and latency targets. ## Boundaries @@ -113,8 +113,8 @@ request mixes, and latency targets. The configured identity is now only Store file `name` (Gateway routes use `store`). Local race tests cover duplicate route names, configuration changes taking effect only after restart, and rejection of old forwarding versions -or mismatched envelope/operation Stores before writes. The public protobuf is -unchanged. Real MongoDB/Kafka quickstart tests pass through Gateway for public +or mismatched envelope/operation Stores before writes. The current public protocol +requires the matching streaming SDK. Real MongoDB/Kafka quickstart checks exercise Gateway for public record operations, native methods, and asynchronous Worker completion. The annotated [Gateway](../../configs/gateway.yaml), diff --git a/docs/document-write-flow.md b/docs/document-write-flow.md index e4ac65b..f50dc9d 100644 --- a/docs/document-write-flow.md +++ b/docs/document-write-flow.md @@ -91,8 +91,7 @@ Gateway and Engine check process memory watermarks before admitting new work. The core validates addresses, document encodings, payloads and write actions. Engine's method queues bound waiting operations and encoded input bytes. Dispatched batches run as collected, without snapshot/output byte quotas or per-allocation -reservations. Each original RPC retains its returned-document allowance derived -from the gRPC send limit. Caller cancellation still stops work when no live caller +reservations. Each streamed result must fit the local gRPC send limit. Caller cancellation still stops work when no live caller remains; memory pressure alone never cancels an admitted batch. In this example, Upsert means "write this complete document without requiring @@ -312,7 +311,7 @@ If every modification needs its own database commit and events, issue sequential calls and wait for each to complete. See the [folding contract](merge-folding.md) for the full boundaries. Repeated Reads fetch one backend observation per address and return independent results; -every copy still counts against the response budget. Repeated synchronous +every copy must fit its result frame. Repeated synchronous Deletes issue one backend delete and return its outcome to all callers. ## 6. Which layer is doing the batching? diff --git a/docs/merge-folding.md b/docs/merge-folding.md index 2ba1456..22eb52a 100644 --- a/docs/merge-folding.md +++ b/docs/merge-folding.md @@ -101,10 +101,9 @@ requirements still apply; folding does not provide exactly-once execution or batch transactions. Reads share one backend observation per address but produce independent payload -and revision copies. Every returned document, including repetitions, consumes -the response budget in original input order. Deduplication therefore does not -allow repeated keys to bypass the public response allowance. The backend's unique-document -read also remains bounded. Missing/error results are returned to all matching +and revision copies. Every streamed result, including repetitions, must fit its +own message ceiling. Internal batch callers retain their local response limits. +The backend's unique-document read remains bounded by the active microbatch. Missing/error results are returned to all matching operations. Snapshot-based Put chains reserve read/output admission space before execution, diff --git a/docs/native-access.md b/docs/native-access.md index 75d0c0d..ab5cf94 100644 --- a/docs/native-access.md +++ b/docs/native-access.md @@ -443,7 +443,7 @@ execution begins do not advertise this pre-execution guarantee. Admitted work fo `DEADLINE_EXCEEDED` respectively. See [memory admission](design/demand-based-admission.md). Execute responses and returned Write documents share the gRPC send ceiling -semantics; returned-document budgets are per original RPC even after batching. +semantics; each returned Write result is checked independently even after batching. Count uses a separate backend response budget of min(the gRPC send ceiling, 256 KiB), enforced by the adapter. This limits backend page sizes without per-allocation memory accounting. Output space is reserved before committing a returned write. A candidate that diff --git a/docs/production-sizing.md b/docs/production-sizing.md index 75aafea..f2be482 100644 --- a/docs/production-sizing.md +++ b/docs/production-sizing.md @@ -59,12 +59,12 @@ using capacity measurements, including MongoDB allocator settings. ## Qualification before rollout -1. Exercise the unchanged public SDK through Gateway, including asynchronous +1. Exercise the matching streaming SDK through Gateway, including asynchronous acceptance and Worker delivery. 2. Measure single-Store and cross-Store requests, large returned documents, conditional writes, slow dependencies and idle Store counts. 3. Record throughput, P99, CPU, RSS, connections, queue age and failures. Include - budget-sensitive cross-Store requests, whose Store groups execute sequentially. + large cross-Store streams and slow receivers under bounded fanout. 4. Verify replica discovery with persistent clients, graceful drain, cancellation, dependency recovery and no mutation replay after a lost response. 5. Set replica limits and scaling thresholds against the workload's SLO and the diff --git a/docs/record-addresses.md b/docs/record-addresses.md index 183d678..6456edd 100644 --- a/docs/record-addresses.md +++ b/docs/record-addresses.md @@ -74,9 +74,8 @@ rendezvous hashing of the full URI and each endpoint to choose an Engine. DNS answer order and Gateway process identity do not affect the choice. Gateway groups operations by Store and chosen Engine, preserving operation order -for repeated records and restoring original result indexes. Groups share the -original RPC's budgets; adding replicas does not multiply the return budget. -Eligible independent groups run with bounded fanout. Native requests select an +for repeated records and restoring original result indexes. Each result must fit +the local message ceiling; independent groups run with bounded fanout. Native requests select an endpoint in round-robin order because they have no record key. Connections are opened lazily per endpoint and reused. `forwarding.max_connections` diff --git a/docs/reliability.md b/docs/reliability.md index 87129a2..84e34c3 100644 --- a/docs/reliability.md +++ b/docs/reliability.md @@ -220,35 +220,28 @@ Changing only a Sink config is refused to protect ordering. ## Capacity, isolation and health -Each Engine or Worker owns process-wide execution limits and byte reservations -for its bound Store. Engine publication has an independent admission pool. -Gateway splits cross-Store calls and enforces its own forwarding limits. Reads and synchronous -merges and folded conditional Put chains reserve snapshot/output space before -execution. Coalesced conditional writes share a bounded snapshot/output working -set and stream larger records through chunks. Each original RPC retains its -own cumulative quotas across those chunks; successful records are not replayed -when another chunk conflicts. Reads and returned-write response reservations -remain per original RPC. Final failed writes release their returned-document -reservation; successful CAS retries retain only the final document's charge. -Read/Write/Delete also reserve up to 1 KiB of failure text plus a result envelope -per operation before execution or publishing. Failure messages are valid UTF-8, -limited to 1 KiB each, and further shortened according to the gRPC response ceiling -per original RPC, retaining at least one byte to satisfy the client contract. -Codes, retryability, and operation indexes remain intact. -This error allowance is separate from the returned-document quota. -Micro-batches split when their input, working-set, and response reservations -exceed the process limit. Dispatched batches wait within their deadlines; direct -synchronous calls use a bounded admission queue, and publication fails fast. -Write/Delete dispatchers have bounded concurrency and preserve record dependencies -across batches, allowing independent calls to pass a refresh wait. -MongoDB group/write limits are shared across requests. Kafka producers have bounded byte buffers and fail fast -when full. Size these budgets alongside per-method waiting queues, -transport buffers, Go object/driver overhead, VM working sets, and replica count. -The admission byte gauge is not process RSS. - -Read budgets include repeated keys and all stores within one original RPC. Search reads split a response -that exceeds their transport budget before treating an individual document as -oversized. Lua output traversal bounds alias expansion, depth and node count +Gateway, Engine and Worker use process memory watermarks to admit new work. +Admitted work follows its caller cancellation and existing execution boundaries; +there are no forwarded memory grants or distributed usage ledgers. Engine method +queues retain their existing operation and input-byte limits. Streaming calls +submit one microbatch at a time and release delivered result documents before +submitting the next. Worker processing is unchanged. + +Each Read or returned Write result must fit its local message ceiling. Returned +write candidates are checked before their own commit. Scalar Delete bounds its +aggregate result envelopes before dispatch. Failure messages remain valid UTF-8, +retain their codes and retryability, and are shortened to fit their local response +boundary. Gateway checks each forwarded frame and propagates gRPC status details. + +Write/Delete dispatchers preserve record dependencies across batches. Kafka +producer buffers, backend transport bounds and Lua sandbox limits remain local. +Capacity planning includes waiting queues, active microbatches, transport and +driver buffers, VM working sets, concurrency and replica count. Logical message +limits are not process RSS limits. + +Repeated read keys produce separate streamed results without sharing a cross-Store +allowance. Search reads split an oversized backend response before treating an +individual document as oversized. Lua output traversal bounds alias expansion, depth and node count before constructing Go output values. The current embedded Lua VM does not provide a strict execution heap quota. Reviewed scripts and container memory limits are still required; independent Engine/Worker and Store deployments limit diff --git a/docs/rolling-upgrades.md b/docs/rolling-upgrades.md index 8667034..cbb999e 100644 --- a/docs/rolling-upgrades.md +++ b/docs/rolling-upgrades.md @@ -12,8 +12,9 @@ already accepted request. ## Streaming protocol upgrade -The private forwarding protocol is version 7. `Forward` is one server-streaming -RPC carrying typed results and a final usage/status settlement. The old unary +The private forwarding protocol is version 8. `Forward` is one server-streaming +RPC carrying typed results and ending with standard gRPC status and EOF. Budget +grants, usage tracking and custom settlement/status envelopes have been removed. The old unary endpoint and byte-chunk protocol have been removed. Public `Read`, `Write`, `Query` and `Scan` are server-streaming RPCs. Upgrade the diff --git a/docs/store-isolation.md b/docs/store-isolation.md index c4a5cc9..09f7371 100644 --- a/docs/store-isolation.md +++ b/docs/store-isolation.md @@ -92,28 +92,23 @@ request-wide declarations, operation counts and completion modes before dispatch Same-record operations remain together and retain their order. There is no cross-Store transaction or global write ordering. -Reads and synchronous writes requesting returned documents process Store groups in first-occurrence order. Only returned-document -allowances cross the forwarding boundary. Each Engine receives the remaining -response grant; coalesced requests retain their original RPC grants. Intermediate -snapshots and merge outputs have no separate byte quota. -Returned-document space is checked **before committing**, never truncated after -successful writes. Native requests apply the smaller of Gateway and Engine limits. - -Writes without returned documents, deletes and async acceptance can -forward Store groups concurrently within `max_fanout`. The extra network hop and -serial budget-sensitive groups have a latency cost; benchmark your workload. - -Gateway never automatically replays Write, Delete or Execute. Complete responses -from healthy Stores are retained when another Store fails. A lost Engine mutation reply -is represented as a failed operation with `retryable=false`, since the effect may -already exist. A missing budget settlement consumes the entire grant; it cannot -be reused by another Store. A proven local rejection before dispatch consumes -nothing. The public result stream itself may be cancelled before a pending result -can reach the caller; completed writes are not rolled back. - -Public gRPC status details survive forwarding, including the Scan admission retry -marker understood by existing SDKs. Client deadlines and cancellation propagate -to Engine. Neither SDK nor public protobuf changes are required. +All Store groups use bounded concurrency within `max_fanout`. Read and Write +results are relayed as individual frames; Query and Scan stream documents followed +by their public page completion metadata. Forwarding carries no byte grants or +usage settlement. Each process applies its own message ceiling and existing +memory admission. Returned-write candidates must fit the Engine's local limit +before their own commit. Align Gateway and Engine message limits. + +Gateway never automatically replays Write, Delete or Execute. Delivered results +remain known when an Engine stream fails. Missing mutation outcomes are reported +with `retryable=false` unless local non-dispatch or Engine's private rejection +trailer proves execution never started. An interrupted public stream cannot +roll back completed writes. + +Errors and status details use standard gRPC trailers, including Scan's admission +retry marker. Successful EOF ends the private stream; there is no separate +settlement frame. Deadlines and cancellation propagate to Engine. The matching +streaming SDK and Server must be upgraded together. ## Readiness, metrics and scaling diff --git a/docs/streaming.md b/docs/streaming.md index 8fca96a..a40d1d1 100644 --- a/docs/streaming.md +++ b/docs/streaming.md @@ -13,8 +13,10 @@ callback may therefore have processed part of a failed page: do not checkpoint it; use idempotent processing when resuming from the previous cursor. Gateway forwards typed Engine results directly. It never serializes or -reassembles a complete response. Forward protocol version 7 has one `Forward` -stream with result frames followed by exactly one final usage/status settlement. +reassembles a complete response. Forward protocol version 8 has one `Forward` +stream containing typed results. EOF and errors use standard gRPC status. The +protocol has no Budget, grant/used fields, Tracker or settlement frame. A private +rejection trailer proves non-execution only when no results have been delivered. No unary or binary-chunk compatibility endpoint remains. Record execution pulls one existing synchronous microbatch at a time, using the diff --git a/gen/forward/forward.pb.go b/gen/forward/forward.pb.go index 580aafd..54ab45d 100644 --- a/gen/forward/forward.pb.go +++ b/gen/forward/forward.pb.go @@ -10,7 +10,6 @@ import ( sink "github.com/liran/sink/gen/sink" protoreflect "google.golang.org/protobuf/reflect/protoreflect" protoimpl "google.golang.org/protobuf/runtime/protoimpl" - anypb "google.golang.org/protobuf/types/known/anypb" reflect "reflect" sync "sync" unsafe "unsafe" @@ -23,57 +22,10 @@ const ( _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) ) -// Public response allowance, including per-record envelope charges. -// Intermediate allocations are owned by each process memory pool. -type Budget struct { - state protoimpl.MessageState `protogen:"open.v1"` - Returns uint64 `protobuf:"varint,4,opt,name=returns,proto3" json:"returns,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache -} - -func (x *Budget) Reset() { - *x = Budget{} - mi := &file_forward_forward_proto_msgTypes[0] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) -} - -func (x *Budget) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*Budget) ProtoMessage() {} - -func (x *Budget) ProtoReflect() protoreflect.Message { - mi := &file_forward_forward_proto_msgTypes[0] - if x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use Budget.ProtoReflect.Descriptor instead. -func (*Budget) Descriptor() ([]byte, []int) { - return file_forward_forward_proto_rawDescGZIP(), []int{0} -} - -func (x *Budget) GetReturns() uint64 { - if x != nil { - return x.Returns - } - return 0 -} - type ForwardRequest struct { state protoimpl.MessageState `protogen:"open.v1"` Version uint32 `protobuf:"varint,1,opt,name=version,proto3" json:"version,omitempty"` Store string `protobuf:"bytes,2,opt,name=store,proto3" json:"store,omitempty"` - Grant *Budget `protobuf:"bytes,4,opt,name=grant,proto3" json:"grant,omitempty"` // Types that are valid to be assigned to Request: // // *ForwardRequest_Read @@ -90,7 +42,7 @@ type ForwardRequest struct { func (x *ForwardRequest) Reset() { *x = ForwardRequest{} - mi := &file_forward_forward_proto_msgTypes[1] + mi := &file_forward_forward_proto_msgTypes[0] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -102,7 +54,7 @@ func (x *ForwardRequest) String() string { func (*ForwardRequest) ProtoMessage() {} func (x *ForwardRequest) ProtoReflect() protoreflect.Message { - mi := &file_forward_forward_proto_msgTypes[1] + mi := &file_forward_forward_proto_msgTypes[0] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -115,7 +67,7 @@ func (x *ForwardRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use ForwardRequest.ProtoReflect.Descriptor instead. func (*ForwardRequest) Descriptor() ([]byte, []int) { - return file_forward_forward_proto_rawDescGZIP(), []int{1} + return file_forward_forward_proto_rawDescGZIP(), []int{0} } func (x *ForwardRequest) GetVersion() uint32 { @@ -132,13 +84,6 @@ func (x *ForwardRequest) GetStore() string { return "" } -func (x *ForwardRequest) GetGrant() *Budget { - if x != nil { - return x.Grant - } - return nil -} - func (x *ForwardRequest) GetRequest() isForwardRequest_Request { if x != nil { return x.Request @@ -256,18 +201,9 @@ func (*ForwardRequest_Count) isForwardRequest_Request() {} func (*ForwardRequest_Scan) isForwardRequest_Request() {} type ForwardResponse struct { - state protoimpl.MessageState `protogen:"open.v1"` - // The final frame carries usage and application status, with no response body. - Complete bool `protobuf:"varint,9,opt,name=complete,proto3" json:"complete,omitempty"` - Version uint32 `protobuf:"varint,1,opt,name=version,proto3" json:"version,omitempty"` - Store string `protobuf:"bytes,2,opt,name=store,proto3" json:"store,omitempty"` - Used *Budget `protobuf:"bytes,4,opt,name=used,proto3" json:"used,omitempty"` - // gRPC application status is carried inside the envelope so usage survives errors. - Code uint32 `protobuf:"varint,5,opt,name=code,proto3" json:"code,omitempty"` - Message string `protobuf:"bytes,6,opt,name=message,proto3" json:"message,omitempty"` - // True only when rejected before invoking the execution service. - NotStarted bool `protobuf:"varint,7,opt,name=not_started,json=notStarted,proto3" json:"not_started,omitempty"` - StatusDetails []*anypb.Any `protobuf:"bytes,8,rep,name=status_details,json=statusDetails,proto3" json:"status_details,omitempty"` + state protoimpl.MessageState `protogen:"open.v1"` + Version uint32 `protobuf:"varint,1,opt,name=version,proto3" json:"version,omitempty"` + Store string `protobuf:"bytes,2,opt,name=store,proto3" json:"store,omitempty"` // Types that are valid to be assigned to Response: // // *ForwardResponse_Read @@ -284,7 +220,7 @@ type ForwardResponse struct { func (x *ForwardResponse) Reset() { *x = ForwardResponse{} - mi := &file_forward_forward_proto_msgTypes[2] + mi := &file_forward_forward_proto_msgTypes[1] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -296,7 +232,7 @@ func (x *ForwardResponse) String() string { func (*ForwardResponse) ProtoMessage() {} func (x *ForwardResponse) ProtoReflect() protoreflect.Message { - mi := &file_forward_forward_proto_msgTypes[2] + mi := &file_forward_forward_proto_msgTypes[1] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -309,14 +245,7 @@ func (x *ForwardResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use ForwardResponse.ProtoReflect.Descriptor instead. func (*ForwardResponse) Descriptor() ([]byte, []int) { - return file_forward_forward_proto_rawDescGZIP(), []int{2} -} - -func (x *ForwardResponse) GetComplete() bool { - if x != nil { - return x.Complete - } - return false + return file_forward_forward_proto_rawDescGZIP(), []int{1} } func (x *ForwardResponse) GetVersion() uint32 { @@ -333,41 +262,6 @@ func (x *ForwardResponse) GetStore() string { return "" } -func (x *ForwardResponse) GetUsed() *Budget { - if x != nil { - return x.Used - } - return nil -} - -func (x *ForwardResponse) GetCode() uint32 { - if x != nil { - return x.Code - } - return 0 -} - -func (x *ForwardResponse) GetMessage() string { - if x != nil { - return x.Message - } - return "" -} - -func (x *ForwardResponse) GetNotStarted() bool { - if x != nil { - return x.NotStarted - } - return false -} - -func (x *ForwardResponse) GetStatusDetails() []*anypb.Any { - if x != nil { - return x.StatusDetails - } - return nil -} - func (x *ForwardResponse) GetResponse() isForwardResponse_Response { if x != nil { return x.Response @@ -488,13 +382,10 @@ var File_forward_forward_proto protoreflect.FileDescriptor const file_forward_forward_proto_rawDesc = "" + "\n" + - "\x15forward/forward.proto\x12\x0fsink.forward.v1\x1a\x0fsink/sink.proto\x1a\x19google/protobuf/any.proto\"P\n" + - "\x06Budget\x12\x18\n" + - "\areturns\x18\x04 \x01(\x04R\areturnsJ\x04\b\x01\x10\x02J\x04\b\x02\x10\x03J\x04\b\x03\x10\x04R\tsnapshotsR\x06inputsR\aoutputs\"\xc6\x03\n" + + "\x15forward/forward.proto\x12\x0fsink.forward.v1\x1a\x0fsink/sink.proto\"\x97\x03\n" + "\x0eForwardRequest\x12\x18\n" + "\aversion\x18\x01 \x01(\rR\aversion\x12\x14\n" + - "\x05store\x18\x02 \x01(\tR\x05store\x12-\n" + - "\x05grant\x18\x04 \x01(\v2\x17.sink.forward.v1.BudgetR\x05grant\x12*\n" + + "\x05store\x18\x02 \x01(\tR\x05store\x12*\n" + "\x04read\x18\n" + " \x01(\v2\x14.sink.v1.ReadRequestH\x00R\x04read\x12-\n" + "\x05write\x18\v \x01(\v2\x15.sink.v1.WriteRequestH\x00R\x05write\x120\n" + @@ -503,17 +394,10 @@ const file_forward_forward_proto_rawDesc = "" + "\x05query\x18\x0e \x01(\v2\x15.sink.v1.QueryRequestH\x00R\x05query\x12-\n" + "\x05count\x18\x0f \x01(\v2\x15.sink.v1.CountRequestH\x00R\x05count\x12*\n" + "\x04scan\x18\x10 \x01(\v2\x14.sink.v1.ScanRequestH\x00R\x04scanB\t\n" + - "\arequest\"\xf5\x04\n" + - "\x0fForwardResponse\x12\x1a\n" + - "\bcomplete\x18\t \x01(\bR\bcomplete\x12\x18\n" + + "\arequest\"\xa0\x03\n" + + "\x0fForwardResponse\x12\x18\n" + "\aversion\x18\x01 \x01(\rR\aversion\x12\x14\n" + "\x05store\x18\x02 \x01(\tR\x05store\x12+\n" + - "\x04used\x18\x04 \x01(\v2\x17.sink.forward.v1.BudgetR\x04used\x12\x12\n" + - "\x04code\x18\x05 \x01(\rR\x04code\x12\x18\n" + - "\amessage\x18\x06 \x01(\tR\amessage\x12\x1f\n" + - "\vnot_started\x18\a \x01(\bR\n" + - "notStarted\x12;\n" + - "\x0estatus_details\x18\b \x03(\v2\x14.google.protobuf.AnyR\rstatusDetails\x12+\n" + "\x04read\x18\n" + " \x01(\v2\x15.sink.v1.ReadResponseH\x00R\x04read\x12.\n" + "\x05write\x18\v \x01(\v2\x16.sink.v1.WriteResponseH\x00R\x05write\x121\n" + @@ -539,52 +423,47 @@ func file_forward_forward_proto_rawDescGZIP() []byte { return file_forward_forward_proto_rawDescData } -var file_forward_forward_proto_msgTypes = make([]protoimpl.MessageInfo, 3) +var file_forward_forward_proto_msgTypes = make([]protoimpl.MessageInfo, 2) var file_forward_forward_proto_goTypes = []any{ - (*Budget)(nil), // 0: sink.forward.v1.Budget - (*ForwardRequest)(nil), // 1: sink.forward.v1.ForwardRequest - (*ForwardResponse)(nil), // 2: sink.forward.v1.ForwardResponse - (*sink.ReadRequest)(nil), // 3: sink.v1.ReadRequest - (*sink.WriteRequest)(nil), // 4: sink.v1.WriteRequest - (*sink.DeleteRequest)(nil), // 5: sink.v1.DeleteRequest - (*sink.ExecuteRequest)(nil), // 6: sink.v1.ExecuteRequest - (*sink.QueryRequest)(nil), // 7: sink.v1.QueryRequest - (*sink.CountRequest)(nil), // 8: sink.v1.CountRequest - (*sink.ScanRequest)(nil), // 9: sink.v1.ScanRequest - (*anypb.Any)(nil), // 10: google.protobuf.Any - (*sink.ReadResponse)(nil), // 11: sink.v1.ReadResponse - (*sink.WriteResponse)(nil), // 12: sink.v1.WriteResponse - (*sink.DeleteResponse)(nil), // 13: sink.v1.DeleteResponse - (*sink.ExecuteResponse)(nil), // 14: sink.v1.ExecuteResponse - (*sink.QueryResponse)(nil), // 15: sink.v1.QueryResponse - (*sink.CountResponse)(nil), // 16: sink.v1.CountResponse - (*sink.ScanResponse)(nil), // 17: sink.v1.ScanResponse + (*ForwardRequest)(nil), // 0: sink.forward.v1.ForwardRequest + (*ForwardResponse)(nil), // 1: sink.forward.v1.ForwardResponse + (*sink.ReadRequest)(nil), // 2: sink.v1.ReadRequest + (*sink.WriteRequest)(nil), // 3: sink.v1.WriteRequest + (*sink.DeleteRequest)(nil), // 4: sink.v1.DeleteRequest + (*sink.ExecuteRequest)(nil), // 5: sink.v1.ExecuteRequest + (*sink.QueryRequest)(nil), // 6: sink.v1.QueryRequest + (*sink.CountRequest)(nil), // 7: sink.v1.CountRequest + (*sink.ScanRequest)(nil), // 8: sink.v1.ScanRequest + (*sink.ReadResponse)(nil), // 9: sink.v1.ReadResponse + (*sink.WriteResponse)(nil), // 10: sink.v1.WriteResponse + (*sink.DeleteResponse)(nil), // 11: sink.v1.DeleteResponse + (*sink.ExecuteResponse)(nil), // 12: sink.v1.ExecuteResponse + (*sink.QueryResponse)(nil), // 13: sink.v1.QueryResponse + (*sink.CountResponse)(nil), // 14: sink.v1.CountResponse + (*sink.ScanResponse)(nil), // 15: sink.v1.ScanResponse } var file_forward_forward_proto_depIdxs = []int32{ - 0, // 0: sink.forward.v1.ForwardRequest.grant:type_name -> sink.forward.v1.Budget - 3, // 1: sink.forward.v1.ForwardRequest.read:type_name -> sink.v1.ReadRequest - 4, // 2: sink.forward.v1.ForwardRequest.write:type_name -> sink.v1.WriteRequest - 5, // 3: sink.forward.v1.ForwardRequest.delete:type_name -> sink.v1.DeleteRequest - 6, // 4: sink.forward.v1.ForwardRequest.execute:type_name -> sink.v1.ExecuteRequest - 7, // 5: sink.forward.v1.ForwardRequest.query:type_name -> sink.v1.QueryRequest - 8, // 6: sink.forward.v1.ForwardRequest.count:type_name -> sink.v1.CountRequest - 9, // 7: sink.forward.v1.ForwardRequest.scan:type_name -> sink.v1.ScanRequest - 0, // 8: sink.forward.v1.ForwardResponse.used:type_name -> sink.forward.v1.Budget - 10, // 9: sink.forward.v1.ForwardResponse.status_details:type_name -> google.protobuf.Any - 11, // 10: sink.forward.v1.ForwardResponse.read:type_name -> sink.v1.ReadResponse - 12, // 11: sink.forward.v1.ForwardResponse.write:type_name -> sink.v1.WriteResponse - 13, // 12: sink.forward.v1.ForwardResponse.delete:type_name -> sink.v1.DeleteResponse - 14, // 13: sink.forward.v1.ForwardResponse.execute:type_name -> sink.v1.ExecuteResponse - 15, // 14: sink.forward.v1.ForwardResponse.query:type_name -> sink.v1.QueryResponse - 16, // 15: sink.forward.v1.ForwardResponse.count:type_name -> sink.v1.CountResponse - 17, // 16: sink.forward.v1.ForwardResponse.scan:type_name -> sink.v1.ScanResponse - 1, // 17: sink.forward.v1.Engine.Forward:input_type -> sink.forward.v1.ForwardRequest - 2, // 18: sink.forward.v1.Engine.Forward:output_type -> sink.forward.v1.ForwardResponse - 18, // [18:19] is the sub-list for method output_type - 17, // [17:18] is the sub-list for method input_type - 17, // [17:17] is the sub-list for extension type_name - 17, // [17:17] is the sub-list for extension extendee - 0, // [0:17] is the sub-list for field type_name + 2, // 0: sink.forward.v1.ForwardRequest.read:type_name -> sink.v1.ReadRequest + 3, // 1: sink.forward.v1.ForwardRequest.write:type_name -> sink.v1.WriteRequest + 4, // 2: sink.forward.v1.ForwardRequest.delete:type_name -> sink.v1.DeleteRequest + 5, // 3: sink.forward.v1.ForwardRequest.execute:type_name -> sink.v1.ExecuteRequest + 6, // 4: sink.forward.v1.ForwardRequest.query:type_name -> sink.v1.QueryRequest + 7, // 5: sink.forward.v1.ForwardRequest.count:type_name -> sink.v1.CountRequest + 8, // 6: sink.forward.v1.ForwardRequest.scan:type_name -> sink.v1.ScanRequest + 9, // 7: sink.forward.v1.ForwardResponse.read:type_name -> sink.v1.ReadResponse + 10, // 8: sink.forward.v1.ForwardResponse.write:type_name -> sink.v1.WriteResponse + 11, // 9: sink.forward.v1.ForwardResponse.delete:type_name -> sink.v1.DeleteResponse + 12, // 10: sink.forward.v1.ForwardResponse.execute:type_name -> sink.v1.ExecuteResponse + 13, // 11: sink.forward.v1.ForwardResponse.query:type_name -> sink.v1.QueryResponse + 14, // 12: sink.forward.v1.ForwardResponse.count:type_name -> sink.v1.CountResponse + 15, // 13: sink.forward.v1.ForwardResponse.scan:type_name -> sink.v1.ScanResponse + 0, // 14: sink.forward.v1.Engine.Forward:input_type -> sink.forward.v1.ForwardRequest + 1, // 15: sink.forward.v1.Engine.Forward:output_type -> sink.forward.v1.ForwardResponse + 15, // [15:16] is the sub-list for method output_type + 14, // [14:15] is the sub-list for method input_type + 14, // [14:14] is the sub-list for extension type_name + 14, // [14:14] is the sub-list for extension extendee + 0, // [0:14] is the sub-list for field type_name } func init() { file_forward_forward_proto_init() } @@ -592,7 +471,7 @@ func file_forward_forward_proto_init() { if File_forward_forward_proto != nil { return } - file_forward_forward_proto_msgTypes[1].OneofWrappers = []any{ + file_forward_forward_proto_msgTypes[0].OneofWrappers = []any{ (*ForwardRequest_Read)(nil), (*ForwardRequest_Write)(nil), (*ForwardRequest_Delete)(nil), @@ -601,7 +480,7 @@ func file_forward_forward_proto_init() { (*ForwardRequest_Count)(nil), (*ForwardRequest_Scan)(nil), } - file_forward_forward_proto_msgTypes[2].OneofWrappers = []any{ + file_forward_forward_proto_msgTypes[1].OneofWrappers = []any{ (*ForwardResponse_Read)(nil), (*ForwardResponse_Write)(nil), (*ForwardResponse_Delete)(nil), @@ -616,7 +495,7 @@ func file_forward_forward_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_forward_forward_proto_rawDesc), len(file_forward_forward_proto_rawDesc)), NumEnums: 0, - NumMessages: 3, + NumMessages: 2, NumExtensions: 0, NumServices: 1, }, diff --git a/gen/forward/forward_grpc.pb.go b/gen/forward/forward_grpc.pb.go index 0a0eeaf..0d06918 100644 --- a/gen/forward/forward_grpc.pb.go +++ b/gen/forward/forward_grpc.pb.go @@ -28,7 +28,7 @@ const ( // // Private Gateway-to-Engine protocol. Never used by public clients. type EngineClient interface { - // Typed result frames followed by exactly one complete settlement frame. + // Typed results; EOF and errors use the standard gRPC stream status. Forward(ctx context.Context, in *ForwardRequest, opts ...grpc.CallOption) (grpc.ServerStreamingClient[ForwardResponse], error) } @@ -65,7 +65,7 @@ type Engine_ForwardClient = grpc.ServerStreamingClient[ForwardResponse] // // Private Gateway-to-Engine protocol. Never used by public clients. type EngineServer interface { - // Typed result frames followed by exactly one complete settlement frame. + // Typed results; EOF and errors use the standard gRPC stream status. Forward(*ForwardRequest, grpc.ServerStreamingServer[ForwardResponse]) error mustEmbedUnimplementedEngineServer() } diff --git a/gen/forward/forward_vtproto.pb.go b/gen/forward/forward_vtproto.pb.go index 9b85bf7..1b0c650 100644 --- a/gen/forward/forward_vtproto.pb.go +++ b/gen/forward/forward_vtproto.pb.go @@ -8,9 +8,7 @@ import ( fmt "fmt" sink "github.com/liran/sink/gen/sink" protohelpers "github.com/planetscale/vtprotobuf/protohelpers" - anypb "github.com/planetscale/vtprotobuf/types/known/anypb" protoimpl "google.golang.org/protobuf/runtime/protoimpl" - anypb1 "google.golang.org/protobuf/types/known/anypb" io "io" ) @@ -21,44 +19,6 @@ const ( _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) ) -func (m *Budget) MarshalVT() (dAtA []byte, err error) { - if m == nil { - return nil, nil - } - size := m.SizeVT() - dAtA = make([]byte, size) - n, err := m.MarshalToSizedBufferVT(dAtA[:size]) - if err != nil { - return nil, err - } - return dAtA[:n], nil -} - -func (m *Budget) MarshalToVT(dAtA []byte) (int, error) { - size := m.SizeVT() - return m.MarshalToSizedBufferVT(dAtA[:size]) -} - -func (m *Budget) MarshalToSizedBufferVT(dAtA []byte) (int, error) { - if m == nil { - return 0, nil - } - i := len(dAtA) - _ = i - var l int - _ = l - if m.unknownFields != nil { - i -= len(m.unknownFields) - copy(dAtA[i:], m.unknownFields) - } - if m.Returns != 0 { - i = protohelpers.EncodeVarint(dAtA, i, uint64(m.Returns)) - i-- - dAtA[i] = 0x20 - } - return len(dAtA) - i, nil -} - func (m *ForwardRequest) MarshalVT() (dAtA []byte, err error) { if m == nil { return nil, nil @@ -98,16 +58,6 @@ func (m *ForwardRequest) MarshalToSizedBufferVT(dAtA []byte) (int, error) { } i -= size } - if m.Grant != nil { - size, err := m.Grant.MarshalToSizedBufferVT(dAtA[:i]) - if err != nil { - return 0, err - } - i -= size - i = protohelpers.EncodeVarint(dAtA, i, uint64(size)) - i-- - dAtA[i] = 0x22 - } if len(m.Store) > 0 { i -= len(m.Store) copy(dAtA[i:], m.Store) @@ -297,60 +247,6 @@ func (m *ForwardResponse) MarshalToSizedBufferVT(dAtA []byte) (int, error) { } i -= size } - if m.Complete { - i-- - if m.Complete { - dAtA[i] = 1 - } else { - dAtA[i] = 0 - } - i-- - dAtA[i] = 0x48 - } - if len(m.StatusDetails) > 0 { - for iNdEx := len(m.StatusDetails) - 1; iNdEx >= 0; iNdEx-- { - size, err := (*anypb.Any)(m.StatusDetails[iNdEx]).MarshalToSizedBufferVT(dAtA[:i]) - if err != nil { - return 0, err - } - i -= size - i = protohelpers.EncodeVarint(dAtA, i, uint64(size)) - i-- - dAtA[i] = 0x42 - } - } - if m.NotStarted { - i-- - if m.NotStarted { - dAtA[i] = 1 - } else { - dAtA[i] = 0 - } - i-- - dAtA[i] = 0x38 - } - if len(m.Message) > 0 { - i -= len(m.Message) - copy(dAtA[i:], m.Message) - i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.Message))) - i-- - dAtA[i] = 0x32 - } - if m.Code != 0 { - i = protohelpers.EncodeVarint(dAtA, i, uint64(m.Code)) - i-- - dAtA[i] = 0x28 - } - if m.Used != nil { - size, err := m.Used.MarshalToSizedBufferVT(dAtA[:i]) - if err != nil { - return 0, err - } - i -= size - i = protohelpers.EncodeVarint(dAtA, i, uint64(size)) - i-- - dAtA[i] = 0x22 - } if len(m.Store) > 0 { i -= len(m.Store) copy(dAtA[i:], m.Store) @@ -501,19 +397,6 @@ func (m *ForwardResponse_Scan) MarshalToSizedBufferVT(dAtA []byte) (int, error) } return len(dAtA) - i, nil } -func (m *Budget) SizeVT() (n int) { - if m == nil { - return 0 - } - var l int - _ = l - if m.Returns != 0 { - n += 1 + protohelpers.SizeOfVarint(uint64(m.Returns)) - } - n += len(m.unknownFields) - return n -} - func (m *ForwardRequest) SizeVT() (n int) { if m == nil { return 0 @@ -527,10 +410,6 @@ func (m *ForwardRequest) SizeVT() (n int) { if l > 0 { n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) } - if m.Grant != nil { - l = m.Grant.SizeVT() - n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) - } if vtmsg, ok := m.Request.(interface{ SizeVT() int }); ok { n += vtmsg.SizeVT() } @@ -635,29 +514,6 @@ func (m *ForwardResponse) SizeVT() (n int) { if l > 0 { n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) } - if m.Used != nil { - l = m.Used.SizeVT() - n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) - } - if m.Code != 0 { - n += 1 + protohelpers.SizeOfVarint(uint64(m.Code)) - } - l = len(m.Message) - if l > 0 { - n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) - } - if m.NotStarted { - n += 2 - } - if len(m.StatusDetails) > 0 { - for _, e := range m.StatusDetails { - l = (*anypb.Any)(e).SizeVT() - n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) - } - } - if m.Complete { - n += 2 - } if vtmsg, ok := m.Response.(interface{ SizeVT() int }); ok { n += vtmsg.SizeVT() } @@ -749,76 +605,6 @@ func (m *ForwardResponse_Scan) SizeVT() (n int) { } return n } -func (m *Budget) UnmarshalVT(dAtA []byte) error { - l := len(dAtA) - iNdEx := 0 - for iNdEx < l { - preIndex := iNdEx - var wire uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return protohelpers.ErrIntOverflow - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - wire |= uint64(b&0x7F) << shift - if b < 0x80 { - break - } - } - fieldNum := int32(wire >> 3) - wireType := int(wire & 0x7) - if wireType == 4 { - return fmt.Errorf("proto: Budget: wiretype end group for non-group") - } - if fieldNum <= 0 { - return fmt.Errorf("proto: Budget: illegal tag %d (wire type %d)", fieldNum, wire) - } - switch fieldNum { - case 4: - if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field Returns", wireType) - } - m.Returns = 0 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return protohelpers.ErrIntOverflow - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - m.Returns |= uint64(b&0x7F) << shift - if b < 0x80 { - break - } - } - default: - iNdEx = preIndex - skippy, err := protohelpers.Skip(dAtA[iNdEx:]) - if err != nil { - return err - } - if (skippy < 0) || (iNdEx+skippy) < 0 { - return protohelpers.ErrInvalidLength - } - if (iNdEx + skippy) > l { - return io.ErrUnexpectedEOF - } - m.unknownFields = append(m.unknownFields, dAtA[iNdEx:iNdEx+skippy]...) - iNdEx += skippy - } - } - - if iNdEx > l { - return io.ErrUnexpectedEOF - } - return nil -} func (m *ForwardRequest) UnmarshalVT(dAtA []byte) error { l := len(dAtA) iNdEx := 0 @@ -899,42 +685,6 @@ func (m *ForwardRequest) UnmarshalVT(dAtA []byte) error { } m.Store = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex - case 4: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Grant", wireType) - } - var msglen int - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return protohelpers.ErrIntOverflow - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - msglen |= int(b&0x7F) << shift - if b < 0x80 { - break - } - } - if msglen < 0 { - return protohelpers.ErrInvalidLength - } - postIndex := iNdEx + msglen - if postIndex < 0 { - return protohelpers.ErrInvalidLength - } - if postIndex > l { - return io.ErrUnexpectedEOF - } - if m.Grant == nil { - m.Grant = &Budget{} - } - if err := m.Grant.UnmarshalVT(dAtA[iNdEx:postIndex]); err != nil { - return err - } - iNdEx = postIndex case 10: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Read", wireType) @@ -1324,167 +1074,6 @@ func (m *ForwardResponse) UnmarshalVT(dAtA []byte) error { } m.Store = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex - case 4: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Used", wireType) - } - var msglen int - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return protohelpers.ErrIntOverflow - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - msglen |= int(b&0x7F) << shift - if b < 0x80 { - break - } - } - if msglen < 0 { - return protohelpers.ErrInvalidLength - } - postIndex := iNdEx + msglen - if postIndex < 0 { - return protohelpers.ErrInvalidLength - } - if postIndex > l { - return io.ErrUnexpectedEOF - } - if m.Used == nil { - m.Used = &Budget{} - } - if err := m.Used.UnmarshalVT(dAtA[iNdEx:postIndex]); err != nil { - return err - } - iNdEx = postIndex - case 5: - if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field Code", wireType) - } - m.Code = 0 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return protohelpers.ErrIntOverflow - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - m.Code |= uint32(b&0x7F) << shift - if b < 0x80 { - break - } - } - case 6: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Message", wireType) - } - var stringLen uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return protohelpers.ErrIntOverflow - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - stringLen |= uint64(b&0x7F) << shift - if b < 0x80 { - break - } - } - intStringLen := int(stringLen) - if intStringLen < 0 { - return protohelpers.ErrInvalidLength - } - postIndex := iNdEx + intStringLen - if postIndex < 0 { - return protohelpers.ErrInvalidLength - } - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.Message = string(dAtA[iNdEx:postIndex]) - iNdEx = postIndex - case 7: - if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field NotStarted", wireType) - } - var v int - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return protohelpers.ErrIntOverflow - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - v |= int(b&0x7F) << shift - if b < 0x80 { - break - } - } - m.NotStarted = bool(v != 0) - case 8: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field StatusDetails", wireType) - } - var msglen int - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return protohelpers.ErrIntOverflow - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - msglen |= int(b&0x7F) << shift - if b < 0x80 { - break - } - } - if msglen < 0 { - return protohelpers.ErrInvalidLength - } - postIndex := iNdEx + msglen - if postIndex < 0 { - return protohelpers.ErrInvalidLength - } - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.StatusDetails = append(m.StatusDetails, &anypb1.Any{}) - if err := (*anypb.Any)(m.StatusDetails[len(m.StatusDetails)-1]).UnmarshalVT(dAtA[iNdEx:postIndex]); err != nil { - return err - } - iNdEx = postIndex - case 9: - if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field Complete", wireType) - } - var v int - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return protohelpers.ErrIntOverflow - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - v |= int(b&0x7F) << shift - if b < 0x80 { - break - } - } - m.Complete = bool(v != 0) case 10: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Read", wireType) diff --git a/internal/app/logging.go b/internal/app/logging.go index ad54734..e2df708 100644 --- a/internal/app/logging.go +++ b/internal/app/logging.go @@ -6,7 +6,6 @@ import ( "strings" "time" - forward "github.com/liran/sink/gen/forward" sink "github.com/liran/sink/gen/sink" "google.golang.org/grpc" "google.golang.org/grpc/codes" @@ -20,21 +19,7 @@ func logUnary(ctx context.Context, req any, info *grpc.UnaryServerInfo, next grp if method != "" { code := status.Code(err) observed := response - if forwarded, ok := observed.(*forward.ForwardResponse); ok { - if code == codes.OK { - code = codes.Code(forwarded.GetCode()) - } - switch body := forwarded.GetResponse().(type) { - case *forward.ForwardResponse_Read: - observed = body.Read - case *forward.ForwardResponse_Write: - observed = body.Write - case *forward.ForwardResponse_Delete: - observed = body.Delete - case *forward.ForwardResponse_Execute: - observed = body.Execute - } - } + logRPC(method, code, observed, time.Since(started)) } return response, err diff --git a/internal/app/logging_test.go b/internal/app/logging_test.go index 544b53c..ea1ef84 100644 --- a/internal/app/logging_test.go +++ b/internal/app/logging_test.go @@ -8,7 +8,6 @@ import ( "strings" "testing" - forward "github.com/liran/sink/gen/forward" sink "github.com/liran/sink/gen/sink" "github.com/liran/sink/internal/config" "github.com/liran/sink/internal/logging" @@ -36,8 +35,6 @@ func TestRPCDiagnosticsCaptureApplicationFailuresWithoutPayload(t *testing.T) { failure := &sink.Failure{Code: sink.FailureCode_FAILURE_CODE_UNAVAILABLE, Message: "private document error"} result := &sink.WriteResult{Status: sink.WriteStatus_WRITE_STATUS_FAILED, Failure: failure} write := &sink.WriteResponse{Results: []*sink.WriteResult{result}} - wrapped := &forward.ForwardResponse_Write{Write: write} - envelope := &forward.ForwardResponse{Response: wrapped} native := &sink.ExecuteResponse{Success: false, Payload: []byte("private response")} tests := []struct { name string @@ -47,9 +44,7 @@ func TestRPCDiagnosticsCaptureApplicationFailuresWithoutPayload(t *testing.T) { }{ {name: "partial failure", response: write, want: "warn"}, {name: "managed failure", response: write, want: "warn"}, - {name: "managed forwarded failure", response: envelope, want: "warn"}, {name: "managed native failure", response: native, want: "warn"}, - {name: "forwarded failure", response: envelope, want: "warn"}, {name: "native failure", response: native, want: "warn"}, {name: "transport failure", err: status.Error(codes.Internal, "private error"), want: "error"}, } diff --git a/internal/engine/memory_test.go b/internal/engine/memory_test.go index 5904c51..4bfaf11 100644 --- a/internal/engine/memory_test.go +++ b/internal/engine/memory_test.go @@ -12,6 +12,8 @@ import ( "github.com/liran/sink/internal/forwarding" "google.golang.org/grpc" "google.golang.org/grpc/codes" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/status" ) type pressureService struct { @@ -40,15 +42,15 @@ func TestEnginePressureRejectsBeforeExecution(t *testing.T) { } write := &sink.WriteRequest{} body := &forward.ForwardRequest_Write{Write: write} - request := &forward.ForwardRequest{Version: forwarding.Version, Store: "primary", Grant: forwarding.FullBudget(1 << 20), Request: body} + request := &forward.ForwardRequest{Version: forwarding.Version, Store: "primary", Request: body} stream := &forwardRecorder{ctx: t.Context()} err = server.Forward(request, stream) response := stream.response - if err != nil || !response.GetNotStarted() || response.GetCode() != uint32(codes.ResourceExhausted) || backend.calls != 0 { + if status.Code(err) != codes.ResourceExhausted || response != nil || backend.calls != 0 { t.Fatalf("pressure reached service: %v %v calls=%d", response, err, backend.calls) } - if response.GetUsed().GetReturns() != 0 { - t.Fatal("rejection consumed response allowance") + if marker := stream.trailer.Get(forwarding.NotStartedTrailer); len(marker) != 1 || marker[0] != "true" { + t.Fatal("rejection omitted not-started evidence") } } @@ -56,6 +58,7 @@ type forwardRecorder struct { grpc.ServerStream ctx context.Context response *forward.ForwardResponse + trailer metadata.MD } func (s *forwardRecorder) Context() context.Context { return s.ctx } @@ -63,3 +66,7 @@ func (s *forwardRecorder) Send(response *forward.ForwardResponse) error { s.response = response return nil } + +func (s *forwardRecorder) SetTrailer(trailer metadata.MD) { + s.trailer = metadata.Join(s.trailer, trailer) +} diff --git a/internal/engine/server.go b/internal/engine/server.go index 433deca..727b0b0 100644 --- a/internal/engine/server.go +++ b/internal/engine/server.go @@ -3,7 +3,6 @@ package engine import ( "errors" - "google.golang.org/grpc" "time" forward "github.com/liran/sink/gen/forward" @@ -12,7 +11,9 @@ import ( "github.com/liran/sink/internal/forwarding" "github.com/liran/sink/internal/metrics" "github.com/liran/sink/internal/protocol" + "google.golang.org/grpc" "google.golang.org/grpc/codes" + "google.golang.org/grpc/metadata" "google.golang.org/grpc/status" ) @@ -44,21 +45,12 @@ func New(opts Options) (*Server, error) { server := &Server{memory: opts.Memory, maxRequestBytes: opts.MaxRequestBytes, service: opts.Service, store: opts.Store, maximum: opts.MaxReadBytes, metrics: opts.Metrics} return server, nil } -func (s *Server) Forward(req *forward.ForwardRequest, stream grpc.ServerStreamingServer[forward.ForwardResponse]) error { +func (s *Server) Forward(req *forward.ForwardRequest, stream grpc.ServerStreamingServer[forward.ForwardResponse]) (err error) { ctx := stream.Context() - response := &forward.ForwardResponse{Version: forwarding.Version, Store: s.store, NotStarted: true, Complete: true} started := time.Now() - defer func() { s.metrics.ObserveForward(req, response, time.Since(started)) }() - tracker := forwarding.NewTracker(req.GetGrant(), s.maximum) + defer func() { s.metrics.ObserveForward(req, status.Code(err), time.Since(started)) }() reject := func(err error) error { - response.Code = uint32(status.Code(err)) - response.Message = status.Convert(err).Message() - response.StatusDetails = status.Convert(err).Proto().GetDetails() - response.Used = tracker.Usage() - err = stream.Send(response) - if err != nil { - response.Code = uint32(status.Code(err)) - } + stream.SetTrailer(metadata.Pairs(forwarding.NotStartedTrailer, "true")) return err } if req.GetVersion() != forwarding.Version { @@ -67,9 +59,6 @@ func (s *Server) Forward(req *forward.ForwardRequest, stream grpc.ServerStreamin if req.GetStore() != s.store { return reject(status.Error(codes.FailedPrecondition, "Engine identity does not match route")) } - if req.GetGrant() == nil { - return reject(status.Error(codes.InvalidArgument, "forwarding budget is required")) - } var request any switch body := req.GetRequest().(type) { case *forward.ForwardRequest_Read: @@ -102,55 +91,40 @@ func (s *Server) Forward(req *forward.ForwardRequest, stream grpc.ServerStreamin s.metrics.AdjustInFlight(1) defer s.metrics.AdjustInFlight(-1) ctx = admitted - response.NotStarted = false - ctx = forwarding.WithTracker(ctx, tracker) - var err error switch body := req.GetRequest().(type) { case *forward.ForwardRequest_Read: output := &resultStream[sink.ReadResponse]{ServerStream: stream, metrics: s.metrics, request: req, ctx: ctx, store: s.store, maximum: s.maximum, send: stream.Send} - err = s.service.Read(body.Read, output) + return s.service.Read(body.Read, output) case *forward.ForwardRequest_Write: output := &resultStream[sink.WriteResponse]{ServerStream: stream, metrics: s.metrics, request: req, ctx: ctx, store: s.store, maximum: s.maximum, send: stream.Send} - err = s.service.Write(body.Write, output) + return s.service.Write(body.Write, output) case *forward.ForwardRequest_Delete: - result, callErr := s.service.Delete(ctx, body.Delete) - err = callErr - if callErr == nil { - frame := &forward.ForwardResponse{Version: forwarding.Version, Store: s.store} - frame.Response = &forward.ForwardResponse_Delete{Delete: result} - if sendErr := stream.Send(frame); sendErr != nil { - return sendErr - } - s.metrics.ObserveForwardResult(req, frame) + output := &resultStream[sink.DeleteResponse]{ServerStream: stream, metrics: s.metrics, request: req, ctx: ctx, store: s.store, maximum: s.maximum, send: stream.Send} + result, err := s.service.Delete(ctx, body.Delete) + if err != nil { + return err } + return output.Send(result) case *forward.ForwardRequest_Execute: - result, callErr := s.service.Execute(ctx, body.Execute) - err = callErr - if callErr == nil { - frame := &forward.ForwardResponse{Version: forwarding.Version, Store: s.store} - frame.Response = &forward.ForwardResponse_Execute{Execute: result} - if sendErr := stream.Send(frame); sendErr != nil { - return sendErr - } - s.metrics.ObserveForwardResult(req, frame) + output := &resultStream[sink.ExecuteResponse]{ServerStream: stream, metrics: s.metrics, request: req, ctx: ctx, store: s.store, maximum: s.maximum, send: stream.Send} + result, err := s.service.Execute(ctx, body.Execute) + if err != nil { + return err } + return output.Send(result) case *forward.ForwardRequest_Query: output := &resultStream[sink.QueryResponse]{ServerStream: stream, metrics: s.metrics, request: req, ctx: ctx, store: s.store, maximum: s.maximum, send: stream.Send} - err = s.service.Query(body.Query, output) + return s.service.Query(body.Query, output) case *forward.ForwardRequest_Count: - result, callErr := s.service.Count(ctx, body.Count) - err = callErr - if callErr == nil { - frame := &forward.ForwardResponse{Version: forwarding.Version, Store: s.store} - frame.Response = &forward.ForwardResponse_Count{Count: result} - if sendErr := stream.Send(frame); sendErr != nil { - return sendErr - } - s.metrics.ObserveForwardResult(req, frame) + output := &resultStream[sink.CountResponse]{ServerStream: stream, metrics: s.metrics, request: req, ctx: ctx, store: s.store, maximum: s.maximum, send: stream.Send} + result, err := s.service.Count(ctx, body.Count) + if err != nil { + return err } + return output.Send(result) case *forward.ForwardRequest_Scan: output := &resultStream[sink.ScanResponse]{ServerStream: stream, metrics: s.metrics, request: req, ctx: ctx, store: s.store, maximum: s.maximum, send: stream.Send} - err = s.service.Scan(body.Scan, output) + return s.service.Scan(body.Scan, output) } - return reject(err) + return status.Error(codes.Internal, "unsupported forwarding request") } diff --git a/internal/engine/stream.go b/internal/engine/stream.go index 28da83f..2d74044 100644 --- a/internal/engine/stream.go +++ b/internal/engine/stream.go @@ -2,6 +2,7 @@ package engine import ( "context" + forward "github.com/liran/sink/gen/forward" sink "github.com/liran/sink/gen/sink" "github.com/liran/sink/internal/forwarding" @@ -34,6 +35,12 @@ func (s *resultStream[T]) Send(result *T) error { frame.Response = &forward.ForwardResponse_Read{Read: result} case *sink.WriteResponse: frame.Response = &forward.ForwardResponse_Write{Write: result} + case *sink.DeleteResponse: + frame.Response = &forward.ForwardResponse_Delete{Delete: result} + case *sink.ExecuteResponse: + frame.Response = &forward.ForwardResponse_Execute{Execute: result} + case *sink.CountResponse: + frame.Response = &forward.ForwardResponse_Count{Count: result} case *sink.QueryResponse: frame.Response = &forward.ForwardResponse_Query{Query: result} case *sink.ScanResponse: diff --git a/internal/forwarding/budget.go b/internal/forwarding/budget.go deleted file mode 100644 index 860fcd6..0000000 --- a/internal/forwarding/budget.go +++ /dev/null @@ -1,84 +0,0 @@ -// Package forwarding contains the private Gateway-to-Engine resource contract. -package forwarding - -import ( - "context" - "sync" - - forward "github.com/liran/sink/gen/forward" - "github.com/liran/sink/internal/storage" -) - -const Version = 7 - -// EnvelopeBytes is reserved in the private transport in addition to the public message cap. -const EnvelopeBytes = 4096 - -type Kind int - -const ( - Returns Kind = iota - kinds -) - -type Tracker struct { - mu sync.Mutex - limits [kinds]int - used [kinds]int -} - -type contextKey int - -const trackerKey contextKey = 1 - -func NewTracker(grant *forward.Budget, maximum int) *Tracker { - t := &Tracker{} - raw := [...]uint64{grant.GetReturns()} - for i, size := range raw { - t.limits[i] = int(min(size, uint64(int(^uint(0)>>1)))) - if Kind(i) == Returns { - t.limits[i] = min(t.limits[i], maximum) - } - } - return t -} - -func WithTracker(ctx context.Context, tracker *Tracker) context.Context { - return context.WithValue(ctx, trackerKey, tracker) -} -func FromContext(ctx context.Context) *Tracker { - tracker, _ := ctx.Value(trackerKey).(*Tracker) - return tracker -} -func (t *Tracker) Limit(kind Kind, maximum int) int { - if t == nil { - return maximum - } - return min(maximum, t.limits[kind]) -} -func (t *Tracker) Observe(kind Kind, used int) { - if t == nil { - return - } - t.mu.Lock() - t.used[kind] = max(t.used[kind], used) - t.mu.Unlock() -} -func (t *Tracker) Fresh(kind Kind, maximum int) *storage.ReadBudget { - if t == nil { - return storage.NewReadBudget(maximum) - } - limit := t.Limit(kind, maximum) - return storage.NewTrackedReadBudget(limit, func(used int) { t.Observe(kind, used) }) -} -func (t *Tracker) Usage() *forward.Budget { - t.mu.Lock() - defer t.mu.Unlock() - usage := &forward.Budget{Returns: uint64(t.used[Returns])} - return usage -} -func FullBudget(maximum int) *forward.Budget { - size := uint64(maximum) - budget := &forward.Budget{Returns: size} - return budget -} diff --git a/internal/forwarding/protocol.go b/internal/forwarding/protocol.go new file mode 100644 index 0000000..24b1ae0 --- /dev/null +++ b/internal/forwarding/protocol.go @@ -0,0 +1,11 @@ +// Package forwarding defines the private Gateway-to-Engine transport contract. +package forwarding + +const Version = 8 + +// EnvelopeBytes leaves room for the private wrapper around a public message. +const EnvelopeBytes = 4096 + +// NotStartedTrailer is set only when Engine rejects before invoking the service. +// Its absence leaves undelivered mutation outcomes unknown. +const NotStartedTrailer = "sink-forward-not-started" diff --git a/internal/gateway/batch.go b/internal/gateway/batch.go index 9ee0e01..a364706 100644 --- a/internal/gateway/batch.go +++ b/internal/gateway/batch.go @@ -11,7 +11,6 @@ import ( "github.com/liran/sink/internal/protocol" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" - "google.golang.org/protobuf/proto" ) type storeGroup struct { @@ -27,6 +26,9 @@ func (s *Server) deleteRecords(ctx context.Context, req *forward.ForwardRequest) if err != nil { return nil, err } + if len(addresses) > s.request.MaxReadBytes/1280 { + return nil, status.Error(codes.ResourceExhausted, "delete result envelopes exceed gRPC send limit") + } ctx, release, err := s.begin(ctx, req) if err != nil { return nil, err @@ -83,41 +85,24 @@ func (s *Server) deleteRecords(ctx context.Context, req *forward.ForwardRequest) } groups[position].indices = append(groups[position].indices, index) } - remaining, err := s.responseBudget(len(addresses)) - if err != nil { - return nil, err - } - run := func(group storeGroup, grant *forward.Budget) *forward.Budget { + run := func(group storeGroup) { sub := splitRequest(req, group.indices) - sub.Grant = grant - result, callErr := s.forward(ctx, group.route, sub) - // An absent settlement consumes its entire grant, including on a lost reply. - used := grant - notStarted := false - if callErr == nil { - used = result.GetUsed() - notStarted = result.GetNotStarted() - if result.GetCode() != 0 { - callErr = forwardedError(result) - } - } + result, notStarted, callErr := s.forward(ctx, group.route, sub) if callErr == nil { - callErr = mergeRecords(response, result, group.indices) + callErr = mergeDeleteResults(response.GetDelete().GetResults(), result.GetDelete().GetResults(), group.indices) } if callErr != nil { failRecords(response, group.indices, callErr, notStarted) } - return used } if len(groups) > 1 { - // These operations do not consume document budgets. Bound fanout independently - // of the number of Stores; requests never create an unbounded goroutine fanout. + // Bound concurrency independently of the number of Stores. var work sync.WaitGroup jobs := make(chan storeGroup) for range min(s.config.MaxFanout, len(groups)) { work.Go(func() { for group := range jobs { - run(group, remaining) + run(group) } }) } @@ -128,12 +113,10 @@ func (s *Server) deleteRecords(ctx context.Context, req *forward.ForwardRequest) work.Wait() } else { for _, group := range groups { - grant := proto.Clone(remaining).(*forward.Budget) - used := run(group, grant) - consume(remaining, used) + run(group) } } - boundFailures(response, s.request.MaxReadBytes, len(addresses)) + boundDeleteFailures(response.GetDelete().GetResults(), s.request.MaxReadBytes) return response, nil } @@ -259,24 +242,13 @@ func failRecords(response *forward.ForwardResponse, indices []int, err error, no } } } -func mergeRecords(response, sub *forward.ForwardResponse, indices []int) error { - switch body := response.GetResponse().(type) { - case *forward.ForwardResponse_Read: - return mergeResults(body.Read.Results, sub.GetRead().GetResults(), indices) - case *forward.ForwardResponse_Write: - return mergeResults(body.Write.Results, sub.GetWrite().GetResults(), indices) - case *forward.ForwardResponse_Delete: - return mergeResults(body.Delete.Results, sub.GetDelete().GetResults(), indices) - } - return status.Error(codes.Internal, "unexpected response type") -} type recordResult interface { *sink.ReadResult | *sink.WriteResult | *sink.DeleteResult GetOperationIndex() uint32 } -func mergeResults[T recordResult](dest, source []T, indices []int) error { +func mergeDeleteResults(dest, source []*sink.DeleteResult, indices []int) error { if len(source) != len(indices) { return status.Error(codes.Internal, "Engine returned an invalid result count") } @@ -290,32 +262,16 @@ func mergeResults[T recordResult](dest, source []T, indices []int) error { } for _, result := range source { index := indices[result.GetOperationIndex()] - switch typed := any(result).(type) { - case *sink.ReadResult: - typed.OperationIndex = uint32(index) - case *sink.WriteResult: - typed.OperationIndex = uint32(index) - case *sink.DeleteResult: - typed.OperationIndex = uint32(index) - } + result.OperationIndex = uint32(index) dest[index] = result } return nil } -func boundFailures(response *forward.ForwardResponse, maximum, count int) { - limit := min(512, max(1, maximum/max(1, count)-128)) - failures := make([]*sink.Failure, 0, count) - for _, result := range response.GetRead().GetResults() { - failures = append(failures, result.GetFailure()) - } - for _, result := range response.GetWrite().GetResults() { - failures = append(failures, result.GetFailure()) - } - for _, result := range response.GetDelete().GetResults() { - failures = append(failures, result.GetFailure()) - } - for _, failure := range failures { +func boundDeleteFailures(results []*sink.DeleteResult, maximum int) { + limit := min(512, max(1, maximum/max(1, len(results))-128)) + for _, result := range results { + failure := result.GetFailure() if failure == nil || len(failure.Message) <= limit { continue } diff --git a/internal/gateway/gateway_test.go b/internal/gateway/gateway_test.go index ecda440..0218b0e 100644 --- a/internal/gateway/gateway_test.go +++ b/internal/gateway/gateway_test.go @@ -19,16 +19,16 @@ func TestRoutesRejectDuplicateStoreNames(t *testing.T) { } func TestInvalidEngineResultsCannotOverwriteKnownSuccess(t *testing.T) { - good := &sink.WriteResult{OperationIndex: 0, Status: sink.WriteStatus_WRITE_STATUS_APPLIED} - cases := [][]*sink.WriteResult{ + good := &sink.DeleteResult{OperationIndex: 0, Status: sink.DeleteStatus_DELETE_STATUS_APPLIED} + cases := [][]*sink.DeleteResult{ {nil}, - {{OperationIndex: 0, Status: sink.WriteStatus_WRITE_STATUS_UNSPECIFIED}}, - {{OperationIndex: 1, Status: sink.WriteStatus_WRITE_STATUS_APPLIED}}, + {{OperationIndex: 0, Status: sink.DeleteStatus_DELETE_STATUS_UNSPECIFIED}}, + {{OperationIndex: 1, Status: sink.DeleteStatus_DELETE_STATUS_APPLIED}}, {good, good}, } for _, source := range cases { - destination := []*sink.WriteResult{good, nil} - if err := mergeResults(destination, source, []int{1}); err == nil { + destination := []*sink.DeleteResult{good, nil} + if err := mergeDeleteResults(destination, source, []int{1}); err == nil { t.Fatal("invalid response accepted") } if destination[0] != good || destination[1] != nil { diff --git a/internal/gateway/methods.go b/internal/gateway/methods.go index 819fe1c..c35c252 100644 --- a/internal/gateway/methods.go +++ b/internal/gateway/methods.go @@ -2,6 +2,7 @@ package gateway import ( "context" + "google.golang.org/grpc" forward "github.com/liran/sink/gen/forward" @@ -38,11 +39,7 @@ func (s *Server) Execute(ctx context.Context, req *sink.ExecuteRequest) (*sink.E return nil, status.Error(codes.InvalidArgument, "command is required") } body := &forward.ForwardRequest_Execute{Execute: req} - grant, err := s.responseBudget(0) - if err != nil { - return nil, err - } - request := &forward.ForwardRequest{Request: body, Grant: grant} + request := &forward.ForwardRequest{Request: body} ctx, release, err := s.begin(ctx, request) if err != nil { return nil, err @@ -52,13 +49,10 @@ func (s *Server) Execute(ctx context.Context, req *sink.ExecuteRequest) (*sink.E if err != nil { return nil, err } - response, err := s.forward(ctx, route, request) + response, _, err := s.forward(ctx, route, request) if err != nil { return nil, err } - if response.GetCode() != 0 { - return nil, forwardedError(response) - } if response.GetExecute() == nil { return nil, status.Error(codes.Internal, "Engine returned an invalid response type") } @@ -71,11 +65,6 @@ func (s *Server) Query(req *sink.QueryRequest, stream grpc.ServerStreamingServer } body := &forward.ForwardRequest_Query{Query: req} request := &forward.ForwardRequest{Request: body} - grant, err := s.responseBudget(0) - if err != nil { - return err - } - request.Grant = grant ctx, release, err := s.begin(stream.Context(), request) if err != nil { return err @@ -102,13 +91,10 @@ func (s *Server) Query(req *sink.QueryRequest, stream grpc.ServerStreamingServer } return stream.Send(result) } - final, err := s.forwardEach(ctx, call) + _, err = s.forwardEach(ctx, call) if err != nil { return err } - if final.GetCode() != 0 { - return forwardedError(final) - } if !complete { return status.Error(codes.Internal, "Engine omitted Query completion") } @@ -120,11 +106,7 @@ func (s *Server) Count(ctx context.Context, req *sink.CountRequest) (*sink.Count return nil, status.Error(codes.InvalidArgument, "command is required") } body := &forward.ForwardRequest_Count{Count: req} - grant, err := s.responseBudget(0) - if err != nil { - return nil, err - } - request := &forward.ForwardRequest{Request: body, Grant: grant} + request := &forward.ForwardRequest{Request: body} ctx, release, err := s.begin(ctx, request) if err != nil { return nil, err @@ -134,13 +116,10 @@ func (s *Server) Count(ctx context.Context, req *sink.CountRequest) (*sink.Count if err != nil { return nil, err } - response, err := s.forward(ctx, route, request) + response, _, err := s.forward(ctx, route, request) if err != nil { return nil, err } - if response.GetCode() != 0 { - return nil, forwardedError(response) - } if response.GetCount() == nil { return nil, status.Error(codes.Internal, "Engine returned an invalid response type") } @@ -153,11 +132,6 @@ func (s *Server) Scan(req *sink.ScanRequest, stream grpc.ServerStreamingServer[s } body := &forward.ForwardRequest_Scan{Scan: req} request := &forward.ForwardRequest{Request: body} - grant, err := s.responseBudget(0) - if err != nil { - return err - } - request.Grant = grant ctx, release, err := s.begin(stream.Context(), request) if err != nil { return err @@ -184,13 +158,10 @@ func (s *Server) Scan(req *sink.ScanRequest, stream grpc.ServerStreamingServer[s } return stream.Send(result) } - final, err := s.forwardEach(ctx, call) + _, err = s.forwardEach(ctx, call) if err != nil { return err } - if final.GetCode() != 0 { - return forwardedError(final) - } if !complete { return status.Error(codes.Internal, "Engine omitted Scan completion") } diff --git a/internal/gateway/record_stream.go b/internal/gateway/record_stream.go index cb5b08e..0107dfb 100644 --- a/internal/gateway/record_stream.go +++ b/internal/gateway/record_stream.go @@ -73,10 +73,6 @@ func (s *Server) streamRecords(ctx context.Context, req *forward.ForwardRequest, } groups[position].indices = append(groups[position].indices, index) } - remaining, err := s.responseBudget(1) - if err != nil { - return err - } // Routing failures contain metadata only and can be sent immediately. for index := range addresses { @@ -105,7 +101,6 @@ func (s *Server) streamRecords(ctx context.Context, req *forward.ForwardRequest, for _, group := range groups { work.Go(func() error { sub := splitRequest(req, group.indices) - sub.Grant = remaining seen := make([]bool, len(group.indices)) delivered := 0 var sendErr error @@ -145,20 +140,13 @@ func (s *Server) streamRecords(ctx context.Context, req *forward.ForwardRequest, delivered++ return nil } - final, err := s.forwardEach(execution, call) + notStarted, err := s.forwardEach(execution, call) if sendErr != nil { return sendErr } if execution.Err() != nil { return status.FromContextError(execution.Err()).Err() } - notStarted := false - if err == nil { - notStarted = final.GetNotStarted() - if final.GetCode() != 0 { - err = forwardedError(final) - } - } if err == nil && delivered != len(group.indices) { err = status.Error(codes.Internal, "Engine omitted record results") } diff --git a/internal/gateway/response_budget_test.go b/internal/gateway/response_budget_test.go deleted file mode 100644 index 55765b3..0000000 --- a/internal/gateway/response_budget_test.go +++ /dev/null @@ -1,34 +0,0 @@ -package gateway - -import ( - "testing" - - forward "github.com/liran/sink/gen/forward" - "github.com/liran/sink/internal/config" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/status" -) - -func TestResponseBudgetComesFromTransportAndSharesOnlyReturns(t *testing.T) { - request := config.Request{MaxReadBytes: 4096} - server := &Server{request: request} - grant, err := server.responseBudget(2) - if err != nil { - t.Fatal(err) - } - if grant.Returns != 4096-2*1280 { - t.Fatalf("unreserved response envelope: %v", grant) - } - used := &forward.Budget{Returns: 100} - consume(grant, used) - if grant.Returns != 4096-2*1280-100 { - t.Fatalf("settlement charged unrelated Store working sets: %v", grant) - } - if _, err := server.responseBudget(4); status.Code(err) != codes.ResourceExhausted { - t.Fatalf("impossible response must fail before forwarding: %v", err) - } - native, err := server.responseBudget(0) - if err != nil || native.GetReturns() != 4096 { - t.Fatalf("native encoded response: %v %v", native, err) - } -} diff --git a/internal/gateway/server.go b/internal/gateway/server.go index 4ec5e73..2bfd040 100644 --- a/internal/gateway/server.go +++ b/internal/gateway/server.go @@ -12,7 +12,6 @@ import ( "github.com/liran/sink/internal/config" "github.com/liran/sink/internal/forwarding" "github.com/liran/sink/internal/protocol" - statuspb "google.golang.org/genproto/googleapis/rpc/status" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" ) @@ -88,8 +87,8 @@ func (s *Server) begin(ctx context.Context, req *forward.ForwardRequest) (contex return execution, release, nil } -// Scalar RPCs carry one typed response plus the final settlement. -func (s *Server) forward(ctx context.Context, route Route, req *forward.ForwardRequest) (*forward.ForwardResponse, error) { +// Scalar RPCs carry one typed response and finish with the gRPC status. +func (s *Server) forward(ctx context.Context, route Route, req *forward.ForwardRequest) (*forward.ForwardResponse, bool, error) { var response *forward.ForwardResponse call := forwardCall{route: route, request: req} call.emit = func(frame *forward.ForwardResponse) error { @@ -99,24 +98,14 @@ func (s *Server) forward(ctx context.Context, route Route, req *forward.ForwardR response = frame return nil } - final, err := s.forwardEach(ctx, call) + notStarted, err := s.forwardEach(ctx, call) if err != nil { - return nil, err - } - if final.GetCode() != 0 { - return final, nil + return nil, notStarted, err } if response == nil { - return nil, status.Error(codes.Internal, "Engine omitted scalar response") + return nil, false, status.Error(codes.Internal, "Engine omitted scalar response") } - response.Used, response.NotStarted = final.Used, final.NotStarted - return response, nil -} -func validUsage(grant, used *forward.Budget) bool { - return used.GetReturns() <= grant.GetReturns() -} -func consume(remaining, used *forward.Budget) { - remaining.Returns -= used.GetReturns() + return response, false, nil } func routeFor(view *snapshot, store string) (Route, error) { route, exists := view.routes[store] @@ -125,33 +114,3 @@ func routeFor(view *snapshot, store string) (Route, error) { } return route, nil } - -func forwardedError(response *forward.ForwardResponse) error { - encoded := &statuspb.Status{Code: int32(response.GetCode()), Message: response.GetMessage(), Details: response.GetStatusDetails()} - return status.FromProto(encoded).Err() -} - -func localRejection(route Route, code codes.Code, message string) *forward.ForwardResponse { - usage := &forward.Budget{} - response := &forward.ForwardResponse{Version: forwarding.Version, Store: route.Store, Used: usage, Code: uint32(code), Message: message, NotStarted: true} - return response -} - -// responseBudget derives public payload capacity from the transport ceiling. -// Result envelopes, revision tokens and bounded failures are reserved before -// forwarding; Store-local working allocations follow process watermark admission. -func (s *Server) responseBudget(operations int) (*forward.Budget, error) { - overhead := 0 - if operations > 0 { - if operations > s.request.MaxReadBytes/1280 { - return nil, status.Error(codes.ResourceExhausted, "result envelopes exceed gRPC send limit") - } - overhead = operations * 1280 - } - bytes := s.request.MaxReadBytes - overhead - if bytes <= 0 { - return nil, status.Error(codes.ResourceExhausted, "response budget is exhausted") - } - budget := &forward.Budget{Returns: uint64(bytes)} - return budget, nil -} diff --git a/internal/gateway/stream.go b/internal/gateway/stream.go index c8002f1..c930b55 100644 --- a/internal/gateway/stream.go +++ b/internal/gateway/stream.go @@ -19,12 +19,12 @@ type forwardCall struct { // forwardEach reads the next result only after the public send completes. The // transport therefore supplies backpressure without an intermediate queue. -func (s *Server) forwardEach(ctx context.Context, call forwardCall) (*forward.ForwardResponse, error) { +func (s *Server) forwardEach(ctx context.Context, call forwardCall) (bool, error) { route, req := call.route, call.request if route.endpoint == "" { targets, release, err := s.pool.destinations(ctx, route) if err != nil { - return localRejection(route, status.Code(err), status.Convert(err).Message()), nil + return true, err } defer release() route = targets[(s.nativeSequence.Add(1)-1)%uint64(len(targets))] @@ -32,7 +32,7 @@ func (s *Server) forwardEach(ctx context.Context, call forwardCall) (*forward.Fo req.Version, req.Store = forwarding.Version, route.Store entry, err := s.pool.acquire(route) if err != nil { - return localRejection(route, status.Code(err), status.Convert(err).Message()), nil + return true, err } defer s.pool.release(entry) ctx, cancel := context.WithCancel(ctx) @@ -41,37 +41,69 @@ func (s *Server) forwardEach(ctx context.Context, call forwardCall) (*forward.Fo defer func() { s.metrics.downstream.Observe(time.Since(started).Seconds()) }() stream, err := entry.client.Forward(ctx, req) if err != nil { - return nil, err + return false, err } - var final *forward.ForwardResponse received := false for { frame, err := stream.Recv() - if err == io.EOF { - if final == nil { - return nil, status.Error(codes.Internal, "Engine omitted stream settlement") - } - return final, nil - } if err != nil { - return nil, err + marker := stream.Trailer().Get(forwarding.NotStartedTrailer) + notStarted := len(marker) == 1 && marker[0] == "true" + if len(marker) > 0 && (!notStarted || received || err == io.EOF) { + return false, status.Error(codes.Internal, "invalid Engine rejection marker") + } + if err == io.EOF { + return false, nil + } + return notStarted, err } - if final != nil || frame.GetVersion() != forwarding.Version || frame.GetStore() != route.Store { - return nil, status.Error(codes.Internal, "invalid Engine stream identity or frame order") + if frame.GetVersion() != forwarding.Version || frame.GetStore() != route.Store { + return false, status.Error(codes.Internal, "invalid Engine stream identity") } - if frame.GetComplete() { - if frame.GetResponse() != nil || frame.GetUsed() == nil || !validUsage(req.GetGrant(), frame.GetUsed()) || frame.GetCode() > uint32(codes.Unauthenticated) || (received && frame.GetNotStarted()) { - return nil, status.Error(codes.Internal, "invalid Engine stream settlement") - } - final = frame - continue + size := responseSize(req, frame) + if size < 0 { + return false, status.Error(codes.Internal, "invalid Engine result type") } - if frame.GetResponse() == nil || frame.GetUsed() != nil || frame.GetCode() != 0 || frame.GetNotStarted() || frame.GetMessage() != "" || len(frame.GetStatusDetails()) != 0 { - return nil, status.Error(codes.Internal, "invalid Engine result frame") + if size > s.request.MaxReadBytes { + return false, status.Error(codes.ResourceExhausted, "Engine result exceeds public message limit") } received = true if err := call.emit(frame); err != nil { - return nil, err + return false, err + } + } +} + +func responseSize(req *forward.ForwardRequest, frame *forward.ForwardResponse) int { + switch body := frame.GetResponse().(type) { + case *forward.ForwardResponse_Read: + if req.GetRead() != nil && body.Read != nil { + return body.Read.SizeVT() + } + case *forward.ForwardResponse_Write: + if req.GetWrite() != nil && body.Write != nil { + return body.Write.SizeVT() + } + case *forward.ForwardResponse_Delete: + if req.GetDelete() != nil && body.Delete != nil { + return body.Delete.SizeVT() + } + case *forward.ForwardResponse_Execute: + if req.GetExecute() != nil && body.Execute != nil { + return body.Execute.SizeVT() + } + case *forward.ForwardResponse_Query: + if req.GetQuery() != nil && body.Query != nil { + return body.Query.SizeVT() + } + case *forward.ForwardResponse_Count: + if req.GetCount() != nil && body.Count != nil { + return body.Count.SizeVT() + } + case *forward.ForwardResponse_Scan: + if req.GetScan() != nil && body.Scan != nil { + return body.Scan.SizeVT() } } + return -1 } diff --git a/internal/metrics/forwarding.go b/internal/metrics/forwarding.go index 9984710..e441ad7 100644 --- a/internal/metrics/forwarding.go +++ b/internal/metrics/forwarding.go @@ -8,20 +8,17 @@ import ( ) // ObserveForward retains the public method labels when the wire RPC is Forward. -func (m *Metrics) ObserveForward(req *forward.ForwardRequest, resp *forward.ForwardResponse, elapsed time.Duration) { +func (m *Metrics) ObserveForward(req *forward.ForwardRequest, code codes.Code, elapsed time.Duration) { if m == nil { return } - method, request, response := forwardObservation(req, resp) + method, request, _ := forwardObservation(req, nil) if method == "" { return } store := m.RequestStore(request) - m.requests.WithLabelValues(store, method, codes.Code(resp.GetCode()).String()).Inc() + m.requests.WithLabelValues(store, method, code.String()).Inc() m.requestDuration.WithLabelValues(store, method).Observe(elapsed.Seconds()) - if resp.GetCode() == 0 { - m.observeOperationResults(method, request, response) - } } // ObserveForwardResult records delivered outcomes without retaining documents. diff --git a/internal/metrics/forwarding_test.go b/internal/metrics/forwarding_test.go index a0af1b0..a9931c6 100644 --- a/internal/metrics/forwarding_test.go +++ b/internal/metrics/forwarding_test.go @@ -8,6 +8,7 @@ import ( forward "github.com/liran/sink/gen/forward" sink "github.com/liran/sink/gen/sink" + "google.golang.org/grpc/codes" ) func TestForwardedRPCUsesPublicMethodMetrics(t *testing.T) { @@ -24,7 +25,8 @@ func TestForwardedRPCUsesPublicMethodMetrics(t *testing.T) { result := &sink.WriteResponse{Results: []*sink.WriteResult{applied}} responseBody := &forward.ForwardResponse_Write{Write: result} response := &forward.ForwardResponse{Response: responseBody} - observed.ObserveForward(request, response, time.Millisecond) + observed.ObserveForward(request, codes.OK, time.Millisecond) + observed.ObserveForwardResult(request, response) recorder := httptest.NewRecorder() httpRequest := httptest.NewRequest("GET", "/metrics", nil) observed.Handler().ServeHTTP(recorder, httpRequest) diff --git a/internal/service/batching_server.go b/internal/service/batching_server.go index 58afac4..0fc390b 100644 --- a/internal/service/batching_server.go +++ b/internal/service/batching_server.go @@ -261,7 +261,7 @@ func (s *BatchingServer) executeReads(ctx context.Context, calls []*batchCall[*s return } operations := make([]*sink.ReadOperation, 0, totalReadOperations(calls)) - budgets := &requestBudgets{} + budgets := &responseGroups{} for _, call := range calls { operations = append(operations, call.request.GetOperations()...) budgets.addContext(call.ctx, len(call.request.GetOperations())) @@ -340,7 +340,7 @@ func (s *BatchingServer) executeWrite(ctx context.Context, calls []*batchCall[*s return } request := combinedWriteRequest(calls) - budgets := &requestBudgets{} + budgets := &responseGroups{} for _, call := range calls { budgets.addContext(call.ctx, len(call.request.GetOperations())) } @@ -398,7 +398,7 @@ func (s *BatchingServer) executeDeleteBatch(ctx context.Context, calls []*batchC return } request := combinedDeleteRequest(calls) - budgets := &requestBudgets{} + budgets := &responseGroups{} for _, call := range calls { budgets.addContext(call.ctx, len(call.request.GetOperations())) } diff --git a/internal/service/failure_budget.go b/internal/service/failure_budget.go index 5e00932..d3712b6 100644 --- a/internal/service/failure_budget.go +++ b/internal/service/failure_budget.go @@ -32,7 +32,7 @@ func boundedFailureMessage(message string, maximum int) string { return strings.Clone(strings.ToValidUTF8(message, "?")) + suffix } -func boundResultFailures[T interface{ GetFailure() *sink.Failure }](results []T, budgets *requestBudgets, maximum int) { +func boundResultFailures[T interface{ GetFailure() *sink.Failure }](results []T, budgets *responseGroups, maximum int) { counts := make([]int, budgets.callerCount()) for index := range results { counts[budgets.owner(index)]++ diff --git a/internal/service/native.go b/internal/service/native.go index 2480afd..a058a20 100644 --- a/internal/service/native.go +++ b/internal/service/native.go @@ -10,7 +10,6 @@ import ( "github.com/liran/sink-go/uri" sink "github.com/liran/sink/gen/sink" - "github.com/liran/sink/internal/forwarding" "github.com/liran/sink/internal/protocol" "github.com/liran/sink/internal/storage" "google.golang.org/grpc/codes" @@ -92,10 +91,7 @@ func nativeStatus(err error) error { } func (s *Server) Execute(ctx context.Context, req *sink.ExecuteRequest) (*sink.ExecuteResponse, error) { - maximum := forwarding.FromContext(ctx).Limit(forwarding.Returns, s.maxReadBytes) - if maximum <= 0 { - return nil, status.Error(codes.ResourceExhausted, "native response budget is exhausted") - } + maximum := s.maxReadBytes if err := protocol.CheckStore(req, s.boundStore); err != nil { return nil, err } @@ -140,10 +136,7 @@ func (s *Server) Execute(ctx context.Context, req *sink.ExecuteRequest) (*sink.E } func (s *Server) scan(ctx context.Context, req *sink.ScanRequest, send func(*sink.ScanResponse) error) error { - maximum := forwarding.FromContext(ctx).Limit(forwarding.Returns, s.maxReadBytes) - if maximum <= 0 { - return status.Error(codes.ResourceExhausted, "native response budget is exhausted") - } + maximum := s.maxReadBytes if err := protocol.CheckStore(req, s.boundStore); err != nil { return err } diff --git a/internal/service/query.go b/internal/service/query.go index b53b262..f2ef48d 100644 --- a/internal/service/query.go +++ b/internal/service/query.go @@ -4,7 +4,6 @@ import ( "context" sink "github.com/liran/sink/gen/sink" - "github.com/liran/sink/internal/forwarding" "github.com/liran/sink/internal/protocol" "github.com/liran/sink/internal/storage" "google.golang.org/grpc/codes" @@ -14,10 +13,7 @@ import ( const maxCountResponseBytes = 256 << 10 func (s *Server) query(ctx context.Context, req *sink.QueryRequest, send func(*sink.QueryResponse) error) error { - maximum := forwarding.FromContext(ctx).Limit(forwarding.Returns, s.maxReadBytes) - if maximum <= 0 { - return status.Error(codes.ResourceExhausted, "native response budget is exhausted") - } + maximum := s.maxReadBytes if err := protocol.CheckStore(req, s.boundStore); err != nil { return err } @@ -73,10 +69,7 @@ func (s *Server) query(ctx context.Context, req *sink.QueryRequest, send func(*s } func (s *Server) Count(ctx context.Context, req *sink.CountRequest) (*sink.CountResponse, error) { - maximum := forwarding.FromContext(ctx).Limit(forwarding.Returns, s.maxReadBytes) - if maximum <= 0 { - return nil, status.Error(codes.ResourceExhausted, "native response budget is exhausted") - } + maximum := s.maxReadBytes if err := protocol.CheckStore(req, s.boundStore); err != nil { return nil, err } diff --git a/internal/service/read.go b/internal/service/read.go index 3b3c18b..e11ce7d 100644 --- a/internal/service/read.go +++ b/internal/service/read.go @@ -4,7 +4,6 @@ import ( "context" sink "github.com/liran/sink/gen/sink" - "github.com/liran/sink/internal/forwarding" "github.com/liran/sink/internal/protocol" "github.com/liran/sink/internal/storage" "google.golang.org/grpc/codes" @@ -15,10 +14,10 @@ func (s *Server) Read(ctx context.Context, req *sink.ReadRequest) (*sink.ReadRes if err := protocol.CheckStore(req, s.boundStore); err != nil { return nil, err } - return s.read(ctx, req, contextBudgets(ctx, len(req.GetOperations()))) + return s.read(ctx, req, responseGroupsFor(ctx, len(req.GetOperations()))) } -func (s *Server) read(ctx context.Context, req *sink.ReadRequest, budgets *requestBudgets) (*sink.ReadResponse, error) { +func (s *Server) read(ctx context.Context, req *sink.ReadRequest, budgets *responseGroups) (*sink.ReadResponse, error) { if req == nil || len(req.GetOperations()) == 0 { return nil, status.Error(codes.InvalidArgument, "read request must contain operations") } @@ -75,7 +74,7 @@ func (s *Server) read(ctx context.Context, req *sink.ReadRequest, budgets *reque defer clear(storageResponse.Results) // Charge copies in each original RPC's order, including repeated keys. // Check the aggregate output size before allocating any document copies. - outputBudgets := budgets.fresh(forwarding.Returns, s.maxReadBytes) + outputBudgets := budgets.fresh(s.maxReadBytes) for index, operationIndex := range operationIndexes { owner := budgets.owner(operationIndex) stored := storageResponse.Results[storageIndexes[index]] diff --git a/internal/service/request_budgets.go b/internal/service/request_budgets.go deleted file mode 100644 index cc1df8c..0000000 --- a/internal/service/request_budgets.go +++ /dev/null @@ -1,71 +0,0 @@ -package service - -import ( - "context" - - "github.com/liran/sink/internal/forwarding" - "github.com/liran/sink/internal/storage" -) - -// requestBudgets retains original RPC ownership after coalescing operations. -// Nil denotes one ordinary RPC with the core's existing byte limits. -type requestBudgets struct { - owners []int - count int - trackers []*forwarding.Tracker -} - -func (b *requestBudgets) callerCount() int { - if b == nil { - return 1 - } - return b.count -} - -func (b *requestBudgets) owner(index int) int { - if b == nil { - return 0 - } - return b.owners[index] -} - -func (b *requestBudgets) fresh(kind forwarding.Kind, maxBytes int) []*storage.ReadBudget { - budgets := make([]*storage.ReadBudget, b.callerCount()) - for index := range budgets { - budgets[index] = b.tracker(index).Fresh(kind, maxBytes) - } - return budgets -} - -func (b *requestBudgets) add(count int) { - for range count { - b.owners = append(b.owners, b.count) - } - b.count++ -} - -func (b *requestBudgets) tracker(owner int) *forwarding.Tracker { - if b == nil || owner >= len(b.trackers) { - return nil - } - return b.trackers[owner] -} -func (b *requestBudgets) addContext(ctx context.Context, count int) { - if streaming, _ := ctx.Value(streamingKey{}).(bool); streaming { - for range count { - b.add(1) - b.trackers = append(b.trackers, forwarding.FromContext(ctx)) - } - return - } - b.add(count) - b.trackers = append(b.trackers, forwarding.FromContext(ctx)) -} -func contextBudgets(ctx context.Context, count int) *requestBudgets { - if streaming, _ := ctx.Value(streamingKey{}).(bool); !streaming && forwarding.FromContext(ctx) == nil { - return nil - } - budgets := &requestBudgets{} - budgets.addContext(ctx, count) - return budgets -} diff --git a/internal/service/response_groups.go b/internal/service/response_groups.go new file mode 100644 index 0000000..60fff94 --- /dev/null +++ b/internal/service/response_groups.go @@ -0,0 +1,61 @@ +package service + +import ( + "context" + + "github.com/liran/sink/internal/storage" +) + +// responseGroups maps coalesced operations to their local response boundaries. +// Streams have one group per result; internal batch calls retain caller ownership. +type responseGroups struct { + owners []int + count int +} + +func (b *responseGroups) callerCount() int { + if b == nil { + return 1 + } + return b.count +} + +func (b *responseGroups) owner(index int) int { + if b == nil { + return 0 + } + return b.owners[index] +} + +func (b *responseGroups) fresh(maxBytes int) []*storage.ReadBudget { + budgets := make([]*storage.ReadBudget, b.callerCount()) + for index := range budgets { + budgets[index] = storage.NewReadBudget(maxBytes) + } + return budgets +} + +func (b *responseGroups) add(count int) { + for range count { + b.owners = append(b.owners, b.count) + } + b.count++ +} + +func (b *responseGroups) addContext(ctx context.Context, count int) { + if streaming, _ := ctx.Value(streamingKey{}).(bool); streaming { + for range count { + b.add(1) + } + return + } + b.add(count) +} +func responseGroupsFor(ctx context.Context, count int) *responseGroups { + if streaming, _ := ctx.Value(streamingKey{}).(bool); !streaming { + return nil + } + budgets := &responseGroups{} + budgets.addContext(ctx, count) + return budgets +} diff --git a/internal/service/server.go b/internal/service/server.go index 0b703cc..4a38397 100644 --- a/internal/service/server.go +++ b/internal/service/server.go @@ -35,10 +35,10 @@ func (s *Server) Write(ctx context.Context, req *sink.WriteRequest) (*sink.Write if err := protocol.CheckStore(req, s.boundStore); err != nil { return nil, err } - return s.write(ctx, req, contextBudgets(ctx, len(req.GetOperations())), nil) + return s.write(ctx, req, responseGroupsFor(ctx, len(req.GetOperations())), nil) } -func (s *Server) write(ctx context.Context, req *sink.WriteRequest, budgets *requestBudgets, completion *writeCompletion) (*sink.WriteResponse, error) { +func (s *Server) write(ctx context.Context, req *sink.WriteRequest, budgets *responseGroups, completion *writeCompletion) (*sink.WriteResponse, error) { if req == nil || len(req.GetOperations()) == 0 { return nil, status.Error(codes.InvalidArgument, "write request must contain operations") } @@ -136,7 +136,7 @@ func (s *Server) Delete(ctx context.Context, req *sink.DeleteRequest) (*sink.Del return s.delete(ctx, req, nil) } -func (s *Server) delete(ctx context.Context, req *sink.DeleteRequest, budgets *requestBudgets) (*sink.DeleteResponse, error) { +func (s *Server) delete(ctx context.Context, req *sink.DeleteRequest, budgets *responseGroups) (*sink.DeleteResponse, error) { if req == nil || len(req.GetOperations()) == 0 { return nil, status.Error(codes.InvalidArgument, "delete request must contain operations") } diff --git a/internal/service/write.go b/internal/service/write.go index 98f34d6..5e89fd3 100644 --- a/internal/service/write.go +++ b/internal/service/write.go @@ -54,7 +54,7 @@ type writeExecutionOptions struct { returns *writeReturns completion *writeCompletion observation *writeObservation - budgets *requestBudgets + budgets *responseGroups WaitUntilVisible bool } diff --git a/internal/service/write_return.go b/internal/service/write_return.go index b3849bd..9a4296a 100644 --- a/internal/service/write_return.go +++ b/internal/service/write_return.go @@ -5,7 +5,6 @@ import ( "errors" sink "github.com/liran/sink/gen/sink" - "github.com/liran/sink/internal/forwarding" "github.com/liran/sink/internal/storage" ) @@ -13,20 +12,19 @@ import ( // bounds retained responses before committing, including per-RPC batching. type writeReturns struct { remaining []int - maximum []int sizes map[int]int - owners *requestBudgets + owners *responseGroups } -func newWriteReturns(req *sink.WriteRequest, budgets *requestBudgets, maximum int) *writeReturns { +func newWriteReturns(req *sink.WriteRequest, budgets *responseGroups, maximum int) *writeReturns { if !hasWriteReturns(req) { return nil } remaining := make([]int, budgets.callerCount()) for index := range remaining { - remaining[index] = budgets.tracker(index).Limit(forwarding.Returns, maximum) + remaining[index] = maximum } - returns := &writeReturns{remaining: remaining, maximum: append([]int(nil), remaining...), sizes: make(map[int]int), owners: budgets} + returns := &writeReturns{remaining: remaining, sizes: make(map[int]int), owners: budgets} return returns } @@ -63,7 +61,6 @@ func (r *writeReturns) reserve(group writeGroup, document storage.Document) erro return storage.ResourceExhaustedError(errors.New("returned write documents exceed response byte budget")) } r.remaining[owner] -= difference - r.owners.tracker(owner).Observe(forwarding.Returns, r.maximum[owner]-r.remaining[owner]) r.sizes[operation.index] = max(charge, r.sizes[operation.index]) } return nil diff --git a/internal/storage/read_budget.go b/internal/storage/read_budget.go index a6122a5..446d27b 100644 --- a/internal/storage/read_budget.go +++ b/internal/storage/read_budget.go @@ -7,13 +7,12 @@ import ( const DefaultMaxReadBytes = 32 << 20 -// ReadBudget is shared across every store and repeated key in one read. Reserve -// before copying a result so small key batches cannot allocate unbounded output. +// ReadBudget bounds document copies within a local result, batch or scan page. +// It does not cross the Gateway-to-Engine transport boundary. type ReadBudget struct { mu sync.Mutex remaining int maximum int - observe func(int) unlimited bool } @@ -25,13 +24,6 @@ func NewReadBudget(maxBytes int) *ReadBudget { return budget } -// NewTrackedReadBudget accepts an explicit zero grant and reports successful charges. -// observe must not call back into this budget. -func NewTrackedReadBudget(maxBytes int, observe func(int)) *ReadBudget { - budget := &ReadBudget{remaining: max(0, maxBytes), maximum: max(0, maxBytes), observe: observe} - return budget -} - // NewUnboundedReadBudget leaves intermediate documents to process admission. // Public response allowances continue to use NewReadBudget. func NewUnboundedReadBudget() *ReadBudget { @@ -55,8 +47,5 @@ func (b *ReadBudget) Reserve(size int) error { return NewOperationError(ErrorCodeResourceExhausted, size >= 0 && size <= b.maximum-overhead, cause) } b.remaining -= size + overhead - if b.observe != nil { - b.observe(b.maximum - b.remaining) - } return nil } diff --git a/internal/storage/search/read_budget_test.go b/internal/storage/search/read_budget_test.go index 3614c7f..2358b18 100644 --- a/internal/storage/search/read_budget_test.go +++ b/internal/storage/search/read_budget_test.go @@ -32,18 +32,17 @@ func TestReadFailuresPreserveDocumentBudget(t *testing.T) { expected := expectedRequest{method: http.MethodPost, path: "/_mget", statusCode: http.StatusOK, responseBody: payload} store, handler := newScriptedStore(t, []expectedRequest{expected}) t.Cleanup(store.Close) - var charged int maximum := charge if scope == "exhausted" { - maximum = 0 + maximum = 1 } - budget := storage.NewTrackedReadBudget(maximum, func(used int) { charged = used }) + budget := storage.NewReadBudget(maximum) request := storage.ReadRequest{ Operations: []storage.ReadOperation{{Address: testAddress("bad")}, {Address: testAddress("good")}}, Budget: budget, } if scope == "operation" { - request.Budget = storage.NewTrackedReadBudget(0, nil) + request.Budget = storage.NewReadBudget(1) for index := range request.Operations { request.Operations[index].Budget = budget } @@ -62,11 +61,11 @@ func TestReadFailuresPreserveDocumentBudget(t *testing.T) { good := response.Results[1] if scope == "exhausted" { code, _ := storage.ErrorDetails(good.Err) - if good.Status != storage.ReadStatusFailed || code != storage.ErrorCodeResourceExhausted || charged != 0 || len(good.Document.Payload) != 0 || len(good.Revision.Data) != 0 { - t.Fatalf("exhausted budget retained a document: %+v, charged %d", good, charged) + if good.Status != storage.ReadStatusFailed || code != storage.ErrorCodeResourceExhausted || len(good.Document.Payload) != 0 || len(good.Revision.Data) != 0 { + t.Fatalf("exhausted budget retained a document: %+v", good) } - } else if good.Status != storage.ReadStatusFound || string(good.Document.Payload) != source || charged != charge { - t.Fatalf("failed document consumed its sibling's budget: %+v, charged %d", good, charged) + } else if good.Status != storage.ReadStatusFound || string(good.Document.Payload) != source { + t.Fatalf("failed document consumed its sibling's budget: %+v", good) } handler.verify() }) diff --git a/internal/storage/search/stream_test.go b/internal/storage/search/stream_test.go index 955d5e8..cfa50ba 100644 --- a/internal/storage/search/stream_test.go +++ b/internal/storage/search/stream_test.go @@ -105,3 +105,117 @@ func TestSearchStreamRejectsMalformedAndDuplicateMetadata(t *testing.T) { } } } + +func TestSearchStreamDecodesMetadataAndDiscardsUnknownFields(t *testing.T) { + body := `{"unknown":{"nested":[null,true,1e300,{"value":[1,2]}]},"_scroll_id":"cursor","terminated_early":false,"timed_out":false,"_clusters":{"total":1,"successful":1,"skipped":0},"_shards":{"total":1,"successful":1,"failed":0},"hits":{"max_score":null,"extra":[{"ignored":true}],"total":{"value":2,"relation":"eq"},"hits":[{"sort":[1]},{"sort":[2]}]}}` + var hits []string + page, err := decodeSearchPage(strings.NewReader(body), func(hit json.RawMessage) error { + hits = append(hits, string(hit)) + return nil + }) + if err != nil || len(hits) != 2 || hits[0] != `{"sort":[1]}` || hits[1] != `{"sort":[2]}` { + t.Fatalf("streamed hits=%v err=%v", hits, err) + } + if page.ScrollID != "cursor" || page.Clusters == nil || page.Shards == nil || page.TimedOut == nil || *page.TimedOut || page.TerminatedEarly { + t.Fatalf("stream lost metadata: %+v", page) + } + if len(page.Hits.Hits) != 2 || page.Hits.Hits[0] != nil || page.Hits.Hits[1] != nil { + t.Fatal("decoder retained delivered hit payloads") + } +} + +func TestSearchStreamRejectsTruncatedHitsAndUnknownFields(t *testing.T) { + for _, body := range []string{ + `{"unknown":`, `{"unknown":{"key":`, `{"unknown":{"key":[`, + `{"unknown":[{"key":`, `{"unknown":[1,}`, `{"unknown":{"key":1]}`, + `{"hits":{"hits":[{"sort":`, `{"hits":{"hits":[null,`, + `{"_scroll_id":[]}`, `{"timed_out":[]}`, `{"terminated_early":[]}`, + `{"_shards":[]}`, `{"_clusters":[]}`, `{"hits":{"total":`, + } { + t.Run(body, func(t *testing.T) { + _, err := decodeSearchPage(strings.NewReader(body), func(json.RawMessage) error { return nil }) + if err == nil { + t.Fatal("truncated or invalid metadata was accepted") + } + }) + } +} + +func TestQueryCallbackErrorStopsDeliveryWithoutRetry(t *testing.T) { + body := `{"hits":{"hits":[{"_source":{"value":1}},{"_source":{"value":2}}],"total":{"value":2,"relation":"eq"}},"timed_out":false,"_shards":{"total":1,"successful":1,"failed":0}}` + expected := expectedRequest{method: http.MethodPost, path: "/items/_search", rawQuery: "rest_total_hits_as_int=false", statusCode: http.StatusOK, responseBody: body} + store, handler := newScriptedStore(t, []expectedRequest{expected}) + t.Cleanup(store.Close) + native := storage.NativeRequest{URI: "sink://primary/items", MaxBytes: 4096} + request := storage.QueryRequest{Request: native, PageSize: 2} + canceled := errors.New("consumer stopped") + emitted := 0 + request.Emit = func(storage.Document) error { emitted++; return canceled } + page, err := store.Query(t.Context(), request) + if !errors.Is(err, canceled) || emitted != 1 || len(page.Documents) != 0 || page.HasMore { + t.Fatalf("callback error lost: page=%+v emitted=%d err=%v", page, emitted, err) + } + handler.verify() +} + +func TestScanStreamResumesByteLimitedPageWithoutCollecting(t *testing.T) { + prefix := `{"timed_out":false,"_shards":{"total":1,"successful":1,"failed":0},"hits":{"hits":` + first := expectedRequest{method: http.MethodPost, path: "/items/_search", statusCode: http.StatusOK, responseBody: prefix + `[{"sort":[1]},{"sort":[2]},{"sort":[3]}]}}`} + second := expectedRequest{method: http.MethodPost, path: "/items/_search", statusCode: http.StatusOK, bodyContains: []string{`"search_after":[2]`}, responseBody: prefix + `[{"sort":[3]}]}}`} + store, handler := newScriptedStore(t, []expectedRequest{first, second}) + t.Cleanup(store.Close) + native := storage.NativeRequest{URI: "sink://primary/items", ContentType: ContentTypeJSON, Payload: []byte(`{"sort":["uid"]}`), MaxBytes: 300} + request := storage.ScanRequest{Request: native, BatchSize: 3} + var delivered []string + request.Emit = func(document storage.Document) error { + delivered = append(delivered, string(document.Payload)) + return nil + } + page, err := store.Scan(t.Context(), request) + if err != nil || len(page.Documents) != 0 || len(page.NextCursor) == 0 || len(delivered) != 2 { + t.Fatalf("first page=%+v delivered=%v err=%v", page, delivered, err) + } + request.Cursor = page.NextCursor + page, err = store.Scan(t.Context(), request) + if err != nil || len(page.Documents) != 0 || len(page.NextCursor) != 0 || len(delivered) != 3 || delivered[2] != `{"sort":[3]}` { + t.Fatalf("resumed page=%+v delivered=%v err=%v", page, delivered, err) + } + handler.verify() +} + +func TestScanStreamFailuresWithholdCursor(t *testing.T) { + for _, scenario := range []string{"callback", "extra-hit", "late-metadata", "duplicate-sort"} { + t.Run(scenario, func(t *testing.T) { + hits := `[{"sort":[1]},{"sort":[2]}]` + if scenario == "extra-hit" { + hits = `[{"sort":[1]},{"sort":[2]},{"sort":[3]}]` + } + if scenario == "duplicate-sort" { + hits = `[{"sort":[1]},{"sort":[1]}]` + } + body := `{"hits":{"hits":` + hits + `},"_shards":{"total":1,"successful":1,"failed":0},"timed_out":` + fmt.Sprint(scenario == "late-metadata") + `}` + expected := expectedRequest{method: http.MethodPost, path: "/items/_search", statusCode: http.StatusOK, responseBody: body} + store, handler := newScriptedStore(t, []expectedRequest{expected}) + t.Cleanup(store.Close) + native := storage.NativeRequest{URI: "sink://primary/items", ContentType: ContentTypeJSON, Payload: []byte(`{"sort":["uid"]}`), MaxBytes: 4096} + request := storage.ScanRequest{Request: native, BatchSize: 1} + stopped := errors.New("consumer stopped") + emitted := 0 + request.Emit = func(storage.Document) error { + emitted++ + if scenario == "callback" { + return stopped + } + return nil + } + page, err := store.Scan(t.Context(), request) + if err == nil || len(page.NextCursor) != 0 || len(page.Documents) != 0 || emitted != 1 { + t.Fatalf("failed stream completed: page=%+v emitted=%d err=%v", page, emitted, err) + } + if scenario == "callback" && !errors.Is(err, stopped) { + t.Fatalf("callback error was replaced: %v", err) + } + handler.verify() + }) + } +} diff --git a/proto/forward/forward.proto b/proto/forward/forward.proto index c92abb3..2554f19 100644 --- a/proto/forward/forward.proto +++ b/proto/forward/forward.proto @@ -3,26 +3,16 @@ package sink.forward.v1; option go_package = "github.com/liran/sink/gen/forward;forward"; import "sink/sink.proto"; -import "google/protobuf/any.proto"; // Private Gateway-to-Engine protocol. Never used by public clients. service Engine { - // Typed result frames followed by exactly one complete settlement frame. + // Typed results; EOF and errors use the standard gRPC stream status. rpc Forward(ForwardRequest) returns (stream ForwardResponse); } -// Public response allowance, including per-record envelope charges. -// Intermediate allocations are owned by each process memory pool. -message Budget { - reserved 1, 2, 3; - reserved "snapshots", "inputs", "outputs"; - uint64 returns = 4; -} - message ForwardRequest { uint32 version = 1; string store = 2; - Budget grant = 4; oneof request { sink.v1.ReadRequest read = 10; sink.v1.WriteRequest write = 11; @@ -35,17 +25,8 @@ message ForwardRequest { } message ForwardResponse { - // The final frame carries usage and application status, with no response body. - bool complete = 9; uint32 version = 1; string store = 2; - Budget used = 4; - // gRPC application status is carried inside the envelope so usage survives errors. - uint32 code = 5; - string message = 6; - // True only when rejected before invoking the execution service. - bool not_started = 7; - repeated google.protobuf.Any status_details = 8; oneof response { sink.v1.ReadResponse read = 10; sink.v1.WriteResponse write = 11; From dc409271603233ec2424a34300d66471ca855557 Mon Sep 17 00:00:00 2001 From: Randy <857271370@qq.com> Date: Sun, 20 Sep 2026 16:53:50 +0800 Subject: [PATCH 3/6] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20refactor:=20order=20pr?= =?UTF-8?q?otobuf=20fields=20by=20tag=20number?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- gen/sink/sink.pb.go | 78 +++++++++++++++++++++---------------------- proto/sink/sink.proto | 14 ++++---- 2 files changed, 46 insertions(+), 46 deletions(-) diff --git a/gen/sink/sink.pb.go b/gen/sink/sink.pb.go index 36a9919..f279823 100644 --- a/gen/sink/sink.pb.go +++ b/gen/sink/sink.pb.go @@ -832,17 +832,17 @@ func (x *WriteRequest) GetLuaPrograms() []*LuaProgram { type WriteOperation struct { state protoimpl.MessageState `protogen:"open.v1"` Address *RecordAddress `protobuf:"bytes,1,opt,name=address,proto3" json:"address,omitempty"` - // Return the logical document submitted by this operation after its commit. - // Only synchronous completion modes support this. These operations do not - // share a folded commit with preceding or following operations. - ReturnDocument bool `protobuf:"varint,4,opt,name=return_document,json=returnDocument,proto3" json:"return_document,omitempty"` // Types that are valid to be assigned to Action: // // *WriteOperation_Put // *WriteOperation_Merge - Action isWriteOperation_Action `protobuf_oneof:"action"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + Action isWriteOperation_Action `protobuf_oneof:"action"` + // Return the logical document submitted by this operation after its commit. + // Only synchronous completion modes support this. These operations do not + // share a folded commit with preceding or following operations. + ReturnDocument bool `protobuf:"varint,4,opt,name=return_document,json=returnDocument,proto3" json:"return_document,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *WriteOperation) Reset() { @@ -882,13 +882,6 @@ func (x *WriteOperation) GetAddress() *RecordAddress { return nil } -func (x *WriteOperation) GetReturnDocument() bool { - if x != nil { - return x.ReturnDocument - } - return false -} - func (x *WriteOperation) GetAction() isWriteOperation_Action { if x != nil { return x.Action @@ -914,6 +907,13 @@ func (x *WriteOperation) GetMerge() *MergeOperation { return nil } +func (x *WriteOperation) GetReturnDocument() bool { + if x != nil { + return x.ReturnDocument + } + return false +} + type isWriteOperation_Action interface { isWriteOperation_Action() } @@ -1977,12 +1977,12 @@ func (x *Projection) GetExclude() bool { // complete and has_more. Metadata is valid only after successful EOF. type QueryResponse struct { state protoimpl.MessageState `protogen:"open.v1"` - Complete bool `protobuf:"varint,3,opt,name=complete,proto3" json:"complete,omitempty"` Documents []*Document `protobuf:"bytes,1,rep,name=documents,proto3" json:"documents,omitempty"` // Determined by fetching one extra result, not by running Count. Result-window // limits apply to this extra result too. A byte limit fails the whole request; // it never produces a silently shortened page. HasMore bool `protobuf:"varint,2,opt,name=has_more,json=hasMore,proto3" json:"has_more,omitempty"` + Complete bool `protobuf:"varint,3,opt,name=complete,proto3" json:"complete,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -2017,13 +2017,6 @@ func (*QueryResponse) Descriptor() ([]byte, []int) { return file_sink_sink_proto_rawDescGZIP(), []int{26} } -func (x *QueryResponse) GetComplete() bool { - if x != nil { - return x.Complete - } - return false -} - func (x *QueryResponse) GetDocuments() []*Document { if x != nil { return x.Documents @@ -2038,6 +2031,13 @@ func (x *QueryResponse) GetHasMore() bool { return false } +func (x *QueryResponse) GetComplete() bool { + if x != nil { + return x.Complete + } + return false +} + // Count is an independent query, not an implicit part of Query. It accepts the // same sources and counts matches before find/HTTP pagination; for aggregate it // counts the output of the supplied pipeline. HTTP counts matching documents @@ -2230,11 +2230,11 @@ func (x *ScanRequest) GetProjection() *Projection { // complete and next_cursor. Checkpoint only after successful EOF. type ScanResponse struct { state protoimpl.MessageState `protogen:"open.v1"` - Complete bool `protobuf:"varint,3,opt,name=complete,proto3" json:"complete,omitempty"` Documents []*Document `protobuf:"bytes,1,rep,name=documents,proto3" json:"documents,omitempty"` // Empty means the scan reached the end observed by this request. Commit a // checkpoint only after processing documents and successful EOF. No close RPC is needed. NextCursor []byte `protobuf:"bytes,2,opt,name=next_cursor,json=nextCursor,proto3" json:"next_cursor,omitempty"` + Complete bool `protobuf:"varint,3,opt,name=complete,proto3" json:"complete,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -2269,13 +2269,6 @@ func (*ScanResponse) Descriptor() ([]byte, []int) { return file_sink_sink_proto_rawDescGZIP(), []int{30} } -func (x *ScanResponse) GetComplete() bool { - if x != nil { - return x.Complete - } - return false -} - func (x *ScanResponse) GetDocuments() []*Document { if x != nil { return x.Documents @@ -2290,6 +2283,13 @@ func (x *ScanResponse) GetNextCursor() []byte { return nil } +func (x *ScanResponse) GetComplete() bool { + if x != nil { + return x.Complete + } + return false +} + var File_sink_sink_proto protoreflect.FileDescriptor const file_sink_sink_proto_rawDesc = "" + @@ -2324,10 +2324,10 @@ const file_sink_sink_proto_rawDesc = "" + "operations\x126\n" + "\flua_programs\x18\x03 \x03(\v2\x13.sink.v1.LuaProgramR\vluaPrograms\"\xd1\x01\n" + "\x0eWriteOperation\x120\n" + - "\aaddress\x18\x01 \x01(\v2\x16.sink.v1.RecordAddressR\aaddress\x12'\n" + - "\x0freturn_document\x18\x04 \x01(\bR\x0ereturnDocument\x12)\n" + + "\aaddress\x18\x01 \x01(\v2\x16.sink.v1.RecordAddressR\aaddress\x12)\n" + "\x03put\x18\x02 \x01(\v2\x15.sink.v1.PutOperationH\x00R\x03put\x12/\n" + - "\x05merge\x18\x03 \x01(\v2\x17.sink.v1.MergeOperationH\x00R\x05mergeB\b\n" + + "\x05merge\x18\x03 \x01(\v2\x17.sink.v1.MergeOperationH\x00R\x05merge\x12'\n" + + "\x0freturn_document\x18\x04 \x01(\bR\x0ereturnDocumentB\b\n" + "\x06action\"e\n" + "\fPutOperation\x12-\n" + "\bdocument\x18\x01 \x01(\v2\x11.sink.v1.DocumentR\bdocument\x12&\n" + @@ -2402,10 +2402,10 @@ const file_sink_sink_proto_rawDesc = "" + "Projection\x12\x16\n" + "\x06fields\x18\x01 \x03(\tR\x06fields\x12\x18\n" + "\aexclude\x18\x02 \x01(\bR\aexclude\"w\n" + - "\rQueryResponse\x12\x1a\n" + - "\bcomplete\x18\x03 \x01(\bR\bcomplete\x12/\n" + + "\rQueryResponse\x12/\n" + "\tdocuments\x18\x01 \x03(\v2\x11.sink.v1.DocumentR\tdocuments\x12\x19\n" + - "\bhas_more\x18\x02 \x01(\bR\ahasMore\":\n" + + "\bhas_more\x18\x02 \x01(\bR\ahasMore\x12\x1a\n" + + "\bcomplete\x18\x03 \x01(\bR\bcomplete\":\n" + "\fCountRequest\x12*\n" + "\acommand\x18\x01 \x01(\v2\x10.sink.v1.CommandR\acommand\"C\n" + "\rCountResponse\x12\x14\n" + @@ -2419,11 +2419,11 @@ const file_sink_sink_proto_rawDesc = "" + "\n" + "projection\x18\x04 \x01(\v2\x13.sink.v1.ProjectionR\n" + "projection\"|\n" + - "\fScanResponse\x12\x1a\n" + - "\bcomplete\x18\x03 \x01(\bR\bcomplete\x12/\n" + + "\fScanResponse\x12/\n" + "\tdocuments\x18\x01 \x03(\v2\x11.sink.v1.DocumentR\tdocuments\x12\x1f\n" + "\vnext_cursor\x18\x02 \x01(\fR\n" + - "nextCursor*m\n" + + "nextCursor\x12\x1a\n" + + "\bcomplete\x18\x03 \x01(\bR\bcomplete*m\n" + "\x10DocumentEncoding\x12!\n" + "\x1dDOCUMENT_ENCODING_UNSPECIFIED\x10\x00\x12\x1a\n" + "\x16DOCUMENT_ENCODING_JSON\x10\x01\x12\x1a\n" + diff --git a/proto/sink/sink.proto b/proto/sink/sink.proto index c1dd5c5..c3bd290 100644 --- a/proto/sink/sink.proto +++ b/proto/sink/sink.proto @@ -103,15 +103,15 @@ message WriteRequest { message WriteOperation { RecordAddress address = 1; - // Return the logical document submitted by this operation after its commit. - // Only synchronous completion modes support this. These operations do not - // share a folded commit with preceding or following operations. - bool return_document = 4; - oneof action { PutOperation put = 2; MergeOperation merge = 3; } + + // Return the logical document submitted by this operation after its commit. + // Only synchronous completion modes support this. These operations do not + // share a folded commit with preceding or following operations. + bool return_document = 4; } // PutOperation writes a complete user document without interpreting its fields. @@ -331,13 +331,13 @@ message Projection { // Document frames contain one document. A final, document-free frame sets // complete and has_more. Metadata is valid only after successful EOF. message QueryResponse { - bool complete = 3; repeated Document documents = 1; // Determined by fetching one extra result, not by running Count. Result-window // limits apply to this extra result too. A byte limit fails the whole request; // it never produces a silently shortened page. bool has_more = 2; + bool complete = 3; } // Count is an independent query, not an implicit part of Query. It accepts the @@ -386,10 +386,10 @@ message ScanRequest { // Document frames contain one document. A final, document-free frame sets // complete and next_cursor. Checkpoint only after successful EOF. message ScanResponse { - bool complete = 3; repeated Document documents = 1; // Empty means the scan reached the end observed by this request. Commit a // checkpoint only after processing documents and successful EOF. No close RPC is needed. bytes next_cursor = 2; + bool complete = 3; } From f0ef1016a65e35550d1f76e53e15a90b5c853d32 Mon Sep 17 00:00:00 2001 From: Randy <857271370@qq.com> Date: Sun, 20 Sep 2026 17:16:43 +0800 Subject: [PATCH 4/6] =?UTF-8?q?=F0=9F=94=A7=20chore:=20align=20SDK=20reque?= =?UTF-8?q?sts=20and=20qualification=20refs?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/ci.yml | 4 ++-- .github/workflows/release-image.yml | 4 ++-- .github/workflows/reliability.yml | 4 ++-- docs/streaming.md | 28 ++++++++++++++++++++++------ go.mod | 2 +- go.sum | 4 ++-- 6 files changed, 31 insertions(+), 15 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9f836b5..90ac160 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -147,8 +147,8 @@ jobs: qualification: name: External test suite - uses: batchstream/sink-production-suite/.github/workflows/server-qualification.yml@730b49c71d1a6344c2986eae7bc9baaf2e1f271b + uses: batchstream/sink-production-suite/.github/workflows/server-qualification.yml@2e9151afcc0e1b1d210afd4c01b5a0c526c0f8d5 with: - suite_ref: 730b49c71d1a6344c2986eae7bc9baaf2e1f271b + suite_ref: 2e9151afcc0e1b1d210afd4c01b5a0c526c0f8d5 sink_ref: ${{ github.event.pull_request.head.sha || github.sha }} sdk_ref: ${{ github.head_ref || 'main' }} diff --git a/.github/workflows/release-image.yml b/.github/workflows/release-image.yml index 5f51835..97bff7a 100644 --- a/.github/workflows/release-image.yml +++ b/.github/workflows/release-image.yml @@ -20,9 +20,9 @@ env: jobs: qualify: name: Public production qualification - uses: batchstream/sink-production-suite/.github/workflows/release-qualification.yml@730b49c71d1a6344c2986eae7bc9baaf2e1f271b + uses: batchstream/sink-production-suite/.github/workflows/release-qualification.yml@2e9151afcc0e1b1d210afd4c01b5a0c526c0f8d5 with: - suite_ref: 730b49c71d1a6344c2986eae7bc9baaf2e1f271b + suite_ref: 2e9151afcc0e1b1d210afd4c01b5a0c526c0f8d5 sink_ref: ${{ github.event.release.tag_name }} binaries: diff --git a/.github/workflows/reliability.yml b/.github/workflows/reliability.yml index 6526688..017cbc1 100644 --- a/.github/workflows/reliability.yml +++ b/.github/workflows/reliability.yml @@ -8,7 +8,7 @@ permissions: jobs: soak: - uses: batchstream/sink-production-suite/.github/workflows/reliability.yml@730b49c71d1a6344c2986eae7bc9baaf2e1f271b + uses: batchstream/sink-production-suite/.github/workflows/reliability.yml@2e9151afcc0e1b1d210afd4c01b5a0c526c0f8d5 with: - suite_ref: 730b49c71d1a6344c2986eae7bc9baaf2e1f271b + suite_ref: 2e9151afcc0e1b1d210afd4c01b5a0c526c0f8d5 sink_ref: ${{ github.sha }} diff --git a/docs/streaming.md b/docs/streaming.md index a40d1d1..1278fdb 100644 --- a/docs/streaming.md +++ b/docs/streaming.md @@ -31,16 +31,32 @@ buffer. Existing message, backend response, scan-page and Lua bounds remain; this change introduces no additional memory-limit setting. Concurrent requests, transport/driver buffers and one large document still contribute to process RSS. -The Go SDK accepts slices for record collections and an optional callback: +The Go SDK methods accept `ctx` and a typed request. Request fields hold record +slices, completion mode, and an optional `OnResult` or `OnDocument` callback: ```go -results, err := client.Read(ctx, addresses) // Collected in request order. -results, err = client.Read(ctx, addresses, onRead) // results is nil. -written, err := client.Write(ctx, mode, operations, onWrite) -queryPage, err := client.Query(ctx, query, onDocument) // queryPage.Documents is nil. -scanPage, err := client.Scan(ctx, scan, onDocument) // scanPage.Documents is nil. +readRequest := sink.ReadRequest{Addresses: addresses} +results, err := client.Read(ctx, readRequest) // Collected in request order. +readRequest.OnResult = onRead +results, err = client.Read(ctx, readRequest) // results is nil. +writeRequest := sink.WriteRequest{ + CompletionMode: mode, + Operations: operations, + OnResult: onWrite, +} +written, err := client.Write(ctx, writeRequest) // written is nil. +query.OnDocument = onDocument +queryPage, err := client.Query(ctx, query) // queryPage.Documents is nil. +scan.OnDocument = onDocument +scanPage, err := client.Scan(ctx, scan) // scanPage.Documents is nil. ``` +Dataset record methods use `DatasetReadRequest.Keys` and +`DatasetWriteRequest.Records`, with the same `OnResult` behavior. Nil callbacks +collect results; non-nil callbacks consume them without collection. Query/Scan +still return page metadata after successful EOF. New per-call options extend +request structs without adding positional arguments. + Callbacks execute serially, provide backpressure, and cancel the RPC when they return an error. They receive owned immutable documents. Callback errors are not retried. Reads retry only unfinished/retryable operations without redelivering diff --git a/go.mod b/go.mod index 34077d0..439f67f 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.27.0 require ( github.com/go-logr/logr v1.4.4 github.com/iceisfun/golua v1.1.1 - github.com/liran/sink-go v0.9.1-0.20260920080920-9c80756d12cd + github.com/liran/sink-go v0.9.1-0.20260920091407-59b2ec143cc2 github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 github.com/prometheus/client_golang v1.24.1 github.com/twmb/franz-go v1.21.7 diff --git a/go.sum b/go.sum index bc5fea7..71ddb9f 100644 --- a/go.sum +++ b/go.sum @@ -30,8 +30,8 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= -github.com/liran/sink-go v0.9.1-0.20260920080920-9c80756d12cd h1:Gl4APsF7Fe/mG13Ipick/cR9E3v/CZpFyew+Lwb40MQ= -github.com/liran/sink-go v0.9.1-0.20260920080920-9c80756d12cd/go.mod h1:yzlNhufcKx518eeC969hAidHTM2ISB+hBag+VDuiPE4= +github.com/liran/sink-go v0.9.1-0.20260920091407-59b2ec143cc2 h1:JfNcjODGVCL3mCGgUTrcONmgbscYi9ULc1/m90av38M= +github.com/liran/sink-go v0.9.1-0.20260920091407-59b2ec143cc2/go.mod h1:yzlNhufcKx518eeC969hAidHTM2ISB+hBag+VDuiPE4= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/pierrec/lz4/v4 v4.1.26 h1:GrpZw1gZttORinvzBdXPUXATeqlJjqUG/D87TKMnhjY= From 884429a51e53d266eaee0a3e47463be5d63aef03 Mon Sep 17 00:00:00 2001 From: Randy <857271370@qq.com> Date: Sun, 20 Sep 2026 17:36:43 +0800 Subject: [PATCH 5/6] =?UTF-8?q?=F0=9F=94=A7=20chore:=20align=20request=20h?= =?UTF-8?q?elper=20docs=20and=20qualification=20refs?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/ci.yml | 4 ++-- .github/workflows/release-image.yml | 4 ++-- .github/workflows/reliability.yml | 4 ++-- docs/streaming.md | 19 +++++++++---------- go.mod | 2 +- go.sum | 4 ++-- 6 files changed, 18 insertions(+), 19 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 90ac160..03153a1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -147,8 +147,8 @@ jobs: qualification: name: External test suite - uses: batchstream/sink-production-suite/.github/workflows/server-qualification.yml@2e9151afcc0e1b1d210afd4c01b5a0c526c0f8d5 + uses: batchstream/sink-production-suite/.github/workflows/server-qualification.yml@2b8254d330ef03714f78ade54aa741c06f9d1cfb with: - suite_ref: 2e9151afcc0e1b1d210afd4c01b5a0c526c0f8d5 + suite_ref: 2b8254d330ef03714f78ade54aa741c06f9d1cfb sink_ref: ${{ github.event.pull_request.head.sha || github.sha }} sdk_ref: ${{ github.head_ref || 'main' }} diff --git a/.github/workflows/release-image.yml b/.github/workflows/release-image.yml index 97bff7a..a24d589 100644 --- a/.github/workflows/release-image.yml +++ b/.github/workflows/release-image.yml @@ -20,9 +20,9 @@ env: jobs: qualify: name: Public production qualification - uses: batchstream/sink-production-suite/.github/workflows/release-qualification.yml@2e9151afcc0e1b1d210afd4c01b5a0c526c0f8d5 + uses: batchstream/sink-production-suite/.github/workflows/release-qualification.yml@2b8254d330ef03714f78ade54aa741c06f9d1cfb with: - suite_ref: 2e9151afcc0e1b1d210afd4c01b5a0c526c0f8d5 + suite_ref: 2b8254d330ef03714f78ade54aa741c06f9d1cfb sink_ref: ${{ github.event.release.tag_name }} binaries: diff --git a/.github/workflows/reliability.yml b/.github/workflows/reliability.yml index 017cbc1..670d8ae 100644 --- a/.github/workflows/reliability.yml +++ b/.github/workflows/reliability.yml @@ -8,7 +8,7 @@ permissions: jobs: soak: - uses: batchstream/sink-production-suite/.github/workflows/reliability.yml@2e9151afcc0e1b1d210afd4c01b5a0c526c0f8d5 + uses: batchstream/sink-production-suite/.github/workflows/reliability.yml@2b8254d330ef03714f78ade54aa741c06f9d1cfb with: - suite_ref: 2e9151afcc0e1b1d210afd4c01b5a0c526c0f8d5 + suite_ref: 2b8254d330ef03714f78ade54aa741c06f9d1cfb sink_ref: ${{ github.sha }} diff --git a/docs/streaming.md b/docs/streaming.md index 1278fdb..828eb1b 100644 --- a/docs/streaming.md +++ b/docs/streaming.md @@ -35,19 +35,15 @@ The Go SDK methods accept `ctx` and a typed request. Request fields hold record slices, completion mode, and an optional `OnResult` or `OnDocument` callback: ```go -readRequest := sink.ReadRequest{Addresses: addresses} +readRequest := sink.NewReadRequest(addresses...) results, err := client.Read(ctx, readRequest) // Collected in request order. -readRequest.OnResult = onRead +readRequest = readRequest.WithOnResult(onRead) results, err = client.Read(ctx, readRequest) // results is nil. -writeRequest := sink.WriteRequest{ - CompletionMode: mode, - Operations: operations, - OnResult: onWrite, -} +writeRequest := sink.NewWriteRequest(operations...).WithOnResult(onWrite) written, err := client.Write(ctx, writeRequest) // written is nil. -query.OnDocument = onDocument +query = query.WithOnDocument(onDocument) queryPage, err := client.Query(ctx, query) // queryPage.Documents is nil. -scan.OnDocument = onDocument +scan = scan.WithOnDocument(onDocument) scanPage, err := client.Scan(ctx, scan) // scanPage.Documents is nil. ``` @@ -55,7 +51,10 @@ Dataset record methods use `DatasetReadRequest.Keys` and `DatasetWriteRequest.Records`, with the same `OnResult` behavior. Nil callbacks collect results; non-nil callbacks consume them without collection. Query/Scan still return page metadata after successful EOF. New per-call options extend -request structs without adding positional arguments. +request structs without adding positional arguments. Constructors and direct +struct literals share defaults: writes/deletes wait until applied, Query uses +page 1 with 100 documents, and Scan uses 100 documents per page. Override them +with `WithCompletionMode`, `WithPage`, `WithPageSize` or `WithBatchSize`. Callbacks execute serially, provide backpressure, and cancel the RPC when they return an error. They receive owned immutable documents. Callback errors are not diff --git a/go.mod b/go.mod index 439f67f..83ba548 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.27.0 require ( github.com/go-logr/logr v1.4.4 github.com/iceisfun/golua v1.1.1 - github.com/liran/sink-go v0.9.1-0.20260920091407-59b2ec143cc2 + github.com/liran/sink-go v0.9.1-0.20260920093327-8979b087f33e github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 github.com/prometheus/client_golang v1.24.1 github.com/twmb/franz-go v1.21.7 diff --git a/go.sum b/go.sum index 71ddb9f..e117693 100644 --- a/go.sum +++ b/go.sum @@ -30,8 +30,8 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= -github.com/liran/sink-go v0.9.1-0.20260920091407-59b2ec143cc2 h1:JfNcjODGVCL3mCGgUTrcONmgbscYi9ULc1/m90av38M= -github.com/liran/sink-go v0.9.1-0.20260920091407-59b2ec143cc2/go.mod h1:yzlNhufcKx518eeC969hAidHTM2ISB+hBag+VDuiPE4= +github.com/liran/sink-go v0.9.1-0.20260920093327-8979b087f33e h1:DIde9B7LKJ2DySqyeOafhUjh0E+Y8j2clyMJqCFXdM4= +github.com/liran/sink-go v0.9.1-0.20260920093327-8979b087f33e/go.mod h1:yzlNhufcKx518eeC969hAidHTM2ISB+hBag+VDuiPE4= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/pierrec/lz4/v4 v4.1.26 h1:GrpZw1gZttORinvzBdXPUXATeqlJjqUG/D87TKMnhjY= From 132fbb8775386c99f31b1068abcb5e1927365e13 Mon Sep 17 00:00:00 2001 From: Randy <857271370@qq.com> Date: Sun, 20 Sep 2026 18:16:49 +0800 Subject: [PATCH 6/6] =?UTF-8?q?=F0=9F=90=9B=20fix:=20restore=20streaming?= =?UTF-8?q?=20failover=20and=20failure=20diagnostics?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/ci.yml | 4 +- .github/workflows/release-image.yml | 4 +- .github/workflows/reliability.yml | 4 +- go.mod | 2 +- go.sum | 4 +- internal/app/logging.go | 77 ++++++++++++++------ internal/app/logging_test.go | 98 +++++++++++++++++++++++++- internal/storage/search/client.go | 14 ++-- internal/storage/search/query.go | 2 - internal/storage/search/stream_test.go | 81 +++++++++++++++++++++ 10 files changed, 250 insertions(+), 40 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 03153a1..7c72ae1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -147,8 +147,8 @@ jobs: qualification: name: External test suite - uses: batchstream/sink-production-suite/.github/workflows/server-qualification.yml@2b8254d330ef03714f78ade54aa741c06f9d1cfb + uses: batchstream/sink-production-suite/.github/workflows/server-qualification.yml@b053b392ede7f05581921f5cbb41f69d03ece9bb with: - suite_ref: 2b8254d330ef03714f78ade54aa741c06f9d1cfb + suite_ref: b053b392ede7f05581921f5cbb41f69d03ece9bb sink_ref: ${{ github.event.pull_request.head.sha || github.sha }} sdk_ref: ${{ github.head_ref || 'main' }} diff --git a/.github/workflows/release-image.yml b/.github/workflows/release-image.yml index a24d589..daa468b 100644 --- a/.github/workflows/release-image.yml +++ b/.github/workflows/release-image.yml @@ -20,9 +20,9 @@ env: jobs: qualify: name: Public production qualification - uses: batchstream/sink-production-suite/.github/workflows/release-qualification.yml@2b8254d330ef03714f78ade54aa741c06f9d1cfb + uses: batchstream/sink-production-suite/.github/workflows/release-qualification.yml@b053b392ede7f05581921f5cbb41f69d03ece9bb with: - suite_ref: 2b8254d330ef03714f78ade54aa741c06f9d1cfb + suite_ref: b053b392ede7f05581921f5cbb41f69d03ece9bb sink_ref: ${{ github.event.release.tag_name }} binaries: diff --git a/.github/workflows/reliability.yml b/.github/workflows/reliability.yml index 670d8ae..6016a84 100644 --- a/.github/workflows/reliability.yml +++ b/.github/workflows/reliability.yml @@ -8,7 +8,7 @@ permissions: jobs: soak: - uses: batchstream/sink-production-suite/.github/workflows/reliability.yml@2b8254d330ef03714f78ade54aa741c06f9d1cfb + uses: batchstream/sink-production-suite/.github/workflows/reliability.yml@b053b392ede7f05581921f5cbb41f69d03ece9bb with: - suite_ref: 2b8254d330ef03714f78ade54aa741c06f9d1cfb + suite_ref: b053b392ede7f05581921f5cbb41f69d03ece9bb sink_ref: ${{ github.sha }} diff --git a/go.mod b/go.mod index 83ba548..efb6beb 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.27.0 require ( github.com/go-logr/logr v1.4.4 github.com/iceisfun/golua v1.1.1 - github.com/liran/sink-go v0.9.1-0.20260920093327-8979b087f33e + github.com/liran/sink-go v0.9.1-0.20260920094052-14251539240b github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 github.com/prometheus/client_golang v1.24.1 github.com/twmb/franz-go v1.21.7 diff --git a/go.sum b/go.sum index e117693..b59ee50 100644 --- a/go.sum +++ b/go.sum @@ -30,8 +30,8 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= -github.com/liran/sink-go v0.9.1-0.20260920093327-8979b087f33e h1:DIde9B7LKJ2DySqyeOafhUjh0E+Y8j2clyMJqCFXdM4= -github.com/liran/sink-go v0.9.1-0.20260920093327-8979b087f33e/go.mod h1:yzlNhufcKx518eeC969hAidHTM2ISB+hBag+VDuiPE4= +github.com/liran/sink-go v0.9.1-0.20260920094052-14251539240b h1:AUB9eXU8qNEJem1CkLGSCOsqSvlAxZjRr8hMvFPsQzs= +github.com/liran/sink-go v0.9.1-0.20260920094052-14251539240b/go.mod h1:yzlNhufcKx518eeC969hAidHTM2ISB+hBag+VDuiPE4= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/pierrec/lz4/v4 v4.1.26 h1:GrpZw1gZttORinvzBdXPUXATeqlJjqUG/D87TKMnhjY= diff --git a/internal/app/logging.go b/internal/app/logging.go index e2df708..699fc71 100644 --- a/internal/app/logging.go +++ b/internal/app/logging.go @@ -6,6 +6,7 @@ import ( "strings" "time" + "github.com/liran/sink/gen/forward" sink "github.com/liran/sink/gen/sink" "google.golang.org/grpc" "google.golang.org/grpc/codes" @@ -17,23 +18,39 @@ func logUnary(ctx context.Context, req any, info *grpc.UnaryServerInfo, next grp response, err := next(ctx, req) method := diagnosticMethod(info.FullMethod) if method != "" { - code := status.Code(err) - observed := response - - logRPC(method, code, observed, time.Since(started)) + var summary rpcSummary + summary.observe(response) + summary.log(method, status.Code(err), time.Since(started)) } return response, err } func logStream(server any, stream grpc.ServerStream, info *grpc.StreamServerInfo, next grpc.StreamHandler) error { - started := time.Now() - err := next(server, stream) - if method := diagnosticMethod(info.FullMethod); method != "" { - logRPC(method, status.Code(err), nil, time.Since(started)) + method := diagnosticMethod(info.FullMethod) + if method == "" { + return next(server, stream) } + started := time.Now() + observed := &diagnosticStream{ServerStream: stream} + err := next(server, observed) + observed.summary.log(method, status.Code(err), time.Since(started)) return err } +// Keep counters only; streamed documents are released after SendMsg returns. +type diagnosticStream struct { + grpc.ServerStream + summary rpcSummary +} + +func (s *diagnosticStream) SendMsg(message any) error { + if err := s.ServerStream.SendMsg(message); err != nil { + return err + } + s.summary.observe(message) + return nil +} + func diagnosticMethod(full string) string { if !strings.HasPrefix(full, "/sink.v1.Sink/") && !strings.HasPrefix(full, "/sink.forward.v1.Engine/") { return "" @@ -47,16 +64,19 @@ func diagnosticMethod(full string) string { } } -func logRPC(method string, code codes.Code, response any, elapsed time.Duration) { - failed := 0 - operations := 0 - failureCode := "" +type rpcSummary struct { + operations int + failed int + failureCode string +} + +func (s *rpcSummary) observe(response any) { observe := func(failure *sink.Failure) { - operations++ + s.operations++ if failure != nil { - failed++ - if failureCode == "" { - failureCode = failure.GetCode().String() + s.failed++ + if s.failureCode == "" { + s.failureCode = failure.GetCode().String() } } } @@ -74,19 +94,34 @@ func logRPC(method string, code codes.Code, response any, elapsed time.Duration) observe(item.GetFailure()) } case *sink.ExecuteResponse: - operations = 1 + s.operations++ if !result.GetSuccess() { - failed = 1 - failureCode = "native_execution_failed" + s.failed++ + if s.failureCode == "" { + s.failureCode = "native_execution_failed" + } } + case *forward.ForwardResponse: + s.observe(result.GetResponse()) + case *forward.ForwardResponse_Read: + s.observe(result.Read) + case *forward.ForwardResponse_Write: + s.observe(result.Write) + case *forward.ForwardResponse_Delete: + s.observe(result.Delete) + case *forward.ForwardResponse_Execute: + s.observe(result.Execute) } +} + +func (s *rpcSummary) log(method string, code codes.Code, elapsed time.Duration) { level := slog.LevelDebug - if code != codes.OK || failed > 0 || elapsed > 5*time.Second { + if code != codes.OK || s.failed > 0 || elapsed > 5*time.Second { level = slog.LevelWarn } if code == codes.Internal || code == codes.DataLoss { level = slog.LevelError } slog.Log(context.Background(), level, "RPC completed", "component", "rpc", "event", "rpc_completed", "method", method, - "status", code.String(), "operations", operations, "failed", failed, "error_code", failureCode, "duration_ms", elapsed.Milliseconds()) + "status", code.String(), "operations", s.operations, "failed", s.failed, "error_code", s.failureCode, "duration_ms", elapsed.Milliseconds()) } diff --git a/internal/app/logging_test.go b/internal/app/logging_test.go index ea1ef84..a3da154 100644 --- a/internal/app/logging_test.go +++ b/internal/app/logging_test.go @@ -8,6 +8,7 @@ import ( "strings" "testing" + "github.com/liran/sink/gen/forward" sink "github.com/liran/sink/gen/sink" "github.com/liran/sink/internal/config" "github.com/liran/sink/internal/logging" @@ -110,8 +111,15 @@ func TestStreamDiagnosticsPreserveOutcomeAndExcludeHealth(t *testing.T) { called := false next := func(server any, stream grpc.ServerStream) error { called = true - if server != runtime || stream != nil { - t.Fatal("interceptor changed stream arguments") + if server != runtime { + t.Fatal("interceptor changed server") + } + if test.level == "" { + if stream != nil { + t.Fatal("interceptor wrapped an unrelated stream") + } + } else if observed, ok := stream.(*diagnosticStream); !ok || observed.ServerStream != nil { + t.Fatal("interceptor lost the underlying stream") } return test.err } @@ -135,3 +143,89 @@ func TestStreamDiagnosticsPreserveOutcomeAndExcludeHealth(t *testing.T) { } } } + +func TestStreamDiagnosticsCountResultsAcrossFramesWithoutPayload(t *testing.T) { + var output bytes.Buffer + options := &slog.HandlerOptions{Level: slog.LevelDebug} + logger := slog.New(slog.NewJSONHandler(&output, options)) + previous := slog.Default() + slog.SetDefault(logger) + defer slog.SetDefault(previous) + failure := &sink.Failure{Code: sink.FailureCode_FAILURE_CODE_UNAVAILABLE, Message: "private failure"} + document := &sink.Document{Payload: []byte("private document")} + read := &sink.ReadResponse{Results: []*sink.ReadResult{{Document: document}, {Failure: failure}}} + write := &sink.WriteResponse{Results: []*sink.WriteResult{{Document: document}, {Failure: failure}}} + deleted := &sink.DeleteResponse{Results: []*sink.DeleteResult{{}, {Failure: failure}}} + executed := &sink.ExecuteResponse{Payload: []byte("private document")} + forwardRead := &forward.ForwardResponse{Response: &forward.ForwardResponse_Read{Read: read}} + forwardWrite := &forward.ForwardResponse{Response: &forward.ForwardResponse_Write{Write: write}} + forwardDelete := &forward.ForwardResponse{Response: &forward.ForwardResponse_Delete{Delete: deleted}} + forwardExecute := &forward.ForwardResponse{Response: &forward.ForwardResponse_Execute{Execute: executed}} + cases := []struct { + method string + message any + operations int + }{ + {method: "Read", message: read, operations: 4}, + {method: "Write", message: write, operations: 4}, + {method: "Delete", message: deleted, operations: 4}, + {method: "Forward", message: forwardRead, operations: 4}, + {method: "Forward", message: forwardWrite, operations: 4}, + {method: "Forward", message: forwardDelete, operations: 4}, + {method: "Forward", message: forwardExecute, operations: 2}, + } + for _, test := range cases { + t.Run(test.method, func(t *testing.T) { + output.Reset() + stream := &diagnosticTestStream{} + prefix := "/sink.v1.Sink/" + if test.method == "Forward" { + prefix = "/sink.forward.v1.Engine/" + } + info := &grpc.StreamServerInfo{FullMethod: prefix + test.method} + next := func(_ any, observed grpc.ServerStream) error { + for range 2 { + if err := observed.SendMsg(test.message); err != nil { + return err + } + } + return nil + } + err := logStream(nil, stream, info, next) + if err != nil || stream.sent != 2 { + t.Fatalf("logging changed delivery: sent=%d err=%v", stream.sent, err) + } + var record map[string]any + if err := json.Unmarshal(output.Bytes(), &record); err != nil { + t.Fatal(err) + } + if record["level"] != "WARN" || record["failed"] != float64(2) || record["operations"] != float64(test.operations) || strings.Contains(output.String(), "private") { + t.Fatalf("incorrect stream summary: %s", output.String()) + } + }) + } +} + +func TestStreamDiagnosticsPreserveSendError(t *testing.T) { + failure := status.Error(codes.Canceled, "client disconnected") + underlying := &diagnosticTestStream{err: failure} + stream := &diagnosticStream{ServerStream: underlying} + response := &sink.WriteResponse{Results: []*sink.WriteResult{{}}} + if err := stream.SendMsg(response); err != failure || stream.summary.operations != 0 { + t.Fatalf("failed send counted or changed: summary=%+v err=%v", stream.summary, err) + } +} + +type diagnosticTestStream struct { + grpc.ServerStream + sent int + err error +} + +func (s *diagnosticTestStream) SendMsg(any) error { + if s.err != nil { + return s.err + } + s.sent++ + return nil +} diff --git a/internal/storage/search/client.go b/internal/storage/search/client.go index 6d60559..b3f477d 100644 --- a/internal/storage/search/client.go +++ b/internal/storage/search/client.go @@ -85,6 +85,11 @@ func (s *Store) perform(ctx context.Context, opts requestOptions) (apiResponse, endpoint.RawQuery = opts.query.Encode() response, err := s.performOnce(ctx, opts, endpoint) if err != nil { + // A successful HTTP response may already have delivered documents. + // Never replay decoding or callback failures, even before the first hit. + if opts.decode != nil && response.statusCode >= 200 && response.statusCode < 300 { + return empty, err + } // Caller cancellation and local response limits say nothing about // endpoint health. Let reads split oversized batches immediately. code, _ := storage.ErrorDetails(err) @@ -144,16 +149,13 @@ func (s *Store) performOnce(ctx context.Context, opts requestOptions, endpoint * return empty, errResponseTooLarge } if opts.decode != nil && httpResponse.StatusCode >= 200 && httpResponse.StatusCode < 300 { + response := apiResponse{statusCode: httpResponse.StatusCode, headers: httpResponse.Header.Clone()} reader := &io.LimitedReader{R: httpResponse.Body, N: maximum + 1} err := opts.decode(reader) if reader.N == 0 { - return empty, errResponseTooLarge - } - if err != nil { - return empty, err + return response, errResponseTooLarge } - response := apiResponse{statusCode: httpResponse.StatusCode, headers: httpResponse.Header.Clone()} - return response, nil + return response, err } body, err := io.ReadAll(io.LimitReader(httpResponse.Body, maximum+1)) if err != nil { diff --git a/internal/storage/search/query.go b/internal/storage/search/query.go index 2f50498..3a87f2e 100644 --- a/internal/storage/search/query.go +++ b/internal/storage/search/query.go @@ -74,8 +74,6 @@ func (s *Store) pageOptions(req storage.NativeRequest) (requestOptions, map[stri func (s *Store) performQuery(ctx context.Context, opts requestOptions) (scanPage, error) { var page scanPage if opts.emitHit != nil { - // Retrying after delivery would duplicate callback side effects. - opts.retrySafe = false opts.decode = func(reader io.Reader) error { decoded, err := decodeSearchPage(reader, opts.emitHit) page = decoded diff --git a/internal/storage/search/stream_test.go b/internal/storage/search/stream_test.go index cfa50ba..8d99807 100644 --- a/internal/storage/search/stream_test.go +++ b/internal/storage/search/stream_test.go @@ -9,6 +9,7 @@ import ( "net/http" "net/http/httptest" "strings" + "sync/atomic" "testing" "time" @@ -219,3 +220,83 @@ func TestScanStreamFailuresWithholdCursor(t *testing.T) { }) } } + +func TestManagedStreamsRecoverOnlyBeforeDecoding(t *testing.T) { + const body = `{"hits":{"hits":[{"_source":{"value":1},"sort":[1]}],"total":{"value":1,"relation":"eq"}},"timed_out":false,"_shards":{"total":1,"successful":1,"failed":0}}` + for _, method := range []string{"Query", "Scan"} { + for _, fault := range []string{"http-503", "transport", "malformed", "truncated", "callback"} { + t.Run(method+"/"+fault, func(t *testing.T) { + var firstCalls, secondCalls atomic.Int32 + firstHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + firstCalls.Add(1) + _, _ = io.Copy(io.Discard, r.Body) + switch fault { + case "http-503": + w.WriteHeader(http.StatusServiceUnavailable) + case "transport": + connection, _, err := w.(http.Hijacker).Hijack() + if err != nil { + t.Error(err) + return + } + _ = connection.Close() + case "malformed": + _, _ = io.WriteString(w, `{"hits":[]}`) + case "truncated": + _, _ = io.WriteString(w, `{"hits":{"hits":[{"_source":{"value":1},"sort":[1]},`) + default: + _, _ = io.WriteString(w, body) + } + }) + first := httptest.NewServer(firstHandler) + t.Cleanup(first.Close) + secondHandler := http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + secondCalls.Add(1) + _, _ = io.WriteString(w, body) + }) + second := httptest.NewServer(secondHandler) + t.Cleanup(second.Close) + opts := Options{Driver: DriverOpenSearch, Store: "primary", Endpoints: []string{first.URL, second.URL}} + store, err := New(opts) + if err != nil { + t.Fatal(err) + } + t.Cleanup(store.Close) + emitted := 0 + stopped := errors.New("consumer stopped") + emit := func(storage.Document) error { + emitted++ + if fault == "callback" { + return stopped + } + return nil + } + native := storage.NativeRequest{URI: "sink://primary/items", ContentType: ContentTypeJSON, Payload: []byte(`{"sort":["uid"]}`), MaxBytes: 4096} + if method == "Query" { + request := storage.QueryRequest{Request: native, PageSize: 1, Emit: emit} + _, err = store.Query(t.Context(), request) + } else { + request := storage.ScanRequest{Request: native, BatchSize: 1, Emit: emit} + _, err = store.Scan(t.Context(), request) + } + recoverable := fault == "http-503" || fault == "transport" + wantSecond, wantEmitted := int32(0), 1 + if recoverable { + wantSecond = 1 + } + if fault == "malformed" { + wantEmitted = 0 + } + if (err == nil) != recoverable || firstCalls.Load() != 1 || secondCalls.Load() != wantSecond || emitted != wantEmitted { + t.Fatalf("unsafe recovery: err=%v calls=%d/%d emitted=%d", err, firstCalls.Load(), secondCalls.Load(), emitted) + } + if fault == "callback" && !errors.Is(err, stopped) { + t.Fatalf("callback error lost: %v", err) + } + if (store.endpoints[0].retryAfter.Load() != 0) != recoverable || store.endpoints[1].retryAfter.Load() != 0 { + t.Fatal("decoding or callback failure changed endpoint health") + } + }) + } + } +}