From 9c80756d12cdd60d68f8aa5340f94a266ed7ccb0 Mon Sep 17 00:00:00 2001 From: Randy <857271370@qq.com> Date: Sun, 20 Sep 2026 16:09:20 +0800 Subject: [PATCH 1/4] =?UTF-8?q?=F0=9F=92=A5=20feat!:=20consume=20streaming?= =?UTF-8?q?=20responses=20with=20optional=20callbacks?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 49 +++++- api/sink/v1/sink.pb.go | 58 +++++--- api/sink/v1/sink_grpc.pb.go | 227 +++++++++++++++------------- api/sink/v1/sink_vtproto.pb.go | 66 +++++++++ balancing_test.go | 2 +- callback.go | 28 ++++ client.go | 239 +++++++++++++++++++----------- client_test.go | 76 +++++++--- dataset.go | 81 +++++++--- dataset_native.go | 8 +- dataset_native_test.go | 21 ++- dataset_test.go | 43 +++--- dns_balancing_integration_test.go | 2 +- integration_test.go | 15 +- native.go | 106 +++++++++---- native_integration_test.go | 2 +- native_test.go | 46 +++++- proto/sink/sink.proto | 26 ++-- query.go | 62 ++++++-- query_test.go | 21 ++- results.go | 126 ++++++---------- scan_deadline_test.go | 21 ++- streaming_test.go | 162 ++++++++++++++++++++ 23 files changed, 1058 insertions(+), 429 deletions(-) create mode 100644 callback.go create mode 100644 streaming_test.go diff --git a/README.md b/README.md index f39110c..099d591 100644 --- a/README.md +++ b/README.md @@ -99,12 +99,12 @@ func main() { _, err = products.Upsert( context.Background(), sink.CompletionWaitUntilVisible, - record, + []sink.Record{record}, ) if err != nil { log.Fatal(err) } - readResults, err := products.Read(context.Background(), sink.StringKey(value.UID)) + readResults, err := products.Read(context.Background(), []sink.Key{sink.StringKey(value.UID)}) if err != nil { log.Fatal(err) } @@ -120,7 +120,7 @@ func main() { `Dataset` binds the stable store, namespace, dataset, and document encoding once. `Read` accepts one or many keys without reconstructing addresses. Each mutation still receives an explicit completion mode because callers of the same dataset -can require different durability or visibility guarantees. Pass any number of +can require different durability or visibility guarantees. Pass a slice of records to `Create`, `Replace`, or `Upsert`; the client validates and encodes the complete collection before sending it, automatically splits large collections by `ClientOptions.MaxOperations`, and preserves global operation indexes: @@ -133,7 +133,7 @@ records := []sink.Record{ results, err := products.Upsert( context.Background(), sink.CompletionReturnAfterAccepted, - records..., + records, ) ``` @@ -179,7 +179,7 @@ record := sink.Record{ results, err := products.Merge( context.Background(), sink.CompletionWaitUntilVisible, - record, + []sink.Record{record}, ) if err != nil { log.Fatal(err) @@ -196,16 +196,49 @@ versioned `sink.v1` array, object, and retry-stable time helpers. See the [Lua merge developer guide](https://github.com/liran/sink/blob/main/docs/lua-merge-guide.md) for the complete function reference and reliability rules. +## Streaming callbacks + +Read, Write, Query and Scan use server-streaming RPCs. The previous unary wire +contract has been removed; upgrade server and SDK together. Record collections +are now slices so the final optional argument can be a callback. Omitting the +callback (or passing nil) collects results and preserves request order. Supplying +one calls it serially as items arrive and returns no collected documents: + +```go +results, err := client.Read(ctx, addresses) +results, err = client.Read(ctx, addresses, func(result sink.ReadResult) error { + // Process the owned result here. Returning an error cancels the stream. + return result.Err() +}) // results == nil + +page, err := client.Scan(ctx, request, func(document sink.Document) error { + return process(document) +}) // page.Documents == nil; NextCursor is available only on success. +``` + +Dataset Read/Create/Replace/Upsert/Merge accept a key/record slice and the same +optional result callback. Query and Scan accept an optional document callback. +Different record results can arrive out of order; `OperationIndex` always refers +to the original input slice. No callback documents are retained by the SDK. +Dataset methods retain only failure metadata for `BatchError`. + +Callbacks can observe partial results before a stream fails. Read retries never +redeliver completed results, and callback errors are never retried. Writes are +never automatically replayed; undelivered outcomes are unknown. Query/Scan +metadata is published only after successful EOF. If a page fails after processing +some documents, keep the previous cursor and use idempotent processing on resume. +The default collector returns received documents/results along with any error. + ## API model - `Dataset` is the primary record API. It binds routing, encoding, and one optional Lua program. `Read` accepts one or many keys; `Create`, `Replace`, `Upsert`, and `Merge` accept one or many `Record` values while keeping completion mode explicit per mutation. Every method splits large collections - automatically and returns all per-record results. -- `Read(ctx, addresses...)` preserves request order and reports found, + automatically and collects per-record results unless a callback is supplied. +- `Read(ctx, addresses, callback...)` preserves request order and reports found, not-found, or failed results independently. -- `Write(ctx, completionMode, operations...)` supports mixed put and merge +- `Write(ctx, completionMode, operations, callback...)` supports mixed put and merge batches. Use `NewPut` and `NewMerge` to construct validated operations. - `Delete(ctx, completionMode, addresses...)` performs hard deletes; deleting an absent record is successful. diff --git a/api/sink/v1/sink.pb.go b/api/sink/v1/sink.pb.go index 9fab054..810dd8e 100644 --- a/api/sink/v1/sink.pb.go +++ b/api/sink/v1/sink.pb.go @@ -645,8 +645,8 @@ func (x *ReadOperation) GetAddress() *RecordAddress { type ReadResponse struct { state protoimpl.MessageState `protogen:"open.v1"` - // Results are returned in request order. operation_index is included to make - // correlation explicit when an implementation executes records in parallel. + // Each frame contains one result. Results may arrive out of order; correlate + // by operation_index. Successful EOF requires exactly one result per operation. Results []*ReadResult `protobuf:"bytes,1,rep,name=results,proto3" json:"results,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache @@ -1096,6 +1096,8 @@ func (x *LuaProgram) GetSha256() []byte { return nil } +// Each frame contains one final result, identified by operation_index. +// A disconnected stream does not establish the outcome of missing operations. type WriteResponse struct { state protoimpl.MessageState `protogen:"open.v1"` Results []*WriteResult `protobuf:"bytes,1,rep,name=results,proto3" json:"results,omitempty"` @@ -1971,8 +1973,11 @@ func (x *Projection) GetExclude() bool { return false } +// Document frames contain one document. A final, document-free frame sets +// complete and has_more. Metadata is valid only after successful EOF. type QueryResponse struct { state protoimpl.MessageState `protogen:"open.v1"` + Complete bool `protobuf:"varint,3,opt,name=complete,proto3" json:"complete,omitempty"` Documents []*Document `protobuf:"bytes,1,rep,name=documents,proto3" json:"documents,omitempty"` // Determined by fetching one extra result, not by running Count. Result-window // limits apply to this extra result too. A byte limit fails the whole request; @@ -2012,6 +2017,13 @@ func (*QueryResponse) Descriptor() ([]byte, []int) { return file_sink_sink_proto_rawDescGZIP(), []int{26} } +func (x *QueryResponse) GetComplete() bool { + if x != nil { + return x.Complete + } + return false +} + func (x *QueryResponse) GetDocuments() []*Document { if x != nil { return x.Documents @@ -2212,13 +2224,16 @@ func (x *ScanRequest) GetProjection() *Projection { } // Documents retain native encodings: BSON documents or complete JSON hits. -// Failed requests return no page; callers may retry from their last saved -// cursor, with live-query semantics and idempotent business processing. +// A failed stream may have delivered documents but never a committed page. +// Resume from the last saved cursor with idempotent business processing. +// Document frames contain one document. A final, document-free frame sets +// complete and next_cursor. Checkpoint only after successful EOF. type ScanResponse struct { state protoimpl.MessageState `protogen:"open.v1"` + Complete bool `protobuf:"varint,3,opt,name=complete,proto3" json:"complete,omitempty"` Documents []*Document `protobuf:"bytes,1,rep,name=documents,proto3" json:"documents,omitempty"` // Empty means the scan reached the end observed by this request. Commit a - // checkpoint only after processing documents. No explicit close is needed. + // checkpoint only after processing documents and successful EOF. No close RPC is needed. NextCursor []byte `protobuf:"bytes,2,opt,name=next_cursor,json=nextCursor,proto3" json:"next_cursor,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache @@ -2254,6 +2269,13 @@ func (*ScanResponse) Descriptor() ([]byte, []int) { return file_sink_sink_proto_rawDescGZIP(), []int{30} } +func (x *ScanResponse) GetComplete() bool { + if x != nil { + return x.Complete + } + return false +} + func (x *ScanResponse) GetDocuments() []*Document { if x != nil { return x.Documents @@ -2379,8 +2401,9 @@ const file_sink_sink_proto_rawDesc = "" + "\n" + "Projection\x12\x16\n" + "\x06fields\x18\x01 \x03(\tR\x06fields\x12\x18\n" + - "\aexclude\x18\x02 \x01(\bR\aexclude\"[\n" + - "\rQueryResponse\x12/\n" + + "\aexclude\x18\x02 \x01(\bR\aexclude\"w\n" + + "\rQueryResponse\x12\x1a\n" + + "\bcomplete\x18\x03 \x01(\bR\bcomplete\x12/\n" + "\tdocuments\x18\x01 \x03(\v2\x11.sink.v1.DocumentR\tdocuments\x12\x19\n" + "\bhas_more\x18\x02 \x01(\bR\ahasMore\":\n" + "\fCountRequest\x12*\n" + @@ -2395,8 +2418,9 @@ const file_sink_sink_proto_rawDesc = "" + "\x06cursor\x18\x03 \x01(\fR\x06cursor\x123\n" + "\n" + "projection\x18\x04 \x01(\v2\x13.sink.v1.ProjectionR\n" + - "projection\"`\n" + - "\fScanResponse\x12/\n" + + "projection\"|\n" + + "\fScanResponse\x12\x1a\n" + + "\bcomplete\x18\x03 \x01(\bR\bcomplete\x12/\n" + "\tdocuments\x18\x01 \x03(\v2\x11.sink.v1.DocumentR\tdocuments\x12\x1f\n" + "\vnext_cursor\x18\x02 \x01(\fR\n" + "nextCursor*m\n" + @@ -2440,15 +2464,15 @@ const file_sink_sink_proto_rawDesc = "" + "\x1fFAILURE_CODE_RESOURCE_EXHAUSTED\x10\x05\x12\x1c\n" + "\x18FAILURE_CODE_UNAVAILABLE\x10\x06\x12\"\n" + "\x1eFAILURE_CODE_DEADLINE_EXCEEDED\x10\a\x12\x19\n" + - "\x15FAILURE_CODE_INTERNAL\x10\b2\x91\x03\n" + - "\x04Sink\x123\n" + - "\x04Read\x12\x14.sink.v1.ReadRequest\x1a\x15.sink.v1.ReadResponse\x126\n" + - "\x05Write\x12\x15.sink.v1.WriteRequest\x1a\x16.sink.v1.WriteResponse\x129\n" + + "\x15FAILURE_CODE_INTERNAL\x10\b2\x99\x03\n" + + "\x04Sink\x125\n" + + "\x04Read\x12\x14.sink.v1.ReadRequest\x1a\x15.sink.v1.ReadResponse0\x01\x128\n" + + "\x05Write\x12\x15.sink.v1.WriteRequest\x1a\x16.sink.v1.WriteResponse0\x01\x129\n" + "\x06Delete\x12\x16.sink.v1.DeleteRequest\x1a\x17.sink.v1.DeleteResponse\x12<\n" + - "\aExecute\x12\x17.sink.v1.ExecuteRequest\x1a\x18.sink.v1.ExecuteResponse\x126\n" + - "\x05Query\x12\x15.sink.v1.QueryRequest\x1a\x16.sink.v1.QueryResponse\x126\n" + - "\x05Count\x12\x15.sink.v1.CountRequest\x1a\x16.sink.v1.CountResponse\x123\n" + - "\x04Scan\x12\x14.sink.v1.ScanRequest\x1a\x15.sink.v1.ScanResponseB-Z+github.com/liran/sink-go/api/sink/v1;sinkv1b\x06proto3" + "\aExecute\x12\x17.sink.v1.ExecuteRequest\x1a\x18.sink.v1.ExecuteResponse\x128\n" + + "\x05Query\x12\x15.sink.v1.QueryRequest\x1a\x16.sink.v1.QueryResponse0\x01\x126\n" + + "\x05Count\x12\x15.sink.v1.CountRequest\x1a\x16.sink.v1.CountResponse\x125\n" + + "\x04Scan\x12\x14.sink.v1.ScanRequest\x1a\x15.sink.v1.ScanResponse0\x01B-Z+github.com/liran/sink-go/api/sink/v1;sinkv1b\x06proto3" var ( file_sink_sink_proto_rawDescOnce sync.Once diff --git a/api/sink/v1/sink_grpc.pb.go b/api/sink/v1/sink_grpc.pb.go index a26ceff..47303f4 100644 --- a/api/sink/v1/sink_grpc.pb.go +++ b/api/sink/v1/sink_grpc.pb.go @@ -37,13 +37,13 @@ const ( // Record requests are batch-native. Atomicity is guaranteed per record, never // for the whole request. Native queries use Execute, Query, Count or paged Scan. type SinkClient interface { - Read(ctx context.Context, in *ReadRequest, opts ...grpc.CallOption) (*ReadResponse, error) - Write(ctx context.Context, in *WriteRequest, opts ...grpc.CallOption) (*WriteResponse, error) + Read(ctx context.Context, in *ReadRequest, opts ...grpc.CallOption) (grpc.ServerStreamingClient[ReadResponse], error) + Write(ctx context.Context, in *WriteRequest, opts ...grpc.CallOption) (grpc.ServerStreamingClient[WriteResponse], error) Delete(ctx context.Context, in *DeleteRequest, opts ...grpc.CallOption) (*DeleteResponse, error) Execute(ctx context.Context, in *ExecuteRequest, opts ...grpc.CallOption) (*ExecuteResponse, error) - Query(ctx context.Context, in *QueryRequest, opts ...grpc.CallOption) (*QueryResponse, error) + Query(ctx context.Context, in *QueryRequest, opts ...grpc.CallOption) (grpc.ServerStreamingClient[QueryResponse], error) Count(ctx context.Context, in *CountRequest, opts ...grpc.CallOption) (*CountResponse, error) - Scan(ctx context.Context, in *ScanRequest, opts ...grpc.CallOption) (*ScanResponse, error) + Scan(ctx context.Context, in *ScanRequest, opts ...grpc.CallOption) (grpc.ServerStreamingClient[ScanResponse], error) } type sinkClient struct { @@ -55,26 +55,44 @@ func NewSinkClient(cc grpc.ClientConnInterface) SinkClient { return client } -func (c *sinkClient) Read(ctx context.Context, in *ReadRequest, opts ...grpc.CallOption) (*ReadResponse, error) { +func (c *sinkClient) Read(ctx context.Context, in *ReadRequest, opts ...grpc.CallOption) (grpc.ServerStreamingClient[ReadResponse], error) { cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) - out := new(ReadResponse) - err := c.cc.Invoke(ctx, Sink_Read_FullMethodName, in, out, cOpts...) + stream, err := c.cc.NewStream(ctx, &Sink_ServiceDesc.Streams[0], Sink_Read_FullMethodName, cOpts...) if err != nil { return nil, err } - return out, nil + x := &grpc.GenericClientStream[ReadRequest, ReadResponse]{ClientStream: stream} + if err := x.ClientStream.SendMsg(in); err != nil { + return nil, err + } + if err := x.ClientStream.CloseSend(); err != nil { + return nil, err + } + return x, nil } -func (c *sinkClient) Write(ctx context.Context, in *WriteRequest, opts ...grpc.CallOption) (*WriteResponse, error) { +// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. +type Sink_ReadClient = grpc.ServerStreamingClient[ReadResponse] + +func (c *sinkClient) Write(ctx context.Context, in *WriteRequest, opts ...grpc.CallOption) (grpc.ServerStreamingClient[WriteResponse], error) { cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) - out := new(WriteResponse) - err := c.cc.Invoke(ctx, Sink_Write_FullMethodName, in, out, cOpts...) + stream, err := c.cc.NewStream(ctx, &Sink_ServiceDesc.Streams[1], Sink_Write_FullMethodName, cOpts...) if err != nil { return nil, err } - return out, nil + x := &grpc.GenericClientStream[WriteRequest, WriteResponse]{ClientStream: stream} + if err := x.ClientStream.SendMsg(in); err != nil { + return nil, err + } + if err := x.ClientStream.CloseSend(); err != nil { + return nil, err + } + return x, nil } +// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. +type Sink_WriteClient = grpc.ServerStreamingClient[WriteResponse] + func (c *sinkClient) Delete(ctx context.Context, in *DeleteRequest, opts ...grpc.CallOption) (*DeleteResponse, error) { cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(DeleteResponse) @@ -95,16 +113,25 @@ func (c *sinkClient) Execute(ctx context.Context, in *ExecuteRequest, opts ...gr return out, nil } -func (c *sinkClient) Query(ctx context.Context, in *QueryRequest, opts ...grpc.CallOption) (*QueryResponse, error) { +func (c *sinkClient) Query(ctx context.Context, in *QueryRequest, opts ...grpc.CallOption) (grpc.ServerStreamingClient[QueryResponse], error) { cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) - out := new(QueryResponse) - err := c.cc.Invoke(ctx, Sink_Query_FullMethodName, in, out, cOpts...) + stream, err := c.cc.NewStream(ctx, &Sink_ServiceDesc.Streams[2], Sink_Query_FullMethodName, cOpts...) if err != nil { return nil, err } - return out, nil + x := &grpc.GenericClientStream[QueryRequest, QueryResponse]{ClientStream: stream} + if err := x.ClientStream.SendMsg(in); err != nil { + return nil, err + } + if err := x.ClientStream.CloseSend(); err != nil { + return nil, err + } + return x, nil } +// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. +type Sink_QueryClient = grpc.ServerStreamingClient[QueryResponse] + func (c *sinkClient) Count(ctx context.Context, in *CountRequest, opts ...grpc.CallOption) (*CountResponse, error) { cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(CountResponse) @@ -115,16 +142,25 @@ func (c *sinkClient) Count(ctx context.Context, in *CountRequest, opts ...grpc.C return out, nil } -func (c *sinkClient) Scan(ctx context.Context, in *ScanRequest, opts ...grpc.CallOption) (*ScanResponse, error) { +func (c *sinkClient) Scan(ctx context.Context, in *ScanRequest, opts ...grpc.CallOption) (grpc.ServerStreamingClient[ScanResponse], error) { cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) - out := new(ScanResponse) - err := c.cc.Invoke(ctx, Sink_Scan_FullMethodName, in, out, cOpts...) + stream, err := c.cc.NewStream(ctx, &Sink_ServiceDesc.Streams[3], Sink_Scan_FullMethodName, cOpts...) if err != nil { return nil, err } - return out, nil + x := &grpc.GenericClientStream[ScanRequest, ScanResponse]{ClientStream: stream} + if err := x.ClientStream.SendMsg(in); err != nil { + return nil, err + } + if err := x.ClientStream.CloseSend(); err != nil { + return nil, err + } + return x, nil } +// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. +type Sink_ScanClient = grpc.ServerStreamingClient[ScanResponse] + // SinkServer is the server API for Sink service. // All implementations must embed UnimplementedSinkServer // for forward compatibility. @@ -134,13 +170,13 @@ func (c *sinkClient) Scan(ctx context.Context, in *ScanRequest, opts ...grpc.Cal // Record requests are batch-native. Atomicity is guaranteed per record, never // for the whole request. Native queries use Execute, Query, Count or paged Scan. type SinkServer interface { - Read(context.Context, *ReadRequest) (*ReadResponse, error) - Write(context.Context, *WriteRequest) (*WriteResponse, error) + Read(*ReadRequest, grpc.ServerStreamingServer[ReadResponse]) error + Write(*WriteRequest, grpc.ServerStreamingServer[WriteResponse]) error Delete(context.Context, *DeleteRequest) (*DeleteResponse, error) Execute(context.Context, *ExecuteRequest) (*ExecuteResponse, error) - Query(context.Context, *QueryRequest) (*QueryResponse, error) + Query(*QueryRequest, grpc.ServerStreamingServer[QueryResponse]) error Count(context.Context, *CountRequest) (*CountResponse, error) - Scan(context.Context, *ScanRequest) (*ScanResponse, error) + Scan(*ScanRequest, grpc.ServerStreamingServer[ScanResponse]) error mustEmbedUnimplementedSinkServer() } @@ -151,11 +187,11 @@ type SinkServer interface { // pointer dereference when methods are called. type UnimplementedSinkServer struct{} -func (UnimplementedSinkServer) Read(context.Context, *ReadRequest) (*ReadResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method Read not implemented") +func (UnimplementedSinkServer) Read(*ReadRequest, grpc.ServerStreamingServer[ReadResponse]) error { + return status.Errorf(codes.Unimplemented, "method Read not implemented") } -func (UnimplementedSinkServer) Write(context.Context, *WriteRequest) (*WriteResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method Write not implemented") +func (UnimplementedSinkServer) Write(*WriteRequest, grpc.ServerStreamingServer[WriteResponse]) error { + return status.Errorf(codes.Unimplemented, "method Write not implemented") } func (UnimplementedSinkServer) Delete(context.Context, *DeleteRequest) (*DeleteResponse, error) { return nil, status.Errorf(codes.Unimplemented, "method Delete not implemented") @@ -163,14 +199,14 @@ func (UnimplementedSinkServer) Delete(context.Context, *DeleteRequest) (*DeleteR func (UnimplementedSinkServer) Execute(context.Context, *ExecuteRequest) (*ExecuteResponse, error) { return nil, status.Errorf(codes.Unimplemented, "method Execute not implemented") } -func (UnimplementedSinkServer) Query(context.Context, *QueryRequest) (*QueryResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method Query not implemented") +func (UnimplementedSinkServer) Query(*QueryRequest, grpc.ServerStreamingServer[QueryResponse]) error { + return status.Errorf(codes.Unimplemented, "method Query not implemented") } func (UnimplementedSinkServer) Count(context.Context, *CountRequest) (*CountResponse, error) { return nil, status.Errorf(codes.Unimplemented, "method Count not implemented") } -func (UnimplementedSinkServer) Scan(context.Context, *ScanRequest) (*ScanResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method Scan not implemented") +func (UnimplementedSinkServer) Scan(*ScanRequest, grpc.ServerStreamingServer[ScanResponse]) error { + return status.Errorf(codes.Unimplemented, "method Scan not implemented") } func (UnimplementedSinkServer) mustEmbedUnimplementedSinkServer() {} func (UnimplementedSinkServer) testEmbeddedByValue() {} @@ -193,42 +229,28 @@ func RegisterSinkServer(s grpc.ServiceRegistrar, srv SinkServer) { s.RegisterService(&Sink_ServiceDesc, srv) } -func _Sink_Read_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(ReadRequest) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return srv.(SinkServer).Read(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: Sink_Read_FullMethodName, +func _Sink_Read_Handler(srv interface{}, stream grpc.ServerStream) error { + m := new(ReadRequest) + if err := stream.RecvMsg(m); err != nil { + return err } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(SinkServer).Read(ctx, req.(*ReadRequest)) - } - return interceptor(ctx, in, info, handler) + return srv.(SinkServer).Read(m, &grpc.GenericServerStream[ReadRequest, ReadResponse]{ServerStream: stream}) } -func _Sink_Write_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(WriteRequest) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return srv.(SinkServer).Write(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: Sink_Write_FullMethodName, - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(SinkServer).Write(ctx, req.(*WriteRequest)) +// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. +type Sink_ReadServer = grpc.ServerStreamingServer[ReadResponse] + +func _Sink_Write_Handler(srv interface{}, stream grpc.ServerStream) error { + m := new(WriteRequest) + if err := stream.RecvMsg(m); err != nil { + return err } - return interceptor(ctx, in, info, handler) + return srv.(SinkServer).Write(m, &grpc.GenericServerStream[WriteRequest, WriteResponse]{ServerStream: stream}) } +// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. +type Sink_WriteServer = grpc.ServerStreamingServer[WriteResponse] + func _Sink_Delete_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(DeleteRequest) if err := dec(in); err != nil { @@ -265,24 +287,17 @@ func _Sink_Execute_Handler(srv interface{}, ctx context.Context, dec func(interf return interceptor(ctx, in, info, handler) } -func _Sink_Query_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(QueryRequest) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return srv.(SinkServer).Query(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: Sink_Query_FullMethodName, +func _Sink_Query_Handler(srv interface{}, stream grpc.ServerStream) error { + m := new(QueryRequest) + if err := stream.RecvMsg(m); err != nil { + return err } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(SinkServer).Query(ctx, req.(*QueryRequest)) - } - return interceptor(ctx, in, info, handler) + return srv.(SinkServer).Query(m, &grpc.GenericServerStream[QueryRequest, QueryResponse]{ServerStream: stream}) } +// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. +type Sink_QueryServer = grpc.ServerStreamingServer[QueryResponse] + func _Sink_Count_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(CountRequest) if err := dec(in); err != nil { @@ -301,24 +316,17 @@ func _Sink_Count_Handler(srv interface{}, ctx context.Context, dec func(interfac return interceptor(ctx, in, info, handler) } -func _Sink_Scan_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(ScanRequest) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return srv.(SinkServer).Scan(ctx, in) +func _Sink_Scan_Handler(srv interface{}, stream grpc.ServerStream) error { + m := new(ScanRequest) + if err := stream.RecvMsg(m); err != nil { + return err } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: Sink_Scan_FullMethodName, - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(SinkServer).Scan(ctx, req.(*ScanRequest)) - } - return interceptor(ctx, in, info, handler) + return srv.(SinkServer).Scan(m, &grpc.GenericServerStream[ScanRequest, ScanResponse]{ServerStream: stream}) } +// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. +type Sink_ScanServer = grpc.ServerStreamingServer[ScanResponse] + // Sink_ServiceDesc is the grpc.ServiceDesc for Sink service. // It's only intended for direct use with grpc.RegisterService, // and not to be introspected or modified (even as a copy) @@ -326,14 +334,6 @@ var Sink_ServiceDesc = grpc.ServiceDesc{ ServiceName: "sink.v1.Sink", HandlerType: (*SinkServer)(nil), Methods: []grpc.MethodDesc{ - { - MethodName: "Read", - Handler: _Sink_Read_Handler, - }, - { - MethodName: "Write", - Handler: _Sink_Write_Handler, - }, { MethodName: "Delete", Handler: _Sink_Delete_Handler, @@ -342,19 +342,32 @@ var Sink_ServiceDesc = grpc.ServiceDesc{ MethodName: "Execute", Handler: _Sink_Execute_Handler, }, - { - MethodName: "Query", - Handler: _Sink_Query_Handler, - }, { MethodName: "Count", Handler: _Sink_Count_Handler, }, + }, + Streams: []grpc.StreamDesc{ + { + StreamName: "Read", + Handler: _Sink_Read_Handler, + ServerStreams: true, + }, + { + StreamName: "Write", + Handler: _Sink_Write_Handler, + ServerStreams: true, + }, + { + StreamName: "Query", + Handler: _Sink_Query_Handler, + ServerStreams: true, + }, { - MethodName: "Scan", - Handler: _Sink_Scan_Handler, + StreamName: "Scan", + Handler: _Sink_Scan_Handler, + ServerStreams: true, }, }, - Streams: []grpc.StreamDesc{}, Metadata: "sink/sink.proto", } diff --git a/api/sink/v1/sink_vtproto.pb.go b/api/sink/v1/sink_vtproto.pb.go index 4b02044..02abd91 100644 --- a/api/sink/v1/sink_vtproto.pb.go +++ b/api/sink/v1/sink_vtproto.pb.go @@ -1491,6 +1491,16 @@ func (m *QueryResponse) MarshalToSizedBufferVT(dAtA []byte) (int, error) { i -= len(m.unknownFields) copy(dAtA[i:], m.unknownFields) } + if m.Complete { + i-- + if m.Complete { + dAtA[i] = 1 + } else { + dAtA[i] = 0 + } + i-- + dAtA[i] = 0x18 + } if m.HasMore { i-- if m.HasMore { @@ -1702,6 +1712,16 @@ func (m *ScanResponse) MarshalToSizedBufferVT(dAtA []byte) (int, error) { i -= len(m.unknownFields) copy(dAtA[i:], m.unknownFields) } + if m.Complete { + i-- + if m.Complete { + dAtA[i] = 1 + } else { + dAtA[i] = 0 + } + i-- + dAtA[i] = 0x18 + } if len(m.NextCursor) > 0 { i -= len(m.NextCursor) copy(dAtA[i:], m.NextCursor) @@ -2287,6 +2307,9 @@ func (m *QueryResponse) SizeVT() (n int) { if m.HasMore { n += 2 } + if m.Complete { + n += 2 + } n += len(m.unknownFields) return n } @@ -2362,6 +2385,9 @@ func (m *ScanResponse) SizeVT() (n int) { if l > 0 { n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) } + if m.Complete { + n += 2 + } n += len(m.unknownFields) return n } @@ -5738,6 +5764,26 @@ func (m *QueryResponse) UnmarshalVT(dAtA []byte) error { } } m.HasMore = bool(v != 0) + case 3: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Complete", wireType) + } + var v int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + m.Complete = bool(v != 0) default: iNdEx = preIndex skippy, err := protohelpers.Skip(dAtA[iNdEx:]) @@ -6211,6 +6257,26 @@ func (m *ScanResponse) UnmarshalVT(dAtA []byte) error { m.NextCursor = []byte{} } iNdEx = postIndex + case 3: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Complete", wireType) + } + var v int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + m.Complete = bool(v != 0) default: iNdEx = preIndex skippy, err := protohelpers.Skip(dAtA[iNdEx:]) diff --git a/balancing_test.go b/balancing_test.go index ba8fc3e..4de30e2 100644 --- a/balancing_test.go +++ b/balancing_test.go @@ -59,7 +59,7 @@ func TestDialBalancesWritesAndFollowsEndpointChanges(t *testing.T) { t.Helper() attempts++ record := sink.Record{Key: sink.StringKey(fmt.Sprintf("record-%d", attempts)), Value: map[string]int{"value": attempts}} - _, err := dataset.Upsert(ctx, sink.CompletionWaitUntilApplied, record) + _, err := dataset.Upsert(ctx, sink.CompletionWaitUntilApplied, []sink.Record{record}) if err != nil { t.Fatalf("write through resolved backends: %v", err) } diff --git a/callback.go b/callback.go new file mode 100644 index 0000000..a2e8a53 --- /dev/null +++ b/callback.go @@ -0,0 +1,28 @@ +package sink + +import "errors" + +// ReadCallback consumes a final read result before the next receive. Returning +// an error cancels the RPC. The SDK retains no result documents when a callback +// is supplied. Already delivered results remain valid if the stream fails. +type ReadCallback func(ReadResult) error + +// WriteCallback consumes a final write result without collecting documents. +// Returning an error cancels the RPC. Unacknowledged writes have an unknown +// outcome and are never replayed by the SDK. +type WriteCallback func(WriteResult) error + +// DocumentCallback consumes a Query or Scan document in query order. +// Returning an error cancels the RPC. The SDK does not collect these documents. +type DocumentCallback func(Document) error + +func oneCallback[T any](callbacks []T) (T, error) { + var empty T + if len(callbacks) > 1 { + return empty, errors.New("at most one result callback is allowed") + } + if len(callbacks) == 0 { + return empty, nil + } + return callbacks[0], nil +} diff --git a/client.go b/client.go index d5d9317..9e5ab1d 100644 --- a/client.go +++ b/client.go @@ -6,7 +6,9 @@ import ( "crypto/tls" "errors" "fmt" + "io" "math/rand/v2" + "sort" "strings" "time" @@ -269,12 +271,13 @@ func (c *Client) CheckHealth(ctx context.Context) error { return nil } -// Read returns results in request order and automatically splits large +// Read collects results in request order, or passes them in arrival order to an +// optional callback without collecting them. It automatically splits large // collections into configured operation-count batches. Transport-level // Unavailable errors and retryable per-operation failures are retried because // reads are idempotent. Returned operation indexes refer to the original // collection. -func (c *Client) Read(ctx context.Context, addresses ...Address) ([]ReadResult, error) { +func (c *Client) Read(ctx context.Context, addresses []Address, callbacks ...ReadCallback) ([]ReadResult, error) { if err := c.validateCollection("read", len(addresses)); err != nil { return nil, err } @@ -287,15 +290,30 @@ func (c *Client) Read(ctx context.Context, addresses ...Address) ([]ReadResult, operation := &sinkv1.ReadOperation{Address: protoAddress} operations[index] = operation } - results := make([]ReadResult, 0, len(addresses)) + callback, err := oneCallback(callbacks) + if err != nil { + return nil, err + } + var results []ReadResult + defer func() { + sort.Slice(results, func(i, j int) bool { return results[i].OperationIndex < results[j].OperationIndex }) + }() for start := 0; start < len(operations); start += c.config.maxOperations { end := min(start+c.config.maxOperations, len(operations)) - batch, err := c.readOperationsWithRetry(ctx, operations[start:end]) + err := c.readOperationsWithRetry(ctx, operations[start:end], func(result ReadResult) error { + result.OperationIndex += start + if result.Failure != nil { + result.Failure.OperationIndex += start + } + if callback != nil { + return callback(result) + } + results = append(results, result) + return nil + }) if err != nil { return results, fmt.Errorf("read records: %w", err) } - remapReadIndexes(batch, start) - results = append(results, batch...) } return results, nil } @@ -304,11 +322,14 @@ func (c *Client) Read(ctx context.Context, addresses ...Address) ([]ReadResult, // collections into configured operation-count batches. It deliberately does // not retry transport failures because the server may already have applied or // durably accepted the mutation. Earlier batches may have completed when a -// later batch returns an error. +// later batch returns an error. Without a callback, acknowledged results are +// collected in request order. With a callback, results arrive incrementally +// and the returned result slice is nil. func (c *Client) Write( ctx context.Context, completionMode CompletionMode, - operations ...WriteOperation, + operations []WriteOperation, + callbacks ...WriteCallback, ) ([]WriteResult, error) { if err := c.validateCollection("write", len(operations)); err != nil { return nil, err @@ -324,24 +345,44 @@ func (c *Client) Write( return nil, fmt.Errorf("write operation %d: %w", index, err) } } - results := make([]WriteResult, 0, len(operations)) + callback, err := oneCallback(callbacks) + if err != nil { + return nil, err + } + var results []WriteResult + defer func() { + sort.Slice(results, func(i, j int) bool { return results[i].OperationIndex < results[j].OperationIndex }) + }() for start := 0; start < len(operations); start += c.config.maxOperations { end := min(start+c.config.maxOperations, len(operations)) - batch, err := c.writeBatch(ctx, completionMode, operations[start:end]) - if err != nil { + call := writeBatchCall{mode: completionMode, operations: operations[start:end]} + call.emit = func(result WriteResult) error { + result.OperationIndex += start + if result.Failure != nil { + result.Failure.OperationIndex += start + } + if callback != nil { + return callback(result) + } + results = append(results, result) + return nil + } + if err := c.writeBatch(ctx, call); err != nil { return results, err } - remapWriteIndexes(batch, start) - results = append(results, batch...) } return results, nil } -func (c *Client) writeBatch( - ctx context.Context, - completionMode CompletionMode, - operations []WriteOperation, -) ([]WriteResult, error) { +type writeBatchCall struct { + mode CompletionMode + operations []WriteOperation + emit WriteCallback +} + +func (c *Client) writeBatch(ctx context.Context, call writeBatchCall) error { + operations, completionMode := call.operations, call.mode + protoOperations := make([]*sinkv1.WriteOperation, len(operations)) luaPrograms := make([]*sinkv1.LuaProgram, 0) seenLuaPrograms := make(map[[sha256.Size]byte]struct{}) @@ -364,20 +405,45 @@ func (c *Client) writeBatch( Operations: protoOperations, LuaPrograms: luaPrograms, } - response, err := c.rpc.Write(ctx, request, c.config.sinkCallOptions...) + ctx, cancel := context.WithCancel(ctx) + defer cancel() + stream, err := c.rpc.Write(ctx, request, c.config.sinkCallOptions...) if err != nil { - return nil, fmt.Errorf("write records: %w", err) - } - results, err := decodeWriteResponse(response, len(operations)) - if err != nil { - return nil, err - } - for index, operation := range operations { - if operation.returnDocument && results[index].Status == WriteApplied && len(results[index].Document.payload) == 0 { - return nil, protocolError("Write", "applied operation omitted the requested document") + return fmt.Errorf("write records: %w", err) + } + seen := make([]bool, len(operations)) + count := 0 + for { + frame, err := stream.Recv() + if err == io.EOF { + if count != len(operations) { + return protocolError("Write", "stream omitted operation results") + } + return nil + } + if err != nil { + return fmt.Errorf("write records: %w", err) + } + if frame == nil || len(frame.GetResults()) != 1 || frame.Results[0] == nil { + return protocolError("Write", "expected one result per frame") + } + raw := frame.Results[0] + index, err := validateResultIndex("Write", raw.GetOperationIndex(), seen) + if err != nil { + return err + } + result, err := decodeWriteResult(raw, index) + if err != nil { + return err } + if operations[index].returnDocument && result.Status == WriteApplied && len(result.Document.payload) == 0 { + return protocolError("Write", "applied operation omitted the requested document") + } + if err := call.emit(result); err != nil { + return err + } + count++ } - return results, nil } // Delete permanently deletes records. Deleting an absent record is successful. @@ -436,24 +502,6 @@ func (c *Client) validateCollection(method string, count int) error { return nil } -func remapReadIndexes(results []ReadResult, offset int) { - for index := range results { - results[index].OperationIndex += offset - if results[index].Failure != nil { - results[index].Failure.OperationIndex += offset - } - } -} - -func remapWriteIndexes(results []WriteResult, offset int) { - for index := range results { - results[index].OperationIndex += offset - if results[index].Failure != nil { - results[index].Failure.OperationIndex += offset - } - } -} - func remapDeleteIndexes(results []DeleteResult, offset int) { for index := range results { results[index].OperationIndex += offset @@ -473,60 +521,83 @@ type pendingRead struct { operation *sinkv1.ReadOperation } -func (c *Client) readOperationsWithRetry( - ctx context.Context, - operations []*sinkv1.ReadOperation, -) ([]ReadResult, error) { - results := make([]ReadResult, len(operations)) - pending := make([]pendingRead, 0, len(operations)) +func (c *Client) readOperationsWithRetry(ctx context.Context, operations []*sinkv1.ReadOperation, emit ReadCallback) error { + pending := make([]pendingRead, len(operations)) for index, operation := range operations { - work := pendingRead{index: index, operation: operation} - pending = append(pending, work) + pending[index] = pendingRead{index: index, operation: operation} } backoff := c.config.readRetry.InitialBackoff for attempt := 1; attempt <= c.config.readRetry.MaxAttempts; attempt++ { - requestOperations := make([]*sinkv1.ReadOperation, 0, len(pending)) - for _, work := range pending { - requestOperations = append(requestOperations, work.operation) + request := &sinkv1.ReadRequest{Operations: make([]*sinkv1.ReadOperation, len(pending))} + for index, work := range pending { + request.Operations[index] = work.operation } - request := &sinkv1.ReadRequest{Operations: requestOperations} - response, err := c.rpc.Read(ctx, request, c.config.sinkCallOptions...) - if err != nil { - if attempt == c.config.readRetry.MaxAttempts || status.Code(err) != codes.Unavailable { - return nil, err - } - if err := waitForBackoff(ctx, jitteredBackoff(backoff, c.config.readRetry.Jitter)); err != nil { - return nil, err + execution, cancel := context.WithCancel(ctx) + stream, transportErr := c.rpc.Read(execution, request, c.config.sinkCallOptions...) + seen := make([]bool, len(pending)) + next := make([]pendingRead, 0) + count := 0 + if transportErr == nil { + for { + frame, err := stream.Recv() + if err == io.EOF { + if count != len(pending) { + cancel() + return protocolError("Read", "stream omitted operation results") + } + break + } + if err != nil { + transportErr = err + break + } + if frame == nil || len(frame.GetResults()) != 1 || frame.Results[0] == nil { + cancel() + return protocolError("Read", "expected one result per frame") + } + raw := frame.Results[0] + index, err := validateResultIndex("Read", raw.GetOperationIndex(), seen) + if err != nil { + cancel() + return err + } + work := pending[index] + result, err := decodeReadResult(raw, work.index) + if err != nil { + cancel() + return err + } + count++ + if attempt < c.config.readRetry.MaxAttempts && result.Failure != nil && result.Failure.Retryable { + next = append(next, work) + } else if err := emit(result); err != nil { + cancel() + return err + } } - backoff = nextBackoff(backoff, c.config.readRetry) - continue } - decoded, err := decodeReadResponse(response, len(pending)) - if err != nil { - return nil, err - } - next := make([]pendingRead, 0) - for index, result := range decoded { - work := pending[index] - result.OperationIndex = work.index - if result.Failure != nil { - result.Failure.OperationIndex = work.index + cancel() + if transportErr != nil { + if attempt == c.config.readRetry.MaxAttempts || status.Code(transportErr) != codes.Unavailable { + return transportErr } - results[work.index] = result - if attempt < c.config.readRetry.MaxAttempts && result.Failure != nil && result.Failure.Retryable { - next = append(next, work) + // Never redeliver a result already observed by the caller. + for index, work := range pending { + if !seen[index] { + next = append(next, work) + } } } if len(next) == 0 { - return results, nil + return transportErr } if err := waitForBackoff(ctx, jitteredBackoff(backoff, c.config.readRetry.Jitter)); err != nil { - return nil, err + return err } pending = next backoff = nextBackoff(backoff, c.config.readRetry) } - return results, nil + return nil } func waitForBackoff(ctx context.Context, delay time.Duration) error { diff --git a/client_test.go b/client_test.go index e1f4ea6..02cbd4d 100644 --- a/client_test.go +++ b/client_test.go @@ -72,7 +72,7 @@ type testSinkServer struct { suppressWriteError bool } -func (s *testSinkServer) Read( +func (s *testSinkServer) readResponse( _ context.Context, request *sinkv1.ReadRequest, ) (*sinkv1.ReadResponse, error) { @@ -226,7 +226,7 @@ func TestDocumentRejectsMissingEncodingAndMalformedPayload(t *testing.T) { } } -func (s *testSinkServer) Write( +func (s *testSinkServer) writeResponse( _ context.Context, request *sinkv1.WriteRequest, ) (*sinkv1.WriteResponse, error) { @@ -412,7 +412,7 @@ func TestClientCoversSinkContract(t *testing.T) { testAddress(t, opaque), } - readResults, err := client.Read(context.Background(), addresses...) + readResults, err := client.Read(context.Background(), addresses) if err != nil { t.Fatalf("Read() error = %v", err) } @@ -446,7 +446,7 @@ func TestClientCoversSinkContract(t *testing.T) { if err != nil { t.Fatalf("sink.NewMerge() error = %v", err) } - writeResults, err := client.Write(context.Background(), sink.CompletionWaitUntilApplied, put, merge) + writeResults, err := client.Write(context.Background(), sink.CompletionWaitUntilApplied, []sink.WriteOperation{put, merge}) if err != nil { t.Fatalf("Write() error = %v", err) } @@ -524,7 +524,7 @@ func TestClientSendsWaitUntilVisibleCompletionMode(t *testing.T) { if err != nil { t.Fatalf("sink.NewPut() error = %v", err) } - _, err = client.Write(t.Context(), sink.CompletionWaitUntilVisible, operation) + _, err = client.Write(t.Context(), sink.CompletionWaitUntilVisible, []sink.WriteOperation{operation}) if err != nil { t.Fatalf("Write() error = %v", err) } @@ -557,7 +557,7 @@ func TestReadRetriesOnlyUnavailable(t *testing.T) { clientOptions := sink.ClientOptions{ReadRetry: retry} client := startTestClient(t, server, clientOptions) address := testAddress(t, sink.StringKey("retry")) - _, err := client.Read(context.Background(), address) + _, err := client.Read(context.Background(), []sink.Address{address}) if err != nil { t.Fatalf("Read() error = %v", err) } @@ -578,7 +578,7 @@ func TestReadRetriesOnlyUnavailable(t *testing.T) { clientOptions := sink.ClientOptions{ReadRetry: retry} client := startTestClient(t, server, clientOptions) address := testAddress(t, sink.StringKey("quota")) - _, err := client.Read(context.Background(), address) + _, err := client.Read(context.Background(), []sink.Address{address}) if status.Code(err) != codes.ResourceExhausted { t.Fatalf("Read() code = %s, want ResourceExhausted", status.Code(err)) } @@ -611,7 +611,7 @@ func TestWriteDeclaresIdenticalLuaProgramOncePerBatch(t *testing.T) { if err != nil { t.Fatalf("sink.NewMerge(second) error = %v", err) } - if _, err := client.Write(context.Background(), sink.CompletionWaitUntilApplied, first, second); err != nil { + if _, err := client.Write(context.Background(), sink.CompletionWaitUntilApplied, []sink.WriteOperation{first, second}); err != nil { t.Fatalf("Write() error = %v", err) } @@ -635,7 +635,7 @@ func TestClientAcceptsMessagesLargerThanGRPCDefault(t *testing.T) { var clientOptions sink.ClientOptions client := startTestClient(t, server, clientOptions) address := testAddress(t, sink.StringKey("large")) - results, err := client.Read(context.Background(), address) + results, err := client.Read(context.Background(), []sink.Address{address}) if err != nil { t.Fatalf("Read() error = %v", err) } @@ -658,7 +658,7 @@ func TestReadRetriesOnlyRetryableOperationFailures(t *testing.T) { testAddress(t, sink.StringKey("retry")), testAddress(t, sink.StringKey("success")), } - results, err := client.Read(context.Background(), addresses...) + results, err := client.Read(context.Background(), addresses) if err != nil { t.Fatalf("Read() error = %v", err) } @@ -694,11 +694,11 @@ func TestClientMethodsSplitBatchesAndRemapIndexes(t *testing.T) { } operations[index] = operation } - readResults, err := client.Read(context.Background(), addresses...) + readResults, err := client.Read(context.Background(), addresses) if err != nil { t.Fatalf("Read() error = %v", err) } - writeResults, err := client.Write(context.Background(), sink.CompletionWaitUntilApplied, operations...) + writeResults, err := client.Write(context.Background(), sink.CompletionWaitUntilApplied, operations) if err != nil { t.Fatalf("Write() error = %v", err) } @@ -740,11 +740,11 @@ func TestClientValidatesCompleteCollectionsBeforeRPC(t *testing.T) { operations[index] = operation } - _, readErr := client.Read(t.Context(), addresses...) + _, readErr := client.Read(t.Context(), addresses) if readErr == nil || !strings.Contains(readErr.Error(), "read operation 2") { t.Fatalf("Read() error = %v", readErr) } - _, writeErr := client.Write(t.Context(), sink.CompletionWaitUntilApplied, operations...) + _, writeErr := client.Write(t.Context(), sink.CompletionWaitUntilApplied, operations) if writeErr == nil || !strings.Contains(writeErr.Error(), "write operation 2") { t.Fatalf("Write() error = %v", writeErr) } @@ -783,11 +783,11 @@ func TestClientMethodsReturnPartialResultsWhenLaterBatchFails(t *testing.T) { operations[index] = operation } - readResults, readErr := client.Read(t.Context(), addresses...) + readResults, readErr := client.Read(t.Context(), addresses) if status.Code(readErr) != codes.Unavailable || len(readResults) != 2 { t.Fatalf("Read() results = %+v, error = %v", readResults, readErr) } - writeResults, writeErr := client.Write(t.Context(), sink.CompletionWaitUntilApplied, operations...) + writeResults, writeErr := client.Write(t.Context(), sink.CompletionWaitUntilApplied, operations) if status.Code(writeErr) != codes.Unavailable || len(writeResults) != 2 { t.Fatalf("Write() results = %+v, error = %v", writeResults, writeErr) } @@ -820,7 +820,7 @@ func TestMutationsAreNotRetried(t *testing.T) { if err != nil { t.Fatalf("sink.NewPut() error = %v", err) } - _, err = client.Write(t.Context(), mode, put) + _, err = client.Write(t.Context(), mode, []sink.WriteOperation{put}) if status.Code(err) != codes.Unavailable { t.Fatalf("Write() code = %s, want Unavailable", status.Code(err)) } @@ -859,7 +859,7 @@ func TestReadRejectsMalformedResponses(t *testing.T) { testAddress(t, sink.StringKey("one")), testAddress(t, sink.StringKey("two")), } - _, err := client.Read(context.Background(), addresses...) + _, err := client.Read(context.Background(), addresses) var protocolErr *sink.ProtocolError if !errors.As(err, &protocolErr) { t.Fatalf("Read() error = %v, want ProtocolError", err) @@ -885,7 +885,7 @@ func TestMutationResponsesRejectDuplicateIndexes(t *testing.T) { } operations[index] = operation } - _, err := client.Write(context.Background(), sink.CompletionWaitUntilApplied, operations...) + _, err := client.Write(context.Background(), sink.CompletionWaitUntilApplied, operations) var writeProtocolErr *sink.ProtocolError if !errors.As(err, &writeProtocolErr) { t.Fatalf("Write() error = %v, want ProtocolError", err) @@ -912,7 +912,7 @@ func TestConstructorsValidateAndCopyInput(t *testing.T) { t.Fatalf("sink.NewPut() error = %v", err) } raw[10] = 'X' - _, err = client.Write(t.Context(), sink.CompletionWaitUntilApplied, operation) + _, err = client.Write(t.Context(), sink.CompletionWaitUntilApplied, []sink.WriteOperation{operation}) if err != nil { t.Fatalf("Write() error = %v", err) } @@ -959,7 +959,7 @@ func TestConstructorsValidateAndCopyInput(t *testing.T) { t.Fatal("sink.NewLuaProgram() accepted empty source") } - results, err := client.Read(t.Context(), address) + results, err := client.Read(t.Context(), []sink.Address{address}) if err != nil { t.Fatalf("Read() error = %v", err) } @@ -1014,3 +1014,37 @@ func TestClientOptionsValidation(t *testing.T) { } } } + +func (s *testSinkServer) Read(req *sinkv1.ReadRequest, stream grpc.ServerStreamingServer[sinkv1.ReadResponse]) error { + response, err := s.readResponse(stream.Context(), req) + if err != nil { + return err + } + if response == nil { + return nil + } + for _, result := range response.Results { + frame := &sinkv1.ReadResponse{Results: []*sinkv1.ReadResult{result}} + if err := stream.Send(frame); err != nil { + return err + } + } + return nil +} + +func (s *testSinkServer) Write(req *sinkv1.WriteRequest, stream grpc.ServerStreamingServer[sinkv1.WriteResponse]) error { + response, err := s.writeResponse(stream.Context(), req) + if err != nil { + return err + } + if response == nil { + return nil + } + for _, result := range response.Results { + frame := &sinkv1.WriteResponse{Results: []*sinkv1.WriteResult{result}} + if err := stream.Send(frame); err != nil { + return err + } + } + return nil +} diff --git a/dataset.go b/dataset.go index 52d9680..5d2fa02 100644 --- a/dataset.go +++ b/dataset.go @@ -69,9 +69,11 @@ func NewDataset(client *Client, opts DatasetOptions) (*Dataset, error) { return dataset, nil } -// Read fetches one or more records by key. It preserves key order, splits large -// collections automatically, and treats not-found results as successful reads. -func (d *Dataset) Read(ctx context.Context, keys ...Key) ([]ReadResult, error) { +// Read fetches one or more records by key, splits large collections automatically, +// and treats not-found results as successful reads. Without a callback, results +// are collected in key order. With a callback, results arrive incrementally +// and the returned result slice is nil. +func (d *Dataset) Read(ctx context.Context, keys []Key, callbacks ...ReadCallback) ([]ReadResult, error) { if err := d.validate("read"); err != nil { return nil, err } @@ -83,8 +85,22 @@ func (d *Dataset) Read(ctx context.Context, keys ...Key) ([]ReadResult, error) { } addresses[index] = address } - results, err := d.client.Read(ctx, addresses...) - resultsErr := ReadResultsError(results) + callback, err := oneCallback(callbacks) + if err != nil { + return nil, err + } + var failures []*OperationError + var onResult ReadCallback + if callback != nil { + onResult = func(result ReadResult) error { + if result.Failure != nil { + failures = append(failures, result.Failure) + } + return callback(result) + } + } + results, err := d.client.Read(ctx, addresses, onResult) + resultsErr := errors.Join(ReadResultsError(results), newBatchError(failures)) if err != nil { if resultsErr != nil { err = errors.Join(err, resultsErr) @@ -101,13 +117,15 @@ func (d *Dataset) Read(ctx context.Context, keys ...Key) ([]ReadResult, error) { func (d *Dataset) Create( ctx context.Context, completionMode CompletionMode, - records ...Record, + records []Record, + callbacks ...WriteCallback, ) ([]WriteResult, error) { opts := datasetPutOptions{ completionMode: completionMode, writeMode: WriteCreate, operation: "create", records: records, + callbacks: callbacks, } return d.put(ctx, opts) } @@ -116,13 +134,15 @@ func (d *Dataset) Create( func (d *Dataset) Replace( ctx context.Context, completionMode CompletionMode, - records ...Record, + records []Record, + callbacks ...WriteCallback, ) ([]WriteResult, error) { opts := datasetPutOptions{ completionMode: completionMode, writeMode: WriteReplace, operation: "replace", records: records, + callbacks: callbacks, } return d.put(ctx, opts) } @@ -131,13 +151,15 @@ func (d *Dataset) Replace( func (d *Dataset) Upsert( ctx context.Context, completionMode CompletionMode, - records ...Record, + records []Record, + callbacks ...WriteCallback, ) ([]WriteResult, error) { opts := datasetPutOptions{ completionMode: completionMode, writeMode: WriteUpsert, operation: "upsert", records: records, + callbacks: callbacks, } return d.put(ctx, opts) } @@ -148,7 +170,8 @@ func (d *Dataset) Upsert( func (d *Dataset) Merge( ctx context.Context, completionMode CompletionMode, - records ...Record, + records []Record, + callbacks ...WriteCallback, ) ([]WriteResult, error) { if err := d.validate("merge"); err != nil { return nil, err @@ -173,10 +196,12 @@ func (d *Dataset) Merge( operation.returnDocument = record.ReturnDocument operations[index] = operation } - return d.write(ctx, "merge", completionMode, operations) + opts := datasetWriteOptions{operation: "merge", mode: completionMode, operations: operations, callbacks: callbacks} + return d.write(ctx, opts) } type datasetPutOptions struct { + callbacks []WriteCallback completionMode CompletionMode writeMode WriteMode operation string @@ -200,7 +225,8 @@ func (d *Dataset) put(ctx context.Context, opts datasetPutOptions) ([]WriteResul operation.returnDocument = record.ReturnDocument operations[index] = operation } - return d.write(ctx, opts.operation, opts.completionMode, operations) + write := datasetWriteOptions{operation: opts.operation, mode: opts.completionMode, operations: operations, callbacks: opts.callbacks} + return d.write(ctx, write) } func (d *Dataset) validate(operation string) error { @@ -224,14 +250,31 @@ func (d *Dataset) encodeRecord(record Record) (Address, Document, error) { return address, document, nil } -func (d *Dataset) write( - ctx context.Context, - operation string, - completionMode CompletionMode, - operations []WriteOperation, -) ([]WriteResult, error) { - results, err := d.client.Write(ctx, completionMode, operations...) - resultsErr := WriteResultsError(results) +type datasetWriteOptions struct { + operation string + mode CompletionMode + operations []WriteOperation + callbacks []WriteCallback +} + +func (d *Dataset) write(ctx context.Context, opts datasetWriteOptions) ([]WriteResult, error) { + operation := opts.operation + callback, err := oneCallback(opts.callbacks) + if err != nil { + return nil, err + } + var failures []*OperationError + var onResult WriteCallback + if callback != nil { + onResult = func(result WriteResult) error { + if result.Failure != nil { + failures = append(failures, result.Failure) + } + return callback(result) + } + } + results, err := d.client.Write(ctx, opts.mode, opts.operations, onResult) + resultsErr := errors.Join(WriteResultsError(results), newBatchError(failures)) if err != nil { if resultsErr != nil { err = errors.Join(err, resultsErr) diff --git a/dataset_native.go b/dataset_native.go index 05a206f..d224845 100644 --- a/dataset_native.go +++ b/dataset_native.go @@ -40,14 +40,14 @@ func (d *Dataset) Execute(ctx context.Context, req ExecuteRequest) (ExecuteRespo // Query fetches a page from this Dataset. An empty Command selects all records. // The Store selects its default query. Query controls and native payload // semantics are the same as Client.Query. -func (d *Dataset) Query(ctx context.Context, req QueryRequest) (QueryResponse, error) { +func (d *Dataset) Query(ctx context.Context, req QueryRequest, callbacks ...DocumentCallback) (QueryResponse, error) { command, err := d.bindNativeCommand(req.Command) if err != nil { var empty QueryResponse return empty, err } req.Command = command - return d.client.Query(ctx, req) + return d.client.Query(ctx, req, callbacks...) } // Count counts matches in this Dataset. An empty Command counts all records; @@ -64,14 +64,14 @@ func (d *Dataset) Count(ctx context.Context, req CountRequest) (CountResponse, e // Scan returns one live page scoped to this Dataset. Reuse the request with // NextCursor to continue. JSON commands must provide a unique stable sort. -func (d *Dataset) Scan(ctx context.Context, req ScanRequest) (ScanResponse, error) { +func (d *Dataset) Scan(ctx context.Context, req ScanRequest, callbacks ...DocumentCallback) (ScanResponse, error) { command, err := d.bindNativeCommand(req.Command) if err != nil { var empty ScanResponse return empty, err } req.Command = command - return d.client.Scan(ctx, req) + return d.client.Scan(ctx, req, callbacks...) } func (d *Dataset) bindNativeCommand(command Command) (Command, error) { diff --git a/dataset_native_test.go b/dataset_native_test.go index 2655ecb..587d4f9 100644 --- a/dataset_native_test.go +++ b/dataset_native_test.go @@ -3,6 +3,7 @@ package sink_test import ( "bytes" "context" + "google.golang.org/grpc" "net/http" "testing" "time" @@ -26,7 +27,7 @@ func (s *datasetNativeServer) Execute(_ context.Context, req *sinkv1.ExecuteRequ return response, nil } -func (s *datasetNativeServer) Scan(_ context.Context, req *sinkv1.ScanRequest) (*sinkv1.ScanResponse, error) { +func (s *datasetNativeServer) scanResponse(_ context.Context, req *sinkv1.ScanRequest) (*sinkv1.ScanResponse, error) { s.scans <- req document := &sinkv1.Document{Encoding: sinkv1.DocumentEncoding_DOCUMENT_ENCODING_JSON, Payload: []byte(`{"number":1}`)} response := &sinkv1.ScanResponse{Documents: []*sinkv1.Document{document}} @@ -209,3 +210,21 @@ func TestDatasetNativePreservesExplicitEncodingAndStoreDefinedOperation(t *testi } } } + +func (s *datasetNativeServer) Scan(req *sinkv1.ScanRequest, stream grpc.ServerStreamingServer[sinkv1.ScanResponse]) error { + response, err := s.scanResponse(stream.Context(), req) + if err != nil { + return err + } + if response == nil { + return nil + } + for _, document := range response.Documents { + frame := &sinkv1.ScanResponse{Documents: []*sinkv1.Document{document}} + if err := stream.Send(frame); err != nil { + return err + } + } + final := &sinkv1.ScanResponse{Complete: true, NextCursor: response.NextCursor} + return stream.Send(final) +} diff --git a/dataset_test.go b/dataset_test.go index 548cec6..76d2d0a 100644 --- a/dataset_test.go +++ b/dataset_test.go @@ -76,15 +76,15 @@ func TestNewDatasetValidatesAndCopiesOptions(t *testing.T) { Key: sink.StringKey("product-1"), Value: datasetProduct{UID: "product-1", Name: "keyboard"}, } - _, err = nilDataset.Upsert(t.Context(), sink.CompletionWaitUntilApplied, record) + _, err = nilDataset.Upsert(t.Context(), sink.CompletionWaitUntilApplied, []sink.Record{record}) if err == nil { t.Fatal("Dataset.Upsert() accepted a nil dataset") } - _, err = nilDataset.Read(t.Context(), record.Key) + _, err = nilDataset.Read(t.Context(), []sink.Key{record.Key}) if err == nil { t.Fatal("Dataset.Read() accepted a nil dataset") } - _, err = dataset.Upsert(t.Context(), 0, record) + _, err = dataset.Upsert(t.Context(), 0, []sink.Record{record}) if err == nil { t.Fatal("Dataset.Upsert() accepted an invalid completion mode") } @@ -106,7 +106,7 @@ func TestDatasetReadBindsRoutingSplitsBatchesAndPreservesOrder(t *testing.T) { sink.StringKey("product-4"), sink.StringKey("product-5"), } - results, err := dataset.Read(t.Context(), keys...) + results, err := dataset.Read(t.Context(), keys) if err != nil { t.Fatalf("Dataset.Read() error = %v", err) } @@ -148,7 +148,7 @@ func TestDatasetReadTreatsNotFoundAsSuccess(t *testing.T) { if err != nil { t.Fatalf("NewDataset() error = %v", err) } - results, err := dataset.Read(t.Context(), sink.StringKey("missing")) + results, err := dataset.Read(t.Context(), []sink.Key{sink.StringKey("missing")}) if err != nil { t.Fatalf("Dataset.Read() error = %v", err) } @@ -174,9 +174,8 @@ func TestDatasetReadCollectsFailuresAfterRetries(t *testing.T) { } results, err := dataset.Read( t.Context(), - sink.StringKey("unavailable"), - sink.StringKey("available"), - ) + []sink.Key{sink.StringKey("unavailable"), + sink.StringKey("available")}) var batchError *sink.BatchError if !errors.As(err, &batchError) || len(batchError.Failures) != 1 || batchError.Failures[0].OperationIndex != 0 { @@ -208,10 +207,9 @@ func TestDatasetReadReturnsPartialResultsOnTransportFailure(t *testing.T) { } results, err := dataset.Read( t.Context(), - sink.StringKey("failed"), - sink.StringKey("found"), - sink.StringKey("transport-failure"), - ) + []sink.Key{sink.StringKey("failed"), + sink.StringKey("found"), + sink.StringKey("transport-failure")}) if status.Code(err) != codes.Unavailable { t.Fatalf("Dataset.Read() error code = %s, want Unavailable", status.Code(err)) } @@ -235,7 +233,7 @@ func TestDatasetReadRejectsInvalidKeysBeforeRPC(t *testing.T) { t.Fatalf("NewDataset() error = %v", err) } var emptyKey sink.Key - _, err = dataset.Read(t.Context(), sink.StringKey("valid"), emptyKey) + _, err = dataset.Read(t.Context(), []sink.Key{sink.StringKey("valid"), emptyKey}) if err == nil || !strings.Contains(err.Error(), "dataset read key 1") { t.Fatalf("Dataset.Read() invalid key error = %v", err) } @@ -283,19 +281,19 @@ func TestDatasetPutMethodsBindRoutingEncodingAndPerCallCompletion(t *testing.T) } createProduct := datasetProduct{UID: expectations[0].key, Name: expectations[0].name, Stock: 12} createRecord := sink.Record{Key: sink.StringKey(createProduct.UID), Value: createProduct} - createResults, err := dataset.Create(t.Context(), sink.CompletionWaitUntilApplied, createRecord) + createResults, err := dataset.Create(t.Context(), sink.CompletionWaitUntilApplied, []sink.Record{createRecord}) if err != nil { t.Fatalf("Dataset.Create() error = %v", err) } replaceProduct := datasetProduct{UID: expectations[1].key, Name: expectations[1].name, Stock: 12} replaceRecord := sink.Record{Key: sink.StringKey(replaceProduct.UID), Value: replaceProduct} - replaceResults, err := dataset.Replace(t.Context(), sink.CompletionWaitUntilVisible, replaceRecord) + replaceResults, err := dataset.Replace(t.Context(), sink.CompletionWaitUntilVisible, []sink.Record{replaceRecord}) if err != nil { t.Fatalf("Dataset.Replace() error = %v", err) } upsertProduct := datasetProduct{UID: expectations[2].key, Name: expectations[2].name, Stock: 12} upsertRecord := sink.Record{Key: sink.StringKey(upsertProduct.UID), Value: upsertProduct} - upsertResults, err := dataset.Upsert(t.Context(), sink.CompletionReturnAfterAccepted, upsertRecord) + upsertResults, err := dataset.Upsert(t.Context(), sink.CompletionReturnAfterAccepted, []sink.Record{upsertRecord}) if err != nil { t.Fatalf("Dataset.Upsert() error = %v", err) } @@ -353,7 +351,7 @@ func TestDatasetUsesConfiguredBSONEncoding(t *testing.T) { } product := datasetProduct{UID: "product-bson", Name: "keyboard", Stock: 4} record := sink.Record{Key: sink.StringKey(product.UID), Value: product} - _, err = dataset.Upsert(t.Context(), sink.CompletionWaitUntilApplied, record) + _, err = dataset.Upsert(t.Context(), sink.CompletionWaitUntilApplied, []sink.Record{record}) if err != nil { t.Fatalf("Dataset.Upsert() error = %v", err) } @@ -393,7 +391,7 @@ func TestDatasetMergeUsesOneBoundProgramForBatch(t *testing.T) { results, err := dataset.Merge( t.Context(), sink.CompletionWaitUntilVisible, - records..., + records, ) if err != nil { t.Fatalf("Dataset.Merge() error = %v", err) @@ -437,8 +435,7 @@ func TestDatasetMergeUsesOneBoundProgramForBatch(t *testing.T) { _, err = putOnlyDataset.Merge( t.Context(), sink.CompletionWaitUntilApplied, - records[0], - ) + []sink.Record{records[0]}) if err == nil || !strings.Contains(err.Error(), "merge program is not configured") { t.Fatalf("Dataset.Merge() without program error = %v", err) } @@ -459,7 +456,7 @@ func TestDatasetUpsertSplitsBatchesAndCollectsOperationFailures(t *testing.T) { product := datasetProduct{UID: uid, Stock: index} records[index] = sink.Record{Key: sink.StringKey(uid), Value: product} } - results, err := dataset.Upsert(t.Context(), sink.CompletionReturnAfterAccepted, records...) + results, err := dataset.Upsert(t.Context(), sink.CompletionReturnAfterAccepted, records) if len(results) != len(records) { t.Fatalf("Dataset.Upsert() results = %d, want %d", len(results), len(records)) } @@ -508,7 +505,7 @@ func TestDatasetUpsertReturnsPartialResultsOnTransportFailure(t *testing.T) { {Key: sink.StringKey("product-2"), Value: datasetProduct{UID: "product-2"}}, {Key: sink.StringKey("product-3"), Value: datasetProduct{UID: "product-3"}}, } - results, err := dataset.Upsert(t.Context(), sink.CompletionWaitUntilApplied, records...) + results, err := dataset.Upsert(t.Context(), sink.CompletionWaitUntilApplied, records) if status.Code(err) != codes.Unavailable { t.Fatalf("Dataset.Upsert() error code = %s, want Unavailable", status.Code(err)) } @@ -537,7 +534,7 @@ func TestDatasetRejectsInvalidRecordsBeforeWrite(t *testing.T) { } var emptyKey sink.Key record := sink.Record{Key: emptyKey, Value: datasetProduct{UID: "product-1"}} - _, err = dataset.Upsert(t.Context(), sink.CompletionWaitUntilApplied, record) + _, err = dataset.Upsert(t.Context(), sink.CompletionWaitUntilApplied, []sink.Record{record}) if err == nil || !strings.Contains(err.Error(), "dataset upsert record 0") { t.Fatalf("Dataset.Upsert() invalid record error = %v", err) } diff --git a/dns_balancing_integration_test.go b/dns_balancing_integration_test.go index b7338b8..76c564c 100644 --- a/dns_balancing_integration_test.go +++ b/dns_balancing_integration_test.go @@ -95,7 +95,7 @@ func testDNSScaleChanges(t *testing.T, opts sink.DialOptions) { t.Helper() attempts++ record := sink.Record{Key: sink.StringKey(fmt.Sprintf("record-%d", attempts)), Value: map[string]int{"value": attempts}} - _, err := dataset.Upsert(ctx, sink.CompletionWaitUntilApplied, record) + _, err := dataset.Upsert(ctx, sink.CompletionWaitUntilApplied, []sink.Record{record}) if err != nil { t.Fatalf("write during DNS scaling: %v", err) } diff --git a/integration_test.go b/integration_test.go index 271a2bd..8afd982 100644 --- a/integration_test.go +++ b/integration_test.go @@ -69,19 +69,19 @@ func TestSinkCompatibility(t *testing.T) { } syncValue := integrationValueFor("sync", "applied") syncRecord := sink.Record{Key: syncKey, Value: syncValue} - writeResults, err := dataset.Upsert(ctx, sink.CompletionWaitUntilVisible, syncRecord) + writeResults, err := dataset.Upsert(ctx, sink.CompletionWaitUntilVisible, []sink.Record{syncRecord}) if err != nil { t.Fatalf("Dataset.Upsert(sync) error = %v", err) } assertWriteStatus(t, writeResults, sink.WriteApplied) - readResults, err := dataset.Read(ctx, syncKey) + readResults, err := dataset.Read(ctx, []sink.Key{syncKey}) if err != nil { t.Fatalf("Dataset.Read(sync) error = %v", err) } assertReadDocument(t, readResults, "sync", "applied") - writeResults, err = dataset.Create(ctx, sink.CompletionWaitUntilApplied, syncRecord) + writeResults, err = dataset.Create(ctx, sink.CompletionWaitUntilApplied, []sink.Record{syncRecord}) if err == nil { t.Fatal("Dataset.Create(duplicate) succeeded") } @@ -90,8 +90,7 @@ func TestSinkCompatibility(t *testing.T) { writeResults, err = dataset.Merge( ctx, sink.CompletionWaitUntilVisible, - syncRecord, - ) + []sink.Record{syncRecord}) if err != nil { t.Fatalf("Dataset.Merge() error = %v", err) } @@ -99,7 +98,7 @@ func TestSinkCompatibility(t *testing.T) { asyncValue := integrationValueFor("async", "accepted") asyncRecord := sink.Record{Key: asyncKey, Value: asyncValue} - writeResults, err = dataset.Upsert(ctx, sink.CompletionReturnAfterAccepted, asyncRecord) + writeResults, err = dataset.Upsert(ctx, sink.CompletionReturnAfterAccepted, []sink.Record{asyncRecord}) if err != nil { t.Fatalf("Dataset.Upsert(async) results=%+v error=%v", writeResults, err) } @@ -128,7 +127,7 @@ func TestSinkCompatibility(t *testing.T) { t.Fatalf("Delete() result %d = %+v", index, result) } } - readResults, err = dataset.Read(ctx, syncKey, asyncKey) + readResults, err = dataset.Read(ctx, []sink.Key{syncKey, asyncKey}) if err != nil { t.Fatalf("Dataset.Read(after delete) error = %v", err) } @@ -194,7 +193,7 @@ func waitForDocument( ticker := time.NewTicker(100 * time.Millisecond) defer ticker.Stop() for { - results, err := opts.Dataset.Read(ctx, opts.Key) + results, err := opts.Dataset.Read(ctx, []sink.Key{opts.Key}) if err == nil && len(results) == 1 && results[0].Status == sink.ReadFound { var document integrationValue decodeErr := results[0].Document.Decode(&document) diff --git a/native.go b/native.go index c7f278f..47b0de8 100644 --- a/native.go +++ b/native.go @@ -6,6 +6,7 @@ import ( "encoding/json" "errors" "fmt" + "io" "mime" "net/http" "reflect" @@ -196,7 +197,9 @@ type ScanResponse struct { // the same request and cursor within ScanRetry and ScanTimeout. Checkpointing // and idempotent processing belong to the caller. Cancellation of one request // does not invalidate an existing cursor. -func (c *Client) Scan(ctx context.Context, req ScanRequest) (ScanResponse, error) { +// An optional callback receives documents without collecting them in Documents. +// NextCursor is returned only after the stream finishes successfully. +func (c *Client) Scan(ctx context.Context, req ScanRequest, callbacks ...DocumentCallback) (ScanResponse, error) { var empty ScanResponse if c == nil || c.rpc == nil { return empty, errors.New("scan requires a client") @@ -221,43 +224,94 @@ func (c *Client) Scan(ctx context.Context, req ScanRequest) (ScanResponse, error ctx, cancel = context.WithTimeout(ctx, c.config.scanTimeout) defer cancel() } - var response *sinkv1.ScanResponse + callback, err := oneCallback(callbacks) + if err != nil { + return empty, err + } + limit := req.BatchSize + if limit == 0 { + limit = 100 + } + result := ScanResponse{} backoff := c.config.scanRetry.InitialBackoff for attempt := 1; attempt <= c.config.scanRetry.MaxAttempts; attempt++ { - if err := ctx.Err(); err != nil { - return empty, fmt.Errorf("scan page: %w", status.FromContextError(err).Err()) + received := 0 + call := scanPageCall{request: request, limit: limit} + call.emit = func(document Document) error { + received++ + if callback != nil { + return callback(document) + } + result.Documents = append(result.Documents, document) + return nil + } + cursor, err := c.scanPage(ctx, call) + if err == nil { + result.NextCursor = cursor + return result, nil } - response, err = c.rpc.Scan(ctx, request, c.config.sinkCallOptions...) - if err == nil || attempt == c.config.scanRetry.MaxAttempts || !retryableScanAdmission(err) { - break + if received > 0 || attempt == c.config.scanRetry.MaxAttempts || !retryableScanAdmission(err) { + return result, fmt.Errorf("scan page: %w", err) } if err := waitForBackoff(ctx, jitteredBackoff(backoff, c.config.scanRetry.Jitter)); err != nil { - return empty, fmt.Errorf("scan page: %w", status.FromContextError(err).Err()) + return result, fmt.Errorf("scan page: %w", status.FromContextError(err).Err()) } backoff = nextBackoff(backoff, c.config.scanRetry) } - if err != nil { - return empty, fmt.Errorf("scan page: %w", err) - } - if response == nil { - return empty, protocolError("Scan", "response is empty") - } - limit := req.BatchSize - if limit == 0 { - limit = 100 - } - if len(response.Documents) > limit || len(response.NextCursor) > 64<<10 || (len(response.NextCursor) > 0 && len(response.Documents) == 0) { - return empty, protocolError("Scan", "invalid page or continuation cursor") + return result, nil +} + +type scanPageCall struct { + request *sinkv1.ScanRequest + limit int + emit DocumentCallback +} + +func (c *Client) scanPage(ctx context.Context, call scanPageCall) ([]byte, error) { + ctx, cancel := context.WithCancel(ctx) + defer cancel() + if err := ctx.Err(); err != nil { + return nil, status.FromContextError(err).Err() } - result := ScanResponse{NextCursor: bytes.Clone(response.NextCursor)} - for _, raw := range response.Documents { - document, err := documentFromProto(raw) + stream, err := c.rpc.Scan(ctx, call.request, c.config.sinkCallOptions...) + if err != nil { + return nil, err + } + var cursor []byte + complete, count := false, 0 + for { + frame, err := stream.Recv() + if err == io.EOF { + if !complete { + return nil, protocolError("Scan", "stream omitted completion") + } + return cursor, nil + } if err != nil { - return empty, protocolError("Scan", err.Error()) + return nil, err + } + if frame == nil || complete { + return nil, protocolError("Scan", "unexpected frame after completion") + } + if frame.GetComplete() { + if len(frame.Documents) != 0 || len(frame.NextCursor) > 64<<10 || (len(frame.NextCursor) > 0 && count == 0) { + return nil, protocolError("Scan", "invalid page completion") + } + complete, cursor = true, bytes.Clone(frame.NextCursor) + continue + } + if len(frame.Documents) != 1 || len(frame.NextCursor) != 0 || count >= call.limit { + return nil, protocolError("Scan", "invalid document frame") + } + document, err := documentFromProto(frame.Documents[0]) + if err != nil { + return nil, protocolError("Scan", err.Error()) + } + count++ + if err := call.emit(document); err != nil { + return nil, err } - result.Documents = append(result.Documents, document) } - return result, nil } func retryableScanAdmission(err error) bool { diff --git a/native_integration_test.go b/native_integration_test.go index ae2ee29..a9de9ee 100644 --- a/native_integration_test.go +++ b/native_integration_test.go @@ -70,7 +70,7 @@ end`)) UpdatedAt time.Time `bson:"updated_at"` }{Count: 1, UpdatedAt: integrationDateTime()} record := sink.Record{Key: key, Value: value, ReturnDocument: true} - results, err := dataset.Merge(ctx, sink.CompletionWaitUntilApplied, record, record) + results, err := dataset.Merge(ctx, sink.CompletionWaitUntilApplied, []sink.Record{record, record}) if err != nil || len(results) != 2 { t.Fatalf("returned Merge: %+v %v", results, err) } diff --git a/native_test.go b/native_test.go index 4e01649..b9b3dad 100644 --- a/native_test.go +++ b/native_test.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "errors" + "google.golang.org/grpc" "net/http" "sync/atomic" "testing" @@ -47,7 +48,7 @@ func (s *nativeRPCServer) Execute(_ context.Context, req *sinkv1.ExecuteRequest) return response, nil } -func (s *nativeRPCServer) Scan(ctx context.Context, req *sinkv1.ScanRequest) (*sinkv1.ScanResponse, error) { +func (s *nativeRPCServer) scanResponse(ctx context.Context, req *sinkv1.ScanRequest) (*sinkv1.ScanResponse, error) { call := s.scanCalls.Add(1) if s.scanRequests != nil { s.scanRequests <- req @@ -73,7 +74,7 @@ func (s *nativeRPCServer) Scan(ctx context.Context, req *sinkv1.ScanRequest) (*s return page, nil } -func (s *nativeRPCServer) Write(_ context.Context, req *sinkv1.WriteRequest) (*sinkv1.WriteResponse, error) { +func (s *nativeRPCServer) writeResponse(_ context.Context, req *sinkv1.WriteRequest) (*sinkv1.WriteResponse, error) { s.writeCalls.Add(1) response := &sinkv1.WriteResponse{} for index, operation := range req.GetOperations() { @@ -219,7 +220,7 @@ func TestReturnedDocumentsReachDatasetAndRejectAsync(t *testing.T) { t.Fatal(err) } record := sink.Record{Key: sink.StringKey("1"), Value: map[string]int{"count": 1}, ReturnDocument: true} - results, err := dataset.Upsert(t.Context(), sink.CompletionWaitUntilApplied, record) + results, err := dataset.Upsert(t.Context(), sink.CompletionWaitUntilApplied, []sink.Record{record}) if err != nil || len(results) != 1 { t.Fatalf("results=%v err=%v", results, err) } @@ -227,7 +228,7 @@ func TestReturnedDocumentsReachDatasetAndRejectAsync(t *testing.T) { if err := results[0].Document.Decode(&returned); err != nil || returned["count"] != 1 { t.Fatalf("document=%v err=%v", returned, err) } - _, err = dataset.Upsert(t.Context(), sink.CompletionReturnAfterAccepted, record) + _, err = dataset.Upsert(t.Context(), sink.CompletionReturnAfterAccepted, []sink.Record{record}) if err == nil || server.writeCalls.Load() != 1 { t.Fatalf("async returning request sent: calls=%d err=%v", server.writeCalls.Load(), err) } @@ -244,9 +245,44 @@ func TestMissingRequestedDocumentIsProtocolFailure(t *testing.T) { t.Fatal(err) } operation = operation.WithReturnedDocument() - _, err = client.Write(t.Context(), sink.CompletionWaitUntilApplied, operation) + _, err = client.Write(t.Context(), sink.CompletionWaitUntilApplied, []sink.WriteOperation{operation}) var protocolErr *sink.ProtocolError if !errors.As(err, &protocolErr) { t.Fatalf("missing requested document error=%v", err) } } + +func (s *nativeRPCServer) Scan(req *sinkv1.ScanRequest, stream grpc.ServerStreamingServer[sinkv1.ScanResponse]) error { + response, err := s.scanResponse(stream.Context(), req) + if err != nil { + return err + } + if response == nil { + return nil + } + for _, document := range response.Documents { + frame := &sinkv1.ScanResponse{Documents: []*sinkv1.Document{document}} + if err := stream.Send(frame); err != nil { + return err + } + } + final := &sinkv1.ScanResponse{Complete: true, NextCursor: response.NextCursor} + return stream.Send(final) +} + +func (s *nativeRPCServer) Write(req *sinkv1.WriteRequest, stream grpc.ServerStreamingServer[sinkv1.WriteResponse]) error { + response, err := s.writeResponse(stream.Context(), req) + if err != nil { + return err + } + if response == nil { + return nil + } + for _, result := range response.Results { + frame := &sinkv1.WriteResponse{Results: []*sinkv1.WriteResult{result}} + if err := stream.Send(frame); err != nil { + return err + } + } + return nil +} diff --git a/proto/sink/sink.proto b/proto/sink/sink.proto index 93a9aab..f3ede88 100644 --- a/proto/sink/sink.proto +++ b/proto/sink/sink.proto @@ -9,13 +9,13 @@ option go_package = "github.com/liran/sink-go/api/sink/v1;sinkv1"; // Record requests are batch-native. Atomicity is guaranteed per record, never // for the whole request. Native queries use Execute, Query, Count or paged Scan. service Sink { - rpc Read(ReadRequest) returns (ReadResponse); - rpc Write(WriteRequest) returns (WriteResponse); + rpc Read(ReadRequest) returns (stream ReadResponse); + rpc Write(WriteRequest) returns (stream WriteResponse); rpc Delete(DeleteRequest) returns (DeleteResponse); rpc Execute(ExecuteRequest) returns (ExecuteResponse); - rpc Query(QueryRequest) returns (QueryResponse); + rpc Query(QueryRequest) returns (stream QueryResponse); rpc Count(CountRequest) returns (CountResponse); - rpc Scan(ScanRequest) returns (ScanResponse); + rpc Scan(ScanRequest) returns (stream ScanResponse); } // RecordAddress is a canonical sink:/// URI. The core treats the @@ -68,8 +68,8 @@ message ReadOperation { } message ReadResponse { - // Results are returned in request order. operation_index is included to make - // correlation explicit when an implementation executes records in parallel. + // Each frame contains one result. Results may arrive out of order; correlate + // by operation_index. Successful EOF requires exactly one result per operation. repeated ReadResult results = 1; } @@ -147,6 +147,8 @@ message LuaProgram { bytes sha256 = 2; } +// Each frame contains one final result, identified by operation_index. +// A disconnected stream does not establish the outcome of missing operations. message WriteResponse { repeated WriteResult results = 1; } @@ -326,7 +328,10 @@ message Projection { bool exclude = 2; // False includes listed fields; true excludes them. } +// Document frames contain one document. A final, document-free frame sets +// complete and has_more. Metadata is valid only after successful EOF. message QueryResponse { + bool complete = 3; repeated Document documents = 1; // Determined by fetching one extra result, not by running Count. Result-window @@ -376,12 +381,15 @@ message ScanRequest { } // Documents retain native encodings: BSON documents or complete JSON hits. -// Failed requests return no page; callers may retry from their last saved -// cursor, with live-query semantics and idempotent business processing. +// A failed stream may have delivered documents but never a committed page. +// Resume from the last saved cursor with idempotent business processing. +// Document frames contain one document. A final, document-free frame sets +// complete and next_cursor. Checkpoint only after successful EOF. message ScanResponse { + bool complete = 3; repeated Document documents = 1; // Empty means the scan reached the end observed by this request. Commit a - // checkpoint only after processing documents. No explicit close is needed. + // checkpoint only after processing documents and successful EOF. No close RPC is needed. bytes next_cursor = 2; } diff --git a/query.go b/query.go index 2f2cbcb..129b0bd 100644 --- a/query.go +++ b/query.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "io" "strings" "unicode/utf8" @@ -39,7 +40,9 @@ type QueryResponse struct { // Use a stable native sort with a unique tie-breaker. Concurrent writes can shift // pages; deep pages are subject to backend offset costs and result-window limits. // HasMore uses one extra result. Query never automatically runs Count or retries. -func (c *Client) Query(ctx context.Context, req QueryRequest) (QueryResponse, error) { +// An optional callback receives documents without collecting them in Documents. +// HasMore is valid only when the stream finishes successfully. +func (c *Client) Query(ctx context.Context, req QueryRequest, callbacks ...DocumentCallback) (QueryResponse, error) { var empty QueryResponse if c == nil || c.rpc == nil { return empty, errors.New("query requires a client") @@ -65,29 +68,60 @@ func (c *Client) Query(ctx context.Context, req QueryRequest) (QueryResponse, er if err != nil { return empty, err } - response, err := c.rpc.Query(ctx, request, c.config.sinkCallOptions...) + callback, err := oneCallback(callbacks) if err != nil { - return empty, fmt.Errorf("query native page: %w", err) + return empty, err } - if response == nil { - return empty, protocolError("Query", "response is empty") + ctx, cancel := context.WithCancel(ctx) + defer cancel() + stream, err := c.rpc.Query(ctx, request, c.config.sinkCallOptions...) + if err != nil { + return empty, fmt.Errorf("query native page: %w", err) } pageSize := req.PageSize if pageSize == 0 { pageSize = 100 } - if len(response.GetDocuments()) > pageSize || (response.GetHasMore() && len(response.GetDocuments()) != pageSize) { - return empty, protocolError("Query", "invalid page length or has_more") - } - result := QueryResponse{HasMore: response.GetHasMore()} - for _, raw := range response.GetDocuments() { - document, err := documentFromProto(raw) + result := QueryResponse{} + complete, hasMore, count := false, false, 0 + for { + frame, err := stream.Recv() + if err == io.EOF { + if !complete { + return result, protocolError("Query", "stream omitted completion") + } + result.HasMore = hasMore + return result, nil + } if err != nil { - return empty, protocolError("Query", err.Error()) + return result, fmt.Errorf("query native page: %w", err) + } + if frame == nil || complete { + return result, protocolError("Query", "unexpected frame after completion") + } + if frame.GetComplete() { + if len(frame.Documents) != 0 || (frame.GetHasMore() && count != pageSize) { + return result, protocolError("Query", "invalid page completion") + } + complete, hasMore = true, frame.GetHasMore() + continue + } + if len(frame.Documents) != 1 || frame.GetHasMore() || count >= pageSize { + return result, protocolError("Query", "invalid document frame") + } + document, err := documentFromProto(frame.Documents[0]) + if err != nil { + return result, protocolError("Query", err.Error()) + } + count++ + if callback != nil { + if err := callback(document); err != nil { + return result, err + } + } else { + result.Documents = append(result.Documents, document) } - result.Documents = append(result.Documents, document) } - return result, nil } type CountRequest struct { diff --git a/query_test.go b/query_test.go index 5b70400..99030e5 100644 --- a/query_test.go +++ b/query_test.go @@ -3,6 +3,7 @@ package sink_test import ( "context" "errors" + "google.golang.org/grpc" "sync/atomic" "testing" @@ -23,7 +24,7 @@ type queryRPCServer struct { estimated bool } -func (s *queryRPCServer) Query(_ context.Context, req *sinkv1.QueryRequest) (*sinkv1.QueryResponse, error) { +func (s *queryRPCServer) queryResponse(_ context.Context, req *sinkv1.QueryRequest) (*sinkv1.QueryResponse, error) { s.queryCalls.Add(1) if s.fail { return nil, status.Error(codes.Unavailable, "connection lost") @@ -114,3 +115,21 @@ func TestQueryAndCountDoNotRetryTransportFailures(t *testing.T) { t.Fatalf("count retried: %d %v", server.countCalls.Load(), err) } } + +func (s *queryRPCServer) Query(req *sinkv1.QueryRequest, stream grpc.ServerStreamingServer[sinkv1.QueryResponse]) error { + response, err := s.queryResponse(stream.Context(), req) + if err != nil { + return err + } + if response == nil { + return nil + } + for _, document := range response.Documents { + frame := &sinkv1.QueryResponse{Documents: []*sinkv1.Document{document}} + if err := stream.Send(frame); err != nil { + return err + } + } + final := &sinkv1.QueryResponse{Complete: true, HasMore: response.HasMore} + return stream.Send(final) +} diff --git a/results.go b/results.go index fe74d69..4906f27 100644 --- a/results.go +++ b/results.go @@ -6,97 +6,67 @@ import ( sinkv1 "github.com/liran/sink-go/api/sink/v1" ) -func decodeReadResponse(response *sinkv1.ReadResponse, count int) ([]ReadResult, error) { - if response == nil { - return nil, protocolError("Read", "response is empty") - } - if len(response.GetResults()) != count { - message := fmt.Sprintf("returned %d results for %d operations", len(response.GetResults()), count) - return nil, protocolError("Read", message) +func decodeReadResult(protoResult *sinkv1.ReadResult, index int) (ReadResult, error) { + var empty ReadResult + if protoResult == nil { + return empty, protocolError("Read", "result is empty") } - results := make([]ReadResult, count) - seen := make([]bool, count) - for _, protoResult := range response.GetResults() { - if protoResult == nil { - return nil, protocolError("Read", "result is empty") + result := ReadResult{OperationIndex: index, Status: protoResult.GetStatus()} + switch result.Status { + case ReadFound: + document, documentErr := documentFromProto(protoResult.GetDocument()) + if documentErr != nil { + return empty, protocolError("Read", fmt.Sprintf("result %d: %v", index, documentErr)) } - index, err := validateResultIndex("Read", protoResult.GetOperationIndex(), seen) - if err != nil { - return nil, err + result.Document = document + result.Revision = revisionFromProto(protoResult.GetRevision()) + case ReadNotFound: + case ReadFailed: + failure, failureErr := operationFailure(index, protoResult.GetFailure()) + if failureErr != nil { + return empty, protocolError("Read", failureErr.Error()) } - result := ReadResult{OperationIndex: index, Status: protoResult.GetStatus()} - switch result.Status { - case ReadFound: - document, documentErr := documentFromProto(protoResult.GetDocument()) - if documentErr != nil { - return nil, protocolError("Read", fmt.Sprintf("result %d: %v", index, documentErr)) - } - result.Document = document - result.Revision = revisionFromProto(protoResult.GetRevision()) - case ReadNotFound: - case ReadFailed: - failure, failureErr := operationFailure(index, protoResult.GetFailure()) - if failureErr != nil { - return nil, protocolError("Read", failureErr.Error()) - } - result.Failure = failure - default: - message := fmt.Sprintf("result %d has unsupported status %s", index, result.Status) - return nil, protocolError("Read", message) - } - results[index] = result + result.Failure = failure + default: + message := fmt.Sprintf("result %d has unsupported status %s", index, result.Status) + return empty, protocolError("Read", message) } - return results, nil + return result, nil } -func decodeWriteResponse(response *sinkv1.WriteResponse, count int) ([]WriteResult, error) { - if response == nil { - return nil, protocolError("Write", "response is empty") +func decodeWriteResult(protoResult *sinkv1.WriteResult, index int) (WriteResult, error) { + var empty WriteResult + if protoResult == nil { + return empty, protocolError("Write", "result is empty") } - if len(response.GetResults()) != count { - message := fmt.Sprintf("returned %d results for %d operations", len(response.GetResults()), count) - return nil, protocolError("Write", message) + result := WriteResult{ + OperationIndex: index, + Status: protoResult.GetStatus(), + Revision: revisionFromProto(protoResult.GetRevision()), } - results := make([]WriteResult, count) - seen := make([]bool, count) - for _, protoResult := range response.GetResults() { - if protoResult == nil { - return nil, protocolError("Write", "result is empty") + if protoResult.GetDocument() != nil { + if result.Status != WriteApplied { + return empty, protocolError("Write", "uncommitted result contains a document") } - index, err := validateResultIndex("Write", protoResult.GetOperationIndex(), seen) + document, err := documentFromProto(protoResult.GetDocument()) if err != nil { - return nil, err - } - result := WriteResult{ - OperationIndex: index, - Status: protoResult.GetStatus(), - Revision: revisionFromProto(protoResult.GetRevision()), - } - if protoResult.GetDocument() != nil { - if result.Status != WriteApplied { - return nil, protocolError("Write", "uncommitted result contains a document") - } - document, err := documentFromProto(protoResult.GetDocument()) - if err != nil { - return nil, protocolError("Write", err.Error()) - } - result.Document = document + return empty, protocolError("Write", err.Error()) } - switch result.Status { - case WriteAccepted, WriteApplied: - case WritePreconditionFailed, WriteFailed: - failure, failureErr := operationFailure(index, protoResult.GetFailure()) - if failureErr != nil { - return nil, protocolError("Write", failureErr.Error()) - } - result.Failure = failure - default: - message := fmt.Sprintf("result %d has unsupported status %s", index, result.Status) - return nil, protocolError("Write", message) + result.Document = document + } + switch result.Status { + case WriteAccepted, WriteApplied: + case WritePreconditionFailed, WriteFailed: + failure, failureErr := operationFailure(index, protoResult.GetFailure()) + if failureErr != nil { + return empty, protocolError("Write", failureErr.Error()) } - results[index] = result + result.Failure = failure + default: + message := fmt.Sprintf("result %d has unsupported status %s", index, result.Status) + return empty, protocolError("Write", message) } - return results, nil + return result, nil } func decodeDeleteResponse(response *sinkv1.DeleteResponse, count int) ([]DeleteResult, error) { diff --git a/scan_deadline_test.go b/scan_deadline_test.go index ca40e30..118d907 100644 --- a/scan_deadline_test.go +++ b/scan_deadline_test.go @@ -2,6 +2,7 @@ package sink_test import ( "context" + "google.golang.org/grpc" "testing" "time" @@ -14,7 +15,7 @@ type scanDeadlineServer struct { deadlines chan time.Time } -func (s *scanDeadlineServer) Scan(ctx context.Context, _ *sinkv1.ScanRequest) (*sinkv1.ScanResponse, error) { +func (s *scanDeadlineServer) scanResponse(ctx context.Context, _ *sinkv1.ScanRequest) (*sinkv1.ScanResponse, error) { deadline, _ := ctx.Deadline() s.deadlines <- deadline response := &sinkv1.ScanResponse{} @@ -50,3 +51,21 @@ func TestScanDeadlineBelongsToCaller(t *testing.T) { }) } } + +func (s *scanDeadlineServer) Scan(req *sinkv1.ScanRequest, stream grpc.ServerStreamingServer[sinkv1.ScanResponse]) error { + response, err := s.scanResponse(stream.Context(), req) + if err != nil { + return err + } + if response == nil { + return nil + } + for _, document := range response.Documents { + frame := &sinkv1.ScanResponse{Documents: []*sinkv1.Document{document}} + if err := stream.Send(frame); err != nil { + return err + } + } + final := &sinkv1.ScanResponse{Complete: true, NextCursor: response.NextCursor} + return stream.Send(final) +} diff --git a/streaming_test.go b/streaming_test.go new file mode 100644 index 0000000..7245366 --- /dev/null +++ b/streaming_test.go @@ -0,0 +1,162 @@ +package sink_test + +import ( + "errors" + "sync/atomic" + "testing" + "time" + + sink "github.com/liran/sink-go" + sinkv1 "github.com/liran/sink-go/api/sink/v1" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +type partialStreamServer struct { + sinkv1.UnimplementedSinkServer + reads atomic.Int32 + writes atomic.Int32 + wait bool + canceled chan struct{} +} + +func (s *partialStreamServer) Read(req *sinkv1.ReadRequest, stream grpc.ServerStreamingServer[sinkv1.ReadResponse]) error { + call := s.reads.Add(1) + for i := len(req.Operations) - 1; i >= 0; i-- { + document := &sinkv1.Document{Encoding: sinkv1.DocumentEncoding_DOCUMENT_ENCODING_JSON, Payload: []byte(`{"value":1}`)} + result := &sinkv1.ReadResult{OperationIndex: uint32(i), Status: sinkv1.ReadStatus_READ_STATUS_FOUND, Document: document} + frame := &sinkv1.ReadResponse{Results: []*sinkv1.ReadResult{result}} + if err := stream.Send(frame); err != nil { + return err + } + if s.wait { + <-stream.Context().Done() + close(s.canceled) + return stream.Context().Err() + } + if call == 1 { + return status.Error(codes.Unavailable, "interrupted after one result") + } + } + return nil +} +func (s *partialStreamServer) Write(_ *sinkv1.WriteRequest, stream grpc.ServerStreamingServer[sinkv1.WriteResponse]) error { + s.writes.Add(1) + result := &sinkv1.WriteResult{OperationIndex: 1, Status: sinkv1.WriteStatus_WRITE_STATUS_APPLIED} + frame := &sinkv1.WriteResponse{Results: []*sinkv1.WriteResult{result}} + if err := stream.Send(frame); err != nil { + return err + } + return status.Error(codes.Unavailable, "write outcome unknown") +} +func TestStreamingReadRetriesOnlyUndeliveredAndDoesNotCollectCallbacks(t *testing.T) { + server := &partialStreamServer{} + opts := sink.ClientOptions{} + client := startTestClient(t, server, opts) + addresses := []sink.Address{testAddress(t, sink.StringKey("a")), testAddress(t, sink.StringKey("b")), testAddress(t, sink.StringKey("c"))} + seen := make(map[int]int) + results, err := client.Read(t.Context(), addresses, func(result sink.ReadResult) error { seen[result.OperationIndex]++; return nil }) + if err != nil || results != nil || len(seen) != 3 || server.reads.Load() != 2 { + t.Fatalf("results=%v seen=%v calls=%d err=%v", results, seen, server.reads.Load(), err) + } + for _, count := range seen { + if count != 1 { + t.Fatal("callback redelivered a completed read") + } + } + // Collecting uses the same stream, restores request order, and retains payloads. + results, err = client.Read(t.Context(), addresses) + if err != nil || len(results) != 3 { + t.Fatalf("collect: %v %v", results, err) + } + for i, result := range results { + if result.OperationIndex != i { + t.Fatal("collecting changed request order") + } + } +} +func TestCallbackErrorCancelsWithoutRetry(t *testing.T) { + server := &partialStreamServer{wait: true, canceled: make(chan struct{})} + opts := sink.ClientOptions{} + client := startTestClient(t, server, opts) + addresses := []sink.Address{testAddress(t, sink.StringKey("a"))} + stop := status.Error(codes.Unavailable, "callback stopped") + results, err := client.Read(t.Context(), addresses, func(sink.ReadResult) error { return stop }) + if !errors.Is(err, stop) || results != nil || server.reads.Load() != 1 { + t.Fatalf("callback cancellation: %v %v", results, err) + } + select { + case <-server.canceled: + case <-time.After(time.Second): + t.Fatal("callback did not cancel the server stream") + } +} +func TestStreamingWritePreservesPartialResultsAndNeverReplays(t *testing.T) { + for _, callback := range []bool{false, true} { + server := &partialStreamServer{} + opts := sink.ClientOptions{} + client := startTestClient(t, server, opts) + address := testAddress(t, sink.StringKey("a")) + operation, err := sink.NewPut(address, testDocument("x"), sink.WriteUpsert) + if err != nil { + t.Fatal(err) + } + operations := []sink.WriteOperation{operation, operation} + var emit sink.WriteCallback + count := 0 + if callback { + emit = func(result sink.WriteResult) error { + count++ + if result.OperationIndex != 1 { + t.Error("lost global index") + } + return nil + } + } + results, err := client.Write(t.Context(), sink.CompletionWaitUntilApplied, operations, emit) + if status.Code(err) != codes.Unavailable || server.writes.Load() != 1 { + t.Fatalf("write replay/status: %v", err) + } + if callback { + if results != nil || count != 1 { + t.Fatal("callback write retained results") + } + } else if len(results) != 1 || results[0].OperationIndex != 1 { + t.Fatal("lost acknowledged write") + } + } +} + +type incompletePageServer struct { + sinkv1.UnimplementedSinkServer + afterCompletion bool +} + +func (s *incompletePageServer) Scan(_ *sinkv1.ScanRequest, stream grpc.ServerStreamingServer[sinkv1.ScanResponse]) error { + document := &sinkv1.Document{Encoding: sinkv1.DocumentEncoding_DOCUMENT_ENCODING_JSON, Payload: []byte(`{"x":1}`)} + frame := &sinkv1.ScanResponse{Documents: []*sinkv1.Document{document}} + if err := stream.Send(frame); err != nil { + return err + } + if s.afterCompletion { + final := &sinkv1.ScanResponse{Complete: true, NextCursor: []byte("uncommitted-cursor")} + if err := stream.Send(final); err != nil { + return err + } + } + return status.Error(codes.Unavailable, "lost page completion") +} +func TestStreamingScanWithholdsCursorOnFailure(t *testing.T) { + for _, final := range []bool{false, true} { + server := &incompletePageServer{afterCompletion: final} + opts := sink.ClientOptions{} + client := startTestClient(t, server, opts) + request := sink.ScanRequest{Command: sink.Command{URI: "sink://primary/items"}} + count := 0 + response, err := client.Scan(t.Context(), request, func(sink.Document) error { count++; return nil }) + if status.Code(err) != codes.Unavailable || count != 1 || response.Documents != nil || len(response.NextCursor) != 0 { + t.Fatalf("unsafe cursor/collection: %+v %v count=%d", response, err, count) + } + } +} From 1fb53e51ae07cabf35b5e1406bae52d2ee447668 Mon Sep 17 00:00:00 2001 From: Randy <857271370@qq.com> Date: Sun, 20 Sep 2026 16:53:42 +0800 Subject: [PATCH 2/4] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20refactor:=20order=20pr?= =?UTF-8?q?otobuf=20fields=20by=20tag=20number?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- api/sink/v1/sink.pb.go | 78 +++++++++++++++++++++--------------------- proto/sink/sink.proto | 14 ++++---- 2 files changed, 46 insertions(+), 46 deletions(-) diff --git a/api/sink/v1/sink.pb.go b/api/sink/v1/sink.pb.go index 810dd8e..79cd00d 100644 --- a/api/sink/v1/sink.pb.go +++ b/api/sink/v1/sink.pb.go @@ -832,17 +832,17 @@ func (x *WriteRequest) GetLuaPrograms() []*LuaProgram { type WriteOperation struct { state protoimpl.MessageState `protogen:"open.v1"` Address *RecordAddress `protobuf:"bytes,1,opt,name=address,proto3" json:"address,omitempty"` - // Return the logical document submitted by this operation after its commit. - // Only synchronous completion modes support this. These operations do not - // share a folded commit with preceding or following operations. - ReturnDocument bool `protobuf:"varint,4,opt,name=return_document,json=returnDocument,proto3" json:"return_document,omitempty"` // Types that are valid to be assigned to Action: // // *WriteOperation_Put // *WriteOperation_Merge - Action isWriteOperation_Action `protobuf_oneof:"action"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + Action isWriteOperation_Action `protobuf_oneof:"action"` + // Return the logical document submitted by this operation after its commit. + // Only synchronous completion modes support this. These operations do not + // share a folded commit with preceding or following operations. + ReturnDocument bool `protobuf:"varint,4,opt,name=return_document,json=returnDocument,proto3" json:"return_document,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *WriteOperation) Reset() { @@ -882,13 +882,6 @@ func (x *WriteOperation) GetAddress() *RecordAddress { return nil } -func (x *WriteOperation) GetReturnDocument() bool { - if x != nil { - return x.ReturnDocument - } - return false -} - func (x *WriteOperation) GetAction() isWriteOperation_Action { if x != nil { return x.Action @@ -914,6 +907,13 @@ func (x *WriteOperation) GetMerge() *MergeOperation { return nil } +func (x *WriteOperation) GetReturnDocument() bool { + if x != nil { + return x.ReturnDocument + } + return false +} + type isWriteOperation_Action interface { isWriteOperation_Action() } @@ -1977,12 +1977,12 @@ func (x *Projection) GetExclude() bool { // complete and has_more. Metadata is valid only after successful EOF. type QueryResponse struct { state protoimpl.MessageState `protogen:"open.v1"` - Complete bool `protobuf:"varint,3,opt,name=complete,proto3" json:"complete,omitempty"` Documents []*Document `protobuf:"bytes,1,rep,name=documents,proto3" json:"documents,omitempty"` // Determined by fetching one extra result, not by running Count. Result-window // limits apply to this extra result too. A byte limit fails the whole request; // it never produces a silently shortened page. HasMore bool `protobuf:"varint,2,opt,name=has_more,json=hasMore,proto3" json:"has_more,omitempty"` + Complete bool `protobuf:"varint,3,opt,name=complete,proto3" json:"complete,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -2017,13 +2017,6 @@ func (*QueryResponse) Descriptor() ([]byte, []int) { return file_sink_sink_proto_rawDescGZIP(), []int{26} } -func (x *QueryResponse) GetComplete() bool { - if x != nil { - return x.Complete - } - return false -} - func (x *QueryResponse) GetDocuments() []*Document { if x != nil { return x.Documents @@ -2038,6 +2031,13 @@ func (x *QueryResponse) GetHasMore() bool { return false } +func (x *QueryResponse) GetComplete() bool { + if x != nil { + return x.Complete + } + return false +} + // Count is an independent query, not an implicit part of Query. It accepts the // same sources and counts matches before find/HTTP pagination; for aggregate it // counts the output of the supplied pipeline. HTTP counts matching documents @@ -2230,11 +2230,11 @@ func (x *ScanRequest) GetProjection() *Projection { // complete and next_cursor. Checkpoint only after successful EOF. type ScanResponse struct { state protoimpl.MessageState `protogen:"open.v1"` - Complete bool `protobuf:"varint,3,opt,name=complete,proto3" json:"complete,omitempty"` Documents []*Document `protobuf:"bytes,1,rep,name=documents,proto3" json:"documents,omitempty"` // Empty means the scan reached the end observed by this request. Commit a // checkpoint only after processing documents and successful EOF. No close RPC is needed. NextCursor []byte `protobuf:"bytes,2,opt,name=next_cursor,json=nextCursor,proto3" json:"next_cursor,omitempty"` + Complete bool `protobuf:"varint,3,opt,name=complete,proto3" json:"complete,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -2269,13 +2269,6 @@ func (*ScanResponse) Descriptor() ([]byte, []int) { return file_sink_sink_proto_rawDescGZIP(), []int{30} } -func (x *ScanResponse) GetComplete() bool { - if x != nil { - return x.Complete - } - return false -} - func (x *ScanResponse) GetDocuments() []*Document { if x != nil { return x.Documents @@ -2290,6 +2283,13 @@ func (x *ScanResponse) GetNextCursor() []byte { return nil } +func (x *ScanResponse) GetComplete() bool { + if x != nil { + return x.Complete + } + return false +} + var File_sink_sink_proto protoreflect.FileDescriptor const file_sink_sink_proto_rawDesc = "" + @@ -2324,10 +2324,10 @@ const file_sink_sink_proto_rawDesc = "" + "operations\x126\n" + "\flua_programs\x18\x03 \x03(\v2\x13.sink.v1.LuaProgramR\vluaPrograms\"\xd1\x01\n" + "\x0eWriteOperation\x120\n" + - "\aaddress\x18\x01 \x01(\v2\x16.sink.v1.RecordAddressR\aaddress\x12'\n" + - "\x0freturn_document\x18\x04 \x01(\bR\x0ereturnDocument\x12)\n" + + "\aaddress\x18\x01 \x01(\v2\x16.sink.v1.RecordAddressR\aaddress\x12)\n" + "\x03put\x18\x02 \x01(\v2\x15.sink.v1.PutOperationH\x00R\x03put\x12/\n" + - "\x05merge\x18\x03 \x01(\v2\x17.sink.v1.MergeOperationH\x00R\x05mergeB\b\n" + + "\x05merge\x18\x03 \x01(\v2\x17.sink.v1.MergeOperationH\x00R\x05merge\x12'\n" + + "\x0freturn_document\x18\x04 \x01(\bR\x0ereturnDocumentB\b\n" + "\x06action\"e\n" + "\fPutOperation\x12-\n" + "\bdocument\x18\x01 \x01(\v2\x11.sink.v1.DocumentR\bdocument\x12&\n" + @@ -2402,10 +2402,10 @@ const file_sink_sink_proto_rawDesc = "" + "Projection\x12\x16\n" + "\x06fields\x18\x01 \x03(\tR\x06fields\x12\x18\n" + "\aexclude\x18\x02 \x01(\bR\aexclude\"w\n" + - "\rQueryResponse\x12\x1a\n" + - "\bcomplete\x18\x03 \x01(\bR\bcomplete\x12/\n" + + "\rQueryResponse\x12/\n" + "\tdocuments\x18\x01 \x03(\v2\x11.sink.v1.DocumentR\tdocuments\x12\x19\n" + - "\bhas_more\x18\x02 \x01(\bR\ahasMore\":\n" + + "\bhas_more\x18\x02 \x01(\bR\ahasMore\x12\x1a\n" + + "\bcomplete\x18\x03 \x01(\bR\bcomplete\":\n" + "\fCountRequest\x12*\n" + "\acommand\x18\x01 \x01(\v2\x10.sink.v1.CommandR\acommand\"C\n" + "\rCountResponse\x12\x14\n" + @@ -2419,11 +2419,11 @@ const file_sink_sink_proto_rawDesc = "" + "\n" + "projection\x18\x04 \x01(\v2\x13.sink.v1.ProjectionR\n" + "projection\"|\n" + - "\fScanResponse\x12\x1a\n" + - "\bcomplete\x18\x03 \x01(\bR\bcomplete\x12/\n" + + "\fScanResponse\x12/\n" + "\tdocuments\x18\x01 \x03(\v2\x11.sink.v1.DocumentR\tdocuments\x12\x1f\n" + "\vnext_cursor\x18\x02 \x01(\fR\n" + - "nextCursor*m\n" + + "nextCursor\x12\x1a\n" + + "\bcomplete\x18\x03 \x01(\bR\bcomplete*m\n" + "\x10DocumentEncoding\x12!\n" + "\x1dDOCUMENT_ENCODING_UNSPECIFIED\x10\x00\x12\x1a\n" + "\x16DOCUMENT_ENCODING_JSON\x10\x01\x12\x1a\n" + diff --git a/proto/sink/sink.proto b/proto/sink/sink.proto index f3ede88..c2699cc 100644 --- a/proto/sink/sink.proto +++ b/proto/sink/sink.proto @@ -103,15 +103,15 @@ message WriteRequest { message WriteOperation { RecordAddress address = 1; - // Return the logical document submitted by this operation after its commit. - // Only synchronous completion modes support this. These operations do not - // share a folded commit with preceding or following operations. - bool return_document = 4; - oneof action { PutOperation put = 2; MergeOperation merge = 3; } + + // Return the logical document submitted by this operation after its commit. + // Only synchronous completion modes support this. These operations do not + // share a folded commit with preceding or following operations. + bool return_document = 4; } // PutOperation writes a complete user document without interpreting its fields. @@ -331,13 +331,13 @@ message Projection { // Document frames contain one document. A final, document-free frame sets // complete and has_more. Metadata is valid only after successful EOF. message QueryResponse { - bool complete = 3; repeated Document documents = 1; // Determined by fetching one extra result, not by running Count. Result-window // limits apply to this extra result too. A byte limit fails the whole request; // it never produces a silently shortened page. bool has_more = 2; + bool complete = 3; } // Count is an independent query, not an implicit part of Query. It accepts the @@ -386,10 +386,10 @@ message ScanRequest { // Document frames contain one document. A final, document-free frame sets // complete and next_cursor. Checkpoint only after successful EOF. message ScanResponse { - bool complete = 3; repeated Document documents = 1; // Empty means the scan reached the end observed by this request. Commit a // checkpoint only after processing documents and successful EOF. No close RPC is needed. bytes next_cursor = 2; + bool complete = 3; } From 59b2ec143cc2d70f3b3fbce40ae44b9bc60615b6 Mon Sep 17 00:00:00 2001 From: Randy <857271370@qq.com> Date: Sun, 20 Sep 2026 17:14:07 +0800 Subject: [PATCH 3/4] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20refactor!:=20unify=20S?= =?UTF-8?q?DK=20calls=20around=20typed=20requests?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 98 ++++++++++++-------- balancing_test.go | 6 +- callback.go | 13 --- client.go | 34 ++----- client_test.go | 148 ++++++++++++++++++++++++------ dataset.go | 105 +++++++-------------- dataset_native.go | 8 +- dataset_test.go | 105 ++++++++++++++++----- dns_balancing_integration_test.go | 6 +- integration_test.go | 49 +++++++--- native.go | 13 ++- native_integration_test.go | 12 ++- native_test.go | 18 +++- query.go | 13 ++- requests.go | 41 +++++++++ streaming_test.go | 141 +++++++++++++++++++++++++++- 16 files changed, 569 insertions(+), 241 deletions(-) create mode 100644 requests.go diff --git a/README.md b/README.md index 099d591..9f738bd 100644 --- a/README.md +++ b/README.md @@ -96,15 +96,16 @@ func main() { Key: sink.StringKey(value.UID), Value: value, } - _, err = products.Upsert( - context.Background(), - sink.CompletionWaitUntilVisible, - []sink.Record{record}, - ) + writeRequest := sink.DatasetWriteRequest{ + CompletionMode: sink.CompletionWaitUntilVisible, + Records: []sink.Record{record}, + } + _, err = products.Upsert(context.Background(), writeRequest) if err != nil { log.Fatal(err) } - readResults, err := products.Read(context.Background(), []sink.Key{sink.StringKey(value.UID)}) + readRequest := sink.DatasetReadRequest{Keys: []sink.Key{sink.StringKey(value.UID)}} + readResults, err := products.Read(context.Background(), readRequest) if err != nil { log.Fatal(err) } @@ -120,8 +121,8 @@ func main() { `Dataset` binds the stable store, namespace, dataset, and document encoding once. `Read` accepts one or many keys without reconstructing addresses. Each mutation still receives an explicit completion mode because callers of the same dataset -can require different durability or visibility guarantees. Pass a slice of -records to `Create`, `Replace`, or `Upsert`; the client validates and encodes the +can require different durability or visibility guarantees. Pass records in +`DatasetWriteRequest.Records` to `Create`, `Replace`, or `Upsert`; the client validates and encodes the complete collection before sending it, automatically splits large collections by `ClientOptions.MaxOperations`, and preserves global operation indexes: @@ -130,11 +131,11 @@ records := []sink.Record{ {Key: sink.StringKey("product-42"), Value: firstProduct}, {Key: sink.StringKey("product-43"), Value: secondProduct}, } -results, err := products.Upsert( - context.Background(), - sink.CompletionReturnAfterAccepted, - records, -) +request := sink.DatasetWriteRequest{ + CompletionMode: sink.CompletionReturnAfterAccepted, + Records: records, +} +results, err := products.Upsert(context.Background(), request) ``` Configure `DocumentEncodingBSON` for MongoDB; it applies `bson` tags and keeps @@ -176,11 +177,11 @@ record := sink.Record{ Key: sink.StringKey("product-42"), Value: incomingProduct, } -results, err := products.Merge( - context.Background(), - sink.CompletionWaitUntilVisible, - []sink.Record{record}, -) +request := sink.DatasetWriteRequest{ + CompletionMode: sink.CompletionWaitUntilVisible, + Records: []sink.Record{record}, +} +results, err := products.Merge(context.Background(), request) if err != nil { log.Fatal(err) } @@ -199,28 +200,42 @@ for the complete function reference and reliability rules. ## Streaming callbacks Read, Write, Query and Scan use server-streaming RPCs. The previous unary wire -contract has been removed; upgrade server and SDK together. Record collections -are now slices so the final optional argument can be a callback. Omitting the -callback (or passing nil) collects results and preserves request order. Supplying -one calls it serially as items arrive and returns no collected documents: +contract has been removed; upgrade server and SDK together. Each method accepts +`ctx` and a typed request. Record requests use `Addresses`, `Operations`, `Keys` +or `Records` slices; completion mode and callbacks belong to the same request. +Future options can be added as fields without changing method signatures. + +`OnResult` handles Read/Write results and `OnDocument` handles Query/Scan documents. +A nil callback collects and returns results. A non-nil callback processes each +item serially without collecting it: Read/Write return a nil result slice; +Query/Scan return nil `Documents` while retaining page metadata on success. ```go -results, err := client.Read(ctx, addresses) -results, err = client.Read(ctx, addresses, func(result sink.ReadResult) error { - // Process the owned result here. Returning an error cancels the stream. +readRequest := sink.ReadRequest{Addresses: addresses} +results, err := client.Read(ctx, readRequest) // Collected in request order. + +readRequest.OnResult = func(result sink.ReadResult) error { + // Returning an error cancels the stream. return result.Err() -}) // results == nil +} +results, err = client.Read(ctx, readRequest) // results == nil -page, err := client.Scan(ctx, request, func(document sink.Document) error { - return process(document) -}) // page.Documents == nil; NextCursor is available only on success. +scanRequest := sink.ScanRequest{ + Command: command, + OnDocument: func(document sink.Document) error { + return process(document) + }, +} +page, err := client.Scan(ctx, scanRequest) +// page.Documents == nil; NextCursor is available only on success. ``` -Dataset Read/Create/Replace/Upsert/Merge accept a key/record slice and the same -optional result callback. Query and Scan accept an optional document callback. -Different record results can arrive out of order; `OperationIndex` always refers -to the original input slice. No callback documents are retained by the SDK. -Dataset methods retain only failure metadata for `BatchError`. +Dataset Read uses `DatasetReadRequest`; Create/Replace/Upsert/Merge share +`DatasetWriteRequest`. Both expose `OnResult`. Query and Scan share their request +types with Client and expose `OnDocument`. Different record results can arrive +out of order; `OperationIndex` always refers to the original input slice. +No callback documents are retained by the SDK. Dataset methods retain only +failure metadata for `BatchError`. Callbacks can observe partial results before a stream fails. Read retries never redeliver completed results, and callback errors are never retried. Writes are @@ -236,11 +251,11 @@ The default collector returns received documents/results along with any error. `Upsert`, and `Merge` accept one or many `Record` values while keeping completion mode explicit per mutation. Every method splits large collections automatically and collects per-record results unless a callback is supplied. -- `Read(ctx, addresses, callback...)` preserves request order and reports found, +- `Read(ctx, ReadRequest)` preserves request order and reports found, not-found, or failed results independently. -- `Write(ctx, completionMode, operations, callback...)` supports mixed put and merge +- `Write(ctx, WriteRequest)` supports mixed put and merge batches. Use `NewPut` and `NewMerge` to construct validated operations. -- `Delete(ctx, completionMode, addresses...)` performs hard deletes; deleting +- `Delete(ctx, DeleteRequest)` performs hard deletes; deleting an absent record is successful. - `Execute(ctx, request)` returns native BSON or HTTP payloads, status and headers for native queries, writes, and administration; MongoDB cursor/session commands @@ -557,7 +572,8 @@ can observe newer data. Execute never retries automatically. Scan retries only temporary admission rejections explicitly marked by Sink, using the identical command and cursor. Unmarked `ResourceExhausted` (including older servers), backend errors, transport failures and invalid pages are not retried. A failed -Scan returns no page; reuse the last saved cursor and process idempotently. +Scan can return partial documents but no next cursor; reuse the last saved +cursor and process idempotently. Persist task completion separately so a completed task does not restart from an empty cursor. A dataset recreation or remapping requires an explicit new scan. @@ -572,7 +588,11 @@ record := sink.Record{ Value: increment, ReturnDocument: true, } -results, err := quotas.Merge(ctx, sink.CompletionWaitUntilApplied, record) +request := sink.DatasetWriteRequest{ + CompletionMode: sink.CompletionWaitUntilApplied, + Records: []sink.Record{record}, +} +results, err := quotas.Merge(ctx, request) if err != nil { return err } diff --git a/balancing_test.go b/balancing_test.go index 4de30e2..1f8b070 100644 --- a/balancing_test.go +++ b/balancing_test.go @@ -59,7 +59,11 @@ func TestDialBalancesWritesAndFollowsEndpointChanges(t *testing.T) { t.Helper() attempts++ record := sink.Record{Key: sink.StringKey(fmt.Sprintf("record-%d", attempts)), Value: map[string]int{"value": attempts}} - _, err := dataset.Upsert(ctx, sink.CompletionWaitUntilApplied, []sink.Record{record}) + upsertRequest := sink.DatasetWriteRequest{ + CompletionMode: sink.CompletionWaitUntilApplied, + Records: []sink.Record{record}, + } + _, err := dataset.Upsert(ctx, upsertRequest) if err != nil { t.Fatalf("write through resolved backends: %v", err) } diff --git a/callback.go b/callback.go index a2e8a53..517ba25 100644 --- a/callback.go +++ b/callback.go @@ -1,7 +1,5 @@ package sink -import "errors" - // ReadCallback consumes a final read result before the next receive. Returning // an error cancels the RPC. The SDK retains no result documents when a callback // is supplied. Already delivered results remain valid if the stream fails. @@ -15,14 +13,3 @@ type WriteCallback func(WriteResult) error // DocumentCallback consumes a Query or Scan document in query order. // Returning an error cancels the RPC. The SDK does not collect these documents. type DocumentCallback func(Document) error - -func oneCallback[T any](callbacks []T) (T, error) { - var empty T - if len(callbacks) > 1 { - return empty, errors.New("at most one result callback is allowed") - } - if len(callbacks) == 0 { - return empty, nil - } - return callbacks[0], nil -} diff --git a/client.go b/client.go index 9e5ab1d..ccc50dc 100644 --- a/client.go +++ b/client.go @@ -277,7 +277,8 @@ func (c *Client) CheckHealth(ctx context.Context) error { // Unavailable errors and retryable per-operation failures are retried because // reads are idempotent. Returned operation indexes refer to the original // collection. -func (c *Client) Read(ctx context.Context, addresses []Address, callbacks ...ReadCallback) ([]ReadResult, error) { +func (c *Client) Read(ctx context.Context, req ReadRequest) ([]ReadResult, error) { + addresses := req.Addresses if err := c.validateCollection("read", len(addresses)); err != nil { return nil, err } @@ -290,10 +291,6 @@ func (c *Client) Read(ctx context.Context, addresses []Address, callbacks ...Rea operation := &sinkv1.ReadOperation{Address: protoAddress} operations[index] = operation } - callback, err := oneCallback(callbacks) - if err != nil { - return nil, err - } var results []ReadResult defer func() { sort.Slice(results, func(i, j int) bool { return results[i].OperationIndex < results[j].OperationIndex }) @@ -305,8 +302,8 @@ func (c *Client) Read(ctx context.Context, addresses []Address, callbacks ...Rea if result.Failure != nil { result.Failure.OperationIndex += start } - if callback != nil { - return callback(result) + if req.OnResult != nil { + return req.OnResult(result) } results = append(results, result) return nil @@ -325,12 +322,8 @@ func (c *Client) Read(ctx context.Context, addresses []Address, callbacks ...Rea // later batch returns an error. Without a callback, acknowledged results are // collected in request order. With a callback, results arrive incrementally // and the returned result slice is nil. -func (c *Client) Write( - ctx context.Context, - completionMode CompletionMode, - operations []WriteOperation, - callbacks ...WriteCallback, -) ([]WriteResult, error) { +func (c *Client) Write(ctx context.Context, req WriteRequest) ([]WriteResult, error) { + completionMode, operations := req.CompletionMode, req.Operations if err := c.validateCollection("write", len(operations)); err != nil { return nil, err } @@ -345,10 +338,6 @@ func (c *Client) Write( return nil, fmt.Errorf("write operation %d: %w", index, err) } } - callback, err := oneCallback(callbacks) - if err != nil { - return nil, err - } var results []WriteResult defer func() { sort.Slice(results, func(i, j int) bool { return results[i].OperationIndex < results[j].OperationIndex }) @@ -361,8 +350,8 @@ func (c *Client) Write( if result.Failure != nil { result.Failure.OperationIndex += start } - if callback != nil { - return callback(result) + if req.OnResult != nil { + return req.OnResult(result) } results = append(results, result) return nil @@ -450,11 +439,8 @@ func (c *Client) writeBatch(ctx context.Context, call writeBatchCall) error { // Large collections are split automatically, and transport failures are not // retried. Earlier batches may have completed when a later batch returns an // error. -func (c *Client) Delete( - ctx context.Context, - completionMode CompletionMode, - addresses ...Address, -) ([]DeleteResult, error) { +func (c *Client) Delete(ctx context.Context, req DeleteRequest) ([]DeleteResult, error) { + completionMode, addresses := req.CompletionMode, req.Addresses if err := c.validateCollection("delete", len(addresses)); err != nil { return nil, err } diff --git a/client_test.go b/client_test.go index 02cbd4d..811fbad 100644 --- a/client_test.go +++ b/client_test.go @@ -412,7 +412,10 @@ func TestClientCoversSinkContract(t *testing.T) { testAddress(t, opaque), } - readResults, err := client.Read(context.Background(), addresses) + readRequest2 := sink.ReadRequest{ + Addresses: addresses, + } + readResults, err := client.Read(context.Background(), readRequest2) if err != nil { t.Fatalf("Read() error = %v", err) } @@ -446,7 +449,11 @@ func TestClientCoversSinkContract(t *testing.T) { if err != nil { t.Fatalf("sink.NewMerge() error = %v", err) } - writeResults, err := client.Write(context.Background(), sink.CompletionWaitUntilApplied, []sink.WriteOperation{put, merge}) + writeRequest2 := sink.WriteRequest{ + CompletionMode: sink.CompletionWaitUntilApplied, + Operations: []sink.WriteOperation{put, merge}, + } + writeResults, err := client.Write(context.Background(), writeRequest2) if err != nil { t.Fatalf("Write() error = %v", err) } @@ -460,11 +467,13 @@ func TestClientCoversSinkContract(t *testing.T) { t.Fatalf("Write() failure code = %s", writeResults[1].Failure.Code) } + deleteRequest2 := sink.DeleteRequest{ + CompletionMode: sink.CompletionReturnAfterAccepted, + Addresses: addresses[:2], + } deleteResults, err := client.Delete( context.Background(), - sink.CompletionReturnAfterAccepted, - addresses[:2]..., - ) + deleteRequest2) if err != nil { t.Fatalf("Delete() error = %v", err) } @@ -524,11 +533,19 @@ func TestClientSendsWaitUntilVisibleCompletionMode(t *testing.T) { if err != nil { t.Fatalf("sink.NewPut() error = %v", err) } - _, err = client.Write(t.Context(), sink.CompletionWaitUntilVisible, []sink.WriteOperation{operation}) + writeRequest2 := sink.WriteRequest{ + CompletionMode: sink.CompletionWaitUntilVisible, + Operations: []sink.WriteOperation{operation}, + } + _, err = client.Write(t.Context(), writeRequest2) if err != nil { t.Fatalf("Write() error = %v", err) } - _, err = client.Delete(t.Context(), sink.CompletionWaitUntilVisible, address) + deleteRequest2 := sink.DeleteRequest{ + CompletionMode: sink.CompletionWaitUntilVisible, + Addresses: []sink.Address{address}, + } + _, err = client.Delete(t.Context(), deleteRequest2) if err != nil { t.Fatalf("Delete() error = %v", err) } @@ -557,7 +574,10 @@ func TestReadRetriesOnlyUnavailable(t *testing.T) { clientOptions := sink.ClientOptions{ReadRetry: retry} client := startTestClient(t, server, clientOptions) address := testAddress(t, sink.StringKey("retry")) - _, err := client.Read(context.Background(), []sink.Address{address}) + readRequest := sink.ReadRequest{ + Addresses: []sink.Address{address}, + } + _, err := client.Read(context.Background(), readRequest) if err != nil { t.Fatalf("Read() error = %v", err) } @@ -578,7 +598,10 @@ func TestReadRetriesOnlyUnavailable(t *testing.T) { clientOptions := sink.ClientOptions{ReadRetry: retry} client := startTestClient(t, server, clientOptions) address := testAddress(t, sink.StringKey("quota")) - _, err := client.Read(context.Background(), []sink.Address{address}) + readRequest := sink.ReadRequest{ + Addresses: []sink.Address{address}, + } + _, err := client.Read(context.Background(), readRequest) if status.Code(err) != codes.ResourceExhausted { t.Fatalf("Read() code = %s, want ResourceExhausted", status.Code(err)) } @@ -611,7 +634,11 @@ func TestWriteDeclaresIdenticalLuaProgramOncePerBatch(t *testing.T) { if err != nil { t.Fatalf("sink.NewMerge(second) error = %v", err) } - if _, err := client.Write(context.Background(), sink.CompletionWaitUntilApplied, []sink.WriteOperation{first, second}); err != nil { + writeRequest2 := sink.WriteRequest{ + CompletionMode: sink.CompletionWaitUntilApplied, + Operations: []sink.WriteOperation{first, second}, + } + if _, err := client.Write(context.Background(), writeRequest2); err != nil { t.Fatalf("Write() error = %v", err) } @@ -635,7 +662,10 @@ func TestClientAcceptsMessagesLargerThanGRPCDefault(t *testing.T) { var clientOptions sink.ClientOptions client := startTestClient(t, server, clientOptions) address := testAddress(t, sink.StringKey("large")) - results, err := client.Read(context.Background(), []sink.Address{address}) + readRequest := sink.ReadRequest{ + Addresses: []sink.Address{address}, + } + results, err := client.Read(context.Background(), readRequest) if err != nil { t.Fatalf("Read() error = %v", err) } @@ -658,7 +688,10 @@ func TestReadRetriesOnlyRetryableOperationFailures(t *testing.T) { testAddress(t, sink.StringKey("retry")), testAddress(t, sink.StringKey("success")), } - results, err := client.Read(context.Background(), addresses) + readRequest2 := sink.ReadRequest{ + Addresses: addresses, + } + results, err := client.Read(context.Background(), readRequest2) if err != nil { t.Fatalf("Read() error = %v", err) } @@ -694,15 +727,26 @@ func TestClientMethodsSplitBatchesAndRemapIndexes(t *testing.T) { } operations[index] = operation } - readResults, err := client.Read(context.Background(), addresses) + readRequest := sink.ReadRequest{ + Addresses: addresses, + } + readResults, err := client.Read(context.Background(), readRequest) if err != nil { t.Fatalf("Read() error = %v", err) } - writeResults, err := client.Write(context.Background(), sink.CompletionWaitUntilApplied, operations) + writeRequest := sink.WriteRequest{ + CompletionMode: sink.CompletionWaitUntilApplied, + Operations: operations, + } + writeResults, err := client.Write(context.Background(), writeRequest) if err != nil { t.Fatalf("Write() error = %v", err) } - deleteResults, err := client.Delete(context.Background(), sink.CompletionWaitUntilApplied, addresses...) + deleteRequest := sink.DeleteRequest{ + CompletionMode: sink.CompletionWaitUntilApplied, + Addresses: addresses, + } + deleteResults, err := client.Delete(context.Background(), deleteRequest) if err != nil { t.Fatalf("Delete() error = %v", err) } @@ -740,15 +784,26 @@ func TestClientValidatesCompleteCollectionsBeforeRPC(t *testing.T) { operations[index] = operation } - _, readErr := client.Read(t.Context(), addresses) + readRequest := sink.ReadRequest{ + Addresses: addresses, + } + _, readErr := client.Read(t.Context(), readRequest) if readErr == nil || !strings.Contains(readErr.Error(), "read operation 2") { t.Fatalf("Read() error = %v", readErr) } - _, writeErr := client.Write(t.Context(), sink.CompletionWaitUntilApplied, operations) + writeRequest := sink.WriteRequest{ + CompletionMode: sink.CompletionWaitUntilApplied, + Operations: operations, + } + _, writeErr := client.Write(t.Context(), writeRequest) if writeErr == nil || !strings.Contains(writeErr.Error(), "write operation 2") { t.Fatalf("Write() error = %v", writeErr) } - _, deleteErr := client.Delete(t.Context(), sink.CompletionWaitUntilApplied, addresses...) + deleteRequest := sink.DeleteRequest{ + CompletionMode: sink.CompletionWaitUntilApplied, + Addresses: addresses, + } + _, deleteErr := client.Delete(t.Context(), deleteRequest) if deleteErr == nil || !strings.Contains(deleteErr.Error(), "delete operation 2") { t.Fatalf("Delete() error = %v", deleteErr) } @@ -783,15 +838,26 @@ func TestClientMethodsReturnPartialResultsWhenLaterBatchFails(t *testing.T) { operations[index] = operation } - readResults, readErr := client.Read(t.Context(), addresses) + readRequest := sink.ReadRequest{ + Addresses: addresses, + } + readResults, readErr := client.Read(t.Context(), readRequest) if status.Code(readErr) != codes.Unavailable || len(readResults) != 2 { t.Fatalf("Read() results = %+v, error = %v", readResults, readErr) } - writeResults, writeErr := client.Write(t.Context(), sink.CompletionWaitUntilApplied, operations) + writeRequest := sink.WriteRequest{ + CompletionMode: sink.CompletionWaitUntilApplied, + Operations: operations, + } + writeResults, writeErr := client.Write(t.Context(), writeRequest) if status.Code(writeErr) != codes.Unavailable || len(writeResults) != 2 { t.Fatalf("Write() results = %+v, error = %v", writeResults, writeErr) } - deleteResults, deleteErr := client.Delete(t.Context(), sink.CompletionWaitUntilApplied, addresses...) + deleteRequest := sink.DeleteRequest{ + CompletionMode: sink.CompletionWaitUntilApplied, + Addresses: addresses, + } + deleteResults, deleteErr := client.Delete(t.Context(), deleteRequest) if status.Code(deleteErr) != codes.Unavailable || len(deleteResults) != 2 { t.Fatalf("Delete() results = %+v, error = %v", deleteResults, deleteErr) } @@ -820,11 +886,19 @@ func TestMutationsAreNotRetried(t *testing.T) { if err != nil { t.Fatalf("sink.NewPut() error = %v", err) } - _, err = client.Write(t.Context(), mode, []sink.WriteOperation{put}) + writeRequest := sink.WriteRequest{ + CompletionMode: mode, + Operations: []sink.WriteOperation{put}, + } + _, err = client.Write(t.Context(), writeRequest) if status.Code(err) != codes.Unavailable { t.Fatalf("Write() code = %s, want Unavailable", status.Code(err)) } - _, err = client.Delete(t.Context(), mode, address) + deleteRequest := sink.DeleteRequest{ + CompletionMode: mode, + Addresses: []sink.Address{address}, + } + _, err = client.Delete(t.Context(), deleteRequest) if status.Code(err) != codes.Unavailable { t.Fatalf("Delete() code = %s, want Unavailable", status.Code(err)) } @@ -859,7 +933,10 @@ func TestReadRejectsMalformedResponses(t *testing.T) { testAddress(t, sink.StringKey("one")), testAddress(t, sink.StringKey("two")), } - _, err := client.Read(context.Background(), addresses) + readRequest := sink.ReadRequest{ + Addresses: addresses, + } + _, err := client.Read(context.Background(), readRequest) var protocolErr *sink.ProtocolError if !errors.As(err, &protocolErr) { t.Fatalf("Read() error = %v, want ProtocolError", err) @@ -885,12 +962,20 @@ func TestMutationResponsesRejectDuplicateIndexes(t *testing.T) { } operations[index] = operation } - _, err := client.Write(context.Background(), sink.CompletionWaitUntilApplied, operations) + writeRequest := sink.WriteRequest{ + CompletionMode: sink.CompletionWaitUntilApplied, + Operations: operations, + } + _, err := client.Write(context.Background(), writeRequest) var writeProtocolErr *sink.ProtocolError if !errors.As(err, &writeProtocolErr) { t.Fatalf("Write() error = %v, want ProtocolError", err) } - _, err = client.Delete(context.Background(), sink.CompletionWaitUntilApplied, addresses...) + deleteRequest := sink.DeleteRequest{ + CompletionMode: sink.CompletionWaitUntilApplied, + Addresses: addresses, + } + _, err = client.Delete(context.Background(), deleteRequest) var deleteProtocolErr *sink.ProtocolError if !errors.As(err, &deleteProtocolErr) { t.Fatalf("Delete() error = %v, want ProtocolError", err) @@ -912,7 +997,11 @@ func TestConstructorsValidateAndCopyInput(t *testing.T) { t.Fatalf("sink.NewPut() error = %v", err) } raw[10] = 'X' - _, err = client.Write(t.Context(), sink.CompletionWaitUntilApplied, []sink.WriteOperation{operation}) + writeRequest2 := sink.WriteRequest{ + CompletionMode: sink.CompletionWaitUntilApplied, + Operations: []sink.WriteOperation{operation}, + } + _, err = client.Write(t.Context(), writeRequest2) if err != nil { t.Fatalf("Write() error = %v", err) } @@ -959,7 +1048,10 @@ func TestConstructorsValidateAndCopyInput(t *testing.T) { t.Fatal("sink.NewLuaProgram() accepted empty source") } - results, err := client.Read(t.Context(), []sink.Address{address}) + readRequest := sink.ReadRequest{ + Addresses: []sink.Address{address}, + } + results, err := client.Read(t.Context(), readRequest) if err != nil { t.Fatalf("Read() error = %v", err) } diff --git a/dataset.go b/dataset.go index 5d2fa02..a11b8e8 100644 --- a/dataset.go +++ b/dataset.go @@ -73,7 +73,8 @@ func NewDataset(client *Client, opts DatasetOptions) (*Dataset, error) { // and treats not-found results as successful reads. Without a callback, results // are collected in key order. With a callback, results arrive incrementally // and the returned result slice is nil. -func (d *Dataset) Read(ctx context.Context, keys []Key, callbacks ...ReadCallback) ([]ReadResult, error) { +func (d *Dataset) Read(ctx context.Context, req DatasetReadRequest) ([]ReadResult, error) { + keys := req.Keys if err := d.validate("read"); err != nil { return nil, err } @@ -85,10 +86,7 @@ func (d *Dataset) Read(ctx context.Context, keys []Key, callbacks ...ReadCallbac } addresses[index] = address } - callback, err := oneCallback(callbacks) - if err != nil { - return nil, err - } + callback := req.OnResult var failures []*OperationError var onResult ReadCallback if callback != nil { @@ -99,7 +97,8 @@ func (d *Dataset) Read(ctx context.Context, keys []Key, callbacks ...ReadCallbac return callback(result) } } - results, err := d.client.Read(ctx, addresses, onResult) + request := ReadRequest{Addresses: addresses, OnResult: onResult} + results, err := d.client.Read(ctx, request) resultsErr := errors.Join(ReadResultsError(results), newBatchError(failures)) if err != nil { if resultsErr != nil { @@ -114,52 +113,31 @@ func (d *Dataset) Read(ctx context.Context, keys []Key, callbacks ...ReadCallbac } // Create writes complete documents only when their keys do not already exist. -func (d *Dataset) Create( - ctx context.Context, - completionMode CompletionMode, - records []Record, - callbacks ...WriteCallback, -) ([]WriteResult, error) { +func (d *Dataset) Create(ctx context.Context, req DatasetWriteRequest) ([]WriteResult, error) { opts := datasetPutOptions{ - completionMode: completionMode, - writeMode: WriteCreate, - operation: "create", - records: records, - callbacks: callbacks, + request: req, + writeMode: WriteCreate, + operation: "create", } return d.put(ctx, opts) } // Replace writes complete documents only when their keys already exist. -func (d *Dataset) Replace( - ctx context.Context, - completionMode CompletionMode, - records []Record, - callbacks ...WriteCallback, -) ([]WriteResult, error) { +func (d *Dataset) Replace(ctx context.Context, req DatasetWriteRequest) ([]WriteResult, error) { opts := datasetPutOptions{ - completionMode: completionMode, - writeMode: WriteReplace, - operation: "replace", - records: records, - callbacks: callbacks, + request: req, + writeMode: WriteReplace, + operation: "replace", } return d.put(ctx, opts) } // Upsert writes complete documents whether or not their keys already exist. -func (d *Dataset) Upsert( - ctx context.Context, - completionMode CompletionMode, - records []Record, - callbacks ...WriteCallback, -) ([]WriteResult, error) { +func (d *Dataset) Upsert(ctx context.Context, req DatasetWriteRequest) ([]WriteResult, error) { opts := datasetPutOptions{ - completionMode: completionMode, - writeMode: WriteUpsert, - operation: "upsert", - records: records, - callbacks: callbacks, + request: req, + writeMode: WriteUpsert, + operation: "upsert", } return d.put(ctx, opts) } @@ -167,20 +145,15 @@ func (d *Dataset) Upsert( // Merge atomically applies the Dataset's bound Lua program to every incoming // record, creating a record when none exists. A Dataset without MergeProgram // rejects Merge before sending an RPC. -func (d *Dataset) Merge( - ctx context.Context, - completionMode CompletionMode, - records []Record, - callbacks ...WriteCallback, -) ([]WriteResult, error) { +func (d *Dataset) Merge(ctx context.Context, req DatasetWriteRequest) ([]WriteResult, error) { if err := d.validate("merge"); err != nil { return nil, err } if !d.hasMergeProgram { return nil, errors.New("dataset merge: merge program is not configured") } - operations := make([]WriteOperation, len(records)) - for index, record := range records { + operations := make([]WriteOperation, len(req.Records)) + for index, record := range req.Records { address, document, err := d.encodeRecord(record) if err != nil { return nil, fmt.Errorf("dataset merge record %d: %w", index, err) @@ -196,24 +169,22 @@ func (d *Dataset) Merge( operation.returnDocument = record.ReturnDocument operations[index] = operation } - opts := datasetWriteOptions{operation: "merge", mode: completionMode, operations: operations, callbacks: callbacks} - return d.write(ctx, opts) + request := WriteRequest{CompletionMode: req.CompletionMode, Operations: operations, OnResult: req.OnResult} + return d.write(ctx, "merge", request) } type datasetPutOptions struct { - callbacks []WriteCallback - completionMode CompletionMode - writeMode WriteMode - operation string - records []Record + request DatasetWriteRequest + writeMode WriteMode + operation string } func (d *Dataset) put(ctx context.Context, opts datasetPutOptions) ([]WriteResult, error) { if err := d.validate(opts.operation); err != nil { return nil, err } - operations := make([]WriteOperation, len(opts.records)) - for index, record := range opts.records { + operations := make([]WriteOperation, len(opts.request.Records)) + for index, record := range opts.request.Records { address, document, err := d.encodeRecord(record) if err != nil { return nil, fmt.Errorf("dataset %s record %d: %w", opts.operation, index, err) @@ -225,8 +196,8 @@ func (d *Dataset) put(ctx context.Context, opts datasetPutOptions) ([]WriteResul operation.returnDocument = record.ReturnDocument operations[index] = operation } - write := datasetWriteOptions{operation: opts.operation, mode: opts.completionMode, operations: operations, callbacks: opts.callbacks} - return d.write(ctx, write) + request := WriteRequest{CompletionMode: opts.request.CompletionMode, Operations: operations, OnResult: opts.request.OnResult} + return d.write(ctx, opts.operation, request) } func (d *Dataset) validate(operation string) error { @@ -250,19 +221,8 @@ func (d *Dataset) encodeRecord(record Record) (Address, Document, error) { return address, document, nil } -type datasetWriteOptions struct { - operation string - mode CompletionMode - operations []WriteOperation - callbacks []WriteCallback -} - -func (d *Dataset) write(ctx context.Context, opts datasetWriteOptions) ([]WriteResult, error) { - operation := opts.operation - callback, err := oneCallback(opts.callbacks) - if err != nil { - return nil, err - } +func (d *Dataset) write(ctx context.Context, operation string, req WriteRequest) ([]WriteResult, error) { + callback := req.OnResult var failures []*OperationError var onResult WriteCallback if callback != nil { @@ -273,7 +233,8 @@ func (d *Dataset) write(ctx context.Context, opts datasetWriteOptions) ([]WriteR return callback(result) } } - results, err := d.client.Write(ctx, opts.mode, opts.operations, onResult) + req.OnResult = onResult + results, err := d.client.Write(ctx, req) resultsErr := errors.Join(WriteResultsError(results), newBatchError(failures)) if err != nil { if resultsErr != nil { diff --git a/dataset_native.go b/dataset_native.go index d224845..05a206f 100644 --- a/dataset_native.go +++ b/dataset_native.go @@ -40,14 +40,14 @@ func (d *Dataset) Execute(ctx context.Context, req ExecuteRequest) (ExecuteRespo // Query fetches a page from this Dataset. An empty Command selects all records. // The Store selects its default query. Query controls and native payload // semantics are the same as Client.Query. -func (d *Dataset) Query(ctx context.Context, req QueryRequest, callbacks ...DocumentCallback) (QueryResponse, error) { +func (d *Dataset) Query(ctx context.Context, req QueryRequest) (QueryResponse, error) { command, err := d.bindNativeCommand(req.Command) if err != nil { var empty QueryResponse return empty, err } req.Command = command - return d.client.Query(ctx, req, callbacks...) + return d.client.Query(ctx, req) } // Count counts matches in this Dataset. An empty Command counts all records; @@ -64,14 +64,14 @@ func (d *Dataset) Count(ctx context.Context, req CountRequest) (CountResponse, e // Scan returns one live page scoped to this Dataset. Reuse the request with // NextCursor to continue. JSON commands must provide a unique stable sort. -func (d *Dataset) Scan(ctx context.Context, req ScanRequest, callbacks ...DocumentCallback) (ScanResponse, error) { +func (d *Dataset) Scan(ctx context.Context, req ScanRequest) (ScanResponse, error) { command, err := d.bindNativeCommand(req.Command) if err != nil { var empty ScanResponse return empty, err } req.Command = command - return d.client.Scan(ctx, req, callbacks...) + return d.client.Scan(ctx, req) } func (d *Dataset) bindNativeCommand(command Command) (Command, error) { diff --git a/dataset_test.go b/dataset_test.go index 76d2d0a..fb50d85 100644 --- a/dataset_test.go +++ b/dataset_test.go @@ -76,15 +76,26 @@ func TestNewDatasetValidatesAndCopiesOptions(t *testing.T) { Key: sink.StringKey("product-1"), Value: datasetProduct{UID: "product-1", Name: "keyboard"}, } - _, err = nilDataset.Upsert(t.Context(), sink.CompletionWaitUntilApplied, []sink.Record{record}) + upsertRequest := sink.DatasetWriteRequest{ + CompletionMode: sink.CompletionWaitUntilApplied, + Records: []sink.Record{record}, + } + _, err = nilDataset.Upsert(t.Context(), upsertRequest) if err == nil { t.Fatal("Dataset.Upsert() accepted a nil dataset") } - _, err = nilDataset.Read(t.Context(), []sink.Key{record.Key}) + readRequest := sink.DatasetReadRequest{ + Keys: []sink.Key{record.Key}, + } + _, err = nilDataset.Read(t.Context(), readRequest) if err == nil { t.Fatal("Dataset.Read() accepted a nil dataset") } - _, err = dataset.Upsert(t.Context(), 0, []sink.Record{record}) + upsertRequest2 := sink.DatasetWriteRequest{ + CompletionMode: 0, + Records: []sink.Record{record}, + } + _, err = dataset.Upsert(t.Context(), upsertRequest2) if err == nil { t.Fatal("Dataset.Upsert() accepted an invalid completion mode") } @@ -106,7 +117,10 @@ func TestDatasetReadBindsRoutingSplitsBatchesAndPreservesOrder(t *testing.T) { sink.StringKey("product-4"), sink.StringKey("product-5"), } - results, err := dataset.Read(t.Context(), keys) + readRequest := sink.DatasetReadRequest{ + Keys: keys, + } + results, err := dataset.Read(t.Context(), readRequest) if err != nil { t.Fatalf("Dataset.Read() error = %v", err) } @@ -148,7 +162,10 @@ func TestDatasetReadTreatsNotFoundAsSuccess(t *testing.T) { if err != nil { t.Fatalf("NewDataset() error = %v", err) } - results, err := dataset.Read(t.Context(), []sink.Key{sink.StringKey("missing")}) + readRequest := sink.DatasetReadRequest{ + Keys: []sink.Key{sink.StringKey("missing")}, + } + results, err := dataset.Read(t.Context(), readRequest) if err != nil { t.Fatalf("Dataset.Read() error = %v", err) } @@ -172,10 +189,13 @@ func TestDatasetReadCollectsFailuresAfterRetries(t *testing.T) { if err != nil { t.Fatalf("NewDataset() error = %v", err) } + readRequest := sink.DatasetReadRequest{ + Keys: []sink.Key{sink.StringKey("unavailable"), + sink.StringKey("available")}, + } results, err := dataset.Read( t.Context(), - []sink.Key{sink.StringKey("unavailable"), - sink.StringKey("available")}) + readRequest) var batchError *sink.BatchError if !errors.As(err, &batchError) || len(batchError.Failures) != 1 || batchError.Failures[0].OperationIndex != 0 { @@ -205,11 +225,14 @@ func TestDatasetReadReturnsPartialResultsOnTransportFailure(t *testing.T) { if err != nil { t.Fatalf("NewDataset() error = %v", err) } + readRequest := sink.DatasetReadRequest{ + Keys: []sink.Key{sink.StringKey("failed"), + sink.StringKey("found"), + sink.StringKey("transport-failure")}, + } results, err := dataset.Read( t.Context(), - []sink.Key{sink.StringKey("failed"), - sink.StringKey("found"), - sink.StringKey("transport-failure")}) + readRequest) if status.Code(err) != codes.Unavailable { t.Fatalf("Dataset.Read() error code = %s, want Unavailable", status.Code(err)) } @@ -233,7 +256,10 @@ func TestDatasetReadRejectsInvalidKeysBeforeRPC(t *testing.T) { t.Fatalf("NewDataset() error = %v", err) } var emptyKey sink.Key - _, err = dataset.Read(t.Context(), []sink.Key{sink.StringKey("valid"), emptyKey}) + readRequest := sink.DatasetReadRequest{ + Keys: []sink.Key{sink.StringKey("valid"), emptyKey}, + } + _, err = dataset.Read(t.Context(), readRequest) if err == nil || !strings.Contains(err.Error(), "dataset read key 1") { t.Fatalf("Dataset.Read() invalid key error = %v", err) } @@ -281,19 +307,31 @@ func TestDatasetPutMethodsBindRoutingEncodingAndPerCallCompletion(t *testing.T) } createProduct := datasetProduct{UID: expectations[0].key, Name: expectations[0].name, Stock: 12} createRecord := sink.Record{Key: sink.StringKey(createProduct.UID), Value: createProduct} - createResults, err := dataset.Create(t.Context(), sink.CompletionWaitUntilApplied, []sink.Record{createRecord}) + createRequest := sink.DatasetWriteRequest{ + CompletionMode: sink.CompletionWaitUntilApplied, + Records: []sink.Record{createRecord}, + } + createResults, err := dataset.Create(t.Context(), createRequest) if err != nil { t.Fatalf("Dataset.Create() error = %v", err) } replaceProduct := datasetProduct{UID: expectations[1].key, Name: expectations[1].name, Stock: 12} replaceRecord := sink.Record{Key: sink.StringKey(replaceProduct.UID), Value: replaceProduct} - replaceResults, err := dataset.Replace(t.Context(), sink.CompletionWaitUntilVisible, []sink.Record{replaceRecord}) + replaceRequest := sink.DatasetWriteRequest{ + CompletionMode: sink.CompletionWaitUntilVisible, + Records: []sink.Record{replaceRecord}, + } + replaceResults, err := dataset.Replace(t.Context(), replaceRequest) if err != nil { t.Fatalf("Dataset.Replace() error = %v", err) } upsertProduct := datasetProduct{UID: expectations[2].key, Name: expectations[2].name, Stock: 12} upsertRecord := sink.Record{Key: sink.StringKey(upsertProduct.UID), Value: upsertProduct} - upsertResults, err := dataset.Upsert(t.Context(), sink.CompletionReturnAfterAccepted, []sink.Record{upsertRecord}) + upsertRequest := sink.DatasetWriteRequest{ + CompletionMode: sink.CompletionReturnAfterAccepted, + Records: []sink.Record{upsertRecord}, + } + upsertResults, err := dataset.Upsert(t.Context(), upsertRequest) if err != nil { t.Fatalf("Dataset.Upsert() error = %v", err) } @@ -351,7 +389,11 @@ func TestDatasetUsesConfiguredBSONEncoding(t *testing.T) { } product := datasetProduct{UID: "product-bson", Name: "keyboard", Stock: 4} record := sink.Record{Key: sink.StringKey(product.UID), Value: product} - _, err = dataset.Upsert(t.Context(), sink.CompletionWaitUntilApplied, []sink.Record{record}) + upsertRequest := sink.DatasetWriteRequest{ + CompletionMode: sink.CompletionWaitUntilApplied, + Records: []sink.Record{record}, + } + _, err = dataset.Upsert(t.Context(), upsertRequest) if err != nil { t.Fatalf("Dataset.Upsert() error = %v", err) } @@ -388,11 +430,13 @@ func TestDatasetMergeUsesOneBoundProgramForBatch(t *testing.T) { {Key: sink.StringKey("product-1"), Value: datasetProduct{UID: "product-1", Stock: 1}}, {Key: sink.StringKey("product-2"), Value: datasetProduct{UID: "product-2", Stock: 2}}, } + mergeRequest := sink.DatasetWriteRequest{ + CompletionMode: sink.CompletionWaitUntilVisible, + Records: records, + } results, err := dataset.Merge( t.Context(), - sink.CompletionWaitUntilVisible, - records, - ) + mergeRequest) if err != nil { t.Fatalf("Dataset.Merge() error = %v", err) } @@ -432,10 +476,13 @@ func TestDatasetMergeUsesOneBoundProgramForBatch(t *testing.T) { if err != nil { t.Fatalf("NewDataset(put only) error = %v", err) } + mergeRequest2 := sink.DatasetWriteRequest{ + CompletionMode: sink.CompletionWaitUntilApplied, + Records: []sink.Record{records[0]}, + } _, err = putOnlyDataset.Merge( t.Context(), - sink.CompletionWaitUntilApplied, - []sink.Record{records[0]}) + mergeRequest2) if err == nil || !strings.Contains(err.Error(), "merge program is not configured") { t.Fatalf("Dataset.Merge() without program error = %v", err) } @@ -456,7 +503,11 @@ func TestDatasetUpsertSplitsBatchesAndCollectsOperationFailures(t *testing.T) { product := datasetProduct{UID: uid, Stock: index} records[index] = sink.Record{Key: sink.StringKey(uid), Value: product} } - results, err := dataset.Upsert(t.Context(), sink.CompletionReturnAfterAccepted, records) + upsertRequest := sink.DatasetWriteRequest{ + CompletionMode: sink.CompletionReturnAfterAccepted, + Records: records, + } + results, err := dataset.Upsert(t.Context(), upsertRequest) if len(results) != len(records) { t.Fatalf("Dataset.Upsert() results = %d, want %d", len(results), len(records)) } @@ -505,7 +556,11 @@ func TestDatasetUpsertReturnsPartialResultsOnTransportFailure(t *testing.T) { {Key: sink.StringKey("product-2"), Value: datasetProduct{UID: "product-2"}}, {Key: sink.StringKey("product-3"), Value: datasetProduct{UID: "product-3"}}, } - results, err := dataset.Upsert(t.Context(), sink.CompletionWaitUntilApplied, records) + upsertRequest := sink.DatasetWriteRequest{ + CompletionMode: sink.CompletionWaitUntilApplied, + Records: records, + } + results, err := dataset.Upsert(t.Context(), upsertRequest) if status.Code(err) != codes.Unavailable { t.Fatalf("Dataset.Upsert() error code = %s, want Unavailable", status.Code(err)) } @@ -534,7 +589,11 @@ func TestDatasetRejectsInvalidRecordsBeforeWrite(t *testing.T) { } var emptyKey sink.Key record := sink.Record{Key: emptyKey, Value: datasetProduct{UID: "product-1"}} - _, err = dataset.Upsert(t.Context(), sink.CompletionWaitUntilApplied, []sink.Record{record}) + upsertRequest := sink.DatasetWriteRequest{ + CompletionMode: sink.CompletionWaitUntilApplied, + Records: []sink.Record{record}, + } + _, err = dataset.Upsert(t.Context(), upsertRequest) if err == nil || !strings.Contains(err.Error(), "dataset upsert record 0") { t.Fatalf("Dataset.Upsert() invalid record error = %v", err) } diff --git a/dns_balancing_integration_test.go b/dns_balancing_integration_test.go index 76c564c..9af9ba2 100644 --- a/dns_balancing_integration_test.go +++ b/dns_balancing_integration_test.go @@ -95,7 +95,11 @@ func testDNSScaleChanges(t *testing.T, opts sink.DialOptions) { t.Helper() attempts++ record := sink.Record{Key: sink.StringKey(fmt.Sprintf("record-%d", attempts)), Value: map[string]int{"value": attempts}} - _, err := dataset.Upsert(ctx, sink.CompletionWaitUntilApplied, []sink.Record{record}) + upsertRequest := sink.DatasetWriteRequest{ + CompletionMode: sink.CompletionWaitUntilApplied, + Records: []sink.Record{record}, + } + _, err := dataset.Upsert(ctx, upsertRequest) if err != nil { t.Fatalf("write during DNS scaling: %v", err) } diff --git a/integration_test.go b/integration_test.go index 8afd982..ad22280 100644 --- a/integration_test.go +++ b/integration_test.go @@ -69,28 +69,42 @@ func TestSinkCompatibility(t *testing.T) { } syncValue := integrationValueFor("sync", "applied") syncRecord := sink.Record{Key: syncKey, Value: syncValue} - writeResults, err := dataset.Upsert(ctx, sink.CompletionWaitUntilVisible, []sink.Record{syncRecord}) + upsertRequest := sink.DatasetWriteRequest{ + CompletionMode: sink.CompletionWaitUntilVisible, + Records: []sink.Record{syncRecord}, + } + writeResults, err := dataset.Upsert(ctx, upsertRequest) if err != nil { t.Fatalf("Dataset.Upsert(sync) error = %v", err) } assertWriteStatus(t, writeResults, sink.WriteApplied) - readResults, err := dataset.Read(ctx, []sink.Key{syncKey}) + readRequest := sink.DatasetReadRequest{ + Keys: []sink.Key{syncKey}, + } + readResults, err := dataset.Read(ctx, readRequest) if err != nil { t.Fatalf("Dataset.Read(sync) error = %v", err) } assertReadDocument(t, readResults, "sync", "applied") - writeResults, err = dataset.Create(ctx, sink.CompletionWaitUntilApplied, []sink.Record{syncRecord}) + createRequest := sink.DatasetWriteRequest{ + CompletionMode: sink.CompletionWaitUntilApplied, + Records: []sink.Record{syncRecord}, + } + writeResults, err = dataset.Create(ctx, createRequest) if err == nil { t.Fatal("Dataset.Create(duplicate) succeeded") } assertWriteFailure(t, writeResults, sink.WritePreconditionFailed, sink.FailurePreconditionFailed) + mergeRequest := sink.DatasetWriteRequest{ + CompletionMode: sink.CompletionWaitUntilVisible, + Records: []sink.Record{syncRecord}, + } writeResults, err = dataset.Merge( ctx, - sink.CompletionWaitUntilVisible, - []sink.Record{syncRecord}) + mergeRequest) if err != nil { t.Fatalf("Dataset.Merge() error = %v", err) } @@ -98,7 +112,11 @@ func TestSinkCompatibility(t *testing.T) { asyncValue := integrationValueFor("async", "accepted") asyncRecord := sink.Record{Key: asyncKey, Value: asyncValue} - writeResults, err = dataset.Upsert(ctx, sink.CompletionReturnAfterAccepted, []sink.Record{asyncRecord}) + upsertRequest2 := sink.DatasetWriteRequest{ + CompletionMode: sink.CompletionReturnAfterAccepted, + Records: []sink.Record{asyncRecord}, + } + writeResults, err = dataset.Upsert(ctx, upsertRequest2) if err != nil { t.Fatalf("Dataset.Upsert(async) results=%+v error=%v", writeResults, err) } @@ -113,12 +131,13 @@ func TestSinkCompatibility(t *testing.T) { t.Fatalf("wait for asynchronous document: %v", err) } + deleteRequest := sink.DeleteRequest{ + CompletionMode: sink.CompletionWaitUntilVisible, + Addresses: []sink.Address{syncAddress, asyncAddress}, + } deleteResults, err := client.Delete( ctx, - sink.CompletionWaitUntilVisible, - syncAddress, - asyncAddress, - ) + deleteRequest) if err != nil { t.Fatalf("Delete() error = %v", err) } @@ -127,7 +146,10 @@ func TestSinkCompatibility(t *testing.T) { t.Fatalf("Delete() result %d = %+v", index, result) } } - readResults, err = dataset.Read(ctx, []sink.Key{syncKey, asyncKey}) + readRequest2 := sink.DatasetReadRequest{ + Keys: []sink.Key{syncKey, asyncKey}, + } + readResults, err = dataset.Read(ctx, readRequest2) if err != nil { t.Fatalf("Dataset.Read(after delete) error = %v", err) } @@ -193,7 +215,10 @@ func waitForDocument( ticker := time.NewTicker(100 * time.Millisecond) defer ticker.Stop() for { - results, err := opts.Dataset.Read(ctx, []sink.Key{opts.Key}) + readRequest := sink.DatasetReadRequest{ + Keys: []sink.Key{opts.Key}, + } + results, err := opts.Dataset.Read(ctx, readRequest) if err == nil && len(results) == 1 && results[0].Status == sink.ReadFound { var document integrationValue decodeErr := results[0].Document.Decode(&document) diff --git a/native.go b/native.go index 47b0de8..7238597 100644 --- a/native.go +++ b/native.go @@ -182,6 +182,9 @@ type ScanRequest struct { BatchSize int Cursor []byte Projection *Projection // Nil preserves the native projection. + // OnDocument consumes each document without collecting it in the response. + // Nil collects documents. Returning an error cancels the stream. + OnDocument DocumentCallback } type ScanResponse struct { @@ -199,7 +202,7 @@ type ScanResponse struct { // does not invalidate an existing cursor. // An optional callback receives documents without collecting them in Documents. // NextCursor is returned only after the stream finishes successfully. -func (c *Client) Scan(ctx context.Context, req ScanRequest, callbacks ...DocumentCallback) (ScanResponse, error) { +func (c *Client) Scan(ctx context.Context, req ScanRequest) (ScanResponse, error) { var empty ScanResponse if c == nil || c.rpc == nil { return empty, errors.New("scan requires a client") @@ -224,10 +227,6 @@ func (c *Client) Scan(ctx context.Context, req ScanRequest, callbacks ...Documen ctx, cancel = context.WithTimeout(ctx, c.config.scanTimeout) defer cancel() } - callback, err := oneCallback(callbacks) - if err != nil { - return empty, err - } limit := req.BatchSize if limit == 0 { limit = 100 @@ -239,8 +238,8 @@ func (c *Client) Scan(ctx context.Context, req ScanRequest, callbacks ...Documen call := scanPageCall{request: request, limit: limit} call.emit = func(document Document) error { received++ - if callback != nil { - return callback(document) + if req.OnDocument != nil { + return req.OnDocument(document) } result.Documents = append(result.Documents, document) return nil diff --git a/native_integration_test.go b/native_integration_test.go index a9de9ee..50baf48 100644 --- a/native_integration_test.go +++ b/native_integration_test.go @@ -63,14 +63,22 @@ end`)) t.Cleanup(func() { cleanup, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() - _, _ = client.Delete(cleanup, sink.CompletionWaitUntilApplied, address) + deleteRequest := sink.DeleteRequest{ + CompletionMode: sink.CompletionWaitUntilApplied, + Addresses: []sink.Address{address}, + } + _, _ = client.Delete(cleanup, deleteRequest) }) value := struct { Count int `bson:"count"` UpdatedAt time.Time `bson:"updated_at"` }{Count: 1, UpdatedAt: integrationDateTime()} record := sink.Record{Key: key, Value: value, ReturnDocument: true} - results, err := dataset.Merge(ctx, sink.CompletionWaitUntilApplied, []sink.Record{record, record}) + mergeRequest := sink.DatasetWriteRequest{ + CompletionMode: sink.CompletionWaitUntilApplied, + Records: []sink.Record{record, record}, + } + results, err := dataset.Merge(ctx, mergeRequest) if err != nil || len(results) != 2 { t.Fatalf("returned Merge: %+v %v", results, err) } diff --git a/native_test.go b/native_test.go index b9b3dad..172ba51 100644 --- a/native_test.go +++ b/native_test.go @@ -220,7 +220,11 @@ func TestReturnedDocumentsReachDatasetAndRejectAsync(t *testing.T) { t.Fatal(err) } record := sink.Record{Key: sink.StringKey("1"), Value: map[string]int{"count": 1}, ReturnDocument: true} - results, err := dataset.Upsert(t.Context(), sink.CompletionWaitUntilApplied, []sink.Record{record}) + upsertRequest := sink.DatasetWriteRequest{ + CompletionMode: sink.CompletionWaitUntilApplied, + Records: []sink.Record{record}, + } + results, err := dataset.Upsert(t.Context(), upsertRequest) if err != nil || len(results) != 1 { t.Fatalf("results=%v err=%v", results, err) } @@ -228,7 +232,11 @@ func TestReturnedDocumentsReachDatasetAndRejectAsync(t *testing.T) { if err := results[0].Document.Decode(&returned); err != nil || returned["count"] != 1 { t.Fatalf("document=%v err=%v", returned, err) } - _, err = dataset.Upsert(t.Context(), sink.CompletionReturnAfterAccepted, []sink.Record{record}) + upsertRequest2 := sink.DatasetWriteRequest{ + CompletionMode: sink.CompletionReturnAfterAccepted, + Records: []sink.Record{record}, + } + _, err = dataset.Upsert(t.Context(), upsertRequest2) if err == nil || server.writeCalls.Load() != 1 { t.Fatalf("async returning request sent: calls=%d err=%v", server.writeCalls.Load(), err) } @@ -245,7 +253,11 @@ func TestMissingRequestedDocumentIsProtocolFailure(t *testing.T) { t.Fatal(err) } operation = operation.WithReturnedDocument() - _, err = client.Write(t.Context(), sink.CompletionWaitUntilApplied, []sink.WriteOperation{operation}) + writeRequest := sink.WriteRequest{ + CompletionMode: sink.CompletionWaitUntilApplied, + Operations: []sink.WriteOperation{operation}, + } + _, err = client.Write(t.Context(), writeRequest) var protocolErr *sink.ProtocolError if !errors.As(err, &protocolErr) { t.Fatalf("missing requested document error=%v", err) diff --git a/query.go b/query.go index 129b0bd..491ece3 100644 --- a/query.go +++ b/query.go @@ -17,6 +17,9 @@ type QueryRequest struct { PageSize int // Zero selects 100; maximum 1000. Sort []SortField // Ordered keys; empty preserves native sorting. Projection *Projection // Nil preserves the native projection. + // OnDocument consumes each document without collecting it in the response. + // Nil collects documents. Returning an error cancels the stream. + OnDocument DocumentCallback } type SortField struct { @@ -42,7 +45,7 @@ type QueryResponse struct { // HasMore uses one extra result. Query never automatically runs Count or retries. // An optional callback receives documents without collecting them in Documents. // HasMore is valid only when the stream finishes successfully. -func (c *Client) Query(ctx context.Context, req QueryRequest, callbacks ...DocumentCallback) (QueryResponse, error) { +func (c *Client) Query(ctx context.Context, req QueryRequest) (QueryResponse, error) { var empty QueryResponse if c == nil || c.rpc == nil { return empty, errors.New("query requires a client") @@ -68,10 +71,6 @@ func (c *Client) Query(ctx context.Context, req QueryRequest, callbacks ...Docum if err != nil { return empty, err } - callback, err := oneCallback(callbacks) - if err != nil { - return empty, err - } ctx, cancel := context.WithCancel(ctx) defer cancel() stream, err := c.rpc.Query(ctx, request, c.config.sinkCallOptions...) @@ -114,8 +113,8 @@ func (c *Client) Query(ctx context.Context, req QueryRequest, callbacks ...Docum return result, protocolError("Query", err.Error()) } count++ - if callback != nil { - if err := callback(document); err != nil { + if req.OnDocument != nil { + if err := req.OnDocument(document); err != nil { return result, err } } else { diff --git a/requests.go b/requests.go new file mode 100644 index 0000000..498f3da --- /dev/null +++ b/requests.go @@ -0,0 +1,41 @@ +package sink + +// ReadRequest selects records and how their results are consumed. +type ReadRequest struct { + Addresses []Address + // OnResult consumes each final result without collecting a result slice. + // Nil collects results in request order. Returning an error cancels the stream. + OnResult ReadCallback +} + +// WriteRequest submits mixed put and merge operations with one completion mode. +type WriteRequest struct { + CompletionMode CompletionMode + Operations []WriteOperation + // OnResult consumes each acknowledgement without collecting a result slice. + // Nil collects results in request order. Returning an error cancels the stream; + // unacknowledged operations may have completed and are never replayed. + OnResult WriteCallback +} + +// DeleteRequest permanently deletes records with one completion mode. +type DeleteRequest struct { + CompletionMode CompletionMode + Addresses []Address +} + +// DatasetReadRequest selects keys within the Dataset's bound resource. +type DatasetReadRequest struct { + Keys []Key + // OnResult has the same delivery and collection semantics as ReadRequest.OnResult. + OnResult ReadCallback +} + +// DatasetWriteRequest supplies records for Create, Replace, Upsert or Merge. +// Routing, encoding and merge programs come from the Dataset. +type DatasetWriteRequest struct { + CompletionMode CompletionMode + Records []Record + // OnResult has the same delivery and collection semantics as WriteRequest.OnResult. + OnResult WriteCallback +} diff --git a/streaming_test.go b/streaming_test.go index 7245366..f0b431e 100644 --- a/streaming_test.go +++ b/streaming_test.go @@ -2,6 +2,7 @@ package sink_test import ( "errors" + "fmt" "sync/atomic" "testing" "time" @@ -56,7 +57,11 @@ func TestStreamingReadRetriesOnlyUndeliveredAndDoesNotCollectCallbacks(t *testin client := startTestClient(t, server, opts) addresses := []sink.Address{testAddress(t, sink.StringKey("a")), testAddress(t, sink.StringKey("b")), testAddress(t, sink.StringKey("c"))} seen := make(map[int]int) - results, err := client.Read(t.Context(), addresses, func(result sink.ReadResult) error { seen[result.OperationIndex]++; return nil }) + readRequest := sink.ReadRequest{ + Addresses: addresses, + OnResult: func(result sink.ReadResult) error { seen[result.OperationIndex]++; return nil }, + } + results, err := client.Read(t.Context(), readRequest) if err != nil || results != nil || len(seen) != 3 || server.reads.Load() != 2 { t.Fatalf("results=%v seen=%v calls=%d err=%v", results, seen, server.reads.Load(), err) } @@ -66,7 +71,10 @@ func TestStreamingReadRetriesOnlyUndeliveredAndDoesNotCollectCallbacks(t *testin } } // Collecting uses the same stream, restores request order, and retains payloads. - results, err = client.Read(t.Context(), addresses) + readRequest2 := sink.ReadRequest{ + Addresses: addresses, + } + results, err = client.Read(t.Context(), readRequest2) if err != nil || len(results) != 3 { t.Fatalf("collect: %v %v", results, err) } @@ -82,7 +90,11 @@ func TestCallbackErrorCancelsWithoutRetry(t *testing.T) { client := startTestClient(t, server, opts) addresses := []sink.Address{testAddress(t, sink.StringKey("a"))} stop := status.Error(codes.Unavailable, "callback stopped") - results, err := client.Read(t.Context(), addresses, func(sink.ReadResult) error { return stop }) + readRequest := sink.ReadRequest{ + Addresses: addresses, + OnResult: func(sink.ReadResult) error { return stop }, + } + results, err := client.Read(t.Context(), readRequest) if !errors.Is(err, stop) || results != nil || server.reads.Load() != 1 { t.Fatalf("callback cancellation: %v %v", results, err) } @@ -114,7 +126,12 @@ func TestStreamingWritePreservesPartialResultsAndNeverReplays(t *testing.T) { return nil } } - results, err := client.Write(t.Context(), sink.CompletionWaitUntilApplied, operations, emit) + writeRequest := sink.WriteRequest{ + CompletionMode: sink.CompletionWaitUntilApplied, + Operations: operations, + OnResult: emit, + } + results, err := client.Write(t.Context(), writeRequest) if status.Code(err) != codes.Unavailable || server.writes.Load() != 1 { t.Fatalf("write replay/status: %v", err) } @@ -154,9 +171,123 @@ func TestStreamingScanWithholdsCursorOnFailure(t *testing.T) { client := startTestClient(t, server, opts) request := sink.ScanRequest{Command: sink.Command{URI: "sink://primary/items"}} count := 0 - response, err := client.Scan(t.Context(), request, func(sink.Document) error { count++; return nil }) + scanRequest := request + scanRequest.OnDocument = func(sink.Document) error { count++; return nil } + response, err := client.Scan(t.Context(), scanRequest) if status.Code(err) != codes.Unavailable || count != 1 || response.Documents != nil || len(response.NextCursor) != 0 { t.Fatalf("unsafe cursor/collection: %+v %v count=%d", response, err, count) } } } + +func TestDatasetRequestCallbacksPreserveBatchFailuresWithoutCollecting(t *testing.T) { + server := &testSinkServer{} + opts := sink.ClientOptions{MaxOperations: 2} + client := startTestClient(t, server, opts) + datasetOptions := sink.DatasetOptions{URI: "sink://primary/items", Encoding: sink.DocumentEncodingJSON} + dataset, err := sink.NewDataset(client, datasetOptions) + if err != nil { + t.Fatal(err) + } + keys := []sink.Key{sink.StringKey("a"), sink.StringKey("b"), sink.StringKey("c")} + readIndexes := make(map[int]int) + readRequest := sink.DatasetReadRequest{ + Keys: keys, + OnResult: func(result sink.ReadResult) error { + readIndexes[result.OperationIndex]++ + return result.Err() + }, + } + readResults, err := dataset.Read(t.Context(), readRequest) + if err != nil || readResults != nil || len(readIndexes) != len(keys) { + t.Fatalf("read callbacks: results=%v indexes=%v err=%v", readResults, readIndexes, err) + } + for index := range keys { + if readIndexes[index] != 1 { + t.Fatalf("read index %d delivered %d times", index, readIndexes[index]) + } + } + records := make([]sink.Record, len(keys)) + for index, key := range keys { + records[index] = sink.Record{Key: key, Value: map[string]int{"value": index}} + } + writeIndexes := make(map[int]int) + writeRequest := sink.DatasetWriteRequest{ + CompletionMode: sink.CompletionWaitUntilApplied, + Records: records, + OnResult: func(result sink.WriteResult) error { + writeIndexes[result.OperationIndex]++ + return nil + }, + } + writeResults, err := dataset.Upsert(t.Context(), writeRequest) + var batchError *sink.BatchError + if writeResults != nil || len(writeIndexes) != len(records) || !errors.As(err, &batchError) { + t.Fatalf("write callbacks: results=%v indexes=%v err=%v", writeResults, writeIndexes, err) + } + if len(batchError.Failures) != 1 || batchError.Failures[0].OperationIndex != 1 { + t.Fatalf("callback lost operation failure: %+v", batchError.Failures) + } + for index := range records { + if writeIndexes[index] != 1 { + t.Fatalf("write index %d delivered %d times", index, writeIndexes[index]) + } + } +} + +func TestNativeRequestCallbacksKeepMetadataWithoutCollecting(t *testing.T) { + for _, scoped := range []bool{false, true} { + t.Run(fmt.Sprintf("dataset=%t", scoped), func(t *testing.T) { + queryServer := &queryRPCServer{} + opts := sink.ClientOptions{} + queryClient := startTestClient(t, queryServer, opts) + scanServer := &nativeRPCServer{} + scanClient := startTestClient(t, scanServer, opts) + command := sdkNativeRequest().Command + datasetOptions := sink.DatasetOptions{URI: command.URI, Encoding: sink.DocumentEncodingJSON} + queryDataset, err := sink.NewDataset(queryClient, datasetOptions) + if err != nil { + t.Fatal(err) + } + scanDataset, err := sink.NewDataset(scanClient, datasetOptions) + if err != nil { + t.Fatal(err) + } + queryCount := 0 + queryRequest := sink.QueryRequest{ + Command: command, + PageSize: 1, + OnDocument: func(document sink.Document) error { + queryCount++ + return nil + }, + } + var queryPage sink.QueryResponse + if scoped { + queryPage, err = queryDataset.Query(t.Context(), queryRequest) + } else { + queryPage, err = queryClient.Query(t.Context(), queryRequest) + } + if err != nil || queryCount != 1 || queryPage.Documents != nil || !queryPage.HasMore { + t.Fatalf("query callback: page=%+v count=%d err=%v", queryPage, queryCount, err) + } + scanCount := 0 + scanRequest := sink.ScanRequest{ + Command: command, + OnDocument: func(document sink.Document) error { + scanCount++ + return nil + }, + } + var scanPage sink.ScanResponse + if scoped { + scanPage, err = scanDataset.Scan(t.Context(), scanRequest) + } else { + scanPage, err = scanClient.Scan(t.Context(), scanRequest) + } + if err != nil || scanCount != 1 || scanPage.Documents != nil || string(scanPage.NextCursor) != "next" { + t.Fatalf("scan callback: page=%+v count=%d err=%v", scanPage, scanCount, err) + } + }) + } +} From 8979b087f33e5c07fab76f7d091f1bef12044458 Mon Sep 17 00:00:00 2001 From: Randy <857271370@qq.com> Date: Sun, 20 Sep 2026 17:33:27 +0800 Subject: [PATCH 4/4] =?UTF-8?q?=E2=9C=A8=20feat:=20add=20request=20helpers?= =?UTF-8?q?=20with=20applied=20completion=20defaults?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 80 +++++++++++++------ client.go | 4 +- dataset_native_test.go | 7 +- dataset_test.go | 2 +- native.go | 8 +- query.go | 9 ++- query_test.go | 3 +- request_helpers.go | 169 ++++++++++++++++++++++++++++++++++++++++ request_helpers_test.go | 125 +++++++++++++++++++++++++++++ requests.go | 3 + streaming_test.go | 59 ++++++-------- 11 files changed, 396 insertions(+), 73 deletions(-) create mode 100644 request_helpers.go create mode 100644 request_helpers_test.go diff --git a/README.md b/README.md index 9f738bd..845326b 100644 --- a/README.md +++ b/README.md @@ -96,15 +96,13 @@ func main() { Key: sink.StringKey(value.UID), Value: value, } - writeRequest := sink.DatasetWriteRequest{ - CompletionMode: sink.CompletionWaitUntilVisible, - Records: []sink.Record{record}, - } + writeRequest := sink.NewDatasetWriteRequest(record). + WithCompletionMode(sink.CompletionWaitUntilVisible) _, err = products.Upsert(context.Background(), writeRequest) if err != nil { log.Fatal(err) } - readRequest := sink.DatasetReadRequest{Keys: []sink.Key{sink.StringKey(value.UID)}} + readRequest := sink.NewDatasetReadRequest(sink.StringKey(value.UID)) readResults, err := products.Read(context.Background(), readRequest) if err != nil { log.Fatal(err) @@ -119,10 +117,10 @@ func main() { ``` `Dataset` binds the stable store, namespace, dataset, and document encoding once. -`Read` accepts one or many keys without reconstructing addresses. Each mutation -still receives an explicit completion mode because callers of the same dataset -can require different durability or visibility guarantees. Pass records in -`DatasetWriteRequest.Records` to `Create`, `Replace`, or `Upsert`; the client validates and encodes the +`Read` accepts one or many keys without reconstructing addresses. Mutations +wait until applied by default; `WithCompletionMode` changes durability or +visibility for an individual call. Pass records to `NewDatasetWriteRequest` or +set `DatasetWriteRequest.Records` directly. The client validates and encodes the complete collection before sending it, automatically splits large collections by `ClientOptions.MaxOperations`, and preserves global operation indexes: @@ -131,10 +129,8 @@ records := []sink.Record{ {Key: sink.StringKey("product-42"), Value: firstProduct}, {Key: sink.StringKey("product-43"), Value: secondProduct}, } -request := sink.DatasetWriteRequest{ - CompletionMode: sink.CompletionReturnAfterAccepted, - Records: records, -} +request := sink.NewDatasetWriteRequest(records...). + WithCompletionMode(sink.CompletionReturnAfterAccepted) results, err := products.Upsert(context.Background(), request) ``` @@ -177,10 +173,8 @@ record := sink.Record{ Key: sink.StringKey("product-42"), Value: incomingProduct, } -request := sink.DatasetWriteRequest{ - CompletionMode: sink.CompletionWaitUntilVisible, - Records: []sink.Record{record}, -} +request := sink.NewDatasetWriteRequest(record). + WithCompletionMode(sink.CompletionWaitUntilVisible) results, err := products.Merge(context.Background(), request) if err != nil { log.Fatal(err) @@ -197,6 +191,49 @@ versioned `sink.v1` array, object, and retry-stable time helpers. See the [Lua merge developer guide](https://github.com/liran/sink/blob/main/docs/lua-merge-guide.md) for the complete function reference and reliability rules. +## Request helpers and defaults + +Helpers accept individual records/keys/addresses, or an existing slice with +`...`. Chain `With...` methods to override fields; each method returns an updated +request value without modifying the original or copying document payloads. +Direct struct literals remain supported and receive the same defaults at call time. + +```go +request := sink.NewDatasetWriteRequest(records...) // APPLIED; collect results. +results, err := products.Upsert(ctx, request) + +streamRequest := request.WithOnResult(processWrite) +results, err = products.Upsert(ctx, streamRequest) // results == nil + +readRequest := sink.NewDatasetReadRequest(keys...).WithOnResult(processRead) +readResults, err := products.Read(ctx, readRequest) // readResults == nil + +queryRequest := sink.NewQueryRequest().WithPage(2).WithPageSize(50) +page, err := products.Query(ctx, queryRequest) + +scanRequest := sink.NewScanRequest().WithBatchSize(50).WithOnDocument(processDocument) +scanPage, err := products.Scan(ctx, scanRequest) +if err != nil { + return err +} +scanRequest = scanRequest.WithCursor(scanPage.NextCursor) +``` + +| Request field | Default when unset | +| --- | --- | +| Write/Delete/DatasetWrite `CompletionMode` | `CompletionWaitUntilApplied` | +| Query `Page` / `PageSize` | `1` / `100` | +| Scan `BatchSize` / `Cursor` | `100` / start a new scan | +| `OnResult` / `OnDocument` | Collect and return results | +| Query `Sort` / Query and Scan `Projection` | Preserve native settings | + +Client record methods have corresponding `NewReadRequest`, `NewWriteRequest` +and `NewDeleteRequest` helpers. Native Client queries and scans use +`WithCommand(command)`; Dataset supplies its bound resource when Command is empty. +`WithSort` keeps the given field order, and `WithProjection` accepts the existing +Projection type. Nil callbacks restore collection. Invalid nonzero completion +modes, negative pagination values and oversized pages still fail validation. + ## Streaming callbacks Read, Write, Query and Scan use server-streaming RPCs. The previous unary wire @@ -248,8 +285,8 @@ The default collector returns received documents/results along with any error. - `Dataset` is the primary record API. It binds routing, encoding, and one optional Lua program. `Read` accepts one or many keys; `Create`, `Replace`, - `Upsert`, and `Merge` accept one or many `Record` values while keeping - completion mode explicit per mutation. Every method splits large collections + `Upsert`, and `Merge` accept one or many `Record` values with APPLIED completion + by default and an optional per-call override. Every method splits large collections automatically and collects per-record results unless a callback is supplied. - `Read(ctx, ReadRequest)` preserves request order and reports found, not-found, or failed results independently. @@ -588,10 +625,7 @@ record := sink.Record{ Value: increment, ReturnDocument: true, } -request := sink.DatasetWriteRequest{ - CompletionMode: sink.CompletionWaitUntilApplied, - Records: []sink.Record{record}, -} +request := sink.NewDatasetWriteRequest(record) // Defaults to APPLIED. results, err := quotas.Merge(ctx, request) if err != nil { return err diff --git a/client.go b/client.go index ccc50dc..cdc8310 100644 --- a/client.go +++ b/client.go @@ -323,7 +323,7 @@ func (c *Client) Read(ctx context.Context, req ReadRequest) ([]ReadResult, error // collected in request order. With a callback, results arrive incrementally // and the returned result slice is nil. func (c *Client) Write(ctx context.Context, req WriteRequest) ([]WriteResult, error) { - completionMode, operations := req.CompletionMode, req.Operations + completionMode, operations := defaultCompletionMode(req.CompletionMode), req.Operations if err := c.validateCollection("write", len(operations)); err != nil { return nil, err } @@ -440,7 +440,7 @@ func (c *Client) writeBatch(ctx context.Context, call writeBatchCall) error { // retried. Earlier batches may have completed when a later batch returns an // error. func (c *Client) Delete(ctx context.Context, req DeleteRequest) ([]DeleteResult, error) { - completionMode, addresses := req.CompletionMode, req.Addresses + completionMode, addresses := defaultCompletionMode(req.CompletionMode), req.Addresses if err := c.validateCollection("delete", len(addresses)); err != nil { return nil, err } diff --git a/dataset_native_test.go b/dataset_native_test.go index 587d4f9..f426849 100644 --- a/dataset_native_test.go +++ b/dataset_native_test.go @@ -48,7 +48,9 @@ func TestDatasetNativeMethodsBindOpaqueURIAndRetainControls(t *testing.T) { t.Fatal(err) } projection := &sink.Projection{Fields: []string{"name"}} - query := sink.QueryRequest{Page: 3, PageSize: 1, Sort: []sink.SortField{{Field: "name", Descending: true}}, Projection: projection} + sortField := sink.SortField{Field: "name", Descending: true} + query := sink.NewQueryRequest().WithPage(3).WithPageSize(1). + WithSort(sortField).WithProjection(projection) if _, err := dataset.Query(t.Context(), query); err != nil { t.Fatal(err) } @@ -63,7 +65,8 @@ func TestDatasetNativeMethodsBindOpaqueURIAndRetainControls(t *testing.T) { } countRequest := <-server.counts assertDatasetNativeResource(t, countRequest.Command, opts.URI) - scan := sink.ScanRequest{BatchSize: 23, Cursor: []byte("checkpoint"), Projection: projection} + scan := sink.NewScanRequest().WithBatchSize(23). + WithCursor([]byte("checkpoint")).WithProjection(projection) if page, err := dataset.Scan(t.Context(), scan); err != nil || len(page.Documents) != 1 { t.Fatalf("scan=%+v err=%v", page, err) } diff --git a/dataset_test.go b/dataset_test.go index fb50d85..2d476fb 100644 --- a/dataset_test.go +++ b/dataset_test.go @@ -92,7 +92,7 @@ func TestNewDatasetValidatesAndCopiesOptions(t *testing.T) { t.Fatal("Dataset.Read() accepted a nil dataset") } upsertRequest2 := sink.DatasetWriteRequest{ - CompletionMode: 0, + CompletionMode: -1, Records: []sink.Record{record}, } _, err = dataset.Upsert(t.Context(), upsertRequest2) diff --git a/native.go b/native.go index 7238597..38b3cb9 100644 --- a/native.go +++ b/native.go @@ -179,7 +179,7 @@ func (c *Client) Execute(ctx context.Context, req ExecuteRequest) (ExecuteRespon type ScanRequest struct { Command Command - BatchSize int + BatchSize int // Zero selects 100; maximum 1000. Cursor []byte Projection *Projection // Nil preserves the native projection. // OnDocument consumes each document without collecting it in the response. @@ -207,6 +207,9 @@ func (c *Client) Scan(ctx context.Context, req ScanRequest) (ScanResponse, error if c == nil || c.rpc == nil { return empty, errors.New("scan requires a client") } + if req.BatchSize == 0 { + req.BatchSize = defaultPageSize + } if req.BatchSize < 0 || req.BatchSize > 1000 { return empty, errors.New("scan batch size must be between 0 and 1000") } @@ -228,9 +231,6 @@ func (c *Client) Scan(ctx context.Context, req ScanRequest) (ScanResponse, error defer cancel() } limit := req.BatchSize - if limit == 0 { - limit = 100 - } result := ScanResponse{} backoff := c.config.scanRetry.InitialBackoff for attempt := 1; attempt <= c.config.scanRetry.MaxAttempts; attempt++ { diff --git a/query.go b/query.go index 491ece3..dc4d226 100644 --- a/query.go +++ b/query.go @@ -50,6 +50,12 @@ func (c *Client) Query(ctx context.Context, req QueryRequest) (QueryResponse, er if c == nil || c.rpc == nil { return empty, errors.New("query requires a client") } + if req.Page == 0 { + req.Page = 1 + } + if req.PageSize == 0 { + req.PageSize = defaultPageSize + } if req.Page < 0 || uint64(req.Page) > uint64(^uint32(0)) || req.PageSize < 0 || req.PageSize > 1000 { return empty, errors.New("query requires a nonnegative uint32 page and page size between 0 and 1000") } @@ -78,9 +84,6 @@ func (c *Client) Query(ctx context.Context, req QueryRequest) (QueryResponse, er return empty, fmt.Errorf("query native page: %w", err) } pageSize := req.PageSize - if pageSize == 0 { - pageSize = 100 - } result := QueryResponse{} complete, hasMore, count := false, false, 0 for { diff --git a/query_test.go b/query_test.go index 99030e5..dc52cec 100644 --- a/query_test.go +++ b/query_test.go @@ -21,6 +21,7 @@ type queryRPCServer struct { countCalls atomic.Int32 fail bool invalidPage bool + lastPage bool estimated bool } @@ -33,7 +34,7 @@ func (s *queryRPCServer) queryResponse(_ context.Context, req *sinkv1.QueryReque s.queries <- req } document := &sinkv1.Document{Encoding: sinkv1.DocumentEncoding_DOCUMENT_ENCODING_JSON, Payload: []byte(`{"_source":{"number":4}}`)} - response := &sinkv1.QueryResponse{Documents: []*sinkv1.Document{document}, HasMore: true} + response := &sinkv1.QueryResponse{Documents: []*sinkv1.Document{document}, HasMore: !s.lastPage} if s.invalidPage { response.Documents = nil } diff --git a/request_helpers.go b/request_helpers.go new file mode 100644 index 0000000..1360b79 --- /dev/null +++ b/request_helpers.go @@ -0,0 +1,169 @@ +package sink + +// Request helpers return an updated value for chaining. They do not modify the +// original request or copy document payloads. Validation happens when calling +// Client or Dataset, just as it does for requests built with struct literals. + +const defaultPageSize = 100 + +// NewReadRequest selects records to collect in request order by default. +func NewReadRequest(addresses ...Address) ReadRequest { + request := ReadRequest{Addresses: addresses} + return request +} + +// WithOnResult consumes results without collecting them. Nil restores collection. +func (r ReadRequest) WithOnResult(callback ReadCallback) ReadRequest { + r.OnResult = callback + return r +} + +// NewWriteRequest submits operations with APPLIED completion by default. +func NewWriteRequest(operations ...WriteOperation) WriteRequest { + request := WriteRequest{CompletionMode: CompletionWaitUntilApplied, Operations: operations} + return request +} + +// WithCompletionMode selects when writes are acknowledged. Zero means APPLIED. +func (r WriteRequest) WithCompletionMode(mode CompletionMode) WriteRequest { + r.CompletionMode = mode + return r +} + +// WithOnResult consumes acknowledgements without collecting them. Nil restores collection. +func (r WriteRequest) WithOnResult(callback WriteCallback) WriteRequest { + r.OnResult = callback + return r +} + +// NewDeleteRequest deletes records with APPLIED completion by default. +func NewDeleteRequest(addresses ...Address) DeleteRequest { + request := DeleteRequest{CompletionMode: CompletionWaitUntilApplied, Addresses: addresses} + return request +} + +// WithCompletionMode selects when deletes are acknowledged. Zero means APPLIED. +func (r DeleteRequest) WithCompletionMode(mode CompletionMode) DeleteRequest { + r.CompletionMode = mode + return r +} + +// NewDatasetReadRequest selects keys to collect in request order by default. +func NewDatasetReadRequest(keys ...Key) DatasetReadRequest { + request := DatasetReadRequest{Keys: keys} + return request +} + +// WithOnResult consumes results without collecting them. Nil restores collection. +func (r DatasetReadRequest) WithOnResult(callback ReadCallback) DatasetReadRequest { + r.OnResult = callback + return r +} + +// NewDatasetWriteRequest supplies records with APPLIED completion by default. +func NewDatasetWriteRequest(records ...Record) DatasetWriteRequest { + request := DatasetWriteRequest{CompletionMode: CompletionWaitUntilApplied, Records: records} + return request +} + +// WithCompletionMode selects when mutations are acknowledged. Zero means APPLIED. +func (r DatasetWriteRequest) WithCompletionMode(mode CompletionMode) DatasetWriteRequest { + r.CompletionMode = mode + return r +} + +// WithOnResult consumes acknowledgements without collecting them. Nil restores collection. +func (r DatasetWriteRequest) WithOnResult(callback WriteCallback) DatasetWriteRequest { + r.OnResult = callback + return r +} + +// NewQueryRequest selects page 1 with 100 documents and collection enabled. +// Dataset supplies the resource; Client callers must supply a Command. +func NewQueryRequest() QueryRequest { + request := QueryRequest{Page: 1, PageSize: defaultPageSize} + return request +} + +// WithCommand supplies the native query and resource. +func (r QueryRequest) WithCommand(command Command) QueryRequest { + r.Command = command + return r +} + +// WithPage selects a one-based page. Zero means page 1. +func (r QueryRequest) WithPage(page int) QueryRequest { + r.Page = page + return r +} + +// WithPageSize selects the page size. Zero means 100; the maximum is 1000. +func (r QueryRequest) WithPageSize(size int) QueryRequest { + r.PageSize = size + return r +} + +// WithSort replaces the native sort with the supplied ordered fields. +// No fields preserves the native sort. +func (r QueryRequest) WithSort(fields ...SortField) QueryRequest { + r.Sort = fields + return r +} + +// WithProjection selects returned fields. Nil preserves native projection. +func (r QueryRequest) WithProjection(projection *Projection) QueryRequest { + r.Projection = projection + return r +} + +// WithOnDocument consumes documents without collecting them. Nil restores collection. +// Page metadata is still returned after successful completion. +func (r QueryRequest) WithOnDocument(callback DocumentCallback) QueryRequest { + r.OnDocument = callback + return r +} + +// NewScanRequest starts a scan with 100 documents per page and collection enabled. +// Dataset supplies the resource; Client callers must supply a Command. +func NewScanRequest() ScanRequest { + request := ScanRequest{BatchSize: defaultPageSize} + return request +} + +// WithCommand supplies the native query and resource. +func (r ScanRequest) WithCommand(command Command) ScanRequest { + r.Command = command + return r +} + +// WithBatchSize selects the page size. Zero means 100; the maximum is 1000. +func (r ScanRequest) WithBatchSize(size int) ScanRequest { + r.BatchSize = size + return r +} + +// WithCursor resumes from a committed checkpoint. Empty starts a new scan. +func (r ScanRequest) WithCursor(cursor []byte) ScanRequest { + r.Cursor = cursor + return r +} + +// WithProjection selects returned fields. Nil preserves native projection. +func (r ScanRequest) WithProjection(projection *Projection) ScanRequest { + r.Projection = projection + return r +} + +// WithOnDocument consumes documents without collecting them. Nil restores collection. +// The next cursor is still returned after successful completion. +func (r ScanRequest) WithOnDocument(callback DocumentCallback) ScanRequest { + r.OnDocument = callback + return r +} + +func defaultCompletionMode(mode CompletionMode) CompletionMode { + if mode == 0 { + return CompletionWaitUntilApplied + } + return mode +} diff --git a/request_helpers_test.go b/request_helpers_test.go new file mode 100644 index 0000000..0b2a937 --- /dev/null +++ b/request_helpers_test.go @@ -0,0 +1,125 @@ +package sink_test + +import ( + "testing" + + sink "github.com/liran/sink-go" + sinkv1 "github.com/liran/sink-go/api/sink/v1" +) + +func TestRequestCompletionDefaultsAndOverridesReachRPC(t *testing.T) { + cases := []struct { + name string + mode sink.CompletionMode + helpers bool + invalid bool + }{ + {name: "literal defaults"}, + {name: "helper defaults", helpers: true}, + {name: "applied", mode: sink.CompletionWaitUntilApplied, helpers: true}, + {name: "accepted", mode: sink.CompletionReturnAfterAccepted, helpers: true}, + {name: "visible", mode: sink.CompletionWaitUntilVisible, helpers: true}, + {name: "negative", mode: -1, helpers: true, invalid: true}, + {name: "unknown", mode: 99, helpers: true, invalid: true}, + } + for _, test := range cases { + t.Run(test.name, func(t *testing.T) { + server := &testSinkServer{} + options := sink.ClientOptions{} + client := startTestClient(t, server, options) + address := testAddress(t, sink.StringKey("defaults")) + operation, err := sink.NewPut(address, testDocument("value"), sink.WriteUpsert) + if err != nil { + t.Fatal(err) + } + datasetOptions := sink.DatasetOptions{URI: "sink://primary/items", Encoding: sink.DocumentEncodingJSON} + dataset, err := sink.NewDataset(client, datasetOptions) + if err != nil { + t.Fatal(err) + } + record := sink.Record{Key: sink.StringKey("defaults"), Value: map[string]int{"value": 1}} + writeRequest := sink.WriteRequest{Operations: []sink.WriteOperation{operation}} + deleteRequest := sink.DeleteRequest{Addresses: []sink.Address{address}} + datasetRequest := sink.DatasetWriteRequest{Records: []sink.Record{record}} + if test.helpers { + writeRequest = sink.NewWriteRequest(operation) + deleteRequest = sink.NewDeleteRequest(address) + datasetRequest = sink.NewDatasetWriteRequest(record) + if test.mode != 0 { + writeRequest = writeRequest.WithCompletionMode(test.mode) + deleteRequest = deleteRequest.WithCompletionMode(test.mode) + datasetRequest = datasetRequest.WithCompletionMode(test.mode) + } + } + writeResults, writeErr := client.Write(t.Context(), writeRequest) + deleteResults, deleteErr := client.Delete(t.Context(), deleteRequest) + datasetResults, datasetErr := dataset.Upsert(t.Context(), datasetRequest) + if test.invalid { + _, writes, deletes := server.counts() + if writeErr == nil || deleteErr == nil || datasetErr == nil || writes != 0 || deletes != 0 { + t.Fatalf("invalid mode reached RPC: write=%v delete=%v dataset=%v calls=%d/%d", writeErr, deleteErr, datasetErr, writes, deletes) + } + return + } + if writeErr != nil || deleteErr != nil || datasetErr != nil { + t.Fatalf("write=%v delete=%v dataset=%v", writeErr, deleteErr, datasetErr) + } + if len(writeResults) != 1 || len(deleteResults) != 1 || len(datasetResults) != 1 { + t.Fatal("default collection lost results") + } + want := test.mode + if want == 0 { + want = sink.CompletionWaitUntilApplied + } + server.mu.Lock() + defer server.mu.Unlock() + for _, request := range server.writeRequests { + if request.CompletionMode != want { + t.Fatalf("write completion = %v, want %v", request.CompletionMode, want) + } + } + if server.deleteRequest.CompletionMode != want { + t.Fatalf("delete completion = %v, want %v", server.deleteRequest.CompletionMode, want) + } + if !test.helpers && (writeRequest.CompletionMode != 0 || deleteRequest.CompletionMode != 0 || datasetRequest.CompletionMode != 0) { + t.Fatal("applying defaults mutated the caller request") + } + }) + } +} + +func TestNativeRequestDefaultsReachRPC(t *testing.T) { + for _, helpers := range []bool{false, true} { + queryServer := &queryRPCServer{queries: make(chan *sinkv1.QueryRequest, 1), lastPage: true} + scanServer := &nativeRPCServer{scanRequests: make(chan *sinkv1.ScanRequest, 1)} + options := sink.ClientOptions{} + queryClient := startTestClient(t, queryServer, options) + scanClient := startTestClient(t, scanServer, options) + command := sdkNativeRequest().Command + queryRequest := sink.QueryRequest{Command: command} + scanRequest := sink.ScanRequest{Command: command} + if helpers { + queryRequest = sink.NewQueryRequest().WithCommand(command) + scanRequest = sink.NewScanRequest().WithCommand(command) + } + page, err := queryClient.Query(t.Context(), queryRequest) + if err != nil || len(page.Documents) != 1 || page.HasMore { + t.Fatalf("default query: %+v %v", page, err) + } + queryWire := <-queryServer.queries + if queryWire.Page != 1 || queryWire.PageSize != 100 || queryWire.Projection != nil || len(queryWire.Sort) != 0 { + t.Fatalf("unexpected query defaults: %v", queryWire) + } + scanPage, err := scanClient.Scan(t.Context(), scanRequest) + if err != nil || len(scanPage.Documents) != 1 || string(scanPage.NextCursor) != "next" { + t.Fatalf("default scan: %+v %v", scanPage, err) + } + scanWire := <-scanServer.scanRequests + if scanWire.BatchSize != 100 || len(scanWire.Cursor) != 0 || scanWire.Projection != nil { + t.Fatalf("unexpected scan defaults: %v", scanWire) + } + if !helpers && (queryRequest.Page != 0 || queryRequest.PageSize != 0 || scanRequest.BatchSize != 0) { + t.Fatal("applying defaults mutated the caller request") + } + } +} diff --git a/requests.go b/requests.go index 498f3da..66fa7d0 100644 --- a/requests.go +++ b/requests.go @@ -10,6 +10,7 @@ type ReadRequest struct { // WriteRequest submits mixed put and merge operations with one completion mode. type WriteRequest struct { + // CompletionMode defaults to CompletionWaitUntilApplied when zero. CompletionMode CompletionMode Operations []WriteOperation // OnResult consumes each acknowledgement without collecting a result slice. @@ -20,6 +21,7 @@ type WriteRequest struct { // DeleteRequest permanently deletes records with one completion mode. type DeleteRequest struct { + // CompletionMode defaults to CompletionWaitUntilApplied when zero. CompletionMode CompletionMode Addresses []Address } @@ -34,6 +36,7 @@ type DatasetReadRequest struct { // DatasetWriteRequest supplies records for Create, Replace, Upsert or Merge. // Routing, encoding and merge programs come from the Dataset. type DatasetWriteRequest struct { + // CompletionMode defaults to CompletionWaitUntilApplied when zero. CompletionMode CompletionMode Records []Record // OnResult has the same delivery and collection semantics as WriteRequest.OnResult. diff --git a/streaming_test.go b/streaming_test.go index f0b431e..7889bed 100644 --- a/streaming_test.go +++ b/streaming_test.go @@ -57,10 +57,10 @@ func TestStreamingReadRetriesOnlyUndeliveredAndDoesNotCollectCallbacks(t *testin client := startTestClient(t, server, opts) addresses := []sink.Address{testAddress(t, sink.StringKey("a")), testAddress(t, sink.StringKey("b")), testAddress(t, sink.StringKey("c"))} seen := make(map[int]int) - readRequest := sink.ReadRequest{ - Addresses: addresses, - OnResult: func(result sink.ReadResult) error { seen[result.OperationIndex]++; return nil }, - } + readRequest := sink.NewReadRequest(addresses...).WithOnResult(func(result sink.ReadResult) error { + seen[result.OperationIndex]++ + return nil + }) results, err := client.Read(t.Context(), readRequest) if err != nil || results != nil || len(seen) != 3 || server.reads.Load() != 2 { t.Fatalf("results=%v seen=%v calls=%d err=%v", results, seen, server.reads.Load(), err) @@ -71,8 +71,9 @@ func TestStreamingReadRetriesOnlyUndeliveredAndDoesNotCollectCallbacks(t *testin } } // Collecting uses the same stream, restores request order, and retains payloads. - readRequest2 := sink.ReadRequest{ - Addresses: addresses, + readRequest2 := readRequest.WithOnResult(nil) + if readRequest.OnResult == nil { + t.Fatal("WithOnResult modified the original request") } results, err = client.Read(t.Context(), readRequest2) if err != nil || len(results) != 3 { @@ -126,11 +127,7 @@ func TestStreamingWritePreservesPartialResultsAndNeverReplays(t *testing.T) { return nil } } - writeRequest := sink.WriteRequest{ - CompletionMode: sink.CompletionWaitUntilApplied, - Operations: operations, - OnResult: emit, - } + writeRequest := sink.NewWriteRequest(operations...).WithOnResult(emit) results, err := client.Write(t.Context(), writeRequest) if status.Code(err) != codes.Unavailable || server.writes.Load() != 1 { t.Fatalf("write replay/status: %v", err) @@ -191,13 +188,10 @@ func TestDatasetRequestCallbacksPreserveBatchFailuresWithoutCollecting(t *testin } keys := []sink.Key{sink.StringKey("a"), sink.StringKey("b"), sink.StringKey("c")} readIndexes := make(map[int]int) - readRequest := sink.DatasetReadRequest{ - Keys: keys, - OnResult: func(result sink.ReadResult) error { - readIndexes[result.OperationIndex]++ - return result.Err() - }, - } + readRequest := sink.NewDatasetReadRequest(keys...).WithOnResult(func(result sink.ReadResult) error { + readIndexes[result.OperationIndex]++ + return result.Err() + }) readResults, err := dataset.Read(t.Context(), readRequest) if err != nil || readResults != nil || len(readIndexes) != len(keys) { t.Fatalf("read callbacks: results=%v indexes=%v err=%v", readResults, readIndexes, err) @@ -212,14 +206,10 @@ func TestDatasetRequestCallbacksPreserveBatchFailuresWithoutCollecting(t *testin records[index] = sink.Record{Key: key, Value: map[string]int{"value": index}} } writeIndexes := make(map[int]int) - writeRequest := sink.DatasetWriteRequest{ - CompletionMode: sink.CompletionWaitUntilApplied, - Records: records, - OnResult: func(result sink.WriteResult) error { - writeIndexes[result.OperationIndex]++ - return nil - }, - } + writeRequest := sink.NewDatasetWriteRequest(records...).WithOnResult(func(result sink.WriteResult) error { + writeIndexes[result.OperationIndex]++ + return nil + }) writeResults, err := dataset.Upsert(t.Context(), writeRequest) var batchError *sink.BatchError if writeResults != nil || len(writeIndexes) != len(records) || !errors.As(err, &batchError) { @@ -254,14 +244,11 @@ func TestNativeRequestCallbacksKeepMetadataWithoutCollecting(t *testing.T) { t.Fatal(err) } queryCount := 0 - queryRequest := sink.QueryRequest{ - Command: command, - PageSize: 1, - OnDocument: func(document sink.Document) error { + queryRequest := sink.NewQueryRequest().WithCommand(command).WithPageSize(1). + WithOnDocument(func(document sink.Document) error { queryCount++ return nil - }, - } + }) var queryPage sink.QueryResponse if scoped { queryPage, err = queryDataset.Query(t.Context(), queryRequest) @@ -272,13 +259,11 @@ func TestNativeRequestCallbacksKeepMetadataWithoutCollecting(t *testing.T) { t.Fatalf("query callback: page=%+v count=%d err=%v", queryPage, queryCount, err) } scanCount := 0 - scanRequest := sink.ScanRequest{ - Command: command, - OnDocument: func(document sink.Document) error { + scanRequest := sink.NewScanRequest().WithCommand(command). + WithOnDocument(func(document sink.Document) error { scanCount++ return nil - }, - } + }) var scanPage sink.ScanResponse if scoped { scanPage, err = scanDataset.Scan(t.Context(), scanRequest)