diff --git a/README.md b/README.md index 2a2d159..375f43a 100644 --- a/README.md +++ b/README.md @@ -17,16 +17,30 @@ Typed Go client for [Weir](https://github.com/batchstream/weir). | Method | Use | | --- | --- | -| `ReadOne` | Read one record. | -| `Read` | Read many records. Results return in input order. | -| `Create` | Write one record that must not exist. | -| `Replace` | Write one record that must exist. | -| `Put` | Create or replace one record. | -| `Delete` | Delete one record. | -| `Mutate` | Write many records in one call. Not a transaction. | -| `AtomicTransform` | Change one record with a Lua function or a backend expression. | -| `ReadStream`, `MutateStream` | Send and receive records incrementally. | -| `Native` | Send a backend's own command. Use it for queries, scans and counts. | +| `Read(ctx, store, options...)` | Read records. Results return in input order. | +| `Mutate(ctx, store, options...)` | Write records. Not a transaction. | +| `Native(ctx, store, request, options...)` | Send a backend's own command. Use it for queries, scans and counts. | + +### Options + +| Option | Applies to | Effect | +| --- | --- | --- | +| `WithDialOptions` | `New` | gRPC options for the connection, such as an interceptor or a custom dialer. | +| `WithRequests` | `Read`, `Mutate` | The records to send. | +| `WithNext` | `Read`, `Mutate` | A callback that produces more records, after those of `WithRequests`. | +| `WithReadConsume`, `WithMutateConsume` | `Read`, `Mutate` | A callback that receives each result as it arrives, instead of the call returning them. | +| `WithQueryVisible` | `Mutate` | Wait until normal queries can see the mutations. | +| `WithChunks` | `Native` | A callback that receives the response body in chunks, instead of `Body` holding it. | + +### Mutation actions + +| Action | Use | +| --- | --- | +| `MutationCreate` | Write a record that must not exist. | +| `MutationReplace` | Write a record that must exist. | +| `MutationPut` | Create or replace a record. | +| `MutationDelete` | Delete a record. | +| `MutationAtomicTransform` | Change a record with a Lua function or a backend expression. | ### Backend formats @@ -58,8 +72,8 @@ documents. Your code owns the document bytes. 4. Never automatically resend a mutation that is `MutationApplied` or `MutationUnknown`, or that has no result. It has applied or may have applied. Reconcile it first. -5. A write may not be visible to the next Search query. Set - `MutationQueryVisible` when it must be. +5. A write may not be visible to the next Search query. Pass + `WithQueryVisible` when it must be. 6. The application listener uses plaintext gRPC. Restrict its network access. ## Requirements @@ -84,7 +98,7 @@ This writes one record to MongoDB through Weir and reads it back. mkdir hello cd hello go mod init example.com/hello -go get github.com/batchstream/weir-go@v0.23.1 go.mongodb.org/mongo-driver/v2/bson +go get github.com/batchstream/weir-go@v0.24.0 go.mongodb.org/mongo-driver/v2/bson ``` **Step 3.** Save this as `main.go`. @@ -122,24 +136,23 @@ func main() { log.Fatal(err) } document := &weir.Document{ContentType: "application/bson", Data: data} - writeRequest := &weir.WriteRequest{Resource: resource, Document: document} - writeOptions := weir.WriteOptions{StoreName: storeName, Request: writeRequest} - written, err := client.Put(ctx, writeOptions) + write := &weir.MutateRequest{Resource: resource, Action: weir.MutationPut, Document: document} + written, err := client.Mutate(ctx, storeName, weir.WithRequests(write)) if err != nil { log.Fatal(err) } - if written.GetFailure() != nil { - log.Fatal(written.GetFailure()) + if written[0].GetFailure() != nil { + log.Fatal(written[0].GetFailure()) } - fmt.Println("outcome:", written.GetOutcome()) + fmt.Println("outcome:", written[0].GetOutcome()) // Read the record back. - readRequest := &weir.ReadRequest{Resource: resource} - readOptions := weir.ReadOneOptions{StoreName: storeName, Request: readRequest} - result, err := client.ReadOne(ctx, readOptions) + read := &weir.ReadRequest{Resource: resource} + results, err := client.Read(ctx, storeName, weir.WithRequests(read)) if err != nil { log.Fatal(err) } + result := results[0] if result.Failure != nil { log.Fatal(result.Failure) } @@ -176,18 +189,18 @@ message: hi | Guide | Contents | | --- | --- | -| [Connections](docs/connections.md) | Addresses, Kubernetes DNS names, keepalive and closing | +| [Connections](docs/connections.md) | Addresses, gRPC options, Kubernetes DNS names, keepalive and closing | | [Operations](docs/operations.md) | Reads, writes, visibility, results and outcomes | | [Lua](docs/lua.md) | Lua transforms | | [Native](docs/native.md) | Backend commands and HTTP helpers | -| [Streaming](docs/streaming.md) | Streams, deadlines and resending refused records | +| [Streaming](docs/streaming.md) | Callbacks, streams, deadlines and resending refused records | | [Development](docs/development.md) | Tests, real backend tests and releases | | [API reference](https://pkg.go.dev/github.com/batchstream/weir-go) | Every exported type and function | | Example | Shows | | --- | --- | | [read](examples/read/main.go) | A typed read | -| [basic](examples/basic/main.go) | A mutation, then many reads in one stream | +| [basic](examples/basic/main.go) | A mutation, then many reads in one call | | [native](examples/native/main.go) | A native command and its response | ## Development diff --git a/api.go b/api.go index a6f4622..316d284 100644 --- a/api.go +++ b/api.go @@ -1,8 +1,6 @@ package weir import ( - "context" - "github.com/batchstream/weir-protocol/api/protocol" pb "github.com/batchstream/weir-protocol/api/weir/v1" ) @@ -19,10 +17,6 @@ type FailureCode = pb.FailureCode type MutationResult = pb.MutationResult type MutationOutcome = pb.MutationOutcome -// MutationOptions controls completion uniformly for every mutation action. -// A nil Options or zero Visibility waits for application acknowledgement. -type MutationOptions = pb.MutationOptions -type MutationVisibility = pb.MutationVisibility type LuaTransform = pb.LuaTransform type ReadRequest = pb.ReadRequest @@ -39,9 +33,6 @@ const ( MutationApplied = pb.MutationOutcome_APPLIED MutationUnknown = pb.MutationOutcome_UNKNOWN - MutationApplicationAcknowledged = pb.MutationVisibility_APPLICATION_ACKNOWLEDGED - MutationQueryVisible = pb.MutationVisibility_QUERY_VISIBLE - NativeCompletionUnconfirmed = pb.NativeCompletion_NATIVE_COMPLETION_UNSPECIFIED NativeNotStarted = pb.NativeCompletion_NATIVE_NOT_STARTED NativeResponseComplete = pb.NativeCompletion_RESPONSE_COMPLETE @@ -64,50 +55,18 @@ const ( // EncodeSegment encodes one decoded resource path segment, including slashes. func EncodeSegment(segment string) string { return protocol.EncodeSegment(segment) } -type WriteRequest struct { - Resource string - Document *Document - Options *MutationOptions -} - -type DeleteRequest struct { - Resource string - Options *MutationOptions -} - -// AtomicTransformRequest requires exactly one of Lua and BackendExpression. -type AtomicTransformRequest struct { - Resource string - Lua *LuaTransform - BackendExpression *Document - Options *MutationOptions -} - // NativeResult retains validated response and completion evidence. Completion is // NativeCompletionUnconfirmed until a terminal acknowledgement arrives; Response // may already be available. An RPC error never revokes a confirmed terminal. // Completion describes response transport, not whether backend writes applied. +// Body holds the response body of a call without WithChunks, as far as it arrived. type NativeResult struct { Response *NativeResponse + Body []byte Completion NativeCompletion Failure *Failure } -// ReadOneOptions describes a single read. Use ReadOptions to share one RPC across -// multiple independent requests in the same Store. -type ReadOneOptions struct { - StoreName string - Request *ReadRequest -} - -// ReadOptions describes a finite read sequence addressed to one Store. Resources -// are canonical paths relative to StoreName. Results have the input order. Do -// not mutate requests or their document bytes until the call returns. -type ReadOptions struct { - StoreName string - Requests []*ReadRequest -} - // MutationAction selects one business operation without a protobuf oneof. type MutationAction uint8 @@ -121,80 +80,12 @@ const ( // MutateRequest describes one mutation. Create, Put and Replace require Document; // Delete accepts no payload; AtomicTransform requires Lua or BackendExpression. -// Options applies to every action. MutationQueryVisible requires visibility to -// subsequent normal queries before an APPLIED result without Failure. APPLIED -// with Failure confirms application only. Nil Options or zero Visibility waits -// for application acknowledgement without a separate visibility guarantee. type MutateRequest struct { Resource string Action MutationAction Document *Document Lua *LuaTransform BackendExpression *Document - Options *MutationOptions -} - -// MutateOptions describes independent mutations addressed to one Store. It is -// not a transaction. Same-resource requests execute in input order; different -// resources may execute concurrently. Cross-RPC ordering follows the database. -// Do not mutate requests, options or their document bytes until the call returns. -type MutateOptions struct { - StoreName string - Requests []*MutateRequest -} - -// ReadStreamOptions produces reads without retaining every input or result. Next -// returns io.EOF after its final item. Items are validated before -// they are sent; an invalid later item does not undo earlier requests. -// Consume receives each confirmed result as its record completes; the index -// identifies the input, starting at 1. -// Next and Consume run concurrently; synchronize any shared application state. -// Both callbacks must honor their context and return promptly. Keep yielded -// requests and document bytes immutable until the corresponding Consume call. -type ReadStreamOptions struct { - StoreName string - Next func(context.Context) (*ReadRequest, error) - Consume func(context.Context, uint64, *ReadResult) error -} - -// MutateStreamOptions incrementally produces mutations to one Store. Next -// returns io.EOF after the final item. Consume receives confirmed results as -// their records complete, identified by input index; a later error does not -// revoke those acknowledgements. Items not consumed remain unacknowledged and -// must never be automatically replayed. -// Next and Consume run concurrently; synchronize any shared application state. -// Both callbacks must honor their context and return promptly. Keep yielded -// requests, options and document bytes immutable until the corresponding -// Consume call. -type MutateStreamOptions struct { - StoreName string - Next func(context.Context) (*MutateRequest, error) - Consume func(context.Context, uint64, *MutationResult) error -} - -type WriteOptions struct { - StoreName string - Request *WriteRequest -} - -type DeleteOptions struct { - StoreName string - Request *DeleteRequest -} - -type AtomicTransformOptions struct { - StoreName string - Request *AtomicTransformRequest -} - -// Native consumes only nonempty response chunks with their validated metadata. -// Consume completes before the next chunk and must honor its context. An empty -// response invokes no callback; its metadata is available in NativeResult. Do -// not mutate response metadata, which remains attached to the final result. -type NativeOptions struct { - StoreName string - Request *NativeRequest - Consume func(context.Context, *NativeResponse, []byte) error } // ReadResult contains exactly one of Document, Missing, or Failure. Missing diff --git a/client.go b/client.go index ea66d4b..bc96238 100644 --- a/client.go +++ b/client.go @@ -13,9 +13,9 @@ import ( "google.golang.org/grpc" ) -// execute consumes one Native request incrementally through its terminal event -// and final gRPC status. -func execute(ctx context.Context, client pb.StoreServiceClient, request *pb.ExecuteRequest, consume func(context.Context, *pb.Event) error) (resultErr error) { +// execute sends one request on a gRPC stream of its own and passes its events to +// consume, which reports the terminal one, then awaits the final gRPC status. +func execute(ctx context.Context, client pb.StoreServiceClient, request *pb.ExecuteRequest, consume func(context.Context, *pb.Event) (bool, error)) (resultErr error) { if client == nil || consume == nil { return errors.New("Execute requires a client and consumer") } @@ -32,7 +32,7 @@ func execute(ctx context.Context, client pb.StoreServiceClient, request *pb.Exec return err } // Commit this attempt before consuming any event; grpc must not transparently - // replay a Native request that could mutate its backend. + // replay a request that could mutate its backend. _ = stream.Context() done := make(chan struct{}) var sendErr error @@ -58,11 +58,11 @@ func execute(ctx context.Context, client pb.StoreServiceClient, request *pb.Exec resultErr = errors.Join(resultErr, sendErr) } }() - state := nativeStreamState{} + terminal := false for { message, err := stream.Recv() if errors.Is(err, io.EOF) { - if !state.terminal { + if !terminal { return errors.New("Execute ended without a terminal business event") } // A peer can finish before Send returns to its caller. Await the @@ -77,53 +77,19 @@ func execute(ctx context.Context, client pb.StoreServiceClient, request *pb.Exec return err } for _, response := range message.Responses { - if err := state.consume(ctx, response, consume); err != nil { + if terminal { + return errors.New("Execute event after terminal business event") + } + if err := protocol.ValidateExecuteResponse(response); err != nil { + return err + } + if response.Index != 1 { + return errors.New("unexpected Execute response ordinal") + } + terminal, err = consume(ctx, response.Event) + if err != nil { return err } } } } - -type nativeStreamState struct { - terminal bool - head bool -} - -// consume checks one Native response in stream order and passes its event on. -func (s *nativeStreamState) consume(ctx context.Context, response *pb.ExecuteResponse, consume func(context.Context, *pb.Event) error) error { - if s.terminal { - return errors.New("Execute event after terminal business event") - } - if err := protocol.ValidateExecuteResponse(response); err != nil { - return err - } - if response.Index != 1 { - return errors.New("unexpected Native response ordinal") - } - if err := s.validate(response.Event); err != nil { - return err - } - return consume(ctx, response.Event) -} - -func (s *nativeStreamState) validate(event *pb.Event) error { - switch value := event.Value.(type) { - case *pb.Event_Head: - if s.head { - return errors.New("invalid Native response head") - } - s.head = true - case *pb.Event_Chunk: - if !s.head { - return errors.New("Native chunk before head") - } - case *pb.Event_NativeEnd: - if value.NativeEnd.Completion == NativeResponseComplete && !s.head || value.NativeEnd.Completion == NativeNotStarted && s.head { - return errors.New("invalid Native completion") - } - s.terminal = true - default: - return errors.New("unexpected Native event") - } - return nil -} diff --git a/client_stream.go b/client_stream.go index 34eaa86..0e36339 100644 --- a/client_stream.go +++ b/client_stream.go @@ -15,85 +15,10 @@ import ( "google.golang.org/protobuf/proto" ) -// ReadStream sends each produced read once there is room and consumes results -// as they complete, over a gRPC stream of its own that follows the Client's -// servers: it moves to another server when its server stops resolving, refuses -// a read or reaches the stream lifetime. A read a server refused is sent again; -// nothing else is. Every result received before a later error is consumed. -func (c *Client) ReadStream(ctx context.Context, options ReadStreamOptions) error { - return c.readStream(ctx, options, false) -} - -func (c *Client) readStream(ctx context.Context, options ReadStreamOptions, known bool) error { - if _, err := c.storeClient(); err != nil { - return err - } - if options.Next == nil || options.Consume == nil { - return errors.New("ReadStream requires a producer and consumer") - } - next := func(ctx context.Context) (*pb.Command, string, error) { - request, err := options.Next(ctx) - if err != nil { - return nil, "", err - } - command, err := readCommand(request) - return command, "", err - } - key := c.lanes.ownKey(options.StoreName, true) - send := func(ctx context.Context, command *pb.Command, queued *queuedRecord) (*ReadResult, error) { - event, shared, err := c.lanes.finish(ctx, key, command, queued) - if !shared { - return c.readAlone(ctx, options.StoreName, command.GetRead()) - } - if err != nil { - return nil, err - } - return readResult(event.GetReadResult()), nil - } - stream := clientStreamOptions[*ReadResult]{key: key, known: known, next: next, send: send, refused: readRefused, consume: options.Consume} - return runClientStream(ctx, c, stream) -} - -// MutateStream sends each produced mutation once there is room and consumes -// results as they complete, over a gRPC stream of its own that follows the -// Client's servers, as ReadStream does. Mutations of one resource run in input -// order, each after the previous one's result, including after a failure. A -// mutation a server refused is sent again; nothing else is. Each consumed result -// is an individual acknowledgement, and every result received before a later -// error is consumed; other mutations stay unacknowledged and may have applied. -func (c *Client) MutateStream(ctx context.Context, options MutateStreamOptions) error { - return c.mutateStream(ctx, options, false) -} - -func (c *Client) mutateStream(ctx context.Context, options MutateStreamOptions, known bool) error { - if _, err := c.storeClient(); err != nil { - return err - } - if options.Next == nil || options.Consume == nil { - return errors.New("MutateStream requires a producer and consumer") - } - next := func(ctx context.Context) (*pb.Command, string, error) { - request, err := options.Next(ctx) - if err != nil { - return nil, "", err - } - command, err := mutateCommand(request) - return command, command.GetMutate().GetResource(), err - } - key := c.lanes.ownKey(options.StoreName, false) - send := func(ctx context.Context, command *pb.Command, queued *queuedRecord) (*MutationResult, error) { - event, shared, err := c.lanes.finish(ctx, key, command, queued) - if !shared { - return c.mutateAlone(ctx, options.StoreName, command) - } - if err != nil { - return nil, err - } - return event.GetMutationResult(), nil - } - stream := clientStreamOptions[*MutationResult]{key: key, known: known, next: next, send: send, refused: mutationRefused, consume: options.Consume} - return runClientStream(ctx, c, stream) -} +// A record stream holds at most these records and bytes unconsumed. A larger +// request proceeds alone. +const recordWindowItems = protocol.RecordWindow +const recordWindowBytes = 8 << 20 // clientStreamOptions describes one record stream of a Client. type clientStreamOptions[R comparable] struct { @@ -109,12 +34,12 @@ type clientStreamOptions[R comparable] struct { // already queued. send func(context.Context, *pb.Command, *queuedRecord) (R, error) refused func(R) bool - consume func(context.Context, uint64, R) error + consume func(context.Context, int, R) error } // streamRecord is one record produced for a stream. type streamRecord struct { - index uint64 + index int size int resource string command *pb.Command @@ -154,9 +79,6 @@ type clientStream[R comparable] struct { } func runClientStream[R comparable](ctx context.Context, c *Client, options clientStreamOptions[R]) error { - if !protocol.ValidStoreName(options.key.store) { - return errors.New("Execute requires a client and valid Store") - } if err := ctx.Err(); err != nil { return err } @@ -190,12 +112,17 @@ func runClientStream[R comparable](ctx context.Context, c *Client, options clien // fail keeps the stream's first error and stops sending. A record that fails // cancels the ones behind it before they can start. func (s *clientStream[R]) fail(err error) { + s.keepFailure(err) + s.cancel() +} + +// keepFailure keeps the stream's first error. +func (s *clientStream[R]) keepFailure(err error) { s.mu.Lock() + defer s.mu.Unlock() if s.failure == nil { s.failure = err } - s.mu.Unlock() - s.cancel() } // produce admits records until the input ends or fails, then closes outcomes @@ -206,7 +133,7 @@ func (s *clientStream[R]) produce() { s.sends.Wait() close(s.outcomes) }() - var count uint64 + count := 0 inflight, bytes := 0, 0 // wait returns one consumed record's credit. wait := func() bool { @@ -238,8 +165,10 @@ func (s *clientStream[R]) produce() { } return } + // Input that fails ends the input only: the records before it finish, and + // their results are the caller's evidence of what they did. if err != nil { - s.fail(fmt.Errorf("request %d: %w", count+1, err)) + s.keepFailure(fmt.Errorf("request %d: %w", count, err)) return } size := proto.Size(command) @@ -249,10 +178,10 @@ func (s *clientStream[R]) produce() { return } } + record := &streamRecord{index: count, size: size, resource: resource, command: command} count++ inflight++ bytes += size - record := &streamRecord{index: count, size: size, resource: resource, command: command} s.admit(record) } } @@ -290,7 +219,6 @@ func (s *clientStream[R]) wakeWriters() { s.unsent = s.unsent[:0] } -// run sends a record and then those waiting behind it, in order. // run sends a resource's records in input order, the first of them already // queued. func (s *clientStream[R]) run(record *streamRecord, queued *queuedRecord) { diff --git a/client_stream_test.go b/client_stream_test.go index cd55876..6554710 100644 --- a/client_stream_test.go +++ b/client_stream_test.go @@ -22,19 +22,18 @@ import ( "google.golang.org/grpc/status" ) -// heldStream is a MutateStream of a Client that a test feeds record by record. +// heldStream is a Mutate of a Client that a test feeds record by record. type heldStream struct { input chan string produced atomic.Int64 mu sync.Mutex - results map[uint64]*MutationResult + results map[int]*MutationResult done chan error } func openHeldStream(ctx context.Context, client *Client) *heldStream { - held := &heldStream{input: make(chan string), results: make(map[uint64]*MutationResult), done: make(chan error, 1)} - options := MutateStreamOptions{StoreName: "records"} - options.Next = func(ctx context.Context) (*MutateRequest, error) { + held := &heldStream{input: make(chan string), results: make(map[int]*MutationResult), done: make(chan error, 1)} + next := func(ctx context.Context) (*MutateRequest, error) { select { case resource, ok := <-held.input: if !ok { @@ -46,13 +45,16 @@ func openHeldStream(ctx context.Context, client *Client) *heldStream { return nil, ctx.Err() } } - options.Consume = func(_ context.Context, index uint64, result *MutationResult) error { + consume := func(_ context.Context, index int, result *MutationResult) error { held.mu.Lock() defer held.mu.Unlock() held.results[index] = result return nil } - go func() { held.done <- client.MutateStream(ctx, options) }() + go func() { + _, err := client.Mutate(ctx, "records", WithNext(next), WithMutateConsume(consume)) + held.done <- err + }() return held } @@ -269,14 +271,13 @@ func TestClientStreamPausedConsumerStopsTheProducer(t *testing.T) { client := laneTestClient(t, peer) release := make(chan struct{}) var produced atomic.Int64 - options := MutateStreamOptions{StoreName: "records"} - options.Next = func(context.Context) (*MutateRequest, error) { + next := func(context.Context) (*MutateRequest, error) { if produced.Add(1) > 2*recordWindowItems { return nil, io.EOF } return putRequest(fmt.Sprintf("records/s:%d", produced.Load()), `{}`), nil } - options.Consume = func(ctx context.Context, _ uint64, _ *MutationResult) error { + consume := func(ctx context.Context, _ int, _ *MutationResult) error { select { case <-release: return nil @@ -285,7 +286,10 @@ func TestClientStreamPausedConsumerStopsTheProducer(t *testing.T) { } } done := make(chan error, 1) - go func() { done <- client.MutateStream(t.Context(), options) }() + go func() { + _, err := client.Mutate(t.Context(), "records", WithNext(next), WithMutateConsume(consume)) + done <- err + }() time.Sleep(100 * time.Millisecond) // The window's records plus the one the producer holds waiting for room. if count := produced.Load(); count > recordWindowItems+1 { @@ -319,7 +323,7 @@ func TestClientStreamFailureKeepsWhatWasConfirmedAndResendsNothing(t *testing.T) } held.mu.Lock() defer held.mu.Unlock() - if len(held.results) != 1 || held.results[1] == nil { + if len(held.results) != 1 || held.results[0] == nil { t.Fatal("confirmed results differ", held.results) } var sent []string @@ -335,31 +339,30 @@ func TestClientStreamReportsItsCallbacksAndInputs(t *testing.T) { peer := newLaneTestPeer() client := laneTestClient(t, peer) failure := errors.New("application failure") - one := func() func(context.Context) (*MutateRequest, error) { - sent := false - return func(context.Context) (*MutateRequest, error) { - if sent { - return nil, io.EOF - } - sent = true - return putRequest("records/s:a", `{}`), nil - } - } - consume := func(context.Context, uint64, *MutationResult) error { return nil } + consume := func(context.Context, int, *MutationResult) error { return nil } + failing := func(context.Context, int, *MutationResult) error { return failure } + empty := func(context.Context) (*MutateRequest, error) { return nil, io.EOF } + broken := func(context.Context) (*MutateRequest, error) { return nil, failure } + invalid := func(context.Context) (*MutateRequest, error) { + request := &MutateRequest{} + return request, nil + } + record := WithRequests(putRequest("records/s:a", `{}`)) tests := []struct { name string - options MutateStreamOptions + store string + options []MutateOption want string }{ - {name: "no callbacks", options: MutateStreamOptions{StoreName: "records"}, want: "requires a producer"}, - {name: "invalid store", options: MutateStreamOptions{StoreName: "", Next: one(), Consume: consume}, want: "valid Store"}, - {name: "empty input", options: MutateStreamOptions{StoreName: "records", Consume: consume, Next: func(context.Context) (*MutateRequest, error) { return nil, io.EOF }}, want: "at least one record"}, - {name: "producer error", options: MutateStreamOptions{StoreName: "records", Consume: consume, Next: func(context.Context) (*MutateRequest, error) { return nil, failure }}, want: "request 1: application failure"}, - {name: "invalid request", options: MutateStreamOptions{StoreName: "records", Consume: consume, Next: func(context.Context) (*MutateRequest, error) { return &MutateRequest{}, nil }}, want: "request 1"}, - {name: "consumer error", options: MutateStreamOptions{StoreName: "records", Next: one(), Consume: func(context.Context, uint64, *MutationResult) error { return failure }}, want: "application failure"}, + {name: "no input", store: "records", options: nil, want: "requires a Store and requests"}, + {name: "invalid store", store: "", options: []MutateOption{record}, want: "requires a Store and requests"}, + {name: "empty input", store: "records", options: []MutateOption{WithNext(empty), WithMutateConsume(consume)}, want: "at least one record"}, + {name: "producer error", store: "records", options: []MutateOption{WithNext(broken), WithMutateConsume(consume)}, want: "request 0: application failure"}, + {name: "invalid request", store: "records", options: []MutateOption{WithNext(invalid), WithMutateConsume(consume)}, want: "request 0"}, + {name: "consumer error", store: "records", options: []MutateOption{record, WithMutateConsume(failing)}, want: "application failure"}, } for _, test := range tests { - err := client.MutateStream(t.Context(), test.options) + _, err := client.Mutate(t.Context(), test.store, test.options...) if err == nil || !strings.Contains(err.Error(), test.want) { t.Errorf("%s: err=%v, want %q", test.name, err, test.want) } @@ -439,8 +442,7 @@ func TestClientStreamSendsALargeRecordAlone(t *testing.T) { before := laneTestReady(t, client, peer)[0] large := strings.Repeat("x", laneRecordBytes+1) sent := 0 - options := MutateStreamOptions{StoreName: "records"} - options.Next = func(context.Context) (*MutateRequest, error) { + next := func(context.Context) (*MutateRequest, error) { if sent == 2 { return nil, io.EOF } @@ -448,14 +450,15 @@ func TestClientStreamSendsALargeRecordAlone(t *testing.T) { return putRequest(fmt.Sprintf("records/s:%d", sent), `{"padding":"`+large+`"}`), nil } results := 0 - options.Consume = func(_ context.Context, _ uint64, result *MutationResult) error { + consume := func(_ context.Context, _ int, result *MutationResult) error { if result.GetOutcome() != MutationApplied { t.Error(result) } results++ return nil } - if err := client.MutateStream(t.Context(), options); err != nil || results != 2 { + _, err := client.Mutate(t.Context(), "records", WithNext(next), WithMutateConsume(consume)) + if err != nil || results != 2 { t.Fatal(err, results) } if opened := peer.streams.Load() - before; opened != 2 { @@ -523,7 +526,7 @@ func TestLoneServerRefusingStreamsReportsItsRecordsNotStarted(t *testing.T) { t.Fatal("the refusal did not reach the consumer", refused) } held.mu.Lock() - result := held.results[1] + result := held.results[0] held.mu.Unlock() if result.GetOutcome() != MutationNotStarted || result.GetFailure().GetCode() != FailureUnavailable { t.Fatal(result) @@ -622,12 +625,12 @@ func TestClientMutateKeepsResultsReceivedBeforeTheStreamBroke(t *testing.T) { t.Fatal(err) } defer client.Close() - options := MutateOptions{StoreName: "records"} + var requests []*MutateRequest for index := range count { - options.Requests = append(options.Requests, putRequest(fmt.Sprintf("records/s:%d", index), `{}`)) + requests = append(requests, putRequest(fmt.Sprintf("records/s:%d", index), `{}`)) } for range 50 { - results, err := client.Mutate(t.Context(), options) + results, err := client.Mutate(t.Context(), "records", WithRequests(requests...)) if status.Code(err) != codes.Unavailable { t.Fatal(err) } @@ -650,9 +653,8 @@ func TestClientStreamConsumesAResultWaitingWhenTheStreamBreaks(t *testing.T) { entered := make(chan struct{}, 8) release := make(chan struct{}) var mu sync.Mutex - seen := make(map[uint64]bool) - options := MutateStreamOptions{StoreName: "records"} - options.Next = func(ctx context.Context) (*MutateRequest, error) { + seen := make(map[int]bool) + next := func(ctx context.Context) (*MutateRequest, error) { select { case resource := <-input: return putRequest(resource, `{}`), nil @@ -660,7 +662,7 @@ func TestClientStreamConsumesAResultWaitingWhenTheStreamBreaks(t *testing.T) { return nil, ctx.Err() } } - options.Consume = func(ctx context.Context, index uint64, _ *MutationResult) error { + consume := func(ctx context.Context, index int, _ *MutationResult) error { if ctx.Err() != nil { return ctx.Err() } @@ -672,7 +674,10 @@ func TestClientStreamConsumesAResultWaitingWhenTheStreamBreaks(t *testing.T) { return nil } done := make(chan error, 1) - go func() { done <- client.MutateStream(t.Context(), options) }() + go func() { + _, err := client.Mutate(t.Context(), "records", WithNext(next), WithMutateConsume(consume)) + done <- err + }() input <- "records/s:one" <-entered input <- "records/s:two" @@ -683,7 +688,7 @@ func TestClientStreamConsumesAResultWaitingWhenTheStreamBreaks(t *testing.T) { if err := <-done; status.Code(err) != codes.Unavailable { t.Fatal(err) } - if !seen[1] || !seen[2] { + if !seen[0] || !seen[1] { t.Fatal("a result received before the break did not reach Consume", seen) } } @@ -694,19 +699,19 @@ func TestClientStreamStopsConsumingOnceConsumeFails(t *testing.T) { failure := errors.New("application failure") sent := 0 calls := 0 - options := MutateStreamOptions{StoreName: "records"} - options.Next = func(context.Context) (*MutateRequest, error) { + next := func(context.Context) (*MutateRequest, error) { if sent == recordWindowItems { return nil, io.EOF } sent++ return putRequest(fmt.Sprintf("records/s:%d", sent), `{}`), nil } - options.Consume = func(context.Context, uint64, *MutationResult) error { + consume := func(context.Context, int, *MutationResult) error { calls++ return failure } - if err := client.MutateStream(t.Context(), options); !errors.Is(err, failure) || calls != 1 { + _, err := client.Mutate(t.Context(), "records", WithNext(next), WithMutateConsume(consume)) + if !errors.Is(err, failure) || calls != 1 { t.Fatal(err, calls) } } @@ -737,27 +742,27 @@ func TestLargeRecordKeepsItsResultWhenItsStreamEndsWithAnError(t *testing.T) { } defer client.Close() large := putRequest("records/s:large", `{"padding":"`+strings.Repeat("x", laneRecordBytes+1)+`"}`) - options := MutateOptions{StoreName: "records", Requests: []*MutateRequest{large}} - results, err := client.Mutate(t.Context(), options) - if status.Code(err) != codes.Internal || results[0].GetOutcome() != MutationApplied { - t.Fatal(results, err) + result, err := mutateOne(t.Context(), client, large) + if status.Code(err) != codes.Internal || result.GetOutcome() != MutationApplied { + t.Fatal(result, err) } } -// Under the race detector this shows that a canceled call stops reading its -// request before returning, as a caller may then reuse its bytes. +// Under the race detector this shows that a canceled call on a stream of its own +// stops reading its requests before returning, as a caller may then reuse their +// bytes. A single record shares a stream instead, whose writer outlives the call. func TestCanceledMutateReturnsAfterItsWriterStoppedReading(t *testing.T) { peer := newLaneTestPeer() client := laneTestClient(t, peer) buffer := []byte(`{"padding":"` + strings.Repeat("x", 256<<10) + `"}`) document := &Document{ContentType: "application/json", Data: buffer} - request := &MutateRequest{Resource: "records/s:reuse", Action: MutationPut, Document: document} - options := MutateOptions{StoreName: "records", Requests: []*MutateRequest{request}} + first := &MutateRequest{Resource: "records/s:reuse", Action: MutationPut, Document: document} + second := &MutateRequest{Resource: "records/s:again", Action: MutationPut, Document: document} deadline := time.Now().Add(time.Second) for attempt := 0; time.Now().Before(deadline); attempt++ { ctx, cancel := context.WithCancel(t.Context()) timer := time.AfterFunc(time.Duration(rand.N(60))*time.Microsecond, cancel) - _, _ = client.Mutate(ctx, options) + _, _ = client.Mutate(ctx, "records", WithRequests(first, second)) timer.Stop() cancel() for index := 12; index < len(buffer)-2; index += 4096 { @@ -815,21 +820,22 @@ func TestClientBatchLeavesInOneMessage(t *testing.T) { t.Fatal(err) } defer client.Close() - mutations := MutateOptions{StoreName: "records"} - reads := ReadOptions{StoreName: "records"} + var mutations []*MutateRequest + var reads []*ReadRequest for index := range recordWindowItems { resource := fmt.Sprintf("records/s:%d", index) - mutations.Requests = append(mutations.Requests, putRequest(resource, `{}`)) - reads.Requests = append(reads.Requests, &ReadRequest{Resource: resource}) + mutations = append(mutations, putRequest(resource, `{}`)) + read := &ReadRequest{Resource: resource} + reads = append(reads, read) } for range 200 { - if _, err := client.Mutate(t.Context(), mutations); err != nil { + if _, err := client.Mutate(t.Context(), "records", WithRequests(mutations...)); err != nil { t.Fatal(err) } if messages := peer.take(); len(messages) != 1 || messages[0] != recordWindowItems { t.Fatal("a batch of mutations was split", messages) } - if _, err := client.Read(t.Context(), reads); err != nil { + if _, err := client.Read(t.Context(), "records", WithRequests(reads...)); err != nil { t.Fatal(err) } if messages := peer.take(); len(messages) != 1 || messages[0] != recordWindowItems { @@ -846,11 +852,11 @@ func TestClientBatchBeyondItsWindowSendsTheRestAsResultsArrive(t *testing.T) { } defer client.Close() const count = 3*recordWindowItems + 1 - options := MutateOptions{StoreName: "records"} + var requests []*MutateRequest for index := range count { - options.Requests = append(options.Requests, putRequest(fmt.Sprintf("records/s:%d", index), `{}`)) + requests = append(requests, putRequest(fmt.Sprintf("records/s:%d", index), `{}`)) } - results, err := client.Mutate(t.Context(), options) + results, err := client.Mutate(t.Context(), "records", WithRequests(requests...)) if err != nil { t.Fatal(err) } @@ -870,14 +876,17 @@ func TestClientBatchBeyondItsWindowSendsTheRestAsResultsArrive(t *testing.T) { } func TestClientBatchOfOneResourceKeepsItsOrder(t *testing.T) { - peer := newLaneTestPeer() - client := laneTestClient(t, peer) + peer := newRefusingPeer() + client := refusingClient(t, peer) const count = 8 - options := MutateOptions{StoreName: "records"} - for range count { - options.Requests = append(options.Requests, putRequest("records/s:same", `{}`)) + var requests []*MutateRequest + var want []string + for index := range count { + data := fmt.Sprintf(`{"n":%d}`, index) + requests = append(requests, putRequest("records/s:same", data)) + want = append(want, data) } - results, err := client.Mutate(t.Context(), options) + results, err := client.Mutate(t.Context(), "records", WithRequests(requests...)) if err != nil { t.Fatal(err) } @@ -886,6 +895,22 @@ func TestClientBatchOfOneResourceKeepsItsOrder(t *testing.T) { t.Fatal(index, result) } } + if applied := peer.drain(); !slices.Equal(applied, want) { + t.Fatal("a resource's records did not keep input order", applied) + } + // Each record waits for the previous one's result, so none shares a message. + counting := &countsMessages{} + countingClient, err := New(serveForTest(t, counting)) + if err != nil { + t.Fatal(err) + } + defer countingClient.Close() + if _, err := countingClient.Mutate(t.Context(), "records", WithRequests(requests...)); err != nil { + t.Fatal(err) + } + if messages := counting.take(); len(messages) != count || slices.Max(messages) != 1 { + t.Fatal("a record did not wait for the previous result of its resource", messages) + } } func TestCanceledBatchEndsWithItsRecordsUnsent(t *testing.T) { @@ -897,10 +922,13 @@ func TestCanceledBatchEndsWithItsRecordsUnsent(t *testing.T) { defer client.Close() ctx, cancel := context.WithCancel(t.Context()) cancel() - options := MutateOptions{StoreName: "records", Requests: []*MutateRequest{putRequest("records/s:a", `{}`)}} - if _, err := client.Mutate(ctx, options); !errors.Is(err, context.Canceled) { + requests := WithRequests(putRequest("records/s:a", `{}`), putRequest("records/s:b", `{}`)) + if _, err := client.Mutate(ctx, "records", requests); !errors.Is(err, context.Canceled) { t.Fatal(err) } + if sent := peer.take(); len(sent) != 0 { + t.Fatal("a canceled call sent records", sent) + } } func TestCloseEndsAStreamWaitingForItsProducer(t *testing.T) { @@ -909,15 +937,17 @@ func TestCloseEndsAStreamWaitingForItsProducer(t *testing.T) { t.Fatal(err) } producing := make(chan struct{}) - options := MutateStreamOptions{StoreName: "records"} - options.Next = func(ctx context.Context) (*MutateRequest, error) { + next := func(ctx context.Context) (*MutateRequest, error) { close(producing) <-ctx.Done() return nil, ctx.Err() } - options.Consume = func(context.Context, uint64, *MutationResult) error { return nil } + consume := func(context.Context, int, *MutationResult) error { return nil } done := make(chan error, 1) - go func() { done <- client.MutateStream(t.Context(), options) }() + go func() { + _, err := client.Mutate(t.Context(), "records", WithNext(next), WithMutateConsume(consume)) + done <- err + }() <-producing if err := client.Close(); err != nil { t.Fatal(err) diff --git a/client_test.go b/client_test.go index 45d07be..43fa818 100644 --- a/client_test.go +++ b/client_test.go @@ -45,6 +45,9 @@ func (p *clientTestPeer) Execute(packed grpc.BidiStreamingServer[pb.ExecuteReque if err := protocol.ValidateExecuteRequest(request); err != nil { return status.Error(codes.InvalidArgument, err.Error()) } + if request.CancelIndex != 0 { + continue + } requestBytes := int64(proto.Size(request)) for previous := p.maxBytes.Load(); requestBytes > previous; previous = p.maxBytes.Load() { if p.maxBytes.CompareAndSwap(previous, requestBytes) { @@ -221,14 +224,17 @@ func clientTestAddress(t *testing.T, peer *clientTestPeer) string { return listener.Addr().String() } -func clientTestConnection(t *testing.T, peer *clientTestPeer) pb.StoreServiceClient { +// clientTestClient returns a Client of peer that sends each record once, so a +// test sees every refusal and counts every send. +func clientTestClient(t *testing.T, peer *clientTestPeer) *Client { t.Helper() - connection, err := Dial("passthrough:///" + clientTestAddress(t, peer)) + client, err := New(clientTestAddress(t, peer)) if err != nil { t.Fatal(err) } - t.Cleanup(func() { _ = connection.Close() }) - return pb.NewStoreServiceClient(connection) + client.resends = nil + t.Cleanup(func() { _ = client.Close() }) + return client } func clientTestReadRequest() *ReadRequest { @@ -236,16 +242,55 @@ func clientTestReadRequest() *ReadRequest { return request } +// readOne returns the result of a Read of one request. +func readOne(ctx context.Context, client *Client, request *ReadRequest) (*ReadResult, error) { + results, err := client.Read(ctx, "records", WithRequests(request)) + if len(results) != 1 { + return nil, fmt.Errorf("read returned %d results: %w", len(results), err) + } + return results[0], err +} + +// mutateOne returns the result of a Mutate of one request. +func mutateOne(ctx context.Context, client *Client, request *MutateRequest, options ...MutateOption) (*MutationResult, error) { + options = append(options, WithRequests(request)) + results, err := client.Mutate(ctx, "records", options...) + if len(results) != 1 { + return nil, fmt.Errorf("mutate returned %d results: %w", len(results), err) + } + return results[0], err +} + +func deleteRequest(resource string) *MutateRequest { + request := &MutateRequest{Resource: resource, Action: MutationDelete} + return request +} + func TestReadRejectsMalformedStreamWithoutEvidence(t *testing.T) { for _, mode := range []string{"invalid_result", "unknown_read_variant", "unknown_event_variant", "read_negative_failure", "missing_result", "zero_index", "skipped_index", "wrong_kind"} { t.Run(mode, func(t *testing.T) { peer := &clientTestPeer{mode: mode} - client := clientTestConnection(t, peer) - options := ReadOneOptions{StoreName: "records", Request: clientTestReadRequest()} - result, err := ReadOne(t.Context(), client, options) + client := clientTestClient(t, peer) + // A peer that never answers leaves a shared stream open until the deadline. + timeout := 5 * time.Second + if mode == "missing_result" { + timeout = 200 * time.Millisecond + } + ctx, cancel := context.WithTimeout(t.Context(), timeout) + defer cancel() + result, err := readOne(ctx, client, clientTestReadRequest()) if err == nil || result != nil { t.Fatal("unvalidated evidence exposed", result, err) } + // A record that needs a stream of its own checks its events the same way. + command, _, err := readCommand(clientTestReadRequest()) + if err != nil { + t.Fatal(err) + } + alone, err := readKind.alone(ctx, client, "records", command) + if err == nil || alone != nil { + t.Fatal("unvalidated evidence exposed on a stream of its own", alone, err) + } }) } } @@ -254,9 +299,8 @@ func TestReadRejectsUnknownFieldsAndFailureCodes(t *testing.T) { for _, mode := range []string{"read_unknown_fields", "read_unknown_fields_final_error", "read_unknown_missing", "read_unknown_failure"} { t.Run(mode, func(t *testing.T) { peer := &clientTestPeer{mode: mode} - client := clientTestConnection(t, peer) - options := ReadOneOptions{StoreName: "records", Request: clientTestReadRequest()} - result, err := ReadOne(t.Context(), client, options) + client := clientTestClient(t, peer) + result, err := readOne(t.Context(), client, clientTestReadRequest()) if err == nil || result != nil { t.Fatal("unknown response accepted as Read evidence", result, err) } @@ -271,10 +315,9 @@ func TestMutationKeepsAppliedEvidenceWithKnownFailureAndLaterError(t *testing.T) for _, mode := range []string{"applied_known_failure", "applied_known_failure_later_error"} { t.Run(mode, func(t *testing.T) { peer := &clientTestPeer{mode: mode} - client := clientTestConnection(t, peer) - request := &MutateRequest{Resource: "records/s:key", Action: MutationDelete} - options := MutateOptions{StoreName: "records", Requests: []*MutateRequest{request, request}} - results, err := Mutate(t.Context(), client, options) + client := clientTestClient(t, peer) + first, second := deleteRequest("records/s:first"), deleteRequest("records/s:second") + results, err := client.Mutate(t.Context(), "records", WithRequests(first, second)) if len(results) != 2 || results[0].GetOutcome() != MutationApplied || results[0].GetFailure().GetCode() != FailureUnavailable { t.Fatal("confirmed application evidence lost", results, err) } @@ -296,10 +339,9 @@ func TestMutationRetainsConfirmedPrefixBeforeUnknownResponse(t *testing.T) { for _, mode := range []string{"applied_then_unknown_fields", "applied_then_unknown_code"} { t.Run(mode, func(t *testing.T) { peer := &clientTestPeer{mode: mode} - client := clientTestConnection(t, peer) - request := &MutateRequest{Resource: "records/s:key", Action: MutationDelete} - options := MutateOptions{StoreName: "records", Requests: []*MutateRequest{request, request}} - results, err := Mutate(t.Context(), client, options) + client := clientTestClient(t, peer) + request := deleteRequest("records/s:key") + results, err := client.Mutate(t.Context(), "records", WithRequests(request, request)) if err == nil || len(results) != 2 || results[0].GetOutcome() != MutationApplied || results[1] != nil { t.Fatal("malformed later response changed confirmed prefix", results, err) } @@ -312,11 +354,10 @@ func TestMutationRetainsConfirmedPrefixBeforeUnknownResponse(t *testing.T) { func TestPutLostResponseHasNoAcknowledgementOrReplay(t *testing.T) { peer := &clientTestPeer{mode: "write_reply_loss"} - client := clientTestConnection(t, peer) + client := clientTestClient(t, peer) document := &Document{ContentType: "application/json", Data: []byte(`{}`)} - request := &WriteRequest{Resource: "records/s:key", Document: document} - options := WriteOptions{StoreName: "records", Request: request} - result, err := Put(t.Context(), client, options) + request := &MutateRequest{Resource: "records/s:key", Action: MutationPut, Document: document} + result, err := mutateOne(t.Context(), client, request) if status.Code(err) != codes.Unavailable || result != nil || peer.streams.Load() != 1 || peer.received.Load() != 1 { t.Fatal("lost response confirmed or replayed mutation", result, err, peer.streams.Load(), peer.received.Load()) } @@ -326,10 +367,8 @@ func TestMutationRejectsInvalidOutcomeAndKeepsAppliedFailure(t *testing.T) { for _, mode := range []string{"invalid_outcome", "unknown_outcome", "negative_failure", "applied_failure", "applied_unknown_failure", "applied_unknown_failure_later_error"} { t.Run(mode, func(t *testing.T) { peer := &clientTestPeer{mode: mode} - client := clientTestConnection(t, peer) - request := &DeleteRequest{Resource: "records/s:key"} - options := DeleteOptions{StoreName: "records", Request: request} - result, err := Delete(t.Context(), client, options) + client := clientTestClient(t, peer) + result, err := mutateOne(t.Context(), client, deleteRequest("records/s:key")) if mode != "applied_failure" { if err == nil || result != nil { t.Fatal("unqualified UNKNOWN accepted", result, err) @@ -343,14 +382,19 @@ func TestMutationRejectsInvalidOutcomeAndKeepsAppliedFailure(t *testing.T) { func TestMutationCancellationStopsCallWithoutReplay(t *testing.T) { peer := &clientTestPeer{mode: "blocked_mutation", canceled: make(chan struct{})} - client := clientTestConnection(t, peer) - request := &DeleteRequest{Resource: "records/s:key"} - options := DeleteOptions{StoreName: "records", Request: request} - ctx, cancel := context.WithTimeout(t.Context(), 50*time.Millisecond) + client := clientTestClient(t, peer) + first, second := deleteRequest("records/s:first"), deleteRequest("records/s:second") + ctx, cancel := context.WithCancel(t.Context()) defer cancel() - result, err := Delete(ctx, client, options) - if err == nil || result != nil { - t.Fatal("blocked mutation succeeded", result, err) + go func() { + for peer.received.Load() == 0 && ctx.Err() == nil { + time.Sleep(time.Millisecond) + } + cancel() + }() + results, err := client.Mutate(ctx, "records", WithRequests(first, second)) + if err == nil || len(results) != 2 || results[0] != nil || results[1] != nil { + t.Fatal("blocked mutation succeeded", results, err) } select { case <-peer.canceled: diff --git a/connection.go b/connection.go index 73a753d..9923442 100644 --- a/connection.go +++ b/connection.go @@ -14,10 +14,10 @@ import ( "github.com/batchstream/weir-protocol/api/protocol" pb "github.com/batchstream/weir-protocol/api/weir/v1" "google.golang.org/grpc" - _ "google.golang.org/grpc/balancer/roundrobin" "google.golang.org/grpc/codes" "google.golang.org/grpc/credentials/insecure" "google.golang.org/grpc/keepalive" + "google.golang.org/grpc/resolver" "google.golang.org/grpc/status" ) @@ -40,11 +40,8 @@ type Client struct { closeErr error } -// Service configs of a Client's own connection and of Dial's. -const ( - endpointConfig = `{"loadBalancingConfig":[{"weir_endpoints":{}}]}` - roundRobinConfig = `{"loadBalancingConfig":[{"round_robin":{}}]}` -) +// endpointConfig is the service config of a Client's connection. +const endpointConfig = `{"loadBalancingConfig":[{"weir_endpoints":{}}]}` // A connection pings a server it has heard nothing from for keepaliveTime and // closes when keepaliveTimeout passes without an answer, so calls leave a @@ -62,14 +59,19 @@ const ( // across them. The Client looks the name up every five seconds: a server // that joins takes its share of streams, and one that stops resolving gets // no new records while its streams finish the ones they carry. -func New(address string) (*Client, error) { +func New(address string, options ...ClientOption) (*Client, error) { if err := validateAddress(address); err != nil { return nil, err } + config := clientOptions{} + for _, option := range options { + if option != nil { + option(&config) + } + } servers := &endpoints{} builder := &endpointResolverBuilder{endpoints: servers, lookup: net.DefaultResolver.LookupHost, interval: resolveInterval, spacing: resolveSpacing} - options := append(connectionOptions(endpointConfig), grpc.WithResolvers(builder)) - connection, err := grpc.NewClient(endpointScheme+":///"+address, options...) + connection, err := grpc.NewClient(endpointScheme+":///"+address, connectionOptions(config.dial, builder)...) if err != nil { return nil, err } @@ -80,13 +82,6 @@ func New(address string) (*Client, error) { return client, nil } -// Dial creates a low-level connection to any gRPC target for the -// package-level operation functions. The caller owns target validation and -// the connection's lifetime. -func Dial(target string) (*grpc.ClientConn, error) { - return grpc.NewClient(target, connectionOptions(roundRobinConfig)...) -} - func validateAddress(address string) error { host, port, err := net.SplitHostPort(address) number, portErr := strconv.Atoi(port) @@ -110,16 +105,21 @@ func validHostname(host string) bool { return true } -func connectionOptions(serviceConfig string) []grpc.DialOption { +// connectionOptions orders a connection's options so that custom ones can replace +// the credentials and keepalive but not what a Client relies on. gRPC takes the +// first resolver of a scheme and the last credentials, keepalive and service +// config. Each stream sets its message sizes itself, which outranks these. +func connectionOptions(custom []grpc.DialOption, servers resolver.Builder) []grpc.DialOption { pings := keepalive.ClientParameters{Time: keepaliveTime, Timeout: keepaliveTimeout, PermitWithoutStream: true} options := []grpc.DialOption{ + grpc.WithResolvers(servers), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithNoProxy(), grpc.WithKeepaliveParams(pings), - grpc.WithDisableRetry(), grpc.WithDisableServiceConfig(), - grpc.WithDefaultServiceConfig(serviceConfig), grpc.WithMaxHeaderListSize(^uint32(0)), grpc.WithDefaultCallOptions(grpc.MaxRetryRPCBufferSize(0), grpc.MaxCallSendMsgSize(protocol.MaxTransportMessageBytes), grpc.MaxCallRecvMsgSize(protocol.MaxTransportMessageBytes)), } + options = append(options, custom...) + options = append(options, grpc.WithDisableRetry(), grpc.WithDisableServiceConfig(), grpc.WithDefaultServiceConfig(endpointConfig)) return options } @@ -130,88 +130,6 @@ func (c *Client) storeClient() (pb.StoreServiceClient, error) { return c.store, nil } -// ReadOne and the other single-record Client methods share a bounded set of -// long-lived streams with concurrent calls of this Client. A canceled call -// stops waiting and cancels only its own record, which may still take effect -// if it is already executing. A record a server refused before it began is -// sent again, up to four sends within the caller's deadline. -func (c *Client) ReadOne(ctx context.Context, options ReadOneOptions) (*ReadResult, error) { - if _, err := c.storeClient(); err != nil { - return nil, err - } - batch := ReadOptions{StoreName: options.StoreName, Requests: []*ReadRequest{options.Request}} - if err := validateReads(ctx, batch); err != nil { - return nil, err - } - operation := &pb.Command_Read{Read: options.Request} - command := &pb.Command{Operation: operation} - key := laneKey{store: options.StoreName, read: true} - send := func() (*ReadResult, error) { - event, shared, err := c.lanes.execute(ctx, key, command) - if !shared { - return c.readAlone(ctx, options.StoreName, options.Request) - } - if err != nil { - return nil, err - } - return readResult(event.GetReadResult()), nil - } - return resendRefused(ctx, c, readRefused, send) -} - -func (c *Client) Create(ctx context.Context, options WriteOptions) (*MutationResult, error) { - request := writeRequest(options.Request, MutationCreate) - return c.mutateOne(ctx, options.StoreName, request) -} - -func (c *Client) Put(ctx context.Context, options WriteOptions) (*MutationResult, error) { - request := writeRequest(options.Request, MutationPut) - return c.mutateOne(ctx, options.StoreName, request) -} - -func (c *Client) Replace(ctx context.Context, options WriteOptions) (*MutationResult, error) { - request := writeRequest(options.Request, MutationReplace) - return c.mutateOne(ctx, options.StoreName, request) -} - -func (c *Client) Delete(ctx context.Context, options DeleteOptions) (*MutationResult, error) { - request := deleteRequest(options.Request) - return c.mutateOne(ctx, options.StoreName, request) -} - -func (c *Client) AtomicTransform(ctx context.Context, options AtomicTransformOptions) (*MutationResult, error) { - request := transformRequest(options.Request) - return c.mutateOne(ctx, options.StoreName, request) -} - -func (c *Client) mutateOne(ctx context.Context, store string, request *MutateRequest) (*MutationResult, error) { - if _, err := c.storeClient(); err != nil { - return nil, err - } - batch := MutateOptions{StoreName: store, Requests: []*MutateRequest{request}} - if err := validateMutations(ctx, batch); err != nil { - return nil, err - } - mutation, err := wireMutation(request) - if err != nil { - return nil, err - } - operation := &pb.Command_Mutate{Mutate: mutation} - command := &pb.Command{Operation: operation} - key := laneKey{store: store} - send := func() (*MutationResult, error) { - event, shared, err := c.lanes.execute(ctx, key, command) - if !shared { - return c.mutateAlone(ctx, store, command) - } - if err != nil { - return nil, err - } - return event.GetMutationResult(), nil - } - return resendRefused(ctx, c, mutationRefused, send) -} - // callError reports a call that ended with its context as the context's error, // whether it used a shared stream or a stream of its own. The server enforces // the propagated deadline too, so its status can arrive before ctx expires. @@ -228,17 +146,6 @@ func callError(ctx context.Context, err error) error { return err } -// Native sends the request again only when a server refused it with -// NativeNotStarted before any response. -func (c *Client) Native(ctx context.Context, options NativeOptions) (*NativeResult, error) { - client, err := c.storeClient() - if err != nil { - return nil, err - } - send := func() (*NativeResult, error) { return Native(ctx, client, options) } - return resendRefused(ctx, c, nativeRefused, send) -} - // Close cancels active requests and closes the connection. Concurrent and // repeated calls wait for the first and return its result. func (c *Client) Close() error { diff --git a/connection_test.go b/connection_test.go index 3161fe1..014913a 100644 --- a/connection_test.go +++ b/connection_test.go @@ -1,12 +1,28 @@ package weir import ( + "context" + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "math/big" + "net" + "os" + "path/filepath" + "strings" + "sync/atomic" "testing" + "time" pb "github.com/batchstream/weir-protocol/api/weir/v1" "google.golang.org/grpc" + "google.golang.org/grpc/credentials" "google.golang.org/grpc/resolver" "google.golang.org/grpc/resolver/manual" + "google.golang.org/grpc/test/bufconn" ) func TestNewRequiresHostAndPort(t *testing.T) { @@ -28,7 +44,7 @@ func TestNewRequiresHostAndPort(t *testing.T) { } // A resolver that returns several servers and its own service config must not -// change the client's round-robin balancing or turn on retries. +// change the client's balancing or turn on retries. func TestConnectionSpreadsCallsAndIgnoresResolverServiceConfig(t *testing.T) { first, second := &clientTestPeer{}, &clientTestPeer{} addresses := []resolver.Address{{Addr: clientTestAddress(t, first)}, {Addr: clientTestAddress(t, second)}} @@ -41,22 +57,237 @@ func TestConnectionSpreadsCallsAndIgnoresResolverServiceConfig(t *testing.T) { state := resolver.State{Addresses: addresses, ServiceConfig: config} builder.InitialState(state) } - options := append(connectionOptions(roundRobinConfig), grpc.WithResolvers(builder)) + options := connectionOptions(nil, builder) connection, err := grpc.NewClient("weirtest:///records", options...) if err != nil { t.Fatal(err) } defer connection.Close() - // Package-level calls open one stream each, so every call is balanced. - store := pb.NewStoreServiceClient(connection) - for range 40 { - request := clientTestReadRequest() - readOptions := ReadOneOptions{StoreName: "records", Request: request} - if _, err := ReadOne(t.Context(), store, readOptions); err != nil { + // A stream of its own is balanced on each call. + client := &Client{store: pb.NewStoreServiceClient(connection)} + command, _, err := readCommand(clientTestReadRequest()) + if err != nil { + t.Fatal(err) + } + calls := int64(0) + deadline := time.Now().Add(5 * time.Second) + for first.streams.Load() == 0 || second.streams.Load() == 0 { + if time.Now().After(deadline) { + t.Fatal("calls did not reach both servers", first.streams.Load(), second.streams.Load()) + } + _, err := readKind.alone(t.Context(), client, "records", command) + if err != nil { t.Fatal(err) } + calls++ + } + if first.streams.Load()+second.streams.Load() != calls { + t.Fatal("calls were not sent exactly once", first.streams.Load(), second.streams.Load(), calls) + } +} + +// countsStreams answers like countsMessages and counts the streams it serves. +type countsStreams struct { + countsMessages + streams atomic.Int64 +} + +func (p *countsStreams) Execute(packed grpc.BidiStreamingServer[pb.ExecuteRequests, pb.ExecuteResponses]) error { + p.streams.Add(1) + return p.countsMessages.Execute(packed) +} + +// sharedAndOwnCalls sends two calls that share a stream and one on a stream of its +// own, and checks that the server served those two streams. +func sharedAndOwnCalls(t *testing.T, client *Client, peer *countsStreams) { + t.Helper() + ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) + defer cancel() + for range 2 { + if _, err := mutateOne(ctx, client, putRequest("records/s:a", `{}`)); err != nil { + t.Fatal("shared stream:", err) + } + } + results, err := client.Mutate(ctx, "records", WithRequests(putRequest("records/s:a", `{}`), putRequest("records/s:b", `{}`))) + if err != nil || results[1].GetOutcome() != MutationApplied { + t.Fatal("own stream:", results, err) + } + if streams := peer.streams.Load(); streams != 2 { + t.Fatal("the calls did not use one shared stream and one of their own", streams) + } +} + +func TestDialOptionsReachAServerThroughACustomDialer(t *testing.T) { + memory := bufconn.Listen(1 << 20) + directory, err := os.MkdirTemp("", "weir") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = os.RemoveAll(directory) }) + socket := filepath.Join(directory, "weir.sock") + unix, err := net.Listen("unix", socket) + if err != nil { + t.Fatal(err) + } + dialers := map[string]struct { + listener net.Listener + dial func(context.Context, string) (net.Conn, error) + }{ + "in memory": {listener: memory, dial: func(ctx context.Context, _ string) (net.Conn, error) { return memory.DialContext(ctx) }}, + "unix socket": {listener: unix, dial: func(ctx context.Context, _ string) (net.Conn, error) { + dialer := net.Dialer{} + return dialer.DialContext(ctx, "unix", socket) + }}, + } + for name, test := range dialers { + t.Run(name, func(t *testing.T) { + server := grpc.NewServer() + peer := &countsStreams{} + pb.RegisterStoreServiceServer(server, peer) + go func() { _ = server.Serve(test.listener) }() + t.Cleanup(server.Stop) + // The address only labels the server the dialer reaches. + client, err := New("127.0.0.1:1", nil, WithDialOptions(nil, grpc.WithContextDialer(test.dial))) + if err != nil { + t.Fatal(err) + } + defer client.Close() + sharedAndOwnCalls(t, client, peer) + }) + } +} + +func TestDialOptionsAddAnInterceptor(t *testing.T) { + var calls atomic.Int64 + interceptor := func(ctx context.Context, desc *grpc.StreamDesc, connection *grpc.ClientConn, method string, streamer grpc.Streamer, options ...grpc.CallOption) (grpc.ClientStream, error) { + calls.Add(1) + return streamer(ctx, desc, connection, method, options...) + } + option := WithDialOptions(grpc.WithStreamInterceptor(interceptor)) + peer := &countsStreams{} + client, err := New(serveForTest(t, peer), option) + if err != nil { + t.Fatal(err) + } + defer client.Close() + sharedAndOwnCalls(t, client, peer) + if calls.Load() == 0 { + t.Fatal("the interceptor did not run") + } +} + +// hidesPeer is a stream whose Context lacks the gRPC peer, as a wrapper's can. +type hidesPeer struct { + grpc.ClientStream +} + +func (hidesPeer) Context() context.Context { return context.Background() } + +func TestAStreamKnowsItsServerBehindAnInterceptorsWrapper(t *testing.T) { + interceptor := func(ctx context.Context, desc *grpc.StreamDesc, connection *grpc.ClientConn, method string, streamer grpc.Streamer, options ...grpc.CallOption) (grpc.ClientStream, error) { + stream, err := streamer(ctx, desc, connection, method, options...) + wrapped := hidesPeer{ClientStream: stream} + return wrapped, err + } + peer := &countsStreams{} + address := serveForTest(t, peer) + client, err := New(address, WithDialOptions(grpc.WithStreamInterceptor(interceptor))) + if err != nil { + t.Fatal(err) + } + defer client.Close() + sharedAndOwnCalls(t, client, peer) + client.lanes.mu.Lock() + defer client.lanes.mu.Unlock() + key := laneKey{store: "records"} + group := client.lanes.groups[key] + if group == nil || len(group.lanes) != 1 || group.lanes[0].address != address { + t.Fatal("a stream cannot be matched to the server that leaves", group) + } +} + +func TestDialOptionsCannotReplaceWhatAClientReliesOn(t *testing.T) { + // Neither a resolver that never answers nor another balancer may take over. + other := manual.NewBuilderWithScheme(endpointScheme) + balancer := grpc.WithDefaultServiceConfig(`{"loadBalancingConfig":[{"round_robin":{}}]}`) + peer := &countsStreams{} + client, err := New(serveForTest(t, peer), WithDialOptions(grpc.WithResolvers(other), balancer)) + if err != nil { + t.Fatal(err) + } + defer client.Close() + sharedAndOwnCalls(t, client, peer) + // The balancer reports its ready servers just after its picker takes over. + deadline := time.Now().Add(5 * time.Second) + for len(client.lanes.endpoints.usable()) == 0 { + if time.Now().After(deadline) { + t.Fatal("another balancer replaced the Client's: it knows no ready server") + } + time.Sleep(time.Millisecond) + } +} + +// tlsTestServer serves TLS with a certificate for one name or IP, and returns the +// pool that trusts it and the server's address, an IP literal with a single server. +func tlsTestServer(t *testing.T, name string, peer pb.StoreServiceServer) (*x509.CertPool, string) { + t.Helper() + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + t.Fatal(err) + } + subject := pkix.Name{CommonName: name} + template := &x509.Certificate{SerialNumber: big.NewInt(1), Subject: subject, NotBefore: time.Now().Add(-time.Hour), NotAfter: time.Now().Add(time.Hour), IsCA: true, BasicConstraintsValid: true, KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign} + if ip := net.ParseIP(name); ip != nil { + template.IPAddresses = []net.IP{ip} + } else { + template.DNSNames = []string{name} + } + der, err := x509.CreateCertificate(rand.Reader, template, template, &key.PublicKey, key) + if err != nil { + t.Fatal(err) + } + certificate, err := x509.ParseCertificate(der) + if err != nil { + t.Fatal(err) + } + pool := x509.NewCertPool() + pool.AddCert(certificate) + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + chain := tls.Certificate{Certificate: [][]byte{der}, PrivateKey: key} + config := &tls.Config{Certificates: []tls.Certificate{chain}} + server := grpc.NewServer(grpc.Creds(credentials.NewTLS(config))) + pb.RegisterStoreServiceServer(server, peer) + go func() { _ = server.Serve(listener) }() + t.Cleanup(server.Stop) + return pool, listener.Addr().String() +} + +func TestDialOptionsReplaceCredentialsAndTLSChecksTheAddressName(t *testing.T) { + peer := &countsStreams{} + pool, address := tlsTestServer(t, "127.0.0.1", peer) + trusted := grpc.WithTransportCredentials(credentials.NewClientTLSFromCert(pool, "")) + client, err := New(address, WithDialOptions(trusted)) + if err != nil { + t.Fatal(err) + } + defer client.Close() + sharedAndOwnCalls(t, client, peer) + + otherPeer := &countsStreams{} + otherPool, otherAddress := tlsTestServer(t, "other.example", otherPeer) + mismatched := grpc.WithTransportCredentials(credentials.NewClientTLSFromCert(otherPool, "")) + other, err := New(otherAddress, WithDialOptions(mismatched)) + if err != nil { + t.Fatal(err) } - if first.streams.Load() == 0 || second.streams.Load() == 0 || first.streams.Load()+second.streams.Load() != 40 { - t.Fatal("calls were not spread across both servers exactly once", first.streams.Load(), second.streams.Load()) + defer other.Close() + ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second) + defer cancel() + _, err = mutateOne(ctx, other, putRequest("records/s:a", `{}`)) + if err == nil || !strings.Contains(err.Error(), "certificate") || otherPeer.streams.Load() != 0 { + t.Fatal("a certificate for another name was accepted, or another error hid it", err, otherPeer.streams.Load()) } } diff --git a/docs/connections.md b/docs/connections.md index a63f968..3aeaeea 100644 --- a/docs/connections.md +++ b/docs/connections.md @@ -11,9 +11,9 @@ deployments. `New` takes a `host:port` address and rejects anything else. It connects on first use and reconnects after failures. When a DNS name resolves to several Weir servers, streams are spread round-robin across them, so every server behind -the address must host the Stores the client uses. Each Read, Mutate, record -stream or Native call uses a gRPC stream of its own; single-record calls share -streams, as described under [Streaming and deadlines](streaming.md). Use a name +the address must host the Stores the client uses. A Read or Mutate of one +request usually shares streams, and any other call uses a gRPC stream of its own, +as described under [Streaming and deadlines](streaming.md). Use a name that lists every server, such as a Kubernetes headless Service; a ClusterIP Service resolves to one virtual IP. The Client looks the name up again every 5 seconds. A server that joins receives new calls once connected, and shared @@ -26,14 +26,43 @@ few seconds before it stops taking records, as the Weir chart's preStop delay arranges. A server that does not host the requested Store fails the call with NotFound. +## gRPC options + +`WithDialOptions` adds gRPC options to a Client's connection, such as a stats +handler, an interceptor or a custom dialer: + +```go +client, err := weir.New("weir:7447", weir.WithDialOptions(grpc.WithStatsHandler(handler))) +``` + +They can replace the Client's defaults for transport credentials and keepalive. +They cannot change what the Client relies on: gRPC never resends a record, message +sizes stay at the protocol's limit, service configs, including +`grpc.WithDefaultServiceConfig`, have no effect, and the Client's own resolver and +balancer follow its servers. + +Stats handlers and interceptors see gRPC streams, not calls. A Read or Mutate sends +its records over streams that can outlive it, and a shared stream carries the +records of many calls, so they see none of a call's context values or deadline. +They suit stream metrics and logging, not per-call tracing. Only Native and a record +that needs a gRPC stream of its own run with the caller's context. + +The address stays `host:port`, so `New` accepts no other gRPC target. A custom +dialer can reach a Unix socket or an in-memory server, with an IP literal and any +port as the address. + +The Weir application listener is plaintext, so TLS applies only to a proxy or +sidecar in front of it. `grpc.WithTransportCredentials` then checks the certificate +against the host name of the address. `grpc.WithCredentialsBundle` conflicts with +the default credentials, so `New` returns an error for it. + ## Reuse and close Reuse the client across goroutines and close it after callers finish. Calls started after `Close` return `ErrClosed`; calls still running end with a -gRPC cancellation error. A call of a Client whose context ends returns the -context's error, so test it with `errors.Is` rather than its gRPC status; the -package operation functions return the gRPC status. A Client sends a record again only when a server refused it +gRPC cancellation error. A call whose context ends returns the context's +error, so test it with `errors.Is` rather than its gRPC status. A Client sends a record again only when a server refused it before it began, as described under [Resending refused records](streaming.md#resending-refused-records). diff --git a/docs/development.md b/docs/development.md index 582757c..21a13e0 100644 --- a/docs/development.md +++ b/docs/development.md @@ -13,7 +13,7 @@ generate schemas. ## Examples [read](../examples/read/main.go) performs a typed read, [basic](../examples/basic/main.go) -mutates a document then reads multiple resources in one stream, and +mutates a document then reads multiple resources in one call, and [native](../examples/native/main.go) consumes native responses. These examples use SDK request and result types. diff --git a/docs/lua.md b/docs/lua.md index 95de993..5c817bb 100644 --- a/docs/lua.md +++ b/docs/lua.md @@ -1,6 +1,6 @@ # Lua transforms -How to change a record with a Lua function through `AtomicTransform`. +How to change a record with a Lua function through `MutationAtomicTransform`. `LuaTransform` accepts UTF-8 Source and optional Input. Source must return exactly one function, called with ordinary Lua tables as `function(current, incoming)`. @@ -16,9 +16,8 @@ source := []byte(`return function(current, incoming) return current end`) lua := &weir.LuaTransform{Source: source, Input: input} -request := &weir.AtomicTransformRequest{Resource: "records/s:first", Lua: lua} -options := weir.AtomicTransformOptions{StoreName: "search", Request: request} -result, err := client.AtomicTransform(ctx, options) +request := &weir.MutateRequest{Resource: "records/s:first", Action: weir.MutationAtomicTransform, Lua: lua} +results, err := client.Mutate(ctx, "search", weir.WithRequests(request)) ``` Mongo Input uses BSON. Lua source and document values have no Weir size, depth, diff --git a/docs/native.md b/docs/native.md index 8c872ee..2e6b12f 100644 --- a/docs/native.md +++ b/docs/native.md @@ -18,16 +18,15 @@ document. Search uses `application/http` and an optional resource prefix. ```go document := &weir.Document{ContentType: "application/vnd.example.command", Data: commandBytes} request := &weir.NativeRequest{Resource: "records", Request: document} -options := weir.NativeOptions{StoreName: "example", Request: request} -options.Consume = func(ctx context.Context, response *weir.NativeResponse, data []byte) error { - return consumeNativeBytes(ctx, response, data) -} -result, err := client.Native(ctx, options) +result, err := client.Native(ctx, "example", request) +// result.Body holds the response body, as far as it arrived. ``` ## Responses -Consume receives nonempty response chunks and finishes before the next chunk. +`NativeResult.Body` holds the response body, as far as it arrived. To avoid +retaining a large one, pass `WithChunks`: its callback receives nonempty response +chunks and finishes before the next chunk, and Body stays nil. `NativeResult.Response` also retains metadata for an empty body when no callback runs. Response.Metadata is an optional opaque Document; Response.BodyContentType describes the separately streamed body. Metadata may @@ -49,17 +48,16 @@ httpRequest, err := http.NewRequestWithContext(ctx, http.MethodGet, "/_doc/examp if err != nil { return err } request, err := weir.NewHTTPNativeRequest("records", httpRequest) if err != nil { return err } -options := weir.NativeOptions{StoreName: "search", Request: request} -options.Consume = func(ctx context.Context, response *weir.NativeResponse, data []byte) error { +consume := func(ctx context.Context, response *weir.NativeResponse, data []byte) error { return consumeNativeBytes(ctx, response, data) } -result, err := client.Native(ctx, options) +result, err := client.Native(ctx, "search", request, weir.WithChunks(consume)) ``` `ParseHTTPNativeResponse(response)` optionally parses application/http metadata containing a complete status line and CRLF header block, with no body bytes or transfer framing. It returns HTTPNativeResponse with StatusCode and Headers; -response body bytes continue through Native's chunk consumer. These helpers are +response body bytes stay in `Body` or go to the `WithChunks` callback. These helpers are ordinary HTTP codecs, independent of any Store type or protobuf backend fields. ## Completion evidence diff --git a/docs/operations.md b/docs/operations.md index 3177a3b..5fb49dc 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -15,8 +15,7 @@ defer client.Close() first := &weir.ReadRequest{Resource: "records/s:example"} second := &weir.ReadRequest{Resource: "records/s:another"} -readOptions := weir.ReadOptions{StoreName: "search", Requests: []*weir.ReadRequest{first, second}} -results, err := client.Read(ctx, readOptions) +results, err := client.Read(ctx, "search", weir.WithRequests(first, second)) if err != nil { return fmt.Errorf("read RPC failed; unacknowledged results %v: %w", results, err) } for index, result := range results { if result.Failure != nil { return fmt.Errorf("read %d failed: %v", index, result.Failure) } @@ -26,7 +25,7 @@ for index, result := range results { ## Resources and documents -Resources are canonical paths relative to `StoreName`. +Resources are canonical paths relative to their Store. `weir.EncodeSegment` canonically encodes each decoded path segment. MongoDB paths are `DATABASE/COLLECTION/KEY`; Search paths are `INDEX/KEY`. Mongo documents use `application/bson` with the first `_id` matching the resource key. Search documents @@ -34,32 +33,33 @@ use `application/json`. The caller owns its document codec; the SDK preserves by ## Methods -Each method takes named options with `StoreName`. `Read` and `Mutate` accept -`Requests`; single-operation helpers accept `Request`: +Each method takes a context and a Store, then options: -| Method | Options / request | Result | +| Method | Options | Result | | --- | --- | --- | -| `Read` | `ReadOptions` / multiple `ReadRequest` | `[]*ReadResult`: Document, Missing, Failure | -| `Mutate` | `MutateOptions` / multiple `MutateRequest` | `[]*MutationResult` | -| `ReadStream` | `ReadStreamOptions` / producer and consumer | per-item `ReadResult` callbacks | -| `MutateStream` | `MutateStreamOptions` / producer and consumer | per-item `MutationResult` callbacks | -| `ReadOne` | `ReadOneOptions` / `ReadRequest` | `ReadResult` | -| `Create`, `Put`, `Replace` | `WriteOptions` / `WriteRequest` | `MutationResult` | -| `Delete` | `DeleteOptions` / `DeleteRequest` | `MutationResult` | -| `AtomicTransform` | `AtomicTransformOptions` / `AtomicTransformRequest` | `MutationResult` | -| `Native` | `NativeOptions` / typed `NativeRequest`, byte consumer | `NativeResult` | +| `Read` | `WithRequests`, `WithNext`, `WithReadConsume` | `[]*ReadResult`: Document, Missing, Failure | +| `Mutate` | `WithRequests`, `WithNext`, `WithMutateConsume`, `WithQueryVisible` | `[]*MutationResult` | +| `Native` | `WithChunks`, after a required `NativeRequest` | `NativeResult` | ## Read and Mutate -`Read` and `Mutate` validate the complete input before sending anything. An empty -input, nil request, malformed resource, unsupported action or invalid document -envelope rejects the entire call without sending earlier valid items. Every -resource is relative to one `StoreName`. Each input is sent as one +`WithRequests` passes the records a call already holds, and `WithNext` a callback +that produces more, as described under +[Streaming and deadlines](streaming.md#withnext-and-consume-callbacks). The call +sends the requests first, then what Next produces, and needs at least one record. +Several `WithRequests` add their requests in order; a later `WithNext` or consume +callback replaces an earlier one. + +A call validates every request of `WithRequests` before sending anything and +before calling Next. A nil request, malformed resource, unsupported action or +invalid document envelope rejects the entire call without sending earlier valid +items. Every resource is relative to the call's Store. Each input is sent as one ExecuteRequest as soon as the window and an earlier mutation of its resource allow. -Results arrive as records complete and are returned in input order, including -repeated resources. Individual backend failures stay in their result positions. A later -RPC failure preserves every validated result already received; unconfirmed -positions remain nil and any such mutation may have applied. Never + +Without a consume callback, results return in input order, one per input, +including repeated resources. Individual backend failures stay in their result +positions. A later RPC failure preserves every validated result already received; +unconfirmed positions remain nil and any such mutation may have applied. Never automatically replay those mutations. ```go @@ -67,18 +67,22 @@ firstDoc := &weir.Document{ContentType: "application/json", Data: []byte(`{"n":1 secondDoc := &weir.Document{ContentType: "application/json", Data: []byte(`{"n":2}`)} first := &weir.MutateRequest{Resource: "records/s:first", Action: weir.MutationPut, Document: firstDoc} second := &weir.MutateRequest{Resource: "records/s:second", Action: weir.MutationPut, Document: secondDoc} -options := weir.MutateOptions{StoreName: "search", Requests: []*weir.MutateRequest{first, second}} -results, err := client.Mutate(ctx, options) +results, err := client.Mutate(ctx, "search", weir.WithRequests(first, second)) // Inspect every result's Outcome/Failure, even when err is nonnil. Never replay // an unacknowledged mutation automatically. ``` +Keep requests and their document bytes unchanged until the call returns. After a +call whose context ended, do not reuse them at all: a single record's shared +stream may still be reading them. + ## Actions and ordering `MutateRequest.Action` selects `MutationCreate`, `MutationPut`, `MutationReplace`, -`MutationDelete` or `MutationAtomicTransform`. Writes require Document, Delete -accepts no payload, and AtomicTransform requires exactly one Lua or -BackendExpression. A batch is not a transaction. Mutations to the same resource +`MutationDelete` or `MutationAtomicTransform`. Create requires absence, Replace +requires an existing resource and Put creates or replaces; all three require +Document. Delete accepts no payload, and AtomicTransform requires exactly one Lua +or BackendExpression. A call is not a transaction. Mutations to the same resource execute in input order, including after an item failure; different resources may run concurrently. Across RPCs, ordering follows the database semantics. @@ -89,34 +93,24 @@ read or mutation and each response carries its indexed result. Backend protocols unacknowledged encoded input; a larger legal request uses the window alone. Credit returns after each Consume callback finishes. A size-dependent wait can hold one produced item that has not yet been sent. Sending and receiving run -concurrently so HTTP/2 flow control advances in both directions. The SDK never -waits for another input or EOF before sending the current item. -`Read` and `Mutate` intentionally accumulate results for their caller; use the -streaming interfaces to avoid retaining the full input and result sequence. - -## Write requests and visibility - -`WriteRequest` contains Resource and Document. Create requires absence; Replace -requires an existing resource; Put creates or replaces. AtomicTransform requires -exactly one Lua or BackendExpression. `MutateRequest`, `WriteRequest`, -`DeleteRequest` and `AtomicTransformRequest` share `Options *MutationOptions`. -Nil options or `MutationApplicationAcknowledged` (the zero Visibility value) -wait for application acknowledgement. Set Visibility to `MutationQueryVisible` -when subsequent normal queries must observe the mutation before the operation -succeeds. +concurrently so HTTP/2 flow control advances in both directions. A call with +`WithNext` never waits for another input or EOF before sending the current item. +Without a consume callback a call accumulates results for its caller; use `WithNext` +and a consume callback to avoid retaining the full input and result sequence. + +## Visibility + +Without `WithQueryVisible` a mutation waits for application acknowledgement. Pass +`WithQueryVisible` when subsequent normal queries must observe the mutations of a +call before they succeed; it applies to every mutation of that call. An APPLIED result without Failure confirms that guarantee; APPLIED with Failure confirms application only and must not be replayed automatically. This includes Lua keep and missing-document deletes. Search adapters wait for refresh; backends without deferred query visibility satisfy this when the mutation is acknowledged. -The SDK forwards the complete options message and rejects unknown option fields -or undeclared visibility values. -Keep options immutable until the call returns or its result is consumed. ```go -completion := &weir.MutationOptions{Visibility: weir.MutationQueryVisible} -request := &weir.WriteRequest{Resource: "records/s:first", Document: document, Options: completion} -options := weir.WriteOptions{StoreName: "search", Request: request} -result, err := client.Put(ctx, options) +request := &weir.MutateRequest{Resource: "records/s:first", Action: weir.MutationPut, Document: document} +results, err := client.Mutate(ctx, "search", weir.WithRequests(request), weir.WithQueryVisible()) ``` ## Read results diff --git a/docs/streaming.md b/docs/streaming.md index 4076323..214a96d 100644 --- a/docs/streaming.md +++ b/docs/streaming.md @@ -9,64 +9,70 @@ one Store and one operation kind. Native sends one request, closes its input and consumes response metadata and body chunks incrementally. Transport and backend protocols govern representable messages and accepted data. -## ReadStream and MutateStream - -`ReadStream` and `MutateStream` take `Next` and `Consume` callbacks. `Next(ctx)` -returns one request, or `io.EOF` to end the finite input. `Consume(ctx, index, -result)` receives each confirmed result as its record completes; the index, -starting at 1, identifies the input. -Inputs are validated as they are produced; an invalid later item does not undo -earlier requests. A Client's stream succeeds with one result per submitted input; -a package function's also requires a final gRPC OK. Results delivered before a -later producer, consumer or transport error remain evidence, and a Client's stream -goes on delivering results it already received after such an error, unless -Consume itself failed. Next and Consume run concurrently; synchronize shared -application state. Callbacks must honor their context and return promptly; -the SDK cancels and joins its sender before returning. Keep yielded requests and -document bytes immutable until the corresponding Consume callback begins. +## WithNext and consume callbacks + +`WithNext` takes a callback whose `next(ctx)` returns one request, or `io.EOF` to +end the finite input. `WithReadConsume` and `WithMutateConsume` take a callback +whose `consume(ctx, index, result)` receives each confirmed result as its record +completes, so not in input order. The index, starting at 0, identifies the record +across `WithRequests` and `WithNext`. With a consume callback the call collects +nothing and returns nil results. Either option works without the other: Next +without a consume callback still returns every result, in input order, with a nil +position for each record that has no result. + +Next starts once the requests of `WithRequests` have entered the window, without +waiting for their results. Mutations of one resource keep input order across both. +Next's requests are validated as they are produced. An invalid item or another +error from Next ends the input there: the records before it still finish, and the +call then returns that error. A call of a Client succeeds with one result per submitted +input. Results delivered before a later producer, consumer or transport error +remain evidence, and a call goes on delivering results it already received after +such an error, unless the consume callback itself failed. Next and the consume +callback run concurrently; synchronize shared application state. Callbacks must +honor their context and return promptly; the SDK cancels and joins its sender +before returning. Keep yielded requests and document bytes immutable until the +call returns, or with a consume callback until the corresponding callback begins. ```go position := 0 -options := weir.ReadStreamOptions{StoreName: "search"} -options.Next = func(ctx context.Context) (*weir.ReadRequest, error) { +next := func(ctx context.Context) (*weir.ReadRequest, error) { if position == len(resourceNames) { return nil, io.EOF } request := &weir.ReadRequest{Resource: resourceNames[position]} position++ return request, nil } -options.Consume = func(ctx context.Context, index uint64, result *weir.ReadResult) error { +consume := func(ctx context.Context, index int, result *weir.ReadResult) error { if result.Failure != nil { return fmt.Errorf("read %d failed: %v", index, result.Failure) } if result.Missing { return nil } return processDocument(ctx, result.Document) } -err := client.ReadStream(ctx, options) +_, err := client.Read(ctx, "search", weir.WithNext(next), weir.WithReadConsume(consume)) ``` Consumers should commit or release each result promptly and avoid retaining unbounded events. -## Read, Mutate, ReadStream and MutateStream +## Calls with a stream of their own -A Client's `Read`, `Mutate`, `ReadStream` and `MutateStream` each send their -records over a gRPC stream of their own, and replace it with one to another server +A `Read` or `Mutate` that does not share streams, as described below, sends its +records over a gRPC stream of its own, and replaces it with one to another server when its server stops resolving or refuses a record, and every minute, so a call may stay open for as long as its producer has records, and an idle one holds no gRPC stream after 10 seconds. A record over 1 MiB uses a further gRPC stream of -its own. `Read` and `Mutate` send the records that fit their window of 32 records -and 8 MiB together, in messages of up to 1 MiB. A stream's mutations of one resource run in input order, -each after the previous one's result, including after a failure; records of -different resources may arrive at the server in any order. A broken gRPC stream -ends the call with its error, and mutations without a result may have applied. -The package operation functions keep one gRPC stream for the whole call, end -with an error when its server goes away and leave following servers to their -caller. +its own. A call without `WithNext` sends the records that fit its window of 32 +records and 8 MiB together, in messages of up to 1 MiB; a call with `WithNext` may +wait on it, so it sends each record as soon as it is produced. A stream's mutations +of one resource run in input order, each after the previous one's result, including +after a failure; records of different resources may arrive at the server in any +order. A broken gRPC stream ends the call with its error, and mutations without a +result may have applied. ## Shared streams of single-record calls -The single-record Client methods (`ReadOne`, `Create`, `Put`, `Replace`, -`Delete` and `AtomicTransform`) share long-lived streams with concurrent calls -of the same Client, so many goroutines in one process pay for a few gRPC streams -instead of one per call. A stream sends the records queued on it together in one +A `Read` or `Mutate` of exactly one request, without `WithNext`, a consume callback +or outgoing gRPC metadata, shares long-lived streams with concurrent calls of the +same Client, so many goroutines in one process pay for a few gRPC streams instead +of one per call. A stream sends the records queued on it together in one message of up to 1 MiB, and the server returns results that complete together in one message, so a busy stream pays for a message per burst rather than per record. Each Store and operation kind takes new records on at @@ -79,7 +85,8 @@ server rejects fails alone, but a broken stream fails every record outstanding on it. A call's deadline travels with its record, so the server drops it if it is still queued when the deadline passes. Canceling a call stops waiting and asks the server to cancel only that record; as with any canceled call, a record -already executing may still take effect. When the server refuses a record +already executing may still take effect. The stream may also still be reading the +record's document after the call returns, so do not reuse its bytes. When the server refuses a record without starting it, as a draining or overloaded server or an unavailable backend does, the refusal returns to the caller. A server that turns a new gRPC stream away before reading it, as a draining or overloaded one does, says so with @@ -94,15 +101,7 @@ Shared streams do not move to a server that refused a record, or rejected a stream before answering one, within those 10 seconds. Streams retire after a minute, which lets the server's connection age rebalance clients behind one virtual IP, and idle streams close after 10 seconds. Shared streams do not carry outgoing gRPC metadata from a call's -context; package operation functions keep one stream per call and send it. - -## Dial and package functions - -`Dial` uses gRPC's normal adaptive flow control and reusable connections. Package -operation functions accept a generated StoreService client for custom -connections and protocol fixtures; ordinary applications use New and typed Client -methods. `Dial` accepts any gRPC target; the caller owns its validation and the -connection lifetime. +context; a call with outgoing metadata uses a stream of its own and sends it. ## Resending refused records @@ -125,6 +124,5 @@ first sends plus a burst of 100, so a server that refuses everything never receives several times its normal traffic. A record still refused at the end keeps its NOT_STARTED evidence, and later mutations of its resource in the same call still run after it. The caller may send it again, unless such a mutation -did not end NOT_STARTED; sending it then would apply it after that mutation. The -package operation functions never resend. Use caller deadlines; they bound -resends as well as calls. +did not end NOT_STARTED; sending it then would apply it after that mutation. Use +caller deadlines; they bound resends as well as calls. diff --git a/endpoints.go b/endpoints.go index 4fd1dda..8ece1f8 100644 --- a/endpoints.go +++ b/endpoints.go @@ -56,6 +56,9 @@ const ( endpointsAttribute endpointKey = iota // endpointPreference carries the server a shared stream asks for. endpointPreference + // endpointPicked carries where the picker records the server it chose for a + // stream, as a stream's peer depends on its dialer and interceptors. + endpointPicked ) func (e *endpoints) onLeave(leave func(gone []string)) { @@ -194,14 +197,15 @@ type endpointPicker struct { } func (p *endpointPicker) Pick(info balancer.PickInfo) (balancer.PickResult, error) { - if address, ok := info.Ctx.Value(endpointPreference).(string); ok { - if subConn := p.byAddress[address]; subConn != nil { - preferred := balancer.PickResult{SubConn: subConn} - return preferred, nil - } + address, preferred := info.Ctx.Value(endpointPreference).(string) + if !preferred || p.byAddress[address] == nil { + index := p.next.Add(1) % uint32(len(p.addresses)) + address = p.addresses[index] + } + if picked, ok := info.Ctx.Value(endpointPicked).(*atomic.Pointer[string]); ok { + picked.Store(&address) } - index := p.next.Add(1) % uint32(len(p.addresses)) - result := balancer.PickResult{SubConn: p.byAddress[p.addresses[index]]} + result := balancer.PickResult{SubConn: p.byAddress[address]} return result, nil } @@ -220,7 +224,7 @@ func (b *endpointResolverBuilder) Build(target resolver.Target, cc resolver.Clie if err != nil { return nil, err } - // A stream's peer address has a canonical port, so the port must be too. + // One server has one address, so its port is canonical. number, err := strconv.ParseUint(port, 10, 16) if err != nil { return nil, err @@ -293,7 +297,7 @@ func (r *endpointResolver) lookup() ([]string, error) { } addresses := make([]string, len(hosts)) for i, host := range hosts { - // A stream's peer address is canonical, so IP literals must be too. + // One server has one address, so IP literals are canonical. if ip, err := netip.ParseAddr(host); err == nil { host = ip.Unmap().String() } diff --git a/examples/basic/main.go b/examples/basic/main.go index c15ba15..7342616 100644 --- a/examples/basic/main.go +++ b/examples/basic/main.go @@ -44,12 +44,10 @@ func run() error { target = weir.EncodeSegment(*index) + "/s:example" document = &weir.Document{ContentType: "application/json", Data: []byte(`{"n":1}`)} } - completion := &weir.MutationOptions{Visibility: weir.MutationQueryVisible} - request := &weir.MutateRequest{Resource: target, Action: weir.MutationPut, Document: document, Options: completion} - opts := weir.MutateOptions{StoreName: *storeName, Requests: []*weir.MutateRequest{request}} - results, err := client.Mutate(ctx, opts) + request := &weir.MutateRequest{Resource: target, Action: weir.MutationPut, Document: document} + results, err := client.Mutate(ctx, *storeName, weir.WithRequests(request), weir.WithQueryVisible()) if err != nil { - if len(results) != 0 && results[0] != nil { + if results[0] != nil { return fmt.Errorf("write RPC incomplete; backend evidence=%v: %w", results[0], err) } return fmt.Errorf("write result unavailable; effects indeterminate: %w", err) @@ -57,12 +55,12 @@ func run() error { if results[0].GetOutcome() != weir.MutationApplied || results[0].GetFailure() != nil { return fmt.Errorf("write: %v", results[0]) } - batch := weir.ReadOptions{StoreName: *storeName} + var batch []*weir.ReadRequest for range *count { read := &weir.ReadRequest{Resource: target} - batch.Requests = append(batch.Requests, read) + batch = append(batch, read) } - reads, err := client.Read(ctx, batch) + reads, err := client.Read(ctx, *storeName, weir.WithRequests(batch...)) if err != nil { return fmt.Errorf("read RPC incomplete: %w", err) } diff --git a/examples/native/main.go b/examples/native/main.go index a963432..dec46c4 100644 --- a/examples/native/main.go +++ b/examples/native/main.go @@ -59,14 +59,13 @@ func run() error { } defer client.Close() total := 0 - opts := weir.NativeOptions{StoreName: *store, Request: request} - opts.Consume = func(_ context.Context, response *weir.NativeResponse, data []byte) error { + consume := func(_ context.Context, response *weir.NativeResponse, data []byte) error { fmt.Printf("metadata=%v content-type=%s chunk=%d bytes\n", response.Metadata, response.BodyContentType, len(data)) total += len(data) // Consume and discard each chunk instead of retaining the whole response. return nil } - result, err := client.Native(ctx, opts) + result, err := client.Native(ctx, *store, request, weir.WithChunks(consume)) if result != nil && result.Completion == weir.NativeNotStarted { return fmt.Errorf("native request was not started: %v; RPC error=%v", result.Failure, err) } diff --git a/examples/read/main.go b/examples/read/main.go index bc48cb8..69e541b 100644 --- a/examples/read/main.go +++ b/examples/read/main.go @@ -22,12 +22,12 @@ func main() { panic(err) } defer client.Close() - readOptions := weir.ReadOptions{StoreName: *store} + var requests []*weir.ReadRequest for _, resource := range strings.Split(*resources, ",") { request := &weir.ReadRequest{Resource: resource} - readOptions.Requests = append(readOptions.Requests, request) + requests = append(requests, request) } - reads, err := client.Read(ctx, readOptions) + reads, err := client.Read(ctx, *store, weir.WithRequests(requests...)) if err != nil { panic(err) } diff --git a/http_native.go b/http_native.go index 6a016a2..44801fe 100644 --- a/http_native.go +++ b/http_native.go @@ -12,7 +12,8 @@ import ( ) // HTTPNativeResponse describes an application/http metadata document. Its body -// belongs to the Native chunk consumer and is never retained by this helper. +// stays in NativeResult.Body or goes to the WithChunks callback, and this helper +// never retains it. type HTTPNativeResponse struct { StatusCode int Headers http.Header diff --git a/integration_test.go b/integration_test.go index 2a3e50b..aed88f8 100644 --- a/integration_test.go +++ b/integration_test.go @@ -5,7 +5,6 @@ package weir_test import ( "context" "encoding/json" - "errors" "fmt" "io" "net/http" @@ -65,7 +64,7 @@ func testLifecycle(t *testing.T, backend string) { defer client.Close() id := fmt.Sprintf("sdk-%s-%d", backend, time.Now().UnixNano()) builder := mutationBuilder{t: t, backend: backend, resource: collection + "/s:" + id, id: id} - write := weir.WriteOptions{StoreName: store, Request: builder.write(1)} + remove := &weir.MutateRequest{Resource: builder.resource, Action: weir.MutationDelete} deleted := false t.Cleanup(func() { if deleted { @@ -79,38 +78,32 @@ func testLifecycle(t *testing.T, backend string) { return } defer cleanupClient.Close() - request := &weir.DeleteRequest{Resource: builder.resource} - remove := weir.DeleteOptions{StoreName: store, Request: request} - result, err := cleanupClient.Delete(cleanup, remove) + result, err := mutateOne(cleanup, cleanupClient, store, remove) if err != nil || result.GetOutcome() != weir.MutationApplied || result.GetFailure() != nil { t.Error("owned record cleanup:", result, err) } }) - result, err := client.Create(ctx, write) + create := builder.write(weir.MutationCreate, 1) + result, err := mutateOne(ctx, client, store, create) applied(t, result, err) - duplicate, err := client.Create(ctx, write) + duplicate, err := mutateOne(ctx, client, store, create) if err != nil || duplicate.GetOutcome() != weir.MutationNotApplied || duplicate.GetFailure().GetCode() != weir.FailurePreconditionFailed { t.Fatalf("duplicate create: %v %v", duplicate, err) } - write.Request = builder.write(4) - result, err = client.Replace(ctx, write) + result, err = mutateOne(ctx, client, store, builder.write(weir.MutationReplace, 4)) applied(t, result, err) - result, err = client.Put(ctx, write) + result, err = mutateOne(ctx, client, store, builder.write(weir.MutationPut, 4)) applied(t, result, err) - transform := weir.AtomicTransformOptions{StoreName: store, Request: builder.transform(4)} - result, err = client.AtomicTransform(ctx, transform) + result, err = mutateOne(ctx, client, store, builder.transform(4)) applied(t, result, err) read := &weir.ReadRequest{Resource: builder.resource} - readOptions := weir.ReadOneOptions{StoreName: store, Request: read} - readResult, err := client.ReadOne(ctx, readOptions) + readResult, err := readOne(ctx, client, store, read) if err != nil || readResult == nil || readResult.Document == nil || number(t, backend, readResult.Document.GetData()) != 4 { t.Fatalf("persisted read: %v %v", readResult, err) } - write.Request = builder.write(5) - result, err = client.Replace(ctx, write) + result, err = mutateOne(ctx, client, store, builder.write(weir.MutationReplace, 5)) applied(t, result, err) - reads := weir.ReadOptions{StoreName: store, Requests: []*weir.ReadRequest{read, read}} - batchResults, err := client.Read(ctx, reads) + batchResults, err := client.Read(ctx, store, weir.WithRequests(read, read)) if err != nil || len(batchResults) != 2 { t.Fatal("batch read failed", batchResults, err) } @@ -120,21 +113,12 @@ func testLifecycle(t *testing.T, backend string) { } } native := nativeRequest(t, backend, collection, id) - var body []byte - nativeOptions := weir.NativeOptions{StoreName: store, Request: native} - nativeOptions.Consume = func(_ context.Context, _ *weir.NativeResponse, data []byte) error { - if len(body)+len(data) > 1<<20 { - return errors.New("fixture response too large") - } - body = append(body, data...) - return nil - } - resultNative, err := client.Native(ctx, nativeOptions) + resultNative, err := client.Native(ctx, store, native) if err != nil || resultNative == nil || resultNative.Completion != weir.NativeResponseComplete { t.Fatalf("Native: %v %v", resultNative, err) } if backend == "mongo" { - raw := bson.Raw(body) + raw := bson.Raw(resultNative.Body) if raw.Lookup("n").AsInt64() != 1 || raw.Lookup("ok").AsFloat64() != 1 { t.Fatal("native count mismatch") } @@ -144,12 +128,10 @@ func testLifecycle(t *testing.T, backend string) { t.Fatalf("native HTTP metadata: %v %v", metadata, err) } } - deleteRequest := &weir.DeleteRequest{Resource: builder.resource} - deleteOptions := weir.DeleteOptions{StoreName: store, Request: deleteRequest} - result, err = client.Delete(ctx, deleteOptions) + result, err = mutateOne(ctx, client, store, remove) applied(t, result, err) deleted = true - readResult, err = client.ReadOne(ctx, readOptions) + readResult, err = readOne(ctx, client, store, read) if err != nil || readResult == nil || !readResult.Missing { t.Fatalf("deleted read: %v %v", readResult, err) } @@ -184,42 +166,35 @@ func testStreaming(t *testing.T, backend string) { } defer cleanupClient.Close() for _, builder := range builders { - request := &weir.DeleteRequest{Resource: builder.resource} - options := weir.DeleteOptions{StoreName: target.store, Request: request} - result, err := cleanupClient.Delete(cleanup, options) + remove := &weir.MutateRequest{Resource: builder.resource, Action: weir.MutationDelete} + result, err := mutateOne(cleanup, cleanupClient, target.store, remove) if err != nil || result.GetOutcome() != weir.MutationApplied || result.GetFailure() != nil { t.Error("owned record cleanup:", result, err) } } }) - first := builders[0].write(0) - updated := builders[0].write(10) - second := builders[1].write(1) - third := builders[2].write(2) - create := &weir.MutateRequest{Resource: first.Resource, Action: weir.MutationCreate, Document: first.Document} - put := &weir.MutateRequest{Resource: updated.Resource, Action: weir.MutationPut, Document: updated.Document} - createSecond := &weir.MutateRequest{Resource: second.Resource, Action: weir.MutationCreate, Document: second.Document} - createThird := &weir.MutateRequest{Resource: third.Resource, Action: weir.MutationCreate, Document: third.Document} - mutations := []*weir.MutateRequest{create, create, put, createSecond, createThird} + create := builders[0].write(weir.MutationCreate, 0) + // The rest follow from a producer, in order after the requests. + produced := []*weir.MutateRequest{builders[0].write(weir.MutationPut, 10), builders[1].write(weir.MutationCreate, 1), builders[2].write(weir.MutationCreate, 2)} + total := 2 + len(produced) // Results arrive as records complete; each index appears once. position, confirmed := 0, 0 - seen := make(map[uint64]bool) - mutate := weir.MutateStreamOptions{StoreName: target.store} - mutate.Next = func(context.Context) (*weir.MutateRequest, error) { - if position == len(mutations) { + seen := make(map[int]bool) + nextMutation := func(context.Context) (*weir.MutateRequest, error) { + if position == len(produced) { return nil, io.EOF } - request := mutations[position] + request := produced[position] position++ return request, nil } - mutate.Consume = func(_ context.Context, index uint64, result *weir.MutationResult) error { + consumeMutation := func(_ context.Context, index int, result *weir.MutationResult) error { confirmed++ - if index == 0 || index > uint64(len(mutations)) || seen[index] { - return fmt.Errorf("unexpected mutation ordinal %d", index) + if index >= total || seen[index] { + return fmt.Errorf("unexpected mutation index %d", index) } seen[index] = true - if index == 2 { + if index == 1 { if result.GetOutcome() != weir.MutationNotApplied || result.GetFailure().GetCode() != weir.FailurePreconditionFailed { return fmt.Errorf("duplicate create evidence: %v", result) } @@ -230,23 +205,22 @@ func testStreaming(t *testing.T, backend string) { } return nil } - if err := client.MutateStream(ctx, mutate); err != nil || confirmed != len(mutations) { + _, err = client.Mutate(ctx, target.store, weir.WithRequests(create, create), weir.WithNext(nextMutation), weir.WithMutateConsume(consumeMutation)) + if err != nil || confirmed != total { t.Fatalf("stream mutation acknowledgements=%d: %v", confirmed, err) } lua := &weir.LuaTransform{Source: []byte(`return function(current, incoming) current.n = current.n + 7 return current end`)} - transform := &weir.AtomicTransformRequest{Resource: first.Resource, Lua: lua} - transformOptions := weir.AtomicTransformOptions{StoreName: target.store, Request: transform} - result, err := client.AtomicTransform(ctx, transformOptions) + transform := &weir.MutateRequest{Resource: create.Resource, Action: weir.MutationAtomicTransform, Lua: lua} + result, err := mutateOne(ctx, client, target.store, transform) applied(t, result, err) - resources := []string{third.Resource, first.Resource, target.collection + "/s:" + idPrefix + "-missing", second.Resource, first.Resource} + resources := []string{builders[2].resource, create.Resource, target.collection + "/s:" + idPrefix + "-missing", builders[1].resource, create.Resource} expected := []int64{2, 17, -1, 1, 17} position, confirmed = 0, 0 clear(seen) - read := weir.ReadStreamOptions{StoreName: target.store} - read.Next = func(context.Context) (*weir.ReadRequest, error) { + nextRead := func(context.Context) (*weir.ReadRequest, error) { if position == len(resources) { return nil, io.EOF } @@ -254,27 +228,38 @@ end`)} position++ return request, nil } - read.Consume = func(_ context.Context, index uint64, result *weir.ReadResult) error { + consumeRead := func(_ context.Context, index int, result *weir.ReadResult) error { confirmed++ - if index == 0 || index > uint64(len(resources)) || seen[index] || result == nil || result.Failure != nil { + if index >= len(resources) || seen[index] || result == nil || result.Failure != nil { return fmt.Errorf("read %d: %v", index, result) } seen[index] = true - if expected[index-1] == -1 { + if expected[index] == -1 { if !result.Missing || result.Document != nil { return fmt.Errorf("read %d did not confirm missing: %v", index, result) } - } else if result.Missing || result.Document == nil || number(t, backend, result.Document.Data) != expected[index-1] { + } else if result.Missing || result.Document == nil || number(t, backend, result.Document.Data) != expected[index] { return fmt.Errorf("read %d did not preserve same-key mutation order: %v", index, result) } return nil } - if err := client.ReadStream(ctx, read); err != nil || confirmed != len(resources) { + _, err = client.Read(ctx, target.store, weir.WithNext(nextRead), weir.WithReadConsume(consumeRead)) + if err != nil || confirmed != len(resources) { t.Fatalf("stream read acknowledgements=%d: %v", confirmed, err) } t.Logf("%s: stream same-key order, duplicate-create continuation, Lua readback and repeated/missing streamed reads verified", backend) } +func mutateOne(ctx context.Context, client *weir.Client, store string, request *weir.MutateRequest) (*weir.MutationResult, error) { + results, err := client.Mutate(ctx, store, weir.WithRequests(request)) + return results[0], err +} + +func readOne(ctx context.Context, client *weir.Client, store string, request *weir.ReadRequest) (*weir.ReadResult, error) { + results, err := client.Read(ctx, store, weir.WithRequests(request)) + return results[0], err +} + func applied(t *testing.T, result *weir.MutationResult, err error) { t.Helper() if err != nil || result.GetOutcome() != weir.MutationApplied || result.GetFailure() != nil { @@ -296,17 +281,17 @@ type mutationBuilder struct { backend, resource, id string } -func (b mutationBuilder) write(n int) *weir.WriteRequest { +func (b mutationBuilder) write(action weir.MutationAction, n int) *weir.MutateRequest { b.t.Helper() document := &weir.Document{ContentType: "application/json", Data: []byte(fmt.Sprintf(`{"n":%d,"unprojected":true}`, n))} if b.backend == "mongo" { value := bson.D{{Key: "_id", Value: b.id}, {Key: "n", Value: n}, {Key: "unprojected", Value: true}} document = bsonDocument(b.t, value) } - request := &weir.WriteRequest{Resource: b.resource, Document: document} + request := &weir.MutateRequest{Resource: b.resource, Action: action, Document: document} return request } -func (b mutationBuilder) transform(n int) *weir.AtomicTransformRequest { +func (b mutationBuilder) transform(n int) *weir.MutateRequest { b.t.Helper() expression := &weir.Document{ContentType: "application/vnd.weir.search-update.v1+json", Data: []byte(fmt.Sprintf(`{"doc":{"n":%d}}`, n))} if b.backend == "mongo" { @@ -315,7 +300,7 @@ func (b mutationBuilder) transform(n int) *weir.AtomicTransformRequest { expression = bsonDocument(b.t, update) expression.ContentType = "application/vnd.weir.mongodb-update.v1+bson" } - request := &weir.AtomicTransformRequest{Resource: b.resource, BackendExpression: expression} + request := &weir.MutateRequest{Resource: b.resource, Action: weir.MutationAtomicTransform, BackendExpression: expression} return request } func number(t *testing.T, backend string, data []byte) int64 { diff --git a/keepalive_test.go b/keepalive_test.go index 7b3f349..d270ab5 100644 --- a/keepalive_test.go +++ b/keepalive_test.go @@ -108,8 +108,7 @@ func TestCallsLeaveAServerThatStopsAnswering(t *testing.T) { defer cancel() first := &MutateRequest{Resource: "records/s:first", Action: MutationPut, Document: document} second := &MutateRequest{Resource: "records/s:second", Action: MutationPut, Document: document} - options := MutateOptions{StoreName: "records", Requests: []*MutateRequest{first, second}} - _, err := client.Mutate(ctx, options) + _, err := client.Mutate(ctx, "records", WithRequests(first, second)) return err } frozenAt := time.Now() diff --git a/lanes.go b/lanes.go index c5334c2..6f2a4a7 100644 --- a/lanes.go +++ b/lanes.go @@ -16,7 +16,6 @@ import ( "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/metadata" - "google.golang.org/grpc/peer" "google.golang.org/grpc/status" "google.golang.org/protobuf/proto" ) @@ -207,13 +206,6 @@ type lane struct { idle *time.Timer } -// execute sends one validated command on a shared stream when one has room and -// otherwise reports false so the caller can use a stream of its own. Callers -// validate first because an invalid request ends the stream for every record. -func (p *lanePool) execute(ctx context.Context, key laneKey, command *pb.Command) (*pb.Event, bool, error) { - return p.finish(ctx, key, command, nil) -} - // queuedRecord is a record enqueue placed on a stream, or why it did not: an // error, or neither for a record that needs a stream of its own. type queuedRecord struct { @@ -249,7 +241,10 @@ func (p *lanePool) enqueue(ctx context.Context, key laneKey, command *pb.Command } // finish waits for the outcome of a record enqueue placed, or sends it now when -// queued is nil, and sends it again when its stream closed before sending it. +// queued is nil, and sends it again when its stream closed before sending it. It +// reports false when no stream has room, so the caller can use one of its own. +// Callers validate first because an invalid request ends the stream for every +// record. func (p *lanePool) finish(ctx context.Context, key laneKey, command *pb.Command, queued *queuedRecord) (*pb.Event, bool, error) { for range laneAttempts { if queued == nil { @@ -391,9 +386,10 @@ type laneOpening struct { func (p *lanePool) open(ctx context.Context, opening laneOpening) (*lane, error) { key, group := opening.key, opening.group laneCtx, cancel := context.WithCancel(p.ctx) - streamCtx := laneCtx + picked := &atomic.Pointer[string]{} + streamCtx := context.WithValue(laneCtx, endpointPicked, picked) if opening.target != "" { - streamCtx = context.WithValue(laneCtx, endpointPreference, opening.target) + streamCtx = context.WithValue(streamCtx, endpointPreference, opening.target) } // A group with one owner sends its caller's outgoing metadata. if outgoing, ok := metadata.FromOutgoingContext(ctx); ok && key.owner != 0 { @@ -411,8 +407,8 @@ func (p *lanePool) open(ctx context.Context, opening laneOpening) (*lane, error) if stopped && err == nil { now := time.Now() address := "" - if reached, ok := peer.FromContext(stream.Context()); ok && reached.Addr != nil { - address = reached.Addr.String() + if chosen := picked.Load(); chosen != nil { + address = *chosen } opened = &lane{pool: p, key: key, stream: stream, ctx: laneCtx, cancel: cancel, expires: now.Add(laneLifetime), address: address, probing: opening.probing, wake: make(chan struct{}, 1), outstanding: make(map[uint64]*laneCall)} opened.load.Store(1) diff --git a/lanes_test.go b/lanes_test.go index 1ae514d..0324d15 100644 --- a/lanes_test.go +++ b/lanes_test.go @@ -210,7 +210,7 @@ func laneTestDial(t *testing.T, servers *laneTestServers, peers []*laneTestPeer) servers.addresses = append(servers.addresses, laneTestListen(t, peer)) } servers.resolver.InitialState(servers.state(servers.addresses...)) - options := append(connectionOptions(endpointConfig), grpc.WithResolvers(servers.resolver)) + options := connectionOptions(nil, servers.resolver) connection, err := grpc.NewClient("weirlanes:///records", options...) if err != nil { t.Fatal(err) @@ -261,11 +261,8 @@ func laneTestListen(t *testing.T, peer *laneTestPeer) string { // each peer's stream count at that point. func laneTestReady(t *testing.T, client *Client, peers ...*laneTestPeer) []int64 { t.Helper() - document := &Document{ContentType: "application/json", Data: []byte(`{}`)} - request := &WriteRequest{Resource: "records/s:ready", Document: document} - options := WriteOptions{StoreName: "records", Request: request} for range 200 { - if _, err := Put(t.Context(), client.store, options); err != nil { + if _, err := putAlone(t.Context(), client, "records/s:ready"); err != nil { t.Fatal(err) } counts := make([]int64, len(peers)) @@ -284,10 +281,16 @@ func laneTestReady(t *testing.T, client *Client, peers ...*laneTestPeer) []int64 } func lanePut(ctx context.Context, client *Client, resource string, size int) (*MutationResult, error) { - document := &Document{ContentType: "application/json", Data: []byte(`{"padding":"` + strings.Repeat("x", size) + `"}`)} - request := &WriteRequest{Resource: resource, Document: document} - options := WriteOptions{StoreName: "records", Request: request} - return client.Put(ctx, options) + return mutateOne(ctx, client, putRequest(resource, `{"padding":"`+strings.Repeat("x", size)+`"}`)) +} + +// putAlone sends one mutation on a gRPC stream of its own. +func putAlone(ctx context.Context, client *Client, resource string) (*MutationResult, error) { + command, _, err := mutateCommand(putRequest(resource, `{}`), nil) + if err != nil { + return nil, err + } + return mutateKind.alone(ctx, client, "records", command) } func TestConcurrentSingleRecordCallsShareABoundedSetOfStreams(t *testing.T) { @@ -584,11 +587,8 @@ func TestRejectedProbeDoesNotHoldCallsOnADrainingServer(t *testing.T) { draining.rejecting.Store(true) // A stream of its own moves round-robin so the first probe reaches the // draining server. - document := &Document{ContentType: "application/json", Data: []byte(`{}`)} - request := &WriteRequest{Resource: "records/s:turn", Document: document} - options := WriteOptions{StoreName: "records", Request: request} turned := healthy.streams.Load() - if _, err := Put(t.Context(), client.store, options); err != nil || healthy.streams.Load() != turned+1 { + if _, err := putAlone(t.Context(), client, "records/s:turn"); err != nil || healthy.streams.Load() != turned+1 { t.Fatal("round-robin did not turn to the healthy server", err) } applied, refused, failed := 0, 0, 0 diff --git a/native.go b/native.go new file mode 100644 index 0000000..7f27f10 --- /dev/null +++ b/native.go @@ -0,0 +1,98 @@ +package weir + +import ( + "context" + "errors" + + pb "github.com/batchstream/weir-protocol/api/weir/v1" +) + +// Native sends one backend command and returns its response. Backend errors +// remain native response data, and a validated result remains evidence alongside +// a later RPC error. The body is collected into NativeResult.Body unless WithChunks +// consumes it incrementally. The request is sent again only when a server refused +// it with NativeNotStarted before any response. +func (c *Client) Native(ctx context.Context, store string, request *NativeRequest, options ...NativeOption) (*NativeResult, error) { + client, err := c.storeClient() + if err != nil { + return nil, err + } + config := nativeOptions{} + for _, option := range options { + if option != nil { + option(&config) + } + } + call := nativeCall{store: store, request: request, chunks: config.chunks} + send := func() (*NativeResult, error) { return call.send(ctx, client) } + result, err := resendRefused(ctx, c, nativeRefused, send) + return result, callError(ctx, err) +} + +// nativeCall describes one send of a Native request. +type nativeCall struct { + store string + request *NativeRequest + chunks func(context.Context, *NativeResponse, []byte) error +} + +func (n nativeCall) send(ctx context.Context, client pb.StoreServiceClient) (*NativeResult, error) { + if n.request == nil { + return nil, errors.New("Native requires a request") + } + variant := &pb.Command_Native{Native: n.request} + command := &pb.Command{Operation: variant} + request := &pb.ExecuteRequest{StoreName: n.store, Index: 1, Command: command} + var result *NativeResult + state := nativeStreamState{} + consume := func(ctx context.Context, event *pb.Event) (bool, error) { + if err := state.validate(event); err != nil { + return false, err + } + switch value := event.Value.(type) { + case *pb.Event_Head: + result = &NativeResult{Response: value.Head} + case *pb.Event_Chunk: + if n.chunks != nil { + return false, n.chunks(ctx, result.Response, value.Chunk) + } + result.Body = append(result.Body, value.Chunk...) + case *pb.Event_NativeEnd: + if result == nil { + result = &NativeResult{} + } + result.Completion = value.NativeEnd.Completion + result.Failure = value.NativeEnd.Failure + return true, nil + } + return false, nil + } + err := execute(ctx, client, request, consume) + return result, err +} + +type nativeStreamState struct { + head bool +} + +// validate checks one Native event in stream order. +func (s *nativeStreamState) validate(event *pb.Event) error { + switch value := event.Value.(type) { + case *pb.Event_Head: + if s.head { + return errors.New("invalid Native response head") + } + s.head = true + case *pb.Event_Chunk: + if !s.head { + return errors.New("Native chunk before head") + } + case *pb.Event_NativeEnd: + if value.NativeEnd.Completion == NativeResponseComplete && !s.head || value.NativeEnd.Completion == NativeNotStarted && s.head { + return errors.New("invalid Native completion") + } + default: + return errors.New("unexpected Native event") + } + return nil +} diff --git a/operations_test.go b/native_test.go similarity index 57% rename from operations_test.go rename to native_test.go index 734ca1e..dbfd712 100644 --- a/operations_test.go +++ b/native_test.go @@ -11,126 +11,8 @@ import ( "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" - "google.golang.org/protobuf/proto" ) -func TestReadPreservesMissingAndBackendFailure(t *testing.T) { - for _, mode := range []string{"read_missing", "read_failure", "read_target_missing"} { - t.Run(mode, func(t *testing.T) { - peer := &clientTestPeer{mode: mode} - client := clientTestConnection(t, peer) - options := ReadOneOptions{StoreName: "records", Request: clientTestReadRequest()} - result, err := ReadOne(t.Context(), client, options) - if err != nil || result == nil || result.Document != nil { - t.Fatal("Read lost backend result", result, err) - } - if mode == "read_missing" && (!result.Missing || result.Failure != nil) { - t.Fatal("Read changed missing into failure", result) - } - if mode == "read_failure" && (result.Missing || result.Failure.GetCode() != FailurePermissionDenied) { - t.Fatal("Read lost failure evidence", result) - } - if mode == "read_target_missing" && (result.Missing || result.Failure.GetCode() != FailureTargetNotFound) { - t.Fatal("missing backend target became successful document absence", result) - } - }) - } -} - -func TestTypedMutationsSelectBackendOperation(t *testing.T) { - acknowledged := &MutationOptions{} - visible := &MutationOptions{Visibility: MutationQueryVisible} - cases := []struct { - name string - options *MutationOptions - }{ - {name: "default"}, - {name: "acknowledged", options: acknowledged}, - {name: "visible", options: visible}, - } - for _, test := range cases { - t.Run(test.name, func(t *testing.T) { - peer := &clientTestPeer{mode: "typed_mutation", mutations: make(chan *pb.MutateRequest, 5)} - client := clientTestConnection(t, peer) - ctx, cancel := context.WithTimeout(t.Context(), 3*time.Second) - defer cancel() - document := &Document{ContentType: "application/json", Data: []byte(`{"n":1}`)} - request := &WriteRequest{Resource: "records/s:key", Document: document, Options: test.options} - write := WriteOptions{StoreName: "records", Request: request} - for _, action := range []string{"create", "put", "replace", "delete", "transform"} { - var result *MutationResult - var err error - switch action { - case "create": - result, err = Create(ctx, client, write) - case "put": - result, err = Put(ctx, client, write) - case "replace": - result, err = Replace(ctx, client, write) - case "delete": - request := &DeleteRequest{Resource: "records/s:key", Options: test.options} - options := DeleteOptions{StoreName: "records", Request: request} - result, err = Delete(ctx, client, options) - case "transform": - program := &LuaTransform{Source: []byte("return function(current, incoming) return current end")} - request := &AtomicTransformRequest{Resource: "records/s:key", Lua: program, Options: test.options} - options := AtomicTransformOptions{StoreName: "records", Request: request} - result, err = AtomicTransform(ctx, client, options) - } - if err != nil || result.GetOutcome() != MutationApplied { - t.Fatal(action, result, err) - } - mutation := <-peer.mutations - if mutation.Resource != request.Resource || !proto.Equal(mutation.Options, test.options) { - t.Fatal("typed operation lost its resource or complete options", action, mutation) - } - selected := false - switch action { - case "create": - selected = mutation.GetCreate() != nil - case "put": - selected = mutation.GetPut() != nil - case "replace": - selected = mutation.GetReplace() != nil - case "delete": - selected = mutation.GetDelete() != nil - case "transform": - selected = mutation.GetAtomicTransform().GetLua() != nil - } - if !selected { - t.Fatal("SDK executed a different mutation", action, mutation) - } - } - if peer.received.Load() != 5 { - t.Fatal("typed mutations were replayed", peer.received.Load()) - } - }) - } -} - -func TestAtomicTransformRejectsAmbiguousFormsBeforeBusinessSend(t *testing.T) { - for _, both := range []bool{false, true} { - t.Run(map[bool]string{false: "neither", true: "both"}[both], func(t *testing.T) { - peer := &clientTestPeer{mode: "typed_mutation"} - client := clientTestConnection(t, peer) - request := &AtomicTransformRequest{Resource: "records/s:key"} - if both { - request.Lua = &LuaTransform{Source: []byte("return function(current, incoming) return current end")} - request.BackendExpression = &Document{ContentType: "application/json", Data: []byte(`{}`)} - } - options := AtomicTransformOptions{StoreName: "records", Request: request} - ctx, cancel := context.WithTimeout(t.Context(), time.Second) - defer cancel() - if result, err := AtomicTransform(ctx, client, options); err == nil || result != nil { - t.Fatal("invalid transform was not rejected", result, err) - } - if peer.received.Load() != 0 { - t.Fatal("invalid transform sent business data", peer.received.Load()) - } - }) - } -} - func (p *clientTestPeer) nativeExecute(stream grpc.BidiStreamingServer[pb.ExecuteRequest, pb.ExecuteResponse]) error { metadata := &Document{ContentType: "application/vnd.example.response", Data: []byte{0xff, 0}} head := &pb.NativeHead{Metadata: metadata, BodyContentType: "application/octet-stream"} @@ -237,33 +119,32 @@ func TestNativeConsumesBytesAndPreservesResponseEvidence(t *testing.T) { for _, test := range cases { t.Run(test.mode, func(t *testing.T) { peer := &clientTestPeer{mode: test.mode} - client := clientTestConnection(t, peer) - options := NativeOptions{StoreName: "records", Request: clientTestNativeRequest()} + client := clientTestClient(t, peer) chunks := 0 failure := errors.New("consumer rejected body") - options.Consume = func(_ context.Context, response *NativeResponse, data []byte) error { + consume := func(_ context.Context, response *NativeResponse, data []byte) error { chunks++ if string(data) != "native body bytes" { - t.Fatal("body bytes changed", response, data) + t.Error("body bytes changed", response, data) } if test.mode == "native_http_error" { httpResponse, err := ParseHTTPNativeResponse(response) if err != nil || httpResponse.StatusCode != 404 { - t.Fatal("complete native HTTP failure lost metadata", response, err) + t.Error("complete native HTTP failure lost metadata", response, err) } } else if test.mode == "native_no_metadata" { if response.Metadata != nil { - t.Fatal("absent metadata changed", response) + t.Error("absent metadata changed", response) } } else if response.Metadata.ContentType != "application/vnd.example.response" { - t.Fatal("opaque metadata changed", response) + t.Error("opaque metadata changed", response) } if test.mode == "native_consumer_chunk_error" { return failure } return nil } - result, err := Native(t.Context(), client, options) + result, err := client.Native(t.Context(), "records", clientTestNativeRequest(), WithChunks(consume)) if result == nil || result.Completion != test.completion || (result.Response != nil) != test.response || (result.Failure != nil) != test.failure || chunks != test.chunks || (err != nil) != test.rpcError { t.Fatal("Native conflated body, completion and RPC evidence", result, err, chunks) } @@ -279,12 +160,10 @@ func TestNativeConsumesBytesAndPreservesResponseEvidence(t *testing.T) { func TestNativeSendsBackendRootRequest(t *testing.T) { peer := &clientTestPeer{mode: "native_no_metadata"} - client := clientTestConnection(t, peer) + client := clientTestClient(t, peer) document := &Document{ContentType: "application/http", Data: []byte("GET /_cluster/health HTTP/1.1\r\n\r\n")} request := &NativeRequest{Request: document} - options := NativeOptions{StoreName: "search", Request: request} - options.Consume = func(context.Context, *NativeResponse, []byte) error { return nil } - result, err := Native(t.Context(), client, options) + result, err := client.Native(t.Context(), "search", request) if err != nil || result == nil || result.Completion != NativeResponseComplete || peer.streams.Load() != 1 || peer.received.Load() != 1 { t.Fatal("Native backend root request failed", result, err, peer.streams.Load(), peer.received.Load()) } @@ -294,11 +173,10 @@ func TestNativeRejectsContradictoryOrMalformedResponseEvidence(t *testing.T) { for _, mode := range []string{"native_empty_chunk", "native_not_started_after_head", "native_not_started_after_chunk", "native_duplicate_head", "native_chunk_before_head", "native_complete_without_head", "native_invalid_metadata", "native_unknown_completion", "native_unknown_fields", "native_unknown_fields_final_error", "native_unknown_failure", "native_missing_end", "native_wrong_kind", "native_wrong_index"} { t.Run(mode, func(t *testing.T) { peer := &clientTestPeer{mode: mode} - client := clientTestConnection(t, peer) - options := NativeOptions{StoreName: "records", Request: clientTestNativeRequest()} + client := clientTestClient(t, peer) chunks := 0 - options.Consume = func(context.Context, *NativeResponse, []byte) error { chunks++; return nil } - result, err := Native(t.Context(), client, options) + consume := func(context.Context, *NativeResponse, []byte) error { chunks++; return nil } + result, err := client.Native(t.Context(), "records", clientTestNativeRequest(), WithChunks(consume)) if err == nil || result != nil && result.Completion != NativeCompletionUnconfirmed { t.Fatal("invalid completion evidence exposed", result, err) } @@ -320,10 +198,8 @@ func TestNativeRejectsInvalidRequestEnvelopeBeforeRPC(t *testing.T) { {Resource: "records", Request: invalid}, } { peer := &clientTestPeer{} - client := clientTestConnection(t, peer) - options := NativeOptions{StoreName: "records", Request: request} - options.Consume = func(context.Context, *NativeResponse, []byte) error { return nil } - if result, err := Native(t.Context(), client, options); err == nil || result != nil { + client := clientTestClient(t, peer) + if result, err := client.Native(t.Context(), "records", request); err == nil || result != nil { t.Fatal("invalid native request envelope accepted", result, err) } if peer.streams.Load() != 0 { @@ -331,3 +207,60 @@ func TestNativeRejectsInvalidRequestEnvelopeBeforeRPC(t *testing.T) { } } } + +func TestNativeCollectsTheBodyUnlessChunksConsumeIt(t *testing.T) { + peer := &clientTestPeer{mode: "native_normal"} + client := clientTestClient(t, peer) + result, err := client.Native(t.Context(), "records", clientTestNativeRequest()) + if err != nil || string(result.Body) != "native body bytes" || result.Completion != NativeResponseComplete { + t.Fatal("the body was not collected", result, err) + } + var chunks [][]byte + consume := func(_ context.Context, _ *NativeResponse, chunk []byte) error { + chunks = append(chunks, chunk) + return nil + } + result, err = client.Native(t.Context(), "records", clientTestNativeRequest(), WithChunks(consume)) + if err != nil || result.Body != nil || len(chunks) != 1 || string(chunks[0]) != "native body bytes" { + t.Fatal("a consumed body was also collected", result, chunks, err) + } +} + +func TestNativeKeepsTheBodyThatArrivedBeforeAnError(t *testing.T) { + peer := &clientTestPeer{mode: "native_final_status_error"} + client := clientTestClient(t, peer) + result, err := client.Native(t.Context(), "records", clientTestNativeRequest()) + if status.Code(err) != codes.Unavailable || result == nil || string(result.Body) != "native body bytes" { + t.Fatal("a later error lost the body", result, err) + } +} + +func TestNativeRequiresARequest(t *testing.T) { + peer := &clientTestPeer{} + client := clientTestClient(t, peer) + if result, err := client.Native(t.Context(), "records", nil); err == nil || result != nil || peer.streams.Load() != 0 { + t.Fatal("a call without a request was accepted", result, err) + } +} + +// answersNothing holds every stream open until its caller leaves. +type answersNothing struct{} + +func (answersNothing) Execute(packed grpc.BidiStreamingServer[pb.ExecuteRequests, pb.ExecuteResponses]) error { + <-packed.Context().Done() + return packed.Context().Err() +} + +func TestNativeReturnsTheErrorOfItsContext(t *testing.T) { + client, err := New(serveForTest(t, answersNothing{})) + if err != nil { + t.Fatal(err) + } + defer client.Close() + ctx, cancel := context.WithTimeout(t.Context(), 100*time.Millisecond) + defer cancel() + result, err := client.Native(ctx, "records", clientTestNativeRequest()) + if !errors.Is(err, context.DeadlineExceeded) { + t.Fatal("a Native past its deadline did not return the context's error", result, err) + } +} diff --git a/next_test.go b/next_test.go new file mode 100644 index 0000000..5f6c05f --- /dev/null +++ b/next_test.go @@ -0,0 +1,675 @@ +package weir + +import ( + "context" + "errors" + "io" + "slices" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/batchstream/weir-protocol/api/protocol" + pb "github.com/batchstream/weir-protocol/api/weir/v1" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/status" +) + +func TestNextLargeDuplexCallConsumesIncrementally(t *testing.T) { + peer := &clientTestPeer{mode: "large_duplex"} + client := clientTestClient(t, peer) + produced, consumed := 0, 0 + var responseBytes uint64 + next := func(context.Context) (*ReadRequest, error) { + if produced == 48 { + return nil, io.EOF + } + produced++ + request := &ReadRequest{Resource: "records/s:key"} + return request, nil + } + consume := func(_ context.Context, _ int, result *ReadResult) error { + consumed++ + if result.Document == nil || len(result.Document.Data) != 1<<20 { + return errors.New("large response document mismatch") + } + responseBytes += uint64(len(result.Document.Data)) + return nil + } + ctx, cancel := context.WithTimeout(t.Context(), 15*time.Second) + defer cancel() + _, err := client.Read(ctx, "records", WithNext(next), WithReadConsume(consume)) + if err != nil { + t.Fatal(err) + } + if produced != 48 || consumed != 48 || responseBytes <= 32<<20 || peer.streams.Load() != 1 || peer.maxBytes.Load() > int64(protocol.MaxTransportMessageBytes) { + t.Fatal("large call was truncated or split into RPCs", produced, consumed, responseBytes, peer.streams.Load(), peer.maxBytes.Load()) + } +} + +func TestPausedConsumerBoundsProducedBytesAndCancels(t *testing.T) { + peer := &clientTestPeer{mode: "typed_mutation"} + client := clientTestClient(t, peer) + document := &Document{ContentType: "application/octet-stream", Data: make([]byte, (2 << 20))} + var produced atomic.Int64 + started := make(chan struct{}) + next := func(ctx context.Context) (*MutateRequest, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + count := produced.Add(1) + resource := "records/s:" + strings.Repeat("k", int(count)) + request := &MutateRequest{Resource: resource, Action: MutationPut, Document: document} + return request, nil + } + once := sync.Once{} + consume := func(ctx context.Context, _ int, result *MutationResult) error { + if result.GetOutcome() != MutationApplied { + return errors.New("invalid first acknowledgement") + } + once.Do(func() { close(started) }) + <-ctx.Done() + return ctx.Err() + } + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + done := make(chan error, 1) + go func() { + _, err := client.Mutate(ctx, "records", WithNext(next), WithMutateConsume(consume)) + done <- err + }() + select { + case <-started: + case <-time.After(3 * time.Second): + t.Fatal("consumer did not start") + } + // At most three encoded 2MiB inputs fit the 8MiB window. The producer may + // yield one unsent item before its size causes it to wait for credit. + timer := time.NewTimer(100 * time.Millisecond) + defer timer.Stop() + select { + case err := <-done: + t.Fatal("call finished while its consumer was paused", err) + case <-timer.C: + } + if count := produced.Load(); count > 5 || peer.received.Load() > 4 || peer.maxBytes.Load() > int64(protocol.MaxTransportMessageBytes) { + t.Fatal("paused consumer allowed unbounded input", count, peer.received.Load(), peer.maxBytes.Load()) + } + cancel() + select { + case err := <-done: + if err == nil { + t.Fatal("cancelled call succeeded") + } + case <-time.After(3 * time.Second): + t.Fatal("cancelled call did not join its sender") + } +} + +func TestCallRejectsDuplicateIndexAndKeepsEarlierResult(t *testing.T) { + peer := &clientTestPeer{mode: "duplicate_index"} + client := clientTestClient(t, peer) + results, err := client.Read(t.Context(), "records", WithRequests(clientTestReadRequest(), clientTestReadRequest())) + if err == nil || len(results) != 2 || results[0] == nil { + t.Fatal("a duplicate index succeeded or erased a confirmed result", results, err) + } + if results[1] != nil { + t.Fatal("a duplicate index acknowledged another input", results) + } +} + +// A call that has one result per input is complete, so what its stream does next +// is not its failure, as with TestReadSucceedsOnceEveryResultArrived. +func TestExtraResultAfterEveryInputsResultKeepsTheCall(t *testing.T) { + peer := &clientTestPeer{mode: "extra_result"} + client := clientTestClient(t, peer) + results, err := client.Read(t.Context(), "records", WithRequests(clientTestReadRequest(), clientTestReadRequest())) + if err != nil || len(results) != 2 || results[0] == nil || results[1] == nil { + t.Fatal("an extra result changed a complete call", results, err) + } +} + +func TestProducerFailureKeepsEarlierAcknowledgements(t *testing.T) { + peer := &clientTestPeer{mode: "batch_read_order"} + client := clientTestClient(t, peer) + confirmed := make(chan struct{}) + produced, consumed := 0, 0 + next := func(ctx context.Context) (*ReadRequest, error) { + if produced == 64 { + select { + case <-confirmed: + case <-ctx.Done(): + return nil, ctx.Err() + } + request := &ReadRequest{Resource: "bad//resource"} + return request, nil + } + produced++ + return clientTestReadRequest(), nil + } + consume := func(_ context.Context, _ int, result *ReadResult) error { + consumed++ + if result == nil { + return errors.New("invalid earlier acknowledgement") + } + if consumed == 64 { + close(confirmed) + } + return nil + } + ctx, cancel := context.WithTimeout(t.Context(), 3*time.Second) + defer cancel() + _, err := client.Read(ctx, "records", WithNext(next), WithReadConsume(consume)) + if err == nil || !strings.Contains(err.Error(), "request 64") || consumed != 64 || peer.streams.Load() != 1 || peer.received.Load() != 64 { + t.Fatal("producer failure replayed requests or discarded earlier evidence", err, consumed, peer.streams.Load(), peer.received.Load()) + } +} + +func TestCancellationJoinsBlockedProducer(t *testing.T) { + peer := &clientTestPeer{} + client := clientTestClient(t, peer) + started, stopped := make(chan struct{}), make(chan struct{}) + next := func(ctx context.Context) (*ReadRequest, error) { + close(started) + <-ctx.Done() + close(stopped) + return nil, ctx.Err() + } + consume := func(context.Context, int, *ReadResult) error { + return errors.New("no input was produced") + } + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + done := make(chan error, 1) + go func() { + _, err := client.Read(ctx, "records", WithNext(next), WithReadConsume(consume)) + done <- err + }() + select { + case <-started: + case <-time.After(time.Second): + t.Fatal("producer did not start") + } + cancel() + select { + case err := <-done: + if err == nil { + t.Fatal("cancelled producer succeeded") + } + select { + case <-stopped: + default: + t.Fatal("call returned before producer finished") + } + case <-time.After(time.Second): + t.Fatal("blocked producer was not cancelled and joined") + } +} + +func TestCallRejectsPrematureSuccessAndJoinsSender(t *testing.T) { + peer := &clientTestPeer{mode: "early_eof"} + client := clientTestClient(t, peer) + next := func(context.Context) (*ReadRequest, error) { return clientTestReadRequest(), nil } + consume := func(context.Context, int, *ReadResult) error { return nil } + ctx, cancel := context.WithTimeout(t.Context(), 3*time.Second) + defer cancel() + _, err := client.Read(ctx, "records", WithNext(next), WithReadConsume(consume)) + if err == nil { + t.Fatal("premature server EOF accepted an unfinished producer") + } +} + +func TestProducedRecordIsSentBeforeTheNextIsProduced(t *testing.T) { + for _, kind := range []string{"read", "mutate"} { + t.Run(kind, func(t *testing.T) { + peer := &clientTestPeer{mode: "typed_mutation"} + client := clientTestClient(t, peer) + firstConsumed := make(chan struct{}) + produced, consumed := 0, 0 + ctx, cancel := context.WithTimeout(t.Context(), 3*time.Second) + defer cancel() + var err error + wait := func(ctx context.Context) error { + if produced != 1 { + return nil + } + select { + case <-firstConsumed: + return nil + case <-ctx.Done(): + return ctx.Err() + } + } + confirm := func(index int) { + if consumed++; index == 0 { + close(firstConsumed) + } + } + if kind == "read" { + next := func(ctx context.Context) (*ReadRequest, error) { + if produced == 2 { + return nil, io.EOF + } + if err := wait(ctx); err != nil { + return nil, err + } + produced++ + return clientTestReadRequest(), nil + } + consume := func(_ context.Context, index int, result *ReadResult) error { + if result.Document == nil { + return errors.New("unexpected read acknowledgement") + } + confirm(index) + return nil + } + _, err = client.Read(ctx, "records", WithNext(next), WithReadConsume(consume)) + } else { + next := func(ctx context.Context) (*MutateRequest, error) { + if produced == 2 { + return nil, io.EOF + } + if err := wait(ctx); err != nil { + return nil, err + } + produced++ + return deleteRequest("records/s:key"), nil + } + consume := func(_ context.Context, index int, result *MutationResult) error { + if result.Outcome != MutationApplied { + return errors.New("unexpected mutation acknowledgement") + } + confirm(index) + return nil + } + _, err = client.Mutate(ctx, "records", WithNext(next), WithMutateConsume(consume)) + } + if err != nil || produced != 2 || consumed != 2 || peer.received.Load() != 2 || peer.streams.Load() != 1 { + t.Fatal("call waited for a batch or EOF before sending its first item", err, produced, consumed, peer.received.Load(), peer.streams.Load()) + } + }) + } +} + +func TestPausedConsumerBoundsItemWindow(t *testing.T) { + peer := &clientTestPeer{mode: "read_missing"} + client := clientTestClient(t, peer) + var produced atomic.Int64 + started := make(chan struct{}) + next := func(context.Context) (*ReadRequest, error) { + produced.Add(1) + return clientTestReadRequest(), nil + } + consume := func(ctx context.Context, _ int, result *ReadResult) error { + if !result.Missing { + return errors.New("unexpected first result") + } + close(started) + <-ctx.Done() + return ctx.Err() + } + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + done := make(chan error, 1) + go func() { + _, err := client.Read(ctx, "records", WithNext(next), WithReadConsume(consume)) + done <- err + }() + select { + case <-started: + case <-time.After(5 * time.Second): + t.Fatal("consumer did not begin") + } + deadline := time.Now().Add(time.Second) + for produced.Load() < recordWindowItems && time.Now().Before(deadline) { + time.Sleep(time.Millisecond) + } + if count := produced.Load(); count != recordWindowItems { + t.Fatal("paused consumer did not stop the producer at the item window", count) + } + time.Sleep(20 * time.Millisecond) + if produced.Load() != recordWindowItems || peer.received.Load() > recordWindowItems { + t.Fatal("paused consumer allowed more items than the window", produced.Load(), peer.received.Load()) + } + cancel() + select { + case err := <-done: + if err == nil { + t.Fatal("cancelled call succeeded") + } + case <-time.After(time.Second): + t.Fatal("item-credit wait did not cancel and join") + } +} + +// readsAfter produces reads of the given resources, then io.EOF. +func readsAfter(resources ...string) func(context.Context) (*ReadRequest, error) { + position := 0 + next := func(context.Context) (*ReadRequest, error) { + if position == len(resources) { + return nil, io.EOF + } + request := &ReadRequest{Resource: resources[position]} + position++ + return request, nil + } + return next +} + +func TestRequestsGoBeforeNextAndKeepOneIndexSequence(t *testing.T) { + peer := newRefusingPeer() + client := refusingClient(t, peer) + first := putRequest("records/s:same", `{"n":1}`) + second := putRequest("records/s:same", `{"n":2}`) + produced := false + next := func(context.Context) (*MutateRequest, error) { + if produced { + return nil, io.EOF + } + produced = true + return putRequest("records/s:same", `{"n":3}`), nil + } + var indexes []int + consume := func(_ context.Context, index int, result *MutationResult) error { + if result.GetOutcome() != MutationApplied { + t.Error(result) + } + indexes = append(indexes, index) + return nil + } + results, err := client.Mutate(t.Context(), "records", WithRequests(first, second), WithNext(next), WithMutateConsume(consume)) + if err != nil || results != nil { + t.Fatal(results, err) + } + // One resource runs in input order across both sources. + if applied := peer.drain(); !slices.Equal(applied, []string{`{"n":1}`, `{"n":2}`, `{"n":3}`}) { + t.Fatal("a resource's records did not keep input order across requests and Next", applied) + } + if !slices.Equal(indexes, []int{0, 1, 2}) { + t.Fatal("the index did not continue from requests into Next", indexes) + } +} + +func TestNextStartsWithoutWaitingForResultsOfRequests(t *testing.T) { + peer := newLaneTestPeer() + client := laneTestClient(t, peer) + laneTestReady(t, client, peer) + nextCalled := make(chan struct{}) + once := sync.Once{} + next := func(ctx context.Context) (*MutateRequest, error) { + once.Do(func() { close(nextCalled) }) + return nil, io.EOF + } + held := putRequest("records/s:held", `{}`) + done := make(chan error, 1) + go func() { + _, err := client.Mutate(t.Context(), "records", WithRequests(held), WithNext(next)) + done <- err + }() + select { + case <-nextCalled: + case <-time.After(5 * time.Second): + t.Fatal("Next waited for the result of a held request") + } + close(peer.release) + if err := <-done; err != nil { + t.Fatal(err) + } +} + +func TestNextWithoutConsumeCollectsEveryResult(t *testing.T) { + peer := &clientTestPeer{mode: "read_missing"} + client := clientTestClient(t, peer) + results, err := client.Read(t.Context(), "records", WithRequests(clientTestReadRequest()), WithNext(readsAfter("records/s:a", "records/s:b"))) + if err != nil || len(results) != 3 { + t.Fatal(results, err) + } + for index, result := range results { + if result == nil || !result.Missing { + t.Fatal(index, result) + } + } +} + +// answersOneOfTwo waits for two records on a stream, then answers only the first +// and ends the stream. +type answersOneOfTwo struct{} + +func (answersOneOfTwo) Execute(packed grpc.BidiStreamingServer[pb.ExecuteRequests, pb.ExecuteResponses]) error { + stream := &recordPeer{BidiStreamingServer: packed} + first, err := stream.Recv() + if err != nil { + return err + } + if _, err := stream.Recv(); err != nil { + return err + } + applied := protocol.Mutation(pb.MutationOutcome_APPLIED, nil) + value := &pb.Event_MutationResult{MutationResult: applied} + event := &pb.Event{Value: value} + answer := &pb.ExecuteResponse{Index: first.Index, Event: event} + if err := stream.Send(answer); err != nil { + return err + } + return status.Error(codes.Unavailable, "the other response was lost") +} + +func TestNextWithoutConsumeKeepsUnconfirmedPositions(t *testing.T) { + client, err := New(serveForTest(t, answersOneOfTwo{})) + if err != nil { + t.Fatal(err) + } + defer client.Close() + produced := 0 + next := func(context.Context) (*MutateRequest, error) { + if produced == 2 { + return nil, io.EOF + } + produced++ + return deleteRequest("records/s:" + strings.Repeat("k", produced)), nil + } + results, err := client.Mutate(t.Context(), "records", WithNext(next)) + if err == nil || len(results) != 2 || results[0].GetOutcome() != MutationApplied || results[1] != nil { + t.Fatal("a record without a result lost its position", results, err) + } +} + +func TestFailingNextLetsEarlierRecordsFinish(t *testing.T) { + invalid := func(context.Context) (*MutateRequest, error) { + request := &MutateRequest{} + return request, nil + } + broken := func(context.Context) (*MutateRequest, error) { + return nil, errors.New("no more input") + } + nexts := map[string]func(context.Context) (*MutateRequest, error){"invalid item": invalid, "error": broken} + for name, next := range nexts { + t.Run(name, func(t *testing.T) { + peer := newRefusingPeer() + client := refusingClient(t, peer) + first := putRequest("records/s:first", `{"n":1}`) + second := putRequest("records/s:second", `{"n":2}`) + results, err := client.Mutate(t.Context(), "records", WithRequests(first, second), WithNext(next)) + if err == nil || !strings.Contains(err.Error(), "request 2") { + t.Fatal("the error does not name the failing input", err) + } + if len(results) != 2 || results[0].GetOutcome() != MutationApplied || results[1].GetOutcome() != MutationApplied { + t.Fatal("a failing Next canceled earlier records", results) + } + if applied := peer.drain(); len(applied) != 2 { + t.Fatal("a failing Next stopped earlier records", applied) + } + }) + } +} + +func TestRequestsLeaveWhileNextBlocks(t *testing.T) { + peer := &clientTestPeer{mode: "read_missing"} + client := clientTestClient(t, peer) + answered := make(chan struct{}) + next := func(ctx context.Context) (*ReadRequest, error) { + select { + case <-answered: + return nil, io.EOF + case <-ctx.Done(): + return nil, ctx.Err() + } + } + consume := func(context.Context, int, *ReadResult) error { + close(answered) + return nil + } + ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second) + defer cancel() + _, err := client.Read(ctx, "records", WithRequests(clientTestReadRequest()), WithNext(next), WithReadConsume(consume)) + if err != nil { + t.Fatal("a request waited for a blocked Next", err) + } +} + +func TestInvalidRequestRejectsTheCallBeforeNext(t *testing.T) { + peer := &clientTestPeer{} + client := clientTestClient(t, peer) + called := false + next := func(context.Context) (*ReadRequest, error) { + called = true + return nil, io.EOF + } + invalid := &ReadRequest{Resource: "bad//resource"} + results, err := client.Read(t.Context(), "records", WithRequests(clientTestReadRequest(), invalid), WithNext(next)) + if err == nil || len(results) != 2 || results[0] != nil || results[1] != nil || called || peer.streams.Load() != 0 { + t.Fatal("an invalid request did not reject the call", results, err, called, peer.streams.Load()) + } +} + +func TestRequestsWithConsumeCollectNothing(t *testing.T) { + peer := &clientTestPeer{mode: "read_missing"} + client := clientTestClient(t, peer) + seen := make(map[int]bool) + consume := func(_ context.Context, index int, result *ReadResult) error { + if !result.Missing { + t.Error(index, result) + } + seen[index] = true + return nil + } + results, err := client.Read(t.Context(), "records", WithRequests(clientTestReadRequest(), clientTestReadRequest()), WithReadConsume(consume)) + if err != nil || results != nil || !seen[0] || !seen[1] || len(seen) != 2 { + t.Fatal(results, err, seen) + } +} + +func TestOptionsCombine(t *testing.T) { + peer := &clientTestPeer{mode: "read_missing"} + client := clientTestClient(t, peer) + var absent ReadOption + results, err := client.Read(t.Context(), "records", absent, WithRequests(clientTestReadRequest()), WithRequests(clientTestReadRequest()), WithNext(readsAfter("records/s:ignored")), WithNext(readsAfter())) + if err != nil || len(results) != 2 || peer.received.Load() != 2 { + t.Fatal("options did not add requests or override Next", results, err, peer.received.Load()) + } +} + +func TestALaterConsumeCallbackReplacesAnEarlierOne(t *testing.T) { + peer := &clientTestPeer{mode: "read_missing"} + client := clientTestClient(t, peer) + replaced := func(context.Context, int, *ReadResult) error { + t.Error("a replaced read callback ran") + return nil + } + reads := 0 + counts := func(context.Context, int, *ReadResult) error { + reads++ + return nil + } + _, err := client.Read(t.Context(), "records", WithRequests(clientTestReadRequest()), WithReadConsume(replaced), WithReadConsume(counts)) + if err != nil || reads != 1 { + t.Fatal(reads, err) + } + mutations := &clientTestPeer{mode: "typed_mutation"} + mutator := clientTestClient(t, mutations) + replacedMutation := func(context.Context, int, *MutationResult) error { + t.Error("a replaced mutation callback ran") + return nil + } + writes := 0 + countsMutation := func(context.Context, int, *MutationResult) error { + writes++ + return nil + } + _, err = mutator.Mutate(t.Context(), "records", WithRequests(deleteRequest("records/s:key")), WithMutateConsume(replacedMutation), WithMutateConsume(countsMutation)) + if err != nil || writes != 1 { + t.Fatal(writes, err) + } +} + +func TestSingleRecordSharesAStreamAndOtherCallsOwnOne(t *testing.T) { + peer := &countsMessages{} + address := serveForTest(t, peer) + client, err := New(address) + if err != nil { + t.Fatal(err) + } + defer client.Close() + own := func() int { + client.lanes.mu.Lock() + defer client.lanes.mu.Unlock() + count := 0 + for key := range client.lanes.groups { + if key.owner != 0 { + count++ + } + } + return count + } + shared := func() int { + client.lanes.mu.Lock() + defer client.lanes.mu.Unlock() + key := laneKey{store: "records"} + if group := client.lanes.groups[key]; group != nil { + return len(group.lanes) + } + return 0 + } + for range 3 { + if _, err := mutateOne(t.Context(), client, putRequest("records/s:a", `{}`)); err != nil { + t.Fatal(err) + } + } + if shared() != 1 || own() != 0 { + t.Fatal("single records did not share one stream", shared(), own()) + } + // A call with a callback owns a stream, which it releases at its end. + called := false + consume := func(context.Context, int, *MutationResult) error { + called = true + if own() != 1 { + t.Error("a call with a callback does not own a stream") + } + return nil + } + record := putRequest("records/s:b", `{}`) + _, err = client.Mutate(t.Context(), "records", WithRequests(record), WithMutateConsume(consume)) + if err != nil || !called { + t.Fatal(called, err) + } + if own() != 0 { + t.Fatal("a finished call kept its stream", own()) + } +} + +func TestSingleRecordWithOutgoingMetadataSendsIt(t *testing.T) { + peer := newLaneTestPeer() + client := laneTestClient(t, peer) + ctx := metadata.AppendToOutgoingContext(t.Context(), "x-request", "seven") + if _, err := mutateOne(ctx, client, putRequest("records/s:a", `{}`)); err != nil { + t.Fatal(err) + } + incoming, _ := peer.outgoing.Load().(metadata.MD) + if values := incoming.Get("x-request"); len(values) != 1 || values[0] != "seven" { + t.Fatal("a single record dropped its caller's metadata", incoming) + } +} diff --git a/operations.go b/operations.go deleted file mode 100644 index 11a8f9b..0000000 --- a/operations.go +++ /dev/null @@ -1,107 +0,0 @@ -package weir - -import ( - "context" - "errors" - - pb "github.com/batchstream/weir-protocol/api/weir/v1" -) - -// ReadOne sends one read without replay. -func ReadOne(ctx context.Context, client pb.StoreServiceClient, options ReadOneOptions) (*ReadResult, error) { - batch := ReadOptions{StoreName: options.StoreName, Requests: []*ReadRequest{options.Request}} - results, err := Read(ctx, client, batch) - if len(results) == 0 { - return nil, err - } - return results[0], err -} - -func Create(ctx context.Context, client pb.StoreServiceClient, options WriteOptions) (*MutationResult, error) { - return writeOne(ctx, client, options, MutationCreate) -} -func Put(ctx context.Context, client pb.StoreServiceClient, options WriteOptions) (*MutationResult, error) { - return writeOne(ctx, client, options, MutationPut) -} -func Replace(ctx context.Context, client pb.StoreServiceClient, options WriteOptions) (*MutationResult, error) { - return writeOne(ctx, client, options, MutationReplace) -} - -func writeOne(ctx context.Context, client pb.StoreServiceClient, options WriteOptions, action MutationAction) (*MutationResult, error) { - request := writeRequest(options.Request, action) - batch := MutateOptions{StoreName: options.StoreName, Requests: []*MutateRequest{request}} - return mutateOne(ctx, client, batch) -} - -func Delete(ctx context.Context, client pb.StoreServiceClient, options DeleteOptions) (*MutationResult, error) { - request := deleteRequest(options.Request) - batch := MutateOptions{StoreName: options.StoreName, Requests: []*MutateRequest{request}} - return mutateOne(ctx, client, batch) -} - -func AtomicTransform(ctx context.Context, client pb.StoreServiceClient, options AtomicTransformOptions) (*MutationResult, error) { - request := transformRequest(options.Request) - batch := MutateOptions{StoreName: options.StoreName, Requests: []*MutateRequest{request}} - return mutateOne(ctx, client, batch) -} - -func writeRequest(input *WriteRequest, action MutationAction) *MutateRequest { - if input == nil { - return nil - } - request := &MutateRequest{Resource: input.Resource, Action: action, Document: input.Document, Options: input.Options} - return request -} - -func deleteRequest(input *DeleteRequest) *MutateRequest { - if input == nil { - return nil - } - request := &MutateRequest{Resource: input.Resource, Action: MutationDelete, Options: input.Options} - return request -} - -func transformRequest(input *AtomicTransformRequest) *MutateRequest { - if input == nil { - return nil - } - request := &MutateRequest{Resource: input.Resource, Action: MutationAtomicTransform, Lua: input.Lua, BackendExpression: input.BackendExpression, Options: input.Options} - return request -} - -func mutateOne(ctx context.Context, client pb.StoreServiceClient, options MutateOptions) (*MutationResult, error) { - results, err := Mutate(ctx, client, options) - if len(results) == 0 { - return nil, err - } - return results[0], err -} - -// Native consumes response bytes incrementally. Backend errors remain native -// response data. A validated result remains evidence alongside later RPC errors. -func Native(ctx context.Context, client pb.StoreServiceClient, options NativeOptions) (*NativeResult, error) { - if options.Request == nil || options.Consume == nil { - return nil, errors.New("Native requires a request and byte consumer") - } - variant := &pb.Command_Native{Native: options.Request} - command := &pb.Command{Operation: variant} - request := &pb.ExecuteRequest{StoreName: options.StoreName, Index: 1, Command: command} - var result *NativeResult - consume := func(ctx context.Context, wire *pb.Event) error { - switch value := wire.Value.(type) { - case *pb.Event_Head: - result = &NativeResult{Response: value.Head} - case *pb.Event_Chunk: - return options.Consume(ctx, result.Response, value.Chunk) - case *pb.Event_NativeEnd: - if result == nil { - result = &NativeResult{} - } - result.Completion = value.NativeEnd.Completion - result.Failure = value.NativeEnd.Failure - } - return nil - } - err := execute(ctx, client, request, consume) - return result, err -} diff --git a/options.go b/options.go new file mode 100644 index 0000000..6c5da3f --- /dev/null +++ b/options.go @@ -0,0 +1,144 @@ +package weir + +import ( + "context" + "slices" + + "google.golang.org/grpc" +) + +// ClientOption configures a Client. +type ClientOption func(*clientOptions) + +type clientOptions struct { + dial []grpc.DialOption +} + +// WithDialOptions adds gRPC options to a Client's connection, such as a stats +// handler, an interceptor or a custom dialer. They can replace its defaults for +// transport credentials and keepalive. They cannot change what a Client relies +// on: gRPC never resends a record, message sizes stay at the protocol's limit, +// service configs have no effect, and its own resolver and balancer follow its +// servers. Stats handlers and interceptors see gRPC streams rather than calls, so +// they suit stream metrics, not per-call tracing. Several WithDialOptions add their +// options in order, and a nil one is ignored. +func WithDialOptions(options ...grpc.DialOption) ClientOption { + option := func(config *clientOptions) { + for _, dial := range options { + if dial != nil { + config.dial = append(config.dial, dial) + } + } + } + return option +} + +// Option configures one Read or Mutate call whose requests have type Q. +type Option[Q any] func(*callOptions[Q]) + +// ReadOption configures a Read. +type ReadOption = Option[*ReadRequest] + +// MutateOption configures a Mutate. +type MutateOption = Option[*MutateRequest] + +// NativeOption configures a Native call. +type NativeOption func(*nativeOptions) + +type callOptions[Q any] struct { + requests []Q + next func(context.Context) (Q, error) + consumeRead func(context.Context, int, *ReadResult) error + consumeMutate func(context.Context, int, *MutationResult) error + queryVisible bool +} + +type nativeOptions struct { + chunks func(context.Context, *NativeResponse, []byte) error +} + +// WithRequests adds requests the call already holds. They are all validated +// before anything is sent, so an invalid one rejects the whole call and WithNext +// is never called. Several WithRequests add their requests in order. Keep the +// requests and their document bytes unchanged until the call returns, and after +// a call its context ended do not reuse them at all: a single record's shared +// stream may still be reading them. +func WithRequests[Q any](requests ...Q) Option[Q] { + option := func(options *callOptions[Q]) { + if options.requests == nil { + options.requests = slices.Clip(requests) + return + } + options.requests = append(options.requests, requests...) + } + return option +} + +// WithNext produces requests one at a time, after those of WithRequests have +// entered the window of 32 records and 8 MiB, without waiting for their results. +// Next returns io.EOF after the last request. Each request is validated as it is +// produced. An invalid request or another error from Next ends the input there: +// the records before it still finish, and the call then returns that error. +// Next runs concurrently with a consume callback, so synchronize what they share. +// It must honor its context, and return promptly once that ends. Keep a produced +// request and its document bytes unchanged until the call returns, or with a +// consume callback until its result arrives there. +// +// Without a consume callback, results accumulate until the call returns, and +// memory grows with the number of records. +func WithNext[Q any](next func(ctx context.Context) (Q, error)) Option[Q] { + option := func(options *callOptions[Q]) { options.next = next } + return option +} + +// WithReadConsume receives each read's result as its record completes, instead +// of the call collecting them, and Read returns nil results. The index is the +// record's position in the input, from 0. Results delivered before a later error +// remain valid. Calls run one at a time and hold back the input when slow; an +// error from consume ends the call. It must honor its context, as a call does not +// return before it does. +func WithReadConsume(consume func(ctx context.Context, index int, result *ReadResult) error) ReadOption { + option := func(options *callOptions[*ReadRequest]) { options.consumeRead = consume } + return option +} + +// WithMutateConsume receives each mutation's result as its record completes, +// instead of the call collecting them, and Mutate returns nil results. The index +// is the record's position in the input, from 0. Each result is an individual +// acknowledgement that a later error does not revoke; records without a result +// may have applied. Calls run one at a time and hold back the input when slow; an +// error from consume ends the call. It must honor its context, as a call does not +// return before it does. +func WithMutateConsume(consume func(ctx context.Context, index int, result *MutationResult) error) MutateOption { + option := func(options *callOptions[*MutateRequest]) { options.consumeMutate = consume } + return option +} + +// WithQueryVisible requires every mutation of the call to be visible to later +// normal queries before an APPLIED result without Failure. APPLIED with Failure +// confirms application only. Without it, a mutation completes on application +// acknowledgement, with no separate visibility guarantee. +func WithQueryVisible() MutateOption { + option := func(options *callOptions[*MutateRequest]) { options.queryVisible = true } + return option +} + +// WithChunks receives a Native response's body incrementally, instead of the call +// collecting it into NativeResult.Body. Each nonempty chunk finishes before the +// next one, and consume must honor its context. An empty response invokes no +// callback. Do not mutate the response metadata, which stays attached to the +// result. +func WithChunks(consume func(ctx context.Context, response *NativeResponse, chunk []byte) error) NativeOption { + option := func(options *nativeOptions) { options.chunks = consume } + return option +} + +func applyOptions[Q any](options []Option[Q]) callOptions[Q] { + applied := callOptions[Q]{} + for _, option := range options { + if option != nil { + option(&applied) + } + } + return applied +} diff --git a/record_stream.go b/record_stream.go deleted file mode 100644 index 57b1d9b..0000000 --- a/record_stream.go +++ /dev/null @@ -1,330 +0,0 @@ -package weir - -import ( - "context" - "errors" - "fmt" - "io" - "math" - "sync" - "sync/atomic" - - "github.com/batchstream/weir-protocol/api/protocol" - pb "github.com/batchstream/weir-protocol/api/weir/v1" - "google.golang.org/grpc" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/status" - "google.golang.org/protobuf/proto" -) - -// ReadStream sends each produced read immediately and consumes indexed results. -// Confirmed callbacks remain valid when a later transport or producer fails. -func ReadStream(ctx context.Context, client pb.StoreServiceClient, options ReadStreamOptions) error { - if options.Next == nil || options.Consume == nil { - return errors.New("ReadStream requires a producer and consumer") - } - next := func(ctx context.Context) (*pb.Command, error) { - request, err := options.Next(ctx) - if err != nil { - return nil, err - } - return readCommand(request) - } - consume := func(ctx context.Context, response *pb.ExecuteResponse) error { - wire := response.Event.GetReadResult() - if wire == nil { - return errors.New("unexpected event in Read stream") - } - return options.Consume(ctx, response.Index, readResult(wire)) - } - optionsInternal := recordStreamOptions{store: options.StoreName, next: next, consume: consume} - return recordStream(ctx, client, optionsInternal) -} - -// MutateStream never replays a request. Every consumed result is an individual -// acknowledgement; later failures leave other items unacknowledged. -func MutateStream(ctx context.Context, client pb.StoreServiceClient, options MutateStreamOptions) error { - if options.Next == nil || options.Consume == nil { - return errors.New("MutateStream requires a producer and consumer") - } - next := func(ctx context.Context) (*pb.Command, error) { - input, err := options.Next(ctx) - if err != nil { - return nil, err - } - return mutateCommand(input) - } - consume := func(ctx context.Context, response *pb.ExecuteResponse) error { - result := response.Event.GetMutationResult() - if result == nil { - return errors.New("unexpected event in Mutate stream") - } - return options.Consume(ctx, response.Index, result) - } - optionsInternal := recordStreamOptions{store: options.StoreName, next: next, consume: consume} - return recordStream(ctx, client, optionsInternal) -} - -// readCommand validates one read and wraps it for sending. -func readCommand(request *ReadRequest) (*pb.Command, error) { - if err := protocol.ValidateReadRequest(request); err != nil { - return nil, err - } - operation := &pb.Command_Read{Read: request} - command := &pb.Command{Operation: operation} - return command, nil -} - -// mutateCommand validates one mutation and wraps it for sending. -func mutateCommand(input *MutateRequest) (*pb.Command, error) { - request, err := wireMutation(input) - if err != nil { - return nil, err - } - if err := protocol.ValidateMutationRequest(request); err != nil { - return nil, err - } - operation := &pb.Command_Mutate{Mutate: request} - command := &pb.Command{Operation: operation} - return command, nil -} - -func readResult(wire *pb.ReadResult) *ReadResult { - result := &ReadResult{Document: wire.GetDocument(), Missing: wire.GetMissing() != nil, Failure: wire.GetFailure()} - return result -} - -// refusedStream returns the Failure a server sends when it turns a stream away -// before reading any of it, which proves that nothing on the stream took -// effect. A lost connection is UNAVAILABLE too, but has no such Failure. -func refusedStream(err error) *pb.Failure { - if status.Code(err) != codes.Unavailable { - return nil - } - for _, detail := range status.Convert(err).Details() { - if failure, ok := detail.(*pb.Failure); ok { - return failure - } - } - return nil -} - -// mutateAlone sends one mutation on a stream of its own. A refused stream is -// the record's refusal, like one on a shared stream. -func (c *Client) mutateAlone(ctx context.Context, store string, command *pb.Command) (*MutationResult, error) { - result, err := sendMutationAlone(ctx, c.store, store, command) - if failure := refusedStream(err); failure != nil && result == nil { - return protocol.Mutation(pb.MutationOutcome_NOT_STARTED, failure), nil - } - return result, callError(ctx, err) -} - -// readAlone sends one read on a stream of its own, as mutateAlone does. -func (c *Client) readAlone(ctx context.Context, store string, request *ReadRequest) (*ReadResult, error) { - alone := ReadOneOptions{StoreName: store, Request: request} - result, err := ReadOne(ctx, c.store, alone) - if failure := refusedStream(err); failure != nil && result == nil { - refused := &ReadResult{Failure: failure} - return refused, nil - } - return result, callError(ctx, err) -} - -// sendMutationAlone sends one wire mutation on a stream of its own. -func sendMutationAlone(ctx context.Context, client pb.StoreServiceClient, store string, command *pb.Command) (*MutationResult, error) { - var result *MutationResult - sent := false - next := func(context.Context) (*pb.Command, error) { - if sent { - return nil, io.EOF - } - sent = true - return command, nil - } - consume := func(_ context.Context, response *pb.ExecuteResponse) error { - result = response.Event.GetMutationResult() - if result == nil { - return errors.New("unexpected event in Mutate stream") - } - return nil - } - options := recordStreamOptions{store: store, next: next, consume: consume} - err := recordStream(ctx, client, options) - return result, err -} - -type recordStreamOptions struct { - store string - next func(context.Context) (*pb.Command, error) - consume func(context.Context, *pb.ExecuteResponse) error -} - -// These backpressure windows permit a larger individual request to proceed alone. -const recordWindowItems = protocol.RecordWindow -const recordWindowBytes = 8 << 20 - -type recordSendResult struct { - count uint64 - err error -} - -func recordStream(ctx context.Context, client pb.StoreServiceClient, options recordStreamOptions) error { - if client == nil || !protocol.ValidStoreName(options.store) { - return errors.New("Execute requires a client and valid Store") - } - if err := ctx.Err(); err != nil { - return err - } - ctx, cancel := context.WithCancel(ctx) - defer cancel() - stream, err := client.Execute(ctx, grpc.MaxCallSendMsgSize(protocol.MaxTransportMessageBytes), grpc.MaxCallRecvMsgSize(protocol.MaxTransportMessageBytes), grpc.MaxRetryRPCBufferSize(0)) - if err != nil { - return err - } - // Calling Context commits this attempt, disabling transparent write replay. - _ = stream.Context() - // Results arrive in completion order; each one returns its record's credit. - var mu sync.Mutex - sizes := make(map[uint64]int, recordWindowItems) - released := make(chan int, recordWindowItems) - done := make(chan recordSendResult, 1) - var finishedInput atomic.Bool - go func() { - result := recordSendResult{} - defer func() { - // Local producer/send errors need not yield a peer response. EOF - // is different: Recv must expose the peer's final status. - if result.err != nil && !errors.Is(result.err, io.EOF) { - cancel() - } - done <- result - }() - inflight, bytes := 0, 0 - for { - // Stop producing when the item window fills. A size-dependent wait - // below can retain only the current unsent item. - for inflight == recordWindowItems { - select { - case size := <-released: - inflight-- - bytes -= size - case <-ctx.Done(): - result.err = ctx.Err() - return - } - } - if err := ctx.Err(); err != nil { - result.err = err - return - } - command, err := options.next(ctx) - if errors.Is(err, io.EOF) { - if result.count == 0 { - result.err = errors.New("Execute requires at least one record") - return - } - finishedInput.Store(true) - result.err = stream.CloseSend() - return - } - if err != nil { - result.err = fmt.Errorf("request %d: %w", result.count+1, err) - return - } - if result.count == math.MaxUint64-1 { - result.err = errors.New("record ordinal exhausted") - return - } - request := &pb.ExecuteRequest{StoreName: options.store, Index: result.count + 1, Command: command} - size := proto.Size(request) - - // A legal item larger than the byte window proceeds alone. Ordinary - // items wait for individual acknowledgements to return byte credit. - for inflight > 0 && size > recordWindowBytes-bytes { - select { - case credit := <-released: - inflight-- - bytes -= credit - case <-ctx.Done(): - result.err = ctx.Err() - return - } - } - if err := ctx.Err(); err != nil { - result.err = err - return - } - // Register before Send: Recv can observe the response before Send - // returns. Only the receiver publishes acknowledged byte credit. - mu.Lock() - sizes[request.Index] = size - mu.Unlock() - inflight++ - bytes += size - result.count = request.Index - message := &pb.ExecuteRequests{Requests: []*pb.ExecuteRequest{request}} - if err := stream.Send(message); err != nil { - result.err = err - return - } - } - }() - var received uint64 - var receiveErr error - // consume confirms one result and returns its record's credit. - consume := func(response *pb.ExecuteResponse) error { - if err := protocol.ValidateExecuteResponse(response); err != nil { - return err - } - mu.Lock() - size, ok := sizes[response.Index] - delete(sizes, response.Index) - mu.Unlock() - if !ok { - return errors.New("Execute response ordinal is missing, duplicated or unexpected") - } - if err := options.consume(ctx, response); err != nil { - return err - } - received++ - released <- size - return nil - } - for receiveErr == nil { - message, err := stream.Recv() - if err != nil { - if errors.Is(err, io.EOF) { - if !finishedInput.Load() { - receiveErr = errors.New("Execute ended before input was closed") - } - } else { - receiveErr = err - } - break - } - receiveErr = protocol.ValidateExecuteResponses(message) - for _, response := range message.GetResponses() { - if receiveErr != nil { - break - } - receiveErr = consume(response) - } - } - if receiveErr != nil { - cancel() - } - sent := <-done - if sent.err != nil && !errors.Is(sent.err, io.EOF) { - if receiveErr == nil { - return sent.err - } - return errors.Join(receiveErr, sent.err) - } - if receiveErr != nil { - return receiveErr - } - if sent.err != nil || received != sent.count { - return fmt.Errorf("Execute incomplete: confirmed %d of %d records", received, sent.count) - } - return nil -} diff --git a/record_stream_test.go b/record_stream_test.go deleted file mode 100644 index 88ad54b..0000000 --- a/record_stream_test.go +++ /dev/null @@ -1,328 +0,0 @@ -package weir - -import ( - "context" - "errors" - "io" - "strings" - "sync/atomic" - "testing" - "time" - - "github.com/batchstream/weir-protocol/api/protocol" -) - -func TestReadStreamLargeDuplexCallConsumesIncrementally(t *testing.T) { - peer := &clientTestPeer{mode: "large_duplex"} - client := clientTestConnection(t, peer) - produced, consumed := 0, uint64(0) - var responseBytes uint64 - options := ReadStreamOptions{StoreName: "records"} - options.Next = func(context.Context) (*ReadRequest, error) { - if produced == 48 { - return nil, io.EOF - } - produced++ - request := &ReadRequest{Resource: "records/s:key"} - return request, nil - } - options.Consume = func(_ context.Context, index uint64, result *ReadResult) error { - consumed++ - if index != consumed || result.Document == nil || len(result.Document.Data) != 1<<20 { - return errors.New("large response index or document mismatch") - } - responseBytes += uint64(len(result.Document.Data)) - return nil - } - ctx, cancel := context.WithTimeout(t.Context(), 15*time.Second) - defer cancel() - if err := ReadStream(ctx, client, options); err != nil { - t.Fatal(err) - } - if produced != 48 || consumed != 48 || responseBytes <= 32<<20 || peer.streams.Load() != 1 || peer.maxBytes.Load() > int64(protocol.MaxTransportMessageBytes) { - t.Fatal("large call was truncated or split into RPCs", produced, consumed, responseBytes, peer.streams.Load(), peer.maxBytes.Load()) - } -} - -func TestMutateStreamPausedConsumerBoundsProducerAndCancels(t *testing.T) { - peer := &clientTestPeer{mode: "typed_mutation"} - client := clientTestConnection(t, peer) - document := &Document{ContentType: "application/octet-stream", Data: make([]byte, (2 << 20))} - var produced atomic.Int64 - started := make(chan struct{}) - options := MutateStreamOptions{StoreName: "records"} - options.Next = func(ctx context.Context) (*MutateRequest, error) { - if err := ctx.Err(); err != nil { - return nil, err - } - produced.Add(1) - request := &MutateRequest{Resource: "records/s:key", Action: MutationPut, Document: document} - return request, nil - } - options.Consume = func(ctx context.Context, index uint64, result *MutationResult) error { - if index != 1 || result.GetOutcome() != MutationApplied { - return errors.New("invalid first acknowledgement") - } - close(started) - <-ctx.Done() - return ctx.Err() - } - ctx, cancel := context.WithCancel(t.Context()) - defer cancel() - done := make(chan error, 1) - go func() { done <- MutateStream(ctx, client, options) }() - select { - case <-started: - case <-time.After(3 * time.Second): - t.Fatal("consumer did not start") - } - // At most three encoded 2MiB inputs fit the 8MiB window. The producer - // may yield one unsent item before its size causes it to wait for credit. - timer := time.NewTimer(100 * time.Millisecond) - defer timer.Stop() - select { - case err := <-done: - t.Fatal("stream finished while its consumer was paused", err) - case <-timer.C: - } - if count := produced.Load(); count > 4 || peer.received.Load() > 3 || peer.maxBytes.Load() > int64(protocol.MaxTransportMessageBytes) { - t.Fatal("paused consumer allowed unbounded input", count, peer.received.Load(), peer.maxBytes.Load()) - } - cancel() - select { - case err := <-done: - if err == nil { - t.Fatal("cancelled stream succeeded") - } - case <-time.After(3 * time.Second): - t.Fatal("cancelled stream did not join its sender") - } -} - -func TestRecordStreamRejectsDuplicateAndExcessIndexesKeepsPrefix(t *testing.T) { - for _, mode := range []string{"duplicate_index", "extra_result"} { - t.Run(mode, func(t *testing.T) { - peer := &clientTestPeer{mode: mode} - client := clientTestConnection(t, peer) - options := ReadOptions{StoreName: "records", Requests: []*ReadRequest{clientTestReadRequest(), clientTestReadRequest()}} - results, err := Read(t.Context(), client, options) - if err == nil || len(results) != 2 || results[0] == nil { - t.Fatal("malformed index succeeded or erased confirmed prefix", results, err) - } - if mode == "duplicate_index" && results[1] != nil { - t.Fatal("duplicate index acknowledged another input", results) - } - }) - } -} - -func TestReadStreamProducerFailureKeepsEarlierAcknowledgements(t *testing.T) { - peer := &clientTestPeer{mode: "batch_read_order"} - client := clientTestConnection(t, peer) - confirmed := make(chan struct{}) - produced, consumed := 0, uint64(0) - options := ReadStreamOptions{StoreName: "records"} - options.Next = func(ctx context.Context) (*ReadRequest, error) { - if produced == 64 { - select { - case <-confirmed: - case <-ctx.Done(): - return nil, ctx.Err() - } - request := &ReadRequest{Resource: "bad//resource"} - return request, nil - } - produced++ - return clientTestReadRequest(), nil - } - options.Consume = func(_ context.Context, index uint64, result *ReadResult) error { - consumed++ - if index != consumed || result == nil { - return errors.New("invalid earlier acknowledgement") - } - if index == 64 { - close(confirmed) - } - return nil - } - ctx, cancel := context.WithTimeout(t.Context(), 3*time.Second) - defer cancel() - err := ReadStream(ctx, client, options) - if err == nil || !strings.Contains(err.Error(), "request 65") || consumed != 64 || peer.streams.Load() != 1 || peer.received.Load() != 64 { - t.Fatal("producer failure replayed requests or discarded earlier evidence", err, consumed, peer.streams.Load(), peer.received.Load()) - } -} - -func TestReadStreamCancellationJoinsBlockedProducer(t *testing.T) { - peer := &clientTestPeer{} - client := clientTestConnection(t, peer) - started, stopped := make(chan struct{}), make(chan struct{}) - options := ReadStreamOptions{StoreName: "records"} - options.Next = func(ctx context.Context) (*ReadRequest, error) { - close(started) - <-ctx.Done() - close(stopped) - return nil, ctx.Err() - } - options.Consume = func(context.Context, uint64, *ReadResult) error { - return errors.New("no input was produced") - } - ctx, cancel := context.WithCancel(t.Context()) - defer cancel() - done := make(chan error, 1) - go func() { done <- ReadStream(ctx, client, options) }() - select { - case <-started: - case <-time.After(time.Second): - t.Fatal("producer did not start") - } - cancel() - select { - case err := <-done: - if err == nil { - t.Fatal("cancelled producer succeeded") - } - select { - case <-stopped: - default: - t.Fatal("stream returned before producer finished") - } - case <-time.After(time.Second): - t.Fatal("blocked producer was not cancelled and joined") - } -} - -func TestReadStreamRejectsPrematureSuccessAndJoinsSender(t *testing.T) { - peer := &clientTestPeer{mode: "early_eof"} - client := clientTestConnection(t, peer) - options := ReadStreamOptions{StoreName: "records"} - options.Next = func(context.Context) (*ReadRequest, error) { return clientTestReadRequest(), nil } - options.Consume = func(context.Context, uint64, *ReadResult) error { return nil } - ctx, cancel := context.WithTimeout(t.Context(), 3*time.Second) - defer cancel() - if err := ReadStream(ctx, client, options); err == nil { - t.Fatal("premature server EOF accepted an unfinished producer") - } -} - -func TestRecordStreamSendsBeforeProducerHasItsNextItem(t *testing.T) { - for _, kind := range []string{"read", "mutate"} { - t.Run(kind, func(t *testing.T) { - peer := &clientTestPeer{mode: "typed_mutation"} - client := clientTestConnection(t, peer) - firstConsumed := make(chan struct{}) - produced, consumed := 0, uint64(0) - ctx, cancel := context.WithTimeout(t.Context(), 3*time.Second) - defer cancel() - var err error - if kind == "read" { - options := ReadStreamOptions{StoreName: "records"} - options.Next = func(ctx context.Context) (*ReadRequest, error) { - if produced == 2 { - return nil, io.EOF - } - if produced == 1 { - select { - case <-firstConsumed: - case <-ctx.Done(): - return nil, ctx.Err() - } - } - produced++ - return clientTestReadRequest(), nil - } - options.Consume = func(_ context.Context, index uint64, result *ReadResult) error { - consumed++ - if index != consumed || result.Document == nil { - return errors.New("unexpected read acknowledgement") - } - if index == 1 { - close(firstConsumed) - } - return nil - } - err = ReadStream(ctx, client, options) - } else { - options := MutateStreamOptions{StoreName: "records"} - options.Next = func(ctx context.Context) (*MutateRequest, error) { - if produced == 2 { - return nil, io.EOF - } - if produced == 1 { - select { - case <-firstConsumed: - case <-ctx.Done(): - return nil, ctx.Err() - } - } - produced++ - request := &MutateRequest{Resource: "records/s:key", Action: MutationDelete} - return request, nil - } - options.Consume = func(_ context.Context, index uint64, result *MutationResult) error { - consumed++ - if index != consumed || result.Outcome != MutationApplied { - return errors.New("unexpected mutation acknowledgement") - } - if index == 1 { - close(firstConsumed) - } - return nil - } - err = MutateStream(ctx, client, options) - } - if err != nil || produced != 2 || consumed != 2 || peer.received.Load() != 2 || peer.streams.Load() != 1 { - t.Fatal("stream waited for a batch or EOF before sending its first item", err, produced, consumed, peer.received.Load(), peer.streams.Load()) - } - }) - } -} - -func TestReadStreamPausedConsumerBoundsItemWindow(t *testing.T) { - peer := &clientTestPeer{mode: "read_missing"} - client := clientTestConnection(t, peer) - var produced atomic.Int64 - started := make(chan struct{}) - options := ReadStreamOptions{StoreName: "records"} - options.Next = func(context.Context) (*ReadRequest, error) { - produced.Add(1) - return clientTestReadRequest(), nil - } - options.Consume = func(ctx context.Context, index uint64, result *ReadResult) error { - if index != 1 || !result.Missing { - return errors.New("unexpected first result") - } - close(started) - <-ctx.Done() - return ctx.Err() - } - ctx, cancel := context.WithCancel(t.Context()) - defer cancel() - done := make(chan error, 1) - go func() { done <- ReadStream(ctx, client, options) }() - select { - case <-started: - case <-time.After(time.Second): - t.Fatal("consumer did not begin") - } - deadline := time.Now().Add(time.Second) - for produced.Load() < recordWindowItems && time.Now().Before(deadline) { - time.Sleep(time.Millisecond) - } - if count := produced.Load(); count != recordWindowItems { - t.Fatal("paused consumer did not stop the producer at the item window", count) - } - time.Sleep(20 * time.Millisecond) - if produced.Load() != recordWindowItems || peer.received.Load() > recordWindowItems { - t.Fatal("paused consumer allowed more items than the window", produced.Load(), peer.received.Load()) - } - cancel() - select { - case err := <-done: - if err == nil { - t.Fatal("cancelled stream succeeded") - } - case <-time.After(time.Second): - t.Fatal("item-credit wait did not cancel and join") - } -} diff --git a/records.go b/records.go index 9e6c889..ed9e8a4 100644 --- a/records.go +++ b/records.go @@ -8,147 +8,299 @@ import ( "github.com/batchstream/weir-protocol/api/protocol" pb "github.com/batchstream/weir-protocol/api/weir/v1" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/status" ) -// Read validates every input before opening one Execute stream. Results -// retain input order. A later stream error preserves already confirmed items; -// unconfirmed positions remain nil. Requests are never automatically replayed. -func Read(ctx context.Context, client pb.StoreServiceClient, options ReadOptions) ([]*ReadResult, error) { - if err := validateReads(ctx, options); err != nil { - return nil, err +// Read sends the reads of WithRequests and then of WithNext, and returns one +// result per read in input order. A later error preserves confirmed results, and +// unconfirmed positions stay nil. With WithReadConsume results go to its callback +// and Read returns nil results. It needs at least one read, or returns an error. +// +// A single read without WithNext, WithReadConsume or outgoing gRPC metadata shares +// long-lived streams with concurrent calls. Any other Read uses a gRPC stream of +// its own that follows the Client's servers. A read a server refused is sent +// again; nothing else is. +func (c *Client) Read(ctx context.Context, store string, options ...ReadOption) ([]*ReadResult, error) { + config := applyOptions(options) + call := recordCall[*ReadRequest, *ReadResult]{ + kind: readKind, + store: store, + requests: config.requests, + next: config.next, + consume: config.consumeRead, + command: readCommand, } - stream := func(ctx context.Context, streamOptions ReadStreamOptions) error { - return ReadStream(ctx, client, streamOptions) - } - return readRecords(ctx, options, stream) + return runRecords(ctx, c, call) } -// Mutate validates every input before opening one Execute stream. -// Same-resource mutations execute in input order, including after item failures. -// A later stream error preserves confirmed results; nil positions remain -// unacknowledged and may have applied. The call is not a transaction or replayed. -func Mutate(ctx context.Context, client pb.StoreServiceClient, options MutateOptions) ([]*MutationResult, error) { - if err := validateMutations(ctx, options); err != nil { - return nil, err +// Mutate sends the mutations of WithRequests and then of WithNext, and returns +// one result per mutation in input order. Mutations of one resource run in input +// order, each after the previous one's result, including after a failure; those +// of different resources may run concurrently. It is not a transaction. A later +// error preserves confirmed results, and unconfirmed positions stay nil and may +// have applied. With WithMutateConsume results go to its callback and Mutate +// returns nil results. It needs at least one mutation, or returns an error. +// +// A single mutation without WithNext, WithMutateConsume or outgoing gRPC metadata +// shares long-lived streams with concurrent calls. Any other Mutate uses a gRPC +// stream of its own that follows the Client's servers. A mutation a server refused +// is sent again; nothing else is. +func (c *Client) Mutate(ctx context.Context, store string, options ...MutateOption) ([]*MutationResult, error) { + config := applyOptions(options) + var completion *pb.MutationOptions + if config.queryVisible { + completion = &pb.MutationOptions{Visibility: pb.MutationVisibility_QUERY_VISIBLE} + } + command := func(request *MutateRequest) (*pb.Command, string, error) { + return mutateCommand(request, completion) } - stream := func(ctx context.Context, streamOptions MutateStreamOptions) error { - return MutateStream(ctx, client, streamOptions) + call := recordCall[*MutateRequest, *MutationResult]{ + kind: mutateKind, + store: store, + requests: config.requests, + next: config.next, + consume: config.consumeMutate, + command: command, } - return mutateRecords(ctx, options, stream) + return runRecords(ctx, c, call) } -// Read sends its reads over a stream of the Client, so it moves between servers -// as ReadStream does. Results retain input order; a later error preserves -// confirmed items and leaves unconfirmed positions nil. -func (c *Client) Read(ctx context.Context, options ReadOptions) ([]*ReadResult, error) { - if err := validateReads(ctx, options); err != nil { - return nil, err - } - if _, err := c.storeClient(); err != nil { - return nil, err - } - stream := func(ctx context.Context, options ReadStreamOptions) error { - return c.readStream(ctx, options, true) +// recordKind holds what differs between the results of reads and of mutations. +type recordKind[R comparable] struct { + name string + read bool + // result returns an event's result, or the zero value for another event. + result func(*pb.Event) R + // refused returns the result of a record a server turned away. + refused func(*pb.Failure) R + // isRefused reports a result a server may accept later. + isRefused func(R) bool +} + +var readKind = recordKind[*ReadResult]{name: "Read", read: true, result: readEventResult, refused: readResultOfRefusal, isRefused: readRefused} + +var mutateKind = recordKind[*MutationResult]{name: "Mutate", result: mutationEventResult, refused: mutationResultOfRefusal, isRefused: mutationRefused} + +func readEventResult(event *pb.Event) *ReadResult { + wire := event.GetReadResult() + if wire == nil { + return nil } - return readRecords(ctx, options, stream) + return readResult(wire) +} + +func readResultOfRefusal(failure *pb.Failure) *ReadResult { + result := &ReadResult{Failure: failure} + return result +} + +func mutationEventResult(event *pb.Event) *MutationResult { + return event.GetMutationResult() } -// Mutate sends its mutations over a stream of the Client, so it moves between -// servers as MutateStream does and each resource's mutations apply in input -// order. A later error preserves confirmed results; nil positions remain -// unacknowledged and may have applied. The call is not a transaction. -func (c *Client) Mutate(ctx context.Context, options MutateOptions) ([]*MutationResult, error) { - if err := validateMutations(ctx, options); err != nil { - return nil, err +func mutationResultOfRefusal(failure *pb.Failure) *MutationResult { + return protocol.Mutation(pb.MutationOutcome_NOT_STARTED, failure) +} + +// recordCall describes one Read or Mutate of a Client. +type recordCall[Q any, R comparable] struct { + kind recordKind[R] + store string + requests []Q + next func(context.Context) (Q, error) + consume func(context.Context, int, R) error + // command returns a validated command and, when it must keep input order, + // its resource. + command func(Q) (*pb.Command, string, error) +} + +func runRecords[Q any, R comparable](ctx context.Context, c *Client, call recordCall[Q, R]) ([]R, error) { + var results []R + if call.consume == nil { + results = make([]R, len(call.requests)) } if _, err := c.storeClient(); err != nil { - return nil, err + return results, err + } + last, err := call.validate(ctx) + if err != nil { + return results, err + } + if call.shares(ctx) { + key := laneKey{store: call.store, read: call.kind.read} + send := call.sender(c, key) + once := func() (R, error) { return send(ctx, last, nil) } + results[0], err = resendRefused(ctx, c, call.kind.isRefused, once) + return results, err + } + produced := 0 + position := 0 + next := func(ctx context.Context) (*pb.Command, string, error) { + var request Q + switch { + case position < len(call.requests): + request = call.requests[position] + position++ + case call.next != nil: + var err error + request, err = call.next(ctx) + if err != nil { + return nil, "", err + } + default: + return nil, "", io.EOF + } + command, resource, err := call.command(request) + if err == nil { + produced++ + } + return command, resource, err + } + consume := call.consume + if consume == nil { + consume = func(_ context.Context, index int, result R) error { + results = growResults(results, index+1) + results[index] = result + return nil + } + } + key := c.lanes.ownKey(call.store, call.kind.read) + // Next may block, so only requests it cannot follow wait to share a message. + stream := clientStreamOptions[R]{key: key, known: call.next == nil, next: next, send: call.sender(c, key), refused: call.kind.isRefused, consume: consume} + err = runClientStream(ctx, c, stream) + if call.consume == nil { + results = growResults(results, produced) } - stream := func(ctx context.Context, options MutateStreamOptions) error { - return c.mutateStream(ctx, options, true) + return results, err +} + +// growResults extends results to a length, leaving the new positions unconfirmed. +func growResults[R any](results []R, length int) []R { + for len(results) < length { + var unconfirmed R + results = append(results, unconfirmed) } - return mutateRecords(ctx, options, stream) + return results } -func validateReads(ctx context.Context, options ReadOptions) error { - if !protocol.ValidStoreName(options.StoreName) || len(options.Requests) == 0 { - return errors.New("Read requires a Store and nonempty requests") +// validate checks the store and every held request before anything is sent. It +// returns the last request's command, which a single shared record sends. +func (call recordCall[Q, R]) validate(ctx context.Context) (*pb.Command, error) { + if !protocol.ValidStoreName(call.store) || (len(call.requests) == 0 && call.next == nil) { + return nil, fmt.Errorf("%s requires a Store and requests or a producer", call.kind.name) } - for index, item := range options.Requests { + var command *pb.Command + for index, request := range call.requests { if err := ctx.Err(); err != nil { - return err + return nil, err } - if err := protocol.ValidateReadRequest(item); err != nil { - return fmt.Errorf("Read request %d: %w", index+1, err) + var err error + command, _, err = call.command(request) + if err != nil { + return nil, fmt.Errorf("%s request %d: %w", call.kind.name, index, err) } } - return nil + return command, nil } -func validateMutations(ctx context.Context, options MutateOptions) error { - if !protocol.ValidStoreName(options.StoreName) || len(options.Requests) == 0 { - return errors.New("Mutate requires a Store and nonempty requests") +// shares reports a call of one record, which needs no stream of its own. A call +// with outgoing metadata keeps one, because shared streams belong to no caller. +func (call recordCall[Q, R]) shares(ctx context.Context) bool { + if len(call.requests) != 1 || call.next != nil || call.consume != nil { + return false } - for index, item := range options.Requests { - if err := ctx.Err(); err != nil { - return err + _, outgoing := metadata.FromOutgoingContext(ctx) + return !outgoing +} + +// sender returns a function that sends a command on a stream of the key's group, +// or on a gRPC stream of its own when none takes it, unless it is already queued. +func (call recordCall[Q, R]) sender(c *Client, key laneKey) func(context.Context, *pb.Command, *queuedRecord) (R, error) { + send := func(ctx context.Context, command *pb.Command, queued *queuedRecord) (R, error) { + var none R + event, shared, err := c.lanes.finish(ctx, key, command, queued) + if !shared { + return call.kind.alone(ctx, c, call.store, command) } - mutation, err := wireMutation(item) if err != nil { - return fmt.Errorf("Mutate request %d: %w", index+1, err) - } - if err := protocol.ValidateMutationRequest(mutation); err != nil { - return fmt.Errorf("Mutate request %d: %w", index+1, err) + return none, err } + return call.kind.result(event), nil } - return nil + return send } -func readRecords(ctx context.Context, options ReadOptions, stream func(context.Context, ReadStreamOptions) error) ([]*ReadResult, error) { - results := make([]*ReadResult, len(options.Requests)) - position := 0 - streamOptions := ReadStreamOptions{StoreName: options.StoreName} - streamOptions.Next = func(context.Context) (*ReadRequest, error) { - if position == len(options.Requests) { - return nil, io.EOF +// alone sends one command on a gRPC stream of its own. A refused stream is the +// record's refusal, like one on a shared stream. +func (kind recordKind[R]) alone(ctx context.Context, c *Client, store string, command *pb.Command) (R, error) { + var none, result R + consume := func(_ context.Context, event *pb.Event) (bool, error) { + result = kind.result(event) + if result == none { + return false, fmt.Errorf("unexpected event in %s stream", kind.name) } - item := options.Requests[position] - position++ - return item, nil + return true, nil } - streamOptions.Consume = func(_ context.Context, index uint64, result *ReadResult) error { - results[index-1] = result - return nil + request := &pb.ExecuteRequest{StoreName: store, Index: 1, Command: command} + err := execute(ctx, c.store, request, consume) + if failure := refusedStream(err); failure != nil && result == none { + return kind.refused(failure), nil } - err := stream(ctx, streamOptions) - return results, err + return result, callError(ctx, err) } -func mutateRecords(ctx context.Context, options MutateOptions, stream func(context.Context, MutateStreamOptions) error) ([]*MutationResult, error) { - results := make([]*MutationResult, len(options.Requests)) - position := 0 - streamOptions := MutateStreamOptions{StoreName: options.StoreName} - streamOptions.Next = func(context.Context) (*MutateRequest, error) { - if position == len(options.Requests) { - return nil, io.EOF +// refusedStream returns the Failure a server sends when it turns a stream away +// before reading any of it, which proves that nothing on the stream took +// effect. A lost connection is UNAVAILABLE too, but has no such Failure. +func refusedStream(err error) *pb.Failure { + if status.Code(err) != codes.Unavailable { + return nil + } + for _, detail := range status.Convert(err).Details() { + if failure, ok := detail.(*pb.Failure); ok { + return failure } - item := options.Requests[position] - position++ - return item, nil } - streamOptions.Consume = func(_ context.Context, index uint64, result *MutationResult) error { - results[index-1] = result - return nil + return nil +} + +// readCommand validates one read and wraps it for sending. Reads keep no order. +func readCommand(request *ReadRequest) (*pb.Command, string, error) { + if err := protocol.ValidateReadRequest(request); err != nil { + return nil, "", err } - err := stream(ctx, streamOptions) - return results, err + operation := &pb.Command_Read{Read: request} + command := &pb.Command{Operation: operation} + return command, "", nil +} + +// mutateCommand validates one mutation and wraps it for sending. Mutations keep +// the input order of their resource. +func mutateCommand(input *MutateRequest, completion *pb.MutationOptions) (*pb.Command, string, error) { + request, err := wireMutation(input, completion) + if err != nil { + return nil, "", err + } + if err := protocol.ValidateMutationRequest(request); err != nil { + return nil, "", err + } + operation := &pb.Command_Mutate{Mutate: request} + command := &pb.Command{Operation: operation} + return command, request.GetResource(), nil +} + +func readResult(wire *pb.ReadResult) *ReadResult { + result := &ReadResult{Document: wire.GetDocument(), Missing: wire.GetMissing() != nil, Failure: wire.GetFailure()} + return result } -func wireMutation(request *MutateRequest) (*pb.MutateRequest, error) { +func wireMutation(request *MutateRequest, completion *pb.MutationOptions) (*pb.MutateRequest, error) { if request == nil { return nil, errors.New("missing mutation") } - mutation := &pb.MutateRequest{Resource: request.Resource, Options: request.Options} + mutation := &pb.MutateRequest{Resource: request.Resource, Options: completion} switch request.Action { case MutationCreate, MutationPut, MutationReplace: if request.Document == nil || request.Lua != nil || request.BackendExpression != nil { diff --git a/records_test.go b/records_test.go index 492a13f..102dbe4 100644 --- a/records_test.go +++ b/records_test.go @@ -10,22 +10,114 @@ import ( "github.com/batchstream/weir-protocol/api/protocol" pb "github.com/batchstream/weir-protocol/api/weir/v1" - "google.golang.org/protobuf/proto" + "google.golang.org/grpc" ) +func TestReadPreservesMissingAndBackendFailure(t *testing.T) { + for _, mode := range []string{"read_missing", "read_failure", "read_target_missing"} { + t.Run(mode, func(t *testing.T) { + peer := &clientTestPeer{mode: mode} + client := clientTestClient(t, peer) + result, err := readOne(t.Context(), client, clientTestReadRequest()) + if err != nil || result == nil || result.Document != nil { + t.Fatal("Read lost backend result", result, err) + } + if mode == "read_missing" && (!result.Missing || result.Failure != nil) { + t.Fatal("Read changed missing into failure", result) + } + if mode == "read_failure" && (result.Missing || result.Failure.GetCode() != FailurePermissionDenied) { + t.Fatal("Read lost failure evidence", result) + } + if mode == "read_target_missing" && (result.Missing || result.Failure.GetCode() != FailureTargetNotFound) { + t.Fatal("missing backend target became successful document absence", result) + } + }) + } +} + +func TestMutationActionsSelectBackendOperation(t *testing.T) { + for _, visible := range []bool{false, true} { + t.Run(fmt.Sprintf("visible=%t", visible), func(t *testing.T) { + peer := &clientTestPeer{mode: "typed_mutation", mutations: make(chan *pb.MutateRequest, 5)} + client := clientTestClient(t, peer) + ctx, cancel := context.WithTimeout(t.Context(), 3*time.Second) + defer cancel() + document := &Document{ContentType: "application/json", Data: []byte(`{"n":1}`)} + program := &LuaTransform{Source: []byte("return function(current, incoming) return current end")} + want := pb.MutationVisibility_APPLICATION_ACKNOWLEDGED + var options []MutateOption + if visible { + want = pb.MutationVisibility_QUERY_VISIBLE + options = append(options, WithQueryVisible()) + } + for _, action := range []MutationAction{MutationCreate, MutationPut, MutationReplace, MutationDelete, MutationAtomicTransform} { + request := &MutateRequest{Resource: "records/s:key", Action: action} + switch action { + case MutationDelete: + case MutationAtomicTransform: + request.Lua = program + default: + request.Document = document + } + result, err := mutateOne(ctx, client, request, options...) + if err != nil || result.GetOutcome() != MutationApplied { + t.Fatal(action, result, err) + } + mutation := <-peer.mutations + if mutation.Resource != request.Resource || mutation.GetOptions().GetVisibility() != want { + t.Fatal("a mutation lost its resource or visibility", action, mutation) + } + selected := action == MutationCreate && mutation.GetCreate() != nil || + action == MutationPut && mutation.GetPut() != nil || + action == MutationReplace && mutation.GetReplace() != nil || + action == MutationDelete && mutation.GetDelete() != nil || + action == MutationAtomicTransform && mutation.GetAtomicTransform().GetLua() != nil + if !selected { + t.Fatal("SDK executed a different mutation", action, mutation) + } + } + if peer.received.Load() != 5 { + t.Fatal("mutations were replayed", peer.received.Load()) + } + }) + } +} + +func TestAtomicTransformRejectsAmbiguousFormsBeforeBusinessSend(t *testing.T) { + for _, both := range []bool{false, true} { + t.Run(map[bool]string{false: "neither", true: "both"}[both], func(t *testing.T) { + peer := &clientTestPeer{mode: "typed_mutation"} + client := clientTestClient(t, peer) + request := &MutateRequest{Resource: "records/s:key", Action: MutationAtomicTransform} + if both { + request.Lua = &LuaTransform{Source: []byte("return function(current, incoming) return current end")} + request.BackendExpression = &Document{ContentType: "application/json", Data: []byte(`{}`)} + } + ctx, cancel := context.WithTimeout(t.Context(), time.Second) + defer cancel() + if result, err := mutateOne(ctx, client, request); err == nil || result != nil { + t.Fatal("invalid transform was not rejected", result, err) + } + if peer.received.Load() != 0 { + t.Fatal("invalid transform sent business data", peer.received.Load()) + } + }) + } +} + func TestReadKeepsInputOrderAndIndividualFailures(t *testing.T) { peer := &clientTestPeer{mode: "batch_read_order"} - client := clientTestConnection(t, peer) - options := ReadOptions{StoreName: "records"} + client := clientTestClient(t, peer) + var requests []*ReadRequest for range 25 { // Repeated resources retain distinct result positions. request := &ReadRequest{Resource: "records/s:repeated"} - options.Requests = append(options.Requests, request) + requests = append(requests, request) } ctx, cancel := context.WithTimeout(t.Context(), 3*time.Second) defer cancel() - results, err := Read(ctx, client, options) - if err != nil || len(results) != len(options.Requests) { + results, err := client.Read(ctx, "records", WithRequests(requests...)) + if err != nil || len(results) != len(requests) { t.Fatal("batch failed", len(results), err) } for index, result := range results { @@ -52,36 +144,31 @@ func TestReadKeepsInputOrderAndIndividualFailures(t *testing.T) { func TestMutateBatchSelectsOperationsAndPreservesOrder(t *testing.T) { peer := &clientTestPeer{mode: "batch_mutate_order", mutations: make(chan *pb.MutateRequest, 5)} - client := clientTestConnection(t, peer) + client := clientTestClient(t, peer) document := &Document{ContentType: "application/json", Data: []byte(`{"n":1}`)} program := &LuaTransform{Source: []byte("return function(current, incoming) return current end")} - options := MutateOptions{StoreName: "records"} + var requests []*MutateRequest for _, action := range []MutationAction{MutationCreate, MutationPut, MutationReplace, MutationDelete, MutationAtomicTransform} { request := &MutateRequest{Resource: "records/s:key", Action: action} - if action == MutationReplace { - request.Options = &MutationOptions{} - } else if action != MutationPut { - request.Options = &MutationOptions{Visibility: MutationQueryVisible} - } if action == MutationAtomicTransform { request.Lua = program } else if action != MutationDelete { request.Document = document } - options.Requests = append(options.Requests, request) + requests = append(requests, request) } - results, err := Mutate(t.Context(), client, options) + results, err := client.Mutate(t.Context(), "records", WithRequests(requests...)) if err != nil || len(results) != 5 { t.Fatal("mutation batch failed", results, err) } - for index, request := range options.Requests { + for index, request := range requests { mutation := <-peer.mutations selected := request.Action == MutationCreate && mutation.GetCreate() != nil || request.Action == MutationPut && mutation.GetPut() != nil || request.Action == MutationReplace && mutation.GetReplace() != nil || request.Action == MutationDelete && mutation.GetDelete() != nil || request.Action == MutationAtomicTransform && mutation.GetAtomicTransform().GetLua() != nil - if !selected || !proto.Equal(mutation.Options, request.Options) { + if !selected { t.Fatal("wrong business operation", index, mutation) } if index == 1 { @@ -97,33 +184,48 @@ func TestMutateBatchSelectsOperationsAndPreservesOrder(t *testing.T) { } } +// allNil reports whether every result of a rejected call stayed unconfirmed. +func allNil[R comparable](results []R, length int) bool { + var none R + if len(results) != length { + return false + } + for _, result := range results { + if result != none { + return false + } + } + return true +} + func TestBatchPreflightRejectsEntireInputBeforeRPC(t *testing.T) { peer := &clientTestPeer{mode: "batch_read_order"} - client := clientTestConnection(t, peer) + client := clientTestClient(t, peer) for _, resource := range []string{"", "weir://other/records/s:key", "/records/s:key", "records/%6B", "records/.."} { request := &ReadRequest{Resource: resource} - options := ReadOptions{StoreName: "records", Requests: []*ReadRequest{clientTestReadRequest(), request}} - if results, err := Read(t.Context(), client, options); err == nil || results != nil { + results, err := client.Read(t.Context(), "records", WithRequests(clientTestReadRequest(), request)) + if err == nil || !allNil(results, 2) { t.Fatal("invalid batch was not rejected", resource, results, err) } } - for _, requests := range [][]*ReadRequest{nil, {}, {clientTestReadRequest(), nil}} { - options := ReadOptions{StoreName: "records", Requests: requests} - if results, err := Read(t.Context(), client, options); err == nil || results != nil { - t.Fatal("empty or nil read accepted", results, err) - } + if results, err := client.Read(t.Context(), "records"); err == nil || len(results) != 0 { + t.Fatal("a call without requests was accepted", results, err) + } + if results, err := client.Read(t.Context(), "records", WithRequests[*ReadRequest]()); err == nil || len(results) != 0 { + t.Fatal("an empty read was accepted", results, err) + } + if results, err := client.Read(t.Context(), "records", WithRequests(clientTestReadRequest(), nil)); err == nil || !allNil(results, 2) { + t.Fatal("a nil read was accepted", results, err) } - options := ReadOptions{StoreName: "invalid/store", Requests: []*ReadRequest{clientTestReadRequest()}} - if results, err := Read(t.Context(), client, options); err == nil || results != nil { + if results, err := client.Read(t.Context(), "invalid/store", WithRequests(clientTestReadRequest())); err == nil || !allNil(results, 1) { t.Fatal("invalid store accepted", results, err) } - options.StoreName = "records" - options.Requests = nil + var requests []*ReadRequest for range 12 { - options.Requests = append(options.Requests, clientTestReadRequest()) + requests = append(requests, clientTestReadRequest()) } - options.Requests = append(options.Requests, nil) - if results, err := Read(t.Context(), client, options); err == nil || results != nil { + requests = append(requests, nil) + if results, err := client.Read(t.Context(), "records", WithRequests(requests...)); err == nil || !allNil(results, 13) { t.Fatal("late invalid input sent earlier valid requests", results, err) } document := &Document{ContentType: "application/json", Data: []byte(`{}`)} @@ -136,8 +238,8 @@ func TestBatchPreflightRejectsEntireInputBeforeRPC(t *testing.T) { {Resource: "records/s:key", Action: MutationAtomicTransform}, {Resource: "records/s:key", Action: MutationAction(99)}, } { - mutation := MutateOptions{StoreName: "records", Requests: []*MutateRequest{valid, invalid}} - if results, err := Mutate(t.Context(), client, mutation); err == nil || results != nil { + results, err := client.Mutate(t.Context(), "records", WithRequests(valid, invalid)) + if err == nil || !allNil(results, 2) { t.Fatal("invalid mutation sent earlier valid item", results, err) } } @@ -148,27 +250,24 @@ func TestBatchPreflightRejectsEntireInputBeforeRPC(t *testing.T) { func TestBatchAcceptsLargeRecord(t *testing.T) { peer := &clientTestPeer{mode: "typed_mutation"} - client := clientTestConnection(t, peer) - mutations := MutateOptions{StoreName: "records"} + client := clientTestClient(t, peer) large := &Document{ContentType: "application/octet-stream", Data: make([]byte, 16<<20)} request := &MutateRequest{Resource: "records/s:key", Action: MutationPut, Document: large} - mutations.Requests = []*MutateRequest{request} - if results, err := Mutate(t.Context(), client, mutations); err != nil || len(results) != 1 || results[0].Outcome != MutationApplied || results[0].Failure != nil { - t.Fatal("large record rejected", len(results), err) + if result, err := mutateOne(t.Context(), client, request); err != nil || result.Outcome != MutationApplied || result.Failure != nil { + t.Fatal("large record rejected", result, err) } if peer.streams.Load() != 1 || peer.received.Load() != 1 { t.Fatal("large record did not use one RPC", peer.streams.Load(), peer.received.Load()) } } -func TestMutateStreamFailurePreservesConfirmedPrefix(t *testing.T) { +func TestMutateFailurePreservesConfirmedPrefix(t *testing.T) { peer := &clientTestPeer{mode: "batch_partial"} - client := clientTestConnection(t, peer) + client := clientTestClient(t, peer) document := &Document{ContentType: "application/json", Data: []byte(`{}`)} first := &MutateRequest{Resource: "records/s:first", Action: MutationPut, Document: document} - second := &MutateRequest{Resource: "records/s:second", Action: MutationDelete} - options := MutateOptions{StoreName: "records", Requests: []*MutateRequest{first, second}} - results, err := Mutate(t.Context(), client, options) + second := deleteRequest("records/s:second") + results, err := client.Mutate(t.Context(), "records", WithRequests(first, second)) if err == nil || len(results) != 2 || results[0].GetOutcome() != MutationApplied || results[1] != nil { t.Fatal("stream failure lost confirmed prefix or acknowledged missing result", results, err) } @@ -177,13 +276,12 @@ func TestMutateStreamFailurePreservesConfirmedPrefix(t *testing.T) { } } -func TestReadStreamFinalFailurePreservesConfirmedResults(t *testing.T) { +func TestReadSucceedsOnceEveryResultArrived(t *testing.T) { peer := &clientTestPeer{mode: "batch_read_final_error"} - client := clientTestConnection(t, peer) - options := ReadOptions{StoreName: "records", Requests: []*ReadRequest{clientTestReadRequest(), clientTestReadRequest(), clientTestReadRequest()}} - results, err := Read(t.Context(), client, options) - if err == nil || len(results) != 3 || results[0] == nil || results[1] == nil || results[2] == nil { - t.Fatal("final transport error or confirmed results lost", results, err) + client := clientTestClient(t, peer) + results, err := client.Read(t.Context(), "records", WithRequests(clientTestReadRequest(), clientTestReadRequest(), clientTestReadRequest())) + if err != nil || len(results) != 3 || results[0] == nil || results[1] == nil || results[2] == nil { + t.Fatal("a call with every result failed or lost one", results, err) } if peer.streams.Load() != 1 || peer.received.Load() != 3 { t.Fatal("read batch replayed", peer.streams.Load(), peer.received.Load()) @@ -192,20 +290,17 @@ func TestReadStreamFinalFailurePreservesConfirmedResults(t *testing.T) { func TestClientBatchPreflightBeforeRPC(t *testing.T) { client := &Client{} - options := ReadOptions{StoreName: "records"} - _, err := client.Read(t.Context(), options) - if err == nil || !strings.Contains(err.Error(), "nonempty requests") { + _, err := client.Read(t.Context(), "records") + if err == nil || !strings.Contains(err.Error(), "requires a Store and requests") { t.Fatal("RPC started before input validation", err) } - mutation := MutateOptions{StoreName: "records"} - _, err = client.Mutate(t.Context(), mutation) - if err == nil || !strings.Contains(err.Error(), "nonempty requests") { + _, err = client.Mutate(t.Context(), "records") + if err == nil || !strings.Contains(err.Error(), "requires a Store and requests") { t.Fatal("RPC started before mutation validation", err) } ctx, cancel := context.WithCancel(t.Context()) cancel() - options.Requests = []*ReadRequest{clientTestReadRequest()} - if _, err := client.Read(ctx, options); !errors.Is(err, context.Canceled) { + if _, err := client.Read(ctx, "records", WithRequests(clientTestReadRequest())); !errors.Is(err, context.Canceled) { t.Fatal("cancelled batch did not stop before its RPC", err) } } @@ -217,8 +312,8 @@ func TestClientSendsBatchesToItsAddress(t *testing.T) { t.Fatal(err) } defer client.Close() - read := ReadOptions{StoreName: "records", Requests: []*ReadRequest{clientTestReadRequest(), clientTestReadRequest()}} - readResults, err := client.Read(t.Context(), read) + reads := []*ReadRequest{clientTestReadRequest(), clientTestReadRequest()} + readResults, err := client.Read(t.Context(), "records", WithRequests(reads...)) // The peer answers by wire index; records of a Client's stream need not be // sent in input order unless they share a resource. if err != nil || len(readResults) != 2 || readResults[0] == nil || readResults[1] == nil || readResults[0].Missing == readResults[1].Missing { @@ -226,9 +321,8 @@ func TestClientSendsBatchesToItsAddress(t *testing.T) { } document := &Document{ContentType: "application/json", Data: []byte(`{}`)} first := &MutateRequest{Resource: "records/s:first", Action: MutationPut, Document: document} - second := &MutateRequest{Resource: "records/s:second", Action: MutationDelete} - mutation := MutateOptions{StoreName: "records", Requests: []*MutateRequest{first, second}} - mutationResults, err := client.Mutate(t.Context(), mutation) + second := deleteRequest("records/s:second") + mutationResults, err := client.Mutate(t.Context(), "records", WithRequests(first, second)) if err != nil || len(mutationResults) != 2 || mutationResults[0] == nil || mutationResults[1] == nil || (mutationResults[0].GetFailure() == nil) == (mutationResults[1].GetFailure() == nil) { t.Fatal("mutation batch failed", mutationResults, err) } @@ -238,23 +332,24 @@ func TestClientSendsBatchesToItsAddress(t *testing.T) { if err := client.Close(); err != nil { t.Fatal(err) } - if _, err := client.Read(t.Context(), read); !errors.Is(err, ErrClosed) { - t.Fatal("closed client accepted a call", err) + closed, err := client.Read(t.Context(), "records", WithRequests(reads...)) + if !errors.Is(err, ErrClosed) || !allNil(closed, 2) { + t.Fatal("closed client accepted a call", closed, err) } } -func TestMutateBatchLargeDocumentsUseOneRPC(t *testing.T) { +func TestLargeDocumentsEachUseAStreamOfTheirOwn(t *testing.T) { peer := &clientTestPeer{mode: "typed_mutation"} - client := clientTestConnection(t, peer) + client := clientTestClient(t, peer) document := &Document{ContentType: "application/octet-stream", Data: make([]byte, (2 << 20))} - options := MutateOptions{StoreName: "records"} + var requests []*MutateRequest for range 20 { request := &MutateRequest{Resource: "records/s:key", Action: MutationPut, Document: document} - options.Requests = append(options.Requests, request) + requests = append(requests, request) } ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) defer cancel() - results, err := Mutate(ctx, client, options) + results, err := client.Mutate(ctx, "records", WithRequests(requests...)) if err != nil || len(results) != 20 { t.Fatal("large typed batch failed", len(results), err) } @@ -263,20 +358,51 @@ func TestMutateBatchLargeDocumentsUseOneRPC(t *testing.T) { t.Fatal("large mutation lost acknowledgement", result) } } - if peer.streams.Load() != 1 || peer.received.Load() != 20 || peer.maxBytes.Load() > int64(protocol.MaxTransportMessageBytes) { - t.Fatal("large batch opened multiple RPCs", peer.streams.Load(), peer.received.Load()) + if peer.streams.Load() != 20 || peer.received.Load() != 20 || peer.maxBytes.Load() > int64(protocol.MaxTransportMessageBytes) { + t.Fatal("large records did not each use a stream", peer.streams.Load(), peer.received.Load()) } } func TestReadAcceptsLargeRequestSequence(t *testing.T) { peer := &clientTestPeer{mode: "batch_read_order"} - client := clientTestConnection(t, peer) - options := ReadOptions{StoreName: "records"} + client := clientTestClient(t, peer) + var requests []*ReadRequest for range 5000 { - options.Requests = append(options.Requests, clientTestReadRequest()) + requests = append(requests, clientTestReadRequest()) } - results, err := Read(t.Context(), client, options) + results, err := client.Read(t.Context(), "records", WithRequests(requests...)) if err != nil || len(results) != 5000 || peer.streams.Load() != 1 || peer.received.Load() != 5000 { t.Fatal("large batch was split or limited by count", len(results), err, peer.streams.Load(), peer.received.Load()) } } + +// streamRefuser turns every stream away before reading it. +type streamRefuser struct{} + +func (streamRefuser) Execute(grpc.BidiStreamingServer[pb.ExecuteRequests, pb.ExecuteResponses]) error { + return refusedStreamError("draining") +} + +func TestARefusedStreamOfItsOwnIsTheRecordsRefusal(t *testing.T) { + client, err := New(serveForTest(t, streamRefuser{})) + if err != nil { + t.Fatal(err) + } + defer client.Close() + readCommandToSend, _, err := readCommand(clientTestReadRequest()) + if err != nil { + t.Fatal(err) + } + read, err := readKind.alone(t.Context(), client, "records", readCommandToSend) + if err != nil || read.Failure.GetCode() != FailureUnavailable { + t.Fatal("a refused read lost its refusal", read, err) + } + mutateCommandToSend, _, err := mutateCommand(deleteRequest("records/s:key"), nil) + if err != nil { + t.Fatal(err) + } + mutation, err := mutateKind.alone(t.Context(), client, "records", mutateCommandToSend) + if err != nil || mutation.GetOutcome() != MutationNotStarted || mutation.GetFailure().GetCode() != FailureUnavailable { + t.Fatal("a refused mutation lost its refusal", mutation, err) + } +} diff --git a/retry_test.go b/retry_test.go index d5fdb43..9895d95 100644 --- a/retry_test.go +++ b/retry_test.go @@ -138,10 +138,7 @@ func putRequest(resource, data string) *MutateRequest { } func refusingPut(ctx context.Context, client *Client, resource string) (*MutationResult, error) { - document := &Document{ContentType: "application/json", Data: []byte(`{}`)} - request := &WriteRequest{Resource: resource, Document: document} - options := WriteOptions{StoreName: "records", Request: request} - return client.Put(ctx, options) + return mutateOne(ctx, client, putRequest(resource, `{}`)) } func TestClientResendsRecordsAServerRefusedBeforeTheyBegan(t *testing.T) { @@ -154,8 +151,7 @@ func TestClientResendsRecordsAServerRefusedBeforeTheyBegan(t *testing.T) { t.Fatal("refused write was not resent until applied", result, err, peer.sent("records/s:write")) } request := &ReadRequest{Resource: "records/s:read"} - options := ReadOneOptions{StoreName: "records", Request: request} - read, err := client.ReadOne(t.Context(), options) + read, err := readOne(t.Context(), client, request) if err != nil || !read.Missing || peer.sent("records/s:read") != 2 { t.Fatal("refused read was not resent", read, err, peer.sent("records/s:read")) } @@ -214,8 +210,7 @@ func TestMutateResendsRefusedRecordsWithoutReorderingAResource(t *testing.T) { peer := newRefusingPeer() client := refusingClient(t, peer) mutate := func(requests ...*MutateRequest) []*MutationResult { - options := MutateOptions{StoreName: "records", Requests: requests} - results, err := client.Mutate(t.Context(), options) + results, err := client.Mutate(t.Context(), "records", WithRequests(requests...)) if err != nil { t.Fatal(err) } @@ -253,8 +248,7 @@ func TestReadResendsRefusedReads(t *testing.T) { client := refusingClient(t, peer) first := &ReadRequest{Resource: "records/s:a"} second := &ReadRequest{Resource: "records/s:b"} - options := ReadOptions{StoreName: "records", Requests: []*ReadRequest{first, second}} - results, err := client.Read(t.Context(), options) + results, err := client.Read(t.Context(), "records", WithRequests(first, second)) if err != nil || !results[0].Missing || !results[1].Missing { t.Fatal("refused read was not resent", results, err) } diff --git a/semantic_contract_test.go b/semantic_contract_test.go index 004d4d6..881fc27 100644 --- a/semantic_contract_test.go +++ b/semantic_contract_test.go @@ -7,18 +7,16 @@ import ( func TestRequestExtensionsRejectBeforeRPC(t *testing.T) { peer := &clientTestPeer{} - client := clientTestConnection(t, peer) + client := clientTestClient(t, peer) read := clientTestReadRequest() addResponseFields(read) - readOptions := ReadOneOptions{StoreName: "records", Request: read} - if result, err := ReadOne(t.Context(), client, readOptions); result != nil || err == nil { + if result, err := readOne(t.Context(), client, read); result != nil || err == nil { t.Fatal("unknown request option was silently ignored", result, err) } document := &Document{ContentType: "application/json", Data: []byte(`{}`)} addResponseFields(document) - write := &WriteRequest{Resource: "records/s:key", Document: document} - writeOptions := WriteOptions{StoreName: "records", Request: write} - if result, err := Put(t.Context(), client, writeOptions); result != nil || err == nil { + write := &MutateRequest{Resource: "records/s:key", Action: MutationPut, Document: document} + if result, err := mutateOne(t.Context(), client, write); result != nil || err == nil { t.Fatal("unknown document option was silently ignored", result, err) } if peer.streams.Load() != 0 { @@ -29,19 +27,18 @@ func TestRequestExtensionsRejectBeforeRPC(t *testing.T) { func TestNativeAcceptsNewStoreFormatsWithoutSchemaSelection(t *testing.T) { for _, mode := range []string{"native_normal", "native_no_metadata"} { peer := &clientTestPeer{mode: mode} - client := clientTestConnection(t, peer) + client := clientTestClient(t, peer) document := &Document{ContentType: "application/vnd.future-store.command"} request := &NativeRequest{Resource: "anything", Request: document} - options := NativeOptions{StoreName: "future-store", Request: request} consumed := false - options.Consume = func(_ context.Context, response *NativeResponse, _ []byte) error { + chunks := func(_ context.Context, response *NativeResponse, _ []byte) error { if mode == "native_normal" && response.Metadata.ContentType != "application/vnd.example.response" { - t.Fatal("unknown metadata format rejected", response) + t.Error("unknown metadata format rejected", response) } consumed = true return nil } - result, err := Native(t.Context(), client, options) + result, err := client.Native(t.Context(), "future-store", request, WithChunks(chunks)) if err != nil || result == nil || result.Completion != NativeResponseComplete || !consumed || peer.received.Load() != 1 { t.Fatal("new Store format needs no SDK or protocol backend selection", result, err) } diff --git a/send_completion_test.go b/send_completion_test.go index 733fff1..3e1b7be 100644 --- a/send_completion_test.go +++ b/send_completion_test.go @@ -81,14 +81,12 @@ func (stream *completionStream) Recv() (*pb.ExecuteResponses, error) { func TestNativeWaitsForSendAfterTerminalEOF(t *testing.T) { stream := &completionStream{delivered: make(chan struct{}), eofSeen: make(chan struct{}), release: make(chan struct{}), closed: make(chan struct{})} - client := &completionClient{stream: stream} + client := &Client{store: &completionClient{stream: stream}} ctx, cancel := context.WithTimeout(t.Context(), 3*time.Second) defer cancel() done := make(chan error, 1) go func() { - options := NativeOptions{StoreName: "records", Request: clientTestNativeRequest()} - options.Consume = func(context.Context, *NativeResponse, []byte) error { return nil } - _, err := Native(ctx, client, options) + _, err := client.Native(ctx, "records", clientTestNativeRequest()) done <- err }() select { @@ -124,17 +122,18 @@ func TestLocalSendFailureCancelsReceiveAndJoins(t *testing.T) { t.Run(mode, func(t *testing.T) { failure := errors.New("local send failed") stream := &completionStream{delivered: make(chan struct{}), sendErr: failure} - client := &completionClient{stream: stream} + client := &Client{store: &completionClient{stream: stream}} ctx, cancel := context.WithTimeout(t.Context(), time.Second) defer cancel() var err error if mode == "native" { - options := NativeOptions{StoreName: "records", Request: clientTestNativeRequest()} - options.Consume = func(context.Context, *NativeResponse, []byte) error { return nil } - _, err = Native(ctx, client, options) + _, err = client.Native(ctx, "records", clientTestNativeRequest()) } else { - options := ReadOptions{StoreName: "records", Requests: []*ReadRequest{clientTestReadRequest()}} - _, err = Read(ctx, client, options) + command, _, commandErr := readCommand(clientTestReadRequest()) + if commandErr != nil { + t.Fatal(commandErr) + } + _, err = readKind.alone(ctx, client, "records", command) } if !errors.Is(err, failure) || ctx.Err() != nil { t.Fatal("local Send failure was lost or receive needed caller deadline", err, ctx.Err()) @@ -145,14 +144,12 @@ func TestLocalSendFailureCancelsReceiveAndJoins(t *testing.T) { func TestNativeRejectsInvalidRequestContentTypeBeforeRPC(t *testing.T) { peer := &clientTestPeer{} - client := clientTestConnection(t, peer) + client := clientTestClient(t, peer) document := &Document{ContentType: string([]byte{0xff})} request := &NativeRequest{Resource: "records", Request: document} - options := NativeOptions{StoreName: "records", Request: request} - options.Consume = func(context.Context, *NativeResponse, []byte) error { return nil } ctx, cancel := context.WithTimeout(t.Context(), time.Second) defer cancel() - end, err := Native(ctx, client, options) + end, err := client.Native(ctx, "records", request) if err == nil || end != nil || ctx.Err() != nil || peer.received.Load() != 0 { t.Fatal("local protobuf error waited for a response or sent malformed input", end, err, ctx.Err(), peer.received.Load()) } diff --git a/visibility_test.go b/visibility_test.go index 1ce7474..63d013a 100644 --- a/visibility_test.go +++ b/visibility_test.go @@ -7,41 +7,38 @@ import ( "testing" pb "github.com/batchstream/weir-protocol/api/weir/v1" - "google.golang.org/protobuf/proto" ) -func TestMutateStreamPreservesPerRecordVisibility(t *testing.T) { +func TestQueryVisibleAppliesToEveryRecordOfACall(t *testing.T) { peer := &clientTestPeer{mode: "typed_mutation", mutations: make(chan *pb.MutateRequest, 2)} - client := clientTestConnection(t, peer) + client := clientTestClient(t, peer) expression := &Document{ContentType: "application/json", Data: []byte(`{"doc":{"n":1}}`)} - completion := &MutationOptions{Visibility: MutationQueryVisible} - first := &MutateRequest{Resource: "records/s:first", Action: MutationAtomicTransform, BackendExpression: expression, Options: completion} - second := &MutateRequest{Resource: "records/s:second", Action: MutationDelete} - requests := []*MutateRequest{first, second} - produced, consumed := 0, uint64(0) - options := MutateStreamOptions{StoreName: "records"} - options.Next = func(context.Context) (*MutateRequest, error) { - if produced == len(requests) { + first := &MutateRequest{Resource: "records/s:first", Action: MutationAtomicTransform, BackendExpression: expression} + second := deleteRequest("records/s:second") + produced := false + next := func(context.Context) (*MutateRequest, error) { + if produced { return nil, io.EOF } - request := requests[produced] - produced++ - return request, nil + produced = true + return second, nil } - options.Consume = func(_ context.Context, index uint64, result *MutationResult) error { + consumed := 0 + consume := func(_ context.Context, _ int, result *MutationResult) error { consumed++ - if index != consumed || result.GetOutcome() != MutationApplied || result.Failure != nil { - return errors.New("mutation evidence was lost or reordered") + if result.GetOutcome() != MutationApplied || result.Failure != nil { + return errors.New("mutation evidence was lost") } return nil } - if err := MutateStream(t.Context(), client, options); err != nil { + _, err := client.Mutate(t.Context(), "records", WithRequests(first), WithNext(next), WithMutateConsume(consume), WithQueryVisible()) + if err != nil { t.Fatal(err) } - for _, request := range requests { + for range 2 { mutation := <-peer.mutations - if mutation.Resource != request.Resource || !proto.Equal(mutation.Options, request.Options) { - t.Fatal("stream changed complete per-record options", mutation, request) + if mutation.GetOptions().GetVisibility() != pb.MutationVisibility_QUERY_VISIBLE { + t.Fatal("a record lost the call's visibility", mutation) } } if peer.streams.Load() != 1 || peer.received.Load() != 2 || consumed != 2 { @@ -49,58 +46,45 @@ func TestMutateStreamPreservesPerRecordVisibility(t *testing.T) { } } -func TestMutationOptionsRejectInvalidBatchBeforeRPC(t *testing.T) { - unknown := &MutationOptions{Visibility: MutationQueryVisible} - unknown.ProtoReflect().SetUnknown([]byte{0x10, 1}) - negative := &MutationOptions{Visibility: MutationVisibility(-1)} - undeclared := &MutationOptions{Visibility: MutationVisibility(2)} - for _, completion := range []*MutationOptions{negative, undeclared, unknown} { - peer := &clientTestPeer{mode: "typed_mutation"} - client := clientTestConnection(t, peer) - first := &MutateRequest{Resource: "records/s:first", Action: MutationDelete} - second := &MutateRequest{Resource: "records/s:second", Action: MutationDelete, Options: completion} - options := MutateOptions{StoreName: "records", Requests: []*MutateRequest{first, second}} - results, err := Mutate(t.Context(), client, options) - if err == nil || results != nil || peer.streams.Load() != 0 || peer.received.Load() != 0 { - t.Fatal("invalid completion option sent part of a batch", completion, results, err, peer.streams.Load(), peer.received.Load()) - } +func TestMutationsWaitForApplicationByDefault(t *testing.T) { + peer := &clientTestPeer{mode: "typed_mutation", mutations: make(chan *pb.MutateRequest, 1)} + client := clientTestClient(t, peer) + if _, err := mutateOne(t.Context(), client, deleteRequest("records/s:key")); err != nil { + t.Fatal(err) + } + if mutation := <-peer.mutations; mutation.Options != nil { + t.Fatal("a mutation sent options nobody asked for", mutation) } } -func TestMutateStreamInvalidOptionsKeepConfirmedPrefix(t *testing.T) { +func TestProducedInvalidMutationKeepsConfirmedPrefix(t *testing.T) { peer := &clientTestPeer{mode: "typed_mutation"} - client := clientTestConnection(t, peer) - completion := &MutationOptions{Visibility: MutationVisibility(2)} + client := clientTestClient(t, peer) confirmed := make(chan struct{}) - produced, consumed := 0, uint64(0) - options := MutateStreamOptions{StoreName: "records"} - options.Next = func(ctx context.Context) (*MutateRequest, error) { + produced, consumed := 0, 0 + next := func(ctx context.Context) (*MutateRequest, error) { produced++ - request := &MutateRequest{Resource: "records/s:key", Action: MutationDelete} if produced == 1 { - return request, nil - } - if produced > 2 { - return nil, io.EOF + return deleteRequest("records/s:key"), nil } select { case <-confirmed: case <-ctx.Done(): return nil, ctx.Err() } - request.Options = completion - return request, nil + invalid := &MutateRequest{Resource: "records/s:key", Action: MutationAction(99)} + return invalid, nil } - options.Consume = func(_ context.Context, index uint64, result *MutationResult) error { + consume := func(_ context.Context, index int, result *MutationResult) error { consumed++ - if index != 1 || result.GetOutcome() != MutationApplied || result.Failure != nil { + if index != 0 || result.GetOutcome() != MutationApplied || result.Failure != nil { return errors.New("confirmed mutation changed") } close(confirmed) return nil } - err := MutateStream(t.Context(), client, options) + _, err := client.Mutate(t.Context(), "records", WithNext(next), WithMutateConsume(consume)) if err == nil || consumed != 1 || peer.streams.Load() != 1 || peer.received.Load() != 1 { - t.Fatal("invalid later option lost evidence, reached the backend or caused replay", err, consumed, peer.streams.Load(), peer.received.Load()) + t.Fatal("invalid later request lost evidence, reached the backend or caused replay", err, consumed, peer.streams.Load(), peer.received.Load()) } }