diff --git a/README.md b/README.md index c8dc8553..235ffa55 100644 --- a/README.md +++ b/README.md @@ -44,14 +44,19 @@ make integration The opt-in `integration` build tag starts MongoDB, Elasticsearch, two Store owners and a discovery node. Tagged tests require a prepared `WEIR_TEST_BINARY`. Coverage includes discovery through every node, direct owner connections, typed -CRUD and Lua transformations, independent persistence checks, same-Store batches, -slice preflight and per-frame validation, finite Scan continuation, Native response evidence, +CRUD and Lua transformations, independent persistence checks, same-Store calls +containing many records, slice preflight and per-record validation, finite Scan continuation, Native response evidence, cancellation, owner restart, directory convergence and persistent client recovery. Separate multi-process tests verify independent single-record callers. One-RPC reads deliver 64 MiB and 256 MiB across 1024 distinct documents while checking bounded result credits and sampled owner RSS. Interrupted streams retain confirmed record outcomes; unconfirmed mutations remain indeterminate. +Each Execute request carries one record and its consecutive index. The client +streams records as they are produced; server aggregation stays within the Store. +Later invalid records stop the stream while preserving earlier acknowledged writes. +Document and Native body formats use the public `ContentType` fields. + Services bind to loopback and own exclusive namespaces. Cleanup verifies exact process and container ownership. Fixture logs, manifests and cleanup receipts are retained in the printed output directory, including startup failures. diff --git a/cmd/weir-lab/main.go b/cmd/weir-lab/main.go index 73d47b2f..e56aa1f0 100644 --- a/cmd/weir-lab/main.go +++ b/cmd/weir-lab/main.go @@ -14,6 +14,7 @@ import ( "os/signal" "path/filepath" "runtime" + "slices" "strconv" "strings" "syscall" @@ -184,10 +185,7 @@ func runLab(ctx context.Context, opts labOptions) (resultErr error) { "topology": "native client and Weir on one host; dedicated database Docker containers with loopback ports", "client_operation_timeout": businessTimeout.String(), "weir_backend_timeout": businessTimeout.String(), - "weir_store_concurrency": fmt.Sprint(opts.concurrency), "weir_max_batch_operations": fmt.Sprint(opts.backendBatchLimit), - "weir_memory_budget": "8GiB (declared process admission budget; not an OS reservation)", - "weir_ingress_max_sessions": "64", "weir_ingress_max_connections": "64", "host_cpus": fmt.Sprint(runtime.NumCPU()), } @@ -202,7 +200,7 @@ func runLab(ctx context.Context, opts labOptions) (resultErr error) { return err } provenance["weir_source"] = identity.Revision - if opts.mongoBinary != "" { + if opts.mongoBinary != "" && slices.Contains(backends, "mongo") { provenance["topology"] = "native client, Weir and isolated MongoDB on one host; Elasticsearch in a dedicated loopback Docker container" } if pinsRaw, err := os.ReadFile("versions.json"); err == nil { diff --git a/go.mod b/go.mod index ac31806a..c6403bef 100644 --- a/go.mod +++ b/go.mod @@ -3,8 +3,8 @@ module github.com/batchstream/weir-tests go 1.27.1 require ( - github.com/batchstream/weir-go v0.6.0 - github.com/batchstream/weir-protocol v0.4.0 + github.com/batchstream/weir-go v0.7.0 + github.com/batchstream/weir-protocol v0.5.0 go.mongodb.org/mongo-driver/v2 v2.9.1 google.golang.org/grpc v1.83.2 ) diff --git a/go.sum b/go.sum index 07cb2f0d..e85e1dbd 100644 --- a/go.sum +++ b/go.sum @@ -1,7 +1,7 @@ -github.com/batchstream/weir-go v0.6.0 h1:S9/+7diNNNbVEH68lg6GbepzXAiCJpF0K74R5pTNFmo= -github.com/batchstream/weir-go v0.6.0/go.mod h1:YsokqwtYw2sN7zVbt4yIgqJUnuoHsDyiA1vQlXPE2VU= -github.com/batchstream/weir-protocol v0.4.0 h1:t4dDLAKeOOGz/wajfsfrYZgbILDvVceoPvJYKxTQuiM= -github.com/batchstream/weir-protocol v0.4.0/go.mod h1:hHmwPoE9yHYAz7Auc3a14Ijo/uxAef4sQ9nBtyXNcGc= +github.com/batchstream/weir-go v0.7.0 h1:PfuZaRH6A8sMOZ5qa0pyLna8A9qsBOydAX3yH2v2t4s= +github.com/batchstream/weir-go v0.7.0/go.mod h1:3hOZQtFCmS7ZbWuM32jOQxdL1sKQD+hdk1d6U5bcglo= +github.com/batchstream/weir-protocol v0.5.0 h1:ARPy3pcEL0EB5pJsGL10R2/MyH+gb0TqO31t18rOrPE= +github.com/batchstream/weir-protocol v0.5.0/go.mod h1:hHmwPoE9yHYAz7Auc3a14Ijo/uxAef4sQ9nBtyXNcGc= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= diff --git a/internal/workload/client.go b/internal/workload/client.go index 0185e273..e46789cb 100644 --- a/internal/workload/client.go +++ b/internal/workload/client.go @@ -3,6 +3,7 @@ package workload import ( "context" "errors" + "io" weir "github.com/batchstream/weir-go" ) @@ -11,8 +12,7 @@ import ( // Only the coordinator owns setup, reset and independent verification. type Client struct { Executor Executor - direct *directPath - weir *weir.Client + io.Closer } type ClientOptions struct { @@ -25,13 +25,12 @@ func OpenClient(ctx context.Context, opts ClientOptions) (*Client, error) { if opts.Dataset == nil || (opts.Path != "direct" && opts.Path != "weir") { return nil, errors.New("client requires a dataset and direct or weir path") } - client := &Client{} if opts.Path == "direct" { direct, err := openDirect(ctx, opts.Dataset) if err != nil { return nil, err } - client.direct, client.Executor = direct, direct + client := &Client{Executor: direct, Closer: direct} return client, nil } open := weir.OpenOptions{Seed: opts.Dataset.Config.WeirSeed, Stores: []string{opts.Dataset.Config.StoreName}} @@ -39,18 +38,7 @@ func OpenClient(ctx context.Context, opts ClientOptions) (*Client, error) { if err != nil { return nil, err } - client.weir = sdk path := &weirPath{dataset: opts.Dataset, client: sdk, info: opts.Evidence} - client.Executor = path + client := &Client{Executor: path, Closer: sdk} return client, nil } - -func (c *Client) Close() error { - if c.direct != nil { - return c.direct.close() - } - if c.weir != nil { - return c.weir.Close() - } - return nil -} diff --git a/internal/workload/dataset.go b/internal/workload/dataset.go index be29b67c..491de996 100644 --- a/internal/workload/dataset.go +++ b/internal/workload/dataset.go @@ -122,7 +122,7 @@ func (d *Dataset) Document(operation Operation) []byte { return d.docs[operation.Record][operation.Revision] } -func (d *Dataset) MediaType() string { +func (d *Dataset) ContentType() string { if d.Config.Backend == "mongo" { return "application/bson" } diff --git a/internal/workload/paths.go b/internal/workload/paths.go index 75f0afbd..d76d8467 100644 --- a/internal/workload/paths.go +++ b/internal/workload/paths.go @@ -163,7 +163,7 @@ func openDirect(ctx context.Context, dataset *Dataset) (*directPath, error) { direct.baseURL = strings.TrimSuffix(address.String(), "/") status, body, err := direct.request(ctx, http.MethodGet, "/", nil) if err != nil || status != http.StatusOK { - _ = direct.close() + _ = direct.Close() return nil, errors.New("direct Search version query failed") } var root struct { @@ -172,7 +172,7 @@ func openDirect(ctx context.Context, dataset *Dataset) (*directPath, error) { } `json:"version"` } if err := json.Unmarshal(body, &root); err != nil || root.Version.Number == "" { - _ = direct.close() + _ = direct.Close() return nil, errors.New("invalid Search version evidence") } direct.info.Protocol = "Elasticsearch HTTP/1.1 JSON" @@ -322,10 +322,10 @@ func (p *Paths) Close() error { if p.client != nil { sdkErr = p.client.Close() } - return errors.Join(sdkErr, p.direct.close()) + return errors.Join(sdkErr, p.direct.Close()) } -func (p *directPath) close() error { +func (p *directPath) Close() error { if p.search != nil { p.search.CloseIdleConnections() } @@ -488,7 +488,7 @@ func (p *weirPath) Execute(ctx context.Context, operation Operation) Outcome { result, err := p.client.AtomicTransform(ctx, opts) return mutationOutcome(result, len(program.Source), err) } - document := &weir.Document{MediaType: p.dataset.MediaType(), Data: p.dataset.Document(operation)} + document := &weir.Document{ContentType: p.dataset.ContentType(), Data: p.dataset.Document(operation)} request := &weir.WriteRequest{Resource: p.dataset.Resource(operation.Record), Document: document} opts := weir.WriteOptions{StoreName: p.dataset.Config.StoreName, Request: request} result, err := p.client.Put(ctx, opts) @@ -507,8 +507,8 @@ func (p *weirPath) readOutcome(result *weir.ReadResult, operation Operation, rpc if result == nil || result.Document == nil || result.Failure != nil || result.Missing { return failed(errors.New("Weir Read lacks a successful document"), false) } - if result.Document.MediaType != p.dataset.MediaType() { - return failed(errors.New("Weir document media type mismatch"), false) + if result.Document.ContentType != p.dataset.ContentType() { + return failed(errors.New("Weir document content type mismatch"), false) } if err := p.dataset.Validate(result.Document.Data, operation); err != nil { return failed(err, false) diff --git a/internal/workload/weir_batch.go b/internal/workload/weir_batch.go index f200d39b..928ff830 100644 --- a/internal/workload/weir_batch.go +++ b/internal/workload/weir_batch.go @@ -27,7 +27,7 @@ func (p *weirPath) ExecuteBatch(ctx context.Context, operations []Operation) []O if operations[0].Write { requests := make([]*weir.MutateRequest, len(operations)) for index, operation := range operations { - document := &weir.Document{MediaType: p.dataset.MediaType(), Data: p.dataset.Document(operation)} + document := &weir.Document{ContentType: p.dataset.ContentType(), Data: p.dataset.Document(operation)} request := &weir.MutateRequest{Resource: p.dataset.Resource(operation.Record), Action: weir.MutationPut, Document: document} requests[index] = request } diff --git a/internal/workload/weir_batch_test.go b/internal/workload/weir_batch_test.go index db7cbfb8..1939b62a 100644 --- a/internal/workload/weir_batch_test.go +++ b/internal/workload/weir_batch_test.go @@ -20,10 +20,11 @@ import ( type acknowledgementPeer struct { pb.UnimplementedStoreServiceServer - mode string - calls atomic.Int64 - applied atomic.Int64 - document []byte + mode string + calls atomic.Int64 + applied atomic.Int64 + document []byte + firstMutation chan struct{} } func (p *acknowledgementPeer) Execute(stream grpc.BidiStreamingServer[pb.ExecuteRequest, pb.ExecuteResponse]) error { @@ -37,7 +38,7 @@ func (p *acknowledgementPeer) Execute(stream grpc.BidiStreamingServer[pb.Execute return err } if request.Command.GetRead() != nil { - document := &pb.Document{MediaType: "application/json", Data: p.document} + document := &pb.Document{ContentType: "application/json", Data: p.document} payload := &pb.ReadResult_Document{Document: document} result := &pb.ReadResult{Result: payload} value := &pb.Event_ReadResult{ReadResult: result} @@ -48,31 +49,33 @@ func (p *acknowledgementPeer) Execute(stream grpc.BidiStreamingServer[pb.Execute } return status.Error(codes.Unavailable, "later read result lost") } - mutations := request.Command.GetMutate().Requests + if request.Command.GetMutate() == nil { + return status.Error(codes.InvalidArgument, "mutation required") + } // Durable writes can precede lost acknowledgements; never infer replay safety. - p.applied.Add(int64(len(mutations))) + if p.applied.Add(1) == 1 && p.firstMutation != nil { + close(p.firstMutation) + } switch p.mode { case "lost_after_apply": - return status.Error(codes.Unavailable, "window acknowledgement lost") + return status.Error(codes.Unavailable, "record acknowledgement lost") case "deadline_after_apply", "canceled_after_apply": <-stream.Context().Done() return status.FromContextError(stream.Context().Err()).Err() } - for index := range mutations { - var failure *pb.Failure - if p.mode == "applied_failure" { - failure = protocol.Fail(pb.FailureCode_UNAVAILABLE, "write applied but replica confirmation failed") - } - result := protocol.Mutation(pb.MutationOutcome_APPLIED, failure) - payload := &pb.Event_MutationResult{MutationResult: result} - event := &pb.Event{Value: payload} - response := &pb.ExecuteResponse{Index: request.Index + uint64(index), Event: event} - if err := stream.Send(response); err != nil { - return err - } - if p.mode == "partial_after_apply" { - return status.Error(codes.Unavailable, "later acknowledgement lost") - } + var failure *pb.Failure + if p.mode == "applied_failure" { + failure = protocol.Fail(pb.FailureCode_UNAVAILABLE, "write applied but replica confirmation failed") + } + result := protocol.Mutation(pb.MutationOutcome_APPLIED, failure) + payload := &pb.Event_MutationResult{MutationResult: result} + event := &pb.Event{Value: payload} + response := &pb.ExecuteResponse{Index: request.Index, Event: event} + if err := stream.Send(response); err != nil { + return err + } + if p.mode == "partial_after_apply" { + return status.Error(codes.Unavailable, "later acknowledgement lost") } } } @@ -123,8 +126,11 @@ func TestSDKStreamMutationAcknowledgementAndNoReplay(t *testing.T) { for _, mode := range []string{"applied_failure", "partial_after_apply", "lost_after_apply", "deadline_after_apply", "canceled_after_apply"} { t.Run(mode, func(t *testing.T) { peer := &acknowledgementPeer{mode: mode} + if mode == "canceled_after_apply" { + peer.firstMutation = make(chan struct{}) + } client := mutationTransport(t, peer) - document := &weir.Document{MediaType: "application/json", Data: []byte(`{"n":1}`)} + document := &weir.Document{ContentType: "application/json", Data: []byte(`{"n":1}`)} first := &weir.MutateRequest{Resource: "records/s:first", Action: weir.MutationPut, Document: document} second := &weir.MutateRequest{Resource: "records/s:second", Action: weir.MutationPut, Document: document} opts := weir.MutateOptions{StoreName: "search", Requests: []*weir.MutateRequest{first, second}} @@ -132,15 +138,20 @@ func TestSDKStreamMutationAcknowledgementAndNoReplay(t *testing.T) { defer cancel() if mode == "canceled_after_apply" { go func() { - for peer.applied.Load() == 0 && ctx.Err() == nil { - time.Sleep(time.Millisecond) + select { + case <-peer.firstMutation: + cancel() + case <-ctx.Done(): } - cancel() }() } replies, rpcErr := weir.Mutate(ctx, client, opts) - if len(replies) != 2 || peer.calls.Load() != 1 || peer.applied.Load() != 2 { - t.Fatalf("whole batch lost accounting or replayed: replies=%v calls=%d applied=%d err=%v", replies, peer.calls.Load(), peer.applied.Load(), rpcErr) + applied := int64(1) + if mode == "applied_failure" { + applied = 2 + } + if len(replies) != 2 || peer.calls.Load() != 1 || peer.applied.Load() != applied { + t.Fatalf("stream lost accounting or replayed: replies=%v calls=%d applied=%d err=%v", replies, peer.calls.Load(), peer.applied.Load(), rpcErr) } if mode == "applied_failure" && rpcErr != nil || mode != "applied_failure" && rpcErr == nil { t.Fatal("unexpected final status", rpcErr) @@ -163,10 +174,10 @@ func TestSDKStreamMutationAcknowledgementAndNoReplay(t *testing.T) { } } -func TestSDKStreamMutationPreflightSendsNoBatch(t *testing.T) { +func TestSDKStreamMutationPreflightSendsNoRecord(t *testing.T) { peer := &acknowledgementPeer{mode: "applied_failure"} client := mutationTransport(t, peer) - document := &weir.Document{MediaType: "application/json", Data: []byte(`{"n":1}`)} + document := &weir.Document{ContentType: "application/json", Data: []byte(`{"n":1}`)} first := &weir.MutateRequest{Resource: "records/s:first", Action: weir.MutationPut, Document: document} invalid := &weir.MutateRequest{Resource: "records/s:second", Action: 0, Document: document} opts := weir.MutateOptions{StoreName: "search", Requests: []*weir.MutateRequest{first, invalid}} diff --git a/tests/integration/backends_test.go b/tests/integration/backends_test.go index 64eab755..ff5a4c78 100644 --- a/tests/integration/backends_test.go +++ b/tests/integration/backends_test.go @@ -103,7 +103,7 @@ func (b *backendData) resource(id string) string { func (b *backendData) write(t *testing.T, id string, n int64) *weir.WriteRequest { t.Helper() - document := &weir.Document{MediaType: "application/json", Data: []byte(fmt.Sprintf("{\"n\":%d}", n))} + document := &weir.Document{ContentType: "application/json", Data: []byte(fmt.Sprintf("{\"n\":%d}", n))} if b.name == "mongo" { value := bson.D{{Key: "_id", Value: id}, {Key: "n", Value: n}} document = bsonDocument(t, value) @@ -115,14 +115,14 @@ func (b *backendData) write(t *testing.T, id string, n int64) *weir.WriteRequest func (b *backendData) transform(t *testing.T, id string) *weir.AtomicTransformRequest { t.Helper() expression := &weir.Document{ - MediaType: "application/vnd.weir.search-update.v1+json", - Data: []byte("{\"doc\":{\"n\":4}}"), + ContentType: "application/vnd.weir.search-update.v1+json", + Data: []byte("{\"doc\":{\"n\":4}}"), } if b.name == "mongo" { fields := bson.D{{Key: "n", Value: int64(1)}} update := bson.D{{Key: "$inc", Value: fields}} expression = bsonDocument(t, update) - expression.MediaType = "application/vnd.weir.mongodb-update.v1+bson" + expression.ContentType = "application/vnd.weir.mongodb-update.v1+bson" } request := &weir.AtomicTransformRequest{Resource: b.resource(id), BackendExpression: expression} return request @@ -134,7 +134,7 @@ func bsonDocument(t *testing.T, value bson.D) *weir.Document { if err != nil { t.Fatal(err) } - document := &weir.Document{MediaType: "application/bson", Data: data} + document := &weir.Document{ContentType: "application/bson", Data: data} return document } @@ -143,8 +143,8 @@ func (b *backendData) number(document *weir.Document) (int64, error) { return 0, errors.New("missing document") } if b.name == "mongo" { - if document.GetMediaType() != "application/bson" { - return 0, fmt.Errorf("MongoDB document media type %q", document.GetMediaType()) + if document.GetContentType() != "application/bson" { + return 0, fmt.Errorf("MongoDB document content type %q", document.GetContentType()) } raw := bson.Raw(document.GetData()) if err := raw.Validate(); err != nil { @@ -159,8 +159,8 @@ func (b *backendData) number(document *weir.Document) (int64, error) { var value struct { N int64 `json:"n"` } - if document.GetMediaType() != "application/json" { - return 0, fmt.Errorf("Search document media type %q", document.GetMediaType()) + if document.GetContentType() != "application/json" { + return 0, fmt.Errorf("Search document content type %q", document.GetContentType()) } if err := json.Unmarshal(document.GetData(), &value); err != nil { return 0, err diff --git a/tests/integration/batch_test.go b/tests/integration/batch_test.go index 873708ac..148558df 100644 --- a/tests/integration/batch_test.go +++ b/tests/integration/batch_test.go @@ -72,8 +72,7 @@ func testPublicBatch(t *testing.T, ctx context.Context, client *weir.Client, bac } targets[input%len(targets)].assertReadResult(t, result, nil, int64(100+input)) } - // Invalid input at the end must be rejected before any earlier mutation - // reaches either target. The SDK preflight and server preflight are tested separately. + // The slice API validates every item before it sends the first record. rejected := make([]*weir.MutateRequest, count) for index := range count { target := targets[index%len(targets)] @@ -87,13 +86,13 @@ func testPublicBatch(t *testing.T, ctx context.Context, client *weir.Client, bac invalidResults, err := client.Mutate(rpc, invalidOptions) cancel() if err == nil || invalidResults != nil { - t.Fatalf("invalid whole batch accepted: length=%d error=%v", len(invalidResults), err) + t.Fatalf("invalid request slice accepted: length=%d error=%v", len(invalidResults), err) } for index := range count { targets[index%len(targets)].assertMissing(t, ctx, fmt.Sprintf("late_invalid_%03d", index)) } testDuplicateURIOrder(t, ctx, client, backend) - t.Log("48 independent streamed SDK mutations across two same-Store targets, ordered results including precondition/missing, duplicate URI execution order, and SDK/server late-invalid preflight verified") + t.Log("48 streamed SDK mutations across two same-Store targets, ordered results including precondition/missing, duplicate URI order, and slice preflight verified") } func provisionBatchTarget(t *testing.T, ctx context.Context, base *backendData) *backendData { @@ -159,8 +158,7 @@ func testDuplicateURIOrder(t *testing.T, ctx context.Context, client *weir.Clien first := &weir.ReadRequest{Resource: backend.resource(id)} missing := &weir.ReadRequest{Resource: backend.resource("ordered_missing")} primary := &weir.ReadRequest{Resource: backend.resource("primary")} - // Exceeds the removed 128-item cap, with duplicates and missing records - // interleaved. These are small documents inside the encoded byte bound. + // Repeated URIs and missing records cross multiple server aggregation groups. reads := make([]*weir.ReadRequest, 513) for index := range reads { switch index % 3 { @@ -195,39 +193,54 @@ func testDuplicateURIOrder(t *testing.T, ctx context.Context, client *weir.Clien func (s *system) testServerPreflight(t *testing.T, backend *backendData) { t.Helper() - const count = 17 - requests := make([]*pb.MutateRequest, count) - for index := range requests { - write := backend.write(t, fmt.Sprintf("wire_invalid_%03d", index), int64(index)) - document := &pb.Document{MediaType: write.Document.MediaType, Data: write.Document.Data} - action := &pb.MutateRequest_Put{Put: document} - request := &pb.MutateRequest{Resource: write.Resource, Action: action} - requests[index] = request - } - requests[count-1].Action = nil - batch := &pb.MutationBatch{Requests: requests} - operation := &pb.Command_Mutate{Mutate: batch} - command := &pb.Command{Operation: operation} - request := &pb.ExecuteRequest{StoreName: backend.name, Index: 1, Command: command} - wire := transport(t, s.cluster.Nodes[0].Application) - rpc, cancel := context.WithTimeout(s.ctx, rpcTimeout) - stream, err := wire.Execute(rpc) - if err != nil { - t.Fatal(err) - } - if err := stream.Send(request); err != nil { - t.Fatal(err) - } - if err := stream.CloseSend(); err != nil { - t.Fatal(err) - } - response, err := stream.Recv() - cancel() - if status.Code(err) != codes.InvalidArgument || response != nil { - t.Fatalf("server late-invalid preflight: response=%v err=%v", response, err) - } - for index := range requests { - backend.assertMissing(t, s.ctx, fmt.Sprintf("wire_invalid_%03d", index)) + for _, phase := range []string{"first_record", "after_confirmed_record"} { + t.Run(backend.name+"/invalid_"+phase, func(t *testing.T) { + wire := transport(t, s.cluster.Nodes[0].Application) + rpc, cancel := context.WithTimeout(s.ctx, rpcTimeout) + defer cancel() + stream, err := wire.Execute(rpc) + if err != nil { + t.Fatal(err) + } + index := uint64(1) + if phase == "after_confirmed_record" { + write := backend.write(t, "wire_confirmed_prefix", 7) + document := &pb.Document{ContentType: write.Document.ContentType, Data: write.Document.Data} + action := &pb.MutateRequest_Put{Put: document} + mutation := &pb.MutateRequest{Resource: write.Resource, Action: action} + operation := &pb.Command_Mutate{Mutate: mutation} + command := &pb.Command{Operation: operation} + request := &pb.ExecuteRequest{StoreName: backend.name, Index: index, Command: command} + if err := stream.Send(request); err != nil { + t.Fatal(err) + } + response, err := stream.Recv() + result := response.GetEvent().GetMutationResult() + if err != nil || response.GetIndex() != index || result.GetOutcome() != pb.MutationOutcome_APPLIED || result.GetFailure() != nil { + t.Fatal("valid prefix lacks successful acknowledgement", response, err) + } + index++ + } + id := "wire_invalid_" + phase + mutation := &pb.MutateRequest{Resource: backend.resource(id)} + operation := &pb.Command_Mutate{Mutate: mutation} + command := &pb.Command{Operation: operation} + request := &pb.ExecuteRequest{StoreName: backend.name, Index: index, Command: command} + if err := stream.Send(request); err != nil { + t.Fatal(err) + } + if err := stream.CloseSend(); err != nil { + t.Fatal(err) + } + response, err := stream.Recv() + if status.Code(err) != codes.InvalidArgument || response != nil { + t.Fatalf("invalid record was accepted: response=%v err=%v", response, err) + } + backend.assertMissing(t, s.ctx, id) + if phase == "after_confirmed_record" { + backend.assertPersisted(t, s.ctx, "wire_confirmed_prefix", 7) + } + }) } } @@ -272,5 +285,5 @@ func (s *system) testLargeDistinctBatch(t *testing.T, backend *backendData) { } evidence.Before, evidence.After, evidence.Method = before, s.nodeMetrics(t), "read" assertStreamBatchMetrics(t, evidence) - t.Log("513 distinct native documents Put and Read in bounded adapter windows on one streamed call; every value independently persisted and reverse-order results verified") + t.Log("513 distinct native documents Put and Read in aggregation groups on one streamed call; every value independently persisted and reverse-order results verified") } diff --git a/tests/integration/lifecycle_test.go b/tests/integration/lifecycle_test.go index d8d50844..1fdb32f2 100644 --- a/tests/integration/lifecycle_test.go +++ b/tests/integration/lifecycle_test.go @@ -214,8 +214,8 @@ func (b *backendData) scanID(document *weir.Document) (string, error) { return "", errors.New("Scan omitted document") } if b.name == "mongo" { - if document.GetMediaType() != "application/bson" { - return "", errors.New("MongoDB Scan changed native media type") + if document.GetContentType() != "application/bson" { + return "", errors.New("MongoDB Scan changed native content type") } raw := bson.Raw(document.GetData()) if err := raw.Validate(); err != nil { @@ -230,7 +230,7 @@ func (b *backendData) scanID(document *weir.Document) (string, error) { var hit struct { ID string `json:"_id"` } - if document.GetMediaType() != "application/json" || json.Unmarshal(document.GetData(), &hit) != nil || hit.ID == "" { + if document.GetContentType() != "application/json" || json.Unmarshal(document.GetData(), &hit) != nil || hit.ID == "" { return "", errors.New("Search Scan omitted native hit _id") } return hit.ID, nil @@ -244,8 +244,9 @@ func testNative(t *testing.T, ctx context.Context, client *weir.Client, backend filter := bson.D{{Key: "_id", Value: "primary"}} command := bson.D{{Key: "count", Value: parts[len(parts)-1]}, {Key: "query", Value: filter}} request.Body = bsonDocument(t, command).Data - request.BodyMediaType = "application/bson" - request.Descriptor = &weir.Document{MediaType: "application/vnd.weir.mongodb-command.v1+protobuf"} + request.BodyContentType = "application/bson" + descriptor := &weir.Document{ContentType: weir.MongoCommandContentType} + request.Descriptor = descriptor } else { descriptor := &weir.SearchHTTPRequest{Method: http.MethodGet, Path: "/_doc/primary"} encoded, err := weir.SearchHTTPDescriptor(descriptor) diff --git a/tests/integration/system_test.go b/tests/integration/system_test.go index b9bc0148..c483b3e2 100644 --- a/tests/integration/system_test.go +++ b/tests/integration/system_test.go @@ -162,8 +162,7 @@ func (s *system) testDiscovery(t *testing.T) { s.testServerPreflight(t, backend) s.testBatchOwnerRouting(t, backend) read := &pb.ReadRequest{Resource: backend.resource("primary")} - batch := &pb.ReadBatch{Requests: []*pb.ReadRequest{read}} - operation := &pb.Command_Read{Read: batch} + operation := &pb.Command_Read{Read: read} command := &pb.Command{Operation: operation} request := &pb.ExecuteRequest{StoreName: backend.name, Index: 1, Command: command} rpc, cancel := context.WithTimeout(ctx, rpcTimeout) @@ -370,16 +369,16 @@ func assertStreamBatchMetrics(t *testing.T, evidence streamBatchEvidence) { delta := right - left if delta != 0 { if index == len(evidence.Before)-1 || delta != float64(evidence.Records) { - t.Fatalf("whole batch relayed or split across owners: node=%d delta=%v", index, delta) + t.Fatalf("record stream relayed or split across owners: node=%d delta=%v", index, delta) } owners++ count, err := observe.Delta(before, after, "weir_store_batch_operations_count", storeLabels) if err != nil || count < 1 { - t.Fatalf("window adapter invocations=%v error=%v", count, err) + t.Fatalf("aggregation invocations=%v error=%v", count, err) } operations, err := observe.Delta(before, after, "weir_store_batch_operations_sum", storeLabels) if err != nil || operations != float64(evidence.Records) { - t.Fatalf("window adapter operations=%v error=%v", operations, err) + t.Fatalf("aggregated operations=%v error=%v", operations, err) } } left, beforeFound = before.Sum("weir_rpc_completions_total", methodLabels) diff --git a/versions.json b/versions.json index 362f8e0e..149b0223 100644 --- a/versions.json +++ b/versions.json @@ -1,13 +1,13 @@ { "go": "1.27.1", - "sdk": "v0.6.0", - "protocol": "v0.4.0", - "weir_source": "b8fce387897e6a7be018a9aed1db35f484f150b7", + "sdk": "v0.7.0", + "protocol": "v0.5.0", + "weir_source": "a1f07b20dc0b7b1e41def2a07fc4191749c497ab", "mongodb_version": "8.0.32", "elasticsearch_version": "8.19.22", "elasticsearch_image": "docker.elastic.co/elasticsearch/elasticsearch@sha256:e98f9c3b09beb2fbb9eaf667d602df3f0e00bd3644138b8458dc17ba1a675595", - "weir_version": "v0.1.1-0.20261004094535-b8fce387897e", - "weir_sum": "h1:zRhILg8EoflGS1bmPe/P/clHVRIoYT4r0a8b1uq/xt0=", + "weir_version": "v0.1.1-0.20261004114516-a1f07b20dc0b", + "weir_sum": "h1:Q/Ysilf6XgVfnuh2YDxKv2XBXbdpxhIY9arUOwaB0iA=", "mongodb_image": "mongo@sha256:d0d926f94df099bff534b7ee5b5986458131a22489dfff8664509af0c1e2ca9c", "mongodb_darwin_arm64_url": "https://fastdl.mongodb.org/osx/mongodb-macos-arm64-8.0.32.tgz", "mongodb_darwin_arm64_sha256": "f81cb258434d548dca7244d599c82eb339043d8dedd0b1b807870c9d263117f2",