diff --git a/README.md b/README.md index 235ffa55..0c942f51 100644 --- a/README.md +++ b/README.md @@ -45,17 +45,17 @@ The opt-in `integration` build tag starts MongoDB, Elasticsearch, two Store owners and a discovery node. Tagged tests require a prepared `WEIR_TEST_BINARY`. Coverage includes discovery through every node, direct owner connections, typed CRUD and Lua transformations, independent persistence checks, same-Store calls -containing many records, slice preflight and per-record validation, finite Scan continuation, Native response evidence, +containing many records, slice preflight and per-record validation, typed Scan filters and include/exclude projections, cross-owner continuation, Native response evidence, cancellation, owner restart, directory convergence and persistent client recovery. -Separate multi-process tests verify independent single-record callers. -One-RPC reads deliver 64 MiB and 256 MiB across 1024 distinct documents while +Lua tests cover missing and existing documents, explicit and implicit decisions, first creation, and concurrent read-modify-write through both owners. Missing records and missing backend targets have distinct outcomes; empty scans succeed and complete HTTP 404 responses retain Native completion evidence. Separate multi-process tests verify independent single-record callers. +One-RPC reads deliver 64 MiB and 256 MiB across 1024 and 4096 distinct documents while checking bounded result credits and sampled owner RSS. Interrupted streams retain confirmed record outcomes; unconfirmed mutations remain indeterminate. Each Execute request carries one record and its consecutive index. The client streams records as they are produced; server aggregation stays within the Store. Later invalid records stop the stream while preserving earlier acknowledged writes. -Document and Native body formats use the public `ContentType` fields. +Documents use explicit `ContentType` fields. Native selects either a BSON MongoDB command or a typed Search HTTP request; SDK consumers receive chunks with validated response metadata. Services bind to loopback and own exclusive namespaces. Cleanup verifies exact process and container ownership. Fixture logs, manifests and cleanup receipts @@ -111,6 +111,9 @@ adapter time, rejections and operations per adapter invocation. MongoDB also records physical command deltas. Adapter invocations can split by namespace, action and byte limits; their counts are distinct from database wire commands. Missing samples and reset counters are reported as unavailable. +`business_operation_timeouts` counts failed logical operations whose request context +expired before the call returned; an expired bulk context counts every failed operation +in that batch. This diagnostic does not override confirmed mutation evidence. A database-saturated comparison requires both paths to qualify independently: at least five CPU intervals cover 80% of measurement time, mean CPU reaches 90% diff --git a/go.mod b/go.mod index c6403bef..3ce375c0 100644 --- a/go.mod +++ b/go.mod @@ -3,8 +3,8 @@ module github.com/batchstream/weir-tests go 1.27.1 require ( - github.com/batchstream/weir-go v0.7.0 - github.com/batchstream/weir-protocol v0.5.0 + github.com/batchstream/weir-go v0.8.0 + github.com/batchstream/weir-protocol v0.6.0 go.mongodb.org/mongo-driver/v2 v2.9.1 google.golang.org/grpc v1.83.2 ) diff --git a/go.sum b/go.sum index e85e1dbd..f044fcb5 100644 --- a/go.sum +++ b/go.sum @@ -1,7 +1,7 @@ -github.com/batchstream/weir-go v0.7.0 h1:PfuZaRH6A8sMOZ5qa0pyLna8A9qsBOydAX3yH2v2t4s= -github.com/batchstream/weir-go v0.7.0/go.mod h1:3hOZQtFCmS7ZbWuM32jOQxdL1sKQD+hdk1d6U5bcglo= -github.com/batchstream/weir-protocol v0.5.0 h1:ARPy3pcEL0EB5pJsGL10R2/MyH+gb0TqO31t18rOrPE= -github.com/batchstream/weir-protocol v0.5.0/go.mod h1:hHmwPoE9yHYAz7Auc3a14Ijo/uxAef4sQ9nBtyXNcGc= +github.com/batchstream/weir-go v0.8.0 h1:NB1K66a61ceLnE7GVUw3N5VfK19D5tunqXzuK59Xkxw= +github.com/batchstream/weir-go v0.8.0/go.mod h1:nNqdGY1ajxYNkRLJWUprF+vQebxd2fOuq0ePAgo2rXs= +github.com/batchstream/weir-protocol v0.6.0 h1:Mpminz34U6IFmqJ0eIl5YA3ClyTok+sxnuLqvIBZHKU= +github.com/batchstream/weir-protocol v0.6.0/go.mod h1:hHmwPoE9yHYAz7Auc3a14Ijo/uxAef4sQ9nBtyXNcGc= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= diff --git a/internal/benchmark/client_worker.go b/internal/benchmark/client_worker.go index ec2422ad..62a2d99a 100644 --- a/internal/benchmark/client_worker.go +++ b/internal/benchmark/client_worker.go @@ -229,7 +229,7 @@ func (s *durationState) nextSingle(index int) workload.Operation { if write { worker.revisions[worker.cursor] = 1 - worker.revisions[worker.cursor] } - operation := workload.Operation{Worker: index, Sequence: worker.sequence, Record: record, Write: write, Revision: worker.revisions[worker.cursor]} + operation := workload.Operation{Record: record, Write: write, Revision: worker.revisions[worker.cursor]} worker.cursor = (worker.cursor + 1) % (worker.end - worker.start) worker.sequence++ return operation diff --git a/internal/benchmark/runner.go b/internal/benchmark/runner.go index 5e9c9fe2..669551e5 100644 --- a/internal/benchmark/runner.go +++ b/internal/benchmark/runner.go @@ -205,6 +205,9 @@ func measure(ctx context.Context, executor workload.Executor, plan *workload.Pla began := time.Now() outcome := executor.Execute(operationCtx, operation) worker.hist.add(time.Since(began)) + if outcome.Status != workload.Success && errors.Is(operationCtx.Err(), context.DeadlineExceeded) { + worker.result.Timeouts++ + } cancel() worker.result.Attempted++ worker.result.RequestBytes += outcome.RequestBytes @@ -236,6 +239,7 @@ func measure(ctx context.Context, executor workload.Executor, plan *workload.Pla result := Result{Path: executor.Name(), Planned: uint64(plan.Count), ElapsedNS: elapsed.Nanoseconds(), Evidence: executor.Evidence()} var hist histogram for _, worker := range workers { + result.Timeouts += worker.result.Timeouts result.Attempted += worker.result.Attempted result.Succeeded += worker.result.Succeeded result.Errors += worker.result.Errors diff --git a/internal/benchmark/runner_test.go b/internal/benchmark/runner_test.go index 36e24488..4737971b 100644 --- a/internal/benchmark/runner_test.go +++ b/internal/benchmark/runner_test.go @@ -7,6 +7,7 @@ import ( "os" "path/filepath" "reflect" + "sync/atomic" "testing" "time" @@ -14,8 +15,9 @@ import ( ) type fakeExecutor struct { - mode string - delay time.Duration + mode string + delay time.Duration + sequence atomic.Uint64 } func (f *fakeExecutor) Name() string { return "direct" } @@ -30,8 +32,10 @@ func (f *fakeExecutor) Execute(ctx context.Context, operation workload.Operation select { case <-timer.C: case <-ctx.Done(): - outcome := workload.Outcome{Status: workload.Indeterminate, Error: ctx.Err().Error()} - return outcome + if f.mode != "success_after_deadline" { + outcome := workload.Outcome{Status: workload.Indeterminate, Error: ctx.Err().Error()} + return outcome + } } } outcome := workload.Outcome{Status: workload.Success, RequestBytes: 3, ResponseBytes: 7} @@ -41,7 +45,7 @@ func (f *fakeExecutor) Execute(ctx context.Context, operation workload.Operation outcome.Error = "post-write acknowledgement failure" } if f.mode == "mixed" { - switch operation.Sequence % 3 { + switch (f.sequence.Add(1) - 1) % 3 { case 1: outcome.Status, outcome.Error = workload.Failed, "known failure" case 2: @@ -213,3 +217,19 @@ func TestWriteReportPreservesRawEvidenceAndNoInvalidRatio(t *testing.T) { t.Fatal("accepted report path collision") } } + +func TestMeasureCountsOperationDeadlines(t *testing.T) { + executor := &fakeExecutor{delay: time.Second} + result := measure(t.Context(), executor, testPlan(t, 8), time.Millisecond) + if result.Timeouts != 8 || result.Attempted != 8 || result.Indeterminate != 8 { + t.Fatalf("deadline accounting: %+v", result) + } +} + +func TestExpiredContextDoesNotEraseConfirmedSuccess(t *testing.T) { + executor := &fakeExecutor{mode: "success_after_deadline", delay: time.Second} + result := measure(t.Context(), executor, testPlan(t, 8), time.Millisecond) + if result.Timeouts != 0 || result.Succeeded != 8 || !result.complete() { + t.Fatalf("confirmed success was counted as timeout: %+v", result) + } +} diff --git a/internal/benchmark/saturation.go b/internal/benchmark/saturation.go index 30b7ba4f..74ef4c3f 100644 --- a/internal/benchmark/saturation.go +++ b/internal/benchmark/saturation.go @@ -236,7 +236,7 @@ func (s *durationState) next(workerIndex int) []workload.Operation { if write { worker.revisions[worker.cursor] = 1 - worker.revisions[worker.cursor] } - operation := workload.Operation{Worker: workerIndex, Sequence: worker.sequence, Record: record, Write: write, Revision: worker.revisions[worker.cursor]} + operation := workload.Operation{Record: record, Write: write, Revision: worker.revisions[worker.cursor]} operations[index] = operation worker.cursor = (worker.cursor + 1) % (worker.end - worker.start) worker.sequence++ @@ -288,6 +288,7 @@ func measureDuration(ctx context.Context, options durationMeasureOptions) Satura began := time.Now() outcomes := options.Executor.ExecuteBatch(operationCtx, operations) elapsed := time.Since(began) + timedOut := errors.Is(operationCtx.Err(), context.DeadlineExceeded) cancel() requests[index]++ if len(outcomes) != len(operations) { @@ -298,6 +299,9 @@ func measureDuration(ctx context.Context, options durationMeasureOptions) Satura } } for operationIndex, outcome := range outcomes { + if timedOut && outcome.Status != workload.Success { + worker.result.Timeouts++ + } worker.hist.add(elapsed) worker.result.Attempted++ if operations[operationIndex].Write { @@ -391,6 +395,7 @@ func measureDuration(ctx context.Context, options durationMeasureOptions) Satura result.Resources.MeasurementNS = elapsed.Nanoseconds() var hist histogram for index, worker := range workers { + result.Timeouts += worker.result.Timeouts result.Attempted += worker.result.Attempted result.Succeeded += worker.result.Succeeded result.Errors += worker.result.Errors diff --git a/internal/benchmark/saturation_test.go b/internal/benchmark/saturation_test.go index 0d00c496..07054f04 100644 --- a/internal/benchmark/saturation_test.go +++ b/internal/benchmark/saturation_test.go @@ -234,3 +234,17 @@ func TestSaturationReportRecordsClientAndBackendRequestBudgets(t *testing.T) { t.Fatal("human-readable report hid the request budgets", markdown) } } + +func TestBulkDurationCountsLogicalOperationsInExpiredBatches(t *testing.T) { + config := workload.Config{Backend: "search", StoreName: "search", Namespace: "weirtest_0123456789abcdef01234567", Records: 8, Concurrency: 2} + dataset, err := workload.New(config) + if err != nil { + t.Fatal(err) + } + executor := &fakeExecutor{delay: time.Second} + opts := durationMeasureOptions{Executor: executor, State: newDurationState(dataset, 2, 4, 0), Duration: time.Second, Timeout: time.Millisecond} + result := measureDuration(t.Context(), opts) + if result.Timeouts != 8 || result.Attempted != 8 || result.Indeterminate != 8 || result.Requests != 2 { + t.Fatalf("expired batches must count logical operations: %+v", result) + } +} diff --git a/internal/workload/dataset.go b/internal/workload/dataset.go index 491de996..00d558b6 100644 --- a/internal/workload/dataset.go +++ b/internal/workload/dataset.go @@ -28,8 +28,6 @@ type Config struct { } type Operation struct { - Worker int - Sequence int Record int Write bool Revision int @@ -161,7 +159,7 @@ func (d *Dataset) Plan(options PlanOptions) (*Plan, error) { } else { plan.Reads++ } - operation := Operation{Worker: worker, Sequence: sequence, Record: record, Write: write, Revision: plan.Expected[record]} + operation := Operation{Record: record, Write: write, Revision: plan.Expected[record]} operations[sequence] = operation } plan.Workers[worker] = operations diff --git a/internal/workload/dataset_test.go b/internal/workload/dataset_test.go index d45539cd..606032ce 100644 --- a/internal/workload/dataset_test.go +++ b/internal/workload/dataset_test.go @@ -35,14 +35,11 @@ func TestDeterministicPlanOwnsWorkerIDsAndReadRevisions(t *testing.T) { actual := make([]int, dataset.Config.Records) reads, writes, count := 0, 0, 0 for worker, operations := range first.Workers { - for sequence, operation := range operations { + for _, operation := range operations { if owner, exists := owners[operation.Record]; exists && owner != worker { t.Fatal("record shared by workers") } owners[operation.Record] = worker - if operation.Worker != worker || operation.Sequence != sequence { - t.Fatal("sequence metadata mismatch") - } if operation.Write { actual[operation.Record] = 1 - actual[operation.Record] writes++ diff --git a/internal/workload/lua.go b/internal/workload/lua.go index 6b73ed1e..ae1437d1 100644 --- a/internal/workload/lua.go +++ b/internal/workload/lua.go @@ -19,7 +19,7 @@ import ( ) func (d *Dataset) luaSource() string { - // The fixture stores BSON int32 and JSON 0/1, both read as lua.v1 int32. + // The fixture stores BSON int32 and JSON 0/1, both read as Lua int32. return `return weir.replace(weir.set(current, "revision", weir.sub(weir.i32("1"), weir.get(current, "revision"))))` } diff --git a/internal/workload/paths.go b/internal/workload/paths.go index d76d8467..abce2465 100644 --- a/internal/workload/paths.go +++ b/internal/workload/paths.go @@ -482,8 +482,8 @@ func (p *weirPath) Execute(ctx context.Context, operation Operation) Outcome { } if operation.Write { if p.dataset.Config.LuaMutations { - program := &weir.ProgramTransform{Runtime: "lua.v1", Source: []byte(p.dataset.luaSource())} - request := &weir.AtomicTransformRequest{Resource: p.dataset.Resource(operation.Record), Program: program} + program := &weir.LuaTransform{Source: []byte(p.dataset.luaSource())} + request := &weir.AtomicTransformRequest{Resource: p.dataset.Resource(operation.Record), Lua: program} opts := weir.AtomicTransformOptions{StoreName: p.dataset.Config.StoreName, Request: request} result, err := p.client.AtomicTransform(ctx, opts) return mutationOutcome(result, len(program.Source), err) diff --git a/tests/integration/batch_test.go b/tests/integration/batch_test.go index 148558df..cf322f30 100644 --- a/tests/integration/batch_test.go +++ b/tests/integration/batch_test.go @@ -16,7 +16,7 @@ import ( func testPublicBatch(t *testing.T, ctx context.Context, client *weir.Client, backend *backendData) { t.Helper() - secondary := provisionBatchTarget(t, ctx, backend) + secondary := backend.provisionTarget(t, ctx, "batch_extra") targets := []*backendData{backend, secondary} const count = 48 mutations := make([]*weir.MutateRequest, count) @@ -95,15 +95,15 @@ func testPublicBatch(t *testing.T, ctx context.Context, client *weir.Client, bac t.Log("48 streamed SDK mutations across two same-Store targets, ordered results including precondition/missing, duplicate URI order, and slice preflight verified") } -func provisionBatchTarget(t *testing.T, ctx context.Context, base *backendData) *backendData { +func (base *backendData) provisionTarget(t *testing.T, ctx context.Context, suffix string) *backendData { t.Helper() - name := base.collection + "_batch_extra" + name := base.collection + "_" + suffix if base.name == "mongo" { database := base.mongo.Database() - if err := database.CreateCollection(ctx, "batch_extra"); err != nil { + if err := database.CreateCollection(ctx, suffix); err != nil { t.Fatal("create second owned Mongo collection:", err) } - secondary := &backendData{name: base.name, collection: database.Name() + "/batch_extra", mongo: database.Collection("batch_extra")} + secondary := &backendData{name: base.name, collection: database.Name() + "/" + suffix, mongo: database.Collection(suffix)} return secondary } secondary := &backendData{name: base.name, collection: name, searchURL: base.searchURL, http: base.http} diff --git a/tests/integration/lifecycle_test.go b/tests/integration/lifecycle_test.go index 1fdb32f2..a31e0a19 100644 --- a/tests/integration/lifecycle_test.go +++ b/tests/integration/lifecycle_test.go @@ -149,7 +149,7 @@ func testCrossOwnerScan(t *testing.T, ctx context.Context, cluster *fixture.Clus for i, owner := range owners { transports[i] = transport(t, cluster.Nodes[owner].Application) } - seen := make(map[string]bool) + seen := make(map[int64]bool) var token []byte exhausted := false for page := 0; page < 5; page++ { @@ -157,14 +157,14 @@ func testCrossOwnerScan(t *testing.T, ctx context.Context, cluster *fixture.Clus count := uint64(0) scan := weir.ScanOptions{StoreName: backend.name, Request: request} scan.Consume = func(_ context.Context, document *weir.Document) error { - id, err := backend.scanID(document) + number, err := backend.number(document) if err != nil { return err } - if seen[id] { - return fmt.Errorf("cross-owner continuation repeated %q", id) + if seen[number] { + return fmt.Errorf("cross-owner continuation repeated %d", number) } - seen[id] = true + seen[number] = true count++ return nil } @@ -187,12 +187,12 @@ func testCrossOwnerScan(t *testing.T, ctx context.Context, cluster *fixture.Clus } token = bytes.Clone(next) } - got := make([]string, 0, len(seen)) - for id := range seen { - got = append(got, id) + got := make([]int64, 0, len(seen)) + for number := range seen { + got = append(got, number) } slices.Sort(got) - want := []string{"page1", "page2", "primary"} + want := []int64{5, 11, 12} if !exhausted || !slices.Equal(got, want) { t.Fatalf("cross-owner complete Scan set: exhausted=%v got=%v want=%v", exhausted, got, want) } @@ -209,33 +209,6 @@ func testCrossOwnerScan(t *testing.T, ctx context.Context, cluster *fixture.Clus t.Log("all finite Scan pages resumed on alternating owners without omissions/duplicates; failed consumer received no checkpoint") } -func (b *backendData) scanID(document *weir.Document) (string, error) { - if document == nil { - return "", errors.New("Scan omitted document") - } - if b.name == "mongo" { - if document.GetContentType() != "application/bson" { - return "", errors.New("MongoDB Scan changed native content type") - } - raw := bson.Raw(document.GetData()) - if err := raw.Validate(); err != nil { - return "", err - } - id, ok := raw.Lookup("_id").StringValueOK() - if !ok { - return "", errors.New("MongoDB Scan omitted _id") - } - return id, nil - } - var hit struct { - ID string `json:"_id"` - } - if document.GetContentType() != "application/json" || json.Unmarshal(document.GetData(), &hit) != nil || hit.ID == "" { - return "", errors.New("Search Scan omitted native hit _id") - } - return hit.ID, nil -} - func testNative(t *testing.T, ctx context.Context, client *weir.Client, backend *backendData) { t.Helper() request := &weir.NativeRequest{Resource: backend.collection} @@ -243,38 +216,28 @@ func testNative(t *testing.T, ctx context.Context, client *weir.Client, backend parts := strings.Split(backend.collection, "/") filter := bson.D{{Key: "_id", Value: "primary"}} command := bson.D{{Key: "count", Value: parts[len(parts)-1]}, {Key: "query", Value: filter}} - request.Body = bsonDocument(t, command).Data - request.BodyContentType = "application/bson" - descriptor := &weir.Document{ContentType: weir.MongoCommandContentType} - request.Descriptor = descriptor + request.MongoDBCommand = bsonDocument(t, command).Data } else { - descriptor := &weir.SearchHTTPRequest{Method: http.MethodGet, Path: "/_doc/primary"} - encoded, err := weir.SearchHTTPDescriptor(descriptor) - if err != nil { - t.Fatal(err) - } - request.Descriptor = encoded + request.SearchHTTP = &weir.SearchHTTPRequest{Method: http.MethodGet, Path: "/_doc/primary"} } - var head *weir.NativeHead + var head *weir.NativeResponse var body []byte options := weir.NativeOptions{StoreName: backend.name, Request: request} - options.Consume = func(_ context.Context, event *weir.Event) error { - if event.Head != nil { - if head != nil { - return errors.New("duplicate Native head") - } - head = event.Head + options.Consume = func(_ context.Context, response *weir.NativeResponse, chunk []byte) error { + if response == nil || head != nil && head != response { + return errors.New("Native response metadata changed between chunks") } - if len(body)+len(event.Chunk) > 1<<20 { + head = response + if len(body)+len(chunk) > 1<<20 { return errors.New("Native fixture response exceeded bound") } - body = append(body, event.Chunk...) + body = append(body, chunk...) return nil } rpc, cancel := context.WithTimeout(ctx, rpcTimeout) end, err := client.Native(rpc, options) cancel() - if err != nil || head == nil || end == nil || end.GetFailure() != nil || end.GetCompletion() != weir.NativeResponseComplete { + if err != nil || head == nil || end == nil || end.Response != head || end.Failure != nil || end.Completion != weir.NativeResponseComplete { t.Fatalf("Native evidence: head=%v end=%v err=%v", head, end, err) } if backend.name == "mongo" { @@ -284,14 +247,14 @@ func testNative(t *testing.T, ctx context.Context, client *weir.Client, backend } return } - metadata, err := weir.DecodeSearchHTTPResponse(head.GetMetadata()) + metadata := head.GetHttp() var response struct { Found bool `json:"found"` Source struct { N int64 `json:"n"` } `json:"_source"` } - if err != nil || metadata.StatusCode != http.StatusOK || json.Unmarshal(body, &response) != nil || !response.Found || response.Source.N != 5 { + if metadata == nil || metadata.StatusCode != http.StatusOK || json.Unmarshal(body, &response) != nil || !response.Found || response.Source.N != 5 { t.Fatalf("Native Search response: metadata=%v body=%s err=%v", metadata, body, err) } } diff --git a/tests/integration/semantics_test.go b/tests/integration/semantics_test.go new file mode 100644 index 00000000..30482384 --- /dev/null +++ b/tests/integration/semantics_test.go @@ -0,0 +1,279 @@ +//go:build integration + +package integration_test + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "testing" + + weir "github.com/batchstream/weir-go" + "go.mongodb.org/mongo-driver/v2/bson" +) + +func (s *system) testLuaSemantics(t *testing.T, backend *backendData) { + t.Helper() + cases := []struct { + name, source string + present bool + n int64 + failure weir.FailureCode + }{ + {name: "merge", source: "return weir.replace(weir.merge(current,input))", present: true, n: 42}, + {name: "implicit_replace", source: "return weir.merge(current,input)", present: true, n: 42}, + {name: "keep", source: "return weir.keep()"}, + {name: "delete", source: "return weir.delete()"}, + {name: "reject", source: "return weir.reject(\"rejected\")", failure: weir.FailurePreconditionFailed}, + {name: "no_return", source: "local v = current"}, + {name: "nil_return", source: "return nil"}, + {name: "current", source: "return current"}, + {name: "invalid", source: "return )", failure: weir.FailureInvalidArgument}, + } + for _, existing := range []bool{false, true} { + for _, tc := range cases { + t.Run(fmt.Sprintf("lua/%s/existing=%t", tc.name, existing), func(t *testing.T) { + id := fmt.Sprintf("lua_%s_%t", tc.name, existing) + if existing { + write := backend.write(t, id, 1) + opts := weir.WriteOptions{StoreName: backend.name, Request: write} + result, err := s.client.Create(s.ctx, opts) + assertApplied(t, result, err) + } + input := backend.write(t, id, 42).Document + program := &weir.LuaTransform{Source: []byte(tc.source), Input: input} + request := &weir.AtomicTransformRequest{Resource: backend.resource(id), Lua: program} + opts := weir.AtomicTransformOptions{StoreName: backend.name, Request: request} + rpc, cancel := context.WithTimeout(s.ctx, rpcTimeout) + result, err := s.client.AtomicTransform(rpc, opts) + cancel() + failure := tc.failure + if tc.name == "current" && !existing { + failure = weir.FailureInvalidArgument + } + if failure != 0 { + if err != nil || result.GetOutcome() != weir.MutationNotApplied || result.GetFailure().GetCode() != failure { + t.Fatalf("rejected Lua: %v %v", result, err) + } + } else { + assertApplied(t, result, err) + } + present, n := tc.present, tc.n + if tc.name != "delete" && !tc.present && existing { + present, n = true, 1 + } + if present { + backend.assertPersisted(t, s.ctx, id, n) + } else { + backend.assertMissing(t, s.ctx, id) + } + }) + } + } + s.testConcurrentLuaCreate(t, backend) +} + +func (s *system) testConcurrentLuaCreate(t *testing.T, backend *backendData) { + t.Helper() + const id = "lua_cross_owner_counter" + const iterations = 4 + one := "weir.i64(\"1\")" + if backend.name == "search" { + one = "weir.i32(\"1\")" + } + source := []byte(`if weir.kind(current)=="missing" then return weir.replace(input) end +return weir.replace(weir.set(current,"n",weir.add(weir.get(current,"n"),` + one + `)))`) + start := make(chan struct{}) + finished := make(chan error, 2) + for _, node := range s.cluster.Nodes[:2] { + wire := transport(t, node.Application) + input := backend.write(t, id, 1).Document + go func() { + <-start + for range iterations { + program := &weir.LuaTransform{Source: source, Input: input} + request := &weir.AtomicTransformRequest{Resource: backend.resource(id), Lua: program} + opts := weir.AtomicTransformOptions{StoreName: backend.name, Request: request} + rpc, cancel := context.WithTimeout(s.ctx, rpcTimeout) + result, err := weir.AtomicTransform(rpc, wire, opts) + cancel() + if err != nil || result.GetOutcome() != weir.MutationApplied || result.GetFailure() != nil { + finished <- fmt.Errorf("cross-owner Lua: result=%v error=%v", result, err) + return + } + } + finished <- nil + }() + } + close(start) + var firstError error + for range 2 { + if err := <-finished; err != nil && firstError == nil { + firstError = err + } + } + if firstError != nil { + t.Fatal(firstError) + } + backend.assertPersisted(t, s.ctx, id, 2*iterations) +} + +func (s *system) testProjectionAndFailures(t *testing.T, backend *backendData) { + t.Helper() + target := backend.provisionTarget(t, s.ctx, "projection.entries") + for n := int64(1); n <= 3; n++ { + write := target.write(t, fmt.Sprint(n), n) + opts := weir.WriteOptions{StoreName: target.name, Request: write} + result, err := s.client.Create(s.ctx, opts) + assertApplied(t, result, err) + } + if target.name == "search" { + refresh := httpRequest{method: http.MethodPost, path: "/" + target.collection + "/_refresh"} + code, body := target.httpDo(t, s.ctx, refresh) + if code != http.StatusOK { + t.Fatalf("refresh projection fixture: %d %s", code, body) + } + } + filter := &weir.Document{ContentType: "application/json", Data: []byte(`{"range":{"n":{"gte":2}}}`)} + if target.name == "mongo" { + bound := bson.D{{Key: "$gte", Value: int64(2)}} + value := bson.D{{Key: "n", Value: bound}} + filter = bsonDocument(t, value) + } + projection := &weir.Projection{Mode: weir.ProjectionInclude, Fields: []string{"n"}} + request := &weir.ScanRequest{Resource: target.collection, Filter: filter, Projection: projection, PageSize: 1} + seen := make(map[int64]bool) + exhausted := false + for page := range 4 { + wire := transport(t, s.cluster.Nodes[page%2].Application) + opts := weir.ScanOptions{StoreName: target.name, Request: request, Consume: func(_ context.Context, document *weir.Document) error { + n, err := target.number(document) + if err != nil { + return err + } + if n < 2 || seen[n] { + return fmt.Errorf("wrong or duplicate projected record %d", n) + } + seen[n] = true + if target.name == "mongo" { + elements, err := bson.Raw(document.Data).Elements() + if err != nil || len(elements) != 1 || elements[0].Key() != "n" { + return fmt.Errorf("unexpected Mongo projection: %v %v", elements, err) + } + } else { + var fields map[string]json.RawMessage + if err := json.Unmarshal(document.Data, &fields); err != nil { + return err + } + if len(fields) != 1 || fields["n"] == nil { + return fmt.Errorf("unexpected Search projection: %s", document.Data) + } + } + return nil + }} + end, err := weir.Scan(s.ctx, wire, opts) + if err != nil || end.GetFailure() != nil || end.GetDocumentCount() > 1 { + t.Fatalf("projected page: %v %v", end, err) + } + if page == 0 { + if end.GetExhausted() || len(end.GetNextContinuationToken()) == 0 { + t.Fatal("first filtered page has no continuation", end) + } + changed := &weir.Projection{Mode: weir.ProjectionExclude, Fields: []string{"n"}} + invalid := &weir.ScanRequest{Resource: target.collection, Filter: filter, Projection: changed, PageSize: 1, ContinuationToken: end.NextContinuationToken} + invalidOpts := weir.ScanOptions{StoreName: target.name, Request: invalid, Consume: func(context.Context, *weir.Document) error { return nil }} + checkpoint, err := weir.Scan(s.ctx, wire, invalidOpts) + if err != nil || checkpoint.GetFailure().GetCode() != weir.FailureInvalidArgument || checkpoint.GetDocumentCount() != 0 || len(checkpoint.GetNextContinuationToken()) != 0 { + t.Fatalf("changed projection accepted token: %v %v", checkpoint, err) + } + request.ContinuationToken = end.NextContinuationToken + } + if end.GetExhausted() { + exhausted = true + break + } + request.ContinuationToken = end.NextContinuationToken + } + if !exhausted || len(seen) != 2 { + t.Fatal("filtered projection omitted documents", seen) + } + exclude := &weir.Projection{Mode: weir.ProjectionExclude, Fields: []string{"n"}} + excluded := &weir.ScanRequest{Resource: target.collection, Projection: exclude, PageSize: 4} + excludedOptions := weir.ScanOptions{StoreName: target.name, Request: excluded, Consume: func(_ context.Context, document *weir.Document) error { + if target.name == "mongo" { + if bson.Raw(document.Data).Lookup("n").Type != 0 { + return fmt.Errorf("excluded Mongo field remains") + } + } else { + var fields map[string]json.RawMessage + if err := json.Unmarshal(document.Data, &fields); err != nil { + return err + } + if fields["n"] != nil { + return fmt.Errorf("excluded Search field remains") + } + } + return nil + }} + end, err := s.client.Scan(s.ctx, excludedOptions) + if err != nil || end.GetFailure() != nil || end.GetDocumentCount() != 3 || !end.GetExhausted() { + t.Fatalf("exclude projection: %v %v", end, err) + } + s.testMissingTargets(t, target) +} + +func (s *system) testMissingTargets(t *testing.T, target *backendData) { + t.Helper() + missing := *target + missing.collection += ".never_created" + requests := []*weir.ReadRequest{ + {Resource: target.resource("1")}, {Resource: target.resource("absent")}, {Resource: missing.resource("1")}, + } + opts := weir.ReadOptions{StoreName: target.name, Requests: requests} + results, err := s.client.Read(s.ctx, opts) + if err != nil || len(results) != 3 { + t.Fatalf("mixed read results: %v %v", results, err) + } + if results[0].Document == nil || !results[1].Missing || results[1].Failure != nil || results[2].Missing || results[2].Failure.GetCode() != weir.FailureTargetNotFound { + t.Fatalf("record/target absence confused: %v", results) + } + emptyFilter := &weir.Document{ContentType: "application/json", Data: []byte(`{"term":{"n":999}}`)} + if target.name == "mongo" { + value := bson.D{{Key: "n", Value: int64(999)}} + emptyFilter = bsonDocument(t, value) + } + request := &weir.ScanRequest{Resource: target.collection, Filter: emptyFilter, PageSize: 1} + scan := weir.ScanOptions{StoreName: target.name, Request: request, Consume: func(context.Context, *weir.Document) error { return nil }} + end, err := s.client.Scan(s.ctx, scan) + if err != nil || end.GetFailure() != nil || !end.GetExhausted() || end.GetDocumentCount() != 0 { + t.Fatalf("empty scan: %v %v", end, err) + } + request = &weir.ScanRequest{Resource: missing.collection, PageSize: 1} + scan.Request = request + end, err = s.client.Scan(s.ctx, scan) + if err != nil || end.GetFailure().GetCode() != weir.FailureTargetNotFound || end.GetExhausted() || len(end.GetNextContinuationToken()) != 0 { + t.Fatalf("missing target scan: %v %v", end, err) + } + nativeMissing := &weir.NativeRequest{Resource: missing.collection} + if target.name == "mongo" { + command := bson.D{{Key: "count", Value: target.mongo.Name() + ".never_created"}} + nativeMissing.MongoDBCommand = bsonDocument(t, command).Data + } else { + nativeMissing.SearchHTTP = &weir.SearchHTTPRequest{Method: http.MethodGet, Path: "/_doc/absent"} + } + nativeMissingOptions := weir.NativeOptions{StoreName: target.name, Request: nativeMissing, Consume: func(context.Context, *weir.NativeResponse, []byte) error { return nil }} + missingResult, err := s.client.Native(s.ctx, nativeMissingOptions) + if err != nil || missingResult == nil || missingResult.Response != nil || missingResult.Completion != weir.NativeNotStarted || missingResult.Failure.GetCode() != weir.FailureTargetNotFound { + t.Fatalf("missing Native target: %v %v", missingResult, err) + } + if target.name == "search" { + httpRequest := &weir.SearchHTTPRequest{Method: http.MethodGet, Path: "/_doc/absent"} + native := &weir.NativeRequest{Resource: target.collection, SearchHTTP: httpRequest} + nativeOpts := weir.NativeOptions{StoreName: target.name, Request: native, Consume: func(context.Context, *weir.NativeResponse, []byte) error { return nil }} + result, err := s.client.Native(s.ctx, nativeOpts) + if err != nil || result == nil || result.Completion != weir.NativeResponseComplete || result.Failure != nil || result.Response.GetHttp().GetStatusCode() != http.StatusNotFound { + t.Fatalf("complete HTTP 404 native response: %v %v", result, err) + } + } +} diff --git a/tests/integration/system_test.go b/tests/integration/system_test.go index c483b3e2..4f398a89 100644 --- a/tests/integration/system_test.go +++ b/tests/integration/system_test.go @@ -88,6 +88,8 @@ func TestSystemIntegration(t *testing.T) { testPublicBatch(t, ctx, client, backend) suite.testLargeDistinctBatch(t, backend) suite.testStreamMemory(t, backend) + suite.testLuaSemantics(t, backend) + suite.testProjectionAndFailures(t, backend) }) { return } diff --git a/versions.json b/versions.json index 149b0223..6d5872ad 100644 --- a/versions.json +++ b/versions.json @@ -1,13 +1,13 @@ { "go": "1.27.1", - "sdk": "v0.7.0", - "protocol": "v0.5.0", - "weir_source": "a1f07b20dc0b7b1e41def2a07fc4191749c497ab", + "sdk": "v0.8.0", + "protocol": "v0.6.0", + "weir_source": "dbcf308fd682905c8d45ea631107c2e454a4af80", "mongodb_version": "8.0.32", "elasticsearch_version": "8.19.22", "elasticsearch_image": "docker.elastic.co/elasticsearch/elasticsearch@sha256:e98f9c3b09beb2fbb9eaf667d602df3f0e00bd3644138b8458dc17ba1a675595", - "weir_version": "v0.1.1-0.20261004114516-a1f07b20dc0b", - "weir_sum": "h1:Q/Ysilf6XgVfnuh2YDxKv2XBXbdpxhIY9arUOwaB0iA=", + "weir_version": "v0.1.1-0.20261004150119-dbcf308fd682", + "weir_sum": "h1:FVKpFJrCgCUE3Ej9QmKIeeFU8ft40KYlRuc/Ld4Vhy8=", "mongodb_image": "mongo@sha256:d0d926f94df099bff534b7ee5b5986458131a22489dfff8664509af0c1e2ca9c", "mongodb_darwin_arm64_url": "https://fastdl.mongodb.org/osx/mongodb-macos-arm64-8.0.32.tgz", "mongodb_darwin_arm64_sha256": "f81cb258434d548dca7244d599c82eb339043d8dedd0b1b807870c9d263117f2",