diff --git a/README.md b/README.md index 8d199502..c8dc8553 100644 --- a/README.md +++ b/README.md @@ -45,9 +45,12 @@ The opt-in `integration` build tag starts MongoDB, Elasticsearch, two Store owners and a discovery node. Tagged tests require a prepared `WEIR_TEST_BINARY`. Coverage includes discovery through every node, direct owner connections, typed CRUD and Lua transformations, independent persistence checks, same-Store batches, -whole-request validation, finite Scan continuation, Native response evidence, +slice preflight and per-frame validation, finite Scan continuation, Native response evidence, cancellation, owner restart, directory convergence and persistent client recovery. Separate multi-process tests verify independent single-record callers. +One-RPC reads deliver 64 MiB and 256 MiB across 1024 distinct documents while +checking bounded result credits and sampled owner RSS. Interrupted streams retain +confirmed record outcomes; unconfirmed mutations remain indeterminate. Services bind to loopback and own exclusive namespaces. Cleanup verifies exact process and container ownership. Fixture logs, manifests and cleanup receipts @@ -98,7 +101,7 @@ distributions and the RPC/adapter ratio show whether aggregation occurred. Reports retain per-round throughput, pooled latency histograms, client process identities, start lag, CPU/RSS, database CPU/memory/I/O/network samples and raw -Weir metrics. Metrics report unary completions, terminal records, queue wait, +Weir metrics. Metrics report stream completions, terminal records, queue wait, adapter time, rejections and operations per adapter invocation. MongoDB also records physical command deltas. Adapter invocations can split by namespace, action and byte limits; their counts are distinct from database wire commands. @@ -112,6 +115,9 @@ concurrency alone cannot prove a plateau. A plateau with low database CPU leaves maximum-capacity comparison unavailable. Reports still show observed throughput and latency at each matched concurrency. No performance ratio is a CI threshold. +The lab records `weir_max_read_size_bytes`: the payload plus 4 KiB metadata +headroom, rounded up to 4 KiB and capped at 2 MiB. This prevents reserving 2 MiB +for every 1 KiB benchmark read; large-document integration still uses 2 MiB. Weir working-memory and process-admission budgets scale with backend concurrency and ingress sessions. Reports record declared budgets and observed usage. `-store-working-memory-mib` allows explicit workspace experiments. diff --git a/cmd/weir-lab/main.go b/cmd/weir-lab/main.go index 3bec708a..73d47b2f 100644 --- a/cmd/weir-lab/main.go +++ b/cmd/weir-lab/main.go @@ -245,8 +245,10 @@ func runLab(ctx context.Context, opts labOptions) (resultErr error) { for backend, memory := range workspace { provenance["weir_"+backend+"_working_memory_mib"] = fmt.Sprint(memory) } + maxReadBytes := min(2<<20, ((opts.payload+4096+4095)/4096)*4096) + provenance["weir_max_read_size_bytes"] = fmt.Sprint(maxReadBytes) provenance["weir_memory_budget"] = fmt.Sprintf("%dMiB (declared process admission budget; not an OS reservation)", memoryMiB) - start := fixture.Options{WeirBinary: binary, MongoBinary: opts.mongoBinary, Backends: backends, OwnerCount: 1, StoreConcurrency: storeWorkers, BatchSize: opts.backendBatchLimit, BackendTimeout: businessTimeout, IngressSessions: max(64, opts.concurrency), DatabaseCPUs: opts.databaseCPUs, ProcessMemoryMiB: memoryMiB, WorkingMemoryMiB: workspace} + start := fixture.Options{WeirBinary: binary, MongoBinary: opts.mongoBinary, Backends: backends, OwnerCount: 1, StoreConcurrency: storeWorkers, BatchSize: opts.backendBatchLimit, MaxReadSizeBytes: maxReadBytes, BackendTimeout: businessTimeout, IngressSessions: max(64, opts.concurrency), DatabaseCPUs: opts.databaseCPUs, ProcessMemoryMiB: memoryMiB, WorkingMemoryMiB: workspace} cluster, err := fixture.Start(ctx, start) if err != nil { return err @@ -303,8 +305,8 @@ func fixtureMemoryBudget(opts labOptions, backends []string, workers int) (map[s workspace := make(map[string]int, len(backends)) // The pinned adapters bound one 2MiB-read batch by at most 41MiB for // MongoDB (native reply, guard and scratch) and 96MiB for Search. - // Add ingress envelopes and 1GiB for Store input/results and other framing. - memoryMiB := max(64, opts.concurrency)*96 + 1024 + // Streaming ingress uses 32MiB per admitted stream. Add 1GiB for Store input/results and other framing. + memoryMiB := max(64, opts.concurrency)*32 + 1024 for _, backend := range backends { perBatchMiB := 41 if opts.luaMutations { diff --git a/cmd/weir-lab/memory_test.go b/cmd/weir-lab/memory_test.go index 8db517d3..97cd72bf 100644 --- a/cmd/weir-lab/memory_test.go +++ b/cmd/weir-lab/memory_test.go @@ -12,19 +12,19 @@ func TestFixtureMemoryCoversActualBackendConcurrency(t *testing.T) { if workspace["mongo"]<<20 < 32*((32<<20)+(128<<10)+(8<<20)) || workspace["search"] < 32*96 { t.Fatal("workspace silently throttles the configured backend concurrency", workspace) } - reservedMiB := 64 + 64/4 + 64*96 + workspace["mongo"] + workspace["search"] + 2*(64+32*2) - if processMiB < reservedMiB || processMiB != 12288 { + reservedMiB := 64 + 64/4 + 64*32 + workspace["mongo"] + workspace["search"] + 2*(64+32*2) + if processMiB < reservedMiB || processMiB != 8192 { t.Fatal("ingress and both backend envelopes do not fit the process", processMiB, reservedMiB) } opts.workingMemoryMiB = 384 workspace, processMiB, err = fixtureMemoryBudget(opts, backends, 32) - if err != nil || workspace["mongo"] != 384 || workspace["search"] != 384 || processMiB != 8192 { + if err != nil || workspace["mongo"] != 384 || workspace["search"] != 384 || processMiB != 4096 { t.Fatal("explicit workspace budget was not honored", workspace, processMiB, err) } opts.workingMemoryMiB = 0 opts.concurrency = 512 workspace, processMiB, err = fixtureMemoryBudget(opts, backends, 32) - if err != nil || workspace["mongo"] != 1312 || workspace["search"] != 3072 || processMiB != 55296 { + if err != nil || workspace["mongo"] != 1312 || workspace["search"] != 3072 || processMiB != 22528 { t.Fatal("512 independent requests do not fit the explicitly declared ingress envelope", workspace, processMiB, err) } if _, _, err := fixtureMemoryBudget(opts, backends, int(^uint(0)>>1)); err == nil { diff --git a/go.mod b/go.mod index 3500fdd2..ac31806a 100644 --- a/go.mod +++ b/go.mod @@ -3,8 +3,8 @@ module github.com/batchstream/weir-tests go 1.27.1 require ( - github.com/batchstream/weir-go v0.5.0 - github.com/batchstream/weir-protocol v0.3.0 + github.com/batchstream/weir-go v0.6.0 + github.com/batchstream/weir-protocol v0.4.0 go.mongodb.org/mongo-driver/v2 v2.9.1 google.golang.org/grpc v1.83.2 ) diff --git a/go.sum b/go.sum index 909f6cb2..07cb2f0d 100644 --- a/go.sum +++ b/go.sum @@ -1,7 +1,7 @@ -github.com/batchstream/weir-go v0.5.0 h1:mcKSCcckl+jU4FkhLhwvIumm33YkOm/aomMwIlAElOw= -github.com/batchstream/weir-go v0.5.0/go.mod h1:poDKCB+xMXpRvpgUewEwB25vIAM9HeGpm3ub0pVEbow= -github.com/batchstream/weir-protocol v0.3.0 h1:6YDOl9yskTP27q4ighNq1+i9y4AGy+/kQTKJvCSQ+Y4= -github.com/batchstream/weir-protocol v0.3.0/go.mod h1:hHmwPoE9yHYAz7Auc3a14Ijo/uxAef4sQ9nBtyXNcGc= +github.com/batchstream/weir-go v0.6.0 h1:S9/+7diNNNbVEH68lg6GbepzXAiCJpF0K74R5pTNFmo= +github.com/batchstream/weir-go v0.6.0/go.mod h1:YsokqwtYw2sN7zVbt4yIgqJUnuoHsDyiA1vQlXPE2VU= +github.com/batchstream/weir-protocol v0.4.0 h1:t4dDLAKeOOGz/wajfsfrYZgbILDvVceoPvJYKxTQuiM= +github.com/batchstream/weir-protocol v0.4.0/go.mod h1:hHmwPoE9yHYAz7Auc3a14Ijo/uxAef4sQ9nBtyXNcGc= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= diff --git a/internal/benchmark/saturation_report.go b/internal/benchmark/saturation_report.go index 712d5fe8..6f4cc1d9 100644 --- a/internal/benchmark/saturation_report.go +++ b/internal/benchmark/saturation_report.go @@ -42,7 +42,7 @@ func (r *SaturationReport) Markdown() string { fmt.Fprintf(&text, "| %d | %d | %d | %s | %d | %d | %d | %d | %.1f | %.3f | %.1f | %.1f | %.1f | %t |\n", pair.BatchSize, pair.Concurrency, pair.Round, result.Path, result.Reads, result.Writes, result.Succeeded, result.Errors+result.Indeterminate, result.OperationsPerSec, float64(result.Latency.P95NS)/1e6, result.Resources.MeanCPUPercent, result.Resources.FullCPUFraction*100, result.Resources.Coverage*100, result.Verified) } } - text.WriteString("\nMeasured Weir adapter and unary RPC evidence (warmup, reset and postflight excluded):\n\n| Batch | Workers | Round | Adapter invocations | Mean adapter batch | Read/Mutate RPCs | Queue mean ms | Adapter mean ms | Rejections | Backend concurrency limit |\n| ---: | ---: | ---: | ---: | ---: | --- | ---: | ---: | ---: | ---: |\n") + text.WriteString("\nMeasured Weir adapter and streaming RPC evidence (warmup, reset and postflight excluded):\n\n| Batch | Workers | Round | Adapter invocations | Mean adapter batch | Execute RPCs | Queue mean ms | Adapter mean ms | Rejections | Backend concurrency limit |\n| ---: | ---: | ---: | ---: | ---: | --- | ---: | ---: | ---: | ---: |\n") var metricWarnings []string for _, pair := range r.Pairs { metrics := pair.Weir.ServerMetrics @@ -50,8 +50,8 @@ func (r *SaturationReport) Markdown() string { fmt.Fprintf(&text, "| %d | %d | %d | unavailable | unavailable | unavailable | unavailable | unavailable | unavailable | unavailable |\n", pair.BatchSize, pair.Concurrency, pair.Round) continue } - fmt.Fprintf(&text, "| %d | %d | %d | %s | %s | %s/%s | %s | %s | %s | %s |\n", pair.BatchSize, pair.Concurrency, pair.Round, - metricCounter(metrics, "weir_store_executions_total"), metricMean(metrics.AdapterBatchAverage, 1), metricCounter(metrics, "weir_rpc_completions_total:read"), metricCounter(metrics, "weir_rpc_completions_total:mutate"), + fmt.Fprintf(&text, "| %d | %d | %d | %s | %s | %s | %s | %s | %s | %s |\n", pair.BatchSize, pair.Concurrency, pair.Round, + metricCounter(metrics, "weir_store_executions_total"), metricMean(metrics.AdapterBatchAverage, 1), metricCounter(metrics, "weir_rpc_completions_total:execute"), metricMean(metrics.QueueWaitMeanSeconds, 1000), metricMean(metrics.ExecutionMeanSeconds, 1000), metricCounter(metrics, "weir_store_rejections_total"), metricMean(metrics.ConcurrencyLimit, 1)) if metrics.Unavailable != "" { warning := fmt.Sprintf("Metrics unavailable (batch %d, workers %d, round %d): %s.", pair.BatchSize, pair.Concurrency, pair.Round, strings.ReplaceAll(metrics.Unavailable, "\n", " ")) @@ -81,7 +81,7 @@ func (r *SaturationReport) Markdown() string { text.WriteString("\nThese QPS and latency observations apply to this hardware and measured concurrency range. A business-throughput improvement can exist without demonstrating the maximum database capacity. An underfull database or missing next-level plateau keeps the database-capacity comparison unavailable.\n") } if r.Parameters.ClientProcesses > 0 { - text.WriteString("\nAggregation evidence: the JSON records one business request per record, Read/Mutate RPC counts, adapter invocation counts, average adapter batch and its cumulative histogram, and RPCs per adapter invocation. An adapter invocation is not necessarily one physical database wire command. MongoDB physical find/update/bulkWrite/getMore/killCursors/commitTransaction/abortTransaction deltas come from owned serverStatus at timed boundaries; missing counters remain unavailable. Elasticsearch has no equivalent physical HTTP-command counter. Every child PID, configured native pool limit, final revision ranges, exit result and sampled CPU/RSS is retained; coordinator sampling CPU is excluded from business-client CPU.\n") + text.WriteString("\nAggregation evidence: the JSON records one business request per record, Execute RPC counts, adapter invocation counts, average adapter batch and its cumulative histogram, and RPCs per adapter invocation. An adapter invocation is not necessarily one physical database wire command. MongoDB physical find/update/bulkWrite/getMore/killCursors/commitTransaction/abortTransaction deltas come from owned serverStatus at timed boundaries; missing counters remain unavailable. Elasticsearch has no equivalent physical HTTP-command counter. Every child PID, configured native pool limit, final revision ranges, exit result and sampled CPU/RSS is retained; coordinator sampling CPU is excluded from business-client CPU.\n") } text.WriteString("\nThe JSON retains before/after raw public Weir metrics and their observed deltas, every resource sample, monitor failures, raw cumulative database/container I/O and network counters, CPU quota/host core denominator, and client/Weir cumulative CPU. Docker CPU uses cumulative Engine usage and read timestamps across each entire sampling interval, never CLI instantaneous percentages. Both interval endpoints must succeed; missing intervals reduce full-run coverage. Samples include monitoring traffic and background database work. Native ps CPU has platform-dependent clock resolution. Native databases share the host CPU budget with the client and Weir. Disk/network utilization or capacity is not inferred from byte counters. A plateau without sustained database CPU saturation is not a database-full-load result. Startup/discovery/reset/postflight and warmup are excluded from throughput; all in-flight completion and validation is included.\n") if r.Incomplete != "" { diff --git a/internal/benchmark/server_metrics.go b/internal/benchmark/server_metrics.go index 569eead2..e2f65b04 100644 --- a/internal/benchmark/server_metrics.go +++ b/internal/benchmark/server_metrics.go @@ -49,7 +49,7 @@ func serverMetricDelta(before, after observe.Snapshot, store string) *ServerMetr } else { metrics.ConcurrencyLimit = &afterLimit } - for _, method := range []string{"read", "mutate"} { + for _, method := range []string{"execute"} { methodLabels := map[string]string{"method": method} value, deltaErr := observe.Delta(before, after, "weir_rpc_completions_total", methodLabels) if deltaErr == nil { @@ -65,10 +65,8 @@ func serverMetricDelta(before, after observe.Snapshot, store string) *ServerMetr if count := metrics.Deltas["weir_store_batch_operations_count"]; count > 0 { average := metrics.Deltas["weir_store_batch_operations_sum"] / count metrics.AdapterBatchAverage = &average - read, readFound := metrics.Deltas["weir_rpc_completions_total:read"] - mutate, mutateFound := metrics.Deltas["weir_rpc_completions_total:mutate"] - if readFound && mutateFound { - ratio := (read + mutate) / count + if streams, found := metrics.Deltas["weir_rpc_completions_total:execute"]; found { + ratio := streams / count metrics.RPCsPerAdapter = &ratio } } else { @@ -98,29 +96,21 @@ func serverMetricDelta(before, after observe.Snapshot, store string) *ServerMetr return metrics } -// Single-request aggregation evidence must match the measured business calls. -// An unused RPC method need not have an exported series; an active one must. +// Single-request aggregation evidence must match every measured business stream. func (m *ServerMetrics) qualifySingleRequests(reads, writes uint64) { m.RPCsPerAdapter = nil if m.Unavailable != "" || m.AdapterBatchAverage == nil { return } - count := m.Deltas["weir_store_batch_operations_count"] - var observed float64 - for method, calls := range map[string]uint64{"read": reads, "mutate": writes} { - value, found := m.Deltas["weir_rpc_completions_total:"+method] - if calls > 0 && !found { - return - } - if found && value != float64(calls) { - m.RPCUnavailable[method] = "timed unary completion count differs from measured single-request calls" - return - } - if found { - observed += value - } + observed, found := m.Deltas["weir_rpc_completions_total:execute"] + if !found { + return + } + if observed != float64(reads+writes) { + m.RPCUnavailable["execute"] = "timed stream completion count differs from measured single-request calls" + return } - if count > 0 { + if count := m.Deltas["weir_store_batch_operations_count"]; count > 0 { ratio := observed / count m.RPCsPerAdapter = &ratio } diff --git a/internal/benchmark/server_metrics_test.go b/internal/benchmark/server_metrics_test.go index 381745af..d4852a59 100644 --- a/internal/benchmark/server_metrics_test.go +++ b/internal/benchmark/server_metrics_test.go @@ -29,7 +29,7 @@ func metricsSnapshot(t *testing.T, measured bool) observe.Snapshot { if measured { rpc += 100 } - fmt.Fprintf(&raw, "weir_rpc_completions_total{method=\"read\",status=\"ok\"} %d\n", rpc) + fmt.Fprintf(&raw, "weir_rpc_completions_total{method=\"execute\",status=\"ok\"} %d\n", rpc) metrics, err := observe.Parse(raw.String()) if err != nil { t.Fatal(err) @@ -42,7 +42,7 @@ func TestServerMetricsUseMeasuredDeltaAndReportMissingObservations(t *testing.T) before := metricsSnapshot(t, false) after := metricsSnapshot(t, true) metrics := serverMetricDelta(before, after, "mongo") - if metrics.Unavailable != "" || metrics.AdapterBatchAverage == nil || *metrics.AdapterBatchAverage != 32 || metrics.Deltas["weir_store_executions_total"] != 100 || metrics.Deltas["weir_rpc_completions_total:read"] != 100 { + if metrics.Unavailable != "" || metrics.AdapterBatchAverage == nil || *metrics.AdapterBatchAverage != 32 || metrics.Deltas["weir_store_executions_total"] != 100 || metrics.Deltas["weir_rpc_completions_total:execute"] != 100 { t.Fatalf("measured batch did not exclude warmup/other store: %+v", metrics) } if metrics.QueueWaitMeanSeconds == nil || *metrics.QueueWaitMeanSeconds != .001 || metrics.ExecutionMeanSeconds == nil || *metrics.ExecutionMeanSeconds != .005 { @@ -51,8 +51,15 @@ func TestServerMetricsUseMeasuredDeltaAndReportMissingObservations(t *testing.T) if metrics.ConcurrencyLimit == nil || *metrics.ConcurrencyLimit != 32 { t.Fatal("configured limit became a delta or came from another store", metrics) } - if metrics.RPCUnavailable["mutate"] == "" { - t.Fatal("missing optional RPC observation silently disappeared") + if len(metrics.RPCUnavailable) != 0 { + t.Fatal("observed stream RPC unexpectedly unavailable") + } + pair := SaturationPair{BatchSize: 1, Concurrency: 32, Round: 1} + pair.Weir.ServerMetrics = metrics + report := &SaturationReport{Pairs: []SaturationPair{pair}} + markdown := report.Markdown() + if !strings.Contains(markdown, "Execute RPCs") || !strings.Contains(markdown, "| 1 | 32 | 1 | 100 | 32.000 | 100 |") || strings.Contains(markdown, "Read/Mutate RPC counts") { + t.Fatal("human-readable report lost observed Execute counts", markdown) } after.Error = "scrape failed" metrics = serverMetricDelta(before, after, "mongo") @@ -83,14 +90,14 @@ func TestSingleRequestRPCProofRequiresActiveMethodCounts(t *testing.T) { metrics := serverMetricDelta(before, after, "mongo") metrics.qualifySingleRequests(100, 0) if metrics.RPCsPerAdapter == nil || *metrics.RPCsPerAdapter != 1 { - t.Fatal("unused mutation series hid verified read-only aggregation evidence") + t.Fatal("read-only stream aggregation evidence lost") } metrics.qualifySingleRequests(100, 1) if metrics.RPCsPerAdapter != nil { - t.Fatal("missing active mutation count became zero") + t.Fatal("mismatched total stream count became valid") } metrics.qualifySingleRequests(99, 0) - if metrics.RPCsPerAdapter != nil || metrics.RPCUnavailable["read"] == "" { - t.Fatal("extra timed unary completions were treated as single-request proof") + if metrics.RPCsPerAdapter != nil || metrics.RPCUnavailable["execute"] == "" { + t.Fatal("extra timed stream completions were treated as single-request proof") } } diff --git a/internal/benchmark/single_saturation.go b/internal/benchmark/single_saturation.go index b0442ccf..a5ec4d5a 100644 --- a/internal/benchmark/single_saturation.go +++ b/internal/benchmark/single_saturation.go @@ -69,7 +69,7 @@ func runSingleSaturation(ctx context.Context, options SaturationOptions) (*Satur } provenance["go"], provenance["client_os_arch"] = runtime.Version(), runtime.GOOS+"/"+runtime.GOARCH provenance["client_process_model"] = "independent OS client processes; one SDK/driver pool each; sequential single-request goroutine workers; total native pool limit equals total client concurrency" - report := &SaturationReport{Schema: 3, Started: time.Now().UTC().Format(time.RFC3339Nano), Parameters: params, Provenance: provenance, CPUQuota: quota, Method: "independent OS client processes send one record per native call or unary SDK RPC; native FindOne/ReplaceOne or single-ID mget/PUT; no client aggregation; worker-owned disjoint IDs; common phase barriers and unchanged pools across warmup and measurement; alternating AB/BA; latency is each business request including validation, queueing and any server aggregation; independent persisted postflight; sustained database CPU requires >=5 intervals, >=80% measurement coverage, mean CPU>=threshold and CPU>=threshold during >=80% measurement duration in every round, plus <=10% next-level throughput gain"} + report := &SaturationReport{Schema: 3, Started: time.Now().UTC().Format(time.RFC3339Nano), Parameters: params, Provenance: provenance, CPUQuota: quota, Method: "independent OS client processes send one record per native call or streaming SDK RPC; native FindOne/ReplaceOne or single-ID mget/PUT; no client aggregation; worker-owned disjoint IDs; common phase barriers and unchanged pools across warmup and measurement; alternating AB/BA; latency is each business request including validation, queueing and any server aggregation; independent persisted postflight; sustained database CPU requires >=5 intervals, >=80% measurement coverage, mean CPU>=threshold and CPU>=threshold during >=80% measurement duration in every round, plus <=10% next-level throughput gain"} if options.Dataset.Config.LuaMutations { report.Method = strings.Replace(report.Method, "native FindOne/ReplaceOne or single-ID mget/PUT", "native single-record snapshot transaction FindOne/compute/ReplaceOne/commit or real-time single-ID mget/compute/version-conditional PUT; SDK Lua AtomicTransform computes the same revision toggle from the current source", 1) } diff --git a/internal/fixture/fixture.go b/internal/fixture/fixture.go index d44c21da..9ef65fb8 100644 --- a/internal/fixture/fixture.go +++ b/internal/fixture/fixture.go @@ -38,6 +38,7 @@ type Options struct { DiscoveryOnly bool StoreConcurrency int BatchSize int + MaxReadSizeBytes int BackendTimeout time.Duration IngressSessions int DatabaseCPUs float64 @@ -46,6 +47,7 @@ type Options struct { } type Node struct { + ProcessID int Application string Peer string Diagnostics string @@ -173,6 +175,12 @@ func validateOptions(options Options) (Options, error) { if options.StoreConcurrency == 0 { options.StoreConcurrency = 2 } + if options.MaxReadSizeBytes == 0 { + options.MaxReadSizeBytes = 2 << 20 + } + if options.MaxReadSizeBytes < 1024 || options.MaxReadSizeBytes > 2<<20 { + return options, errors.New("invalid maximum read document size") + } if options.BatchSize == 0 { options.BatchSize = 32 } @@ -285,7 +293,7 @@ func (c *Cluster) writeConfiguration(index int) error { stores := make([]map[string]any, 0, len(c.options.Backends)) if node.Owner { for _, backend := range c.options.Backends { - store := map[string]any{"name": backend, "max_concurrency": c.options.StoreConcurrency, "max_batch_operations": c.options.BatchSize, "max_read_size": "2MiB", "backend_timeout": c.options.BackendTimeout.String()} + store := map[string]any{"name": backend, "max_concurrency": c.options.StoreConcurrency, "max_batch_operations": c.options.BatchSize, "max_read_size": fmt.Sprintf("%dB", c.options.MaxReadSizeBytes), "backend_timeout": c.options.BackendTimeout.String()} if memory, configured := c.options.WorkingMemoryMiB[backend]; configured { store["working_memory"] = fmt.Sprintf("%dMiB", memory) } @@ -336,6 +344,7 @@ func (c *Cluster) startNode(ctx context.Context, index int) error { } p := &process{cmd: cmd, done: make(chan struct{}), logPath: logPath} c.processes[index] = p + c.Nodes[index].ProcessID = cmd.Process.Pid go func() { p.err = cmd.Wait() _ = log.Close() diff --git a/internal/fixture/fixture_test.go b/internal/fixture/fixture_test.go index dafb59aa..8a12bbb9 100644 --- a/internal/fixture/fixture_test.go +++ b/internal/fixture/fixture_test.go @@ -36,6 +36,8 @@ func TestValidateOptionsRejectsInvalidBeforeExternalWork(t *testing.T) { {WeirBinary: binary, OwnerCount: 9}, {WeirBinary: binary, StoreConcurrency: -1}, {WeirBinary: binary, BatchSize: -1}, + {WeirBinary: binary, MaxReadSizeBytes: 1023}, + {WeirBinary: binary, MaxReadSizeBytes: (2 << 20) + 1}, {WeirBinary: binary, BackendTimeout: -time.Second}, {WeirBinary: binary, ProcessMemoryMiB: -1}, {WeirBinary: binary, MongoBinary: filepath.Join(t.TempDir(), "missing-mongod")}, diff --git a/internal/workload/paths.go b/internal/workload/paths.go index ada690e0..75f0afbd 100644 --- a/internal/workload/paths.go +++ b/internal/workload/paths.go @@ -101,7 +101,7 @@ func Open(ctx context.Context, dataset *Dataset) (*Paths, error) { } paths.client = client info := direct.info - info.Protocol = "gRPC unary Read/Mutate batches via published Weir SDK" + info.Protocol = "bidirectional gRPC Execute streams via published Weir SDK" info.Endpoint = dataset.Config.WeirSeed info.RetryPolicy = "SDK never replays business requests" if dataset.Config.LuaMutations { @@ -497,8 +497,12 @@ func (p *weirPath) Execute(ctx context.Context, operation Operation) Outcome { request := &weir.ReadRequest{Resource: p.dataset.Resource(operation.Record)} opts := weir.ReadOneOptions{StoreName: p.dataset.Config.StoreName, Request: request} result, err := p.client.ReadOne(ctx, opts) - if err != nil { - return failed(err, false) + return p.readOutcome(result, operation, err) +} + +func (p *weirPath) readOutcome(result *weir.ReadResult, operation Operation, rpcErr error) Outcome { + if result == nil && rpcErr != nil { + return failed(rpcErr, false) } if result == nil || result.Document == nil || result.Failure != nil || result.Missing { return failed(errors.New("Weir Read lacks a successful document"), false) diff --git a/internal/workload/weir_batch.go b/internal/workload/weir_batch.go index 5573baa8..f200d39b 100644 --- a/internal/workload/weir_batch.go +++ b/internal/workload/weir_batch.go @@ -54,36 +54,20 @@ func (p *weirPath) ExecuteBatch(ctx context.Context, operations []Operation) []O } outcomes := make([]Outcome, len(operations)) for index, reply := range replies { - if err != nil { - outcomes[index] = failed(err, false) - continue - } - if reply == nil || reply.Document == nil || reply.Failure != nil || reply.Missing || reply.Document.MediaType != p.dataset.MediaType() { - outcomes[index] = failed(errors.New("Weir batch read lacks matching successful document"), false) - continue - } - if err := p.dataset.Validate(reply.Document.Data, operations[index]); err != nil { - outcomes[index] = failed(err, false) - continue - } - outcome := Outcome{Status: Success, ResponseBytes: uint64(len(reply.Document.Data))} - outcomes[index] = outcome + outcomes[index] = p.readOutcome(reply, operations[index], err) } return outcomes } // A confirmed APPLIED item can also contain a post-write business failure. // Preserve that evidence, but exclude it from successful throughput. A failed -// unary RPC supplies no item acknowledgements; all its writes remain unknown. +// stream retains confirmed item evidence; only unconfirmed writes remain unknown. func mutationOutcome(result *weir.MutationResult, requestBytes int, rpcErr error) Outcome { if result != nil && result.GetOutcome() == weir.MutationApplied { outcome := Outcome{Status: Success, Applied: true, RequestBytes: uint64(requestBytes)} - if result.GetFailure() != nil || rpcErr != nil { + if result.GetFailure() != nil { outcome.Status = Failed - failure := rpcErr - if result.GetFailure() != nil { - failure = errors.Join(fmt.Errorf("Weir APPLIED with failure: %v", result.GetFailure()), rpcErr) - } + failure := fmt.Errorf("Weir APPLIED with failure: %v", result.GetFailure()) outcome.Error = failure.Error() } return outcome diff --git a/internal/workload/weir_batch_test.go b/internal/workload/weir_batch_test.go index a27a4507..db7cbfb8 100644 --- a/internal/workload/weir_batch_test.go +++ b/internal/workload/weir_batch_test.go @@ -2,6 +2,7 @@ package workload import ( "context" + "io" "net" "sync/atomic" "testing" @@ -19,30 +20,82 @@ import ( type acknowledgementPeer struct { pb.UnimplementedStoreServiceServer - mode string - calls atomic.Int64 - applied atomic.Int64 + mode string + calls atomic.Int64 + applied atomic.Int64 + document []byte } -func (p *acknowledgementPeer) Mutate(ctx context.Context, request *pb.MutateBatchRequest) (*pb.MutateBatchResponse, error) { +func (p *acknowledgementPeer) Execute(stream grpc.BidiStreamingServer[pb.ExecuteRequest, pb.ExecuteResponse]) error { p.calls.Add(1) - // Simulate durable writes before acknowledgement is lost. Neither the SDK - // nor the benchmark adapter can infer these outcomes from an RPC error. - p.applied.Add(int64(len(request.Requests))) - switch p.mode { - case "lost_after_apply": - return nil, status.Error(codes.Unavailable, "entire batch acknowledgement lost") - case "deadline_after_apply", "canceled_after_apply": - <-ctx.Done() - return nil, status.FromContextError(ctx.Err()).Err() + for { + request, err := stream.Recv() + if err == io.EOF { + return nil + } + if err != nil { + return err + } + if request.Command.GetRead() != nil { + document := &pb.Document{MediaType: "application/json", Data: p.document} + payload := &pb.ReadResult_Document{Document: document} + result := &pb.ReadResult{Result: payload} + value := &pb.Event_ReadResult{ReadResult: result} + event := &pb.Event{Value: value} + response := &pb.ExecuteResponse{Index: request.Index, Event: event} + if err := stream.Send(response); err != nil { + return err + } + return status.Error(codes.Unavailable, "later read result lost") + } + mutations := request.Command.GetMutate().Requests + // Durable writes can precede lost acknowledgements; never infer replay safety. + p.applied.Add(int64(len(mutations))) + switch p.mode { + case "lost_after_apply": + return status.Error(codes.Unavailable, "window acknowledgement lost") + case "deadline_after_apply", "canceled_after_apply": + <-stream.Context().Done() + return status.FromContextError(stream.Context().Err()).Err() + } + for index := range mutations { + var failure *pb.Failure + if p.mode == "applied_failure" { + failure = protocol.Fail(pb.FailureCode_UNAVAILABLE, "write applied but replica confirmation failed") + } + result := protocol.Mutation(pb.MutationOutcome_APPLIED, failure) + payload := &pb.Event_MutationResult{MutationResult: result} + event := &pb.Event{Value: payload} + response := &pb.ExecuteResponse{Index: request.Index + uint64(index), Event: event} + if err := stream.Send(response); err != nil { + return err + } + if p.mode == "partial_after_apply" { + return status.Error(codes.Unavailable, "later acknowledgement lost") + } + } } - results := make([]*pb.MutationResult, len(request.Requests)) - for index := range results { - failure := protocol.Fail(pb.FailureCode_UNAVAILABLE, "write applied but replica confirmation failed") - results[index] = protocol.Mutation(pb.MutationOutcome_APPLIED, failure) +} + +func TestSDKStreamReadPreservesConfirmedPrefixThroughLaterFailure(t *testing.T) { + dataset := testDataset(t, "search") + first := Operation{Record: 0} + second := Operation{Record: 1} + peer := &acknowledgementPeer{document: dataset.Document(first)} + client := mutationTransport(t, peer) + firstRequest := &weir.ReadRequest{Resource: dataset.Resource(first.Record)} + secondRequest := &weir.ReadRequest{Resource: dataset.Resource(second.Record)} + options := weir.ReadOptions{StoreName: "search", Requests: []*weir.ReadRequest{firstRequest, secondRequest}} + replies, err := weir.Read(t.Context(), client, options) + if err == nil || len(replies) != 2 || replies[0] == nil || replies[1] != nil || peer.calls.Load() != 1 { + t.Fatal("unexpected partial read evidence", replies, err) + } + path := &weirPath{dataset: dataset} + confirmed := path.readOutcome(replies[0], first, err) + unknown := path.readOutcome(replies[1], second, err) + if confirmed.Status != Success || confirmed.ResponseBytes == 0 || unknown.Status != Failed || unknown.ResponseBytes != 0 { + t.Fatal("later stream error changed confirmed read accounting", confirmed, unknown) } - response := &pb.MutateBatchResponse{Results: results} - return response, nil } func mutationTransport(t *testing.T, peer *acknowledgementPeer) pb.StoreServiceClient { @@ -66,8 +119,8 @@ func mutationTransport(t *testing.T, peer *acknowledgementPeer) pb.StoreServiceC return pb.NewStoreServiceClient(connection) } -func TestSDKUnaryMutationAcknowledgementAndNoReplay(t *testing.T) { - for _, mode := range []string{"applied_failure", "lost_after_apply", "deadline_after_apply", "canceled_after_apply"} { +func TestSDKStreamMutationAcknowledgementAndNoReplay(t *testing.T) { + for _, mode := range []string{"applied_failure", "partial_after_apply", "lost_after_apply", "deadline_after_apply", "canceled_after_apply"} { t.Run(mode, func(t *testing.T) { peer := &acknowledgementPeer{mode: mode} client := mutationTransport(t, peer) @@ -92,21 +145,25 @@ func TestSDKUnaryMutationAcknowledgementAndNoReplay(t *testing.T) { if mode == "applied_failure" && rpcErr != nil || mode != "applied_failure" && rpcErr == nil { t.Fatal("unexpected final status", rpcErr) } - for _, reply := range replies { + for index, reply := range replies { outcome := mutationOutcome(reply, len(document.Data), rpcErr) if mode == "applied_failure" { if outcome.Status != Failed || !outcome.Applied || outcome.RequestBytes != uint64(len(document.Data)) || outcome.Error == "" { t.Fatal("confirmed APPLIED business failure lost evidence", reply, outcome) } + } else if mode == "partial_after_apply" && index == 0 { + if outcome.Status != Success || !outcome.Applied || outcome.RequestBytes != uint64(len(document.Data)) || outcome.Error != "" { + t.Fatal("later stream failure revoked a successful acknowledgement", reply, outcome) + } } else if reply != nil || outcome.Status != Indeterminate || outcome.Applied || outcome.RequestBytes != 0 || outcome.Error == "" { - t.Fatal("failed unary RPC invented a partial acknowledgement", reply, outcome) + t.Fatal("failed stream invented a partial acknowledgement", reply, outcome) } } }) } } -func TestSDKUnaryMutationPreflightSendsNoBatch(t *testing.T) { +func TestSDKStreamMutationPreflightSendsNoBatch(t *testing.T) { peer := &acknowledgementPeer{mode: "applied_failure"} client := mutationTransport(t, peer) document := &weir.Document{MediaType: "application/json", Data: []byte(`{"n":1}`)} diff --git a/tests/integration/batch_test.go b/tests/integration/batch_test.go index 685f7701..873708ac 100644 --- a/tests/integration/batch_test.go +++ b/tests/integration/batch_test.go @@ -93,7 +93,7 @@ func testPublicBatch(t *testing.T, ctx context.Context, client *weir.Client, bac targets[index%len(targets)].assertMissing(t, ctx, fmt.Sprintf("late_invalid_%03d", index)) } testDuplicateURIOrder(t, ctx, client, backend) - t.Log("48 independent unary SDK mutations across two same-Store targets, ordered results including precondition/missing, duplicate URI execution order, and SDK/server late-invalid preflight verified") + t.Log("48 independent streamed SDK mutations across two same-Store targets, ordered results including precondition/missing, duplicate URI execution order, and SDK/server late-invalid preflight verified") } func provisionBatchTarget(t *testing.T, ctx context.Context, base *backendData) *backendData { @@ -205,10 +205,23 @@ func (s *system) testServerPreflight(t *testing.T, backend *backendData) { requests[index] = request } requests[count-1].Action = nil - request := &pb.MutateBatchRequest{StoreName: backend.name, Requests: requests} + batch := &pb.MutationBatch{Requests: requests} + operation := &pb.Command_Mutate{Mutate: batch} + command := &pb.Command{Operation: operation} + request := &pb.ExecuteRequest{StoreName: backend.name, Index: 1, Command: command} wire := transport(t, s.cluster.Nodes[0].Application) rpc, cancel := context.WithTimeout(s.ctx, rpcTimeout) - response, err := wire.Mutate(rpc, request) + stream, err := wire.Execute(rpc) + if err != nil { + t.Fatal(err) + } + if err := stream.Send(request); err != nil { + t.Fatal(err) + } + if err := stream.CloseSend(); err != nil { + t.Fatal(err) + } + response, err := stream.Recv() cancel() if status.Code(err) != codes.InvalidArgument || response != nil { t.Fatalf("server late-invalid preflight: response=%v err=%v", response, err) @@ -241,8 +254,8 @@ func (s *system) testLargeDistinctBatch(t *testing.T, backend *backendData) { for _, result := range results { assertApplied(t, result, nil) } - evidence := unaryBatchEvidence{Before: before, After: s.nodeMetrics(t), Store: backend.name, Method: "mutate", Records: count, SingleInvocation: true} - assertUnaryBatchMetrics(t, evidence) + evidence := streamBatchEvidence{Before: before, After: s.nodeMetrics(t), Store: backend.name, Method: "mutate", Records: count} + assertStreamBatchMetrics(t, evidence) for index := range mutations { backend.assertPersisted(t, s.ctx, fmt.Sprintf("large_distinct_%03d", index), int64(10000+index)) } @@ -258,6 +271,6 @@ func (s *system) testLargeDistinctBatch(t *testing.T, backend *backendData) { backend.assertReadResult(t, result, nil, int64(10000+count-1-index)) } evidence.Before, evidence.After, evidence.Method = before, s.nodeMetrics(t), "read" - assertUnaryBatchMetrics(t, evidence) - t.Log("513 distinct native documents Put and Read in one physical adapter batch per unary call; every value independently persisted and reverse-order results verified") + assertStreamBatchMetrics(t, evidence) + t.Log("513 distinct native documents Put and Read in bounded adapter windows on one streamed call; every value independently persisted and reverse-order results verified") } diff --git a/tests/integration/stream_memory_test.go b/tests/integration/stream_memory_test.go new file mode 100644 index 00000000..b2e461a6 --- /dev/null +++ b/tests/integration/stream_memory_test.go @@ -0,0 +1,161 @@ +//go:build integration + +package integration_test + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "os/exec" + "strconv" + "strings" + "testing" + "time" + + weir "github.com/batchstream/weir-go" + "github.com/batchstream/weir-tests/internal/fixture" + "github.com/batchstream/weir-tests/internal/observe" + "go.mongodb.org/mongo-driver/v2/bson" +) + +// Sample only processes started by this suite, including both possible owners. +func ownerResidentBytes(ctx context.Context, nodes []fixture.Node) (uint64, error) { + var largest uint64 + for _, node := range nodes { + if !node.Owner { + continue + } + if node.ProcessID <= 0 { + return 0, errors.New("owned Weir PID unavailable") + } + command := exec.CommandContext(ctx, "ps", "-o", "rss=", "-p", strconv.Itoa(node.ProcessID)) + output, err := command.Output() + if err != nil { + return 0, err + } + kib, err := strconv.ParseUint(strings.TrimSpace(string(output)), 10, 64) + if err != nil { + return 0, err + } + largest = max(largest, kib*1024) + } + return largest, nil +} + +func (s *system) testStreamMemory(t *testing.T, backend *backendData) { + t.Helper() + const payloadBytes = 64 << 10 + const distinctRecords = 1024 + padding := strings.Repeat("x", payloadBytes) + requests := make([]*weir.MutateRequest, distinctRecords) + reads := make([]*weir.ReadRequest, distinctRecords) + var jsonData []byte + if backend.name != "mongo" { + fields := map[string]any{"n": 42, "padding": padding} + data, err := json.Marshal(fields) + if err != nil { + t.Fatal(err) + } + jsonData = data + } + for index := range distinctRecords { + id := fmt.Sprintf("stream_memory_%04d", index) + write := backend.write(t, id, 42) + if backend.name == "mongo" { + fields := bson.D{{Key: "_id", Value: id}, {Key: "n", Value: int64(42)}, {Key: "padding", Value: padding}} + write.Document = bsonDocument(t, fields) + } else { + write.Document.Data = jsonData + } + request := &weir.MutateRequest{Resource: write.Resource, Action: weir.MutationPut, Document: write.Document} + requests[index] = request + read := &weir.ReadRequest{Resource: write.Resource} + reads[index] = read + } + rpc, cancel := context.WithTimeout(s.ctx, 90*time.Second) + writeOptions := weir.MutateOptions{StoreName: backend.name, Requests: requests} + results, err := s.client.Mutate(rpc, writeOptions) + cancel() + if err != nil || len(results) != distinctRecords { + t.Fatal("prepare distinct stream documents", len(results), err) + } + for _, result := range results { + assertApplied(t, result, nil) + } + backend.assertPersisted(t, s.ctx, "stream_memory_0000", 42) + backend.assertPersisted(t, s.ctx, "stream_memory_1023", 42) + var shortPeak uint64 + for _, count := range []int{1024, 4096} { + before := s.nodeMetrics(t) + ctx, stop := context.WithTimeout(s.ctx, 90*time.Second) + produced, consumed := 0, uint64(0) + var bytes, peak uint64 + sample := func() error { + resident, err := ownerResidentBytes(ctx, s.cluster.Nodes) + if err != nil { + return err + } + peak = max(peak, resident) + for _, node := range s.cluster.Nodes { + if !node.Owner { + continue + } + snapshot := observe.Fetch(ctx, node.Diagnostics) + labels := map[string]string{"store": backend.name} + retained, found := snapshot.Sum("weir_store_result_reserved_bytes", labels) + limit, limitFound := snapshot.Sum("weir_store_result_reserved_bytes_limit", labels) + if !found || !limitFound || retained < 0 || retained > limit { + return fmt.Errorf("Store result credits invalid: retained=%v limit=%v error=%s", retained, limit, snapshot.Error) + } + } + return nil + } + if err := sample(); err != nil { + stop() + t.Fatal(err) + } + options := weir.ReadStreamOptions{ + StoreName: backend.name, + Next: func(context.Context) (*weir.ReadRequest, error) { + if produced == count { + return nil, io.EOF + } + request := reads[produced%distinctRecords] + produced++ + return request, nil + }, + Consume: func(_ context.Context, index uint64, result *weir.ReadResult) error { + if index != consumed+1 || result == nil || result.Document == nil || result.Failure != nil || result.Missing { + return fmt.Errorf("unexpected stream result at %d", index) + } + if n, err := backend.number(result.Document); err != nil || n != 42 || len(result.Document.Data) < payloadBytes { + return fmt.Errorf("large streamed document changed: n=%v error=%v", n, err) + } + consumed++ + bytes += uint64(len(result.Document.Data)) + if consumed%128 == 0 { + return sample() + } + return nil + }, + } + err := s.client.ReadStream(ctx, options) + if err == nil { + err = sample() + } + stop() + if err != nil || consumed != uint64(count) || bytes < uint64(count)*payloadBytes { + t.Fatalf("large call failed: count=%d consumed=%d bytes=%d error=%v", count, consumed, bytes, err) + } + evidence := streamBatchEvidence{Before: before, After: s.nodeMetrics(t), Store: backend.name, Method: "read", Records: count} + assertStreamBatchMetrics(t, evidence) + if count == 1024 { + shortPeak = peak + } else if peak > shortPeak+(96<<20) { + t.Fatalf("4x logical output grew Weir RSS excessively: short=%d long=%d", shortPeak, peak) + } + t.Logf("%s: one Execute stream delivered %d records (%d bytes), sampled max owner RSS=%d bytes", backend.name, count, bytes, peak) + } +} diff --git a/tests/integration/system_test.go b/tests/integration/system_test.go index 49d7c2dd..b9bc0148 100644 --- a/tests/integration/system_test.go +++ b/tests/integration/system_test.go @@ -87,6 +87,7 @@ func TestSystemIntegration(t *testing.T) { testCrossOwnerScan(t, ctx, cluster, backend) testPublicBatch(t, ctx, client, backend) suite.testLargeDistinctBatch(t, backend) + suite.testStreamMemory(t, backend) }) { return } @@ -161,12 +162,25 @@ func (s *system) testDiscovery(t *testing.T) { s.testServerPreflight(t, backend) s.testBatchOwnerRouting(t, backend) read := &pb.ReadRequest{Resource: backend.resource("primary")} - request := &pb.ReadBatchRequest{StoreName: backend.name, Requests: []*pb.ReadRequest{read}} + batch := &pb.ReadBatch{Requests: []*pb.ReadRequest{read}} + operation := &pb.Command_Read{Read: batch} + command := &pb.Command{Operation: operation} + request := &pb.ExecuteRequest{StoreName: backend.name, Index: 1, Command: command} rpc, cancel := context.WithTimeout(ctx, rpcTimeout) - response, err := wire.Read(rpc, request) + stream, err := wire.Execute(rpc) + if err != nil { + t.Fatal(err) + } + if err := stream.Send(request); err != nil { + t.Fatal(err) + } + if err := stream.CloseSend(); err != nil { + t.Fatal(err) + } + response, err := stream.Recv() cancel() if response != nil || status.Code(err) != codes.Unavailable { - t.Fatalf("non-owner %s unary Read: want Unavailable and no business response, got response=%v err=%v", backend.name, response, err) + t.Fatalf("non-owner %s streamed Read: want Unavailable and no business response, got response=%v err=%v", backend.name, response, err) } } } @@ -327,20 +341,19 @@ func (s *system) nodeMetrics(t *testing.T) []observe.Snapshot { return snapshots } -type unaryBatchEvidence struct { - Before, After []observe.Snapshot - Store, Method string - Records int - SingleInvocation bool +type streamBatchEvidence struct { + Before, After []observe.Snapshot + Store, Method string + Records int } -func assertUnaryBatchMetrics(t *testing.T, evidence unaryBatchEvidence) { +func assertStreamBatchMetrics(t *testing.T, evidence streamBatchEvidence) { t.Helper() var owners int var rpcCount float64 labels := map[string]string{"store": evidence.Store, "operation": evidence.Method} storeLabels := map[string]string{"store": evidence.Store} - methodLabels := map[string]string{"method": evidence.Method} + methodLabels := map[string]string{"method": "execute"} for index, before := range evidence.Before { after := evidence.After[index] left, beforeFound := before.Sum("weir_store_records_total", labels) @@ -360,21 +373,19 @@ func assertUnaryBatchMetrics(t *testing.T, evidence unaryBatchEvidence) { t.Fatalf("whole batch relayed or split across owners: node=%d delta=%v", index, delta) } owners++ - if evidence.SingleInvocation { - count, err := observe.Delta(before, after, "weir_store_batch_operations_count", storeLabels) - if err != nil || count != 1 { - t.Fatalf("physical adapter invocations=%v error=%v", count, err) - } - operations, err := observe.Delta(before, after, "weir_store_batch_operations_sum", storeLabels) - if err != nil || operations != float64(evidence.Records) { - t.Fatalf("physical adapter batch operations=%v error=%v", operations, err) - } + count, err := observe.Delta(before, after, "weir_store_batch_operations_count", storeLabels) + if err != nil || count < 1 { + t.Fatalf("window adapter invocations=%v error=%v", count, err) + } + operations, err := observe.Delta(before, after, "weir_store_batch_operations_sum", storeLabels) + if err != nil || operations != float64(evidence.Records) { + t.Fatalf("window adapter operations=%v error=%v", operations, err) } } left, beforeFound = before.Sum("weir_rpc_completions_total", methodLabels) right, afterFound = after.Sum("weir_rpc_completions_total", methodLabels) if !beforeFound || !afterFound { - t.Fatalf("node %d unary completion observation unavailable", index) + t.Fatalf("node %d streamed completion observation unavailable", index) } if right < left { t.Fatal("RPC completion metrics reset") @@ -382,7 +393,7 @@ func assertUnaryBatchMetrics(t *testing.T, evidence unaryBatchEvidence) { rpcCount += right - left } if owners != 1 || rpcCount != 1 { - t.Fatalf("single owner unary batch: owners=%d completed RPCs=%v", owners, rpcCount) + t.Fatalf("single owner streamed batch: owners=%d completed RPCs=%v", owners, rpcCount) } } @@ -399,11 +410,11 @@ func (s *system) testBatchOwnerRouting(t *testing.T, backend *backendData) { results, err := s.client.Read(rpc, opts) cancel() if err != nil || len(results) != len(reads) { - t.Fatalf("routed unary batch: results=%d err=%v", len(results), err) + t.Fatalf("routed streamed batch: results=%d err=%v", len(results), err) } for _, result := range results { backend.assertReadResult(t, result, nil, 5) } - evidence := unaryBatchEvidence{Before: before, After: s.nodeMetrics(t), Store: backend.name, Method: "read", Records: len(reads)} - assertUnaryBatchMetrics(t, evidence) + evidence := streamBatchEvidence{Before: before, After: s.nodeMetrics(t), Store: backend.name, Method: "read", Records: len(reads)} + assertStreamBatchMetrics(t, evidence) } diff --git a/versions.json b/versions.json index 0b2ea7f2..362f8e0e 100644 --- a/versions.json +++ b/versions.json @@ -1,13 +1,13 @@ { "go": "1.27.1", - "sdk": "v0.5.0", - "protocol": "v0.3.0", - "weir_source": "19b435678ed6b102a5b22dfcc39d345a2561b22f", + "sdk": "v0.6.0", + "protocol": "v0.4.0", + "weir_source": "b8fce387897e6a7be018a9aed1db35f484f150b7", "mongodb_version": "8.0.32", "elasticsearch_version": "8.19.22", "elasticsearch_image": "docker.elastic.co/elasticsearch/elasticsearch@sha256:e98f9c3b09beb2fbb9eaf667d602df3f0e00bd3644138b8458dc17ba1a675595", - "weir_version": "v0.1.1-0.20261004071043-19b435678ed6", - "weir_sum": "h1:hgbEUuc+9PA/tcADap9ULd4JEqr9AI3p15VSC1sDDLU=", + "weir_version": "v0.1.1-0.20261004094535-b8fce387897e", + "weir_sum": "h1:zRhILg8EoflGS1bmPe/P/clHVRIoYT4r0a8b1uq/xt0=", "mongodb_image": "mongo@sha256:d0d926f94df099bff534b7ee5b5986458131a22489dfff8664509af0c1e2ca9c", "mongodb_darwin_arm64_url": "https://fastdl.mongodb.org/osx/mongodb-macos-arm64-8.0.32.tgz", "mongodb_darwin_arm64_sha256": "f81cb258434d548dca7244d599c82eb339043d8dedd0b1b807870c9d263117f2",