From ca627925309acbba2d06fbf5192ebbe8643a453d Mon Sep 17 00:00:00 2001 From: Randy <857271370@qq.com> Date: Wed, 23 Sep 2026 03:43:04 +0800 Subject: [PATCH 1/2] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20refactor:=20simplify?= =?UTF-8?q?=20Dataset=20program=20state=20and=20error=20aggregation?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dataset.go | 38 +++++++++++--------------------------- dataset_test.go | 3 --- types.go | 5 +++-- 3 files changed, 14 insertions(+), 32 deletions(-) diff --git a/dataset.go b/dataset.go index fbe74f2..3b5c62d 100644 --- a/dataset.go +++ b/dataset.go @@ -31,11 +31,10 @@ type Record struct { // Record methods return decoded results and a BatchError when any individual // operation fails; native methods retain the corresponding Client semantics. type Dataset struct { - client *Client - uri string - encoding DocumentEncoding - mergeProgram LuaProgram - hasMergeProgram bool + client *Client + uri string + encoding DocumentEncoding + mergeProgram *LuaProgram } // NewDataset binds stable dataset settings to a Client. MergeProgram is @@ -59,12 +58,9 @@ func NewDataset(client *Client, opts DatasetOptions) (*Dataset, error) { if err := opts.MergeProgram.validate(); err != nil { return nil, fmt.Errorf("create dataset: %w", err) } - program, err := NewLuaProgram(opts.MergeProgram.Source()) - if err != nil { - return nil, fmt.Errorf("create dataset: copy merge program: %w", err) - } - dataset.mergeProgram = program - dataset.hasMergeProgram = true + // Snapshot the value; LuaProgram owns immutable buffers. + program := *opts.MergeProgram + dataset.mergeProgram = &program } return dataset, nil } @@ -99,16 +95,10 @@ func (d *Dataset) Read(ctx context.Context, req DatasetReadRequest) ([]ReadResul } request := ReadRequest{Addresses: addresses, OnResult: onResult} results, err := d.client.Read(ctx, request) - resultsErr := errors.Join(ReadResultsError(results), newBatchError(failures)) + err = errors.Join(err, ReadResultsError(results), newBatchError(failures)) if err != nil { - if resultsErr != nil { - err = errors.Join(err, resultsErr) - } return results, fmt.Errorf("dataset read: %w", err) } - if resultsErr != nil { - return results, fmt.Errorf("dataset read: %w", resultsErr) - } return results, nil } @@ -149,7 +139,7 @@ func (d *Dataset) Merge(ctx context.Context, req DatasetWriteRequest) ([]WriteRe if err := d.validate("merge"); err != nil { return nil, err } - if !d.hasMergeProgram { + if d.mergeProgram == nil { return nil, errors.New("dataset merge: merge program is not configured") } operations := make([]WriteOperation, len(req.Records)) @@ -160,7 +150,7 @@ func (d *Dataset) Merge(ctx context.Context, req DatasetWriteRequest) ([]WriteRe } mergeOptions := MergeOptions{ Incoming: document, - Program: d.mergeProgram, + Program: *d.mergeProgram, } operation, err := NewMerge(address, mergeOptions) if err != nil { @@ -235,15 +225,9 @@ func (d *Dataset) write(ctx context.Context, operation string, req WriteRequest) } req.OnResult = onResult results, err := d.client.Write(ctx, req) - resultsErr := errors.Join(WriteResultsError(results), newBatchError(failures)) + err = errors.Join(err, WriteResultsError(results), newBatchError(failures)) if err != nil { - if resultsErr != nil { - err = errors.Join(err, resultsErr) - } return results, fmt.Errorf("dataset %s: %w", operation, err) } - if resultsErr != nil { - return results, fmt.Errorf("dataset %s: %w", operation, resultsErr) - } return results, nil } diff --git a/dataset_test.go b/dataset_test.go index 7a3ed6b..d4def91 100644 --- a/dataset_test.go +++ b/dataset_test.go @@ -461,9 +461,6 @@ func TestDatasetMergeUsesOneBoundProgramForBatch(t *testing.T) { if merge == nil { t.Fatalf("Dataset.Merge() operation %d = %+v", index, operation) } - if got := merge.ProtoReflect().GetUnknown(); len(got) != 0 { - t.Fatalf("Dataset.Merge() legacy compatibility field %d = %x", index, got) - } if len(merge.GetLuaProgram().GetSource()) != 0 || !bytes.Equal(merge.GetLuaProgram().GetSha256(), wantDigest[:]) { t.Fatalf("Dataset.Merge() program reference %d = %+v", index, merge.GetLuaProgram()) diff --git a/types.go b/types.go index 4b1ad0e..043cbc0 100644 --- a/types.go +++ b/types.go @@ -334,8 +334,9 @@ type WriteOperation struct { } // WithReturnedDocument requests the logical document used by this operation's -// successful commit. It requires synchronous completion and disables folding -// with other operations on this record. Backend-generated fields are excluded. +// successful commit. It requires synchronous completion and an independent +// commit for this operation; surrounding operations may still fold. +// Backend-generated fields are excluded. func (o WriteOperation) WithReturnedDocument() WriteOperation { o.returnDocument = true return o From f6d571dda3c862b178157d4443831601730f6084 Mon Sep 17 00:00:00 2001 From: Randy <857271370@qq.com> Date: Wed, 23 Sep 2026 03:43:04 +0800 Subject: [PATCH 2/2] =?UTF-8?q?=F0=9F=93=9D=20docs:=20describe=20the=20cur?= =?UTF-8?q?rent=20SDK=20contract=20without=20migration=20history?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 40 ++++++++++++++-------------------------- 1 file changed, 14 insertions(+), 26 deletions(-) diff --git a/README.md b/README.md index 093cdca..59030e0 100644 --- a/README.md +++ b/README.md @@ -40,8 +40,7 @@ empty/dot segments, invalid UTF-8 and noncanonical typed keys are rejected. Store names are lowercase ASCII letters/digits with `.`, `_` and `-` after the first character. The entire URI is at most 16 KiB. -Deploy this SDK with the matching URI-protocol Sink build in a new cluster. -There is no old address constructor or mixed-version wire compatibility field. +Use this SDK with a Sink build that implements the same protocol. Native `Command` fields remain backend-specific and do not use record affinity. ## Quick start @@ -116,7 +115,7 @@ func main() { } ``` -`Dataset` binds the stable store, namespace, dataset, and document encoding once. +`Dataset` binds a resource URI, document encoding, and optional merge program. `Read` accepts one or many keys without reconstructing addresses. Mutations wait until applied by default; `WithCompletionMode` changes durability or visibility for an individual call. Pass records to `NewDatasetWriteRequest` or @@ -236,8 +235,7 @@ modes, negative pagination values and oversized pages still fail validation. ## Streaming callbacks -Read, Write, Query and Scan use server-streaming RPCs. The previous unary wire -contract has been removed; upgrade server and SDK together. Each method accepts +Read, Write, Query and Scan use server-streaming RPCs. Each method accepts `ctx` and a typed request. Record requests use `Addresses`, `Operations`, `Keys` or `Records` slices; completion mode and callbacks belong to the same request. Future options can be added as fields without changing method signatures. @@ -376,16 +374,14 @@ response, err := client.Execute(ctx, request) // response.Payload, StatusCode, and Headers remain available on NativeError. ``` -MongoDB Execute on revision-protected Sink servers supports `insert`, `update`, +MongoDB Execute supports `insert`, `update`, `delete`, and `findAndModify`, plus an explicit set of read/diagnostic and index commands. Inserted/replacement documents, operator updates, and update pipelines atomically receive fresh Sink revisions, so concurrent record Merges detect the change and recompute. The server rejects metadata tampering, unsafe commands such as `drop`/`renameCollection`, and unknown commands before execution; it does not fall back to unrestricted passthrough. All writers to a collection must use the -same revision protocol and metadata field. This requires server-side support: -older unrestricted servers do not advance revisions for native writes, and a -client upgrade alone does not protect mixed native/record mutations. +same revision protocol and metadata field. Search Execute validates supported routes before forwarding native payloads. Document writes, `_bulk`, queries, mapping updates and routine refresh/flush @@ -394,8 +390,7 @@ settings/templates, lifecycle policies and unknown administrative/plugin write routes return `INVALID_ARGUMENT` before reaching the backend. GET/HEAD/OPTIONS can still inspect native endpoints. See the server's [native access contract](https://github.com/batchstream/sink/blob/main/docs/native-access.md#elasticsearch-and-opensearch-endpoints) -for supported routes. This protection requires an updated server; upgrading -this SDK alone does not restrict older servers. External administration and +for supported routes. External administration and existing lifecycle policies must be coordinated with record clients, which must discard old revisions and snapshots after an index change. Native mutations do not run Lua merges or @@ -532,7 +527,7 @@ accepts ordered `bson.D` arguments and encodes the command once. Native BSON typ and ordered fields are preserved. For search Stores, supply an index-relative Path such as `/_search`, `/_mapping` or `/_doc/id`; empty Execute Path selects the index itself for inspection with GET/HEAD. Explicit index creation/deletion is rejected -by updated servers. Query/Count/Scan default to `POST //_search`. For nonempty +by Sink. Query/Count/Scan default to `POST //_search`. For nonempty payloads, ContentType defaults to the Dataset document encoding; explicit ContentType takes precedence, including NDJSON for bulk bodies. Use Client for database, cluster and multi-index endpoints. Dataset binding is a convenience; @@ -578,8 +573,6 @@ for { Nil preserves native projection; a non-nil empty projection selects all fields. Projection is executed by the backend, reducing document transfer and payload memory. Search paths are relative to `_source`, and hit metadata is preserved. -This field requires a server with Scan projection support; upgrade the server -before clients that rely on it. Older servers ignore the new protobuf field. MongoDB Scan supports find queries in `_id` ascending order by default, or an explicit `_id` descending sort, with simple collation. Projections can exclude @@ -607,7 +600,7 @@ Scan reads live data: inserts before the checkpoint may be missed, later inserts may appear, and updates/deletes can change results. Retries from the same cursor can observe newer data. Execute never retries automatically. Scan retries only temporary admission rejections explicitly marked by Sink, using the identical -command and cursor. Unmarked `ResourceExhausted` (including older servers), +command and cursor. Unmarked `ResourceExhausted`, backend errors, transport failures and invalid pages are not retried. A failed Scan can return partial documents but no next cursor; reuse the last saved cursor and process idempotently. @@ -696,7 +689,7 @@ then gracefully drain the server. Abrupt termination can fail in-flight calls; load balancing does not make mutating RPCs safe to replay. The SDK interval applies to Gateway discovery; Gateway has a separate -`gateway.dns_refresh_interval` for Engine discovery. Size each rollout's serving +`forwarding.dns_refresh_interval` for Engine discovery. Size each rollout's serving overlap for endpoint publication, upstream DNS caches, refresh and lookup delays, plus accepted requests that still retain an old Engine address snapshot. Kubernetes `preStop` is part of the total termination grace period, whereas @@ -704,11 +697,6 @@ Sink's `shutdown_timeout` bounds gRPC draining after SIGTERM. A refresh interval shorter than `preStop` is not sufficient by itself. See the server's [rollout timing and qualification guide](https://github.com/batchstream/sink/blob/main/docs/rolling-upgrades.md). -The URI-only SDK release line starting with v0.8.0 pairs with Sink v0.15.0 or -later compatible releases. Upgrade the three server roles and clients together -using the [configuration migration guide](https://github.com/batchstream/sink/blob/main/docs/configuration-migration.md); -v0.7.x and the earlier multi-Store protocol are not wire-compatible with this line. - Reads retry transport-level `Unavailable` failures and retryable per-operation failures with bounded exponential backoff and jitter. Only failed operations are resubmitted after a partial batch response. The default is three attempts, @@ -742,15 +730,15 @@ use. ## Compatibility and development -Native access and returned documents require the matching Sink server update. -Upgrade the server before using these options. Older servers report Unimplemented -for native RPCs; a missing requested write document is a ProtocolError after the -write may already have been applied, so it must not trigger an automatic retry. +Use matching SDK and server protocols. A missing requested write document is a +`ProtocolError` after the write may already have been applied, so it must not +trigger an automatic retry. The generated protocol matches the current Sink server contract. CI runs descriptor contract tests, race-enabled unit tests against an in-memory gRPC server, malformed-response tests, static analysis, and an end-to-end -compatibility test against the current Sink main branch with MongoDB and Kafka. +integration test against the matching Sink branch when available, or main, with +MongoDB and Kafka. The compatibility workflow also runs weekly so server-side drift is detected without requiring a client commit.