Skip to content
This repository was archived by the owner on Nov 8, 2026. It is now read-only.
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 1 addition & 2 deletions .github/coverage-minimums.json
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@
{
"github.com/batchstream/sink-go": 84,
"github.com/batchstream/sink-go/uri": 91
"github.com/batchstream/sink-go": 84
}
11 changes: 4 additions & 7 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,10 @@ jobs:
- name: Fuzz canonical URI boundaries
run: |
mkdir -p .reports/fuzz
go test ./uri -run='^$' -fuzz='^FuzzCanonicalURI$' -fuzztime=30s -parallel=2 2>&1 | tee .reports/fuzz/uri.log
protocol_dir="$(go list -m -f '{{.Dir}}' github.com/batchstream/sink-protocol)"
fuzz_dir="$(mktemp -d)"
cp -R "${protocol_dir}/uri/." "${fuzz_dir}/"
(cd "${fuzz_dir}" && GO111MODULE=off go test -run='^$' -fuzz='^FuzzCanonicalURI$' -fuzztime=30s -parallel=2 .) 2>&1 | tee .reports/fuzz/uri.log
grep -q 'new interesting:' .reports/fuzz/uri.log

- name: Preserve fuzz evidence
Expand All @@ -60,7 +63,6 @@ jobs:
name: uri-fuzzing
path: |
.reports/fuzz/
uri/testdata/fuzz/
include-hidden-files: true
retention-days: 14

Expand Down Expand Up @@ -102,11 +104,6 @@ jobs:
go-version-file: go.mod
cache: true

- name: Verify matching protobuf contract
run: |
sed 's#github.com/batchstream/sink/gen/sink;sink#github.com/batchstream/sink-go/api/sink/v1;sinkv1#' \
sink-server/proto/sink/sink.proto | diff -u - proto/sink/sink.proto

- name: Start Sink with MongoDB and Kafka
run: docker compose --file sink-server/examples/quickstart/compose.yaml up --detach --build engine worker gateway

Expand Down
23 changes: 1 addition & 22 deletions Makefile
Original file line number Diff line number Diff line change
@@ -1,29 +1,8 @@
.PHONY: proto proto-check test test-race test-integration lint clean
.PHONY: test test-race test-integration lint clean

BIN_DIR := $(CURDIR)/.bin
PROTOC_GEN_GO_VERSION := v1.36.6
PROTOC_GEN_GO_GRPC_VERSION := v1.5.1
PROTOC_GEN_GO_VTPROTO_VERSION := v0.6.1-0.20240319094008-0393e58bdf10
STATICCHECK_VERSION := v0.8.1

proto:
@mkdir -p $(BIN_DIR)
GOBIN=$(BIN_DIR) go install google.golang.org/protobuf/cmd/protoc-gen-go@$(PROTOC_GEN_GO_VERSION)
GOBIN=$(BIN_DIR) go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@$(PROTOC_GEN_GO_GRPC_VERSION)
GOBIN=$(BIN_DIR) go install github.com/planetscale/vtprotobuf/cmd/protoc-gen-go-vtproto@$(PROTOC_GEN_GO_VTPROTO_VERSION)
PATH="$(BIN_DIR):$$PATH" protoc \
--proto_path=proto \
--go_out=. --go_opt=module=github.com/batchstream/sink-go \
--go-grpc_out=. --go-grpc_opt=module=github.com/batchstream/sink-go \
--go-vtproto_out=. --go-vtproto_opt=module=github.com/batchstream/sink-go,features=marshal+unmarshal+size \
proto/sink/sink.proto
@perl -0pi -e 's/\treturn &sinkClient\{cc\}\n/\tclient := \&sinkClient{cc}\n\treturn client\n/' api/sink/v1/sink_grpc.pb.go
@perl -0pi -e 's/^(\s*)m\.([A-Za-z]+) = append\(m\.\2, &([A-Za-z]+)\{\}\)\n/$$1item := \&$${3}{}\n$$1m.$$2 = append(m.$$2, item)\n/gm' api/sink/v1/sink_vtproto.pb.go
@gofmt -w api/sink/v1/sink_grpc.pb.go api/sink/v1/sink_vtproto.pb.go

proto-check: proto
git diff --exit-code -- api/sink/v1

test:
go test ./... -count=1

Expand Down
11 changes: 5 additions & 6 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ canonical URI for Engine affinity. All Gateways with the same Engine membership
select the same Engine for a record. Membership changes may temporarily split
traffic. No exclusive ownership or cross-replica ordering is promised.

The shared `github.com/batchstream/sink-go/uri` package builds, parses and validates
The shared `github.com/batchstream/sink-protocol/uri` package builds, parses and validates
URIs. String keys use `s:`, int64 keys `i:`, bytes keys `b:` plus unpadded base64url,
and opaque keys `o:<base64url type>:<base64url data>`. The builder escapes each
path segment; encoded slashes are part of a segment. Alternate URI spellings,
Expand Down Expand Up @@ -299,7 +299,7 @@ The default collector returns received documents/results along with any error.
search hits, with an opaque `NextCursor` for resuming on any server.
- String, int64, byte, and opaque keys are supported.
- `CheckHealth` uses the standard gRPC health service.
- `Raw` exposes the generated `api/sink/v1` client for advanced use.
- `Raw` exposes the generated `github.com/batchstream/sink-protocol/sink/v1` client for advanced use.

Use `CompletionWaitUntilApplied` for storage acknowledgement,
`CompletionWaitUntilVisible` when a following search read must observe the
Expand Down Expand Up @@ -734,19 +734,18 @@ Use matching SDK and server protocols. A missing requested write document is a
`ProtocolError` after the write may already have been applied, so it must not
trigger an automatic retry.

The generated protocol matches the current Sink server contract. CI
runs descriptor contract tests, race-enabled unit tests against an in-memory
The SDK uses the generated public API from `sink-protocol`, the same versioned
contract consumed by the Sink server. CI runs race-enabled unit tests against an in-memory
gRPC server, malformed-response tests, static analysis, and an end-to-end
integration test against the matching Sink branch when available, or main, with
MongoDB and Kafka.
The compatibility workflow also runs weekly so server-side drift is detected
The compatibility workflow also runs weekly to detect server-side behavior drift
without requiring a client commit.

```shell
make test
make test-race
make lint
make proto-check
```

To run the external compatibility test against an already running server:
Expand Down
3 changes: 0 additions & 3 deletions api/sink/v1/doc.go

This file was deleted.

Loading
Loading